Vendor import of llvm-project main llvmorg-14-init-13186-g0c553cc1af2e.vendor/llvm-project/llvmorg-14-init-13186-g0c553cc1af2e

1630 files changed, 80042 insertions, 46749 deletions

diff --git a/clang/include/clang/APINotes/Types.h b/clang/include/clang/APINotes/Types.h
index d9bf2f07291f..0d97e9ad8623 100644
--- a/clang/include/clang/APINotes/Types.h
+++ b/clang/include/clang/APINotes/Types.h
@@ -240,7 +240,7 @@ public:
   }
   void setSwiftImportAsNonGeneric(llvm::Optional<bool> Value) {
     SwiftImportAsNonGenericSpecified = Value.hasValue();
-    SwiftImportAsNonGeneric = Value.hasValue() ? *Value : false;
+    SwiftImportAsNonGeneric = Value.getValueOr(false);
   }
 
   llvm::Optional<bool> getSwiftObjCMembers() const {
@@ -249,7 +249,7 @@ public:
   }
   void setSwiftObjCMembers(llvm::Optional<bool> Value) {
     SwiftObjCMembersSpecified = Value.hasValue();
-    SwiftObjCMembers = Value.hasValue() ? *Value : false;
+    SwiftObjCMembers = Value.getValueOr(false);
   }
 
   /// Strip off any information within the class information structure that is
@@ -368,7 +368,7 @@ public:
   }
   void setSwiftImportAsAccessors(llvm::Optional<bool> Value) {
     SwiftImportAsAccessorsSpecified = Value.hasValue();
-    SwiftImportAsAccessors = Value.hasValue() ? *Value : false;
+    SwiftImportAsAccessors = Value.getValueOr(false);
   }
 
   friend bool operator==(const ObjCPropertyInfo &, const ObjCPropertyInfo &);
@@ -433,7 +433,7 @@ public:
   }
   void setNoEscape(llvm::Optional<bool> Value) {
     NoEscapeSpecified = Value.hasValue();
-    NoEscape = Value.hasValue() ? *Value : false;
+    NoEscape = Value.getValueOr(false);
   }
 
   llvm::Optional<RetainCountConventionKind> getRetainCountConvention() const {
@@ -671,7 +671,7 @@ public:
   }
   void setFlagEnum(llvm::Optional<bool> Value) {
     HasFlagEnum = Value.hasValue();
-    IsFlagEnum = Value.hasValue() ? *Value : false;
+    IsFlagEnum = Value.getValueOr(false);
   }
 
   TagInfo &operator|=(const TagInfo &RHS) {
diff --git a/clang/include/clang/AST/ASTContext.h b/clang/include/clang/AST/ASTContext.h
index d336342e4cda..63f2c948c79b 100644
--- a/clang/include/clang/AST/ASTContext.h
+++ b/clang/include/clang/AST/ASTContext.h
@@ -248,6 +248,7 @@ class ASTContext : public RefCountedBase<ASTContext> {
   mutable llvm::ContextualFoldingSet<TemplateSpecializationType, ASTContext&>
     TemplateSpecializationTypes;
   mutable llvm::FoldingSet<ParenType> ParenTypes;
+  mutable llvm::FoldingSet<UsingType> UsingTypes;
   mutable llvm::FoldingSet<ElaboratedType> ElaboratedTypes;
   mutable llvm::FoldingSet<DependentNameType> DependentNameTypes;
   mutable llvm::ContextualFoldingSet<DependentTemplateSpecializationType,
@@ -264,8 +265,8 @@ class ASTContext : public RefCountedBase<ASTContext> {
   mutable llvm::FoldingSet<AtomicType> AtomicTypes;
   llvm::FoldingSet<AttributedType> AttributedTypes;
   mutable llvm::FoldingSet<PipeType> PipeTypes;
-  mutable llvm::FoldingSet<ExtIntType> ExtIntTypes;
-  mutable llvm::FoldingSet<DependentExtIntType> DependentExtIntTypes;
+  mutable llvm::FoldingSet<BitIntType> BitIntTypes;
+  mutable llvm::FoldingSet<DependentBitIntType> DependentBitIntTypes;
 
   mutable llvm::FoldingSet<QualifiedTemplateName> QualifiedTemplateNames;
   mutable llvm::FoldingSet<DependentTemplateName> DependentTemplateNames;
@@ -1350,13 +1351,13 @@ public:
   /// Return a write_only pipe type for the specified type.
   QualType getWritePipeType(QualType T) const;
 
-  /// Return an extended integer type with the specified signedness and bit
+  /// Return a bit-precise integer type with the specified signedness and bit
   /// count.
-  QualType getExtIntType(bool Unsigned, unsigned NumBits) const;
+  QualType getBitIntType(bool Unsigned, unsigned NumBits) const;
 
-  /// Return a dependent extended integer type with the specified signedness and
-  /// bit count.
-  QualType getDependentExtIntType(bool Unsigned, Expr *BitsExpr) const;
+  /// Return a dependent bit-precise integer type with the specified signedness
+  /// and bit count.
+  QualType getDependentBitIntType(bool Unsigned, Expr *BitsExpr) const;
 
   /// Gets the struct used to keep track of the extended descriptor for
   /// pointer to blocks.
@@ -1555,6 +1556,9 @@ public:
     return getTypeDeclTypeSlow(Decl);
   }
 
+  QualType getUsingType(const UsingShadowDecl *Found,
+                        QualType Underlying) const;
+
   /// Return the unique reference to the type for the specified
   /// typedef-name decl.
   QualType getTypedefType(const TypedefNameDecl *Decl,
@@ -1564,6 +1568,9 @@ public:
 
   QualType getEnumType(const EnumDecl *Decl) const;
 
+  QualType
+  getUnresolvedUsingType(const UnresolvedUsingTypenameDecl *Decl) const;
+
   QualType getInjectedClassNameType(CXXRecordDecl *Decl, QualType TST) const;
 
   QualType getAttributedType(attr::Kind attrKind,
diff --git a/clang/include/clang/AST/ASTDiagnostic.h b/clang/include/clang/AST/ASTDiagnostic.h
index d6549e12d92a..4cd909751725 100644
--- a/clang/include/clang/AST/ASTDiagnostic.h
+++ b/clang/include/clang/AST/ASTDiagnostic.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_CLANG_AST_ASTDIAGNOSTIC_H
 #define LLVM_CLANG_AST_ASTDIAGNOSTIC_H
 
+#include "clang/AST/Type.h"
 #include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/DiagnosticAST.h"
 
@@ -31,6 +32,11 @@ namespace clang {
       SmallVectorImpl<char> &Output,
       void *Cookie,
       ArrayRef<intptr_t> QualTypeVals);
+
+  /// Returns a desugared version of the QualType, and marks ShouldAKA as true
+  /// whenever we remove significant sugar from the type.
+  QualType desugarForDiagnostic(ASTContext &Context, QualType QT,
+                                bool &ShouldAKA);
 }  // end namespace clang
 
 #endif
diff --git a/clang/include/clang/AST/ASTImporterLookupTable.h b/clang/include/clang/AST/ASTImporterLookupTable.h
index 47dca2033839..918c2b9e350c 100644
--- a/clang/include/clang/AST/ASTImporterLookupTable.h
+++ b/clang/include/clang/AST/ASTImporterLookupTable.h
@@ -75,6 +75,10 @@ public:
   // The function should be called when the old context is definitely different
   // from the new.
   void update(NamedDecl *ND, DeclContext *OldDC);
+  // Same as 'update' but allow if 'ND' is not in the table or the old context
+  // is the same as the new.
+  // FIXME: The old redeclaration context is not handled.
+  void updateForced(NamedDecl *ND, DeclContext *OldDC);
   using LookupResult = DeclList;
   LookupResult lookup(DeclContext *DC, DeclarationName Name) const;
   // Check if the `ND` is within the lookup table (with its current name) in
diff --git a/clang/include/clang/AST/AbstractBasicReader.h b/clang/include/clang/AST/AbstractBasicReader.h
index 5505d661b44e..442039044cfe 100644
--- a/clang/include/clang/AST/AbstractBasicReader.h
+++ b/clang/include/clang/AST/AbstractBasicReader.h
@@ -21,7 +21,7 @@ inline T makeNullableFromOptional(const Optional<T> &value) {
 
 template <class T>
 inline T *makePointerFromOptional(Optional<T *> value) {
-  return (value ? *value : nullptr);
+  return value.getValueOr(nullptr);
 }
 
 // PropertyReader is a class concept that requires the following method:
diff --git a/clang/include/clang/AST/DeclTemplate.h b/clang/include/clang/AST/DeclTemplate.h
index d33babef958e..f7a2e3146d06 100644
--- a/clang/include/clang/AST/DeclTemplate.h
+++ b/clang/include/clang/AST/DeclTemplate.h
@@ -1211,13 +1211,12 @@ class TemplateTypeParmDecl final : public TypeDecl,
   DefArgStorage DefaultArgument;
 
   TemplateTypeParmDecl(DeclContext *DC, SourceLocation KeyLoc,
-                       SourceLocation IdLoc, IdentifierInfo *Id,
-                       bool Typename, bool HasTypeConstraint,
-                       Optional<unsigned> NumExpanded)
+                       SourceLocation IdLoc, IdentifierInfo *Id, bool Typename,
+                       bool HasTypeConstraint, Optional<unsigned> NumExpanded)
       : TypeDecl(TemplateTypeParm, DC, IdLoc, Id, KeyLoc), Typename(Typename),
-      HasTypeConstraint(HasTypeConstraint), TypeConstraintInitialized(false),
-      ExpandedParameterPack(NumExpanded),
-      NumExpanded(NumExpanded ? *NumExpanded : 0) {}
+        HasTypeConstraint(HasTypeConstraint), TypeConstraintInitialized(false),
+        ExpandedParameterPack(NumExpanded),
+        NumExpanded(NumExpanded.getValueOr(0)) {}
 
 public:
   static TemplateTypeParmDecl *Create(const ASTContext &C, DeclContext *DC,
diff --git a/clang/include/clang/AST/Expr.h b/clang/include/clang/AST/Expr.h
index 2c63406fba18..e2c36e12393f 100644
--- a/clang/include/clang/AST/Expr.h
+++ b/clang/include/clang/AST/Expr.h
@@ -572,6 +572,12 @@ public:
   bool isConstantInitializer(ASTContext &Ctx, bool ForRef,
                              const Expr **Culprit = nullptr) const;
 
+  /// If this expression is an unambiguous reference to a single declaration,
+  /// in the style of __builtin_function_start, return that declaration.  Note
+  /// that this may return a non-static member function or field in C++ if this
+  /// expression is a member pointer constant.
+  const ValueDecl *getAsBuiltinConstantDeclRef(const ASTContext &Context) const;
+
   /// EvalStatus is a struct with detailed info about an evaluation in progress.
   struct EvalStatus {
     /// Whether the evaluated expression has side effects.
diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index 565eb0c9cf99..3fd1b6d30080 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -2224,6 +2224,47 @@ public:
   }
 };
 
+/// This represents 'compare' clause in the '#pragma omp atomic'
+/// directive.
+///
+/// \code
+/// #pragma omp atomic compare
+/// \endcode
+/// In this example directive '#pragma omp atomic' has 'compare' clause.
+class OMPCompareClause final : public OMPClause {
+public:
+  /// Build 'compare' clause.
+  ///
+  /// \param StartLoc Starting location of the clause.
+  /// \param EndLoc Ending location of the clause.
+  OMPCompareClause(SourceLocation StartLoc, SourceLocation EndLoc)
+      : OMPClause(llvm::omp::OMPC_compare, StartLoc, EndLoc) {}
+
+  /// Build an empty clause.
+  OMPCompareClause()
+      : OMPClause(llvm::omp::OMPC_compare, SourceLocation(), SourceLocation()) {
+  }
+
+  child_range children() {
+    return child_range(child_iterator(), child_iterator());
+  }
+
+  const_child_range children() const {
+    return const_child_range(const_child_iterator(), const_child_iterator());
+  }
+
+  child_range used_children() {
+    return child_range(child_iterator(), child_iterator());
+  }
+  const_child_range used_children() const {
+    return const_child_range(const_child_iterator(), const_child_iterator());
+  }
+
+  static bool classof(const OMPClause *T) {
+    return T->getClauseKind() == llvm::omp::OMPC_compare;
+  }
+};
+
 /// This represents 'seq_cst' clause in the '#pragma omp atomic'
 /// directive.
 ///
diff --git a/clang/include/clang/AST/PropertiesBase.td b/clang/include/clang/AST/PropertiesBase.td
index a087cb406b29..9282b24eb68c 100644
--- a/clang/include/clang/AST/PropertiesBase.td
+++ b/clang/include/clang/AST/PropertiesBase.td
@@ -107,6 +107,8 @@ def DeclRef : RefPropertyType<"Decl"> { let ConstWhenWriting = 1; }
     SubclassPropertyType<"TemplateTypeParmDecl", DeclRef>;
   def TemplateTemplateParmDeclRef :
     SubclassPropertyType<"TemplateTemplateParmDecl", DeclRef>;
+  def UsingShadowDeclRef :
+    SubclassPropertyType<"UsingShadowDecl", DeclRef>;
   def ValueDeclRef :
     SubclassPropertyType<"ValueDecl", DeclRef>;
 def ElaboratedTypeKeyword : EnumPropertyType;
diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index 9797eac53dde..f62dc36de556 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -981,6 +981,7 @@ DEF_TRAVERSE_TYPE(FunctionProtoType, {
     TRY_TO(TraverseStmt(NE));
 })
 
+DEF_TRAVERSE_TYPE(UsingType, {})
 DEF_TRAVERSE_TYPE(UnresolvedUsingType, {})
 DEF_TRAVERSE_TYPE(TypedefType, {})
 
@@ -1072,8 +1073,8 @@ DEF_TRAVERSE_TYPE(AtomicType, { TRY_TO(TraverseType(T->getValueType())); })
 
 DEF_TRAVERSE_TYPE(PipeType, { TRY_TO(TraverseType(T->getElementType())); })
 
-DEF_TRAVERSE_TYPE(ExtIntType, {})
-DEF_TRAVERSE_TYPE(DependentExtIntType,
+DEF_TRAVERSE_TYPE(BitIntType, {})
+DEF_TRAVERSE_TYPE(DependentBitIntType,
                   { TRY_TO(TraverseStmt(T->getNumBitsExpr())); })
 
 #undef DEF_TRAVERSE_TYPE
@@ -1252,6 +1253,7 @@ DEF_TRAVERSE_TYPELOC(FunctionProtoType, {
     TRY_TO(TraverseStmt(NE));
 })
 
+DEF_TRAVERSE_TYPELOC(UsingType, {})
 DEF_TRAVERSE_TYPELOC(UnresolvedUsingType, {})
 DEF_TRAVERSE_TYPELOC(TypedefType, {})
 
@@ -1358,8 +1360,8 @@ DEF_TRAVERSE_TYPELOC(AtomicType, { TRY_TO(TraverseTypeLoc(TL.getValueLoc())); })
 
 DEF_TRAVERSE_TYPELOC(PipeType, { TRY_TO(TraverseTypeLoc(TL.getValueLoc())); })
 
-DEF_TRAVERSE_TYPELOC(ExtIntType, {})
-DEF_TRAVERSE_TYPELOC(DependentExtIntType, {
+DEF_TRAVERSE_TYPELOC(BitIntType, {})
+DEF_TRAVERSE_TYPELOC(DependentBitIntType, {
   TRY_TO(TraverseStmt(TL.getTypePtr()->getNumBitsExpr()));
 })
 
@@ -2095,7 +2097,13 @@ bool RecursiveASTVisitor<Derived>::TraverseFunctionHelper(FunctionDecl *D) {
   }
 
   if (VisitBody) {
-    TRY_TO(TraverseStmt(D->getBody())); // Function body.
+    TRY_TO(TraverseStmt(D->getBody()));
+    // Body may contain using declarations whose shadows are parented to the
+    // FunctionDecl itself.
+    for (auto *Child : D->decls()) {
+      if (isa<UsingShadowDecl>(Child))
+        TRY_TO(TraverseDecl(Child));
+    }
   }
   return true;
 }
@@ -3227,6 +3235,11 @@ bool RecursiveASTVisitor<Derived>::VisitOMPCaptureClause(OMPCaptureClause *) {
 }
 
 template <typename Derived>
+bool RecursiveASTVisitor<Derived>::VisitOMPCompareClause(OMPCompareClause *) {
+  return true;
+}
+
+template <typename Derived>
 bool RecursiveASTVisitor<Derived>::VisitOMPSeqCstClause(OMPSeqCstClause *) {
   return true;
 }
diff --git a/clang/include/clang/AST/TextNodeDumper.h b/clang/include/clang/AST/TextNodeDumper.h
index 0eb0031de11f..41bbf2ec593a 100644
--- a/clang/include/clang/AST/TextNodeDumper.h
+++ b/clang/include/clang/AST/TextNodeDumper.h
@@ -311,6 +311,7 @@ public:
   void VisitFunctionType(const FunctionType *T);
   void VisitFunctionProtoType(const FunctionProtoType *T);
   void VisitUnresolvedUsingType(const UnresolvedUsingType *T);
+  void VisitUsingType(const UsingType *T);
   void VisitTypedefType(const TypedefType *T);
   void VisitUnaryTransformType(const UnaryTransformType *T);
   void VisitTagType(const TagType *T);
diff --git a/clang/include/clang/AST/Type.h b/clang/include/clang/AST/Type.h
index 4c89c297bf34..a69c0ae67d0a 100644
--- a/clang/include/clang/AST/Type.h
+++ b/clang/include/clang/AST/Type.h
@@ -129,6 +129,7 @@ class TemplateArgumentLoc;
 class TemplateTypeParmDecl;
 class TypedefNameDecl;
 class UnresolvedUsingTypenameDecl;
+class UsingShadowDecl;
 
 using CanQualType = CanQual<Type>;
 
@@ -2128,7 +2129,7 @@ public:
   bool isOCLExtOpaqueType() const;              // Any OpenCL extension type
 
   bool isPipeType() const;                      // OpenCL pipe type
-  bool isExtIntType() const;                    // Extended Int Type
+  bool isBitIntType() const;                    // Bit-precise integer type
   bool isOpenCLSpecificType() const;            // Any OpenCL specific type
 
   /// Determines if this type, which must satisfy
@@ -4368,6 +4369,27 @@ public:
   }
 };
 
+class UsingType : public Type, public llvm::FoldingSetNode {
+  UsingShadowDecl *Found;
+  friend class ASTContext; // ASTContext creates these.
+
+  UsingType(const UsingShadowDecl *Found, QualType Underlying, QualType Canon);
+
+public:
+  UsingShadowDecl *getFoundDecl() const { return Found; }
+  QualType getUnderlyingType() const;
+
+  bool isSugared() const { return true; }
+  QualType desugar() const { return getUnderlyingType(); }
+
+  void Profile(llvm::FoldingSetNodeID &ID) { Profile(ID, Found); }
+  static void Profile(llvm::FoldingSetNodeID &ID,
+                      const UsingShadowDecl *Found) {
+    ID.AddPointer(Found);
+  }
+  static bool classof(const Type *T) { return T->getTypeClass() == Using; }
+};
+
 class TypedefType : public Type {
   TypedefNameDecl *Decl;
 
@@ -6307,13 +6329,13 @@ public:
 };
 
 /// A fixed int type of a specified bitwidth.
-class ExtIntType final : public Type, public llvm::FoldingSetNode {
+class BitIntType final : public Type, public llvm::FoldingSetNode {
   friend class ASTContext;
   unsigned IsUnsigned : 1;
   unsigned NumBits : 24;
 
 protected:
-  ExtIntType(bool isUnsigned, unsigned NumBits);
+  BitIntType(bool isUnsigned, unsigned NumBits);
 
 public:
   bool isUnsigned() const { return IsUnsigned; }
@@ -6333,16 +6355,16 @@ public:
     ID.AddInteger(NumBits);
   }
 
-  static bool classof(const Type *T) { return T->getTypeClass() == ExtInt; }
+  static bool classof(const Type *T) { return T->getTypeClass() == BitInt; }
 };
 
-class DependentExtIntType final : public Type, public llvm::FoldingSetNode {
+class DependentBitIntType final : public Type, public llvm::FoldingSetNode {
   friend class ASTContext;
   const ASTContext &Context;
   llvm::PointerIntPair<Expr*, 1, bool> ExprAndUnsigned;
 
 protected:
-  DependentExtIntType(const ASTContext &Context, bool IsUnsigned,
+  DependentBitIntType(const ASTContext &Context, bool IsUnsigned,
                       Expr *NumBits);
 
 public:
@@ -6360,7 +6382,7 @@ public:
                       bool IsUnsigned, Expr *NumBitsExpr);
 
   static bool classof(const Type *T) {
-    return T->getTypeClass() == DependentExtInt;
+    return T->getTypeClass() == DependentBitInt;
   }
 };
 
@@ -6891,8 +6913,8 @@ inline bool Type::isPipeType() const {
   return isa<PipeType>(CanonicalType);
 }
 
-inline bool Type::isExtIntType() const {
-  return isa<ExtIntType>(CanonicalType);
+inline bool Type::isBitIntType() const {
+  return isa<BitIntType>(CanonicalType);
 }
 
 #define EXT_OPAQUE_TYPE(ExtType, Id, Ext) \
@@ -6998,7 +7020,7 @@ inline bool Type::isIntegerType() const {
     return IsEnumDeclComplete(ET->getDecl()) &&
       !IsEnumDeclScoped(ET->getDecl());
   }
-  return isExtIntType();
+  return isBitIntType();
 }
 
 inline bool Type::isFixedPointType() const {
@@ -7056,7 +7078,7 @@ inline bool Type::isScalarType() const {
          isa<MemberPointerType>(CanonicalType) ||
          isa<ComplexType>(CanonicalType) ||
          isa<ObjCObjectPointerType>(CanonicalType) ||
-         isExtIntType();
+         isBitIntType();
 }
 
 inline bool Type::isIntegralOrEnumerationType() const {
@@ -7069,7 +7091,7 @@ inline bool Type::isIntegralOrEnumerationType() const {
   if (const auto *ET = dyn_cast<EnumType>(CanonicalType))
     return IsEnumDeclComplete(ET->getDecl());
 
-  return isExtIntType();
+  return isBitIntType();
 }
 
 inline bool Type::isBooleanType() const {
diff --git a/clang/include/clang/AST/TypeLoc.h b/clang/include/clang/AST/TypeLoc.h
index bb668c1980fe..7a036836e8c4 100644
--- a/clang/include/clang/AST/TypeLoc.h
+++ b/clang/include/clang/AST/TypeLoc.h
@@ -665,6 +665,16 @@ public:
   }
 };
 
+/// Wrapper for source info for types used via transparent aliases.
+class UsingTypeLoc : public InheritingConcreteTypeLoc<TypeSpecTypeLoc,
+                                                      UsingTypeLoc, UsingType> {
+public:
+  QualType getUnderlyingType() const {
+    return getTypePtr()->getUnderlyingType();
+  }
+  UsingShadowDecl *getFoundDecl() const { return getTypePtr()->getFoundDecl(); }
+};
+
 /// Wrapper for source info for typedefs.
 class TypedefTypeLoc : public InheritingConcreteTypeLoc<TypeSpecTypeLoc,
                                                         TypedefTypeLoc,
@@ -2561,12 +2571,12 @@ inline T TypeLoc::getAsAdjusted() const {
   }
   return Cur.getAs<T>();
 }
-class ExtIntTypeLoc final
-    : public InheritingConcreteTypeLoc<TypeSpecTypeLoc, ExtIntTypeLoc,
-                                        ExtIntType> {};
-class DependentExtIntTypeLoc final
-    : public InheritingConcreteTypeLoc<TypeSpecTypeLoc, DependentExtIntTypeLoc,
-                                        DependentExtIntType> {};
+class BitIntTypeLoc final
+    : public InheritingConcreteTypeLoc<TypeSpecTypeLoc, BitIntTypeLoc,
+                                       BitIntType> {};
+class DependentBitIntTypeLoc final
+    : public InheritingConcreteTypeLoc<TypeSpecTypeLoc, DependentBitIntTypeLoc,
+                                       DependentBitIntType> {};
 
 } // namespace clang
 
diff --git a/clang/include/clang/AST/TypeProperties.td b/clang/include/clang/AST/TypeProperties.td
index 438d5af5a2e2..8cbb67589b94 100644
--- a/clang/include/clang/AST/TypeProperties.td
+++ b/clang/include/clang/AST/TypeProperties.td
@@ -358,7 +358,20 @@ let Class = UnresolvedUsingType in {
   }
 
   def : Creator<[{
-    return ctx.getTypeDeclType(cast<UnresolvedUsingTypenameDecl>(declaration));
+    return ctx.getUnresolvedUsingType(cast<UnresolvedUsingTypenameDecl>(declaration));
+  }]>;
+}
+
+let Class = UsingType in {
+  def : Property<"foundDeclaration", UsingShadowDeclRef> {
+    let Read = [{ node->getFoundDecl() }];
+  }
+  def : Property<"underlyingType", QualType> {
+    let Read = [{ node->getUnderlyingType() }];
+  }
+
+  def : Creator<[{
+    return ctx.getUsingType(foundDeclaration, underlyingType);
   }]>;
 }
 
@@ -882,7 +895,7 @@ let Class = PipeType in {
   }]>;
 }
 
-let Class = ExtIntType in {
+let Class = BitIntType in {
   def : Property<"isUnsigned", Bool> {
     let Read = [{ node->isUnsigned() }];
   }
@@ -891,11 +904,11 @@ let Class = ExtIntType in {
   }
 
   def : Creator<[{
-    return ctx.getExtIntType(isUnsigned, numBits);
+    return ctx.getBitIntType(isUnsigned, numBits);
   }]>;
 }
 
-let Class = DependentExtIntType in {
+let Class = DependentBitIntType in {
   def : Property<"isUnsigned", Bool> {
     let Read = [{ node->isUnsigned() }];
   }
@@ -903,6 +916,6 @@ let Class = DependentExtIntType in {
     let Read = [{ node->getNumBitsExpr() }];
   }
   def : Creator<[{
-    return ctx.getDependentExtIntType(isUnsigned, numBitsExpr);
+    return ctx.getDependentBitIntType(isUnsigned, numBitsExpr);
   }]>;
 }
diff --git a/clang/include/clang/ASTMatchers/ASTMatchers.h b/clang/include/clang/ASTMatchers/ASTMatchers.h
index 5221d05477d0..55cce324b436 100644
--- a/clang/include/clang/ASTMatchers/ASTMatchers.h
+++ b/clang/include/clang/ASTMatchers/ASTMatchers.h
@@ -4128,25 +4128,34 @@ AST_MATCHER_P(DeclRefExpr, to, internal::Matcher<Decl>,
           InnerMatcher.matches(*DeclNode, Finder, Builder));
 }
 
-/// Matches a \c DeclRefExpr that refers to a declaration through a
-/// specific using shadow declaration.
+/// Matches if a node refers to a declaration through a specific
+/// using shadow declaration.
 ///
-/// Given
+/// Examples:
 /// \code
-///   namespace a { void f() {} }
+///   namespace a { int f(); }
 ///   using a::f;
-///   void g() {
-///     f();     // Matches this ..
-///     a::f();  // .. but not this.
-///   }
+///   int x = f();
 /// \endcode
 /// declRefExpr(throughUsingDecl(anything()))
-///   matches \c f()
-AST_MATCHER_P(DeclRefExpr, throughUsingDecl,
-              internal::Matcher<UsingShadowDecl>, InnerMatcher) {
+///   matches \c f
+///
+/// \code
+///   namespace a { class X{}; }
+///   using a::X;
+///   X x;
+/// \code
+/// typeLoc(loc(usingType(throughUsingDecl(anything()))))
+///   matches \c X
+///
+/// Usable as: Matcher<DeclRefExpr>, Matcher<UsingType>
+AST_POLYMORPHIC_MATCHER_P(throughUsingDecl,
+                          AST_POLYMORPHIC_SUPPORTED_TYPES(DeclRefExpr,
+                                                          UsingType),
+                          internal::Matcher<UsingShadowDecl>, Inner) {
   const NamedDecl *FoundDecl = Node.getFoundDecl();
   if (const UsingShadowDecl *UsingDecl = dyn_cast<UsingShadowDecl>(FoundDecl))
-    return InnerMatcher.matches(*UsingDecl, Finder, Builder);
+    return Inner.matches(*UsingDecl, Finder, Builder);
   return false;
 }
 
@@ -4872,7 +4881,7 @@ AST_POLYMORPHIC_MATCHER_P2(forEachArgumentWithParamType,
     }
   }
 
-  int ParamIndex = 0;
+  unsigned ParamIndex = 0;
   bool Matched = false;
   unsigned NumArgs = Node.getNumArgs();
   if (FProto && FProto->isVariadic())
@@ -4886,7 +4895,7 @@ AST_POLYMORPHIC_MATCHER_P2(forEachArgumentWithParamType,
 
       // This test is cheaper compared to the big matcher in the next if.
       // Therefore, please keep this order.
-      if (FProto) {
+      if (FProto && FProto->getNumParams() > ParamIndex) {
         QualType ParamType = FProto->getParamType(ParamIndex);
         if (ParamMatcher.matches(ParamType, Finder, &ParamMatches)) {
           Result.addMatch(ParamMatches);
@@ -6843,7 +6852,7 @@ extern const AstTypeMatcher<DecltypeType> decltypeType;
 AST_TYPE_TRAVERSE_MATCHER(hasDeducedType, getDeducedType,
                           AST_POLYMORPHIC_SUPPORTED_TYPES(AutoType));
 
-/// Matches \c DecltypeType nodes to find out the underlying type.
+/// Matches \c DecltypeType or \c UsingType nodes to find the underlying type.
 ///
 /// Given
 /// \code
@@ -6853,9 +6862,10 @@ AST_TYPE_TRAVERSE_MATCHER(hasDeducedType, getDeducedType,
 /// decltypeType(hasUnderlyingType(isInteger()))
 ///   matches the type of "a"
 ///
-/// Usable as: Matcher<DecltypeType>
+/// Usable as: Matcher<DecltypeType>, Matcher<UsingType>
 AST_TYPE_TRAVERSE_MATCHER(hasUnderlyingType, getUnderlyingType,
-                          AST_POLYMORPHIC_SUPPORTED_TYPES(DecltypeType));
+                          AST_POLYMORPHIC_SUPPORTED_TYPES(DecltypeType,
+                                                          UsingType));
 
 /// Matches \c FunctionType nodes.
 ///
@@ -7183,6 +7193,18 @@ AST_MATCHER_P(ElaboratedType, namesType, internal::Matcher<QualType>,
   return InnerMatcher.matches(Node.getNamedType(), Finder, Builder);
 }
 
+/// Matches types specified through a using declaration.
+///
+/// Given
+/// \code
+///   namespace a { struct S {}; }
+///   using a::S;
+///   S s;
+/// \endcode
+///
+/// \c usingType() matches the type of the variable declaration of \c s.
+extern const AstTypeMatcher<UsingType> usingType;
+
 /// Matches types that represent the result of substituting a type for a
 /// template type parameter.
 ///
diff --git a/clang/include/clang/ASTMatchers/ASTMatchersInternal.h b/clang/include/clang/ASTMatchers/ASTMatchersInternal.h
index a77611001fb1..ab7a445dbcd4 100644
--- a/clang/include/clang/ASTMatchers/ASTMatchersInternal.h
+++ b/clang/include/clang/ASTMatchers/ASTMatchersInternal.h
@@ -1090,6 +1090,12 @@ private:
     if (const auto *S = dyn_cast<ElaboratedType>(&Node)) {
       return matchesSpecialized(S->desugar(), Finder, Builder);
     }
+    // Similarly types found via using declarations.
+    // These are *usually* meaningless sugar, and this matches the historical
+    // behavior prior to the introduction of UsingType.
+    if (const auto *S = dyn_cast<UsingType>(&Node)) {
+      return matchesSpecialized(S->desugar(), Finder, Builder);
+    }
     return false;
   }
 
diff --git a/clang/include/clang/Analysis/Analyses/ThreadSafetyCommon.h b/clang/include/clang/Analysis/Analyses/ThreadSafetyCommon.h
index 4a58fe870944..a0ae44131b45 100644
--- a/clang/include/clang/Analysis/Analyses/ThreadSafetyCommon.h
+++ b/clang/include/clang/Analysis/Analyses/ThreadSafetyCommon.h
@@ -155,7 +155,7 @@ public:
       return false;
 
     // Ignore anonymous functions.
-    if (!dyn_cast_or_null<NamedDecl>(AC.getDecl()))
+    if (!isa_and_nonnull<NamedDecl>(AC.getDecl()))
       return false;
 
     SortedGraph = AC.getAnalysis<PostOrderCFGView>();
diff --git a/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h b/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h
index 69a5c2e47b66..4a3c0239f8e1 100644
--- a/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h
+++ b/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h
@@ -15,11 +15,20 @@
 #ifndef LLVM_CLANG_ANALYSIS_FLOWSENSITIVE_DATAFLOWENVIRONMENT_H
 #define LLVM_CLANG_ANALYSIS_FLOWSENSITIVE_DATAFLOWENVIRONMENT_H
 
+#include "clang/Analysis/FlowSensitive/DataflowLattice.h"
+
 namespace clang {
 namespace dataflow {
 
 /// Holds the state of the program (store and heap) at a given program point.
-class Environment {};
+class Environment {
+public:
+  bool operator==(const Environment &) const { return true; }
+
+  LatticeJoinEffect join(const Environment &) {
+    return LatticeJoinEffect::Unchanged;
+  }
+};
 
 } // namespace dataflow
 } // namespace clang
diff --git a/clang/include/clang/Analysis/FlowSensitive/DataflowWorklist.h b/clang/include/clang/Analysis/FlowSensitive/DataflowWorklist.h
index 90095735ad3d..52d84eb13c56 100644
--- a/clang/include/clang/Analysis/FlowSensitive/DataflowWorklist.h
+++ b/clang/include/clang/Analysis/FlowSensitive/DataflowWorklist.h
@@ -61,11 +61,12 @@ struct ReversePostOrderCompare {
 /// the same block multiple times at once.
 struct ForwardDataflowWorklist
     : DataflowWorklistBase<ReversePostOrderCompare, 20> {
+  ForwardDataflowWorklist(const CFG &Cfg, PostOrderCFGView *POV)
+      : DataflowWorklistBase(Cfg, POV,
+                             ReversePostOrderCompare{POV->getComparator()}) {}
+
   ForwardDataflowWorklist(const CFG &Cfg, AnalysisDeclContext &Ctx)
-      : DataflowWorklistBase(
-            Cfg, Ctx.getAnalysis<PostOrderCFGView>(),
-            ReversePostOrderCompare{
-                Ctx.getAnalysis<PostOrderCFGView>()->getComparator()}) {}
+      : ForwardDataflowWorklist(Cfg, Ctx.getAnalysis<PostOrderCFGView>()) {}
 
   void enqueueSuccessors(const CFGBlock *Block) {
     for (auto B : Block->succs())
diff --git a/clang/include/clang/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.h b/clang/include/clang/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.h
index 9448b911f471..6193b9860d33 100644
--- a/clang/include/clang/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.h
+++ b/clang/include/clang/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.h
@@ -76,6 +76,24 @@ struct TypeErasedDataflowAnalysisState {
   Environment Env;
 };
 
+/// Transfers the state of a basic block by evaluating each of its statements in
+/// the context of `Analysis` and the states of its predecessors that are
+/// available in `BlockStates`. `HandleTransferredStmt` (if provided) will be
+/// applied to each statement in the block, after it is evaluated.
+///
+/// Requirements:
+///
+///   All predecessors of `Block` except those with loop back edges must have
+///   already been transferred. States in `BlockStates` that are set to
+///   `llvm::None` represent basic blocks that are not evaluated yet.
+TypeErasedDataflowAnalysisState transferBlock(
+    std::vector<llvm::Optional<TypeErasedDataflowAnalysisState>> &BlockStates,
+    const CFGBlock &Block, const Environment &InitEnv,
+    TypeErasedDataflowAnalysis &Analysis,
+    std::function<void(const CFGStmt &,
+                       const TypeErasedDataflowAnalysisState &)>
+        HandleTransferredStmt = nullptr);
+
 /// Performs dataflow analysis and returns a mapping from basic block IDs to
 /// dataflow analysis states that model the respective basic blocks. Indices
 /// of the returned vector correspond to basic block IDs.
diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index fab3f3edfb83..10c5c7f1b879 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -3865,6 +3865,14 @@ def ReleaseHandle : InheritableParamAttr {
   let Documentation = [ReleaseHandleDocs];
 }
 
+def DiagnoseAsBuiltin : InheritableAttr {
+  let Spellings = [Clang<"diagnose_as_builtin">];
+  let Args = [DeclArgument<Function, "Function">,
+              VariadicUnsignedArgument<"ArgIndices">];
+  let Subjects = SubjectList<[Function]>;
+  let Documentation = [DiagnoseAsBuiltinDocs];
+}
+
 def Builtin : InheritableAttr {
   let Spellings = [];
   let Args = [UnsignedArgument<"ID">];
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index 10cce4c2d689..8a7424a88c9f 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -187,6 +187,10 @@ primary use is for COFF object files which explicitly specify what interfaces
 are imported from external modules. See the dllimport_ documentation on MSDN
 for more information.
 
+Note that a dllimport function may still be inlined, if its definition is
+available and it doesn't reference any non-dllimport functions or global
+variables.
+
 .. _dllimport: https://msdn.microsoft.com/en-us/library/3y1sfaz2.aspx
   }];
 }
@@ -5980,6 +5984,50 @@ attribute requires a string literal argument to identify the handle being releas
   }];
 }
 
+def DiagnoseAsBuiltinDocs : Documentation {
+  let Category = DocCatFunction;
+  let Content = [{
+The ``diagnose_as_builtin` attribute indicates that Fortify diagnostics are to
+be applied to the declared function as if it were the function specified by the
+attribute. The builtin function whose diagnostics are to be mimicked should be
+given. In addition, the order in which arguments should be applied must also
+be given.
+
+For example, the attribute can be used as follows.
+
+  .. code-block:: c
+
+    __attribute__((diagnose_as_builtin(__builtin_memset, 3, 2, 1)))
+    void *mymemset(int n, int c, void *s) {
+      // ...
+    }
+
+This indicates that calls to ``mymemset`` should be diagnosed as if they were
+calls to ``__builtin_memset``. The arguments ``3, 2, 1`` indicate by index the
+order in which arguments of ``mymemset`` should be applied to
+``__builtin_memset``. The third argument should be applied first, then the
+second, and then the first. Thus (when Fortify warnings are enabled) the call
+``mymemset(n, c, s)`` will diagnose overflows as if it were the call
+``__builtin_memset(s, c, n)``.
+
+For variadic functions, the variadic arguments must come in the same order as
+they would to the builtin function, after all normal arguments. For instance,
+to diagnose a new function as if it were `sscanf`, we can use the attribute as
+follows.
+
+  .. code-block:: c
+  __attribute__((diagnose_as_builtin(sscanf, 1, 2)))
+  int mysscanf(const char *str, const char *format, ...)  {
+    // ...
+  }
+
+Then the call `mysscanf("abc def", "%4s %4s", buf1, buf2)` will be diagnosed as
+if it were the call `sscanf("abc def", "%4s %4s", buf1, buf2)`.
+
+This attribute cannot be applied to non-static member functions.
+}];
+}
+
 def ArmSveVectorBitsDocs : Documentation {
   let Category = DocCatType;
   let Content = [{
diff --git a/clang/include/clang/Basic/Builtins.def b/clang/include/clang/Basic/Builtins.def
index ad8b66aa490b..45d445163749 100644
--- a/clang/include/clang/Basic/Builtins.def
+++ b/clang/include/clang/Basic/Builtins.def
@@ -646,8 +646,10 @@ BUILTIN(__builtin_call_with_static_chain, "v.", "nt")
 BUILTIN(__builtin_elementwise_abs, "v.", "nct")
 BUILTIN(__builtin_elementwise_max, "v.", "nct")
 BUILTIN(__builtin_elementwise_min, "v.", "nct")
+BUILTIN(__builtin_elementwise_ceil, "v.", "nct")
 BUILTIN(__builtin_reduce_max, "v.", "nct")
 BUILTIN(__builtin_reduce_min, "v.", "nct")
+BUILTIN(__builtin_reduce_xor, "v.", "nct")
 
 BUILTIN(__builtin_matrix_transpose, "v.", "nFt")
 BUILTIN(__builtin_matrix_column_major_load, "v.", "nFt")
@@ -1574,6 +1576,7 @@ BUILTIN(__builtin_smulll_overflow, "bSLLiCSLLiCSLLi*", "n")
 
 // Clang builtins (not available in GCC).
 BUILTIN(__builtin_addressof, "v*v&", "nct")
+BUILTIN(__builtin_function_start, "v*v&", "nct")
 BUILTIN(__builtin_operator_new, "v*z", "tc")
 BUILTIN(__builtin_operator_delete, "vv*", "tn")
 BUILTIN(__builtin_char_memchr, "c*cC*iz", "n")
@@ -1611,7 +1614,6 @@ LANGBUILTIN(__builtin_coro_alloc, "b", "n", COR_LANG)
 LANGBUILTIN(__builtin_coro_begin, "v*v*", "n", COR_LANG)
 LANGBUILTIN(__builtin_coro_end, "bv*Ib", "n", COR_LANG)
 LANGBUILTIN(__builtin_coro_suspend, "cIb", "n", COR_LANG)
-LANGBUILTIN(__builtin_coro_param, "bv*v*", "n", COR_LANG)
 
 // OpenCL v2.0 s6.13.16, s9.17.3.5 - Pipe functions.
 // We need the generic prototype, since the packet type could be anything.
diff --git a/clang/include/clang/Basic/BuiltinsAArch64NeonSVEBridge.def b/clang/include/clang/Basic/BuiltinsAArch64NeonSVEBridge.def
new file mode 100644
index 000000000000..8e3229984d8b
--- /dev/null
+++ b/clang/include/clang/Basic/BuiltinsAArch64NeonSVEBridge.def
@@ -0,0 +1,39 @@
+#ifdef GET_SVE_BUILTINS
+BUILTIN(__builtin_sve_svget_neonq_s8, "V16Scq16Sc", "n")
+BUILTIN(__builtin_sve_svget_neonq_s16, "V8sq8s", "n")
+BUILTIN(__builtin_sve_svget_neonq_s32, "V4iq4i", "n")
+BUILTIN(__builtin_sve_svget_neonq_s64, "V2Wiq2Wi", "n")
+BUILTIN(__builtin_sve_svget_neonq_u8, "V16Ucq16Uc", "n")
+BUILTIN(__builtin_sve_svget_neonq_u16, "V16Usq16Us", "n")
+BUILTIN(__builtin_sve_svget_neonq_u32, "V4Uiq4Ui", "n")
+BUILTIN(__builtin_sve_svget_neonq_u64, "V2UWiq2UWi", "n")
+BUILTIN(__builtin_sve_svget_neonq_f16, "V8hq8h", "n")
+BUILTIN(__builtin_sve_svget_neonq_f32, "V4fq4f", "n")
+BUILTIN(__builtin_sve_svget_neonq_f64, "V2dq2d", "n")
+BUILTIN(__builtin_sve_svget_neonq_bf16, "V8yq8y", "n")
+BUILTIN(__builtin_sve_svset_neonq_s8, "q16Scq16ScV16Sc", "n")
+BUILTIN(__builtin_sve_svset_neonq_s16, "q8sq8sV8s", "n")
+BUILTIN(__builtin_sve_svset_neonq_s32, "q4iq4iV4i", "n")
+BUILTIN(__builtin_sve_svset_neonq_s64, "q2Wiq2WiV2Wi", "n")
+BUILTIN(__builtin_sve_svset_neonq_u8, "q16Ucq16UcV16Uc", "n")
+BUILTIN(__builtin_sve_svset_neonq_u16, "q8Usq8UsV8s", "n")
+BUILTIN(__builtin_sve_svset_neonq_u32, "q4Uiq4UiV4Ui", "n")
+BUILTIN(__builtin_sve_svset_neonq_u64, "q2UWiq2UWiV2UWi", "n")
+BUILTIN(__builtin_sve_svset_neonq_f16, "q8hq8hV8h", "n")
+BUILTIN(__builtin_sve_svset_neonq_f32, "q4fq4fV4f", "n")
+BUILTIN(__builtin_sve_svset_neonq_f64, "q2dq2dV2d", "n")
+BUILTIN(__builtin_sve_svset_neonq_bf16, "q8yq8yV8y", "n")
+BUILTIN(__builtin_sve_svdup_neonq_s8, "q16ScV16Sc", "n")
+BUILTIN(__builtin_sve_svdup_neonq_s16, "q8sV8s", "n")
+BUILTIN(__builtin_sve_svdup_neonq_s32, "q4iV4i", "n")
+BUILTIN(__builtin_sve_svdup_neonq_s64, "q4iV4i", "n")
+BUILTIN(__builtin_sve_svdup_neonq_u8, "q16UcV16Uc", "n")
+BUILTIN(__builtin_sve_svdup_neonq_u16, "q8UsV8Us", "n")
+BUILTIN(__builtin_sve_svdup_neonq_u32, "q4UiV4Ui", "n")
+BUILTIN(__builtin_sve_svdup_neonq_u64, "q2UWiV2UWi", "n")
+BUILTIN(__builtin_sve_svdup_neonq_f16, "q8hV8h", "n")
+BUILTIN(__builtin_sve_svdup_neonq_f32, "q4fV4f", "n")
+BUILTIN(__builtin_sve_svdup_neonq_f64, "q2dV2d", "n")
+BUILTIN(__builtin_sve_svdup_neonq_bf16, "q8yV8y", "n")
+#endif
+
diff --git a/clang/include/clang/Basic/BuiltinsAArch64NeonSVEBridge_cg.def b/clang/include/clang/Basic/BuiltinsAArch64NeonSVEBridge_cg.def
new file mode 100644
index 000000000000..7717ba67b427
--- /dev/null
+++ b/clang/include/clang/Basic/BuiltinsAArch64NeonSVEBridge_cg.def
@@ -0,0 +1,39 @@
+#ifdef GET_SVE_LLVM_INTRINSIC_MAP
+SVEMAP2(svget_neonq_s8, SVETypeFlags::EltTyInt8),
+SVEMAP2(svget_neonq_s16, SVETypeFlags::EltTyInt16),
+SVEMAP2(svget_neonq_s32, SVETypeFlags::EltTyInt32),
+SVEMAP2(svget_neonq_s64, SVETypeFlags::EltTyInt64),
+SVEMAP2(svget_neonq_u8, SVETypeFlags::EltTyInt8),
+SVEMAP2(svget_neonq_u16, SVETypeFlags::EltTyInt16),
+SVEMAP2(svget_neonq_u32, SVETypeFlags::EltTyInt32),
+SVEMAP2(svget_neonq_u64, SVETypeFlags::EltTyInt64),
+SVEMAP2(svget_neonq_f16, SVETypeFlags::EltTyFloat16),
+SVEMAP2(svget_neonq_f32, SVETypeFlags::EltTyFloat32),
+SVEMAP2(svget_neonq_f64, SVETypeFlags::EltTyFloat64),
+SVEMAP2(svget_neonq_bf16, SVETypeFlags::EltTyBFloat16),
+SVEMAP2(svset_neonq_s8, SVETypeFlags::EltTyInt8),
+SVEMAP2(svset_neonq_s16, SVETypeFlags::EltTyInt16),
+SVEMAP2(svset_neonq_s32, SVETypeFlags::EltTyInt32),
+SVEMAP2(svset_neonq_s64, SVETypeFlags::EltTyInt64),
+SVEMAP2(svset_neonq_u8, SVETypeFlags::EltTyInt8),
+SVEMAP2(svset_neonq_u16, SVETypeFlags::EltTyInt16),
+SVEMAP2(svset_neonq_u32, SVETypeFlags::EltTyInt32),
+SVEMAP2(svset_neonq_u64, SVETypeFlags::EltTyInt64),
+SVEMAP2(svset_neonq_f16, SVETypeFlags::EltTyFloat16),
+SVEMAP2(svset_neonq_f32, SVETypeFlags::EltTyFloat32),
+SVEMAP2(svset_neonq_f64, SVETypeFlags::EltTyFloat64),
+SVEMAP2(svset_neonq_bf16, SVETypeFlags::EltTyBFloat16),
+SVEMAP2(svdup_neonq_s8, SVETypeFlags::EltTyInt8),
+SVEMAP2(svdup_neonq_s16, SVETypeFlags::EltTyInt16),
+SVEMAP2(svdup_neonq_s32, SVETypeFlags::EltTyInt32),
+SVEMAP2(svdup_neonq_s64, SVETypeFlags::EltTyInt64),
+SVEMAP2(svdup_neonq_u8, SVETypeFlags::EltTyInt8),
+SVEMAP2(svdup_neonq_u16, SVETypeFlags::EltTyInt16),
+SVEMAP2(svdup_neonq_u32, SVETypeFlags::EltTyInt32),
+SVEMAP2(svdup_neonq_u64, SVETypeFlags::EltTyInt64),
+SVEMAP2(svdup_neonq_f16, SVETypeFlags::EltTyFloat16),
+SVEMAP2(svdup_neonq_f32, SVETypeFlags::EltTyFloat32),
+SVEMAP2(svdup_neonq_f64, SVETypeFlags::EltTyFloat64),
+SVEMAP2(svdup_neonq_bf16, SVETypeFlags::EltTyBFloat16),
+#endif
+
diff --git a/clang/include/clang/Basic/BuiltinsHexagon.def b/clang/include/clang/Basic/BuiltinsHexagon.def
index 0001bd556117..0f62c235bb62 100644
--- a/clang/include/clang/Basic/BuiltinsHexagon.def
+++ b/clang/include/clang/Basic/BuiltinsHexagon.def
@@ -17,8 +17,10 @@
 #   define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE) BUILTIN(ID, TYPE, ATTRS)
 #endif
 
+#pragma push_macro("V69")
+#define V69 "v69"
 #pragma push_macro("V68")
-#define V68 "v68"
+#define V68 "v68|" V69
 #pragma push_macro("V67")
 #define V67 "v67|" V68
 #pragma push_macro("V66")
@@ -34,8 +36,10 @@
 #pragma push_macro("V5")
 #define V5 "v5|" V55
 
+#pragma push_macro("HVXV69")
+#define HVXV69 "hvxv69"
 #pragma push_macro("HVXV68")
-#define HVXV68 "hvxv68"
+#define HVXV68 "hvxv68|" HVXV69
 #pragma push_macro("HVXV67")
 #define HVXV67 "hvxv67|" HVXV68
 #pragma push_macro("HVXV66")
@@ -128,6 +132,7 @@ TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpyub_rtt_acc_128B,"V64iV64iV32iLLi","", "
 #pragma pop_macro("HVXV66")
 #pragma pop_macro("HVXV67")
 #pragma pop_macro("HVXV68")
+#pragma pop_macro("HVXV69")
 
 #pragma pop_macro("V5")
 #pragma pop_macro("V55")
@@ -137,6 +142,7 @@ TARGET_BUILTIN(__builtin_HEXAGON_V6_vrmpyub_rtt_acc_128B,"V64iV64iV32iLLi","", "
 #pragma pop_macro("V66")
 #pragma pop_macro("V67")
 #pragma pop_macro("V68")
+#pragma pop_macro("V69")
 
 #undef BUILTIN
 #undef TARGET_BUILTIN
diff --git a/clang/include/clang/Basic/BuiltinsHexagonDep.def b/clang/include/clang/Basic/BuiltinsHexagonDep.def
index 152c9c4dd8ad..2eb4ca69c7bd 100644
--- a/clang/include/clang/Basic/BuiltinsHexagonDep.def
+++ b/clang/include/clang/Basic/BuiltinsHexagonDep.def
@@ -1739,3 +1739,150 @@ TARGET_BUILTIN(__builtin_HEXAGON_V6_v6mpyvubs10, "V32iV32iV32iUIi", "", HVXV68)
 TARGET_BUILTIN(__builtin_HEXAGON_V6_v6mpyvubs10_128B, "V64iV64iV64iUIi", "", HVXV68)
 TARGET_BUILTIN(__builtin_HEXAGON_V6_v6mpyvubs10_vxx, "V32iV32iV32iV32iUIi", "", HVXV68)
 TARGET_BUILTIN(__builtin_HEXAGON_V6_v6mpyvubs10_vxx_128B, "V64iV64iV64iV64iUIi", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vabs_hf, "V16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vabs_hf_128B, "V32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vabs_sf, "V16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vabs_sf_128B, "V32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_hf, "V16iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_hf_128B, "V32iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_hf_hf, "V16iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_hf_hf_128B, "V32iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_qf16, "V16iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_qf16_128B, "V32iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_qf16_mix, "V16iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_qf16_mix_128B, "V32iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_qf32, "V16iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_qf32_128B, "V32iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_qf32_mix, "V16iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_qf32_mix_128B, "V32iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_sf, "V16iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_sf_128B, "V32iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_sf_hf, "V32iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_sf_hf_128B, "V64iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_sf_sf, "V16iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vadd_sf_sf_128B, "V32iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vassign_fp, "V16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vassign_fp_128B, "V32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vconv_hf_qf16, "V16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vconv_hf_qf16_128B, "V32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vconv_hf_qf32, "V16iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vconv_hf_qf32_128B, "V32iV64i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vconv_sf_qf32, "V16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vconv_sf_qf32_128B, "V32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_b_hf, "V16iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_b_hf_128B, "V32iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_h_hf, "V16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_h_hf_128B, "V32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_hf_b, "V32iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_hf_b_128B, "V64iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_hf_h, "V16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_hf_h_128B, "V32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_hf_sf, "V16iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_hf_sf_128B, "V32iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_hf_ub, "V32iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_hf_ub_128B, "V64iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_hf_uh, "V16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_hf_uh_128B, "V32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_sf_hf, "V32iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_sf_hf_128B, "V64iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_ub_hf, "V16iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_ub_hf_128B, "V32iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_uh_hf, "V16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vcvt_uh_hf_128B, "V32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpy_sf_hf, "V16iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpy_sf_hf_128B, "V32iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpy_sf_hf_acc, "V16iV16iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vdmpy_sf_hf_acc_128B, "V32iV32iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vfmax_hf, "V16iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vfmax_hf_128B, "V32iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vfmax_sf, "V16iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vfmax_sf_128B, "V32iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vfmin_hf, "V16iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vfmin_hf_128B, "V32iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vfmin_sf, "V16iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vfmin_sf_128B, "V32iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vfneg_hf, "V16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vfneg_hf_128B, "V32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vfneg_sf, "V16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vfneg_sf_128B, "V32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vgthf, "V64bV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vgthf_128B, "V128bV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vgthf_and, "V64bV64bV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vgthf_and_128B, "V128bV128bV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vgthf_or, "V64bV64bV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vgthf_or_128B, "V128bV128bV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vgthf_xor, "V64bV64bV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vgthf_xor_128B, "V128bV128bV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtsf, "V64bV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtsf_128B, "V128bV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtsf_and, "V64bV64bV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtsf_and_128B, "V128bV128bV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtsf_or, "V64bV64bV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtsf_or_128B, "V128bV128bV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtsf_xor, "V64bV64bV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vgtsf_xor_128B, "V128bV128bV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vmax_hf, "V16iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vmax_hf_128B, "V32iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vmax_sf, "V16iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vmax_sf_128B, "V32iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vmin_hf, "V16iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vmin_hf_128B, "V32iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vmin_sf, "V16iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vmin_sf_128B, "V32iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_hf_hf, "V16iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_hf_hf_128B, "V32iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_hf_hf_acc, "V16iV16iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_hf_hf_acc_128B, "V32iV32iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf16, "V16iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf16_128B, "V32iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf16_hf, "V16iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf16_hf_128B, "V32iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf16_mix_hf, "V16iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf16_mix_hf_128B, "V32iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf32, "V16iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf32_128B, "V32iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf32_hf, "V32iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf32_hf_128B, "V64iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf32_mix_hf, "V32iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf32_mix_hf_128B, "V64iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf32_qf16, "V32iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf32_qf16_128B, "V64iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf32_sf, "V16iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_qf32_sf_128B, "V32iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_sf_hf, "V32iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_sf_hf_128B, "V64iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_sf_hf_acc, "V32iV32iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_sf_hf_acc_128B, "V64iV64iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_sf_sf, "V16iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpy_sf_sf_128B, "V32iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_hf, "V16iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_hf_128B, "V32iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_hf_hf, "V16iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_hf_hf_128B, "V32iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_qf16, "V16iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_qf16_128B, "V32iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_qf16_mix, "V16iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_qf16_mix_128B, "V32iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_qf32, "V16iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_qf32_128B, "V32iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_qf32_mix, "V16iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_qf32_mix_128B, "V32iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_sf, "V16iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_sf_128B, "V32iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_sf_hf, "V32iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_sf_hf_128B, "V64iV32iV32i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_sf_sf, "V16iV16iV16i", "", HVXV68)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vsub_sf_sf_128B, "V32iV32iV32i", "", HVXV68)
+
+// V69 HVX Instructions.
+
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrvuhubrndsat, "V16iV32iV16i", "", HVXV69)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrvuhubrndsat_128B, "V32iV64iV32i", "", HVXV69)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrvuhubsat, "V16iV32iV16i", "", HVXV69)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrvuhubsat_128B, "V32iV64iV32i", "", HVXV69)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrvwuhrndsat, "V16iV32iV16i", "", HVXV69)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrvwuhrndsat_128B, "V32iV64iV32i", "", HVXV69)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrvwuhsat, "V16iV32iV16i", "", HVXV69)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vasrvwuhsat_128B, "V32iV64iV32i", "", HVXV69)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyuhvs, "V16iV16iV16i", "", HVXV69)
+TARGET_BUILTIN(__builtin_HEXAGON_V6_vmpyuhvs_128B, "V32iV32iV32i", "", HVXV69)
diff --git a/clang/include/clang/Basic/BuiltinsHexagonMapCustomDep.def b/clang/include/clang/Basic/BuiltinsHexagonMapCustomDep.def
index 93f560fc5adc..389eadb21d01 100644
--- a/clang/include/clang/Basic/BuiltinsHexagonMapCustomDep.def
+++ b/clang/include/clang/Basic/BuiltinsHexagonMapCustomDep.def
@@ -8,199 +8,7 @@
 // Automatically generated file, do not edit!
 //===----------------------------------------------------------------------===//
 
-CUSTOM_BUILTIN_MAPPING(A2_add, 0)
-CUSTOM_BUILTIN_MAPPING(A2_addi, 0)
-CUSTOM_BUILTIN_MAPPING(A2_addp, 0)
-CUSTOM_BUILTIN_MAPPING(A2_and, 0)
-CUSTOM_BUILTIN_MAPPING(A2_andir, 0)
-CUSTOM_BUILTIN_MAPPING(A2_neg, 0)
-CUSTOM_BUILTIN_MAPPING(A2_not, 0)
-CUSTOM_BUILTIN_MAPPING(A2_or, 0)
-CUSTOM_BUILTIN_MAPPING(A2_orir, 0)
-CUSTOM_BUILTIN_MAPPING(A2_sub, 0)
-CUSTOM_BUILTIN_MAPPING(A2_subp, 0)
-CUSTOM_BUILTIN_MAPPING(A2_subri, 0)
-CUSTOM_BUILTIN_MAPPING(A2_sxtb, 0)
-CUSTOM_BUILTIN_MAPPING(A2_sxth, 0)
-CUSTOM_BUILTIN_MAPPING(A2_xor, 0)
-CUSTOM_BUILTIN_MAPPING(A2_zxtb, 0)
-CUSTOM_BUILTIN_MAPPING(A2_zxth, 0)
-CUSTOM_BUILTIN_MAPPING(M2_dpmpyss_s0, 0)
-CUSTOM_BUILTIN_MAPPING(M2_dpmpyuu_s0, 0)
-CUSTOM_BUILTIN_MAPPING(M2_mpyi, 0)
-CUSTOM_BUILTIN_MAPPING(M2_mpysmi, 0)
-CUSTOM_BUILTIN_MAPPING(M2_mpyui, 0)
-CUSTOM_BUILTIN_MAPPING(S2_asl_i_p, 0)
-CUSTOM_BUILTIN_MAPPING(S2_asl_i_r, 0)
-CUSTOM_BUILTIN_MAPPING(S2_asr_i_p, 0)
-CUSTOM_BUILTIN_MAPPING(S2_asr_i_r, 0)
-CUSTOM_BUILTIN_MAPPING(S2_lsr_i_p, 0)
-CUSTOM_BUILTIN_MAPPING(S2_lsr_i_r, 0)
-CUSTOM_BUILTIN_MAPPING(V6_pred_and, 64)
-CUSTOM_BUILTIN_MAPPING(V6_pred_and_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_pred_and_n, 64)
-CUSTOM_BUILTIN_MAPPING(V6_pred_and_n_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_pred_not, 64)
-CUSTOM_BUILTIN_MAPPING(V6_pred_not_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_pred_or, 64)
-CUSTOM_BUILTIN_MAPPING(V6_pred_or_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_pred_or_n, 64)
-CUSTOM_BUILTIN_MAPPING(V6_pred_or_n_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_pred_scalar2, 64)
-CUSTOM_BUILTIN_MAPPING(V6_pred_scalar2_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_pred_xor, 64)
-CUSTOM_BUILTIN_MAPPING(V6_pred_xor_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vS32b_nqpred_ai, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vS32b_nqpred_ai_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vS32b_nt_nqpred_ai, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vS32b_nt_nqpred_ai_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vS32b_nt_qpred_ai, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vS32b_nt_qpred_ai_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vS32b_qpred_ai, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vS32b_qpred_ai_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vaddbnq, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vaddbnq_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vaddbq, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vaddbq_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vaddhnq, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vaddhnq_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vaddhq, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vaddhq_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vaddwnq, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vaddwnq_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vaddwq, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vaddwq_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vandqrt, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vandqrt_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vandqrt_acc, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vandqrt_acc_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vandvrt, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vandvrt_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vandvrt_acc, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vandvrt_acc_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_veqb, 64)
-CUSTOM_BUILTIN_MAPPING(V6_veqb_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_veqb_and, 64)
-CUSTOM_BUILTIN_MAPPING(V6_veqb_and_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_veqb_or, 64)
-CUSTOM_BUILTIN_MAPPING(V6_veqb_or_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_veqb_xor, 64)
-CUSTOM_BUILTIN_MAPPING(V6_veqb_xor_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_veqh, 64)
-CUSTOM_BUILTIN_MAPPING(V6_veqh_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_veqh_and, 64)
-CUSTOM_BUILTIN_MAPPING(V6_veqh_and_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_veqh_or, 64)
-CUSTOM_BUILTIN_MAPPING(V6_veqh_or_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_veqh_xor, 64)
-CUSTOM_BUILTIN_MAPPING(V6_veqh_xor_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_veqw, 64)
-CUSTOM_BUILTIN_MAPPING(V6_veqw_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_veqw_and, 64)
-CUSTOM_BUILTIN_MAPPING(V6_veqw_and_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_veqw_or, 64)
-CUSTOM_BUILTIN_MAPPING(V6_veqw_or_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_veqw_xor, 64)
-CUSTOM_BUILTIN_MAPPING(V6_veqw_xor_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vgtb, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vgtb_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vgtb_and, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vgtb_and_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vgtb_or, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vgtb_or_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vgtb_xor, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vgtb_xor_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vgth, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vgth_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vgth_and, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vgth_and_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vgth_or, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vgth_or_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vgth_xor, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vgth_xor_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vgtub, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vgtub_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vgtub_and, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vgtub_and_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vgtub_or, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vgtub_or_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vgtub_xor, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vgtub_xor_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vgtuh, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vgtuh_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vgtuh_and, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vgtuh_and_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vgtuh_or, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vgtuh_or_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vgtuh_xor, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vgtuh_xor_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vgtuw, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vgtuw_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vgtuw_and, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vgtuw_and_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vgtuw_or, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vgtuw_or_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vgtuw_xor, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vgtuw_xor_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vgtw, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vgtw_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vgtw_and, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vgtw_and_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vgtw_or, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vgtw_or_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vgtw_xor, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vgtw_xor_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vmux, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vmux_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vsubbnq, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vsubbnq_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vsubbq, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vsubbq_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vsubhnq, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vsubhnq_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vsubhq, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vsubhq_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vsubwnq, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vsubwnq_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vsubwq, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vsubwq_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vswap, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vswap_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_pred_scalar2v2, 64)
-CUSTOM_BUILTIN_MAPPING(V6_pred_scalar2v2_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_shuffeqh, 64)
-CUSTOM_BUILTIN_MAPPING(V6_shuffeqh_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_shuffeqw, 64)
-CUSTOM_BUILTIN_MAPPING(V6_shuffeqw_128B, 128)
 CUSTOM_BUILTIN_MAPPING(V6_vaddcarry, 64)
 CUSTOM_BUILTIN_MAPPING(V6_vaddcarry_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vandnqrt, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vandnqrt_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vandnqrt_acc, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vandnqrt_acc_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vandvnqv, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vandvnqv_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vandvqv, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vandvqv_128B, 128)
 CUSTOM_BUILTIN_MAPPING(V6_vsubcarry, 64)
 CUSTOM_BUILTIN_MAPPING(V6_vsubcarry_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vgathermhq, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vgathermhq_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vgathermhwq, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vgathermhwq_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vgathermwq, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vgathermwq_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vprefixqb, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vprefixqb_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vprefixqh, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vprefixqh_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vprefixqw, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vprefixqw_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vscattermhq, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vscattermhq_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vscattermhwq, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vscattermhwq_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vscattermwq, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vscattermwq_128B, 128)
-CUSTOM_BUILTIN_MAPPING(V6_vaddcarrysat, 64)
-CUSTOM_BUILTIN_MAPPING(V6_vaddcarrysat_128B, 128)
diff --git a/clang/include/clang/Basic/BuiltinsSVE.def b/clang/include/clang/Basic/BuiltinsSVE.def
index 2839ca992d98..c70525e967b1 100644
--- a/clang/include/clang/Basic/BuiltinsSVE.def
+++ b/clang/include/clang/Basic/BuiltinsSVE.def
@@ -15,6 +15,7 @@
 
 #define GET_SVE_BUILTINS
 #include "clang/Basic/arm_sve_builtins.inc"
+#include "clang/Basic/BuiltinsAArch64NeonSVEBridge.def"
 #undef GET_SVE_BUILTINS
 
 #undef BUILTIN
diff --git a/clang/include/clang/Basic/CodeGenOptions.def b/clang/include/clang/Basic/CodeGenOptions.def
index 94b3003a9c33..723302f108e2 100644
--- a/clang/include/clang/Basic/CodeGenOptions.def
+++ b/clang/include/clang/Basic/CodeGenOptions.def
@@ -368,9 +368,6 @@ ENUM_CODEGENOPT(DebuggerTuning, llvm::DebuggerKind, 3,
 /// emitted.
 VALUE_CODEGENOPT(DwarfVersion, 3, 0)
 
-/// Whether to use experimental new variable location tracking.
-CODEGENOPT(ValueTrackingVariableLocations, 1, 0)
-
 /// Whether we should emit CodeView debug information. It's possible to emit
 /// CodeView and DWARF into the same object.
 CODEGENOPT(EmitCodeView, 1, 0)
diff --git a/clang/include/clang/Basic/CodeGenOptions.h b/clang/include/clang/Basic/CodeGenOptions.h
index 664e4998b8de..33ec03a17136 100644
--- a/clang/include/clang/Basic/CodeGenOptions.h
+++ b/clang/include/clang/Basic/CodeGenOptions.h
@@ -227,6 +227,9 @@ public:
   /// Output filename for the split debug info, not used in the skeleton CU.
   std::string SplitDwarfOutput;
 
+  /// Output filename used in the COFF debug information.
+  std::string ObjectFilenameForDebug;
+
   /// The name of the relocation model to use.
   llvm::Reloc::Model RelocationModel;
 
@@ -395,7 +398,7 @@ public:
   /// Executable and command-line used to create a given CompilerInvocation.
   /// Most of the time this will be the full -cc1 command.
   const char *Argv0 = nullptr;
-  ArrayRef<const char *> CommandLineArgs;
+  std::vector<std::string> CommandLineArgs;
 
   /// The minimum hotness value a diagnostic needs in order to be included in
   /// optimization diagnostics.
diff --git a/clang/include/clang/Basic/Cuda.h b/clang/include/clang/Basic/Cuda.h
index 8c08ab3f5d74..04967925e9a4 100644
--- a/clang/include/clang/Basic/Cuda.h
+++ b/clang/include/clang/Basic/Cuda.h
@@ -95,6 +95,8 @@ enum class CudaArch {
   GFX1033,
   GFX1034,
   GFX1035,
+  Generic, // A processor model named 'generic' if the target backend defines a
+           // public one.
   LAST,
 };
 
@@ -103,7 +105,8 @@ static inline bool IsNVIDIAGpuArch(CudaArch A) {
 }
 
 static inline bool IsAMDGpuArch(CudaArch A) {
-  return A >= CudaArch::GFX600 && A < CudaArch::LAST;
+  // Generic processor model is for testing only.
+  return A >= CudaArch::GFX600 && A < CudaArch::Generic;
 }
 
 const char *CudaArchToString(CudaArch A);
diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td
index 2f50918b527b..a7fd2f26478c 100644
--- a/clang/include/clang/Basic/DiagnosticDriverKinds.td
+++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td
@@ -74,6 +74,14 @@ def err_drv_no_hip_runtime : Error<
   "cannot find HIP runtime; provide its path via '--rocm-path', or pass "
   "'-nogpuinc' to build without HIP runtime">;
 
+def err_drv_no_hipspv_device_lib : Error<
+  "cannot find HIP device library%select{| for %1}0; provide its path via "
+  "'--hip-path' or '--hip-device-lib-path', or pass '-nogpulib' to build "
+  "without HIP device library">;
+def err_drv_hipspv_no_hip_path : Error<
+  "'--hip-path' must be specified when offloading to "
+  "SPIR-V%select{| unless %1 is given}0.">;
+
 def err_drv_undetermined_amdgpu_arch : Error<
   "cannot determine AMDGPU architecture: %0; consider passing it via "
   "'--march'">;
@@ -108,6 +116,10 @@ def warn_drv_unsupported_option_for_target : Warning<
   "ignoring '%0' option as it is not currently supported for target '%1'">,
   InGroup<OptionIgnored>;
 
+def warn_drv_spirv_linking_multiple_inputs_unsupported: Warning<
+  "Linking multiple input files is not supported for SPIR-V yet">,
+  InGroup<OptionIgnored>;
+
 def err_drv_invalid_thread_model_for_target : Error<
   "invalid thread model '%0' in '%1' for this target">;
 def err_drv_invalid_linker_name : Error<
@@ -431,11 +443,13 @@ def err_analyzer_checker_option_invalid_input : Error<
 def err_analyzer_checker_incompatible_analyzer_option : Error<
   "checker cannot be enabled with analyzer option '%0' == %1">;
 
-def err_drv_invalid_hvx_length : Error<
-  "-mhvx-length is not supported without a -mhvx/-mhvx= flag">;
-def warn_drv_vectorize_needs_hvx : Warning<
-  "auto-vectorization requires HVX, use -mhvx to enable it">,
+def warn_drv_needs_hvx : Warning<
+  "%0 requires HVX, use -mhvx/-mhvx= to enable it">,
   InGroup<OptionIgnored>;
+def err_drv_needs_hvx : Error<
+  "%0 requires HVX, use -mhvx/-mhvx= to enable it">;
+def err_drv_needs_hvx_version : Error<
+  "%0 is not supported on HVX %1">;
 
 def err_drv_module_header_wrong_kind : Error<
   "header file '%0' input type '%1' does not match type of prior input "
@@ -551,7 +565,7 @@ def warn_drv_moutline_unsupported_opt : Warning<
   InGroup<OptionIgnored>;
 
 def warn_drv_moutline_atomics_unsupported_opt : Warning<
-  "'%0' does not support '-moutline-atomics'; flag ignored">,
+  "'%0' does not support '-%1'; flag ignored">,
   InGroup<OptionIgnored>;
 
 def warn_drv_darwin_sdk_invalid_settings : Warning<
@@ -608,7 +622,14 @@ def err_cc1_round_trip_ok_then_fail : Error<
   "generated arguments parse failed in round-trip">;
 def err_cc1_round_trip_mismatch : Error<
   "generated arguments do not match in round-trip">;
+def err_cc1_unbounded_vscale_min : Error<
+  "minimum vscale must be an unsigned integer greater than 0">;
 
 def err_drv_ssp_missing_offset_argument : Error<
   "'%0' is used without '-mstack-protector-guard-offset', and there is no default">;
+
+def err_drv_only_one_offload_target_supported_in : Error<
+  "Only one offload target is supported in %0.">;
+def err_drv_invalid_or_unsupported_offload_target : Error<
+  "Invalid or unsupported offload target: '%0'.">;
 }
diff --git a/clang/include/clang/Basic/DiagnosticGroups.td b/clang/include/clang/Basic/DiagnosticGroups.td
index 90df3a424406..c0642efaee4e 100644
--- a/clang/include/clang/Basic/DiagnosticGroups.td
+++ b/clang/include/clang/Basic/DiagnosticGroups.td
@@ -191,6 +191,7 @@ def DeprecatedVolatile : DiagGroup<"deprecated-volatile">;
 def DeprecatedWritableStr : DiagGroup<"deprecated-writable-strings",
                                       [CXX11CompatDeprecatedWritableStr]>;
 def DeprecatedPragma : DiagGroup<"deprecated-pragma">;
+def DeprecatedType : DiagGroup<"deprecated-type">;
 // FIXME: Why is DeprecatedImplementations not in this group?
 def Deprecated : DiagGroup<"deprecated", [DeprecatedAnonEnumEnumConversion,
                                           DeprecatedArrayCompare,
@@ -208,6 +209,7 @@ def Deprecated : DiagGroup<"deprecated", [DeprecatedAnonEnumEnumConversion,
                                           DeprecatedPragma,
                                           DeprecatedRegister,
                                           DeprecatedThisCapture,
+                                          DeprecatedType,
                                           DeprecatedVolatile,
                                           DeprecatedWritableStr]>,
                  DiagCategory<"Deprecations">;
@@ -1044,6 +1046,13 @@ def : DiagGroup<"unused-local-typedefs", [UnusedLocalTypedef]>;
 def NonGCC : DiagGroup<"non-gcc",
     [SignCompare, Conversion, LiteralRange]>;
 
+def CXX14Attrs : DiagGroup<"c++14-attribute-extensions">;
+def CXX17Attrs : DiagGroup<"c++17-attribute-extensions">;
+def CXX20Attrs : DiagGroup<"c++20-attribute-extensions">;
+def FutureAttrs : DiagGroup<"future-attribute-extensions", [CXX14Attrs,
+                                                            CXX17Attrs,
+                                                            CXX20Attrs]>;
+
 // A warning group for warnings about using C++11 features as extensions in
 // earlier C++ versions.
 def CXX11 : DiagGroup<"c++11-extensions", [CXX11ExtraSemi, CXX11InlineNamespace,
@@ -1051,15 +1060,15 @@ def CXX11 : DiagGroup<"c++11-extensions", [CXX11ExtraSemi, CXX11InlineNamespace,
 
 // A warning group for warnings about using C++14 features as extensions in
 // earlier C++ versions.
-def CXX14 : DiagGroup<"c++14-extensions", [CXX14BinaryLiteral]>;
+def CXX14 : DiagGroup<"c++14-extensions", [CXX14BinaryLiteral, CXX14Attrs]>;
 
 // A warning group for warnings about using C++17 features as extensions in
 // earlier C++ versions.
-def CXX17 : DiagGroup<"c++17-extensions">;
+def CXX17 : DiagGroup<"c++17-extensions", [CXX17Attrs]>;
 
 // A warning group for warnings about using C++20 features as extensions in
 // earlier C++ versions.
-def CXX20 : DiagGroup<"c++20-extensions", [CXX20Designator]>;
+def CXX20 : DiagGroup<"c++20-extensions", [CXX20Designator, CXX20Attrs]>;
 
 // A warning group for warnings about using C++2b features as extensions in
 // earlier C++ versions.
diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td
index 92e877074ad3..9dc036c03faa 100644
--- a/clang/include/clang/Basic/DiagnosticParseKinds.td
+++ b/clang/include/clang/Basic/DiagnosticParseKinds.td
@@ -1503,6 +1503,15 @@ def warn_pragma_force_cuda_host_device_bad_arg : Warning<
 def err_pragma_cannot_end_force_cuda_host_device : Error<
   "force_cuda_host_device end pragma without matching "
   "force_cuda_host_device begin">;
+
+def warn_ext_int_deprecated : Warning<
+  "'_ExtInt' is deprecated; use '_BitInt' instead">, InGroup<DeprecatedType>;
+def ext_bit_int : Extension<
+  "'_BitInt' in %select{C17 and earlier|C++}0 is a Clang extension">,
+  InGroup<DiagGroup<"bit-int-extension">>;
+def warn_c17_compat_bit_int : Warning<
+  "'_BitInt' is incompatible with C standards before C2x">,
+  InGroup<CPre2xCompat>, DefaultIgnore;
 } // end of Parse Issue category.
 
 let CategoryName = "Modules Issue" in {
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index fb5bd53f7432..f2089bfda04d 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -838,6 +838,9 @@ def warn_fortify_scanf_overflow : Warning<
   "%2, but the corresponding specifier may require size %3">,
   InGroup<FortifySource>;
 
+def err_function_start_invalid_type: Error<
+  "argument must be a function">;
+
 /// main()
 // static main() is not an error in C, just in C++.
 def warn_static_main : Warning<"'main' should not be declared static">,
@@ -2956,6 +2959,17 @@ def err_attribute_invalid_argument : Error<
 def err_attribute_wrong_number_arguments : Error<
   "%0 attribute %plural{0:takes no arguments|1:takes one argument|"
   ":requires exactly %1 arguments}1">;
+def err_attribute_wrong_number_arguments_for : Error <
+  "%0 attribute references function %1, which %plural{0:takes no arguments|1:takes one argument|"
+  ":takes exactly %2 arguments}2">;
+def err_attribute_bounds_for_function : Error<
+  "%0 attribute references parameter %1, but the function %2 has only %3 parameters">;
+def err_attribute_no_member_function : Error<
+  "%0 attribute cannot be applied to non-static member functions">;
+def err_attribute_parameter_types : Error<
+  "%0 attribute parameter types do not match: parameter %1 of function %2 has type %3, "
+  "but parameter %4 of function %5 has type %6">;
+
 def err_attribute_too_many_arguments : Error<
   "%0 attribute takes no more than %1 argument%s1">;
 def err_attribute_too_few_arguments : Error<
@@ -3013,7 +3027,7 @@ def err_attribute_sizeless_type : Error<
   "%0 attribute cannot be applied to sizeless type %1">;
 def err_attribute_argument_n_type : Error<
   "%0 attribute requires parameter %1 to be %select{int or bool|an integer "
-  "constant|a string|an identifier|a constant expression}2">;
+  "constant|a string|an identifier|a constant expression|a builtin function}2">;
 def err_attribute_argument_type : Error<
   "%0 attribute requires %select{int or bool|an integer "
   "constant|a string|an identifier}1">;
@@ -8337,8 +8351,8 @@ def err_atomic_exclusive_builtin_pointer_size : Error<
   " 1,2,4 or 8 byte type (%0 invalid)">;
 def err_atomic_builtin_ext_int_size : Error<
   "Atomic memory operand must have a power-of-two size">;
-def err_atomic_builtin_ext_int_prohibit : Error<
-  "argument to atomic builtin of type '_ExtInt' is not supported">;
+def err_atomic_builtin_bit_int_prohibit : Error<
+  "argument to atomic builtin of type '_BitInt' is not supported">;
 def err_atomic_op_needs_atomic : Error<
   "address argument to atomic operation must be a pointer to _Atomic "
   "type (%0 invalid)">;
@@ -8374,8 +8388,8 @@ def err_overflow_builtin_must_be_int : Error<
 def err_overflow_builtin_must_be_ptr_int : Error<
   "result argument to overflow builtin must be a pointer "
   "to a non-const integer (%0 invalid)">;
-def err_overflow_builtin_ext_int_max_size : Error<
-  "__builtin_mul_overflow does not support signed _ExtInt operands of more "
+def err_overflow_builtin_bit_int_max_size : Error<
+  "__builtin_mul_overflow does not support 'signed _BitInt' operands of more "
   "than %0 bits">;
 
 def err_atomic_load_store_uses_lib : Error<
@@ -8626,11 +8640,11 @@ def warn_unused_volatile : Warning<
   InGroup<DiagGroup<"unused-volatile-lvalue">>;
 
 def ext_cxx14_attr : Extension<
-  "use of the %0 attribute is a C++14 extension">, InGroup<CXX14>;
+  "use of the %0 attribute is a C++14 extension">, InGroup<CXX14Attrs>;
 def ext_cxx17_attr : Extension<
-  "use of the %0 attribute is a C++17 extension">, InGroup<CXX17>;
+  "use of the %0 attribute is a C++17 extension">, InGroup<CXX17Attrs>;
 def ext_cxx20_attr : Extension<
-  "use of the %0 attribute is a C++20 extension">, InGroup<CXX20>;
+  "use of the %0 attribute is a C++20 extension">, InGroup<CXX20Attrs>;
 
 def warn_unused_comparison : Warning<
   "%select{equality|inequality|relational|three-way}0 comparison result unused">,
@@ -9140,15 +9154,22 @@ def warn_cxx17_compat_defaulted_comparison : Warning<
   "before C++20">, InGroup<CXXPre20Compat>, DefaultIgnore;
 def err_defaulted_comparison_template : Error<
   "comparison operator template cannot be defaulted">;
-def err_defaulted_comparison_out_of_class : Error<
-  "%sub{select_defaulted_comparison_kind}0 can only be defaulted in a class "
-  "definition">;
+def err_defaulted_comparison_num_args : Error<
+  "%select{non-member|member}0 %sub{select_defaulted_comparison_kind}1"
+  " comparison operator must have %select{2|1}0 parameters">;
 def err_defaulted_comparison_param : Error<
   "invalid parameter type for defaulted %sub{select_defaulted_comparison_kind}0"
   "; found %1, expected %2%select{| or %4}3">;
+def err_defaulted_comparison_param_unknown : Error<
+  "invalid parameter type for non-member defaulted"
+  " %sub{select_defaulted_comparison_kind}0"
+  "; found %1, expected class or reference to a constant class">;
 def err_defaulted_comparison_param_mismatch : Error<
   "parameters for defaulted %sub{select_defaulted_comparison_kind}0 "
   "must have the same type%diff{ (found $ vs $)|}1,2">;
+def err_defaulted_comparison_not_friend : Error<
+  "%sub{select_defaulted_comparison_kind}0 is not a friend of"
+  " %select{|incomplete class }1%2">;
 def err_defaulted_comparison_non_const : Error<
   "defaulted member %sub{select_defaulted_comparison_kind}0 must be "
   "const-qualified">;
@@ -9811,6 +9832,8 @@ def err_ppc_builtin_only_on_arch : Error<
   "this builtin is only valid on POWER%0 or later CPUs">;
 def err_ppc_builtin_requires_vsx : Error<
   "this builtin requires VSX to be enabled">;
+def err_ppc_builtin_requires_htm : Error<
+  "this builtin requires HTM to be enabled">;
 def err_ppc_builtin_requires_abi : Error<
   "this builtin requires ABI -mabi=%0">;
 def err_ppc_invalid_use_mma_type : Error<
@@ -10486,7 +10509,7 @@ def err_omp_atomic_capture_not_compound_statement : Error<
 def note_omp_atomic_capture: Note<
   "%select{expected assignment expression|expected compound statement|expected exactly two expression statements|expected in right hand side of the first expression}0">;
 def err_omp_atomic_several_clauses : Error<
-  "directive '#pragma omp atomic' cannot contain more than one 'read', 'write', 'update' or 'capture' clause">;
+  "directive '#pragma omp atomic' cannot contain more than one 'read', 'write', 'update', 'capture', or 'compare' clause">;
 def err_omp_several_mem_order_clauses : Error<
   "directive '#pragma omp %0' cannot contain more than one %select{'seq_cst', 'relaxed', |}1'acq_rel', 'acquire' or 'release' clause">;
 def err_omp_atomic_incompatible_mem_order_clause : Error<
@@ -11020,14 +11043,12 @@ def err_implied_coroutine_type_not_found : Error<
   "a coroutine; include <experimental/coroutine> if your version "
   "of libcxx is less than 14.0">;
 def warn_deprecated_coroutine_namespace : Warning<
-  "Please move from std::experimental::%0 to std::%0. "
-  "Support for std::experimental::%0 will be removed in LLVM 15.">,
+  "support for std::experimental::%0 will be removed in LLVM 15; "
+  "use std::%0 instead">,
   InGroup<DeprecatedExperimentalCoroutine>;
-def err_mixed_use_std_and_experimental_namespace_for_coroutine : Error <
-  "Found mixed use of std namespace and std::experimental namespace for "
-  "coroutine, which is disallowed. The coroutine components in "
-  "std::experimental namespace is deprecated. Please use coroutine components "
-  "under std namespace.">;
+def err_mixed_use_std_and_experimental_namespace_for_coroutine : Error<
+  "mixed use of std and std::experimental namespaces for "
+  "coroutine components">;
 def err_implicit_coroutine_std_nothrow_type_not_found : Error<
   "std::nothrow was not found; include <new> before defining a coroutine which "
   "uses get_return_object_on_allocation_failure()">;
@@ -11050,8 +11071,6 @@ def err_coroutine_type_missing_specialization : Error<
   "specialization %0">;
 def err_coroutine_promise_incompatible_return_functions : Error<
   "the coroutine promise type %0 declares both 'return_value' and 'return_void'">;
-def err_coroutine_promise_requires_return_function : Error<
-  "the coroutine promise type %0 must declare either 'return_value' or 'return_void'">;
 def note_coroutine_promise_implicit_await_transform_required_here : Note<
   "call to 'await_transform' implicitly required by 'co_await' here">;
 def note_coroutine_promise_suspend_implicitly_required : Note<
@@ -11355,8 +11374,10 @@ def err_builtin_launder_invalid_arg : Error<
 def err_builtin_invalid_arg_type: Error <
   "%ordinal0 argument must be a "
   "%select{vector, integer or floating point type|matrix|"
-   "pointer to a valid matrix element type|"
-   "signed integer or floating point type|vector type}1 (was %2)">;
+  "pointer to a valid matrix element type|"
+  "signed integer or floating point type|vector type|"
+  "floating point type|"
+  "vector of integers}1 (was %2)">;
 
 def err_builtin_matrix_disabled: Error<
   "matrix types extension is disabled. Pass -fenable-matrix to enable it">;
@@ -11423,9 +11444,9 @@ def warn_sycl_kernel_return_type : Warning<
   "function template with 'sycl_kernel' attribute must have a 'void' return type">,
   InGroup<IgnoredAttributes>;
 
-def err_ext_int_bad_size : Error<"%select{signed|unsigned}0 _ExtInt must "
+def err_bit_int_bad_size : Error<"%select{signed|unsigned}0 _BitInt must "
                                  "have a bit size of at least %select{2|1}0">;
-def err_ext_int_max_size : Error<"%select{signed|unsigned}0 _ExtInt of bit "
+def err_bit_int_max_size : Error<"%select{signed|unsigned}0 _BitInt of bit "
                                  "sizes greater than %1 not supported">;
 
 // errors of expect.with.probability
diff --git a/clang/include/clang/Basic/Module.h b/clang/include/clang/Basic/Module.h
index 3476b05d2e92..de7857347bc2 100644
--- a/clang/include/clang/Basic/Module.h
+++ b/clang/include/clang/Basic/Module.h
@@ -153,6 +153,10 @@ public:
     return Kind == ModuleInterfaceUnit || Kind == PrivateModuleFragment;
   }
 
+  /// Does this Module scope describe a fragment of the global module within
+  /// some C++ module.
+  bool isGlobalModule() const { return Kind == GlobalModuleFragment; }
+
 private:
   /// The submodules of this module, indexed by name.
   std::vector<Module *> SubModules;
diff --git a/clang/include/clang/Basic/Specifiers.h b/clang/include/clang/Basic/Specifiers.h
index 66cdba3f912e..82b3c1f3c274 100644
--- a/clang/include/clang/Basic/Specifiers.h
+++ b/clang/include/clang/Basic/Specifiers.h
@@ -59,7 +59,7 @@ namespace clang {
     TST_char32,       // C++11 char32_t
     TST_int,
     TST_int128,
-    TST_extint,       // Extended Int types.
+    TST_bitint,       // Bit-precise integer types.
     TST_half,         // OpenCL half, ARM NEON __fp16
     TST_Float16,      // C11 extension ISO/IEC TS 18661-3
     TST_Accum,        // ISO/IEC JTC1 SC22 WG14 N1169 Extension
diff --git a/clang/include/clang/Basic/TargetInfo.h b/clang/include/clang/Basic/TargetInfo.h
index 3e1e09417c66..437feba85e23 100644
--- a/clang/include/clang/Basic/TargetInfo.h
+++ b/clang/include/clang/Basic/TargetInfo.h
@@ -582,9 +582,13 @@ public:
     return (getPointerWidth(0) >= 64) || getTargetOpts().ForceEnableInt128;
   } // FIXME
 
-  /// Determine whether the _ExtInt type is supported on this target. This
+  /// Determine whether the _BitInt type is supported on this target. This
   /// limitation is put into place for ABI reasons.
-  virtual bool hasExtIntType() const {
+  /// FIXME: _BitInt is a required type in C23, so there's not much utility in
+  /// asking whether the target supported it or not; I think this should be
+  /// removed once backends have been alerted to the type and have had the
+  /// chance to do implementation work if needed.
+  virtual bool hasBitIntType() const {
     return false;
   }
 
diff --git a/clang/include/clang/Basic/TokenKinds.def b/clang/include/clang/Basic/TokenKinds.def
index 0dd5936aa3e6..e55244e1c3ac 100644
--- a/clang/include/clang/Basic/TokenKinds.def
+++ b/clang/include/clang/Basic/TokenKinds.def
@@ -300,6 +300,7 @@ KEYWORD(if                          , KEYALL)
 KEYWORD(inline                      , KEYC99|KEYCXX|KEYGNU)
 KEYWORD(int                         , KEYALL)
 KEYWORD(_ExtInt                     , KEYALL)
+KEYWORD(_BitInt                     , KEYALL)
 KEYWORD(long                        , KEYALL)
 KEYWORD(register                    , KEYALL)
 KEYWORD(restrict                    , KEYC99)
diff --git a/clang/include/clang/Basic/TypeNodes.td b/clang/include/clang/Basic/TypeNodes.td
index 011394c3ef45..b65e8ab521c0 100644
--- a/clang/include/clang/Basic/TypeNodes.td
+++ b/clang/include/clang/Basic/TypeNodes.td
@@ -75,6 +75,7 @@ def DependentSizedMatrixType : TypeNode<MatrixType>, AlwaysDependent;
 def FunctionType : TypeNode<Type, 1>;
 def FunctionProtoType : TypeNode<FunctionType>;
 def FunctionNoProtoType : TypeNode<FunctionType>;
+def UsingType : TypeNode<Type>, NeverCanonical;
 def UnresolvedUsingType : TypeNode<Type>, AlwaysDependent;
 def ParenType : TypeNode<Type>, NeverCanonical;
 def TypedefType : TypeNode<Type>, NeverCanonical;
@@ -107,5 +108,5 @@ def ObjCInterfaceType : TypeNode<ObjCObjectType>, LeafType;
 def ObjCObjectPointerType : TypeNode<Type>;
 def PipeType : TypeNode<Type>;
 def AtomicType : TypeNode<Type>;
-def ExtIntType : TypeNode<Type>;
-def DependentExtIntType : TypeNode<Type>, AlwaysDependent;
+def BitIntType : TypeNode<Type>;
+def DependentBitIntType : TypeNode<Type>, AlwaysDependent;
diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h
index 8b1f7091e701..9ae34a2eaf01 100644
--- a/clang/include/clang/Driver/Driver.h
+++ b/clang/include/clang/Driver/Driver.h
@@ -595,6 +595,21 @@ private:
 
   /// @}
 
+  /// Retrieves a ToolChain for a particular device \p Target triple
+  ///
+  /// \param[in] HostTC is the host ToolChain paired with the device
+  ///
+  /// \param[in] Action (e.g. OFK_Cuda/OFK_OpenMP/OFK_SYCL) is an Offloading
+  /// action that is optionally passed to a ToolChain (used by CUDA, to specify
+  /// if it's used in conjunction with OpenMP)
+  ///
+  /// Will cache ToolChains for the life of the driver object, and create them
+  /// on-demand.
+  const ToolChain &getOffloadingDeviceToolChain(
+      const llvm::opt::ArgList &Args, const llvm::Triple &Target,
+      const ToolChain &HostTC,
+      const Action::OffloadKind &TargetDeviceOffloadKind) const;
+
   /// Get bitmasks for which option flags to include and exclude based on
   /// the driver mode.
   std::pair<unsigned, unsigned> getIncludeExcludeOptionFlagMasks(bool IsClCompatMode) const;
diff --git a/clang/include/clang/Driver/Job.h b/clang/include/clang/Driver/Job.h
index 8b287638a271..6e3b51f2a799 100644
--- a/clang/include/clang/Driver/Job.h
+++ b/clang/include/clang/Driver/Job.h
@@ -204,6 +204,10 @@ public:
   ///         from the parent process will be used.
   virtual void setEnvironment(llvm::ArrayRef<const char *> NewEnvironment);
 
+  void replaceArguments(llvm::opt::ArgStringList List) {
+    Arguments = std::move(List);
+  }
+
   const char *getExecutable() const { return Executable; }
 
   const llvm::opt::ArgStringList &getArguments() const { return Arguments; }
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 4e6dd2050344..dc8bd831f2a2 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -985,6 +985,9 @@ defm hip_fp32_correctly_rounded_divide_sqrt : BoolFOption<"hip-fp32-correctly-ro
   BothFlags<[], " that single precision floating-point divide and sqrt used in "
   "the program source are correctly rounded (HIP device compilation only)">>,
   ShouldParseIf<hip.KeyPath>;
+def hipspv_pass_plugin_EQ : Joined<["--"], "hipspv-pass-plugin=">,
+  Group<Link_Group>, MetaVarName<"<dsopath>">,
+  HelpText<"path to a pass plugin for HIP to SPIR-V passes.">;
 defm gpu_allow_device_init : BoolFOption<"gpu-allow-device-init",
   LangOpts<"GPUAllowDeviceInit">, DefaultFalse,
   PosFlag<SetTrue, [CC1Option], "Allow">, NegFlag<SetFalse, [], "Don't allow">,
@@ -1132,6 +1135,13 @@ defm autolink : BoolFOption<"autolink",
   NegFlag<SetFalse, [CC1Option], "Disable generation of linker directives for automatic library linking">,
   PosFlag<SetTrue>>;
 
+// In the future this option will be supported by other offloading
+// languages and accept other values such as CPU/GPU architectures,
+// offload kinds and target aliases.
+def offload_EQ : CommaJoined<["--"], "offload=">, Flags<[NoXarchOption]>,
+  HelpText<"Specify comma-separated list of offloading target triples"
+           " (HIP only)">;
+
 // C++ Coroutines TS
 defm coroutines_ts : BoolFOption<"coroutines-ts",
   LangOpts<"Coroutines">, Default<cpp20.KeyPath>,
@@ -2392,7 +2402,8 @@ def fomit_frame_pointer : Flag<["-"], "fomit-frame-pointer">, Group<f_Group>;
 def fopenmp : Flag<["-"], "fopenmp">, Group<f_Group>, Flags<[CC1Option, NoArgumentUnused, FlangOption, FC1Option]>,
   HelpText<"Parse OpenMP pragmas and generate parallel code.">;
 def fno_openmp : Flag<["-"], "fno-openmp">, Group<f_Group>, Flags<[NoArgumentUnused]>;
-def fopenmp_version_EQ : Joined<["-"], "fopenmp-version=">, Group<f_Group>, Flags<[CC1Option, NoArgumentUnused]>;
+def fopenmp_version_EQ : Joined<["-"], "fopenmp-version=">, Group<f_Group>, Flags<[CC1Option, NoArgumentUnused]>,
+  HelpText<"Set OpenMP version (e.g. 45 for OpenMP 4.5, 50 for OpenMP 5.0). Default value is 50.">;
 defm openmp_extensions: BoolFOption<"openmp-extensions",
   LangOpts<"OpenMPExtensions">, DefaultTrue,
   PosFlag<SetTrue, [CC1Option, NoArgumentUnused],
@@ -2433,16 +2444,16 @@ def fopenmp_target_debug : Flag<["-"], "fopenmp-target-debug">, Group<f_Group>,
   HelpText<"Enable debugging in the OpenMP offloading device RTL">;
 def fno_openmp_target_debug : Flag<["-"], "fno-openmp-target-debug">, Group<f_Group>, Flags<[NoArgumentUnused]>;
 def fopenmp_target_debug_EQ : Joined<["-"], "fopenmp-target-debug=">, Group<f_Group>, Flags<[CC1Option, NoArgumentUnused, HelpHidden]>;
-def fopenmp_assume_teams_oversubscription : Flag<["-"], "fopenmp-assume-teams-oversubscription">, 
+def fopenmp_assume_teams_oversubscription : Flag<["-"], "fopenmp-assume-teams-oversubscription">,
   Group<f_Group>, Flags<[CC1Option, NoArgumentUnused, HelpHidden]>;
-def fopenmp_assume_threads_oversubscription : Flag<["-"], "fopenmp-assume-threads-oversubscription">, 
+def fopenmp_assume_threads_oversubscription : Flag<["-"], "fopenmp-assume-threads-oversubscription">,
   Group<f_Group>, Flags<[CC1Option, NoArgumentUnused, HelpHidden]>;
-def fno_openmp_assume_teams_oversubscription : Flag<["-"], "fno-openmp-assume-teams-oversubscription">, 
+def fno_openmp_assume_teams_oversubscription : Flag<["-"], "fno-openmp-assume-teams-oversubscription">,
   Group<f_Group>, Flags<[CC1Option, NoArgumentUnused, HelpHidden]>;
-def fno_openmp_assume_threads_oversubscription : Flag<["-"], "fno-openmp-assume-threads-oversubscription">, 
+def fno_openmp_assume_threads_oversubscription : Flag<["-"], "fno-openmp-assume-threads-oversubscription">,
   Group<f_Group>, Flags<[CC1Option, NoArgumentUnused, HelpHidden]>;
 defm openmp_target_new_runtime: BoolFOption<"openmp-target-new-runtime",
-  LangOpts<"OpenMPTargetNewRuntime">, DefaultFalse,
+  LangOpts<"OpenMPTargetNewRuntime">, DefaultTrue,
   PosFlag<SetTrue, [CC1Option], "Use the new bitcode library for OpenMP offloading">,
   NegFlag<SetFalse>>;
 defm openmp_optimistic_collapse : BoolFOption<"openmp-optimistic-collapse",
@@ -3338,6 +3349,11 @@ def mno_fix_cortex_a53_835769 : Flag<["-"], "mno-fix-cortex-a53-835769">,
 def mmark_bti_property : Flag<["-"], "mmark-bti-property">,
   Group<m_aarch64_Features_Group>,
   HelpText<"Add .note.gnu.property with BTI to assembly files (AArch64 only)">;
+def mno_bti_at_return_twice : Flag<["-"], "mno-bti-at-return-twice">,
+  Group<m_arm_Features_Group>,
+  HelpText<"Do not add a BTI instruction after a setjmp or other"
+           " return-twice construct (Arm only)">;
+
 foreach i = {1-31} in
   def ffixed_x#i : Flag<["-"], "ffixed-x"#i>, Group<m_Group>,
     HelpText<"Reserve the x"#i#" register (AArch64/RISC-V only)">;
@@ -3352,8 +3368,7 @@ def msve_vector_bits_EQ : Joined<["-"], "msve-vector-bits=">, Group<m_aarch64_Fe
 
 def mvscale_min_EQ : Joined<["-"], "mvscale-min=">,
   Group<m_aarch64_Features_Group>, Flags<[NoXarchOption,CC1Option]>,
-  HelpText<"Specify the vscale minimum. Defaults to the"
-           " vector length agnostic value of \"0\". (AArch64 only)">,
+  HelpText<"Specify the vscale minimum. Defaults to \"1\". (AArch64 only)">,
   MarshallingInfoInt<LangOpts<"VScaleMin">>;
 def mvscale_max_EQ : Joined<["-"], "mvscale-max=">,
   Group<m_aarch64_Features_Group>, Flags<[NoXarchOption,CC1Option]>,
@@ -3759,6 +3774,8 @@ def nobuiltininc : Flag<["-"], "nobuiltininc">, Flags<[CC1Option, CoreOption]>,
   MarshallingInfoNegativeFlag<HeaderSearchOpts<"UseBuiltinIncludes">>;
 def nogpuinc : Flag<["-"], "nogpuinc">, HelpText<"Do not add include paths for CUDA/HIP and"
   " do not include the default CUDA/HIP wrapper headers">;
+def nohipwrapperinc : Flag<["-"], "nohipwrapperinc">,
+  HelpText<"Do not include the default HIP wrapper headers and include paths">;
 def : Flag<["-"], "nocudainc">, Alias<nogpuinc>;
 def nogpulib : Flag<["-"], "nogpulib">,
   HelpText<"Do not link device library for CUDA/HIP device compilation">;
@@ -3785,6 +3802,11 @@ def o : JoinedOrSeparate<["-"], "o">, Flags<[NoXarchOption, RenderAsInput,
   CC1Option, CC1AsOption, FC1Option, FlangOption]>,
   HelpText<"Write output to <file>">, MetaVarName<"<file>">,
   MarshallingInfoString<FrontendOpts<"OutputFile">>;
+def object_file_name_EQ : Joined<["-"], "object-file-name=">, Flags<[CC1Option, CC1AsOption, CoreOption]>,
+  HelpText<"Set the output <file> for debug infos">, MetaVarName<"<file>">,
+  MarshallingInfoString<CodeGenOpts<"ObjectFilenameForDebug">>;
+def object_file_name : Separate<["-"], "object-file-name">, Flags<[CC1Option, CC1AsOption, CoreOption]>,
+    Alias<object_file_name_EQ>;
 def pagezero__size : JoinedOrSeparate<["-"], "pagezero_size">;
 def pass_exit_codes : Flag<["-", "--"], "pass-exit-codes">, Flags<[Unsupported]>;
 def pedantic_errors : Flag<["-", "--"], "pedantic-errors">, Group<pedantic_Group>, Flags<[CC1Option]>,
@@ -4138,6 +4160,8 @@ def mv67t : Flag<["-"], "mv67t">, Group<m_hexagon_Features_Group>,
   Alias<mcpu_EQ>, AliasArgs<["hexagonv67t"]>;
 def mv68 : Flag<["-"], "mv68">, Group<m_hexagon_Features_Group>,
   Alias<mcpu_EQ>, AliasArgs<["hexagonv68"]>;
+def mv69 : Flag<["-"], "mv69">, Group<m_hexagon_Features_Group>,
+  Alias<mcpu_EQ>, AliasArgs<["hexagonv69"]>;
 def mhexagon_hvx : Flag<["-"], "mhvx">, Group<m_hexagon_Features_HVX_Group>,
   HelpText<"Enable Hexagon Vector eXtensions">;
 def mhexagon_hvx_EQ : Joined<["-"], "mhvx=">,
@@ -4149,6 +4173,18 @@ def mno_hexagon_hvx : Flag<["-"], "mno-hvx">,
 def mhexagon_hvx_length_EQ : Joined<["-"], "mhvx-length=">,
   Group<m_hexagon_Features_HVX_Group>, HelpText<"Set Hexagon Vector Length">,
   Values<"64B,128B">;
+def mhexagon_hvx_qfloat : Flag<["-"], "mhvx-qfloat">,
+  Group<m_hexagon_Features_HVX_Group>,
+  HelpText<"Enable Hexagon HVX QFloat instructions">;
+def mno_hexagon_hvx_qfloat : Flag<["-"], "mno-hvx-qfloat">,
+  Group<m_hexagon_Features_HVX_Group>,
+  HelpText<"Disable Hexagon HVX QFloat instructions">;
+def mhexagon_hvx_ieee_fp : Flag<["-"], "mhvx-ieee-fp">,
+  Group<m_hexagon_Features_Group>,
+  HelpText<"Enable Hexagon HVX IEEE floating-point">;
+def mno_hexagon_hvx_ieee_fp : Flag<["-"], "mno-hvx-ieee-fp">,
+  Group<m_hexagon_Features_Group>,
+  HelpText<"Disable Hexagon HVX IEEE floating-point">;
 def ffixed_r19: Flag<["-"], "ffixed-r19">,
   HelpText<"Reserve register r19 (Hexagon only)">;
 def mmemops : Flag<["-"], "mmemops">, Group<m_hexagon_Features_Group>,
diff --git a/clang/include/clang/Driver/Tool.h b/clang/include/clang/Driver/Tool.h
index cc0a09fb2747..42cf99a4a970 100644
--- a/clang/include/clang/Driver/Tool.h
+++ b/clang/include/clang/Driver/Tool.h
@@ -52,6 +52,7 @@ public:
   const ToolChain &getToolChain() const { return TheToolChain; }
 
   virtual bool hasIntegratedAssembler() const { return false; }
+  virtual bool hasIntegratedBackend() const { return true; }
   virtual bool canEmitIR() const { return false; }
   virtual bool hasIntegratedCPP() const = 0;
   virtual bool isLinkJob() const { return false; }
diff --git a/clang/include/clang/Driver/ToolChain.h b/clang/include/clang/Driver/ToolChain.h
index dad861d586cb..4afc9bf36b5f 100644
--- a/clang/include/clang/Driver/ToolChain.h
+++ b/clang/include/clang/Driver/ToolChain.h
@@ -348,10 +348,7 @@ public:
   /// is LLD. If it's set, it can be assumed that the linker is LLD built
   /// at the same revision as clang, and clang can make assumptions about
   /// LLD's supported flags, error output, etc.
-  /// If LinkerIsLLDDarwinNew is non-nullptr, it's set if the linker is
-  /// the new version in lld/MachO.
-  std::string GetLinkerPath(bool *LinkerIsLLD = nullptr,
-                            bool *LinkerIsLLDDarwinNew = nullptr) const;
+  std::string GetLinkerPath(bool *LinkerIsLLD = nullptr) const;
 
   /// Returns the linker path for emitting a static library.
   std::string GetStaticLibToolPath() const;
@@ -387,6 +384,9 @@ public:
   /// Check if the toolchain should use the integrated assembler.
   virtual bool useIntegratedAs() const;
 
+  /// Check if the toolchain should use the integrated backend.
+  virtual bool useIntegratedBackend() const { return true; }
+
   /// Check if the toolchain should use AsmParser to parse inlineAsm when
   /// integrated assembler is not default.
   virtual bool parseInlineAsmUsingAsmParser() const { return false; }
diff --git a/clang/include/clang/Format/Format.h b/clang/include/clang/Format/Format.h
index d38bc6e3f0e6..0f97e80d425e 100644
--- a/clang/include/clang/Format/Format.h
+++ b/clang/include/clang/Format/Format.h
@@ -2669,6 +2669,7 @@ struct FormatStyle {
   bool isCpp() const { return Language == LK_Cpp || Language == LK_ObjC; }
   bool isCSharp() const { return Language == LK_CSharp; }
   bool isJson() const { return Language == LK_Json; }
+  bool isJavaScript() const { return Language == LK_JavaScript; }
 
   /// Language, this format style is targeted at.
   /// \version 3.5
diff --git a/clang/include/clang/Lex/ModuleMap.h b/clang/include/clang/Lex/ModuleMap.h
index 41f85a1f572d..08c61a5dc560 100644
--- a/clang/include/clang/Lex/ModuleMap.h
+++ b/clang/include/clang/Lex/ModuleMap.h
@@ -538,8 +538,11 @@ public:
   ///
   /// We model the global module fragment as a submodule of the module
   /// interface unit. Unfortunately, we can't create the module interface
-  /// unit's Module until later, because we don't know what it will be called.
-  Module *createGlobalModuleFragmentForModuleUnit(SourceLocation Loc);
+  /// unit's Module until later, because we don't know what it will be called
+  /// usually. See C++20 [module.unit]/7.2 for the case we could know its
+  /// parent.
+  Module *createGlobalModuleFragmentForModuleUnit(SourceLocation Loc,
+                                                  Module *Parent = nullptr);
 
   /// Create a global module fragment for a C++ module interface unit.
   Module *createPrivateModuleFragmentForInterfaceUnit(Module *Parent,
diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index 92a703b42173..741a484390b2 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -2564,6 +2564,10 @@ private:
   /// full validation of the syntactic structure of attributes.
   bool TrySkipAttributes();
 
+  /// Diagnoses use of _ExtInt as being deprecated, and diagnoses use of
+  /// _BitInt as an extension when appropriate.
+  void DiagnoseBitIntUse(const Token &Tok);
+
 public:
   TypeResult
   ParseTypeName(SourceRange *Range = nullptr,
diff --git a/clang/include/clang/Sema/DeclSpec.h b/clang/include/clang/Sema/DeclSpec.h
index ed5be2da3acd..2704a9c1fc78 100644
--- a/clang/include/clang/Sema/DeclSpec.h
+++ b/clang/include/clang/Sema/DeclSpec.h
@@ -266,7 +266,7 @@ public:
   static const TST TST_char32 = clang::TST_char32;
   static const TST TST_int = clang::TST_int;
   static const TST TST_int128 = clang::TST_int128;
-  static const TST TST_extint = clang::TST_extint;
+  static const TST TST_bitint = clang::TST_bitint;
   static const TST TST_half = clang::TST_half;
   static const TST TST_BFloat16 = clang::TST_BFloat16;
   static const TST TST_float = clang::TST_float;
@@ -404,7 +404,7 @@ private:
             T == TST_underlyingType || T == TST_atomic);
   }
   static bool isExprRep(TST T) {
-    return (T == TST_typeofExpr || T == TST_decltype || T == TST_extint);
+    return (T == TST_typeofExpr || T == TST_decltype || T == TST_bitint);
   }
   static bool isTemplateIdRep(TST T) {
     return (T == TST_auto || T == TST_decltype_auto);
@@ -703,7 +703,7 @@ public:
   bool SetTypePipe(bool isPipe, SourceLocation Loc,
                        const char *&PrevSpec, unsigned &DiagID,
                        const PrintingPolicy &Policy);
-  bool SetExtIntType(SourceLocation KWLoc, Expr *BitWidth,
+  bool SetBitIntType(SourceLocation KWLoc, Expr *BitWidth,
                      const char *&PrevSpec, unsigned &DiagID,
                      const PrintingPolicy &Policy);
   bool SetTypeSpecSat(SourceLocation Loc, const char *&PrevSpec,
diff --git a/clang/include/clang/Sema/ParsedAttr.h b/clang/include/clang/Sema/ParsedAttr.h
index ff2303c84bd2..6403179cb327 100644
--- a/clang/include/clang/Sema/ParsedAttr.h
+++ b/clang/include/clang/Sema/ParsedAttr.h
@@ -1097,6 +1097,7 @@ enum AttributeArgumentNType {
   AANT_ArgumentString,
   AANT_ArgumentIdentifier,
   AANT_ArgumentConstantExpr,
+  AANT_ArgumentBuiltinFunction,
 };
 
 /// These constants match the enumerated choices of
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 1a82a9498d1d..79834554a50d 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -1324,12 +1324,15 @@ public:
 
     bool isImmediateFunctionContext() const {
       return Context == ExpressionEvaluationContext::ImmediateFunctionContext ||
-             InImmediateFunctionContext;
+             (Context == ExpressionEvaluationContext::DiscardedStatement &&
+              InImmediateFunctionContext);
     }
 
     bool isDiscardedStatementContext() const {
       return Context == ExpressionEvaluationContext::DiscardedStatement ||
-             InDiscardedStatement;
+             (Context ==
+                  ExpressionEvaluationContext::ImmediateFunctionContext &&
+              InDiscardedStatement);
     }
   };
 
@@ -2043,7 +2046,7 @@ public:
                          SourceLocation Loc);
   QualType BuildWritePipeType(QualType T,
                          SourceLocation Loc);
-  QualType BuildExtIntType(bool IsUnsigned, Expr *BitWidth, SourceLocation Loc);
+  QualType BuildBitIntType(bool IsUnsigned, Expr *BitWidth, SourceLocation Loc);
 
   TypeSourceInfo *GetTypeForDeclarator(Declarator &D, Scope *S);
   TypeSourceInfo *GetTypeForDeclaratorCast(Declarator &D, QualType FromTy);
@@ -2219,6 +2222,17 @@ private:
     return ModuleScopes.empty() ? nullptr : ModuleScopes.back().Module;
   }
 
+  /// Helper function to judge if we are in module purview.
+  /// Return false if we are not in a module.
+  bool isCurrentModulePurview() const {
+    return getCurrentModule() ? getCurrentModule()->isModulePurview() : false;
+  }
+
+  /// Enter the scope of the global module.
+  Module *PushGlobalModuleFragment(SourceLocation BeginLoc, bool IsImplicit);
+  /// Leave the scope of the global module.
+  void PopGlobalModuleFragment();
+
   VisibleModuleSet VisibleModules;
 
 public:
@@ -11176,6 +11190,9 @@ public:
   /// Called on well-formed 'capture' clause.
   OMPClause *ActOnOpenMPCaptureClause(SourceLocation StartLoc,
                                       SourceLocation EndLoc);
+  /// Called on well-formed 'compare' clause.
+  OMPClause *ActOnOpenMPCompareClause(SourceLocation StartLoc,
+                                      SourceLocation EndLoc);
   /// Called on well-formed 'seq_cst' clause.
   OMPClause *ActOnOpenMPSeqCstClause(SourceLocation StartLoc,
                                      SourceLocation EndLoc);
@@ -12769,8 +12786,8 @@ private:
   bool CheckPPCMMAType(QualType Type, SourceLocation TypeLoc);
 
   bool SemaBuiltinElementwiseMath(CallExpr *TheCall);
-  bool SemaBuiltinElementwiseMathOneArg(CallExpr *TheCall);
-  bool SemaBuiltinReduceMath(CallExpr *TheCall);
+  bool PrepareBuiltinElementwiseMathOneArgCall(CallExpr *TheCall);
+  bool PrepareBuiltinReduceMathOneArgCall(CallExpr *TheCall);
 
   // Matrix builtin handling.
   ExprResult SemaBuiltinMatrixTranspose(CallExpr *TheCall,
diff --git a/clang/include/clang/Serialization/TypeBitCodes.def b/clang/include/clang/Serialization/TypeBitCodes.def
index e92e05810648..85571ea290f7 100644
--- a/clang/include/clang/Serialization/TypeBitCodes.def
+++ b/clang/include/clang/Serialization/TypeBitCodes.def
@@ -58,9 +58,10 @@ TYPE_BIT_CODE(DependentSizedExtVector, DEPENDENT_SIZED_EXT_VECTOR, 46)
 TYPE_BIT_CODE(DependentAddressSpace, DEPENDENT_ADDRESS_SPACE, 47)
 TYPE_BIT_CODE(DependentVector, DEPENDENT_SIZED_VECTOR, 48)
 TYPE_BIT_CODE(MacroQualified, MACRO_QUALIFIED, 49)
-TYPE_BIT_CODE(ExtInt, EXT_INT, 50)
-TYPE_BIT_CODE(DependentExtInt, DEPENDENT_EXT_INT, 51)
+TYPE_BIT_CODE(BitInt, BIT_INT, 50)
+TYPE_BIT_CODE(DependentBitInt, DEPENDENT_BIT_INT, 51)
 TYPE_BIT_CODE(ConstantMatrix, CONSTANT_MATRIX, 52)
 TYPE_BIT_CODE(DependentSizedMatrix, DEPENDENT_SIZE_MATRIX, 53)
+TYPE_BIT_CODE(Using, USING, 54)
 
 #undef TYPE_BIT_CODE
diff --git a/clang/include/clang/StaticAnalyzer/Checkers/SValExplainer.h b/clang/include/clang/StaticAnalyzer/Checkers/SValExplainer.h
index 31a4ed50a723..48cd3395e936 100644
--- a/clang/include/clang/StaticAnalyzer/Checkers/SValExplainer.h
+++ b/clang/include/clang/StaticAnalyzer/Checkers/SValExplainer.h
@@ -32,7 +32,7 @@ private:
     std::string Str;
     llvm::raw_string_ostream OS(Str);
     S->printPretty(OS, nullptr, PrintingPolicy(ACtx.getLangOpts()));
-    return OS.str();
+    return Str;
   }
 
   bool isThisObject(const SymbolicRegion *R) {
@@ -69,7 +69,7 @@ public:
     std::string Str;
     llvm::raw_string_ostream OS(Str);
     OS << "concrete memory address '" << I << "'";
-    return OS.str();
+    return Str;
   }
 
   std::string VisitNonLocSymbolVal(nonloc::SymbolVal V) {
@@ -82,7 +82,7 @@ public:
     llvm::raw_string_ostream OS(Str);
     OS << (I.isSigned() ? "signed " : "unsigned ") << I.getBitWidth()
        << "-bit integer '" << I << "'";
-    return OS.str();
+    return Str;
   }
 
   std::string VisitNonLocLazyCompoundVal(nonloc::LazyCompoundVal V) {
@@ -123,7 +123,7 @@ public:
     OS << "(" << Visit(S->getLHS()) << ") "
        << std::string(BinaryOperator::getOpcodeStr(S->getOpcode())) << " "
        << S->getRHS();
-    return OS.str();
+    return Str;
   }
 
   // TODO: IntSymExpr doesn't appear in practice.
@@ -177,7 +177,7 @@ public:
     else
       OS << "'" << Visit(R->getIndex()) << "'";
     OS << " of " + Visit(R->getSuperRegion());
-    return OS.str();
+    return Str;
   }
 
   std::string VisitNonParamVarRegion(const NonParamVarRegion *R) {
diff --git a/clang/include/clang/StaticAnalyzer/Core/AnalyzerOptions.def b/clang/include/clang/StaticAnalyzer/Core/AnalyzerOptions.def
index aab8e1284bf6..c0cade46d614 100644
--- a/clang/include/clang/StaticAnalyzer/Core/AnalyzerOptions.def
+++ b/clang/include/clang/StaticAnalyzer/Core/AnalyzerOptions.def
@@ -336,6 +336,18 @@ ANALYZER_OPTION(
     "might be modeled by the analyzer to never return NULL.",
     false)
 
+ANALYZER_OPTION(
+    bool, ShouldIgnoreBisonGeneratedFiles, "ignore-bison-generated-files",
+    "If enabled, any files containing the \"/* A Bison parser, made by\" "
+    "won't be analyzed.",
+    true)
+
+ANALYZER_OPTION(
+    bool, ShouldIgnoreFlexGeneratedFiles, "ignore-flex-generated-files",
+    "If enabled, any files containing the \"/* A lexical scanner generated by "
+    "flex\" won't be analyzed.",
+    true)
+
 //===----------------------------------------------------------------------===//
 // Unsigned analyzer options.
 //===----------------------------------------------------------------------===//
diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallDescription.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallDescription.h
index 396c9a4de440..cd972b7837d0 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallDescription.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallDescription.h
@@ -134,6 +134,9 @@ public:
       std::initializer_list<std::pair<CallDescription, T>> &&List)
       : LinearMap(List) {}
 
+  template <typename InputIt>
+  CallDescriptionMap(InputIt First, InputIt Last) : LinearMap(First, Last) {}
+
   ~CallDescriptionMap() = default;
 
   // These maps are usually stored once per checker, so let's make sure
@@ -141,6 +144,9 @@ public:
   CallDescriptionMap(const CallDescriptionMap &) = delete;
   CallDescriptionMap &operator=(const CallDescription &) = delete;
 
+  CallDescriptionMap(CallDescriptionMap &&) = default;
+  CallDescriptionMap &operator=(CallDescriptionMap &&) = default;
+
   LLVM_NODISCARD const T *lookup(const CallEvent &Call) const {
     // Slow path: linear lookup.
     // TODO: Implement some sort of fast path.
diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/RangedConstraintManager.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/RangedConstraintManager.h
index a80484610131..3a0bec9d04e5 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/RangedConstraintManager.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/RangedConstraintManager.h
@@ -140,6 +140,30 @@ public:
     /// Complexity: O(N)
     ///             where N = size(Original)
     RangeSet add(RangeSet Original, const llvm::APSInt &Point);
+    /// Create a new set which is a union of two given ranges.
+    /// Possible intersections are not checked here.
+    ///
+    /// Complexity: O(N + M)
+    ///             where N = size(LHS), M = size(RHS)
+    RangeSet unite(RangeSet LHS, RangeSet RHS);
+    /// Create a new set by uniting given range set with the given range.
+    /// All intersections and adjacent ranges are handled here.
+    ///
+    /// Complexity: O(N)
+    ///             where N = size(Original)
+    RangeSet unite(RangeSet Original, Range Element);
+    /// Create a new set by uniting given range set with the given point.
+    /// All intersections and adjacent ranges are handled here.
+    ///
+    /// Complexity: O(N)
+    ///             where N = size(Original)
+    RangeSet unite(RangeSet Original, llvm::APSInt Point);
+    /// Create a new set by uniting given range set with the given range
+    /// between points. All intersections and adjacent ranges are handled here.
+    ///
+    /// Complexity: O(N)
+    ///             where N = size(Original)
+    RangeSet unite(RangeSet Original, llvm::APSInt From, llvm::APSInt To);
 
     RangeSet getEmptySet() { return &EmptySet; }
 
@@ -224,6 +248,9 @@ public:
     ContainerType *construct(ContainerType &&From);
 
     RangeSet intersect(const ContainerType &LHS, const ContainerType &RHS);
+    /// NOTE: This function relies on the fact that all values in the
+    /// containers are persistent (created via BasicValueFactory::getValue).
+    ContainerType unite(const ContainerType &LHS, const ContainerType &RHS);
 
     // Many operations include producing new APSInt values and that's why
     // we need this factory.
diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/Store.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/Store.h
index d2461705d128..bdf9662d5d99 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/Store.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/Store.h
@@ -172,9 +172,9 @@ public:
   ///    dynamic_cast.
   ///  - We don't know (base is a symbolic region and we don't have
   ///    enough info to determine if the cast will succeed at run time).
-  /// The function returns an SVal representing the derived class; it's
-  /// valid only if Failed flag is set to false.
-  SVal attemptDownCast(SVal Base, QualType DerivedPtrType, bool &Failed);
+  /// The function returns an optional with SVal representing the derived class
+  /// in case of a successful cast and `None` otherwise.
+  Optional<SVal> evalBaseToDerived(SVal Base, QualType DerivedPtrType);
 
   const ElementRegion *GetElementZeroRegion(const SubRegion *R, QualType T);
 
diff --git a/clang/include/clang/Testing/TestClangConfig.h b/clang/include/clang/Testing/TestClangConfig.h
index 5d6be4f65d0a..92d5cc3cff99 100644
--- a/clang/include/clang/Testing/TestClangConfig.h
+++ b/clang/include/clang/Testing/TestClangConfig.h
@@ -73,7 +73,7 @@ struct TestClangConfig {
     std::string Result;
     llvm::raw_string_ostream OS(Result);
     OS << "{ Language=" << Language << ", Target=" << Target << " }";
-    return OS.str();
+    return Result;
   }
 
   friend std::ostream &operator<<(std::ostream &OS,
diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
index af02fa2e7e87..7d0b8f2138f9 100644
--- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
+++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
@@ -26,57 +26,83 @@ namespace dependencies {
 /// the dependency scanning filesystem.
 ///
 /// It represents one of the following:
-/// - an opened source file with minimized contents and a stat value.
-/// - an opened source file with original contents and a stat value.
-/// - a directory entry with its stat value.
-/// - an error value to represent a file system error.
-/// - a placeholder with an invalid stat indicating a not yet initialized entry.
+/// - opened file with original contents and a stat value,
+/// - opened file with original contents, minimized contents and a stat value,
+/// - directory entry with its stat value,
+/// - filesystem error,
+/// - uninitialized entry with unknown status.
 class CachedFileSystemEntry {
 public:
-  /// Default constructor creates an entry with an invalid stat.
-  CachedFileSystemEntry() : MaybeStat(llvm::vfs::Status()) {}
+  /// Creates an uninitialized entry.
+  CachedFileSystemEntry()
+      : MaybeStat(llvm::vfs::Status()), MinimizedContentsAccess(nullptr) {}
 
-  CachedFileSystemEntry(std::error_code Error) : MaybeStat(std::move(Error)) {}
+  /// Initialize the cached file system entry.
+  void init(llvm::ErrorOr<llvm::vfs::Status> &&MaybeStatus, StringRef Filename,
+            llvm::vfs::FileSystem &FS);
 
-  /// Create an entry that represents an opened source file with minimized or
-  /// original contents.
+  /// Initialize the entry as file with minimized or original contents.
   ///
   /// The filesystem opens the file even for `stat` calls open to avoid the
   /// issues with stat + open of minimized files that might lead to a
-  /// mismatching size of the file. If file is not minimized, the full file is
-  /// read and copied into memory to ensure that it's not memory mapped to avoid
-  /// running out of file descriptors.
-  static CachedFileSystemEntry createFileEntry(StringRef Filename,
-                                               llvm::vfs::FileSystem &FS,
-                                               bool Minimize = true);
+  /// mismatching size of the file.
+  llvm::ErrorOr<llvm::vfs::Status> initFile(StringRef Filename,
+                                            llvm::vfs::FileSystem &FS);
 
-  /// Create an entry that represents a directory on the filesystem.
-  static CachedFileSystemEntry createDirectoryEntry(llvm::vfs::Status &&Stat);
+  /// Minimize contents of the file.
+  void minimizeFile();
 
-  /// \returns True if the entry is valid.
-  bool isValid() const { return !MaybeStat || MaybeStat->isStatusKnown(); }
+  /// \returns True if the entry is initialized.
+  bool isInitialized() const {
+    return !MaybeStat || MaybeStat->isStatusKnown();
+  }
 
   /// \returns True if the current entry points to a directory.
   bool isDirectory() const { return MaybeStat && MaybeStat->isDirectory(); }
 
-  /// \returns The error or the file's contents.
-  llvm::ErrorOr<StringRef> getContents() const {
+  /// \returns The error or the file's original contents.
+  llvm::ErrorOr<StringRef> getOriginalContents() const {
+    if (!MaybeStat)
+      return MaybeStat.getError();
+    assert(!MaybeStat->isDirectory() && "not a file");
+    assert(isInitialized() && "not initialized");
+    assert(OriginalContents && "not read");
+    return OriginalContents->getBuffer();
+  }
+
+  /// \returns The error or the file's minimized contents.
+  llvm::ErrorOr<StringRef> getMinimizedContents() const {
     if (!MaybeStat)
       return MaybeStat.getError();
     assert(!MaybeStat->isDirectory() && "not a file");
-    assert(isValid() && "not initialized");
-    return Contents.str();
+    assert(isInitialized() && "not initialized");
+    llvm::MemoryBuffer *Buffer = MinimizedContentsAccess.load();
+    assert(Buffer && "not minimized");
+    return Buffer->getBuffer();
+  }
+
+  /// \returns True if this entry represents a file that can be read.
+  bool isReadable() const { return MaybeStat && !MaybeStat->isDirectory(); }
+
+  /// \returns True if this cached entry needs to be updated.
+  bool needsUpdate(bool ShouldBeMinimized) const {
+    return isReadable() && needsMinimization(ShouldBeMinimized);
+  }
+
+  /// \returns True if the contents of this entry need to be minimized.
+  bool needsMinimization(bool ShouldBeMinimized) const {
+    return ShouldBeMinimized && !MinimizedContentsAccess.load();
   }
 
   /// \returns The error or the status of the entry.
   llvm::ErrorOr<llvm::vfs::Status> getStatus() const {
-    assert(isValid() && "not initialized");
+    assert(isInitialized() && "not initialized");
     return MaybeStat;
   }
 
   /// \returns the name of the file.
   StringRef getName() const {
-    assert(isValid() && "not initialized");
+    assert(isInitialized() && "not initialized");
     return MaybeStat->getName();
   }
 
@@ -86,19 +112,16 @@ public:
     return PPSkippedRangeMapping;
   }
 
-  CachedFileSystemEntry(CachedFileSystemEntry &&) = default;
-  CachedFileSystemEntry &operator=(CachedFileSystemEntry &&) = default;
-
-  CachedFileSystemEntry(const CachedFileSystemEntry &) = delete;
-  CachedFileSystemEntry &operator=(const CachedFileSystemEntry &) = delete;
-
 private:
   llvm::ErrorOr<llvm::vfs::Status> MaybeStat;
-  // Store the contents in a small string to allow a
-  // move from the small string for the minimized contents.
-  // Note: small size of 1 allows us to store an empty string with an implicit
-  // null terminator without any allocations.
-  llvm::SmallString<1> Contents;
+  std::unique_ptr<llvm::MemoryBuffer> OriginalContents;
+
+  /// Owning storage for the minimized file contents.
+  std::unique_ptr<llvm::MemoryBuffer> MinimizedContentsStorage;
+  /// Atomic view of the minimized file contents.
+  /// This prevents data races when multiple threads call `needsMinimization`.
+  std::atomic<llvm::MemoryBuffer *> MinimizedContentsAccess;
+
   PreprocessorSkippedRangeMapping PPSkippedRangeMapping;
 };
 
@@ -115,61 +138,70 @@ public:
     CachedFileSystemEntry Value;
   };
 
+  DependencyScanningFilesystemSharedCache();
+
   /// Returns a cache entry for the corresponding key.
   ///
   /// A new cache entry is created if the key is not in the cache. This is a
   /// thread safe call.
-  SharedFileSystemEntry &get(StringRef Key, bool Minimized);
+  SharedFileSystemEntry &get(StringRef Key);
 
 private:
-  class SingleCache {
-  public:
-    SingleCache();
-
-    SharedFileSystemEntry &get(StringRef Key);
-
-  private:
-    struct CacheShard {
-      std::mutex CacheLock;
-      llvm::StringMap<SharedFileSystemEntry, llvm::BumpPtrAllocator> Cache;
-    };
-    std::unique_ptr<CacheShard[]> CacheShards;
-    unsigned NumShards;
+  struct CacheShard {
+    std::mutex CacheLock;
+    llvm::StringMap<SharedFileSystemEntry, llvm::BumpPtrAllocator> Cache;
   };
-
-  SingleCache CacheMinimized;
-  SingleCache CacheOriginal;
+  std::unique_ptr<CacheShard[]> CacheShards;
+  unsigned NumShards;
 };
 
 /// This class is a local cache, that caches the 'stat' and 'open' calls to the
 /// underlying real file system. It distinguishes between minimized and original
 /// files.
 class DependencyScanningFilesystemLocalCache {
-private:
-  using SingleCache =
-      llvm::StringMap<const CachedFileSystemEntry *, llvm::BumpPtrAllocator>;
-
-  SingleCache CacheMinimized;
-  SingleCache CacheOriginal;
+  llvm::StringMap<const CachedFileSystemEntry *, llvm::BumpPtrAllocator> Cache;
 
-  SingleCache &selectCache(bool Minimized) {
-    return Minimized ? CacheMinimized : CacheOriginal;
+public:
+  const CachedFileSystemEntry *getCachedEntry(StringRef Filename) {
+    return Cache[Filename];
   }
+};
+
+/// Reference to a CachedFileSystemEntry.
+/// If the underlying entry is an opened file, this wrapper returns the correct
+/// contents (original or minimized) and ensures consistency with file size
+/// reported by status.
+class EntryRef {
+  /// For entry that is an opened file, this bit signifies whether its contents
+  /// are minimized.
+  bool Minimized;
+
+  /// The underlying cached entry.
+  const CachedFileSystemEntry &Entry;
 
 public:
-  void setCachedEntry(StringRef Filename, bool Minimized,
-                      const CachedFileSystemEntry *Entry) {
-    SingleCache &Cache = selectCache(Minimized);
-    bool IsInserted = Cache.try_emplace(Filename, Entry).second;
-    (void)IsInserted;
-    assert(IsInserted && "local cache is updated more than once");
+  EntryRef(bool Minimized, const CachedFileSystemEntry &Entry)
+      : Minimized(Minimized), Entry(Entry) {}
+
+  llvm::ErrorOr<llvm::vfs::Status> getStatus() const {
+    auto MaybeStat = Entry.getStatus();
+    if (!MaybeStat || MaybeStat->isDirectory())
+      return MaybeStat;
+    return llvm::vfs::Status::copyWithNewSize(*MaybeStat,
+                                              getContents()->size());
+  }
+
+  bool isDirectory() const { return Entry.isDirectory(); }
+
+  StringRef getName() const { return Entry.getName(); }
+
+  llvm::ErrorOr<StringRef> getContents() const {
+    return Minimized ? Entry.getMinimizedContents()
+                     : Entry.getOriginalContents();
   }
 
-  const CachedFileSystemEntry *getCachedEntry(StringRef Filename,
-                                              bool Minimized) {
-    SingleCache &Cache = selectCache(Minimized);
-    auto It = Cache.find(Filename);
-    return It == Cache.end() ? nullptr : It->getValue();
+  const PreprocessorSkippedRangeMapping *getPPSkippedRangeMapping() const {
+    return Minimized ? &Entry.getPPSkippedRangeMapping() : nullptr;
   }
 };
 
@@ -204,19 +236,13 @@ private:
   /// Check whether the file should be minimized.
   bool shouldMinimize(StringRef Filename);
 
-  llvm::ErrorOr<const CachedFileSystemEntry *>
-  getOrCreateFileSystemEntry(const StringRef Filename);
-
-  /// Create a cached file system entry based on the initial status result.
-  CachedFileSystemEntry
-  createFileSystemEntry(llvm::ErrorOr<llvm::vfs::Status> &&MaybeStatus,
-                        StringRef Filename, bool ShouldMinimize);
+  llvm::ErrorOr<EntryRef> getOrCreateFileSystemEntry(StringRef Filename);
 
   /// The global cache shared between worker threads.
   DependencyScanningFilesystemSharedCache &SharedCache;
   /// The local cache is used by the worker thread to cache file system queries
   /// locally instead of querying the global cache every time.
-  DependencyScanningFilesystemLocalCache Cache;
+  DependencyScanningFilesystemLocalCache LocalCache;
   /// The optional mapping structure which records information about the
   /// excluded conditional directive skip mappings that are used by the
   /// currently active preprocessor.
diff --git a/clang/include/clang/Tooling/Inclusions/IncludeStyle.h b/clang/include/clang/Tooling/Inclusions/IncludeStyle.h
index d54f8a402e2d..d5638642d017 100644
--- a/clang/include/clang/Tooling/Inclusions/IncludeStyle.h
+++ b/clang/include/clang/Tooling/Inclusions/IncludeStyle.h
@@ -106,7 +106,7 @@ struct IncludeStyle {
   ///       Priority:        2
   ///       SortPriority:    2
   ///       CaseSensitive:   true
-  ///     - Regex:           '^(<|"(gtest|gmock|isl|json)/)'
+  ///     - Regex:           '^((<|")(gtest|gmock|isl|json)/)'
   ///       Priority:        3
   ///     - Regex:           '<[[:alnum:].]+>'
   ///       Priority:        4
diff --git a/clang/include/clang/module.modulemap b/clang/include/clang/module.modulemap
index e850a1cd4b9a..2b73cd5451b7 100644
--- a/clang/include/clang/module.modulemap
+++ b/clang/include/clang/module.modulemap
@@ -34,6 +34,8 @@ module Clang_Basic {
   textual header "Basic/AArch64SVEACLETypes.def"
   textual header "Basic/BuiltinsAArch64.def"
   textual header "Basic/BuiltinsAMDGPU.def"
+  textual header "Basic/BuiltinsAArch64NeonSVEBridge.def"
+  textual header "Basic/BuiltinsAArch64NeonSVEBridge_cg.def"
   textual header "Basic/BuiltinsARM.def"
   textual header "Basic/BuiltinsBPF.def"
   textual header "Basic/Builtins.def"
diff --git a/clang/lib/ARCMigrate/ARCMT.cpp b/clang/lib/ARCMigrate/ARCMT.cpp
index 4851c434d765..68ee7c59270e 100644
--- a/clang/lib/ARCMigrate/ARCMT.cpp
+++ b/clang/lib/ARCMigrate/ARCMT.cpp
@@ -162,9 +162,7 @@ static bool HasARCRuntime(CompilerInvocation &origCI) {
     return triple.getOSMajorVersion() >= 11;
 
   if (triple.getOS() == llvm::Triple::MacOSX) {
-    unsigned Major, Minor, Micro;
-    triple.getOSVersion(Major, Minor, Micro);
-    return Major > 10 || (Major == 10 && Minor >= 7);
+    return triple.getOSVersion() >= VersionTuple(10, 7);
   }
 
   return false;
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index 2d85d72e5b8a..008b703d4c1a 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -2286,8 +2286,8 @@ TypeInfo ASTContext::getTypeInfoImpl(const Type *T) const {
     Align = toBits(Layout.getAlignment());
     break;
   }
-  case Type::ExtInt: {
-    const auto *EIT = cast<ExtIntType>(T);
+  case Type::BitInt: {
+    const auto *EIT = cast<BitIntType>(T);
     Align =
         std::min(static_cast<unsigned>(std::max(
                      getCharWidth(), llvm::PowerOf2Ceil(EIT->getNumBits()))),
@@ -2349,6 +2349,9 @@ TypeInfo ASTContext::getTypeInfoImpl(const Type *T) const {
   case Type::ObjCTypeParam:
     return getTypeInfo(cast<ObjCTypeParamType>(T)->desugar().getTypePtr());
 
+  case Type::Using:
+    return getTypeInfo(cast<UsingType>(T)->desugar().getTypePtr());
+
   case Type::Typedef: {
     const TypedefNameDecl *Typedef = cast<TypedefType>(T)->getDecl();
     TypeInfo Info = getTypeInfo(Typedef->getUnderlyingType().getTypePtr());
@@ -3569,8 +3572,8 @@ QualType ASTContext::getVariableArrayDecayedType(QualType type) const {
   case Type::Auto:
   case Type::DeducedTemplateSpecialization:
   case Type::PackExpansion:
-  case Type::ExtInt:
-  case Type::DependentExtInt:
+  case Type::BitInt:
+  case Type::DependentBitInt:
     llvm_unreachable("type should never be variably-modified");
 
   // These types can be variably-modified but should never need to
@@ -4482,34 +4485,34 @@ QualType ASTContext::getWritePipeType(QualType T) const {
   return getPipeType(T, false);
 }
 
-QualType ASTContext::getExtIntType(bool IsUnsigned, unsigned NumBits) const {
+QualType ASTContext::getBitIntType(bool IsUnsigned, unsigned NumBits) const {
   llvm::FoldingSetNodeID ID;
-  ExtIntType::Profile(ID, IsUnsigned, NumBits);
+  BitIntType::Profile(ID, IsUnsigned, NumBits);
 
   void *InsertPos = nullptr;
-  if (ExtIntType *EIT = ExtIntTypes.FindNodeOrInsertPos(ID, InsertPos))
+  if (BitIntType *EIT = BitIntTypes.FindNodeOrInsertPos(ID, InsertPos))
     return QualType(EIT, 0);
 
-  auto *New = new (*this, TypeAlignment) ExtIntType(IsUnsigned, NumBits);
-  ExtIntTypes.InsertNode(New, InsertPos);
+  auto *New = new (*this, TypeAlignment) BitIntType(IsUnsigned, NumBits);
+  BitIntTypes.InsertNode(New, InsertPos);
   Types.push_back(New);
   return QualType(New, 0);
 }
 
-QualType ASTContext::getDependentExtIntType(bool IsUnsigned,
+QualType ASTContext::getDependentBitIntType(bool IsUnsigned,
                                             Expr *NumBitsExpr) const {
   assert(NumBitsExpr->isInstantiationDependent() && "Only good for dependent");
   llvm::FoldingSetNodeID ID;
-  DependentExtIntType::Profile(ID, *this, IsUnsigned, NumBitsExpr);
+  DependentBitIntType::Profile(ID, *this, IsUnsigned, NumBitsExpr);
 
   void *InsertPos = nullptr;
-  if (DependentExtIntType *Existing =
-          DependentExtIntTypes.FindNodeOrInsertPos(ID, InsertPos))
+  if (DependentBitIntType *Existing =
+          DependentBitIntTypes.FindNodeOrInsertPos(ID, InsertPos))
     return QualType(Existing, 0);
 
   auto *New = new (*this, TypeAlignment)
-      DependentExtIntType(*this, IsUnsigned, NumBitsExpr);
-  DependentExtIntTypes.InsertNode(New, InsertPos);
+      DependentBitIntType(*this, IsUnsigned, NumBitsExpr);
+  DependentBitIntTypes.InsertNode(New, InsertPos);
 
   Types.push_back(New);
   return QualType(New, 0);
@@ -4568,9 +4571,7 @@ QualType ASTContext::getTypeDeclTypeSlow(const TypeDecl *Decl) const {
     assert(Enum->isFirstDecl() && "enum has previous declaration");
     return getEnumType(Enum);
   } else if (const auto *Using = dyn_cast<UnresolvedUsingTypenameDecl>(Decl)) {
-    Type *newType = new (*this, TypeAlignment) UnresolvedUsingType(Using);
-    Decl->TypeForDecl = newType;
-    Types.push_back(newType);
+    return getUnresolvedUsingType(Using);
   } else
     llvm_unreachable("TypeDecl without a type?");
 
@@ -4593,6 +4594,27 @@ QualType ASTContext::getTypedefType(const TypedefNameDecl *Decl,
   return QualType(newType, 0);
 }
 
+QualType ASTContext::getUsingType(const UsingShadowDecl *Found,
+                                  QualType Underlying) const {
+  llvm::FoldingSetNodeID ID;
+  UsingType::Profile(ID, Found);
+
+  void *InsertPos = nullptr;
+  UsingType *T = UsingTypes.FindNodeOrInsertPos(ID, InsertPos);
+  if (T)
+    return QualType(T, 0);
+
+  assert(!Underlying.hasLocalQualifiers());
+  assert(Underlying == getTypeDeclType(cast<TypeDecl>(Found->getTargetDecl())));
+  QualType Canon = Underlying.getCanonicalType();
+
+  UsingType *NewType =
+      new (*this, TypeAlignment) UsingType(Found, Underlying, Canon);
+  Types.push_back(NewType);
+  UsingTypes.InsertNode(NewType, InsertPos);
+  return QualType(NewType, 0);
+}
+
 QualType ASTContext::getRecordType(const RecordDecl *Decl) const {
   if (Decl->TypeForDecl) return QualType(Decl->TypeForDecl, 0);
 
@@ -4619,6 +4641,22 @@ QualType ASTContext::getEnumType(const EnumDecl *Decl) const {
   return QualType(newType, 0);
 }
 
+QualType ASTContext::getUnresolvedUsingType(
+    const UnresolvedUsingTypenameDecl *Decl) const {
+  if (Decl->TypeForDecl)
+    return QualType(Decl->TypeForDecl, 0);
+
+  if (const UnresolvedUsingTypenameDecl *CanonicalDecl =
+          Decl->getCanonicalDecl())
+    if (CanonicalDecl->TypeForDecl)
+      return QualType(Decl->TypeForDecl = CanonicalDecl->TypeForDecl, 0);
+
+  Type *newType = new (*this, TypeAlignment) UnresolvedUsingType(Decl);
+  Decl->TypeForDecl = newType;
+  Types.push_back(newType);
+  return QualType(newType, 0);
+}
+
 QualType ASTContext::getAttributedType(attr::Kind attrKind,
                                        QualType modifiedType,
                                        QualType equivalentType) {
@@ -6444,7 +6482,7 @@ unsigned ASTContext::getIntegerRank(const Type *T) const {
 
   // Results in this 'losing' to any type of the same size, but winning if
   // larger.
-  if (const auto *EIT = dyn_cast<ExtIntType>(T))
+  if (const auto *EIT = dyn_cast<BitIntType>(T))
     return 0 + (EIT->getNumBits() << 3);
 
   switch (cast<BuiltinType>(T)->getKind()) {
@@ -7885,7 +7923,7 @@ void ASTContext::getObjCEncodingForTypeImpl(QualType T, std::string &S,
     return;
 
   case Type::Pipe:
-  case Type::ExtInt:
+  case Type::BitInt:
 #define ABSTRACT_TYPE(KIND, BASE)
 #define TYPE(KIND, BASE)
 #define DEPENDENT_TYPE(KIND, BASE) \
@@ -9234,7 +9272,7 @@ void getIntersectionOfProtocols(ASTContext &Context,
   // Remove any implied protocols from the list of inherited protocols.
   if (!ImpliedProtocols.empty()) {
     llvm::erase_if(IntersectionSet, [&](ObjCProtocolDecl *proto) -> bool {
-      return ImpliedProtocols.count(proto) > 0;
+      return ImpliedProtocols.contains(proto);
     });
   }
 
@@ -10101,12 +10139,12 @@ QualType ASTContext::mergeTypes(QualType LHS, QualType RHS,
     assert(LHS != RHS &&
            "Equivalent pipe types should have already been handled!");
     return {};
-  case Type::ExtInt: {
-    // Merge two ext-int types, while trying to preserve typedef info.
-    bool LHSUnsigned  = LHS->castAs<ExtIntType>()->isUnsigned();
-    bool RHSUnsigned = RHS->castAs<ExtIntType>()->isUnsigned();
-    unsigned LHSBits = LHS->castAs<ExtIntType>()->getNumBits();
-    unsigned RHSBits = RHS->castAs<ExtIntType>()->getNumBits();
+  case Type::BitInt: {
+    // Merge two bit-precise int types, while trying to preserve typedef info.
+    bool LHSUnsigned = LHS->castAs<BitIntType>()->isUnsigned();
+    bool RHSUnsigned = RHS->castAs<BitIntType>()->isUnsigned();
+    unsigned LHSBits = LHS->castAs<BitIntType>()->getNumBits();
+    unsigned RHSBits = RHS->castAs<BitIntType>()->getNumBits();
 
     // Like unsigned/int, shouldn't have a type if they don't match.
     if (LHSUnsigned != RHSUnsigned)
@@ -10256,7 +10294,7 @@ unsigned ASTContext::getIntWidth(QualType T) const {
     T = ET->getDecl()->getIntegerType();
   if (T->isBooleanType())
     return 1;
-  if(const auto *EIT = T->getAs<ExtIntType>())
+  if (const auto *EIT = T->getAs<BitIntType>())
     return EIT->getNumBits();
   // For builtin types, just use the standard type sizing method
   return (unsigned)getTypeSize(T);
@@ -10271,9 +10309,9 @@ QualType ASTContext::getCorrespondingUnsignedType(QualType T) const {
     return getVectorType(getCorrespondingUnsignedType(VTy->getElementType()),
                          VTy->getNumElements(), VTy->getVectorKind());
 
-  // For _ExtInt, return an unsigned _ExtInt with same width.
-  if (const auto *EITy = T->getAs<ExtIntType>())
-    return getExtIntType(/*IsUnsigned=*/true, EITy->getNumBits());
+  // For _BitInt, return an unsigned _BitInt with same width.
+  if (const auto *EITy = T->getAs<BitIntType>())
+    return getBitIntType(/*IsUnsigned=*/true, EITy->getNumBits());
 
   // For enums, get the underlying integer type of the enum, and let the general
   // integer type signchanging code handle it.
@@ -10339,9 +10377,9 @@ QualType ASTContext::getCorrespondingSignedType(QualType T) const {
     return getVectorType(getCorrespondingSignedType(VTy->getElementType()),
                          VTy->getNumElements(), VTy->getVectorKind());
 
-  // For _ExtInt, return a signed _ExtInt with same width.
-  if (const auto *EITy = T->getAs<ExtIntType>())
-    return getExtIntType(/*IsUnsigned=*/false, EITy->getNumBits());
+  // For _BitInt, return a signed _BitInt with same width.
+  if (const auto *EITy = T->getAs<BitIntType>())
+    return getBitIntType(/*IsUnsigned=*/false, EITy->getNumBits());
 
   // For enums, get the underlying integer type of the enum, and let the general
   // integer type signchanging code handle it.
diff --git a/clang/lib/AST/ASTDiagnostic.cpp b/clang/lib/AST/ASTDiagnostic.cpp
index 7e435e8b35b8..724ede272fbf 100644
--- a/clang/lib/AST/ASTDiagnostic.cpp
+++ b/clang/lib/AST/ASTDiagnostic.cpp
@@ -26,7 +26,8 @@ using namespace clang;
 
 // Returns a desugared version of the QualType, and marks ShouldAKA as true
 // whenever we remove significant sugar from the type.
-static QualType Desugar(ASTContext &Context, QualType QT, bool &ShouldAKA) {
+QualType clang::desugarForDiagnostic(ASTContext &Context, QualType QT,
+                                     bool &ShouldAKA) {
   QualifierCollector QC;
 
   while (true) {
@@ -37,6 +38,11 @@ static QualType Desugar(ASTContext &Context, QualType QT, bool &ShouldAKA) {
       QT = ET->desugar();
       continue;
     }
+    // ... or a using type ...
+    if (const UsingType *UT = dyn_cast<UsingType>(Ty)) {
+      QT = UT->desugar();
+      continue;
+    }
     // ... or a paren type ...
     if (const ParenType *PT = dyn_cast<ParenType>(Ty)) {
       QT = PT->desugar();
@@ -76,7 +82,7 @@ static QualType Desugar(ASTContext &Context, QualType QT, bool &ShouldAKA) {
     if (const FunctionType *FT = dyn_cast<FunctionType>(Ty)) {
       bool DesugarReturn = false;
       QualType SugarRT = FT->getReturnType();
-      QualType RT = Desugar(Context, SugarRT, DesugarReturn);
+      QualType RT = desugarForDiagnostic(Context, SugarRT, DesugarReturn);
       if (auto nullability = AttributedType::stripOuterNullability(SugarRT)) {
         RT = Context.getAttributedType(
             AttributedType::getNullabilityAttrKind(*nullability), RT, RT);
@@ -87,7 +93,7 @@ static QualType Desugar(ASTContext &Context, QualType QT, bool &ShouldAKA) {
       const FunctionProtoType *FPT = dyn_cast<FunctionProtoType>(FT);
       if (FPT) {
         for (QualType SugarPT : FPT->param_types()) {
-          QualType PT = Desugar(Context, SugarPT, DesugarArgument);
+          QualType PT = desugarForDiagnostic(Context, SugarPT, DesugarArgument);
           if (auto nullability =
                   AttributedType::stripOuterNullability(SugarPT)) {
             PT = Context.getAttributedType(
@@ -115,7 +121,8 @@ static QualType Desugar(ASTContext &Context, QualType QT, bool &ShouldAKA) {
         for (unsigned I = 0, N = TST->getNumArgs(); I != N; ++I) {
           const TemplateArgument &Arg = TST->getArg(I);
           if (Arg.getKind() == TemplateArgument::Type)
-            Args.push_back(Desugar(Context, Arg.getAsType(), DesugarArgument));
+            Args.push_back(desugarForDiagnostic(Context, Arg.getAsType(),
+                                                DesugarArgument));
           else
             Args.push_back(Arg);
         }
@@ -129,6 +136,29 @@ static QualType Desugar(ASTContext &Context, QualType QT, bool &ShouldAKA) {
       }
     }
 
+    if (const auto *AT = dyn_cast<ArrayType>(Ty)) {
+      QualType ElementTy =
+          desugarForDiagnostic(Context, AT->getElementType(), ShouldAKA);
+      if (const auto *CAT = dyn_cast<ConstantArrayType>(AT))
+        QT = Context.getConstantArrayType(
+            ElementTy, CAT->getSize(), CAT->getSizeExpr(),
+            CAT->getSizeModifier(), CAT->getIndexTypeCVRQualifiers());
+      else if (const auto *VAT = dyn_cast<VariableArrayType>(AT))
+        QT = Context.getVariableArrayType(
+            ElementTy, VAT->getSizeExpr(), VAT->getSizeModifier(),
+            VAT->getIndexTypeCVRQualifiers(), VAT->getBracketsRange());
+      else if (const auto *DSAT = dyn_cast<DependentSizedArrayType>(AT))
+        QT = Context.getDependentSizedArrayType(
+            ElementTy, DSAT->getSizeExpr(), DSAT->getSizeModifier(),
+            DSAT->getIndexTypeCVRQualifiers(), DSAT->getBracketsRange());
+      else if (const auto *IAT = dyn_cast<IncompleteArrayType>(AT))
+        QT = Context.getIncompleteArrayType(ElementTy, IAT->getSizeModifier(),
+                                            IAT->getIndexTypeCVRQualifiers());
+      else
+        llvm_unreachable("Unhandled array type");
+      break;
+    }
+
     // Don't desugar magic Objective-C types.
     if (QualType(Ty,0) == Context.getObjCIdType() ||
         QualType(Ty,0) == Context.getObjCClassType() ||
@@ -181,24 +211,25 @@ break; \
   // If we have a pointer-like type, desugar the pointee as well.
   // FIXME: Handle other pointer-like types.
   if (const PointerType *Ty = QT->getAs<PointerType>()) {
-    QT = Context.getPointerType(Desugar(Context, Ty->getPointeeType(),
-                                        ShouldAKA));
+    QT = Context.getPointerType(
+        desugarForDiagnostic(Context, Ty->getPointeeType(), ShouldAKA));
   } else if (const auto *Ty = QT->getAs<ObjCObjectPointerType>()) {
-    QT = Context.getObjCObjectPointerType(Desugar(Context, Ty->getPointeeType(),
-                                                  ShouldAKA));
+    QT = Context.getObjCObjectPointerType(
+        desugarForDiagnostic(Context, Ty->getPointeeType(), ShouldAKA));
   } else if (const LValueReferenceType *Ty = QT->getAs<LValueReferenceType>()) {
-    QT = Context.getLValueReferenceType(Desugar(Context, Ty->getPointeeType(),
-                                                ShouldAKA));
+    QT = Context.getLValueReferenceType(
+        desugarForDiagnostic(Context, Ty->getPointeeType(), ShouldAKA));
   } else if (const RValueReferenceType *Ty = QT->getAs<RValueReferenceType>()) {
-    QT = Context.getRValueReferenceType(Desugar(Context, Ty->getPointeeType(),
-                                                ShouldAKA));
+    QT = Context.getRValueReferenceType(
+        desugarForDiagnostic(Context, Ty->getPointeeType(), ShouldAKA));
   } else if (const auto *Ty = QT->getAs<ObjCObjectType>()) {
     if (Ty->getBaseType().getTypePtr() != Ty && !ShouldAKA) {
-      QualType BaseType = Desugar(Context, Ty->getBaseType(), ShouldAKA);
-      QT = Context.getObjCObjectType(BaseType, Ty->getTypeArgsAsWritten(),
-                                     llvm::makeArrayRef(Ty->qual_begin(),
-                                                        Ty->getNumProtocols()),
-                                     Ty->isKindOfTypeAsWritten());
+      QualType BaseType =
+          desugarForDiagnostic(Context, Ty->getBaseType(), ShouldAKA);
+      QT = Context.getObjCObjectType(
+          BaseType, Ty->getTypeArgsAsWritten(),
+          llvm::makeArrayRef(Ty->qual_begin(), Ty->getNumProtocols()),
+          Ty->isKindOfTypeAsWritten());
     }
   }
 
@@ -251,7 +282,8 @@ ConvertTypeToDiagnosticString(ASTContext &Context, QualType Ty,
       continue;  // Same canonical types
     std::string CompareS = CompareTy.getAsString(Context.getPrintingPolicy());
     bool ShouldAKA = false;
-    QualType CompareDesugar = Desugar(Context, CompareTy, ShouldAKA);
+    QualType CompareDesugar =
+        desugarForDiagnostic(Context, CompareTy, ShouldAKA);
     std::string CompareDesugarStr =
         CompareDesugar.getAsString(Context.getPrintingPolicy());
     if (CompareS != S && CompareDesugarStr != S)
@@ -286,7 +318,7 @@ ConvertTypeToDiagnosticString(ASTContext &Context, QualType Ty,
   // sugar gives us something "significantly different".
   if (!Repeated) {
     bool ShouldAKA = false;
-    QualType DesugaredTy = Desugar(Context, Ty, ShouldAKA);
+    QualType DesugaredTy = desugarForDiagnostic(Context, Ty, ShouldAKA);
     if (ShouldAKA || ForceAKA) {
       if (DesugaredTy == Ty) {
         DesugaredTy = Ty.getCanonicalType();
@@ -308,7 +340,7 @@ ConvertTypeToDiagnosticString(ASTContext &Context, QualType Ty,
       OS << "'" << S << "' (vector of " << VTy->getNumElements() << " '"
          << VTy->getElementType().getAsString(Context.getPrintingPolicy())
          << "' " << Values << ")";
-      return OS.str();
+      return DecoratedString;
     }
   }
 
diff --git a/clang/lib/AST/ASTImporter.cpp b/clang/lib/AST/ASTImporter.cpp
index 710e40bbb4b7..7f78da10e0b3 100644
--- a/clang/lib/AST/ASTImporter.cpp
+++ b/clang/lib/AST/ASTImporter.cpp
@@ -354,6 +354,7 @@ namespace clang {
     ExpectedType VisitTypeOfExprType(const TypeOfExprType *T);
     // FIXME: DependentTypeOfExprType
     ExpectedType VisitTypeOfType(const TypeOfType *T);
+    ExpectedType VisitUsingType(const UsingType *T);
     ExpectedType VisitDecltypeType(const DecltypeType *T);
     ExpectedType VisitUnaryTransformType(const UnaryTransformType *T);
     ExpectedType VisitAutoType(const AutoType *T);
@@ -1340,6 +1341,17 @@ ExpectedType ASTNodeImporter::VisitTypeOfType(const TypeOfType *T) {
   return Importer.getToContext().getTypeOfType(*ToUnderlyingTypeOrErr);
 }
 
+ExpectedType ASTNodeImporter::VisitUsingType(const UsingType *T) {
+  Expected<UsingShadowDecl *> FoundOrErr = import(T->getFoundDecl());
+  if (!FoundOrErr)
+    return FoundOrErr.takeError();
+  Expected<QualType> UnderlyingOrErr = import(T->getUnderlyingType());
+  if (!UnderlyingOrErr)
+    return UnderlyingOrErr.takeError();
+
+  return Importer.getToContext().getUsingType(*FoundOrErr, *UnderlyingOrErr);
+}
+
 ExpectedType ASTNodeImporter::VisitDecltypeType(const DecltypeType *T) {
   // FIXME: Make sure that the "to" context supports C++0x!
   ExpectedExpr ToExprOrErr = import(T->getUnderlyingExpr());
@@ -6066,20 +6078,24 @@ ASTNodeImporter::VisitFunctionTemplateDecl(FunctionTemplateDecl *D) {
   if (Error Err = importInto(TemplatedFD, D->getTemplatedDecl()))
     return std::move(Err);
 
-  // Template parameters of the ClassTemplateDecl and FunctionTemplateDecl are
-  // shared, if the FunctionTemplateDecl is a deduction guide for the class.
-  // At import the ClassTemplateDecl object is always created first (FIXME: is
-  // this really true?) because the dependency, then the FunctionTemplateDecl.
-  // The DeclContext of the template parameters is changed when the
-  // FunctionTemplateDecl is created, but was set already when the class
-  // template was created. So here it is not the TU (default value) any more.
-  // FIXME: The DeclContext of the parameters is now set finally to the
-  // CXXDeductionGuideDecl object that was imported later. This may not be the
-  // same that is in the original AST, specially if there are multiple deduction
-  // guides.
-  DeclContext *OldParamDC = nullptr;
-  if (Params->size() > 0)
-    OldParamDC = Params->getParam(0)->getDeclContext();
+  // At creation of the template the template parameters are "adopted"
+  // (DeclContext is changed). After this possible change the lookup table
+  // must be updated.
+  // At deduction guides the DeclContext of the template parameters may be
+  // different from what we would expect, it may be the class template, or a
+  // probably different CXXDeductionGuideDecl. This may come from the fact that
+  // the template parameter objects may be shared between deduction guides or
+  // the class template, and at creation of multiple FunctionTemplateDecl
+  // objects (for deduction guides) the same parameters are re-used. The
+  // "adoption" happens multiple times with different parent, even recursively
+  // for TemplateTemplateParmDecl. The same happens at import when the
+  // FunctionTemplateDecl objects are created, but in different order.
+  // In this way the DeclContext of these template parameters is not necessarily
+  // the same as in the "from" context.
+  SmallVector<DeclContext *, 2> OldParamDC;
+  OldParamDC.reserve(Params->size());
+  llvm::transform(*Params, std::back_inserter(OldParamDC),
+                  [](NamedDecl *ND) { return ND->getDeclContext(); });
 
   FunctionTemplateDecl *ToFunc;
   if (GetImportedOrCreateDecl(ToFunc, D, Importer.getToContext(), DC, Loc, Name,
@@ -6091,7 +6107,12 @@ ASTNodeImporter::VisitFunctionTemplateDecl(FunctionTemplateDecl *D) {
   ToFunc->setAccess(D->getAccess());
   ToFunc->setLexicalDeclContext(LexicalDC);
   LexicalDC->addDeclInternal(ToFunc);
-  updateLookupTableForTemplateParameters(*Params, OldParamDC);
+
+  ASTImporterLookupTable *LT = Importer.SharedState->getLookupTable();
+  if (LT && !OldParamDC.empty()) {
+    for (unsigned int I = 0; I < OldParamDC.size(); ++I)
+      LT->updateForced(Params->getParam(I), OldParamDC[I]);
+  }
 
   if (FoundByLookup) {
     auto *Recent =
diff --git a/clang/lib/AST/ASTImporterLookupTable.cpp b/clang/lib/AST/ASTImporterLookupTable.cpp
index ef42561c6f94..b7d17a5e92d0 100644
--- a/clang/lib/AST/ASTImporterLookupTable.cpp
+++ b/clang/lib/AST/ASTImporterLookupTable.cpp
@@ -140,6 +140,11 @@ void ASTImporterLookupTable::update(NamedDecl *ND, DeclContext *OldDC) {
   add(ND);
 }
 
+void ASTImporterLookupTable::updateForced(NamedDecl *ND, DeclContext *OldDC) {
+  LookupTable[OldDC][ND->getDeclName()].remove(ND);
+  add(ND);
+}
+
 ASTImporterLookupTable::LookupResult
 ASTImporterLookupTable::lookup(DeclContext *DC, DeclarationName Name) const {
   auto DCI = LookupTable.find(DC->getPrimaryContext());
diff --git a/clang/lib/AST/ASTStructuralEquivalence.cpp b/clang/lib/AST/ASTStructuralEquivalence.cpp
index 7fd24e2aa9ad..0813a5204a5e 100644
--- a/clang/lib/AST/ASTStructuralEquivalence.cpp
+++ b/clang/lib/AST/ASTStructuralEquivalence.cpp
@@ -945,6 +945,12 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
       return false;
     break;
 
+  case Type::Using:
+    if (!IsStructurallyEquivalent(Context, cast<UsingType>(T1)->getFoundDecl(),
+                                  cast<UsingType>(T2)->getFoundDecl()))
+      return false;
+    break;
+
   case Type::Typedef:
     if (!IsStructurallyEquivalent(Context, cast<TypedefType>(T1)->getDecl(),
                                   cast<TypedefType>(T2)->getDecl()))
@@ -1205,18 +1211,18 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
                                   cast<PipeType>(T2)->getElementType()))
       return false;
     break;
-  case Type::ExtInt: {
-    const auto *Int1 = cast<ExtIntType>(T1);
-    const auto *Int2 = cast<ExtIntType>(T2);
+  case Type::BitInt: {
+    const auto *Int1 = cast<BitIntType>(T1);
+    const auto *Int2 = cast<BitIntType>(T2);
 
     if (Int1->isUnsigned() != Int2->isUnsigned() ||
         Int1->getNumBits() != Int2->getNumBits())
       return false;
     break;
   }
-  case Type::DependentExtInt: {
-    const auto *Int1 = cast<DependentExtIntType>(T1);
-    const auto *Int2 = cast<DependentExtIntType>(T2);
+  case Type::DependentBitInt: {
+    const auto *Int1 = cast<DependentBitIntType>(T1);
+    const auto *Int2 = cast<DependentBitIntType>(T2);
 
     if (Int1->isUnsigned() != Int2->isUnsigned() ||
         !IsStructurallyEquivalent(Context, Int1->getNumBitsExpr(),
diff --git a/clang/lib/AST/AttrImpl.cpp b/clang/lib/AST/AttrImpl.cpp
index a3b46752c511..c2f13cf63830 100644
--- a/clang/lib/AST/AttrImpl.cpp
+++ b/clang/lib/AST/AttrImpl.cpp
@@ -60,7 +60,7 @@ std::string LoopHintAttr::getValueString(const PrintingPolicy &Policy) const {
   else
     OS << "disable";
   OS << ")";
-  return OS.str();
+  return ValueName;
 }
 
 // Return a string suitable for identifying this attribute in diagnostics.
diff --git a/clang/lib/AST/Comment.cpp b/clang/lib/AST/Comment.cpp
index fae3640d5ff7..43820fc566e4 100644
--- a/clang/lib/AST/Comment.cpp
+++ b/clang/lib/AST/Comment.cpp
@@ -108,12 +108,7 @@ Comment::child_iterator Comment::child_end() const {
 }
 
 bool TextComment::isWhitespaceNoCache() const {
-  for (StringRef::const_iterator I = Text.begin(), E = Text.end();
-       I != E; ++I) {
-    if (!clang::isWhitespace(*I))
-      return false;
-  }
-  return true;
+  return llvm::all_of(Text, clang::isWhitespace);
 }
 
 bool ParagraphComment::isWhitespaceNoCache() const {
diff --git a/clang/lib/AST/CommentBriefParser.cpp b/clang/lib/AST/CommentBriefParser.cpp
index 2a5f7452b776..bf9e17993497 100644
--- a/clang/lib/AST/CommentBriefParser.cpp
+++ b/clang/lib/AST/CommentBriefParser.cpp
@@ -8,15 +8,12 @@
 
 #include "clang/AST/CommentBriefParser.h"
 #include "clang/AST/CommentCommandTraits.h"
+#include "clang/Basic/CharInfo.h"
 
 namespace clang {
 namespace comments {
 
 namespace {
-inline bool isWhitespace(char C) {
-  return C == ' ' || C == '\n' || C == '\r' ||
-         C == '\t' || C == '\f' || C == '\v';
-}
 
 /// Convert all whitespace into spaces, remove leading and trailing spaces,
 /// compress multiple spaces into one.
@@ -26,12 +23,11 @@ void cleanupBrief(std::string &S) {
   for (std::string::iterator I = S.begin(), E = S.end();
        I != E; ++I) {
     const char C = *I;
-    if (isWhitespace(C)) {
+    if (clang::isWhitespace(C)) {
       if (!PrevWasSpace) {
         *O++ = ' ';
         PrevWasSpace = true;
       }
-      continue;
     } else {
       *O++ = C;
       PrevWasSpace = false;
@@ -44,12 +40,7 @@ void cleanupBrief(std::string &S) {
 }
 
 bool isWhitespace(StringRef Text) {
-  for (StringRef::const_iterator I = Text.begin(), E = Text.end();
-       I != E; ++I) {
-    if (!isWhitespace(*I))
-      return false;
-  }
-  return true;
+  return llvm::all_of(Text, clang::isWhitespace);
 }
 } // unnamed namespace
 
diff --git a/clang/lib/AST/Decl.cpp b/clang/lib/AST/Decl.cpp
index 68dfef248f65..e63560f1b6fe 100644
--- a/clang/lib/AST/Decl.cpp
+++ b/clang/lib/AST/Decl.cpp
@@ -604,8 +604,14 @@ static LinkageInfo getExternalLinkageFor(const NamedDecl *D) {
   //   - A name declared at namespace scope that does not have internal linkage
   //     by the previous rules and that is introduced by a non-exported
   //     declaration has module linkage.
-  if (isInModulePurview(D) && !isExportedFromModuleInterfaceUnit(
-                                  cast<NamedDecl>(D->getCanonicalDecl())))
+  //
+  // [basic.namespace.general]/p2
+  //   A namespace is never attached to a named module and never has a name with
+  //   module linkage.
+  if (isInModulePurview(D) &&
+      !isExportedFromModuleInterfaceUnit(
+          cast<NamedDecl>(D->getCanonicalDecl())) &&
+      !isa<NamespaceDecl>(D))
     return LinkageInfo(ModuleLinkage, DefaultVisibility, false);
 
   return LinkageInfo::external();
@@ -1583,7 +1589,7 @@ std::string NamedDecl::getQualifiedNameAsString() const {
   std::string QualName;
   llvm::raw_string_ostream OS(QualName);
   printQualifiedName(OS, getASTContext().getPrintingPolicy());
-  return OS.str();
+  return QualName;
 }
 
 void NamedDecl::printQualifiedName(raw_ostream &OS) const {
diff --git a/clang/lib/AST/DeclarationName.cpp b/clang/lib/AST/DeclarationName.cpp
index 56cf4b457a48..b2232ddfced3 100644
--- a/clang/lib/AST/DeclarationName.cpp
+++ b/clang/lib/AST/DeclarationName.cpp
@@ -236,7 +236,7 @@ std::string DeclarationName::getAsString() const {
   std::string Result;
   llvm::raw_string_ostream OS(Result);
   OS << *this;
-  return OS.str();
+  return Result;
 }
 
 void *DeclarationName::getFETokenInfoSlow() const {
@@ -460,7 +460,7 @@ std::string DeclarationNameInfo::getAsString() const {
   std::string Result;
   llvm::raw_string_ostream OS(Result);
   OS << *this;
-  return OS.str();
+  return Result;
 }
 
 raw_ostream &clang::operator<<(raw_ostream &OS, DeclarationNameInfo DNInfo) {
diff --git a/clang/lib/AST/Expr.cpp b/clang/lib/AST/Expr.cpp
index d3cb2ff3734c..2530beb89d17 100644
--- a/clang/lib/AST/Expr.cpp
+++ b/clang/lib/AST/Expr.cpp
@@ -202,6 +202,23 @@ bool Expr::isKnownToHaveBooleanValue(bool Semantic) const {
   return false;
 }
 
+const ValueDecl *
+Expr::getAsBuiltinConstantDeclRef(const ASTContext &Context) const {
+  Expr::EvalResult Eval;
+
+  if (EvaluateAsConstantExpr(Eval, Context)) {
+    APValue &Value = Eval.Val;
+
+    if (Value.isMemberPointer())
+      return Value.getMemberPointerDecl();
+
+    if (Value.isLValue() && Value.getLValueOffset().isZero())
+      return Value.getLValueBase().dyn_cast<const ValueDecl *>();
+  }
+
+  return nullptr;
+}
+
 // Amusing macro metaprogramming hack: check whether a class provides
 // a more specific implementation of getExprLoc().
 //
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index 99babd58b027..469339e8cd62 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -1954,11 +1954,12 @@ static bool EvaluateIgnoredValue(EvalInfo &Info, const Expr *E) {
   return true;
 }
 
-/// Should this call expression be treated as a string literal?
-static bool IsStringLiteralCall(const CallExpr *E) {
+/// Should this call expression be treated as a constant?
+static bool IsConstantCall(const CallExpr *E) {
   unsigned Builtin = E->getBuiltinCallee();
   return (Builtin == Builtin::BI__builtin___CFStringMakeConstantString ||
-          Builtin == Builtin::BI__builtin___NSStringMakeConstantString);
+          Builtin == Builtin::BI__builtin___NSStringMakeConstantString ||
+          Builtin == Builtin::BI__builtin_function_start);
 }
 
 static bool IsGlobalLValue(APValue::LValueBase B) {
@@ -2004,7 +2005,7 @@ static bool IsGlobalLValue(APValue::LValueBase B) {
   case Expr::ObjCBoxedExprClass:
     return cast<ObjCBoxedExpr>(E)->isExpressibleAsConstantInitializer();
   case Expr::CallExprClass:
-    return IsStringLiteralCall(cast<CallExpr>(E));
+    return IsConstantCall(cast<CallExpr>(E));
   // For GCC compatibility, &&label has static storage duration.
   case Expr::AddrLabelExprClass:
     return true;
@@ -2931,6 +2932,11 @@ handleCompareOpForVectorHelper(const APTy &LHSValue, BinaryOperatorKind Opcode,
     break;
   }
 
+  // The boolean operations on these vector types use an instruction that
+  // results in a mask of '-1' for the 'truth' value.  Ensure that we negate 1
+  // to -1 to make sure that we produce the correct value.
+  Result.negate();
+
   return true;
 }
 
@@ -8962,7 +8968,7 @@ bool PointerExprEvaluator::visitNonBuiltinCallExpr(const CallExpr *E) {
 }
 
 bool PointerExprEvaluator::VisitCallExpr(const CallExpr *E) {
-  if (IsStringLiteralCall(E))
+  if (IsConstantCall(E))
     return Success(E);
 
   if (unsigned BuiltinOp = E->getBuiltinCallee())
@@ -10179,7 +10185,8 @@ namespace {
     bool VisitInitListExpr(const InitListExpr *E);
     bool VisitUnaryImag(const UnaryOperator *E);
     bool VisitBinaryOperator(const BinaryOperator *E);
-    // FIXME: Missing: unary -, unary ~, conditional operator (for GNU
+    bool VisitUnaryOperator(const UnaryOperator *E);
+    // FIXME: Missing: conditional operator (for GNU
     //                 conditional select), shufflevector, ExtVectorElementExpr
   };
 } // end anonymous namespace
@@ -10364,6 +10371,83 @@ bool VectorExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
   return Success(LHSValue, E);
 }
 
+static llvm::Optional<APValue> handleVectorUnaryOperator(ASTContext &Ctx,
+                                                         QualType ResultTy,
+                                                         UnaryOperatorKind Op,
+                                                         APValue Elt) {
+  switch (Op) {
+  case UO_Plus:
+    // Nothing to do here.
+    return Elt;
+  case UO_Minus:
+    if (Elt.getKind() == APValue::Int) {
+      Elt.getInt().negate();
+    } else {
+      assert(Elt.getKind() == APValue::Float &&
+             "Vector can only be int or float type");
+      Elt.getFloat().changeSign();
+    }
+    return Elt;
+  case UO_Not:
+    // This is only valid for integral types anyway, so we don't have to handle
+    // float here.
+    assert(Elt.getKind() == APValue::Int &&
+           "Vector operator ~ can only be int");
+    Elt.getInt().flipAllBits();
+    return Elt;
+  case UO_LNot: {
+    if (Elt.getKind() == APValue::Int) {
+      Elt.getInt() = !Elt.getInt();
+      // operator ! on vectors returns -1 for 'truth', so negate it.
+      Elt.getInt().negate();
+      return Elt;
+    }
+    assert(Elt.getKind() == APValue::Float &&
+           "Vector can only be int or float type");
+    // Float types result in an int of the same size, but -1 for true, or 0 for
+    // false.
+    APSInt EltResult{Ctx.getIntWidth(ResultTy),
+                     ResultTy->isUnsignedIntegerType()};
+    if (Elt.getFloat().isZero())
+      EltResult.setAllBits();
+    else
+      EltResult.clearAllBits();
+
+    return APValue{EltResult};
+  }
+  default:
+    // FIXME: Implement the rest of the unary operators.
+    return llvm::None;
+  }
+}
+
+bool VectorExprEvaluator::VisitUnaryOperator(const UnaryOperator *E) {
+  Expr *SubExpr = E->getSubExpr();
+  const auto *VD = SubExpr->getType()->castAs<VectorType>();
+  // This result element type differs in the case of negating a floating point
+  // vector, since the result type is the a vector of the equivilant sized
+  // integer.
+  const QualType ResultEltTy = VD->getElementType();
+  UnaryOperatorKind Op = E->getOpcode();
+
+  APValue SubExprValue;
+  if (!Evaluate(SubExprValue, Info, SubExpr))
+    return false;
+
+  assert(SubExprValue.getVectorLength() == VD->getNumElements() &&
+         "Vector length doesn't match type?");
+
+  SmallVector<APValue, 4> ResultElements;
+  for (unsigned EltNum = 0; EltNum < VD->getNumElements(); ++EltNum) {
+    llvm::Optional<APValue> Elt = handleVectorUnaryOperator(
+        Info.Ctx, ResultEltTy, Op, SubExprValue.getVectorElt(EltNum));
+    if (!Elt)
+      return false;
+    ResultElements.push_back(*Elt);
+  }
+  return Success(APValue(ResultElements.data(), ResultElements.size()), E);
+}
+
 //===----------------------------------------------------------------------===//
 // Array Evaluation
 //===----------------------------------------------------------------------===//
@@ -11077,7 +11161,7 @@ EvaluateBuiltinClassifyType(QualType T, const LangOptions &LangOpts) {
   case Type::ObjCInterface:
   case Type::ObjCObjectPointer:
   case Type::Pipe:
-  case Type::ExtInt:
+  case Type::BitInt:
     // GCC classifies vectors as None. We follow its lead and classify all
     // other types that don't fit into the regular classification the same way.
     return GCCTypeClass::None;
diff --git a/clang/lib/AST/ItaniumMangle.cpp b/clang/lib/AST/ItaniumMangle.cpp
index 07579d04e275..7afc1250a36f 100644
--- a/clang/lib/AST/ItaniumMangle.cpp
+++ b/clang/lib/AST/ItaniumMangle.cpp
@@ -2263,8 +2263,8 @@ bool CXXNameMangler::mangleUnresolvedTypeOrSimpleId(QualType Ty,
   case Type::Atomic:
   case Type::Pipe:
   case Type::MacroQualified:
-  case Type::ExtInt:
-  case Type::DependentExtInt:
+  case Type::BitInt:
+  case Type::DependentBitInt:
     llvm_unreachable("type is illegal as a nested name specifier");
 
   case Type::SubstTemplateTypeParmPack:
@@ -2380,6 +2380,9 @@ bool CXXNameMangler::mangleUnresolvedTypeOrSimpleId(QualType Ty,
     break;
   }
 
+  case Type::Using:
+    return mangleUnresolvedTypeOrSimpleId(cast<UsingType>(Ty)->desugar(),
+                                          Prefix);
   case Type::Elaborated:
     return mangleUnresolvedTypeOrSimpleId(
         cast<ElaboratedType>(Ty)->getNamedType(), Prefix);
@@ -3967,26 +3970,20 @@ void CXXNameMangler::mangleType(const PipeType *T) {
   Out << "8ocl_pipe";
 }
 
-void CXXNameMangler::mangleType(const ExtIntType *T) {
-  Out << "U7_ExtInt";
-  llvm::APSInt BW(32, true);
-  BW = T->getNumBits();
-  TemplateArgument TA(Context.getASTContext(), BW, getASTContext().IntTy);
-  mangleTemplateArgs(TemplateName(), &TA, 1);
-  if (T->isUnsigned())
-    Out << "j";
-  else
-    Out << "i";
+void CXXNameMangler::mangleType(const BitIntType *T) {
+  // 5.1.5.2 Builtin types
+  // <type> ::= DB <number | instantiation-dependent expression> _
+  //        ::= DU <number | instantiation-dependent expression> _
+  Out << "D" << (T->isUnsigned() ? "U" : "B") << T->getNumBits() << "_";
 }
 
-void CXXNameMangler::mangleType(const DependentExtIntType *T) {
-  Out << "U7_ExtInt";
-  TemplateArgument TA(T->getNumBitsExpr());
-  mangleTemplateArgs(TemplateName(), &TA, 1);
-  if (T->isUnsigned())
-    Out << "j";
-  else
-    Out << "i";
+void CXXNameMangler::mangleType(const DependentBitIntType *T) {
+  // 5.1.5.2 Builtin types
+  // <type> ::= DB <number | instantiation-dependent expression> _
+  //        ::= DU <number | instantiation-dependent expression> _
+  Out << "D" << (T->isUnsigned() ? "U" : "B");
+  mangleExpression(T->getNumBitsExpr());
+  Out << "_";
 }
 
 void CXXNameMangler::mangleIntegerLiteral(QualType T,
diff --git a/clang/lib/AST/JSONNodeDumper.cpp b/clang/lib/AST/JSONNodeDumper.cpp
index 86879b8c3533..ae585def4d07 100644
--- a/clang/lib/AST/JSONNodeDumper.cpp
+++ b/clang/lib/AST/JSONNodeDumper.cpp
@@ -744,11 +744,18 @@ void JSONNodeDumper::VisitNamedDecl(const NamedDecl *ND) {
     JOS.attribute("name", ND->getNameAsString());
     // FIXME: There are likely other contexts in which it makes no sense to ask
     // for a mangled name.
-    if (!isa<RequiresExprBodyDecl>(ND->getDeclContext())) {
-      std::string MangledName = ASTNameGen.getName(ND);
-      if (!MangledName.empty())
-        JOS.attribute("mangledName", MangledName);
-    }
+    if (isa<RequiresExprBodyDecl>(ND->getDeclContext()))
+      return;
+
+    // Mangled names are not meaningful for locals, and may not be well-defined
+    // in the case of VLAs.
+    auto *VD = dyn_cast<VarDecl>(ND);
+    if (VD && VD->hasLocalStorage())
+      return;
+
+    std::string MangledName = ASTNameGen.getName(ND);
+    if (!MangledName.empty())
+      JOS.attribute("mangledName", MangledName);
   }
 }
 
diff --git a/clang/lib/AST/MicrosoftMangle.cpp b/clang/lib/AST/MicrosoftMangle.cpp
index 79a448a2435c..8802b6e500a6 100644
--- a/clang/lib/AST/MicrosoftMangle.cpp
+++ b/clang/lib/AST/MicrosoftMangle.cpp
@@ -3365,26 +3365,26 @@ void MicrosoftMangleContextImpl::mangleCXXName(GlobalDecl GD,
   return Mangler.mangle(GD);
 }
 
-void MicrosoftCXXNameMangler::mangleType(const ExtIntType *T, Qualifiers,
+void MicrosoftCXXNameMangler::mangleType(const BitIntType *T, Qualifiers,
                                          SourceRange Range) {
   llvm::SmallString<64> TemplateMangling;
   llvm::raw_svector_ostream Stream(TemplateMangling);
   MicrosoftCXXNameMangler Extra(Context, Stream);
   Stream << "?$";
   if (T->isUnsigned())
-    Extra.mangleSourceName("_UExtInt");
+    Extra.mangleSourceName("_UBitInt");
   else
-    Extra.mangleSourceName("_ExtInt");
+    Extra.mangleSourceName("_BitInt");
   Extra.mangleIntegerLiteral(llvm::APSInt::getUnsigned(T->getNumBits()));
 
   mangleArtificialTagType(TTK_Struct, TemplateMangling, {"__clang"});
 }
 
-void MicrosoftCXXNameMangler::mangleType(const DependentExtIntType *T,
+void MicrosoftCXXNameMangler::mangleType(const DependentBitIntType *T,
                                          Qualifiers, SourceRange Range) {
   DiagnosticsEngine &Diags = Context.getDiags();
   unsigned DiagID = Diags.getCustomDiagID(
-      DiagnosticsEngine::Error, "cannot mangle this DependentExtInt type yet");
+      DiagnosticsEngine::Error, "cannot mangle this DependentBitInt type yet");
   Diags.Report(Range.getBegin(), DiagID) << Range;
 }
 
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index f721e56f7fdd..1bd049b88005 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -126,6 +126,7 @@ const OMPClauseWithPreInit *OMPClauseWithPreInit::get(const OMPClause *C) {
   case OMPC_write:
   case OMPC_update:
   case OMPC_capture:
+  case OMPC_compare:
   case OMPC_seq_cst:
   case OMPC_acq_rel:
   case OMPC_acquire:
@@ -217,6 +218,7 @@ const OMPClauseWithPostUpdate *OMPClauseWithPostUpdate::get(const OMPClause *C)
   case OMPC_write:
   case OMPC_update:
   case OMPC_capture:
+  case OMPC_compare:
   case OMPC_seq_cst:
   case OMPC_acq_rel:
   case OMPC_acquire:
@@ -1792,6 +1794,10 @@ void OMPClausePrinter::VisitOMPCaptureClause(OMPCaptureClause *) {
   OS << "capture";
 }
 
+void OMPClausePrinter::VisitOMPCompareClause(OMPCompareClause *) {
+  OS << "compare";
+}
+
 void OMPClausePrinter::VisitOMPSeqCstClause(OMPSeqCstClause *) {
   OS << "seq_cst";
 }
@@ -2454,7 +2460,7 @@ std::string OMPTraitInfo::getMangledName() const {
                                                 Property.RawString);
     }
   }
-  return OS.str();
+  return MangledName;
 }
 
 OMPTraitInfo::OMPTraitInfo(StringRef MangledName) {
diff --git a/clang/lib/AST/ParentMap.cpp b/clang/lib/AST/ParentMap.cpp
index 2ff5c9d8aeb5..da21e573c320 100644
--- a/clang/lib/AST/ParentMap.cpp
+++ b/clang/lib/AST/ParentMap.cpp
@@ -133,8 +133,7 @@ void ParentMap::setParent(const Stmt *S, const Stmt *Parent) {
 
 Stmt* ParentMap::getParent(Stmt* S) const {
   MapTy* M = (MapTy*) Impl;
-  MapTy::iterator I = M->find(S);
-  return I == M->end() ? nullptr : I->second;
+  return M->lookup(S);
 }
 
 Stmt *ParentMap::getParentIgnoreParens(Stmt *S) const {
diff --git a/clang/lib/AST/QualTypeNames.cpp b/clang/lib/AST/QualTypeNames.cpp
index 673821078345..561757b1ba64 100644
--- a/clang/lib/AST/QualTypeNames.cpp
+++ b/clang/lib/AST/QualTypeNames.cpp
@@ -418,6 +418,13 @@ QualType getFullyQualifiedType(QualType QT, const ASTContext &Ctx,
     return QT;
   }
 
+  // We don't consider the alias introduced by `using a::X` as a new type.
+  // The qualified name is still a::X.
+  if (isa<UsingType>(QT.getTypePtr())) {
+    return getFullyQualifiedType(QT.getSingleStepDesugaredType(Ctx), Ctx,
+                                 WithGlobalNsPrefix);
+  }
+
   // Remove the part of the type related to the type being a template
   // parameter (we won't report it as part of the 'type name' and it
   // is actually make the code below to be more complex (to handle
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index 4339c249e027..09853e0f0e49 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -551,6 +551,8 @@ void OMPClauseProfiler::VisitOMPUpdateClause(const OMPUpdateClause *) {}
 
 void OMPClauseProfiler::VisitOMPCaptureClause(const OMPCaptureClause *) {}
 
+void OMPClauseProfiler::VisitOMPCompareClause(const OMPCompareClause *) {}
+
 void OMPClauseProfiler::VisitOMPSeqCstClause(const OMPSeqCstClause *) {}
 
 void OMPClauseProfiler::VisitOMPAcqRelClause(const OMPAcqRelClause *) {}
diff --git a/clang/lib/AST/TextNodeDumper.cpp b/clang/lib/AST/TextNodeDumper.cpp
index b21e806e307c..67c934847c7f 100644
--- a/clang/lib/AST/TextNodeDumper.cpp
+++ b/clang/lib/AST/TextNodeDumper.cpp
@@ -1534,6 +1534,10 @@ void TextNodeDumper::VisitUnresolvedUsingType(const UnresolvedUsingType *T) {
   dumpDeclRef(T->getDecl());
 }
 
+void TextNodeDumper::VisitUsingType(const UsingType *T) {
+  dumpDeclRef(T->getFoundDecl());
+}
+
 void TextNodeDumper::VisitTypedefType(const TypedefType *T) {
   dumpDeclRef(T->getDecl());
 }
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
index e0ac3f5b1351..c771fe264b0c 100644
--- a/clang/lib/AST/Type.cpp
+++ b/clang/lib/AST/Type.cpp
@@ -338,25 +338,25 @@ VectorType::VectorType(TypeClass tc, QualType vecType, unsigned nElements,
   VectorTypeBits.NumElements = nElements;
 }
 
-ExtIntType::ExtIntType(bool IsUnsigned, unsigned NumBits)
-    : Type(ExtInt, QualType{}, TypeDependence::None), IsUnsigned(IsUnsigned),
+BitIntType::BitIntType(bool IsUnsigned, unsigned NumBits)
+    : Type(BitInt, QualType{}, TypeDependence::None), IsUnsigned(IsUnsigned),
       NumBits(NumBits) {}
 
-DependentExtIntType::DependentExtIntType(const ASTContext &Context,
+DependentBitIntType::DependentBitIntType(const ASTContext &Context,
                                          bool IsUnsigned, Expr *NumBitsExpr)
-    : Type(DependentExtInt, QualType{},
+    : Type(DependentBitInt, QualType{},
            toTypeDependence(NumBitsExpr->getDependence())),
       Context(Context), ExprAndUnsigned(NumBitsExpr, IsUnsigned) {}
 
-bool DependentExtIntType::isUnsigned() const {
+bool DependentBitIntType::isUnsigned() const {
   return ExprAndUnsigned.getInt();
 }
 
-clang::Expr *DependentExtIntType::getNumBitsExpr() const {
+clang::Expr *DependentBitIntType::getNumBitsExpr() const {
   return ExprAndUnsigned.getPointer();
 }
 
-void DependentExtIntType::Profile(llvm::FoldingSetNodeID &ID,
+void DependentBitIntType::Profile(llvm::FoldingSetNodeID &ID,
                                   const ASTContext &Context, bool IsUnsigned,
                                   Expr *NumBitsExpr) {
   ID.AddBoolean(IsUnsigned);
@@ -1932,7 +1932,7 @@ bool Type::isIntegralType(const ASTContext &Ctx) const {
     if (const auto *ET = dyn_cast<EnumType>(CanonicalType))
       return ET->getDecl()->isComplete();
 
-  return isExtIntType();
+  return isBitIntType();
 }
 
 bool Type::isIntegralOrUnscopedEnumerationType() const {
@@ -1940,7 +1940,7 @@ bool Type::isIntegralOrUnscopedEnumerationType() const {
     return BT->getKind() >= BuiltinType::Bool &&
            BT->getKind() <= BuiltinType::Int128;
 
-  if (isExtIntType())
+  if (isBitIntType())
     return true;
 
   return isUnscopedEnumerationType();
@@ -2023,7 +2023,9 @@ bool Type::isSignedIntegerType() const {
       return ET->getDecl()->getIntegerType()->isSignedIntegerType();
   }
 
-  if (const ExtIntType *IT = dyn_cast<ExtIntType>(CanonicalType))
+  if (const auto *IT = dyn_cast<BitIntType>(CanonicalType))
+    return IT->isSigned();
+  if (const auto *IT = dyn_cast<DependentBitIntType>(CanonicalType))
     return IT->isSigned();
 
   return false;
@@ -2040,9 +2042,10 @@ bool Type::isSignedIntegerOrEnumerationType() const {
       return ET->getDecl()->getIntegerType()->isSignedIntegerType();
   }
 
-  if (const ExtIntType *IT = dyn_cast<ExtIntType>(CanonicalType))
+  if (const auto *IT = dyn_cast<BitIntType>(CanonicalType))
+    return IT->isSigned();
+  if (const auto *IT = dyn_cast<DependentBitIntType>(CanonicalType))
     return IT->isSigned();
-
 
   return false;
 }
@@ -2070,7 +2073,9 @@ bool Type::isUnsignedIntegerType() const {
       return ET->getDecl()->getIntegerType()->isUnsignedIntegerType();
   }
 
-  if (const ExtIntType *IT = dyn_cast<ExtIntType>(CanonicalType))
+  if (const auto *IT = dyn_cast<BitIntType>(CanonicalType))
+    return IT->isUnsigned();
+  if (const auto *IT = dyn_cast<DependentBitIntType>(CanonicalType))
     return IT->isUnsigned();
 
   return false;
@@ -2087,7 +2092,9 @@ bool Type::isUnsignedIntegerOrEnumerationType() const {
       return ET->getDecl()->getIntegerType()->isUnsignedIntegerType();
   }
 
-  if (const ExtIntType *IT = dyn_cast<ExtIntType>(CanonicalType))
+  if (const auto *IT = dyn_cast<BitIntType>(CanonicalType))
+    return IT->isUnsigned();
+  if (const auto *IT = dyn_cast<DependentBitIntType>(CanonicalType))
     return IT->isUnsigned();
 
   return false;
@@ -2129,7 +2136,7 @@ bool Type::isRealType() const {
            BT->getKind() <= BuiltinType::Ibm128;
   if (const auto *ET = dyn_cast<EnumType>(CanonicalType))
       return ET->getDecl()->isComplete() && !ET->getDecl()->isScoped();
-  return isExtIntType();
+  return isBitIntType();
 }
 
 bool Type::isArithmeticType() const {
@@ -2145,7 +2152,7 @@ bool Type::isArithmeticType() const {
     // false for scoped enumerations since that will disable any
     // unwanted implicit conversions.
     return !ET->getDecl()->isScoped() && ET->getDecl()->isComplete();
-  return isa<ComplexType>(CanonicalType) || isExtIntType();
+  return isa<ComplexType>(CanonicalType) || isBitIntType();
 }
 
 Type::ScalarTypeKind Type::getScalarTypeKind() const {
@@ -2174,7 +2181,7 @@ Type::ScalarTypeKind Type::getScalarTypeKind() const {
     if (CT->getElementType()->isRealFloatingType())
       return STK_FloatingComplex;
     return STK_IntegralComplex;
-  } else if (isExtIntType()) {
+  } else if (isBitIntType()) {
     return STK_Integral;
   }
 
@@ -2381,7 +2388,7 @@ bool QualType::isCXX98PODType(const ASTContext &Context) const {
   case Type::MemberPointer:
   case Type::Vector:
   case Type::ExtVector:
-  case Type::ExtInt:
+  case Type::BitInt:
     return true;
 
   case Type::Enum:
@@ -3400,6 +3407,17 @@ QualType TypedefType::desugar() const {
   return getDecl()->getUnderlyingType();
 }
 
+UsingType::UsingType(const UsingShadowDecl *Found, QualType Underlying,
+                     QualType Canon)
+    : Type(Using, Canon, Underlying->getDependence()),
+      Found(const_cast<UsingShadowDecl *>(Found)) {
+  assert(Underlying == getUnderlyingType());
+}
+
+QualType UsingType::getUnderlyingType() const {
+  return QualType(cast<TypeDecl>(Found->getTargetDecl())->getTypeForDecl(), 0);
+}
+
 QualType MacroQualifiedType::desugar() const { return getUnderlyingType(); }
 
 QualType MacroQualifiedType::getModifiedType() const {
@@ -3849,7 +3867,7 @@ static CachedProperties computeCachedProperties(const Type *T) {
     // here in error recovery.
     return CachedProperties(ExternalLinkage, false);
 
-  case Type::ExtInt:
+  case Type::BitInt:
   case Type::Builtin:
     // C++ [basic.link]p8:
     //   A type is said to have linkage if and only if:
@@ -3949,7 +3967,7 @@ LinkageInfo LinkageComputer::computeTypeLinkageInfo(const Type *T) {
     assert(T->isInstantiationDependentType());
     return LinkageInfo::external();
 
-  case Type::ExtInt:
+  case Type::BitInt:
   case Type::Builtin:
     return LinkageInfo::external();
 
@@ -4169,8 +4187,8 @@ bool Type::canHaveNullability(bool ResultIfUnknown) const {
   case Type::ObjCInterface:
   case Type::Atomic:
   case Type::Pipe:
-  case Type::ExtInt:
-  case Type::DependentExtInt:
+  case Type::BitInt:
+  case Type::DependentBitInt:
     return false;
   }
   llvm_unreachable("bad type kind!");
diff --git a/clang/lib/AST/TypePrinter.cpp b/clang/lib/AST/TypePrinter.cpp
index eca9af3e5f36..2a33a69f288d 100644
--- a/clang/lib/AST/TypePrinter.cpp
+++ b/clang/lib/AST/TypePrinter.cpp
@@ -212,6 +212,7 @@ bool TypePrinter::canPrefixQualifiers(const Type *T,
     case Type::Builtin:
     case Type::Complex:
     case Type::UnresolvedUsing:
+    case Type::Using:
     case Type::Typedef:
     case Type::TypeOfExpr:
     case Type::TypeOf:
@@ -232,8 +233,8 @@ bool TypePrinter::canPrefixQualifiers(const Type *T,
     case Type::ObjCInterface:
     case Type::Atomic:
     case Type::Pipe:
-    case Type::ExtInt:
-    case Type::DependentExtInt:
+    case Type::BitInt:
+    case Type::DependentBitInt:
       CanPrefixQualifiers = true;
       break;
 
@@ -1046,6 +1047,21 @@ void TypePrinter::printUnresolvedUsingBefore(const UnresolvedUsingType *T,
 void TypePrinter::printUnresolvedUsingAfter(const UnresolvedUsingType *T,
                                             raw_ostream &OS) {}
 
+void TypePrinter::printUsingBefore(const UsingType *T, raw_ostream &OS) {
+  // After `namespace b { using a::X }`, is the type X within B a::X or b::X?
+  //
+  // - b::X is more formally correct given the UsingType model
+  // - b::X makes sense if "re-exporting" a symbol in a new namespace
+  // - a::X makes sense if "importing" a symbol for convenience
+  //
+  // The "importing" use seems much more common, so we print a::X.
+  // This could be a policy option, but the right choice seems to rest more
+  // with the intent of the code than the caller.
+  printTypeSpec(T->getFoundDecl()->getUnderlyingDecl(), OS);
+}
+
+void TypePrinter::printUsingAfter(const UsingType *T, raw_ostream &OS) {}
+
 void TypePrinter::printTypedefBefore(const TypedefType *T, raw_ostream &OS) {
   printTypeSpec(T->getDecl(), OS);
 }
@@ -1200,26 +1216,26 @@ void TypePrinter::printPipeBefore(const PipeType *T, raw_ostream &OS) {
 
 void TypePrinter::printPipeAfter(const PipeType *T, raw_ostream &OS) {}
 
-void TypePrinter::printExtIntBefore(const ExtIntType *T, raw_ostream &OS) {
+void TypePrinter::printBitIntBefore(const BitIntType *T, raw_ostream &OS) {
   if (T->isUnsigned())
     OS << "unsigned ";
-  OS << "_ExtInt(" << T->getNumBits() << ")";
+  OS << "_BitInt(" << T->getNumBits() << ")";
   spaceBeforePlaceHolder(OS);
 }
 
-void TypePrinter::printExtIntAfter(const ExtIntType *T, raw_ostream &OS) {}
+void TypePrinter::printBitIntAfter(const BitIntType *T, raw_ostream &OS) {}
 
-void TypePrinter::printDependentExtIntBefore(const DependentExtIntType *T,
+void TypePrinter::printDependentBitIntBefore(const DependentBitIntType *T,
                                              raw_ostream &OS) {
   if (T->isUnsigned())
     OS << "unsigned ";
-  OS << "_ExtInt(";
+  OS << "_BitInt(";
   T->getNumBitsExpr()->printPretty(OS, nullptr, Policy);
   OS << ")";
   spaceBeforePlaceHolder(OS);
 }
 
-void TypePrinter::printDependentExtIntAfter(const DependentExtIntType *T,
+void TypePrinter::printDependentBitIntAfter(const DependentBitIntType *T,
                                             raw_ostream &OS) {}
 
 /// Appends the given scope to the end of a string.
diff --git a/clang/lib/ASTMatchers/ASTMatchersInternal.cpp b/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
index 7680eb38283e..815058f32de4 100644
--- a/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
+++ b/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
@@ -1059,6 +1059,7 @@ const AstTypeMatcher<UnaryTransformType> unaryTransformType;
 const AstTypeMatcher<RecordType> recordType;
 const AstTypeMatcher<TagType> tagType;
 const AstTypeMatcher<ElaboratedType> elaboratedType;
+const AstTypeMatcher<UsingType> usingType;
 const AstTypeMatcher<SubstTemplateTypeParmType> substTemplateTypeParmType;
 const AstTypeMatcher<TemplateTypeParmType> templateTypeParmType;
 const AstTypeMatcher<InjectedClassNameType> injectedClassNameType;
diff --git a/clang/lib/ASTMatchers/Dynamic/Diagnostics.cpp b/clang/lib/ASTMatchers/Dynamic/Diagnostics.cpp
index ba2f49e6b623..41ab0ed70fda 100644
--- a/clang/lib/ASTMatchers/Dynamic/Diagnostics.cpp
+++ b/clang/lib/ASTMatchers/Dynamic/Diagnostics.cpp
@@ -204,7 +204,7 @@ std::string Diagnostics::toString() const {
   std::string S;
   llvm::raw_string_ostream OS(S);
   printToStream(OS);
-  return OS.str();
+  return S;
 }
 
 void Diagnostics::printToStreamFull(llvm::raw_ostream &OS) const {
@@ -223,7 +223,7 @@ std::string Diagnostics::toStringFull() const {
   std::string S;
   llvm::raw_string_ostream OS(S);
   printToStreamFull(OS);
-  return OS.str();
+  return S;
 }
 
 }  // namespace dynamic
diff --git a/clang/lib/ASTMatchers/Dynamic/Marshallers.h b/clang/lib/ASTMatchers/Dynamic/Marshallers.h
index 783fb203c408..fa9d42247e24 100644
--- a/clang/lib/ASTMatchers/Dynamic/Marshallers.h
+++ b/clang/lib/ASTMatchers/Dynamic/Marshallers.h
@@ -1035,7 +1035,6 @@ public:
   void getArgKinds(ASTNodeKind ThisKind, unsigned,
                    std::vector<ArgKind> &ArgKinds) const override {
     ArgKinds.push_back(ArgKind::MakeNodeArg(ThisKind));
-    return;
   }
   bool isConvertibleTo(ASTNodeKind Kind, unsigned *Specificity = nullptr,
                        ASTNodeKind *LeastDerivedKind = nullptr) const override {
diff --git a/clang/lib/ASTMatchers/Dynamic/Parser.cpp b/clang/lib/ASTMatchers/Dynamic/Parser.cpp
index c6a77bb6c2e0..cab1476acf94 100644
--- a/clang/lib/ASTMatchers/Dynamic/Parser.cpp
+++ b/clang/lib/ASTMatchers/Dynamic/Parser.cpp
@@ -645,7 +645,7 @@ bool Parser::parseMatcherExpressionImpl(const TokenInfo &NameToken,
   Tokenizer->SkipNewlines();
 
   {
-    ScopedContextEntry SCE(this, Ctor ? *Ctor : nullptr);
+    ScopedContextEntry SCE(this, Ctor.getValueOr(nullptr));
 
     while (Tokenizer->nextTokenKind() != TokenInfo::TK_Eof) {
       if (Tokenizer->nextTokenKind() == TokenInfo::TK_CloseParen) {
diff --git a/clang/lib/ASTMatchers/Dynamic/Registry.cpp b/clang/lib/ASTMatchers/Dynamic/Registry.cpp
index 878547923d27..4f3efdb0a663 100644
--- a/clang/lib/ASTMatchers/Dynamic/Registry.cpp
+++ b/clang/lib/ASTMatchers/Dynamic/Registry.cpp
@@ -228,6 +228,7 @@ RegistryMaps::RegistryMaps() {
   REGISTER_MATCHER(eachOf);
   REGISTER_MATCHER(elaboratedType);
   REGISTER_MATCHER(elaboratedTypeLoc);
+  REGISTER_MATCHER(usingType);
   REGISTER_MATCHER(enumConstantDecl);
   REGISTER_MATCHER(enumDecl);
   REGISTER_MATCHER(enumType);
diff --git a/clang/lib/Analysis/AnalysisDeclContext.cpp b/clang/lib/Analysis/AnalysisDeclContext.cpp
index d8466ac34a3d..06f1f813aeed 100644
--- a/clang/lib/Analysis/AnalysisDeclContext.cpp
+++ b/clang/lib/Analysis/AnalysisDeclContext.cpp
@@ -387,7 +387,7 @@ std::string AnalysisDeclContext::getFunctionName(const Decl *D) {
     OS << ' ' << OMD->getSelector().getAsString() << ']';
   }
 
-  return OS.str();
+  return Str;
 }
 
 LocationContextManager &AnalysisDeclContext::getLocationContextManager() {
diff --git a/clang/lib/Analysis/CFG.cpp b/clang/lib/Analysis/CFG.cpp
index abf65e3efce9..9ef3b5b6277a 100644
--- a/clang/lib/Analysis/CFG.cpp
+++ b/clang/lib/Analysis/CFG.cpp
@@ -1820,8 +1820,6 @@ void CFGBuilder::addScopesEnd(LocalScope::const_iterator B,
 
   for (VarDecl *VD : llvm::reverse(DeclsWithEndedScope))
     appendScopeEnd(Block, VD, S);
-
-  return;
 }
 
 /// addAutomaticObjDtors - Add to current block automatic objects destructors
diff --git a/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp b/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp
index bb7eb9971068..413e8d14bf0a 100644
--- a/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp
+++ b/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp
@@ -11,15 +11,82 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include <utility>
 #include <vector>
 
+#include "clang/Analysis/Analyses/PostOrderCFGView.h"
 #include "clang/Analysis/CFG.h"
 #include "clang/Analysis/FlowSensitive/DataflowEnvironment.h"
+#include "clang/Analysis/FlowSensitive/DataflowWorklist.h"
 #include "clang/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.h"
+#include "llvm/ADT/None.h"
 #include "llvm/ADT/Optional.h"
+#include "llvm/Support/raw_ostream.h"
 
-using namespace clang;
-using namespace dataflow;
+namespace clang {
+namespace dataflow {
+
+/// Computes the input state for a given basic block by joining the output
+/// states of its predecessors.
+///
+/// Requirements:
+///
+///   All predecessors of `Block` except those with loop back edges must have
+///   already been transferred. States in `BlockStates` that are set to
+///   `llvm::None` represent basic blocks that are not evaluated yet.
+static TypeErasedDataflowAnalysisState computeBlockInputState(
+    std::vector<llvm::Optional<TypeErasedDataflowAnalysisState>> &BlockStates,
+    const CFGBlock &Block, const Environment &InitEnv,
+    TypeErasedDataflowAnalysis &Analysis) {
+  // FIXME: Consider passing `Block` to `Analysis.typeErasedInitialElement()`
+  // to enable building analyses like computation of dominators that initialize
+  // the state of each basic block differently.
+  TypeErasedDataflowAnalysisState State = {Analysis.typeErasedInitialElement(),
+                                           InitEnv};
+  for (const CFGBlock *Pred : Block.preds()) {
+    // Skip if the `Block` is unreachable or control flow cannot get past it.
+    if (!Pred || Pred->hasNoReturnElement())
+      continue;
+
+    // Skip if `Pred` was not evaluated yet. This could happen if `Pred` has a
+    // loop back edge to `Block`.
+    const llvm::Optional<TypeErasedDataflowAnalysisState> &MaybePredState =
+        BlockStates[Pred->getBlockID()];
+    if (!MaybePredState.hasValue())
+      continue;
+
+    const TypeErasedDataflowAnalysisState &PredState =
+        MaybePredState.getValue();
+    Analysis.joinTypeErased(State.Lattice, PredState.Lattice);
+    State.Env.join(PredState.Env);
+  }
+  return State;
+}
+
+TypeErasedDataflowAnalysisState transferBlock(
+    std::vector<llvm::Optional<TypeErasedDataflowAnalysisState>> &BlockStates,
+    const CFGBlock &Block, const Environment &InitEnv,
+    TypeErasedDataflowAnalysis &Analysis,
+    std::function<void(const CFGStmt &,
+                       const TypeErasedDataflowAnalysisState &)>
+        HandleTransferredStmt) {
+  TypeErasedDataflowAnalysisState State =
+      computeBlockInputState(BlockStates, Block, InitEnv, Analysis);
+  for (const CFGElement &Element : Block) {
+    // FIXME: Evaluate other kinds of `CFGElement`.
+    const llvm::Optional<CFGStmt> Stmt = Element.getAs<CFGStmt>();
+    if (!Stmt.hasValue())
+      continue;
+
+    // FIXME: Evaluate the statement contained in `Stmt`.
+
+    State.Lattice = Analysis.transferTypeErased(Stmt.getValue().getStmt(),
+                                                State.Lattice, State.Env);
+    if (HandleTransferredStmt != nullptr)
+      HandleTransferredStmt(Stmt.getValue(), State);
+  }
+  return State;
+}
 
 std::vector<llvm::Optional<TypeErasedDataflowAnalysisState>>
 runTypeErasedDataflowAnalysis(const CFG &Cfg,
@@ -29,7 +96,59 @@ runTypeErasedDataflowAnalysis(const CFG &Cfg,
   // are specified in the header. This could be done by remembering
   // what options were used to build `Cfg` and asserting on them here.
 
-  // FIXME: Implement work list-based algorithm to compute the fixed
-  // point of `Analysis::transform` for every basic block in `Cfg`.
-  return {};
+  PostOrderCFGView POV(&Cfg);
+  ForwardDataflowWorklist Worklist(Cfg, &POV);
+
+  std::vector<llvm::Optional<TypeErasedDataflowAnalysisState>> BlockStates;
+  BlockStates.resize(Cfg.size(), llvm::None);
+
+  // The entry basic block doesn't contain statements so it can be skipped.
+  const CFGBlock &Entry = Cfg.getEntry();
+  BlockStates[Entry.getBlockID()] = {Analysis.typeErasedInitialElement(),
+                                     InitEnv};
+  Worklist.enqueueSuccessors(&Entry);
+
+  // Bugs in lattices and transfer functions can prevent the analysis from
+  // converging. To limit the damage (infinite loops) that these bugs can cause,
+  // limit the number of iterations.
+  // FIXME: Consider making the maximum number of iterations configurable.
+  // FIXME: Set up statistics (see llvm/ADT/Statistic.h) to count average number
+  // of iterations, number of functions that time out, etc.
+  unsigned Iterations = 0;
+  static constexpr unsigned MaxIterations = 1 << 16;
+  while (const CFGBlock *Block = Worklist.dequeue()) {
+    if (++Iterations > MaxIterations) {
+      llvm::errs() << "Maximum number of iterations reached, giving up.\n";
+      break;
+    }
+
+    const llvm::Optional<TypeErasedDataflowAnalysisState> &OldBlockState =
+        BlockStates[Block->getBlockID()];
+    TypeErasedDataflowAnalysisState NewBlockState =
+        transferBlock(BlockStates, *Block, InitEnv, Analysis);
+
+    if (OldBlockState.hasValue() &&
+        Analysis.isEqualTypeErased(OldBlockState.getValue().Lattice,
+                                   NewBlockState.Lattice) &&
+        OldBlockState->Env == NewBlockState.Env) {
+      // The state of `Block` didn't change after transfer so there's no need to
+      // revisit its successors.
+      continue;
+    }
+
+    BlockStates[Block->getBlockID()] = std::move(NewBlockState);
+
+    // Do not add unreachable successor blocks to `Worklist`.
+    if (Block->hasNoReturnElement())
+      continue;
+
+    Worklist.enqueueSuccessors(Block);
+  }
+  // FIXME: Consider evaluating unreachable basic blocks (those that have a
+  // state set to `llvm::None` at this point) to also analyze dead code.
+
+  return BlockStates;
 }
+
+} // namespace dataflow
+} // namespace clang
diff --git a/clang/lib/Analysis/ThreadSafety.cpp b/clang/lib/Analysis/ThreadSafety.cpp
index b196ffa73cbf..9cc990bd35a3 100644
--- a/clang/lib/Analysis/ThreadSafety.cpp
+++ b/clang/lib/Analysis/ThreadSafety.cpp
@@ -418,7 +418,6 @@ public:
 private:
   Context::Factory ContextFactory;
   std::vector<VarDefinition> VarDefinitions;
-  std::vector<unsigned> CtxIndices;
   std::vector<std::pair<const Stmt *, Context>> SavedContexts;
 
 public:
@@ -731,8 +730,6 @@ void LocalVariableMap::traverseCFG(CFG *CFGraph,
                                    std::vector<CFGBlockInfo> &BlockInfo) {
   PostOrderCFGView::CFGBlockSet VisitedBlocks(CFGraph);
 
-  CtxIndices.resize(CFGraph->getNumBlockIDs());
-
   for (const auto *CurrBlock : *SortedGraph) {
     unsigned CurrBlockID = CurrBlock->getBlockID();
     CFGBlockInfo *CurrBlockInfo = &BlockInfo[CurrBlockID];
diff --git a/clang/lib/Analysis/UninitializedValues.cpp b/clang/lib/Analysis/UninitializedValues.cpp
index 67cd39728c35..a38ae34f4b81 100644
--- a/clang/lib/Analysis/UninitializedValues.cpp
+++ b/clang/lib/Analysis/UninitializedValues.cpp
@@ -591,8 +591,8 @@ public:
 
         if (AtPredExit == MayUninitialized) {
           // If the predecessor's terminator is an "asm goto" that initializes
-          // the variable, then it won't be counted as "initialized" on the
-          // non-fallthrough paths.
+          // the variable, then don't count it as "initialized" on the indirect
+          // paths.
           CFGTerminator term = Pred->getTerminator();
           if (const auto *as = dyn_cast_or_null<GCCAsmStmt>(term.getStmt())) {
             const CFGBlock *fallthrough = *Pred->succ_begin();
@@ -810,13 +810,22 @@ void TransferFunctions::VisitGCCAsmStmt(GCCAsmStmt *as) {
   if (!as->isAsmGoto())
     return;
 
-  for (const Expr *o : as->outputs())
-    if (const VarDecl *VD = findVar(o).getDecl())
+  ASTContext &C = ac.getASTContext();
+  for (const Expr *O : as->outputs()) {
+    const Expr *Ex = stripCasts(C, O);
+
+    // Strip away any unary operators. Invalid l-values are reported by other
+    // semantic analysis passes.
+    while (const auto *UO = dyn_cast<UnaryOperator>(Ex))
+      Ex = stripCasts(C, UO->getSubExpr());
+
+    if (const VarDecl *VD = findVar(Ex).getDecl())
       if (vals[VD] != Initialized)
         // If the variable isn't initialized by the time we get here, then we
         // mark it as potentially uninitialized for those cases where it's used
         // on an indirect path, where it's not guaranteed to be defined.
         vals[VD] = MayUninitialized;
+  }
 }
 
 void TransferFunctions::VisitObjCMessageExpr(ObjCMessageExpr *ME) {
diff --git a/clang/lib/Basic/Cuda.cpp b/clang/lib/Basic/Cuda.cpp
index e82a3a705e70..2d75578b3de0 100644
--- a/clang/lib/Basic/Cuda.cpp
+++ b/clang/lib/Basic/Cuda.cpp
@@ -123,6 +123,7 @@ static const CudaArchToStringMap arch_names[] = {
     GFX(1033), // gfx1033
     GFX(1034), // gfx1034
     GFX(1035), // gfx1035
+    {CudaArch::Generic, "generic", ""},
     // clang-format on
 };
 #undef SM
diff --git a/clang/lib/Basic/OpenMPKinds.cpp b/clang/lib/Basic/OpenMPKinds.cpp
index 9e74e05bd863..1761c6d3d89b 100644
--- a/clang/lib/Basic/OpenMPKinds.cpp
+++ b/clang/lib/Basic/OpenMPKinds.cpp
@@ -163,6 +163,7 @@ unsigned clang::getOpenMPSimpleClauseType(OpenMPClauseKind Kind, StringRef Str,
   case OMPC_read:
   case OMPC_write:
   case OMPC_capture:
+  case OMPC_compare:
   case OMPC_seq_cst:
   case OMPC_acq_rel:
   case OMPC_acquire:
@@ -428,6 +429,7 @@ const char *clang::getOpenMPSimpleClauseTypeName(OpenMPClauseKind Kind,
   case OMPC_read:
   case OMPC_write:
   case OMPC_capture:
+  case OMPC_compare:
   case OMPC_seq_cst:
   case OMPC_acq_rel:
   case OMPC_acquire:
diff --git a/clang/lib/Basic/SourceLocation.cpp b/clang/lib/Basic/SourceLocation.cpp
index 6986fcd322f2..6e5e55fb09ce 100644
--- a/clang/lib/Basic/SourceLocation.cpp
+++ b/clang/lib/Basic/SourceLocation.cpp
@@ -90,7 +90,7 @@ SourceLocation::printToString(const SourceManager &SM) const {
   std::string S;
   llvm::raw_string_ostream OS(S);
   print(OS, SM);
-  return OS.str();
+  return S;
 }
 
 LLVM_DUMP_METHOD void SourceLocation::dump(const SourceManager &SM) const {
@@ -149,7 +149,7 @@ SourceRange::printToString(const SourceManager &SM) const {
   std::string S;
   llvm::raw_string_ostream OS(S);
   print(OS, SM);
-  return OS.str();
+  return S;
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/clang/lib/Basic/Targets/AArch64.cpp b/clang/lib/Basic/Targets/AArch64.cpp
index 4d403ae1809d..4089a393b762 100644
--- a/clang/lib/Basic/Targets/AArch64.cpp
+++ b/clang/lib/Basic/Targets/AArch64.cpp
@@ -307,6 +307,9 @@ void AArch64TargetInfo::getTargetDefines(const LangOptions &Opts,
   if (FPU & SveMode)
     Builder.defineMacro("__ARM_FEATURE_SVE", "1");
 
+  if ((FPU & NeonMode) && (FPU & SveMode))
+    Builder.defineMacro("__ARM_NEON_SVE_BRIDGE", "1");
+
   if (HasSVE2)
     Builder.defineMacro("__ARM_FEATURE_SVE2", "1");
 
@@ -474,10 +477,12 @@ ArrayRef<Builtin::Info> AArch64TargetInfo::getTargetBuiltins() const {
 Optional<std::pair<unsigned, unsigned>>
 AArch64TargetInfo::getVScaleRange(const LangOptions &LangOpts) const {
   if (LangOpts.VScaleMin || LangOpts.VScaleMax)
-    return std::pair<unsigned, unsigned>(LangOpts.VScaleMin,
-                                         LangOpts.VScaleMax);
+    return std::pair<unsigned, unsigned>(
+        LangOpts.VScaleMin ? LangOpts.VScaleMin : 1, LangOpts.VScaleMax);
+
   if (hasFeature("sve"))
-    return std::pair<unsigned, unsigned>(0, 16);
+    return std::pair<unsigned, unsigned>(1, 16);
+
   return None;
 }
 
diff --git a/clang/lib/Basic/Targets/AArch64.h b/clang/lib/Basic/Targets/AArch64.h
index dea59a9b015d..74745df3be8d 100644
--- a/clang/lib/Basic/Targets/AArch64.h
+++ b/clang/lib/Basic/Targets/AArch64.h
@@ -150,7 +150,7 @@ public:
   const char *getBFloat16Mangling() const override { return "u6__bf16"; };
   bool hasInt128Type() const override;
 
-  bool hasExtIntType() const override { return true; }
+  bool hasBitIntType() const override { return true; }
 };
 
 class LLVM_LIBRARY_VISIBILITY AArch64leTargetInfo : public AArch64TargetInfo {
diff --git a/clang/lib/Basic/Targets/AMDGPU.h b/clang/lib/Basic/Targets/AMDGPU.h
index 8b9d7ce79c16..974922191488 100644
--- a/clang/lib/Basic/Targets/AMDGPU.h
+++ b/clang/lib/Basic/Targets/AMDGPU.h
@@ -426,7 +426,7 @@ public:
 
   void setAuxTarget(const TargetInfo *Aux) override;
 
-  bool hasExtIntType() const override { return true; }
+  bool hasBitIntType() const override { return true; }
 
   // Record offload arch features since they are needed for defining the
   // pre-defined macros.
diff --git a/clang/lib/Basic/Targets/ARC.h b/clang/lib/Basic/Targets/ARC.h
index 3c0c5f6df2f4..5411cd2cd869 100644
--- a/clang/lib/Basic/Targets/ARC.h
+++ b/clang/lib/Basic/Targets/ARC.h
@@ -66,7 +66,7 @@ public:
     return false;
   }
 
-  bool hasExtIntType() const override { return true; }
+  bool hasBitIntType() const override { return true; }
 
   bool isCLZForZeroUndef() const override { return false; }
 };
diff --git a/clang/lib/Basic/Targets/ARM.cpp b/clang/lib/Basic/Targets/ARM.cpp
index f330780300f2..c619d6cde41d 100644
--- a/clang/lib/Basic/Targets/ARM.cpp
+++ b/clang/lib/Basic/Targets/ARM.cpp
@@ -465,6 +465,8 @@ bool ARMTargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
   HWDiv = 0;
   DotProd = 0;
   HasMatMul = 0;
+  HasPAC = 0;
+  HasBTI = 0;
   HasFloat16 = true;
   ARMCDECoprocMask = 0;
   HasBFloat16 = false;
@@ -547,6 +549,9 @@ bool ARMTargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
       HasBFloat16 = true;
     } else if (Feature == "-fpregs") {
       FPRegsDisabled = true;
+    } else if (Feature == "+pacbti") {
+      HasPAC = 1;
+      HasBTI = 1;
     }
   }
 
@@ -890,6 +895,12 @@ void ARMTargetInfo::getTargetDefines(const LangOptions &Opts,
   if (HasMatMul)
     Builder.defineMacro("__ARM_FEATURE_MATMUL_INT8", "1");
 
+  if (HasPAC)
+    Builder.defineMacro("__ARM_FEATURE_PAUTH", "1");
+
+  if (HasBTI)
+    Builder.defineMacro("__ARM_FEATURE_BTI", "1");
+
   if (HasBFloat16) {
     Builder.defineMacro("__ARM_FEATURE_BF16", "1");
     Builder.defineMacro("__ARM_FEATURE_BF16_VECTOR_ARITHMETIC", "1");
@@ -900,7 +911,7 @@ void ARMTargetInfo::getTargetDefines(const LangOptions &Opts,
     Builder.defineMacro("__ARM_FEATURE_BTI_DEFAULT", "1");
 
   if (Opts.hasSignReturnAddress()) {
-    unsigned Value = Opts.isSignReturnAddressWithAKey() ? 1 : 2;
+    unsigned Value = 1;
     if (Opts.isSignReturnAddressScopeAll())
       Value |= 1 << 2;
     Builder.defineMacro("__ARM_FEATURE_PAC_DEFAULT", Twine(Value));
diff --git a/clang/lib/Basic/Targets/ARM.h b/clang/lib/Basic/Targets/ARM.h
index 7d0011d134ea..40c658f3f40e 100644
--- a/clang/lib/Basic/Targets/ARM.h
+++ b/clang/lib/Basic/Targets/ARM.h
@@ -79,6 +79,8 @@ class LLVM_LIBRARY_VISIBILITY ARMTargetInfo : public TargetInfo {
   unsigned DotProd : 1;
   unsigned HasMatMul : 1;
   unsigned FPRegsDisabled : 1;
+  unsigned HasPAC : 1;
+  unsigned HasBTI : 1;
 
   enum {
     LDREX_B = (1 << 0), /// byte (8-bit)
@@ -191,8 +193,8 @@ public:
 
   bool hasSjLjLowering() const override;
 
-  bool hasExtIntType() const override { return true; }
-  
+  bool hasBitIntType() const override { return true; }
+
   const char *getBFloat16Mangling() const override { return "u6__bf16"; };
 };
 
diff --git a/clang/lib/Basic/Targets/Hexagon.cpp b/clang/lib/Basic/Targets/Hexagon.cpp
index 9c37dee7e89a..161369242926 100644
--- a/clang/lib/Basic/Targets/Hexagon.cpp
+++ b/clang/lib/Basic/Targets/Hexagon.cpp
@@ -68,6 +68,9 @@ void HexagonTargetInfo::getTargetDefines(const LangOptions &Opts,
   } else if (CPU == "hexagonv68") {
     Builder.defineMacro("__HEXAGON_V68__");
     Builder.defineMacro("__HEXAGON_ARCH__", "68");
+  } else if (CPU == "hexagonv69") {
+    Builder.defineMacro("__HEXAGON_V69__");
+    Builder.defineMacro("__HEXAGON_ARCH__", "69");
   }
 
   if (hasFeature("hvx-length64b")) {
@@ -128,6 +131,10 @@ bool HexagonTargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
     else if (F == "+audio")
       HasAudio = true;
   }
+  if (CPU.compare("hexagonv68") >= 0) {
+    HasLegalHalfType = true;
+    HasFloat16 = true;
+  }
   return true;
 }
 
@@ -214,7 +221,7 @@ static constexpr CPUSuffix Suffixes[] = {
     {{"hexagonv60"}, {"60"}}, {{"hexagonv62"},  {"62"}},
     {{"hexagonv65"}, {"65"}}, {{"hexagonv66"},  {"66"}},
     {{"hexagonv67"}, {"67"}}, {{"hexagonv67t"}, {"67t"}},
-    {{"hexagonv68"}, {"68"}},
+    {{"hexagonv68"}, {"68"}}, {{"hexagonv69"},  {"69"}},
 };
 
 const char *HexagonTargetInfo::getHexagonCPUSuffix(StringRef Name) {
diff --git a/clang/lib/Basic/Targets/Hexagon.h b/clang/lib/Basic/Targets/Hexagon.h
index d6c7da5f1e40..94441998f355 100644
--- a/clang/lib/Basic/Targets/Hexagon.h
+++ b/clang/lib/Basic/Targets/Hexagon.h
@@ -139,7 +139,7 @@ public:
     return CPU.find('t') != std::string::npos;
   }
 
-  bool hasExtIntType() const override { return true; }
+  bool hasBitIntType() const override { return true; }
 };
 } // namespace targets
 } // namespace clang
diff --git a/clang/lib/Basic/Targets/Lanai.h b/clang/lib/Basic/Targets/Lanai.h
index 9af5427b81c4..56c6cced938a 100644
--- a/clang/lib/Basic/Targets/Lanai.h
+++ b/clang/lib/Basic/Targets/Lanai.h
@@ -87,7 +87,7 @@ public:
 
   const char *getClobbers() const override { return ""; }
 
-  bool hasExtIntType() const override { return true; }
+  bool hasBitIntType() const override { return true; }
 };
 } // namespace targets
 } // namespace clang
diff --git a/clang/lib/Basic/Targets/Mips.h b/clang/lib/Basic/Targets/Mips.h
index b475c03889a1..b54d36e1c95f 100644
--- a/clang/lib/Basic/Targets/Mips.h
+++ b/clang/lib/Basic/Targets/Mips.h
@@ -406,7 +406,7 @@ public:
   unsigned getUnwindWordWidth() const override;
 
   bool validateTarget(DiagnosticsEngine &Diags) const override;
-  bool hasExtIntType() const override { return true; }
+  bool hasBitIntType() const override { return true; }
 };
 } // namespace targets
 } // namespace clang
diff --git a/clang/lib/Basic/Targets/NVPTX.cpp b/clang/lib/Basic/Targets/NVPTX.cpp
index 3561b22677bc..75e82d819900 100644
--- a/clang/lib/Basic/Targets/NVPTX.cpp
+++ b/clang/lib/Basic/Targets/NVPTX.cpp
@@ -215,6 +215,7 @@ void NVPTXTargetInfo::getTargetDefines(const LangOptions &Opts,
       case CudaArch::GFX1033:
       case CudaArch::GFX1034:
       case CudaArch::GFX1035:
+      case CudaArch::Generic:
       case CudaArch::LAST:
         break;
       case CudaArch::UNUSED:
diff --git a/clang/lib/Basic/Targets/NVPTX.h b/clang/lib/Basic/Targets/NVPTX.h
index ef751b8e1a8d..589f24f4bb03 100644
--- a/clang/lib/Basic/Targets/NVPTX.h
+++ b/clang/lib/Basic/Targets/NVPTX.h
@@ -121,7 +121,7 @@ public:
 
   void fillValidCPUList(SmallVectorImpl<StringRef> &Values) const override {
     for (int i = static_cast<int>(CudaArch::SM_20);
-         i < static_cast<int>(CudaArch::LAST); ++i)
+         i < static_cast<int>(CudaArch::Generic); ++i)
       Values.emplace_back(CudaArchToString(static_cast<CudaArch>(i)));
   }
 
@@ -175,7 +175,7 @@ public:
     return CCCR_Warning;
   }
 
-  bool hasExtIntType() const override { return true; }
+  bool hasBitIntType() const override { return true; }
 };
 } // namespace targets
 } // namespace clang
diff --git a/clang/lib/Basic/Targets/OSTargets.cpp b/clang/lib/Basic/Targets/OSTargets.cpp
index 53748bf067cd..f8f12daaa072 100644
--- a/clang/lib/Basic/Targets/OSTargets.cpp
+++ b/clang/lib/Basic/Targets/OSTargets.cpp
@@ -48,12 +48,12 @@ void getDarwinDefines(MacroBuilder &Builder, const LangOptions &Opts,
     Builder.defineMacro("_REENTRANT");
 
   // Get the platform type and version number from the triple.
-  unsigned Maj, Min, Rev;
+  VersionTuple OsVersion;
   if (Triple.isMacOSX()) {
-    Triple.getMacOSXVersion(Maj, Min, Rev);
+    Triple.getMacOSXVersion(OsVersion);
     PlatformName = "macos";
   } else {
-    Triple.getOSVersion(Maj, Min, Rev);
+    OsVersion = Triple.getOSVersion();
     PlatformName = llvm::Triple::getOSTypeName(Triple.getOS());
     if (PlatformName == "ios" && Triple.isMacCatalystEnvironment())
       PlatformName = "maccatalyst";
@@ -63,29 +63,29 @@ void getDarwinDefines(MacroBuilder &Builder, const LangOptions &Opts,
   // generating code for Win32 ABI. No need to emit
   // __ENVIRONMENT_XX_OS_VERSION_MIN_REQUIRED__.
   if (PlatformName == "win32") {
-    PlatformMinVersion = VersionTuple(Maj, Min, Rev);
+    PlatformMinVersion = OsVersion;
     return;
   }
 
   // Set the appropriate OS version define.
   if (Triple.isiOS()) {
-    assert(Maj < 100 && Min < 100 && Rev < 100 && "Invalid version!");
+    assert(OsVersion < VersionTuple(100) && "Invalid version!");
     char Str[7];
-    if (Maj < 10) {
-      Str[0] = '0' + Maj;
-      Str[1] = '0' + (Min / 10);
-      Str[2] = '0' + (Min % 10);
-      Str[3] = '0' + (Rev / 10);
-      Str[4] = '0' + (Rev % 10);
+    if (OsVersion.getMajor() < 10) {
+      Str[0] = '0' + OsVersion.getMajor();
+      Str[1] = '0' + (OsVersion.getMinor().getValueOr(0) / 10);
+      Str[2] = '0' + (OsVersion.getMinor().getValueOr(0) % 10);
+      Str[3] = '0' + (OsVersion.getSubminor().getValueOr(0) / 10);
+      Str[4] = '0' + (OsVersion.getSubminor().getValueOr(0) % 10);
       Str[5] = '\0';
     } else {
       // Handle versions >= 10.
-      Str[0] = '0' + (Maj / 10);
-      Str[1] = '0' + (Maj % 10);
-      Str[2] = '0' + (Min / 10);
-      Str[3] = '0' + (Min % 10);
-      Str[4] = '0' + (Rev / 10);
-      Str[5] = '0' + (Rev % 10);
+      Str[0] = '0' + (OsVersion.getMajor() / 10);
+      Str[1] = '0' + (OsVersion.getMajor() % 10);
+      Str[2] = '0' + (OsVersion.getMinor().getValueOr(0) / 10);
+      Str[3] = '0' + (OsVersion.getMinor().getValueOr(0) % 10);
+      Str[4] = '0' + (OsVersion.getSubminor().getValueOr(0) / 10);
+      Str[5] = '0' + (OsVersion.getSubminor().getValueOr(0) % 10);
       Str[6] = '\0';
     }
     if (Triple.isTvOS())
@@ -95,13 +95,13 @@ void getDarwinDefines(MacroBuilder &Builder, const LangOptions &Opts,
                           Str);
 
   } else if (Triple.isWatchOS()) {
-    assert(Maj < 10 && Min < 100 && Rev < 100 && "Invalid version!");
+    assert(OsVersion < VersionTuple(10) && "Invalid version!");
     char Str[6];
-    Str[0] = '0' + Maj;
-    Str[1] = '0' + (Min / 10);
-    Str[2] = '0' + (Min % 10);
-    Str[3] = '0' + (Rev / 10);
-    Str[4] = '0' + (Rev % 10);
+    Str[0] = '0' + OsVersion.getMajor();
+    Str[1] = '0' + (OsVersion.getMinor().getValueOr(0) / 10);
+    Str[2] = '0' + (OsVersion.getMinor().getValueOr(0) % 10);
+    Str[3] = '0' + (OsVersion.getSubminor().getValueOr(0) / 10);
+    Str[4] = '0' + (OsVersion.getSubminor().getValueOr(0) % 10);
     Str[5] = '\0';
     Builder.defineMacro("__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__", Str);
   } else if (Triple.isMacOSX()) {
@@ -109,22 +109,22 @@ void getDarwinDefines(MacroBuilder &Builder, const LangOptions &Opts,
     // define (because we only get a single digit for the minor and micro
     // revision numbers). So, we limit them to the maximum representable
     // version.
-    assert(Maj < 100 && Min < 100 && Rev < 100 && "Invalid version!");
+    assert(OsVersion < VersionTuple(100) && "Invalid version!");
     char Str[7];
-    if (Maj < 10 || (Maj == 10 && Min < 10)) {
-      Str[0] = '0' + (Maj / 10);
-      Str[1] = '0' + (Maj % 10);
-      Str[2] = '0' + std::min(Min, 9U);
-      Str[3] = '0' + std::min(Rev, 9U);
+    if (OsVersion < VersionTuple(10, 10)) {
+      Str[0] = '0' + (OsVersion.getMajor() / 10);
+      Str[1] = '0' + (OsVersion.getMajor() % 10);
+      Str[2] = '0' + std::min(OsVersion.getMinor().getValueOr(0), 9U);
+      Str[3] = '0' + std::min(OsVersion.getSubminor().getValueOr(0), 9U);
       Str[4] = '\0';
     } else {
       // Handle versions > 10.9.
-      Str[0] = '0' + (Maj / 10);
-      Str[1] = '0' + (Maj % 10);
-      Str[2] = '0' + (Min / 10);
-      Str[3] = '0' + (Min % 10);
-      Str[4] = '0' + (Rev / 10);
-      Str[5] = '0' + (Rev % 10);
+      Str[0] = '0' + (OsVersion.getMajor() / 10);
+      Str[1] = '0' + (OsVersion.getMajor() % 10);
+      Str[2] = '0' + (OsVersion.getMinor().getValueOr(0) / 10);
+      Str[3] = '0' + (OsVersion.getMinor().getValueOr(0) % 10);
+      Str[4] = '0' + (OsVersion.getSubminor().getValueOr(0) / 10);
+      Str[5] = '0' + (OsVersion.getSubminor().getValueOr(0) % 10);
       Str[6] = '\0';
     }
     Builder.defineMacro("__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__", Str);
@@ -134,7 +134,7 @@ void getDarwinDefines(MacroBuilder &Builder, const LangOptions &Opts,
   if (Triple.isOSDarwin())
     Builder.defineMacro("__MACH__");
 
-  PlatformMinVersion = VersionTuple(Maj, Min, Rev);
+  PlatformMinVersion = OsVersion;
 }
 
 static void addMinGWDefines(const llvm::Triple &Triple, const LangOptions &Opts,
@@ -203,6 +203,7 @@ static void addVisualCDefines(const LangOptions &Opts, MacroBuilder &Builder) {
   }
 
   Builder.defineMacro("_INTEGRAL_MAX_BITS", "64");
+  Builder.defineMacro("__STDC_NO_THREADS__");
 
   // Starting with VS 2022 17.1, MSVC predefines the below macro to inform
   // users of the execution character set defined at compile time.
diff --git a/clang/lib/Basic/Targets/OSTargets.h b/clang/lib/Basic/Targets/OSTargets.h
index 7fbe2cbc5653..3c1830d5f8e8 100644
--- a/clang/lib/Basic/Targets/OSTargets.h
+++ b/clang/lib/Basic/Targets/OSTargets.h
@@ -148,9 +148,7 @@ public:
       return 64;
     }
 
-    unsigned Major, Minor, Micro;
-    T.getOSVersion(Major, Minor, Micro);
-    if (llvm::VersionTuple(Major, Minor, Micro) < MinVersion)
+    if (T.getOSVersion() < MinVersion)
       return 64;
     return OSTargetInfo<Target>::getExnObjectAlignment();
   }
@@ -294,7 +292,7 @@ protected:
     Builder.defineMacro("__HAIKU__");
     Builder.defineMacro("__ELF__");
     DefineStd(Builder, "unix", Opts);
-    if (this->HasFloat128) 
+    if (this->HasFloat128)
       Builder.defineMacro("__FLOAT128__");
   }
 
@@ -376,10 +374,9 @@ protected:
     Builder.defineMacro("__ELF__");
     if (Triple.isAndroid()) {
       Builder.defineMacro("__ANDROID__", "1");
-      unsigned Maj, Min, Rev;
-      Triple.getEnvironmentVersion(Maj, Min, Rev);
       this->PlatformName = "android";
-      this->PlatformMinVersion = VersionTuple(Maj, Min, Rev);
+      this->PlatformMinVersion = Triple.getEnvironmentVersion();
+      const unsigned Maj = this->PlatformMinVersion.getMajor();
       if (Maj) {
         Builder.defineMacro("__ANDROID_MIN_SDK_VERSION__", Twine(Maj));
         // This historical but ambiguous name for the minSdkVersion macro. Keep
@@ -693,23 +690,32 @@ protected:
     if (Opts.EnableAIXExtendedAltivecABI)
       Builder.defineMacro("__EXTABI__");
 
-    unsigned Major, Minor, Micro;
-    Triple.getOSVersion(Major, Minor, Micro);
+    VersionTuple OsVersion = Triple.getOSVersion();
 
     // Define AIX OS-Version Macros.
     // Includes logic for legacy versions of AIX; no specific intent to support.
-    std::pair<int, int> OsVersion = {Major, Minor};
-    if (OsVersion >= std::make_pair(3, 2)) Builder.defineMacro("_AIX32");
-    if (OsVersion >= std::make_pair(4, 1)) Builder.defineMacro("_AIX41");
-    if (OsVersion >= std::make_pair(4, 3)) Builder.defineMacro("_AIX43");
-    if (OsVersion >= std::make_pair(5, 0)) Builder.defineMacro("_AIX50");
-    if (OsVersion >= std::make_pair(5, 1)) Builder.defineMacro("_AIX51");
-    if (OsVersion >= std::make_pair(5, 2)) Builder.defineMacro("_AIX52");
-    if (OsVersion >= std::make_pair(5, 3)) Builder.defineMacro("_AIX53");
-    if (OsVersion >= std::make_pair(6, 1)) Builder.defineMacro("_AIX61");
-    if (OsVersion >= std::make_pair(7, 1)) Builder.defineMacro("_AIX71");
-    if (OsVersion >= std::make_pair(7, 2)) Builder.defineMacro("_AIX72");
-    if (OsVersion >= std::make_pair(7, 3)) Builder.defineMacro("_AIX73");
+    if (OsVersion >= VersionTuple(3, 2))
+      Builder.defineMacro("_AIX32");
+    if (OsVersion >= VersionTuple(4, 1))
+      Builder.defineMacro("_AIX41");
+    if (OsVersion >= VersionTuple(4, 3))
+      Builder.defineMacro("_AIX43");
+    if (OsVersion >= VersionTuple(5, 0))
+      Builder.defineMacro("_AIX50");
+    if (OsVersion >= VersionTuple(5, 1))
+      Builder.defineMacro("_AIX51");
+    if (OsVersion >= VersionTuple(5, 2))
+      Builder.defineMacro("_AIX52");
+    if (OsVersion >= VersionTuple(5, 3))
+      Builder.defineMacro("_AIX53");
+    if (OsVersion >= VersionTuple(6, 1))
+      Builder.defineMacro("_AIX61");
+    if (OsVersion >= VersionTuple(7, 1))
+      Builder.defineMacro("_AIX71");
+    if (OsVersion >= VersionTuple(7, 2))
+      Builder.defineMacro("_AIX72");
+    if (OsVersion >= VersionTuple(7, 3))
+      Builder.defineMacro("_AIX73");
 
     // FIXME: Do not define _LONG_LONG when -fno-long-long is specified.
     Builder.defineMacro("_LONG_LONG");
diff --git a/clang/lib/Basic/Targets/PNaCl.h b/clang/lib/Basic/Targets/PNaCl.h
index d5bfc369583f..b5cf73d73e95 100644
--- a/clang/lib/Basic/Targets/PNaCl.h
+++ b/clang/lib/Basic/Targets/PNaCl.h
@@ -69,7 +69,7 @@ public:
 
   const char *getClobbers() const override { return ""; }
 
-  bool hasExtIntType() const override { return true; }
+  bool hasBitIntType() const override { return true; }
 };
 
 // We attempt to use PNaCl (le32) frontend and Mips32EL backend.
diff --git a/clang/lib/Basic/Targets/PPC.cpp b/clang/lib/Basic/Targets/PPC.cpp
index c3c61ed443ca..7f7b44b658eb 100644
--- a/clang/lib/Basic/Targets/PPC.cpp
+++ b/clang/lib/Basic/Targets/PPC.cpp
@@ -623,14 +623,11 @@ void PPCTargetInfo::addP10SpecificFeatures(
   Features["pcrelative-memops"] = true;
   Features["prefix-instrs"] = true;
   Features["isa-v31-instructions"] = true;
-  return;
 }
 
 // Add features specific to the "Future" CPU.
 void PPCTargetInfo::addFutureSpecificFeatures(
-    llvm::StringMap<bool> &Features) const {
-  return;
-}
+    llvm::StringMap<bool> &Features) const {}
 
 bool PPCTargetInfo::hasFeature(StringRef Feature) const {
   return llvm::StringSwitch<bool>(Feature)
diff --git a/clang/lib/Basic/Targets/PPC.h b/clang/lib/Basic/Targets/PPC.h
index f19d3ebbc066..60701072ac4b 100644
--- a/clang/lib/Basic/Targets/PPC.h
+++ b/clang/lib/Basic/Targets/PPC.h
@@ -351,7 +351,7 @@ public:
   const char *getFloat128Mangling() const override { return "u9__ieee128"; }
   const char *getIbm128Mangling() const override { return "g"; }
 
-  bool hasExtIntType() const override { return true; }
+  bool hasBitIntType() const override { return true; }
 
   bool isSPRegName(StringRef RegName) const override {
     return RegName.equals("r1") || RegName.equals("x1");
diff --git a/clang/lib/Basic/Targets/RISCV.cpp b/clang/lib/Basic/Targets/RISCV.cpp
index 93562dde2f54..770d37a1c1be 100644
--- a/clang/lib/Basic/Targets/RISCV.cpp
+++ b/clang/lib/Basic/Targets/RISCV.cpp
@@ -246,6 +246,9 @@ bool RISCVTargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
     ISAInfo = std::move(*ParseResult);
   }
 
+  if (ABI.empty())
+    ABI = llvm::RISCV::computeDefaultABIFromArch(*ISAInfo).str();
+
   return true;
 }
 
diff --git a/clang/lib/Basic/Targets/RISCV.h b/clang/lib/Basic/Targets/RISCV.h
index f7ffe9febcd0..5331ed4a50ae 100644
--- a/clang/lib/Basic/Targets/RISCV.h
+++ b/clang/lib/Basic/Targets/RISCV.h
@@ -95,7 +95,7 @@ public:
   bool handleTargetFeatures(std::vector<std::string> &Features,
                             DiagnosticsEngine &Diags) override;
 
-  bool hasExtIntType() const override { return true; }
+  bool hasBitIntType() const override { return true; }
 };
 class LLVM_LIBRARY_VISIBILITY RISCV32TargetInfo : public RISCVTargetInfo {
 public:
diff --git a/clang/lib/Basic/Targets/SPIR.h b/clang/lib/Basic/Targets/SPIR.h
index 8cf18b6c20f1..a40d4b3ca27e 100644
--- a/clang/lib/Basic/Targets/SPIR.h
+++ b/clang/lib/Basic/Targets/SPIR.h
@@ -162,7 +162,7 @@ public:
     supportAllOpenCLOpts();
   }
 
-  bool hasExtIntType() const override { return true; }
+  bool hasBitIntType() const override { return true; }
 
   bool hasInt128Type() const override { return false; }
 };
diff --git a/clang/lib/Basic/Targets/Sparc.h b/clang/lib/Basic/Targets/Sparc.h
index 22a1621fcb9f..177a117520da 100644
--- a/clang/lib/Basic/Targets/Sparc.h
+++ b/clang/lib/Basic/Targets/Sparc.h
@@ -48,8 +48,6 @@ public:
 
   bool hasFeature(StringRef Feature) const override;
 
-  bool hasSjLjLowering() const override { return true; }
-
   ArrayRef<Builtin::Info> getTargetBuiltins() const override {
     // FIXME: Implement!
     return None;
@@ -178,8 +176,7 @@ public:
   void getTargetDefines(const LangOptions &Opts,
                         MacroBuilder &Builder) const override;
 
-  bool hasSjLjLowering() const override { return true; }
-  bool hasExtIntType() const override { return true; }
+  bool hasBitIntType() const override { return true; }
 };
 
 // SPARCV8el is the 32-bit little-endian mode selected by Triple::sparcel.
@@ -232,7 +229,7 @@ public:
     return getCPUGeneration(CPU) == CG_V9;
   }
 
-  bool hasExtIntType() const override { return true; }
+  bool hasBitIntType() const override { return true; }
 };
 } // namespace targets
 } // namespace clang
diff --git a/clang/lib/Basic/Targets/SystemZ.h b/clang/lib/Basic/Targets/SystemZ.h
index d3e3ed50dd47..92cefeea5d26 100644
--- a/clang/lib/Basic/Targets/SystemZ.h
+++ b/clang/lib/Basic/Targets/SystemZ.h
@@ -170,7 +170,7 @@ public:
 
   const char *getLongDoubleMangling() const override { return "g"; }
 
-  bool hasExtIntType() const override { return true; }
+  bool hasBitIntType() const override { return true; }
 
   int getEHDataRegisterNumber(unsigned RegNo) const override {
     return RegNo < 4 ? 6 + RegNo : -1;
diff --git a/clang/lib/Basic/Targets/WebAssembly.h b/clang/lib/Basic/Targets/WebAssembly.h
index 16534d3ef99b..075486990558 100644
--- a/clang/lib/Basic/Targets/WebAssembly.h
+++ b/clang/lib/Basic/Targets/WebAssembly.h
@@ -137,7 +137,7 @@ private:
     }
   }
 
-  bool hasExtIntType() const override { return true; }
+  bool hasBitIntType() const override { return true; }
 
   bool hasProtectedVisibility() const override { return false; }
 
diff --git a/clang/lib/Basic/Targets/X86.h b/clang/lib/Basic/Targets/X86.h
index b9b2ac79815b..c952b8c9a336 100644
--- a/clang/lib/Basic/Targets/X86.h
+++ b/clang/lib/Basic/Targets/X86.h
@@ -462,7 +462,7 @@ public:
 
   ArrayRef<Builtin::Info> getTargetBuiltins() const override;
 
-  bool hasExtIntType() const override { return true; }
+  bool hasBitIntType() const override { return true; }
 };
 
 class LLVM_LIBRARY_VISIBILITY NetBSDI386TargetInfo
@@ -472,10 +472,9 @@ public:
       : NetBSDTargetInfo<X86_32TargetInfo>(Triple, Opts) {}
 
   unsigned getFloatEvalMethod() const override {
-    unsigned Major, Minor, Micro;
-    getTriple().getOSVersion(Major, Minor, Micro);
+    VersionTuple OsVersion = getTriple().getOSVersion();
     // New NetBSD uses the default rounding mode.
-    if (Major >= 7 || (Major == 6 && Minor == 99 && Micro >= 26) || Major == 0)
+    if (OsVersion >= VersionTuple(6, 99, 26) || OsVersion.getMajor() == 0)
       return X86_32TargetInfo::getFloatEvalMethod();
     // NetBSD before 6.99.26 defaults to "double" rounding.
     return 1;
@@ -769,7 +768,7 @@ public:
 
   ArrayRef<Builtin::Info> getTargetBuiltins() const override;
 
-  bool hasExtIntType() const override { return true; }
+  bool hasBitIntType() const override { return true; }
 };
 
 // x86-64 Windows target
diff --git a/clang/lib/Basic/Targets/XCore.h b/clang/lib/Basic/Targets/XCore.h
index c33766751aa1..25f20581839d 100644
--- a/clang/lib/Basic/Targets/XCore.h
+++ b/clang/lib/Basic/Targets/XCore.h
@@ -76,7 +76,7 @@ public:
 
   bool allowsLargerPreferedTypeAlignment() const override { return false; }
 
-  bool hasExtIntType() const override { return true; }
+  bool hasBitIntType() const override { return true; }
 };
 } // namespace targets
 } // namespace clang
diff --git a/clang/lib/Basic/Version.cpp b/clang/lib/Basic/Version.cpp
index af3118b0f6da..e205da7adec1 100644
--- a/clang/lib/Basic/Version.cpp
+++ b/clang/lib/Basic/Version.cpp
@@ -82,7 +82,7 @@ std::string getClangFullRepositoryVersion() {
       OS << LLVMRepo << ' ';
     OS << LLVMRev << ')';
   }
-  return OS.str();
+  return buf;
 }
 
 std::string getClangFullVersion() {
@@ -102,7 +102,7 @@ std::string getClangToolFullVersion(StringRef ToolName) {
     OS << " " << repo;
   }
 
-  return OS.str();
+  return buf;
 }
 
 std::string getClangFullCPPVersion() {
@@ -120,7 +120,7 @@ std::string getClangFullCPPVersion() {
     OS << " " << repo;
   }
 
-  return OS.str();
+  return buf;
 }
 
 } // end namespace clang
diff --git a/clang/lib/CodeGen/ABIInfo.h b/clang/lib/CodeGen/ABIInfo.h
index 56f0dd4322d2..0d12183055e1 100644
--- a/clang/lib/CodeGen/ABIInfo.h
+++ b/clang/lib/CodeGen/ABIInfo.h
@@ -105,7 +105,7 @@ namespace swiftcall {
                                 uint64_t &Members) const;
 
     // Implement the Type::IsPromotableIntegerType for ABI specific needs. The
-    // only difference is that this considers _ExtInt as well.
+    // only difference is that this considers bit-precise integer types as well.
     bool isPromotableIntegerTypeForABI(QualType Ty) const;
 
     /// A convenience method to return an indirect ABIArgInfo with an
diff --git a/clang/lib/CodeGen/Address.h b/clang/lib/CodeGen/Address.h
index 6a8e57f8db33..37c20291c0e8 100644
--- a/clang/lib/CodeGen/Address.h
+++ b/clang/lib/CodeGen/Address.h
@@ -23,15 +23,29 @@ namespace CodeGen {
 /// An aligned address.
 class Address {
   llvm::Value *Pointer;
+  llvm::Type *ElementType;
   CharUnits Alignment;
+
+protected:
+  Address(std::nullptr_t) : Pointer(nullptr), ElementType(nullptr) {}
+
 public:
-  Address(llvm::Value *pointer, CharUnits alignment)
-      : Pointer(pointer), Alignment(alignment) {
-    assert((!alignment.isZero() || pointer == nullptr) &&
-           "creating valid address with invalid alignment");
+  Address(llvm::Value *pointer, llvm::Type *elementType, CharUnits alignment)
+      : Pointer(pointer), ElementType(elementType), Alignment(alignment) {
+    assert(pointer != nullptr && "Pointer cannot be null");
+    assert(elementType != nullptr && "Element type cannot be null");
+    assert(llvm::cast<llvm::PointerType>(pointer->getType())
+               ->isOpaqueOrPointeeTypeMatches(elementType) &&
+           "Incorrect pointer element type");
+    assert(!alignment.isZero() && "Alignment cannot be zero");
   }
 
-  static Address invalid() { return Address(nullptr, CharUnits()); }
+  // Deprecated: Use constructor with explicit element type instead.
+  Address(llvm::Value *Pointer, CharUnits Alignment)
+      : Address(Pointer, Pointer->getType()->getPointerElementType(),
+                Alignment) {}
+
+  static Address invalid() { return Address(nullptr); }
   bool isValid() const { return Pointer != nullptr; }
 
   llvm::Value *getPointer() const {
@@ -45,11 +59,9 @@ public:
   }
 
   /// Return the type of the values stored in this address.
-  ///
-  /// When IR pointer types lose their element type, we should simply
-  /// store it in Address instead for the convenience of writing code.
   llvm::Type *getElementType() const {
-    return getType()->getElementType();
+    assert(isValid());
+    return ElementType;
   }
 
   /// Return the address space that this address resides in.
@@ -67,30 +79,42 @@ public:
     assert(isValid());
     return Alignment;
   }
+
+  /// Return address with different pointer, but same element type and
+  /// alignment.
+  Address withPointer(llvm::Value *NewPointer) const {
+    return Address(NewPointer, ElementType, Alignment);
+  }
+
+  /// Return address with different alignment, but same pointer and element
+  /// type.
+  Address withAlignment(CharUnits NewAlignment) const {
+    return Address(Pointer, ElementType, NewAlignment);
+  }
 };
 
 /// A specialization of Address that requires the address to be an
 /// LLVM Constant.
 class ConstantAddress : public Address {
+  ConstantAddress(std::nullptr_t) : Address(nullptr) {}
+
 public:
-  ConstantAddress(llvm::Constant *pointer, CharUnits alignment)
-    : Address(pointer, alignment) {}
+  ConstantAddress(llvm::Constant *pointer, llvm::Type *elementType,
+                  CharUnits alignment)
+      : Address(pointer, elementType, alignment) {}
 
   static ConstantAddress invalid() {
-    return ConstantAddress(nullptr, CharUnits());
+    return ConstantAddress(nullptr);
   }
 
   llvm::Constant *getPointer() const {
     return llvm::cast<llvm::Constant>(Address::getPointer());
   }
 
-  ConstantAddress getBitCast(llvm::Type *ty) const {
-    return ConstantAddress(llvm::ConstantExpr::getBitCast(getPointer(), ty),
-                           getAlignment());
-  }
-
-  ConstantAddress getElementBitCast(llvm::Type *ty) const {
-    return getBitCast(ty->getPointerTo(getAddressSpace()));
+  ConstantAddress getElementBitCast(llvm::Type *ElemTy) const {
+    llvm::Constant *BitCast = llvm::ConstantExpr::getBitCast(
+        getPointer(), ElemTy->getPointerTo(getAddressSpace()));
+    return ConstantAddress(BitCast, ElemTy, getAlignment());
   }
 
   static bool isaImpl(Address addr) {
@@ -98,7 +122,7 @@ public:
   }
   static ConstantAddress castImpl(Address addr) {
     return ConstantAddress(llvm::cast<llvm::Constant>(addr.getPointer()),
-                           addr.getAlignment());
+                           addr.getElementType(), addr.getAlignment());
   }
 };
 
diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp
index 510f3911939c..bacac0a20d4d 100644
--- a/clang/lib/CodeGen/BackendUtil.cpp
+++ b/clang/lib/CodeGen/BackendUtil.cpp
@@ -94,10 +94,16 @@ using namespace llvm;
   llvm::PassPluginLibraryInfo get##Ext##PluginInfo();
 #include "llvm/Support/Extension.def"
 
+namespace llvm {
+extern cl::opt<bool> DebugInfoCorrelate;
+}
+
 namespace {
 
 // Default filename used for profile generation.
-static constexpr StringLiteral DefaultProfileGenName = "default_%m.profraw";
+std::string getDefaultProfileGenName() {
+  return DebugInfoCorrelate ? "default_%p.proflite" : "default_%m.profraw";
+}
 
 class EmitAssemblyHelper {
   DiagnosticsEngine &Diags;
@@ -597,8 +603,6 @@ static bool initTargetOptions(DiagnosticsEngine &Diags,
   Options.ForceDwarfFrameSection = CodeGenOpts.ForceDwarfFrameSection;
   Options.EmitCallSiteInfo = CodeGenOpts.EmitCallSiteInfo;
   Options.EnableAIXExtendedAltivecABI = CodeGenOpts.EnableAIXExtendedAltivecABI;
-  Options.ValueTrackingVariableLocations =
-      CodeGenOpts.ValueTrackingVariableLocations;
   Options.XRayOmitFunctionIndex = CodeGenOpts.XRayOmitFunctionIndex;
   Options.LoopAlignment = CodeGenOpts.LoopAlignment;
 
@@ -640,6 +644,7 @@ static bool initTargetOptions(DiagnosticsEngine &Diags,
   Options.MCOptions.Argv0 = CodeGenOpts.Argv0;
   Options.MCOptions.CommandLineArgs = CodeGenOpts.CommandLineArgs;
   Options.DebugStrictDwarf = CodeGenOpts.DebugStrictDwarf;
+  Options.ObjectFilenameForDebug = CodeGenOpts.ObjectFilenameForDebug;
 
   return true;
 }
@@ -886,7 +891,7 @@ void EmitAssemblyHelper::CreatePasses(legacy::PassManager &MPM,
     if (!CodeGenOpts.InstrProfileOutput.empty())
       PMBuilder.PGOInstrGen = CodeGenOpts.InstrProfileOutput;
     else
-      PMBuilder.PGOInstrGen = std::string(DefaultProfileGenName);
+      PMBuilder.PGOInstrGen = getDefaultProfileGenName();
   }
   if (CodeGenOpts.hasProfileIRUse()) {
     PMBuilder.PGOInstrUse = CodeGenOpts.ProfileInstrumentUsePath;
@@ -1231,7 +1236,7 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
   if (CodeGenOpts.hasProfileIRInstr())
     // -fprofile-generate.
     PGOOpt = PGOOptions(CodeGenOpts.InstrProfileOutput.empty()
-                            ? std::string(DefaultProfileGenName)
+                            ? getDefaultProfileGenName()
                             : CodeGenOpts.InstrProfileOutput,
                         "", "", PGOOptions::IRInstr, PGOOptions::NoCSAction,
                         CodeGenOpts.DebugInfoForProfiling);
@@ -1269,13 +1274,13 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
              "Cannot run CSProfileGen pass with ProfileGen or SampleUse "
              " pass");
       PGOOpt->CSProfileGenFile = CodeGenOpts.InstrProfileOutput.empty()
-                                     ? std::string(DefaultProfileGenName)
+                                     ? getDefaultProfileGenName()
                                      : CodeGenOpts.InstrProfileOutput;
       PGOOpt->CSAction = PGOOptions::CSIRInstr;
     } else
       PGOOpt = PGOOptions("",
                           CodeGenOpts.InstrProfileOutput.empty()
-                              ? std::string(DefaultProfileGenName)
+                              ? getDefaultProfileGenName()
                               : CodeGenOpts.InstrProfileOutput,
                           "", PGOOptions::NoAction, PGOOptions::CSIRInstr,
                           CodeGenOpts.DebugInfoForProfiling);
@@ -1577,7 +1582,8 @@ static void runThinLTOBackend(
     return;
 
   auto AddStream = [&](size_t Task) {
-    return std::make_unique<CachedFileStream>(std::move(OS));
+    return std::make_unique<CachedFileStream>(std::move(OS),
+                                              CGOpts.ObjectFilenameForDebug);
   };
   lto::Config Conf;
   if (CGOpts.SaveTempsFilePrefix != "") {
diff --git a/clang/lib/CodeGen/CGAtomic.cpp b/clang/lib/CodeGen/CGAtomic.cpp
index b68e6328acdf..e81c5ba5055c 100644
--- a/clang/lib/CodeGen/CGAtomic.cpp
+++ b/clang/lib/CodeGen/CGAtomic.cpp
@@ -1079,8 +1079,8 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
       if (AS == LangAS::opencl_generic)
         return V;
       auto DestAS = getContext().getTargetAddressSpace(LangAS::opencl_generic);
-      auto T = V->getType();
-      auto *DestType = T->getPointerElementType()->getPointerTo(DestAS);
+      auto T = llvm::cast<llvm::PointerType>(V->getType());
+      auto *DestType = llvm::PointerType::getWithSamePointeeType(T, DestAS);
 
       return getTargetHooks().performAddrSpaceCast(
           *this, V, AS, LangAS::opencl_generic, DestType, false);
@@ -1321,15 +1321,14 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
         ResVal = Builder.CreateNot(ResVal);
 
       Builder.CreateStore(
-          ResVal,
-          Builder.CreateBitCast(Dest, ResVal->getType()->getPointerTo()));
+          ResVal, Builder.CreateElementBitCast(Dest, ResVal->getType()));
     }
 
     if (RValTy->isVoidType())
       return RValue::get(nullptr);
 
     return convertTempToRValue(
-        Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()),
+        Builder.CreateElementBitCast(Dest, ConvertTypeForMem(RValTy)),
         RValTy, E->getExprLoc());
   }
 
@@ -1382,8 +1381,7 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
       return RValue::get(nullptr);
 
     return convertTempToRValue(
-        Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo(
-                                        Dest.getAddressSpace())),
+        Builder.CreateElementBitCast(Dest, ConvertTypeForMem(RValTy)),
         RValTy, E->getExprLoc());
   }
 
@@ -1455,17 +1453,14 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
 
   assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits());
   return convertTempToRValue(
-      Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo(
-                                      Dest.getAddressSpace())),
+      Builder.CreateElementBitCast(Dest, ConvertTypeForMem(RValTy)),
       RValTy, E->getExprLoc());
 }
 
 Address AtomicInfo::emitCastToAtomicIntPointer(Address addr) const {
-  unsigned addrspace =
-    cast<llvm::PointerType>(addr.getPointer()->getType())->getAddressSpace();
   llvm::IntegerType *ty =
     llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
-  return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace));
+  return CGF.Builder.CreateElementBitCast(addr, ty);
 }
 
 Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const {
diff --git a/clang/lib/CodeGen/CGBlocks.cpp b/clang/lib/CodeGen/CGBlocks.cpp
index 2da2014345d8..7bb6dbb8a8ac 100644
--- a/clang/lib/CodeGen/CGBlocks.cpp
+++ b/clang/lib/CodeGen/CGBlocks.cpp
@@ -2721,8 +2721,7 @@ void CodeGenFunction::emitByrefStructureInit(const AutoVarEmission &emission) {
   Address addr = emission.Addr;
 
   // That's an alloca of the byref structure type.
-  llvm::StructType *byrefType = cast<llvm::StructType>(
-    cast<llvm::PointerType>(addr.getPointer()->getType())->getElementType());
+  llvm::StructType *byrefType = cast<llvm::StructType>(addr.getElementType());
 
   unsigned nextHeaderIndex = 0;
   CharUnits nextHeaderOffset;
diff --git a/clang/lib/CodeGen/CGBuilder.h b/clang/lib/CodeGen/CGBuilder.h
index 4fad44a105cd..7c9f41e84eaf 100644
--- a/clang/lib/CodeGen/CGBuilder.h
+++ b/clang/lib/CodeGen/CGBuilder.h
@@ -86,7 +86,8 @@ public:
   llvm::LoadInst *CreateAlignedLoad(llvm::Type *Ty, llvm::Value *Addr,
                                     CharUnits Align,
                                     const llvm::Twine &Name = "") {
-    assert(Addr->getType()->getPointerElementType() == Ty);
+    assert(llvm::cast<llvm::PointerType>(Addr->getType())
+               ->isOpaqueOrPointeeTypeMatches(Ty));
     return CreateAlignedLoad(Ty, Addr, Align.getAsAlign(), Name);
   }
 
@@ -115,13 +116,15 @@ public:
   /// Emit a load from an i1 flag variable.
   llvm::LoadInst *CreateFlagLoad(llvm::Value *Addr,
                                  const llvm::Twine &Name = "") {
-    assert(Addr->getType()->getPointerElementType() == getInt1Ty());
+    assert(llvm::cast<llvm::PointerType>(Addr->getType())
+               ->isOpaqueOrPointeeTypeMatches(getInt1Ty()));
     return CreateAlignedLoad(getInt1Ty(), Addr, CharUnits::One(), Name);
   }
 
   /// Emit a store to an i1 flag variable.
   llvm::StoreInst *CreateFlagStore(bool Value, llvm::Value *Addr) {
-    assert(Addr->getType()->getPointerElementType() == getInt1Ty());
+    assert(llvm::cast<llvm::PointerType>(Addr->getType())
+               ->isOpaqueOrPointeeTypeMatches(getInt1Ty()));
     return CreateAlignedStore(getInt1(Value), Addr, CharUnits::One());
   }
 
@@ -165,8 +168,9 @@ public:
   /// preserving information like the alignment and address space.
   Address CreateElementBitCast(Address Addr, llvm::Type *Ty,
                                const llvm::Twine &Name = "") {
-    auto PtrTy = Ty->getPointerTo(Addr.getAddressSpace());
-    return CreateBitCast(Addr, PtrTy, Name);
+    auto *PtrTy = Ty->getPointerTo(Addr.getAddressSpace());
+    return Address(CreateBitCast(Addr.getPointer(), PtrTy, Name),
+                   Ty, Addr.getAlignment());
   }
 
   using CGBuilderBaseTy::CreatePointerBitCastOrAddrSpaceCast;
@@ -194,6 +198,7 @@ public:
 
     return Address(CreateStructGEP(Addr.getElementType(),
                                    Addr.getPointer(), Index, Name),
+                   ElTy->getElementType(Index),
                    Addr.getAlignment().alignmentAtOffset(Offset));
   }
 
@@ -215,6 +220,7 @@ public:
     return Address(
         CreateInBoundsGEP(Addr.getElementType(), Addr.getPointer(),
                           {getSize(CharUnits::Zero()), getSize(Index)}, Name),
+        ElTy->getElementType(),
         Addr.getAlignment().alignmentAtOffset(Index * EltSize));
   }
 
@@ -231,6 +237,7 @@ public:
 
     return Address(CreateInBoundsGEP(Addr.getElementType(), Addr.getPointer(),
                                      getSize(Index), Name),
+                   ElTy,
                    Addr.getAlignment().alignmentAtOffset(Index * EltSize));
   }
 
@@ -247,15 +254,32 @@ public:
 
     return Address(CreateGEP(Addr.getElementType(), Addr.getPointer(),
                              getSize(Index), Name),
+                   Addr.getElementType(),
                    Addr.getAlignment().alignmentAtOffset(Index * EltSize));
   }
 
+  /// Create GEP with single dynamic index. The address alignment is reduced
+  /// according to the element size.
+  using CGBuilderBaseTy::CreateGEP;
+  Address CreateGEP(Address Addr, llvm::Value *Index,
+                    const llvm::Twine &Name = "") {
+    const llvm::DataLayout &DL = BB->getParent()->getParent()->getDataLayout();
+    CharUnits EltSize =
+        CharUnits::fromQuantity(DL.getTypeAllocSize(Addr.getElementType()));
+
+    return Address(CreateGEP(Addr.getElementType(), Addr.getPointer(), Index,
+                             Name),
+                   Addr.getElementType(),
+                   Addr.getAlignment().alignmentOfArrayElement(EltSize));
+  }
+
   /// Given a pointer to i8, adjust it by a given constant offset.
   Address CreateConstInBoundsByteGEP(Address Addr, CharUnits Offset,
                                      const llvm::Twine &Name = "") {
     assert(Addr.getElementType() == TypeCache.Int8Ty);
     return Address(CreateInBoundsGEP(Addr.getElementType(), Addr.getPointer(),
                                      getSize(Offset), Name),
+                   Addr.getElementType(),
                    Addr.getAlignment().alignmentAtOffset(Offset));
   }
   Address CreateConstByteGEP(Address Addr, CharUnits Offset,
@@ -263,6 +287,7 @@ public:
     assert(Addr.getElementType() == TypeCache.Int8Ty);
     return Address(CreateGEP(Addr.getElementType(), Addr.getPointer(),
                              getSize(Offset), Name),
+                   Addr.getElementType(),
                    Addr.getAlignment().alignmentAtOffset(Offset));
   }
 
@@ -278,8 +303,9 @@ public:
         /*isSigned=*/true);
     if (!GEP->accumulateConstantOffset(DL, Offset))
       llvm_unreachable("offset of GEP with constants is always computable");
-    return Address(GEP, Addr.getAlignment().alignmentAtOffset(
-                            CharUnits::fromQuantity(Offset.getSExtValue())));
+    return Address(GEP, GEP->getResultElementType(),
+                   Addr.getAlignment().alignmentAtOffset(
+                       CharUnits::fromQuantity(Offset.getSExtValue())));
   }
 
   using CGBuilderBaseTy::CreateMemCpy;
@@ -330,8 +356,14 @@ public:
 
     return Address(CreatePreserveStructAccessIndex(ElTy, Addr.getPointer(),
                                                    Index, FieldIndex, DbgInfo),
+                   ElTy->getElementType(Index),
                    Addr.getAlignment().alignmentAtOffset(Offset));
   }
+
+  using CGBuilderBaseTy::CreateLaunderInvariantGroup;
+  Address CreateLaunderInvariantGroup(Address Addr) {
+    return Addr.withPointer(CreateLaunderInvariantGroup(Addr.getPointer()));
+  }
 };
 
 }  // end namespace CodeGen
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 5d6df59cc405..1982b40ff667 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -96,13 +96,33 @@ llvm::Constant *CodeGenModule::getBuiltinLibFunction(const FunctionDecl *FD,
   StringRef Name;
   GlobalDecl D(FD);
 
+  // TODO: This list should be expanded or refactored after all GCC-compatible
+  // std libcall builtins are implemented.
+  static SmallDenseMap<unsigned, StringRef, 8> F128Builtins{
+      {Builtin::BI__builtin_printf, "__printfieee128"},
+      {Builtin::BI__builtin_vsnprintf, "__vsnprintfieee128"},
+      {Builtin::BI__builtin_vsprintf, "__vsprintfieee128"},
+      {Builtin::BI__builtin_sprintf, "__sprintfieee128"},
+      {Builtin::BI__builtin_snprintf, "__snprintfieee128"},
+      {Builtin::BI__builtin_fprintf, "__fprintfieee128"},
+      {Builtin::BI__builtin_nexttowardf128, "__nexttowardieee128"},
+  };
+
   // If the builtin has been declared explicitly with an assembler label,
   // use the mangled name. This differs from the plain label on platforms
   // that prefix labels.
   if (FD->hasAttr<AsmLabelAttr>())
     Name = getMangledName(D);
-  else
-    Name = Context.BuiltinInfo.getName(BuiltinID) + 10;
+  else {
+    // TODO: This mutation should also be applied to other targets other than
+    // PPC, after backend supports IEEE 128-bit style libcalls.
+    if (getTriple().isPPC64() &&
+        &getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad() &&
+        F128Builtins.find(BuiltinID) != F128Builtins.end())
+      Name = F128Builtins[BuiltinID];
+    else
+      Name = Context.BuiltinInfo.getName(BuiltinID) + 10;
+  }
 
   llvm::FunctionType *Ty =
     cast<llvm::FunctionType>(getTypes().ConvertType(FD->getType()));
@@ -667,7 +687,7 @@ getIntegerWidthAndSignedness(const clang::ASTContext &context,
                              const clang::QualType Type) {
   assert(Type->isIntegerType() && "Given type is not an integer.");
   unsigned Width = Type->isBooleanType()  ? 1
-                   : Type->isExtIntType() ? context.getIntWidth(Type)
+                   : Type->isBitIntType() ? context.getIntWidth(Type)
                                           : context.getTypeInfo(Type).Width;
   bool Signed = Type->isSignedIntegerType();
   return {Width, Signed};
@@ -1482,8 +1502,7 @@ Value *CodeGenFunction::EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID,
     Value *ArgValue = EmitScalarExpr(E->getArg(1));
 
     llvm::Type *ArgType = ArgValue->getType();
-    llvm::Type *IndexType =
-        IndexAddress.getPointer()->getType()->getPointerElementType();
+    llvm::Type *IndexType = IndexAddress.getElementType();
     llvm::Type *ResultType = ConvertType(E->getType());
 
     Value *ArgZero = llvm::Constant::getNullValue(ArgType);
@@ -3113,6 +3132,14 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
                                             "elt.abs");
     return RValue::get(Result);
   }
+
+  case Builtin::BI__builtin_elementwise_ceil: {
+    Value *Op0 = EmitScalarExpr(E->getArg(0));
+    Value *Result = Builder.CreateUnaryIntrinsic(llvm::Intrinsic::ceil, Op0,
+                                                 nullptr, "elt.ceil");
+    return RValue::get(Result);
+  }
+
   case Builtin::BI__builtin_elementwise_max: {
     Value *Op0 = EmitScalarExpr(E->getArg(0));
     Value *Op1 = EmitScalarExpr(E->getArg(1));
@@ -3184,6 +3211,13 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     return RValue::get(Result);
   }
 
+  case Builtin::BI__builtin_reduce_xor: {
+    Value *Op0 = EmitScalarExpr(E->getArg(0));
+    Value *Result = Builder.CreateUnaryIntrinsic(
+        llvm::Intrinsic::vector_reduce_xor, Op0, nullptr, "rdx.xor");
+    return RValue::get(Result);
+  }
+
   case Builtin::BI__builtin_matrix_transpose: {
     const auto *MatrixTy = E->getArg(0)->getType()->getAs<ConstantMatrixType>();
     Value *MatValue = EmitScalarExpr(E->getArg(0));
@@ -4478,6 +4512,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
   }
   case Builtin::BI__builtin_addressof:
     return RValue::get(EmitLValue(E->getArg(0)).getPointer(*this));
+  case Builtin::BI__builtin_function_start:
+    return RValue::get(CGM.GetFunctionStart(
+        E->getArg(0)->getAsBuiltinConstantDeclRef(CGM.getContext())));
   case Builtin::BI__builtin_operator_new:
     return EmitBuiltinNewDeleteCall(
         E->getCallee()->getType()->castAs<FunctionProtoType>(), E, false);
@@ -4674,8 +4711,6 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     return EmitCoroutineIntrinsic(E, Intrinsic::coro_end);
   case Builtin::BI__builtin_coro_suspend:
     return EmitCoroutineIntrinsic(E, Intrinsic::coro_suspend);
-  case Builtin::BI__builtin_coro_param:
-    return EmitCoroutineIntrinsic(E, Intrinsic::coro_param);
 
   // OpenCL v2.0 s6.13.16.2, Built-in pipe read and write functions
   case Builtin::BIread_pipe:
@@ -5221,9 +5256,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     llvm::Type *BPP = Int8PtrPtrTy;
 
     DestAddr = Address(Builder.CreateBitCast(DestAddr.getPointer(), BPP, "cp"),
-                       DestAddr.getAlignment());
+                       Int8PtrTy, DestAddr.getAlignment());
     SrcAddr = Address(Builder.CreateBitCast(SrcAddr.getPointer(), BPP, "ap"),
-                      SrcAddr.getAlignment());
+                      Int8PtrTy, SrcAddr.getAlignment());
 
     Value *ArgPtr = Builder.CreateLoad(SrcAddr, "ap.val");
     return RValue::get(Builder.CreateStore(ArgPtr, DestAddr));
@@ -6385,6 +6420,7 @@ static const ARMVectorIntrinsicInfo AArch64SISDIntrinsicMap[] = {
 static const ARMVectorIntrinsicInfo AArch64SVEIntrinsicMap[] = {
 #define GET_SVE_LLVM_INTRINSIC_MAP
 #include "clang/Basic/arm_sve_builtin_cg.inc"
+#include "clang/Basic/BuiltinsAArch64NeonSVEBridge_cg.def"
 #undef GET_SVE_LLVM_INTRINSIC_MAP
 };
 
@@ -9308,6 +9344,54 @@ Value *CodeGenFunction::EmitAArch64SVEBuiltinExpr(unsigned BuiltinID,
     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_sve_tbl2, VTy);
     return Builder.CreateCall(F, {V0, V1, Ops[1]});
   }
+
+  case SVE::BI__builtin_sve_svset_neonq_s8:
+  case SVE::BI__builtin_sve_svset_neonq_s16:
+  case SVE::BI__builtin_sve_svset_neonq_s32:
+  case SVE::BI__builtin_sve_svset_neonq_s64:
+  case SVE::BI__builtin_sve_svset_neonq_u8:
+  case SVE::BI__builtin_sve_svset_neonq_u16:
+  case SVE::BI__builtin_sve_svset_neonq_u32:
+  case SVE::BI__builtin_sve_svset_neonq_u64:
+  case SVE::BI__builtin_sve_svset_neonq_f16:
+  case SVE::BI__builtin_sve_svset_neonq_f32:
+  case SVE::BI__builtin_sve_svset_neonq_f64:
+  case SVE::BI__builtin_sve_svset_neonq_bf16: {
+    return Builder.CreateInsertVector(Ty, Ops[0], Ops[1], Builder.getInt64(0));
+  }
+
+  case SVE::BI__builtin_sve_svget_neonq_s8:
+  case SVE::BI__builtin_sve_svget_neonq_s16:
+  case SVE::BI__builtin_sve_svget_neonq_s32:
+  case SVE::BI__builtin_sve_svget_neonq_s64:
+  case SVE::BI__builtin_sve_svget_neonq_u8:
+  case SVE::BI__builtin_sve_svget_neonq_u16:
+  case SVE::BI__builtin_sve_svget_neonq_u32:
+  case SVE::BI__builtin_sve_svget_neonq_u64:
+  case SVE::BI__builtin_sve_svget_neonq_f16:
+  case SVE::BI__builtin_sve_svget_neonq_f32:
+  case SVE::BI__builtin_sve_svget_neonq_f64:
+  case SVE::BI__builtin_sve_svget_neonq_bf16: {
+    return Builder.CreateExtractVector(Ty, Ops[0], Builder.getInt64(0));
+  }
+
+  case SVE::BI__builtin_sve_svdup_neonq_s8:
+  case SVE::BI__builtin_sve_svdup_neonq_s16:
+  case SVE::BI__builtin_sve_svdup_neonq_s32:
+  case SVE::BI__builtin_sve_svdup_neonq_s64:
+  case SVE::BI__builtin_sve_svdup_neonq_u8:
+  case SVE::BI__builtin_sve_svdup_neonq_u16:
+  case SVE::BI__builtin_sve_svdup_neonq_u32:
+  case SVE::BI__builtin_sve_svdup_neonq_u64:
+  case SVE::BI__builtin_sve_svdup_neonq_f16:
+  case SVE::BI__builtin_sve_svdup_neonq_f32:
+  case SVE::BI__builtin_sve_svdup_neonq_f64:
+  case SVE::BI__builtin_sve_svdup_neonq_bf16: {
+    Value *Insert = Builder.CreateInsertVector(Ty, UndefValue::get(Ty), Ops[0],
+                                               Builder.getInt64(0));
+    return Builder.CreateIntrinsic(Intrinsic::aarch64_sve_dupq_lane, {Ty},
+                                   {Insert, Builder.getInt64(0)});
+  }
   }
 
   /// Should not happen
@@ -15331,7 +15415,8 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
     // If the user wants the entire vector, just load the entire vector.
     if (NumBytes == 16) {
       Value *BC = Builder.CreateBitCast(Ops[0], ResTy->getPointerTo());
-      Value *LD = Builder.CreateLoad(Address(BC, CharUnits::fromQuantity(1)));
+      Value *LD =
+          Builder.CreateLoad(Address(BC, ResTy, CharUnits::fromQuantity(1)));
       if (!IsLE)
         return LD;
 
@@ -15392,8 +15477,8 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
             RevMask.push_back(15 - Idx);
           StVec = Builder.CreateShuffleVector(Ops[2], Ops[2], RevMask);
         }
-        return Builder.CreateStore(StVec,
-                                   Address(BC, CharUnits::fromQuantity(1)));
+        return Builder.CreateStore(
+            StVec, Address(BC, Ops[2]->getType(), CharUnits::fromQuantity(1)));
       }
       auto *ConvTy = Int64Ty;
       unsigned NumElts = 0;
@@ -15427,8 +15512,8 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
         Function *F = CGM.getIntrinsic(Intrinsic::bswap, ConvTy);
         Elt = Builder.CreateCall(F, Elt);
       }
-      return Builder.CreateStore(Elt,
-                                 Address(PtrBC, CharUnits::fromQuantity(1)));
+      return Builder.CreateStore(
+          Elt, Address(PtrBC, ConvTy, CharUnits::fromQuantity(1)));
     };
     unsigned Stored = 0;
     unsigned RemainingBytes = NumBytes;
@@ -16222,7 +16307,8 @@ Value *EmitAMDGPUWorkGroupSize(CodeGenFunction &CGF, unsigned Index) {
   auto *DstTy =
       CGF.Int16Ty->getPointerTo(GEP->getType()->getPointerAddressSpace());
   auto *Cast = CGF.Builder.CreateBitCast(GEP, DstTy);
-  auto *LD = CGF.Builder.CreateLoad(Address(Cast, CharUnits::fromQuantity(2)));
+  auto *LD = CGF.Builder.CreateLoad(
+      Address(Cast, CGF.Int16Ty, CharUnits::fromQuantity(2)));
   llvm::MDBuilder MDHelper(CGF.getLLVMContext());
   llvm::MDNode *RNode = MDHelper.createRange(APInt(16, 1),
       APInt(16, CGF.getTarget().getMaxOpenCLWorkGroupSize() + 1));
@@ -16242,7 +16328,8 @@ Value *EmitAMDGPUGridSize(CodeGenFunction &CGF, unsigned Index) {
   auto *DstTy =
       CGF.Int32Ty->getPointerTo(GEP->getType()->getPointerAddressSpace());
   auto *Cast = CGF.Builder.CreateBitCast(GEP, DstTy);
-  auto *LD = CGF.Builder.CreateLoad(Address(Cast, CharUnits::fromQuantity(4)));
+  auto *LD = CGF.Builder.CreateLoad(
+      Address(Cast, CGF.Int32Ty, CharUnits::fromQuantity(4)));
   LD->setMetadata(llvm::LLVMContext::MD_invariant_load,
                   llvm::MDNode::get(CGF.getLLVMContext(), None));
   return LD;
@@ -16314,8 +16401,7 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
     llvm::Value *Result = Builder.CreateExtractValue(Tmp, 0);
     llvm::Value *Flag = Builder.CreateExtractValue(Tmp, 1);
 
-    llvm::Type *RealFlagType
-      = FlagOutPtr.getPointer()->getType()->getPointerElementType();
+    llvm::Type *RealFlagType = FlagOutPtr.getElementType();
 
     llvm::Value *FlagExt = Builder.CreateZExt(Flag, RealFlagType);
     Builder.CreateStore(FlagExt, FlagOutPtr);
@@ -16572,6 +16658,15 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
     llvm::Value *RayInverseDir = EmitScalarExpr(E->getArg(4));
     llvm::Value *TextureDescr = EmitScalarExpr(E->getArg(5));
 
+    // The builtins take these arguments as vec4 where the last element is
+    // ignored. The intrinsic takes them as vec3.
+    RayOrigin = Builder.CreateShuffleVector(RayOrigin, RayOrigin,
+                                            ArrayRef<int>{0, 1, 2});
+    RayDir =
+        Builder.CreateShuffleVector(RayDir, RayDir, ArrayRef<int>{0, 1, 2});
+    RayInverseDir = Builder.CreateShuffleVector(RayInverseDir, RayInverseDir,
+                                                ArrayRef<int>{0, 1, 2});
+
     Function *F = CGM.getIntrinsic(Intrinsic::amdgcn_image_bvh_intersect_ray,
                                    {NodePtr->getType(), RayDir->getType()});
     return Builder.CreateCall(F, {NodePtr, RayExtent, RayOrigin, RayDir,
@@ -17938,7 +18033,7 @@ RValue CodeGenFunction::EmitBuiltinAlignTo(const CallExpr *E, bool AlignUp) {
     if (getLangOpts().isSignedOverflowDefined())
       Result = Builder.CreateGEP(Int8Ty, Base, Difference, "aligned_result");
     else
-      Result = EmitCheckedInBoundsGEP(Base, Difference,
+      Result = EmitCheckedInBoundsGEP(Int8Ty, Base, Difference,
                                       /*SignedIndices=*/true,
                                       /*isSubtraction=*/!AlignUp,
                                       E->getExprLoc(), "aligned_result");
@@ -18501,6 +18596,7 @@ getIntrinsicForHexagonNonGCCBuiltin(unsigned BuiltinID) {
     CUSTOM_BUILTIN_MAPPING(S2_storerf_pcr, 0)
     CUSTOM_BUILTIN_MAPPING(S2_storeri_pcr, 0)
     CUSTOM_BUILTIN_MAPPING(S2_storerd_pcr, 0)
+    // Legacy builtins that take a vector in place of a vector predicate.
     CUSTOM_BUILTIN_MAPPING(V6_vmaskedstoreq, 64)
     CUSTOM_BUILTIN_MAPPING(V6_vmaskedstorenq, 64)
     CUSTOM_BUILTIN_MAPPING(V6_vmaskedstorentq, 64)
@@ -18534,8 +18630,8 @@ Value *CodeGenFunction::EmitHexagonBuiltinExpr(unsigned BuiltinID,
   auto MakeCircOp = [this, E](unsigned IntID, bool IsLoad) {
     // The base pointer is passed by address, so it needs to be loaded.
     Address A = EmitPointerWithAlignment(E->getArg(0));
-    Address BP = Address(
-        Builder.CreateBitCast(A.getPointer(), Int8PtrPtrTy), A.getAlignment());
+    Address BP = Address(Builder.CreateBitCast(
+        A.getPointer(), Int8PtrPtrTy), Int8PtrTy, A.getAlignment());
     llvm::Value *Base = Builder.CreateLoad(BP);
     // The treatment of both loads and stores is the same: the arguments for
     // the builtin are the same as the arguments for the intrinsic.
@@ -18579,7 +18675,7 @@ Value *CodeGenFunction::EmitHexagonBuiltinExpr(unsigned BuiltinID,
     // per call.
     Address DestAddr = EmitPointerWithAlignment(E->getArg(1));
     DestAddr = Address(Builder.CreateBitCast(DestAddr.getPointer(), Int8PtrTy),
-                       DestAddr.getAlignment());
+                       Int8Ty, DestAddr.getAlignment());
     llvm::Value *DestAddress = DestAddr.getPointer();
 
     // Operands are Base, Dest, Modifier.
@@ -18626,8 +18722,8 @@ Value *CodeGenFunction::EmitHexagonBuiltinExpr(unsigned BuiltinID,
   case Hexagon::BI__builtin_HEXAGON_V6_vsubcarry_128B: {
     // Get the type from the 0-th argument.
     llvm::Type *VecType = ConvertType(E->getArg(0)->getType());
-    Address PredAddr = Builder.CreateBitCast(
-        EmitPointerWithAlignment(E->getArg(2)), VecType->getPointerTo(0));
+    Address PredAddr = Builder.CreateElementBitCast(
+        EmitPointerWithAlignment(E->getArg(2)), VecType);
     llvm::Value *PredIn = V2Q(Builder.CreateLoad(PredAddr));
     llvm::Value *Result = Builder.CreateCall(CGM.getIntrinsic(ID),
         {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1)), PredIn});
@@ -18638,6 +18734,27 @@ Value *CodeGenFunction::EmitHexagonBuiltinExpr(unsigned BuiltinID,
     return Builder.CreateExtractValue(Result, 0);
   }
 
+  case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstoreq:
+  case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorenq:
+  case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentq:
+  case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentnq:
+  case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstoreq_128B:
+  case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorenq_128B:
+  case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentq_128B:
+  case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentnq_128B: {
+    SmallVector<llvm::Value*,4> Ops;
+    const Expr *PredOp = E->getArg(0);
+    // There will be an implicit cast to a boolean vector. Strip it.
+    if (auto *Cast = dyn_cast<ImplicitCastExpr>(PredOp)) {
+      if (Cast->getCastKind() == CK_BitCast)
+        PredOp = Cast->getSubExpr();
+      Ops.push_back(V2Q(EmitScalarExpr(PredOp)));
+    }
+    for (int i = 1, e = E->getNumArgs(); i != e; ++i)
+      Ops.push_back(EmitScalarExpr(E->getArg(i)));
+    return Builder.CreateCall(CGM.getIntrinsic(ID), Ops);
+  }
+
   case Hexagon::BI__builtin_HEXAGON_L2_loadrub_pci:
   case Hexagon::BI__builtin_HEXAGON_L2_loadrb_pci:
   case Hexagon::BI__builtin_HEXAGON_L2_loadruh_pci:
@@ -18674,40 +18791,6 @@ Value *CodeGenFunction::EmitHexagonBuiltinExpr(unsigned BuiltinID,
     return MakeBrevLd(Intrinsic::hexagon_L2_loadri_pbr, Int32Ty);
   case Hexagon::BI__builtin_brev_ldd:
     return MakeBrevLd(Intrinsic::hexagon_L2_loadrd_pbr, Int64Ty);
-
-  default: {
-    if (ID == Intrinsic::not_intrinsic)
-      return nullptr;
-
-    auto IsVectorPredTy = [](llvm::Type *T) {
-      return T->isVectorTy() &&
-             cast<llvm::VectorType>(T)->getElementType()->isIntegerTy(1);
-    };
-
-    llvm::Function *IntrFn = CGM.getIntrinsic(ID);
-    llvm::FunctionType *IntrTy = IntrFn->getFunctionType();
-    SmallVector<llvm::Value*,4> Ops;
-    for (unsigned i = 0, e = IntrTy->getNumParams(); i != e; ++i) {
-      llvm::Type *T = IntrTy->getParamType(i);
-      const Expr *A = E->getArg(i);
-      if (IsVectorPredTy(T)) {
-        // There will be an implicit cast to a boolean vector. Strip it.
-        if (auto *Cast = dyn_cast<ImplicitCastExpr>(A)) {
-          if (Cast->getCastKind() == CK_BitCast)
-            A = Cast->getSubExpr();
-        }
-        Ops.push_back(V2Q(EmitScalarExpr(A)));
-      } else {
-        Ops.push_back(EmitScalarExpr(A));
-      }
-    }
-
-    llvm::Value *Call = Builder.CreateCall(IntrFn, Ops);
-    if (IsVectorPredTy(IntrTy->getReturnType()))
-      Call = Q2V(Call);
-
-    return Call;
-  } // default
   } // switch
 
   return nullptr;
diff --git a/clang/lib/CodeGen/CGCUDANV.cpp b/clang/lib/CodeGen/CGCUDANV.cpp
index a1b4431ca8c4..c4e3f7f54f4f 100644
--- a/clang/lib/CodeGen/CGCUDANV.cpp
+++ b/clang/lib/CodeGen/CGCUDANV.cpp
@@ -814,6 +814,9 @@ llvm::Function *CGNVCUDARuntime::makeModuleCtorFunction() {
         Linkage,
         /*Initializer=*/llvm::ConstantPointerNull::get(VoidPtrPtrTy),
         "__hip_gpubin_handle");
+    if (Linkage == llvm::GlobalValue::LinkOnceAnyLinkage)
+      GpuBinaryHandle->setComdat(
+          CGM.getModule().getOrInsertComdat(GpuBinaryHandle->getName()));
     GpuBinaryHandle->setAlignment(CGM.getPointerAlign().getAsAlign());
     // Prevent the weak symbol in different shared libraries being merged.
     if (Linkage != llvm::GlobalValue::InternalLinkage)
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index d830a7e01709..d70f78fea6b4 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -1261,8 +1261,7 @@ static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty,
     //
     // FIXME: Assert that we aren't truncating non-padding bits when have access
     // to that information.
-    Src = CGF.Builder.CreateBitCast(Src,
-                                    Ty->getPointerTo(Src.getAddressSpace()));
+    Src = CGF.Builder.CreateElementBitCast(Src, Ty);
     return CGF.Builder.CreateLoad(Src);
   }
 
@@ -1832,11 +1831,6 @@ void CodeGenModule::getDefaultFunctionAttributes(StringRef Name,
     if (LangOpts.getFPExceptionMode() == LangOptions::FPE_Ignore)
       FuncAttrs.addAttribute("no-trapping-math", "true");
 
-    // Strict (compliant) code is the default, so only add this attribute to
-    // indicate that we are trying to workaround a problem case.
-    if (!CodeGenOpts.StrictFloatCastOverflow)
-      FuncAttrs.addAttribute("strict-float-cast-overflow", "false");
-
     // TODO: Are these all needed?
     // unsafe/inf/nan/nsz are handled by instruction-level FastMathFlags.
     if (LangOpts.NoHonorInfs)
@@ -1971,7 +1965,7 @@ static bool DetermineNoUndef(QualType QTy, CodeGenTypes &Types,
       // there's no internal padding (typeSizeEqualsStoreSize).
       return false;
   }
-  if (QTy->isExtIntType())
+  if (QTy->isBitIntType())
     return true;
   if (QTy->isReferenceType())
     return true;
@@ -2686,8 +2680,8 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
     case ABIArgInfo::Indirect:
     case ABIArgInfo::IndirectAliased: {
       assert(NumIRArgs == 1);
-      Address ParamAddr =
-          Address(Fn->getArg(FirstIRArg), ArgI.getIndirectAlign());
+      Address ParamAddr = Address(Fn->getArg(FirstIRArg), ConvertTypeForMem(Ty),
+                                  ArgI.getIndirectAlign());
 
       if (!hasScalarEvaluationKind(Ty)) {
         // Aggregates and complex variables are accessed by reference. All we
@@ -3475,12 +3469,19 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI,
     case TEK_Aggregate:
       // Do nothing; aggregrates get evaluated directly into the destination.
       break;
-    case TEK_Scalar:
-      EmitStoreOfScalar(Builder.CreateLoad(ReturnValue),
-                        MakeNaturalAlignAddrLValue(&*AI, RetTy),
-                        /*isInit*/ true);
+    case TEK_Scalar: {
+      LValueBaseInfo BaseInfo;
+      TBAAAccessInfo TBAAInfo;
+      CharUnits Alignment =
+          CGM.getNaturalTypeAlignment(RetTy, &BaseInfo, &TBAAInfo);
+      Address ArgAddr(&*AI, ConvertType(RetTy), Alignment);
+      LValue ArgVal =
+          LValue::MakeAddr(ArgAddr, RetTy, getContext(), BaseInfo, TBAAInfo);
+      EmitStoreOfScalar(
+          Builder.CreateLoad(ReturnValue), ArgVal, /*isInit*/ true);
       break;
     }
+    }
     break;
   }
 
@@ -4134,8 +4135,7 @@ void CodeGenFunction::EmitCallArgs(
   }
 
   // If we still have any arguments, emit them using the type of the argument.
-  for (auto *A : llvm::make_range(std::next(ArgRange.begin(), ArgTypes.size()),
-                                  ArgRange.end()))
+  for (auto *A : llvm::drop_begin(ArgRange, ArgTypes.size()))
     ArgTypes.push_back(IsVariadic ? getVarArgType(A) : A->getType());
   assert((int)ArgTypes.size() == (ArgRange.end() - ArgRange.begin()));
 
@@ -4308,11 +4308,8 @@ void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
       type->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee()) {
     // If we're using inalloca, use the argument memory.  Otherwise, use a
     // temporary.
-    AggValueSlot Slot;
-    if (args.isUsingInAlloca())
-      Slot = createPlaceholderSlot(*this, type);
-    else
-      Slot = CreateAggTemp(type, "agg.tmp");
+    AggValueSlot Slot = args.isUsingInAlloca()
+        ? createPlaceholderSlot(*this, type) : CreateAggTemp(type, "agg.tmp");
 
     bool DestroyedInCallee = true, NeedsEHCleanup = true;
     if (const auto *RD = type->getAsCXXRecordDecl())
@@ -4651,13 +4648,13 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
     //
     // In other cases, we assert that the types match up (until pointers stop
     // having pointee types).
-    llvm::Type *TypeFromVal;
     if (Callee.isVirtual())
-      TypeFromVal = Callee.getVirtualFunctionType();
-    else
-      TypeFromVal =
-          Callee.getFunctionPointer()->getType()->getPointerElementType();
-    assert(IRFuncTy == TypeFromVal);
+      assert(IRFuncTy == Callee.getVirtualFunctionType());
+    else {
+      llvm::PointerType *PtrTy =
+          llvm::cast<llvm::PointerType>(Callee.getFunctionPointer()->getType());
+      assert(PtrTy->isOpaqueOrPointeeTypeMatches(IRFuncTy));
+    }
   }
 #endif
 
@@ -4872,7 +4869,8 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
           I->copyInto(*this, AI);
         } else {
           // Skip the extra memcpy call.
-          auto *T = V->getType()->getPointerElementType()->getPointerTo(
+          auto *T = llvm::PointerType::getWithSamePointeeType(
+              cast<llvm::PointerType>(V->getType()),
               CGM.getDataLayout().getAllocaAddrSpace());
           IRCallArgs[FirstIRArg] = getTargetHooks().performAddrSpaceCast(
               *this, V, LangAS::Default, CGM.getASTAllocaAddressSpace(), T,
@@ -4967,8 +4965,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
           Builder.CreateMemCpy(TempAlloca, Src, SrcSize);
           Src = TempAlloca;
         } else {
-          Src = Builder.CreateBitCast(Src,
-                                      STy->getPointerTo(Src.getAddressSpace()));
+          Src = Builder.CreateElementBitCast(Src, STy);
         }
 
         assert(NumIRArgs == STy->getNumElements());
diff --git a/clang/lib/CodeGen/CGCall.h b/clang/lib/CodeGen/CGCall.h
index e3d9fec6d363..c8594068c3fc 100644
--- a/clang/lib/CodeGen/CGCall.h
+++ b/clang/lib/CodeGen/CGCall.h
@@ -115,7 +115,8 @@ public:
     AbstractInfo = abstractInfo;
     assert(functionPtr && "configuring callee without function pointer");
     assert(functionPtr->getType()->isPointerTy());
-    assert(functionPtr->getType()->getPointerElementType()->isFunctionTy());
+    assert(functionPtr->getType()->isOpaquePointerTy() ||
+           functionPtr->getType()->getPointerElementType()->isFunctionTy());
   }
 
   static CGCallee forBuiltin(unsigned builtinID,
diff --git a/clang/lib/CodeGen/CGClass.cpp b/clang/lib/CodeGen/CGClass.cpp
index 0df64d4d5d26..8f99ff0d50ff 100644
--- a/clang/lib/CodeGen/CGClass.cpp
+++ b/clang/lib/CodeGen/CGClass.cpp
@@ -127,18 +127,18 @@ CodeGenModule::getDynamicOffsetAlignment(CharUnits actualBaseAlign,
 
 Address CodeGenFunction::LoadCXXThisAddress() {
   assert(CurFuncDecl && "loading 'this' without a func declaration?");
-  assert(isa<CXXMethodDecl>(CurFuncDecl));
+  auto *MD = cast<CXXMethodDecl>(CurFuncDecl);
 
   // Lazily compute CXXThisAlignment.
   if (CXXThisAlignment.isZero()) {
     // Just use the best known alignment for the parent.
     // TODO: if we're currently emitting a complete-object ctor/dtor,
     // we can always use the complete-object alignment.
-    auto RD = cast<CXXMethodDecl>(CurFuncDecl)->getParent();
-    CXXThisAlignment = CGM.getClassPointerAlignment(RD);
+    CXXThisAlignment = CGM.getClassPointerAlignment(MD->getParent());
   }
 
-  return Address(LoadCXXThis(), CXXThisAlignment);
+  llvm::Type *Ty = ConvertType(MD->getThisType()->getPointeeType());
+  return Address(LoadCXXThis(), Ty, CXXThisAlignment);
 }
 
 /// Emit the address of a field using a member data pointer.
@@ -286,7 +286,7 @@ ApplyNonVirtualAndVirtualOffset(CodeGenFunction &CGF, Address addr,
   }
   alignment = alignment.alignmentAtOffset(nonVirtualOffset);
 
-  return Address(ptr, alignment);
+  return Address(ptr, CGF.Int8Ty, alignment);
 }
 
 Address CodeGenFunction::GetAddressOfBaseClass(
@@ -326,9 +326,9 @@ Address CodeGenFunction::GetAddressOfBaseClass(
   }
 
   // Get the base pointer type.
+  llvm::Type *BaseValueTy = ConvertType((PathEnd[-1])->getType());
   llvm::Type *BasePtrTy =
-      ConvertType((PathEnd[-1])->getType())
-          ->getPointerTo(Value.getType()->getPointerAddressSpace());
+      BaseValueTy->getPointerTo(Value.getType()->getPointerAddressSpace());
 
   QualType DerivedTy = getContext().getRecordType(Derived);
   CharUnits DerivedAlign = CGM.getClassPointerAlignment(Derived);
@@ -342,7 +342,7 @@ Address CodeGenFunction::GetAddressOfBaseClass(
       EmitTypeCheck(TCK_Upcast, Loc, Value.getPointer(),
                     DerivedTy, DerivedAlign, SkippedChecks);
     }
-    return Builder.CreateBitCast(Value, BasePtrTy);
+    return Builder.CreateElementBitCast(Value, BaseValueTy);
   }
 
   llvm::BasicBlock *origBB = nullptr;
@@ -379,7 +379,7 @@ Address CodeGenFunction::GetAddressOfBaseClass(
                                           VirtualOffset, Derived, VBase);
 
   // Cast to the destination type.
-  Value = Builder.CreateBitCast(Value, BasePtrTy);
+  Value = Builder.CreateElementBitCast(Value, BaseValueTy);
 
   // Build a phi if we needed a null check.
   if (NullCheckValue) {
@@ -406,16 +406,16 @@ CodeGenFunction::GetAddressOfDerivedClass(Address BaseAddr,
 
   QualType DerivedTy =
     getContext().getCanonicalType(getContext().getTagDeclType(Derived));
-  unsigned AddrSpace =
-    BaseAddr.getPointer()->getType()->getPointerAddressSpace();
-  llvm::Type *DerivedPtrTy = ConvertType(DerivedTy)->getPointerTo(AddrSpace);
+  unsigned AddrSpace = BaseAddr.getAddressSpace();
+  llvm::Type *DerivedValueTy = ConvertType(DerivedTy);
+  llvm::Type *DerivedPtrTy = DerivedValueTy->getPointerTo(AddrSpace);
 
   llvm::Value *NonVirtualOffset =
     CGM.GetNonVirtualBaseClassOffset(Derived, PathBegin, PathEnd);
 
   if (!NonVirtualOffset) {
     // No offset, we can just cast back.
-    return Builder.CreateBitCast(BaseAddr, DerivedPtrTy);
+    return Builder.CreateElementBitCast(BaseAddr, DerivedValueTy);
   }
 
   llvm::BasicBlock *CastNull = nullptr;
@@ -453,7 +453,7 @@ CodeGenFunction::GetAddressOfDerivedClass(Address BaseAddr,
     Value = PHI;
   }
 
-  return Address(Value, CGM.getClassPointerAlignment(Derived));
+  return Address(Value, DerivedValueTy, CGM.getClassPointerAlignment(Derived));
 }
 
 llvm::Value *CodeGenFunction::GetVTTParameter(GlobalDecl GD,
@@ -996,16 +996,8 @@ namespace {
 
   private:
     void emitMemcpyIR(Address DestPtr, Address SrcPtr, CharUnits Size) {
-      llvm::PointerType *DPT = DestPtr.getType();
-      llvm::Type *DBP =
-        llvm::Type::getInt8PtrTy(CGF.getLLVMContext(), DPT->getAddressSpace());
-      DestPtr = CGF.Builder.CreateBitCast(DestPtr, DBP);
-
-      llvm::PointerType *SPT = SrcPtr.getType();
-      llvm::Type *SBP =
-        llvm::Type::getInt8PtrTy(CGF.getLLVMContext(), SPT->getAddressSpace());
-      SrcPtr = CGF.Builder.CreateBitCast(SrcPtr, SBP);
-
+      DestPtr = CGF.Builder.CreateElementBitCast(DestPtr, CGF.Int8Ty);
+      SrcPtr = CGF.Builder.CreateElementBitCast(SrcPtr, CGF.Int8Ty);
       CGF.Builder.CreateMemCpy(DestPtr, SrcPtr, Size.getQuantity());
     }
 
@@ -2068,8 +2060,8 @@ void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
 
   if (SlotAS != ThisAS) {
     unsigned TargetThisAS = getContext().getTargetAddressSpace(ThisAS);
-    llvm::Type *NewType =
-        ThisPtr->getType()->getPointerElementType()->getPointerTo(TargetThisAS);
+    llvm::Type *NewType = llvm::PointerType::getWithSamePointeeType(
+        This.getType(), TargetThisAS);
     ThisPtr = getTargetHooks().performAddrSpaceCast(*this, This.getPointer(),
                                                     ThisAS, SlotAS, NewType);
   }
@@ -2507,9 +2499,6 @@ void CodeGenFunction::InitializeVTablePointer(const VPtr &Vptr) {
 
   // Apply the offsets.
   Address VTableField = LoadCXXThisAddress();
-  unsigned ThisAddrSpace =
-      VTableField.getPointer()->getType()->getPointerAddressSpace();
-
   if (!NonVirtualOffset.isZero() || VirtualOffset)
     VTableField = ApplyNonVirtualAndVirtualOffset(
         *this, VTableField, NonVirtualOffset, VirtualOffset, Vptr.VTableClass,
@@ -2525,8 +2514,7 @@ void CodeGenFunction::InitializeVTablePointer(const VPtr &Vptr) {
           ->getPointerTo(GlobalsAS);
   // vtable field is is derived from `this` pointer, therefore they should be in
   // the same addr space. Note that this might not be LLVM address space 0.
-  VTableField = Builder.CreateBitCast(VTableField,
-                                      VTablePtrTy->getPointerTo(ThisAddrSpace));
+  VTableField = Builder.CreateElementBitCast(VTableField, VTablePtrTy);
   VTableAddressPoint = Builder.CreateBitCast(VTableAddressPoint, VTablePtrTy);
 
   llvm::StoreInst *Store = Builder.CreateStore(VTableAddressPoint, VTableField);
diff --git a/clang/lib/CodeGen/CGCleanup.h b/clang/lib/CodeGen/CGCleanup.h
index 1b54c0018d27..76f3a48f32f3 100644
--- a/clang/lib/CodeGen/CGCleanup.h
+++ b/clang/lib/CodeGen/CGCleanup.h
@@ -242,7 +242,7 @@ class alignas(8) EHCleanupScope : public EHScope {
 
   /// An optional i1 variable indicating whether this cleanup has been
   /// activated yet.
-  llvm::AllocaInst *ActiveFlag;
+  Address ActiveFlag;
 
   /// Extra information required for cleanups that have resolved
   /// branches through them.  This has to be allocated on the side
@@ -290,7 +290,8 @@ public:
                  EHScopeStack::stable_iterator enclosingEH)
       : EHScope(EHScope::Cleanup, enclosingEH),
         EnclosingNormal(enclosingNormal), NormalBlock(nullptr),
-        ActiveFlag(nullptr), ExtInfo(nullptr), FixupDepth(fixupDepth) {
+        ActiveFlag(Address::invalid()), ExtInfo(nullptr),
+        FixupDepth(fixupDepth) {
     CleanupBits.IsNormalCleanup = isNormal;
     CleanupBits.IsEHCleanup = isEH;
     CleanupBits.IsActive = true;
@@ -320,13 +321,13 @@ public:
   bool isLifetimeMarker() const { return CleanupBits.IsLifetimeMarker; }
   void setLifetimeMarker() { CleanupBits.IsLifetimeMarker = true; }
 
-  bool hasActiveFlag() const { return ActiveFlag != nullptr; }
+  bool hasActiveFlag() const { return ActiveFlag.isValid(); }
   Address getActiveFlag() const {
-    return Address(ActiveFlag, CharUnits::One());
+    return ActiveFlag;
   }
   void setActiveFlag(Address Var) {
     assert(Var.getAlignment().isOne());
-    ActiveFlag = cast<llvm::AllocaInst>(Var.getPointer());
+    ActiveFlag = Var;
   }
 
   void setTestFlagInNormalCleanup() {
diff --git a/clang/lib/CodeGen/CGCoroutine.cpp b/clang/lib/CodeGen/CGCoroutine.cpp
index ca071d3d2e80..2041d2a5b4c9 100644
--- a/clang/lib/CodeGen/CGCoroutine.cpp
+++ b/clang/lib/CodeGen/CGCoroutine.cpp
@@ -597,6 +597,10 @@ void CodeGenFunction::EmitCoroutineBody(const CoroutineBodyStmt &S) {
       CGM.getIntrinsic(llvm::Intrinsic::coro_begin), {CoroId, Phi});
   CurCoro.Data->CoroBegin = CoroBegin;
 
+  // We need to emit `get_­return_­object` first. According to:
+  // [dcl.fct.def.coroutine]p7
+  // The call to get_­return_­object is sequenced before the call to
+  // initial_­suspend and is invoked at most once.
   GetReturnObjectManager GroManager(*this, S);
   GroManager.EmitGroAlloca();
 
diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp
index af651e6f44b7..6e189a61dd20 100644
--- a/clang/lib/CodeGen/CGDebugInfo.cpp
+++ b/clang/lib/CodeGen/CGDebugInfo.cpp
@@ -768,7 +768,7 @@ llvm::DIType *CGDebugInfo::CreateType(const BuiltinType *BT) {
       }
 
       // Element count = (VLENB / SEW) x LMUL
-      SmallVector<int64_t, 9> Expr(
+      SmallVector<int64_t, 12> Expr(
           // The DW_OP_bregx operation has two operands: a register which is
           // specified by an unsigned LEB128 number, followed by a signed LEB128
           // offset.
@@ -782,6 +782,8 @@ llvm::DIType *CGDebugInfo::CreateType(const BuiltinType *BT) {
         Expr.push_back(llvm::dwarf::DW_OP_div);
       else
         Expr.push_back(llvm::dwarf::DW_OP_mul);
+      // Element max index = count - 1
+      Expr.append({llvm::dwarf::DW_OP_constu, 1, llvm::dwarf::DW_OP_minus});
 
       auto *LowerBound =
           llvm::ConstantAsMetadata::get(llvm::ConstantInt::getSigned(
@@ -884,9 +886,9 @@ llvm::DIType *CGDebugInfo::CreateType(const AutoType *Ty) {
   return DBuilder.createUnspecifiedType("auto");
 }
 
-llvm::DIType *CGDebugInfo::CreateType(const ExtIntType *Ty) {
+llvm::DIType *CGDebugInfo::CreateType(const BitIntType *Ty) {
 
-  StringRef Name = Ty->isUnsigned() ? "unsigned _ExtInt" : "_ExtInt";
+  StringRef Name = Ty->isUnsigned() ? "unsigned _BitInt" : "_BitInt";
   llvm::dwarf::TypeKind Encoding = Ty->isUnsigned()
                                        ? llvm::dwarf::DW_ATE_unsigned
                                        : llvm::dwarf::DW_ATE_signed;
@@ -3353,6 +3355,9 @@ static QualType UnwrapTypeForDebugInfo(QualType T, const ASTContext &C) {
     case Type::Elaborated:
       T = cast<ElaboratedType>(T)->getNamedType();
       break;
+    case Type::Using:
+      T = cast<UsingType>(T)->getUnderlyingType();
+      break;
     case Type::Paren:
       T = cast<ParenType>(T)->getInnerType();
       break;
@@ -3531,8 +3536,8 @@ llvm::DIType *CGDebugInfo::CreateTypeNode(QualType Ty, llvm::DIFile *Unit,
   case Type::Atomic:
     return CreateType(cast<AtomicType>(Ty), Unit);
 
-  case Type::ExtInt:
-    return CreateType(cast<ExtIntType>(Ty));
+  case Type::BitInt:
+    return CreateType(cast<BitIntType>(Ty));
   case Type::Pipe:
     return CreateType(cast<PipeType>(Ty), Unit);
 
@@ -3545,6 +3550,7 @@ llvm::DIType *CGDebugInfo::CreateTypeNode(QualType Ty, llvm::DIFile *Unit,
   case Type::Decayed:
   case Type::DeducedTemplateSpecialization:
   case Type::Elaborated:
+  case Type::Using:
   case Type::Paren:
   case Type::MacroQualified:
   case Type::SubstTemplateTypeParm:
@@ -3633,6 +3639,9 @@ llvm::DICompositeType *CGDebugInfo::CreateLimitedType(const RecordType *Ty) {
     // Record exports it symbols to the containing structure.
     if (CXXRD->isAnonymousStructOrUnion())
         Flags |= llvm::DINode::FlagExportSymbols;
+
+    Flags |= getAccessFlag(CXXRD->getAccess(),
+                           dyn_cast<CXXRecordDecl>(CXXRD->getDeclContext()));
   }
 
   llvm::DINodeArray Annotations = CollectBTFDeclTagAnnotations(D);
diff --git a/clang/lib/CodeGen/CGDebugInfo.h b/clang/lib/CodeGen/CGDebugInfo.h
index a7b72fa5f5a6..14ff0eeabd21 100644
--- a/clang/lib/CodeGen/CGDebugInfo.h
+++ b/clang/lib/CodeGen/CGDebugInfo.h
@@ -177,7 +177,7 @@ class CGDebugInfo {
   llvm::DIType *CreateType(const BuiltinType *Ty);
   llvm::DIType *CreateType(const ComplexType *Ty);
   llvm::DIType *CreateType(const AutoType *Ty);
-  llvm::DIType *CreateType(const ExtIntType *Ty);
+  llvm::DIType *CreateType(const BitIntType *Ty);
   llvm::DIType *CreateQualifiedType(QualType Ty, llvm::DIFile *Fg,
                                     TypeLoc TL = TypeLoc());
   llvm::DIType *CreateQualifiedType(const FunctionProtoType *Ty,
diff --git a/clang/lib/CodeGen/CGDecl.cpp b/clang/lib/CodeGen/CGDecl.cpp
index 941671c61482..e09279c1d455 100644
--- a/clang/lib/CodeGen/CGDecl.cpp
+++ b/clang/lib/CodeGen/CGDecl.cpp
@@ -405,7 +405,8 @@ void CodeGenFunction::EmitStaticVarDecl(const VarDecl &D,
 
   // Store into LocalDeclMap before generating initializer to handle
   // circular references.
-  setAddrOfLocalVar(&D, Address(addr, alignment));
+  llvm::Type *elemTy = ConvertTypeForMem(D.getType());
+  setAddrOfLocalVar(&D, Address(addr, elemTy, alignment));
 
   // We can't have a VLA here, but we can have a pointer to a VLA,
   // even though that doesn't really make any sense.
@@ -458,8 +459,7 @@ void CodeGenFunction::EmitStaticVarDecl(const VarDecl &D,
   // RAUW's the GV uses of this constant will be invalid.
   llvm::Constant *castedAddr =
     llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast(var, expectedType);
-  if (var != castedAddr)
-    LocalDeclMap.find(&D)->second = Address(castedAddr, alignment);
+  LocalDeclMap.find(&D)->second = Address(castedAddr, elemTy, alignment);
   CGM.setStaticLocalDeclAddress(&D, castedAddr);
 
   CGM.getSanitizerMetadata()->reportGlobalToASan(var, D);
@@ -1146,7 +1146,7 @@ Address CodeGenModule::createUnnamedGlobalFrom(const VarDecl &D,
     CacheEntry->setAlignment(Align.getAsAlign());
   }
 
-  return Address(CacheEntry, Align);
+  return Address(CacheEntry, CacheEntry->getValueType(), Align);
 }
 
 static Address createUnnamedGlobalForMemcpyFrom(CodeGenModule &CGM,
@@ -1193,7 +1193,7 @@ static void emitStoresForConstant(CodeGenModule &CGM, const VarDecl &D,
     bool valueAlreadyCorrect =
         constant->isNullValue() || isa<llvm::UndefValue>(constant);
     if (!valueAlreadyCorrect) {
-      Loc = Builder.CreateBitCast(Loc, Ty->getPointerTo(Loc.getAddressSpace()));
+      Loc = Builder.CreateElementBitCast(Loc, Ty);
       emitStoresForInitAfterBZero(CGM, constant, Loc, isVolatile, Builder,
                                   IsAutoInit);
     }
diff --git a/clang/lib/CodeGen/CGDeclCXX.cpp b/clang/lib/CodeGen/CGDeclCXX.cpp
index d22f9dc3b68c..3579761f1429 100644
--- a/clang/lib/CodeGen/CGDeclCXX.cpp
+++ b/clang/lib/CodeGen/CGDeclCXX.cpp
@@ -172,7 +172,7 @@ void CodeGenFunction::EmitInvariantStart(llvm::Constant *Addr, CharUnits Size) {
 }
 
 void CodeGenFunction::EmitCXXGlobalVarDeclInit(const VarDecl &D,
-                                               llvm::Constant *DeclPtr,
+                                               llvm::GlobalVariable *GV,
                                                bool PerformInit) {
 
   const Expr *Init = D.getInit();
@@ -194,14 +194,16 @@ void CodeGenFunction::EmitCXXGlobalVarDeclInit(const VarDecl &D,
   // "shared" address space qualifier, but the constructor of StructWithCtor
   // expects "this" in the "generic" address space.
   unsigned ExpectedAddrSpace = getContext().getTargetAddressSpace(T);
-  unsigned ActualAddrSpace = DeclPtr->getType()->getPointerAddressSpace();
+  unsigned ActualAddrSpace = GV->getAddressSpace();
+  llvm::Constant *DeclPtr = GV;
   if (ActualAddrSpace != ExpectedAddrSpace) {
-    llvm::Type *LTy = CGM.getTypes().ConvertTypeForMem(T);
-    llvm::PointerType *PTy = llvm::PointerType::get(LTy, ExpectedAddrSpace);
+    llvm::PointerType *PTy = llvm::PointerType::getWithSamePointeeType(
+        GV->getType(), ExpectedAddrSpace);
     DeclPtr = llvm::ConstantExpr::getAddrSpaceCast(DeclPtr, PTy);
   }
 
-  ConstantAddress DeclAddr(DeclPtr, getContext().getDeclAlign(&D));
+  ConstantAddress DeclAddr(
+      DeclPtr, GV->getValueType(), getContext().getDeclAlign(&D));
 
   if (!T->isReferenceType()) {
     if (getLangOpts().OpenMP && !getLangOpts().OpenMPSimd &&
diff --git a/clang/lib/CodeGen/CGException.cpp b/clang/lib/CodeGen/CGException.cpp
index aff9c77d53c7..91ecbecc843f 100644
--- a/clang/lib/CodeGen/CGException.cpp
+++ b/clang/lib/CodeGen/CGException.cpp
@@ -400,8 +400,8 @@ void CodeGenFunction::EmitAnyExprToExn(const Expr *e, Address addr) {
 
   // __cxa_allocate_exception returns a void*;  we need to cast this
   // to the appropriate type for the object.
-  llvm::Type *ty = ConvertTypeForMem(e->getType())->getPointerTo();
-  Address typedAddr = Builder.CreateBitCast(addr, ty);
+  llvm::Type *ty = ConvertTypeForMem(e->getType());
+  Address typedAddr = Builder.CreateElementBitCast(addr, ty);
 
   // FIXME: this isn't quite right!  If there's a final unelided call
   // to a copy constructor, then according to [except.terminate]p1 we
@@ -421,13 +421,13 @@ void CodeGenFunction::EmitAnyExprToExn(const Expr *e, Address addr) {
 Address CodeGenFunction::getExceptionSlot() {
   if (!ExceptionSlot)
     ExceptionSlot = CreateTempAlloca(Int8PtrTy, "exn.slot");
-  return Address(ExceptionSlot, getPointerAlign());
+  return Address(ExceptionSlot, Int8PtrTy, getPointerAlign());
 }
 
 Address CodeGenFunction::getEHSelectorSlot() {
   if (!EHSelectorSlot)
     EHSelectorSlot = CreateTempAlloca(Int32Ty, "ehselector.slot");
-  return Address(EHSelectorSlot, CharUnits::fromQuantity(4));
+  return Address(EHSelectorSlot, Int32Ty, CharUnits::fromQuantity(4));
 }
 
 llvm::Value *CodeGenFunction::getExceptionFromSlot() {
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 4332e74dbb24..34b4951a7f72 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -71,7 +71,7 @@ Address CodeGenFunction::CreateTempAllocaWithoutCast(llvm::Type *Ty,
                                                      llvm::Value *ArraySize) {
   auto Alloca = CreateTempAlloca(Ty, Name, ArraySize);
   Alloca->setAlignment(Align.getAsAlign());
-  return Address(Alloca, Align);
+  return Address(Alloca, Ty, Align);
 }
 
 /// CreateTempAlloca - This creates a alloca and inserts it into the entry
@@ -101,7 +101,7 @@ Address CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, CharUnits Align,
         Ty->getPointerTo(DestAddrSpace), /*non-null*/ true);
   }
 
-  return Address(V, Align);
+  return Address(V, Ty, Align);
 }
 
 /// CreateTempAlloca - This creates an alloca and inserts it into the entry
@@ -144,7 +144,7 @@ Address CodeGenFunction::CreateMemTemp(QualType Ty, CharUnits Align,
                                     /*ArraySize=*/nullptr, Alloca);
 
   if (Ty->isConstantMatrixType()) {
-    auto *ArrayTy = cast<llvm::ArrayType>(Result.getType()->getElementType());
+    auto *ArrayTy = cast<llvm::ArrayType>(Result.getElementType());
     auto *VectorTy = llvm::FixedVectorType::get(ArrayTy->getElementType(),
                                                 ArrayTy->getNumElements());
 
@@ -1099,7 +1099,7 @@ Address CodeGenFunction::EmitPointerWithAlignment(const Expr *E,
           if (InnerBaseInfo.getAlignmentSource() != AlignmentSource::Decl) {
             if (BaseInfo)
               BaseInfo->mergeForCast(TargetTypeBaseInfo);
-            Addr = Address(Addr.getPointer(), Align);
+            Addr = Address(Addr.getPointer(), Addr.getElementType(), Align);
           }
         }
 
@@ -1111,10 +1111,12 @@ Address CodeGenFunction::EmitPointerWithAlignment(const Expr *E,
                                       CodeGenFunction::CFITCK_UnrelatedCast,
                                       CE->getBeginLoc());
         }
-        return CE->getCastKind() != CK_AddressSpaceConversion
-                   ? Builder.CreateBitCast(Addr, ConvertType(E->getType()))
-                   : Builder.CreateAddrSpaceCast(Addr,
-                                                 ConvertType(E->getType()));
+
+        if (CE->getCastKind() == CK_AddressSpaceConversion)
+         return Builder.CreateAddrSpaceCast(Addr, ConvertType(E->getType()));
+
+        llvm::Type *ElemTy = ConvertTypeForMem(E->getType()->getPointeeType());
+        return Builder.CreateElementBitCast(Addr, ElemTy);
       }
       break;
 
@@ -1160,7 +1162,8 @@ Address CodeGenFunction::EmitPointerWithAlignment(const Expr *E,
   // Otherwise, use the alignment of the type.
   CharUnits Align =
       CGM.getNaturalPointeeTypeAlignment(E->getType(), BaseInfo, TBAAInfo);
-  return Address(EmitScalarExpr(E), Align);
+  llvm::Type *ElemTy = ConvertTypeForMem(E->getType()->getPointeeType());
+  return Address(EmitScalarExpr(E), ElemTy, Align);
 }
 
 llvm::Value *CodeGenFunction::EmitNonNullRValueCheck(RValue RV, QualType T) {
@@ -1306,7 +1309,8 @@ LValue CodeGenFunction::EmitLValue(const Expr *E) {
     const ConstantExpr *CE = cast<ConstantExpr>(E);
     if (llvm::Value *Result = ConstantEmitter(*this).tryEmitConstantExpr(CE)) {
       QualType RetType = cast<CallExpr>(CE->getSubExpr()->IgnoreImplicit())
-                             ->getCallReturnType(getContext());
+                             ->getCallReturnType(getContext())
+                             ->getPointeeType();
       return MakeNaturalAlignAddrLValue(Result, RetType);
     }
     return EmitLValue(cast<ConstantExpr>(E)->getSubExpr());
@@ -1342,10 +1346,11 @@ LValue CodeGenFunction::EmitLValue(const Expr *E) {
     if (LV.isSimple()) {
       // Defend against branches out of gnu statement expressions surrounded by
       // cleanups.
-      llvm::Value *V = LV.getPointer(*this);
+      Address Addr = LV.getAddress(*this);
+      llvm::Value *V = Addr.getPointer();
       Scope.ForceCleanup({&V});
-      return LValue::MakeAddr(Address(V, LV.getAlignment()), LV.getType(),
-                              getContext(), LV.getBaseInfo(), LV.getTBAAInfo());
+      return LValue::MakeAddr(Addr.withPointer(V), LV.getType(), getContext(),
+                              LV.getBaseInfo(), LV.getTBAAInfo());
     }
     // FIXME: Is it possible to create an ExprWithCleanups that produces a
     // bitfield lvalue or some other non-simple lvalue?
@@ -1777,16 +1782,14 @@ llvm::Value *CodeGenFunction::EmitFromMemory(llvm::Value *Value, QualType Ty) {
 // MatrixType), if it points to a array (the memory type of MatrixType).
 static Address MaybeConvertMatrixAddress(Address Addr, CodeGenFunction &CGF,
                                          bool IsVector = true) {
-  auto *ArrayTy = dyn_cast<llvm::ArrayType>(
-      cast<llvm::PointerType>(Addr.getPointer()->getType())->getElementType());
+  auto *ArrayTy = dyn_cast<llvm::ArrayType>(Addr.getElementType());
   if (ArrayTy && IsVector) {
     auto *VectorTy = llvm::FixedVectorType::get(ArrayTy->getElementType(),
                                                 ArrayTy->getNumElements());
 
     return Address(CGF.Builder.CreateElementBitCast(Addr, VectorTy));
   }
-  auto *VectorTy = dyn_cast<llvm::VectorType>(
-      cast<llvm::PointerType>(Addr.getPointer()->getType())->getElementType());
+  auto *VectorTy = dyn_cast<llvm::VectorType>(Addr.getElementType());
   if (VectorTy && !IsVector) {
     auto *ArrayTy = llvm::ArrayType::get(
         VectorTy->getElementType(),
@@ -2475,10 +2478,11 @@ CodeGenFunction::EmitLoadOfReference(LValue RefLVal,
       Builder.CreateLoad(RefLVal.getAddress(*this), RefLVal.isVolatile());
   CGM.DecorateInstructionWithTBAA(Load, RefLVal.getTBAAInfo());
 
+  QualType PointeeType = RefLVal.getType()->getPointeeType();
   CharUnits Align = CGM.getNaturalTypeAlignment(
-      RefLVal.getType()->getPointeeType(), PointeeBaseInfo, PointeeTBAAInfo,
+      PointeeType, PointeeBaseInfo, PointeeTBAAInfo,
       /* forPointeeType= */ true);
-  return Address(Load, Align);
+  return Address(Load, ConvertTypeForMem(PointeeType), Align);
 }
 
 LValue CodeGenFunction::EmitLoadOfReferenceLValue(LValue RefLVal) {
@@ -2528,7 +2532,7 @@ static LValue EmitGlobalVarDeclLValue(CodeGenFunction &CGF,
   llvm::Type *RealVarTy = CGF.getTypes().ConvertTypeForMem(VD->getType());
   V = EmitBitCastOfLValueToProperType(CGF, V, RealVarTy);
   CharUnits Alignment = CGF.getContext().getDeclAlign(VD);
-  Address Addr(V, Alignment);
+  Address Addr(V, RealVarTy, Alignment);
   // Emit reference to the private copy of the variable if it is an OpenMP
   // threadprivate variable.
   if (CGF.getLangOpts().OpenMP && !CGF.getLangOpts().OpenMPSimd &&
@@ -2610,7 +2614,7 @@ static LValue EmitGlobalNamedRegister(const VarDecl *VD, CodeGenModule &CGM) {
 
   llvm::Value *Ptr =
     llvm::MetadataAsValue::get(CGM.getLLVMContext(), M->getOperand(0));
-  return LValue::MakeGlobalReg(Address(Ptr, Alignment), VD->getType());
+  return LValue::MakeGlobalReg(Ptr, Alignment, VD->getType());
 }
 
 /// Determine whether we can emit a reference to \p VD from the current
@@ -2706,7 +2710,7 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
                                         /* BaseInfo= */ nullptr,
                                         /* TBAAInfo= */ nullptr,
                                         /* forPointeeType= */ true);
-        Addr = Address(Val, Alignment);
+        Addr = Address(Val, ConvertTypeForMem(E->getType()), Alignment);
       }
       return MakeAddrLValue(Addr, T, AlignmentSource::Decl);
     }
@@ -2783,9 +2787,10 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
     // Otherwise, it might be static local we haven't emitted yet for
     // some reason; most likely, because it's in an outer function.
     } else if (VD->isStaticLocal()) {
-      addr = Address(CGM.getOrCreateStaticVarDecl(
-          *VD, CGM.getLLVMLinkageVarDefinition(VD, /*IsConstant=*/false)),
-                     getContext().getDeclAlign(VD));
+      llvm::Constant *var = CGM.getOrCreateStaticVarDecl(
+          *VD, CGM.getLLVMLinkageVarDefinition(VD, /*IsConstant=*/false));
+      addr = Address(
+          var, ConvertTypeForMem(VD->getType()), getContext().getDeclAlign(VD));
 
     // No other cases for now.
     } else {
@@ -3586,7 +3591,7 @@ static llvm::Value *emitArraySubscriptGEP(CodeGenFunction &CGF,
                                           SourceLocation loc,
                                     const llvm::Twine &name = "arrayidx") {
   if (inbounds) {
-    return CGF.EmitCheckedInBoundsGEP(ptr, indices, signedIndices,
+    return CGF.EmitCheckedInBoundsGEP(elemType, ptr, indices, signedIndices,
                                       CodeGenFunction::NotSubtraction, loc,
                                       name);
   } else {
@@ -3698,7 +3703,7 @@ static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr,
                                                         idx, DbgInfo);
   }
 
-  return Address(eltPtr, eltAlign);
+  return Address(eltPtr, CGF.ConvertTypeForMem(eltType), eltAlign);
 }
 
 LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E,
@@ -4380,8 +4385,7 @@ LValue CodeGenFunction::EmitLValueForField(LValue base,
         hasAnyVptr(FieldType, getContext()))
       // Because unions can easily skip invariant.barriers, we need to add
       // a barrier every time CXXRecord field with vptr is referenced.
-      addr = Address(Builder.CreateLaunderInvariantGroup(addr.getPointer()),
-                     addr.getAlignment());
+      addr = Builder.CreateLaunderInvariantGroup(addr);
 
     if (IsInPreservedAIRegion ||
         (getDebugInfo() && rec->hasAttr<BPFPreserveAccessIndexAttr>())) {
@@ -4539,10 +4543,10 @@ EmitConditionalOperatorLValue(const AbstractConditionalOperator *expr) {
       // because it can't be used.
       if (auto *ThrowExpr = dyn_cast<CXXThrowExpr>(live->IgnoreParens())) {
         EmitCXXThrowExpr(ThrowExpr);
-        llvm::Type *Ty =
-            llvm::PointerType::getUnqual(ConvertType(dead->getType()));
+        llvm::Type *ElemTy = ConvertType(dead->getType());
+        llvm::Type *Ty = llvm::PointerType::getUnqual(ElemTy);
         return MakeAddrLValue(
-            Address(llvm::UndefValue::get(Ty), CharUnits::One()),
+            Address(llvm::UndefValue::get(Ty), ElemTy, CharUnits::One()),
             dead->getType());
       }
       return EmitLValue(live);
@@ -4584,11 +4588,13 @@ EmitConditionalOperatorLValue(const AbstractConditionalOperator *expr) {
   EmitBlock(contBlock);
 
   if (lhs && rhs) {
-    llvm::PHINode *phi =
-        Builder.CreatePHI(lhs->getPointer(*this)->getType(), 2, "cond-lvalue");
-    phi->addIncoming(lhs->getPointer(*this), lhsBlock);
-    phi->addIncoming(rhs->getPointer(*this), rhsBlock);
-    Address result(phi, std::min(lhs->getAlignment(), rhs->getAlignment()));
+    Address lhsAddr = lhs->getAddress(*this);
+    Address rhsAddr = rhs->getAddress(*this);
+    llvm::PHINode *phi = Builder.CreatePHI(lhsAddr.getType(), 2, "cond-lvalue");
+    phi->addIncoming(lhsAddr.getPointer(), lhsBlock);
+    phi->addIncoming(rhsAddr.getPointer(), rhsBlock);
+    Address result(phi, lhsAddr.getElementType(),
+                   std::min(lhsAddr.getAlignment(), rhsAddr.getAlignment()));
     AlignmentSource alignSource =
       std::max(lhs->getBaseInfo().getAlignmentSource(),
                rhs->getBaseInfo().getAlignmentSource());
diff --git a/clang/lib/CodeGen/CGExprAgg.cpp b/clang/lib/CodeGen/CGExprAgg.cpp
index 5b56a587fa5f..3b996b89a1d7 100644
--- a/clang/lib/CodeGen/CGExprAgg.cpp
+++ b/clang/lib/CodeGen/CGExprAgg.cpp
@@ -301,7 +301,7 @@ void AggExprEmitter::withReturnValueSlot(
   if (!UseTemp)
     return;
 
-  assert(Dest.getPointer() != Src.getAggregatePointer());
+  assert(Dest.isIgnored() || Dest.getPointer() != Src.getAggregatePointer());
   EmitFinalDestCopy(E->getType(), Src);
 
   if (!RequiresDestruction && LifetimeStartInst) {
@@ -493,7 +493,7 @@ void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType,
   CharUnits elementSize = CGF.getContext().getTypeSizeInChars(elementType);
   CharUnits elementAlign =
     DestPtr.getAlignment().alignmentOfArrayElement(elementSize);
-  llvm::Type *llvmElementType = begin->getType()->getPointerElementType();
+  llvm::Type *llvmElementType = CGF.ConvertTypeForMem(elementType);
 
   // Consider initializing the array by copying from a global. For this to be
   // more efficient than per-element initialization, the size of the elements
@@ -513,7 +513,8 @@ void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType,
       Emitter.finalize(GV);
       CharUnits Align = CGM.getContext().getTypeAlignInChars(ArrayQTy);
       GV->setAlignment(Align.getAsAlign());
-      EmitFinalDestCopy(ArrayQTy, CGF.MakeAddrLValue(GV, ArrayQTy, Align));
+      Address GVAddr(GV, GV->getValueType(), Align);
+      EmitFinalDestCopy(ArrayQTy, CGF.MakeAddrLValue(GVAddr, ArrayQTy));
       return;
     }
   }
@@ -565,8 +566,8 @@ void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType,
       if (endOfInit.isValid()) Builder.CreateStore(element, endOfInit);
     }
 
-    LValue elementLV =
-      CGF.MakeAddrLValue(Address(element, elementAlign), elementType);
+    LValue elementLV = CGF.MakeAddrLValue(
+        Address(element, llvmElementType, elementAlign), elementType);
     EmitInitializationToLValue(E->getInit(i), elementLV);
   }
 
diff --git a/clang/lib/CodeGen/CGExprCXX.cpp b/clang/lib/CodeGen/CGExprCXX.cpp
index cc838bf38c6c..0571c498c377 100644
--- a/clang/lib/CodeGen/CGExprCXX.cpp
+++ b/clang/lib/CodeGen/CGExprCXX.cpp
@@ -1052,13 +1052,8 @@ void CodeGenFunction::EmitNewArrayInitializer(
       InitListElements =
           cast<ConstantArrayType>(ILE->getType()->getAsArrayTypeUnsafe())
               ->getSize().getZExtValue();
-      CurPtr =
-          Address(Builder.CreateInBoundsGEP(CurPtr.getElementType(),
-                                            CurPtr.getPointer(),
-                                            Builder.getSize(InitListElements),
-                                            "string.init.end"),
-                  CurPtr.getAlignment().alignmentAtOffset(InitListElements *
-                                                          ElementSize));
+      CurPtr = Builder.CreateConstInBoundsGEP(
+          CurPtr, InitListElements, "string.init.end");
 
       // Zero out the rest, if any remain.
       llvm::ConstantInt *ConstNum = dyn_cast<llvm::ConstantInt>(NumElements);
@@ -1135,7 +1130,7 @@ void CodeGenFunction::EmitNewArrayInitializer(
     }
 
     // Switch back to initializing one base element at a time.
-    CurPtr = Builder.CreateBitCast(CurPtr, BeginPtr.getType());
+    CurPtr = Builder.CreateElementBitCast(CurPtr, BeginPtr.getElementType());
   }
 
   // If all elements have already been initialized, skip any further
@@ -1594,7 +1589,7 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) {
     // In these cases, discard the computed alignment and use the
     // formal alignment of the allocated type.
     if (BaseInfo.getAlignmentSource() != AlignmentSource::Decl)
-      allocation = Address(allocation.getPointer(), allocAlign);
+      allocation = allocation.withAlignment(allocAlign);
 
     // Set up allocatorArgs for the call to operator delete if it's not
     // the reserved global operator.
@@ -1664,7 +1659,7 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) {
           allocationAlign, getContext().toCharUnitsFromBits(AllocatorAlign));
     }
 
-    allocation = Address(RV.getScalarVal(), allocationAlign);
+    allocation = Address(RV.getScalarVal(), Int8Ty, allocationAlign);
   }
 
   // Emit a null check on the allocation result if the allocation
@@ -1725,8 +1720,7 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) {
   // of optimization level.
   if (CGM.getCodeGenOpts().StrictVTablePointers &&
       allocator->isReservedGlobalPlacementOperator())
-    result = Address(Builder.CreateLaunderInvariantGroup(result.getPointer()),
-                     result.getAlignment());
+    result = Builder.CreateLaunderInvariantGroup(result);
 
   // Emit sanitizer checks for pointer value now, so that in the case of an
   // array it was checked only once and not at each constructor call. We may
diff --git a/clang/lib/CodeGen/CGExprConstant.cpp b/clang/lib/CodeGen/CGExprConstant.cpp
index ff900ed077e6..cf1f2e0eab92 100644
--- a/clang/lib/CodeGen/CGExprConstant.cpp
+++ b/clang/lib/CodeGen/CGExprConstant.cpp
@@ -899,7 +899,7 @@ static ConstantAddress tryEmitGlobalCompoundLiteral(CodeGenModule &CGM,
   CharUnits Align = CGM.getContext().getTypeAlignInChars(E->getType());
   if (llvm::GlobalVariable *Addr =
           CGM.getAddrOfConstantCompoundLiteralIfEmitted(E))
-    return ConstantAddress(Addr, Align);
+    return ConstantAddress(Addr, Addr->getValueType(), Align);
 
   LangAS addressSpace = E->getType().getAddressSpace();
 
@@ -921,7 +921,7 @@ static ConstantAddress tryEmitGlobalCompoundLiteral(CodeGenModule &CGM,
   emitter.finalize(GV);
   GV->setAlignment(Align.getAsAlign());
   CGM.setAddrOfConstantCompoundLiteral(E, GV);
-  return ConstantAddress(GV, Align);
+  return ConstantAddress(GV, GV->getValueType(), Align);
 }
 
 static llvm::Constant *
@@ -1988,6 +1988,9 @@ ConstantLValueEmitter::VisitAddrLabelExpr(const AddrLabelExpr *E) {
 ConstantLValue
 ConstantLValueEmitter::VisitCallExpr(const CallExpr *E) {
   unsigned builtin = E->getBuiltinCallee();
+  if (builtin == Builtin::BI__builtin_function_start)
+    return CGM.GetFunctionStart(
+        E->getArg(0)->getAsBuiltinConstantDeclRef(CGM.getContext()));
   if (builtin != Builtin::BI__builtin___CFStringMakeConstantString &&
       builtin != Builtin::BI__builtin___NSStringMakeConstantString)
     return nullptr;
diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp
index ae9434f96529..e32462eb635c 100644
--- a/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/clang/lib/CodeGen/CGExprScalar.cpp
@@ -1240,7 +1240,18 @@ Value *ScalarExprEmitter::EmitScalarCast(Value *Src, QualType SrcType,
 
   if (isa<llvm::IntegerType>(DstElementTy)) {
     assert(SrcElementTy->isFloatingPointTy() && "Unknown real conversion");
-    if (DstElementType->isSignedIntegerOrEnumerationType())
+    bool IsSigned = DstElementType->isSignedIntegerOrEnumerationType();
+
+    // If we can't recognize overflow as undefined behavior, assume that
+    // overflow saturates. This protects against normal optimizations if we are
+    // compiling with non-standard FP semantics.
+    if (!CGF.CGM.getCodeGenOpts().StrictFloatCastOverflow) {
+      llvm::Intrinsic::ID IID =
+          IsSigned ? llvm::Intrinsic::fptosi_sat : llvm::Intrinsic::fptoui_sat;
+      return Builder.CreateCall(CGF.CGM.getIntrinsic(IID, {DstTy, SrcTy}), Src);
+    }
+
+    if (IsSigned)
       return Builder.CreateFPToSI(Src, DstTy, "conv");
     return Builder.CreateFPToUI(Src, DstTy, "conv");
   }
@@ -2631,12 +2642,12 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
           = CGF.getContext().getAsVariableArrayType(type)) {
       llvm::Value *numElts = CGF.getVLASize(vla).NumElts;
       if (!isInc) numElts = Builder.CreateNSWNeg(numElts, "vla.negsize");
+      llvm::Type *elemTy = value->getType()->getPointerElementType();
       if (CGF.getLangOpts().isSignedOverflowDefined())
-        value = Builder.CreateGEP(value->getType()->getPointerElementType(),
-                                  value, numElts, "vla.inc");
+        value = Builder.CreateGEP(elemTy, value, numElts, "vla.inc");
       else
         value = CGF.EmitCheckedInBoundsGEP(
-            value, numElts, /*SignedIndices=*/false, isSubtraction,
+            elemTy, value, numElts, /*SignedIndices=*/false, isSubtraction,
             E->getExprLoc(), "vla.inc");
 
     // Arithmetic on function pointers (!) is just +-1.
@@ -2647,7 +2658,8 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
       if (CGF.getLangOpts().isSignedOverflowDefined())
         value = Builder.CreateGEP(CGF.Int8Ty, value, amt, "incdec.funcptr");
       else
-        value = CGF.EmitCheckedInBoundsGEP(value, amt, /*SignedIndices=*/false,
+        value = CGF.EmitCheckedInBoundsGEP(CGF.Int8Ty, value, amt,
+                                           /*SignedIndices=*/false,
                                            isSubtraction, E->getExprLoc(),
                                            "incdec.funcptr");
       value = Builder.CreateBitCast(value, input->getType());
@@ -2655,13 +2667,13 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
     // For everything else, we can just do a simple increment.
     } else {
       llvm::Value *amt = Builder.getInt32(amount);
+      llvm::Type *elemTy = CGF.ConvertTypeForMem(type);
       if (CGF.getLangOpts().isSignedOverflowDefined())
-        value = Builder.CreateGEP(value->getType()->getPointerElementType(),
-                                  value, amt, "incdec.ptr");
+        value = Builder.CreateGEP(elemTy, value, amt, "incdec.ptr");
       else
-        value = CGF.EmitCheckedInBoundsGEP(value, amt, /*SignedIndices=*/false,
-                                           isSubtraction, E->getExprLoc(),
-                                           "incdec.ptr");
+        value = CGF.EmitCheckedInBoundsGEP(
+            elemTy, value, amt, /*SignedIndices=*/false, isSubtraction,
+            E->getExprLoc(), "incdec.ptr");
     }
 
   // Vector increment/decrement.
@@ -2771,9 +2783,9 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
     if (CGF.getLangOpts().isSignedOverflowDefined())
       value = Builder.CreateGEP(CGF.Int8Ty, value, sizeValue, "incdec.objptr");
     else
-      value = CGF.EmitCheckedInBoundsGEP(value, sizeValue,
-                                         /*SignedIndices=*/false, isSubtraction,
-                                         E->getExprLoc(), "incdec.objptr");
+      value = CGF.EmitCheckedInBoundsGEP(
+          CGF.Int8Ty, value, sizeValue, /*SignedIndices=*/false, isSubtraction,
+          E->getExprLoc(), "incdec.objptr");
     value = Builder.CreateBitCast(value, input->getType());
   }
 
@@ -3508,16 +3520,15 @@ static Value *emitPointerArithmetic(CodeGenFunction &CGF,
     // GEP indexes are signed, and scaling an index isn't permitted to
     // signed-overflow, so we use the same semantics for our explicit
     // multiply.  We suppress this if overflow is not undefined behavior.
+    llvm::Type *elemTy = pointer->getType()->getPointerElementType();
     if (CGF.getLangOpts().isSignedOverflowDefined()) {
       index = CGF.Builder.CreateMul(index, numElements, "vla.index");
-      pointer = CGF.Builder.CreateGEP(
-          pointer->getType()->getPointerElementType(), pointer, index,
-          "add.ptr");
+      pointer = CGF.Builder.CreateGEP(elemTy, pointer, index, "add.ptr");
     } else {
       index = CGF.Builder.CreateNSWMul(index, numElements, "vla.index");
-      pointer =
-          CGF.EmitCheckedInBoundsGEP(pointer, index, isSigned, isSubtraction,
-                                     op.E->getExprLoc(), "add.ptr");
+      pointer = CGF.EmitCheckedInBoundsGEP(
+          elemTy, pointer, index, isSigned, isSubtraction, op.E->getExprLoc(),
+          "add.ptr");
     }
     return pointer;
   }
@@ -3531,12 +3542,13 @@ static Value *emitPointerArithmetic(CodeGenFunction &CGF,
     return CGF.Builder.CreateBitCast(result, pointer->getType());
   }
 
+  llvm::Type *elemTy = CGF.ConvertTypeForMem(elementType);
   if (CGF.getLangOpts().isSignedOverflowDefined())
-    return CGF.Builder.CreateGEP(
-        pointer->getType()->getPointerElementType(), pointer, index, "add.ptr");
+    return CGF.Builder.CreateGEP(elemTy, pointer, index, "add.ptr");
 
-  return CGF.EmitCheckedInBoundsGEP(pointer, index, isSigned, isSubtraction,
-                                    op.E->getExprLoc(), "add.ptr");
+  return CGF.EmitCheckedInBoundsGEP(
+      elemTy, pointer, index, isSigned, isSubtraction, op.E->getExprLoc(),
+      "add.ptr");
 }
 
 // Construct an fmuladd intrinsic to represent a fused mul-add of MulOp and
@@ -5057,12 +5069,12 @@ static GEPOffsetAndOverflow EmitGEPOffsetInBytes(Value *BasePtr, Value *GEPVal,
 }
 
 Value *
-CodeGenFunction::EmitCheckedInBoundsGEP(Value *Ptr, ArrayRef<Value *> IdxList,
+CodeGenFunction::EmitCheckedInBoundsGEP(llvm::Type *ElemTy, Value *Ptr,
+                                        ArrayRef<Value *> IdxList,
                                         bool SignedIndices, bool IsSubtraction,
                                         SourceLocation Loc, const Twine &Name) {
   llvm::Type *PtrTy = Ptr->getType();
-  Value *GEPVal = Builder.CreateInBoundsGEP(
-      PtrTy->getPointerElementType(), Ptr, IdxList, Name);
+  Value *GEPVal = Builder.CreateInBoundsGEP(ElemTy, Ptr, IdxList, Name);
 
   // If the pointer overflow sanitizer isn't enabled, do nothing.
   if (!SanOpts.has(SanitizerKind::PointerOverflow))
diff --git a/clang/lib/CodeGen/CGNonTrivialStruct.cpp b/clang/lib/CodeGen/CGNonTrivialStruct.cpp
index ad505fc5a0d4..e3b0e069b830 100644
--- a/clang/lib/CodeGen/CGNonTrivialStruct.cpp
+++ b/clang/lib/CodeGen/CGNonTrivialStruct.cpp
@@ -366,11 +366,11 @@ template <class Derived> struct GenFuncBase {
         llvm::ConstantInt::get(NumElts->getType(), BaseEltSize);
     llvm::Value *SizeInBytes =
         CGF.Builder.CreateNUWMul(BaseEltSizeVal, NumElts);
-    Address BC = CGF.Builder.CreateBitCast(DstAddr, CGF.CGM.Int8PtrTy);
+    Address BC = CGF.Builder.CreateElementBitCast(DstAddr, CGF.CGM.Int8Ty);
     llvm::Value *DstArrayEnd =
         CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, BC.getPointer(), SizeInBytes);
-    DstArrayEnd = CGF.Builder.CreateBitCast(DstArrayEnd, CGF.CGM.Int8PtrPtrTy,
-                                            "dstarray.end");
+    DstArrayEnd = CGF.Builder.CreateBitCast(
+        DstArrayEnd, CGF.CGM.Int8PtrPtrTy, "dstarray.end");
     llvm::BasicBlock *PreheaderBB = CGF.Builder.GetInsertBlock();
 
     // Create the header block and insert the phi instructions.
@@ -426,9 +426,9 @@ template <class Derived> struct GenFuncBase {
     assert(Addr.isValid() && "invalid address");
     if (Offset.getQuantity() == 0)
       return Addr;
-    Addr = CGF->Builder.CreateBitCast(Addr, CGF->CGM.Int8PtrTy);
+    Addr = CGF->Builder.CreateElementBitCast(Addr, CGF->CGM.Int8Ty);
     Addr = CGF->Builder.CreateConstInBoundsGEP(Addr, Offset.getQuantity());
-    return CGF->Builder.CreateBitCast(Addr, CGF->CGM.Int8PtrPtrTy);
+    return CGF->Builder.CreateElementBitCast(Addr, CGF->CGM.Int8PtrTy);
   }
 
   Address getAddrWithOffset(Address Addr, CharUnits StructFieldOffset,
@@ -491,9 +491,8 @@ template <class Derived> struct GenFuncBase {
 
     for (unsigned I = 0; I < N; ++I) {
       Alignments[I] = Addrs[I].getAlignment();
-      Ptrs[I] =
-          CallerCGF.Builder.CreateBitCast(Addrs[I], CallerCGF.CGM.Int8PtrPtrTy)
-              .getPointer();
+      Ptrs[I] = CallerCGF.Builder.CreateElementBitCast(
+          Addrs[I], CallerCGF.CGM.Int8PtrTy).getPointer();
     }
 
     if (llvm::Function *F =
@@ -554,19 +553,21 @@ struct GenBinaryFunc : CopyStructVisitor<Derived, IsMove>,
         return;
 
       QualType RT = QualType(FD->getParent()->getTypeForDecl(), 0);
-      llvm::PointerType *PtrTy = this->CGF->ConvertType(RT)->getPointerTo();
+      llvm::Type *Ty = this->CGF->ConvertType(RT);
       Address DstAddr = this->getAddrWithOffset(Addrs[DstIdx], Offset);
       LValue DstBase = this->CGF->MakeAddrLValue(
-          this->CGF->Builder.CreateBitCast(DstAddr, PtrTy), FT);
+          this->CGF->Builder.CreateElementBitCast(DstAddr, Ty), FT);
       DstLV = this->CGF->EmitLValueForField(DstBase, FD);
       Address SrcAddr = this->getAddrWithOffset(Addrs[SrcIdx], Offset);
       LValue SrcBase = this->CGF->MakeAddrLValue(
-          this->CGF->Builder.CreateBitCast(SrcAddr, PtrTy), FT);
+          this->CGF->Builder.CreateElementBitCast(SrcAddr, Ty), FT);
       SrcLV = this->CGF->EmitLValueForField(SrcBase, FD);
     } else {
-      llvm::PointerType *Ty = this->CGF->ConvertTypeForMem(FT)->getPointerTo();
-      Address DstAddr = this->CGF->Builder.CreateBitCast(Addrs[DstIdx], Ty);
-      Address SrcAddr = this->CGF->Builder.CreateBitCast(Addrs[SrcIdx], Ty);
+      llvm::Type *Ty = this->CGF->ConvertTypeForMem(FT);
+      Address DstAddr =
+          this->CGF->Builder.CreateElementBitCast(Addrs[DstIdx], Ty);
+      Address SrcAddr =
+          this->CGF->Builder.CreateElementBitCast(Addrs[SrcIdx], Ty);
       DstLV = this->CGF->MakeAddrLValue(DstAddr, FT);
       SrcLV = this->CGF->MakeAddrLValue(SrcAddr, FT);
     }
@@ -817,7 +818,7 @@ void CodeGenFunction::destroyNonTrivialCStruct(CodeGenFunction &CGF,
 void CodeGenFunction::defaultInitNonTrivialCStructVar(LValue Dst) {
   GenDefaultInitialize Gen(getContext());
   Address DstPtr =
-      Builder.CreateBitCast(Dst.getAddress(*this), CGM.Int8PtrPtrTy);
+      Builder.CreateElementBitCast(Dst.getAddress(*this), CGM.Int8PtrTy);
   Gen.setCGF(this);
   QualType QT = Dst.getType();
   QT = Dst.isVolatile() ? QT.withVolatile() : QT;
@@ -830,7 +831,7 @@ static void callSpecialFunction(G &&Gen, StringRef FuncName, QualType QT,
                                 std::array<Address, N> Addrs) {
   auto SetArtificialLoc = ApplyDebugLocation::CreateArtificial(CGF);
   for (unsigned I = 0; I < N; ++I)
-    Addrs[I] = CGF.Builder.CreateBitCast(Addrs[I], CGF.CGM.Int8PtrPtrTy);
+    Addrs[I] = CGF.Builder.CreateElementBitCast(Addrs[I], CGF.CGM.Int8PtrTy);
   QT = IsVolatile ? QT.withVolatile() : QT;
   Gen.callFunc(FuncName, QT, Addrs, CGF);
 }
diff --git a/clang/lib/CodeGen/CGObjC.cpp b/clang/lib/CodeGen/CGObjC.cpp
index ac26f0d4232c..b5bcf157036d 100644
--- a/clang/lib/CodeGen/CGObjC.cpp
+++ b/clang/lib/CodeGen/CGObjC.cpp
@@ -3915,8 +3915,8 @@ static llvm::Value *emitIsPlatformVersionAtLeast(CodeGenFunction &CGF,
     Args.push_back(
         llvm::ConstantInt::get(CGM.Int32Ty, getBaseMachOPlatformID(TT)));
     Args.push_back(llvm::ConstantInt::get(CGM.Int32Ty, Version.getMajor()));
-    Args.push_back(llvm::ConstantInt::get(CGM.Int32Ty, Min ? *Min : 0));
-    Args.push_back(llvm::ConstantInt::get(CGM.Int32Ty, SMin ? *SMin : 0));
+    Args.push_back(llvm::ConstantInt::get(CGM.Int32Ty, Min.getValueOr(0)));
+    Args.push_back(llvm::ConstantInt::get(CGM.Int32Ty, SMin.getValueOr(0)));
   };
 
   assert(!Version.empty() && "unexpected empty version");
@@ -3952,8 +3952,8 @@ CodeGenFunction::EmitBuiltinAvailable(const VersionTuple &Version) {
   Optional<unsigned> Min = Version.getMinor(), SMin = Version.getSubminor();
   llvm::Value *Args[] = {
       llvm::ConstantInt::get(CGM.Int32Ty, Version.getMajor()),
-      llvm::ConstantInt::get(CGM.Int32Ty, Min ? *Min : 0),
-      llvm::ConstantInt::get(CGM.Int32Ty, SMin ? *SMin : 0),
+      llvm::ConstantInt::get(CGM.Int32Ty, Min.getValueOr(0)),
+      llvm::ConstantInt::get(CGM.Int32Ty, SMin.getValueOr(0))
   };
 
   llvm::Value *CallRes =
diff --git a/clang/lib/CodeGen/CGObjCGNU.cpp b/clang/lib/CodeGen/CGObjCGNU.cpp
index e016644150b4..b2bf60d2c0fc 100644
--- a/clang/lib/CodeGen/CGObjCGNU.cpp
+++ b/clang/lib/CodeGen/CGObjCGNU.cpp
@@ -978,7 +978,9 @@ class CGObjCGNUstep2 : public CGObjCGNUstep {
     // Look for an existing one
     llvm::StringMap<llvm::Constant*>::iterator old = ObjCStrings.find(Str);
     if (old != ObjCStrings.end())
-      return ConstantAddress(old->getValue(), Align);
+      return ConstantAddress(
+          old->getValue(), old->getValue()->getType()->getPointerElementType(),
+          Align);
 
     bool isNonASCII = SL->containsNonAscii();
 
@@ -1000,7 +1002,7 @@ class CGObjCGNUstep2 : public CGObjCGNUstep {
       auto *ObjCStr = llvm::ConstantExpr::getIntToPtr(
           llvm::ConstantInt::get(Int64Ty, str), IdTy);
       ObjCStrings[Str] = ObjCStr;
-      return ConstantAddress(ObjCStr, Align);
+      return ConstantAddress(ObjCStr, IdTy->getPointerElementType(), Align);
     }
 
     StringRef StringClass = CGM.getLangOpts().ObjCConstantStringClass;
@@ -1114,7 +1116,7 @@ class CGObjCGNUstep2 : public CGObjCGNUstep {
     llvm::Constant *ObjCStr = llvm::ConstantExpr::getBitCast(ObjCStrGV, IdTy);
     ObjCStrings[Str] = ObjCStr;
     ConstantStrings.push_back(ObjCStr);
-    return ConstantAddress(ObjCStr, Align);
+    return ConstantAddress(ObjCStr, IdTy->getPointerElementType(), Align);
   }
 
   void PushProperty(ConstantArrayBuilder &PropertiesArray,
@@ -2476,7 +2478,7 @@ ConstantAddress CGObjCGNU::GenerateConstantString(const StringLiteral *SL) {
   // Look for an existing one
   llvm::StringMap<llvm::Constant*>::iterator old = ObjCStrings.find(Str);
   if (old != ObjCStrings.end())
-    return ConstantAddress(old->getValue(), Align);
+    return ConstantAddress(old->getValue(), Int8Ty, Align);
 
   StringRef StringClass = CGM.getLangOpts().ObjCConstantStringClass;
 
@@ -2503,7 +2505,7 @@ ConstantAddress CGObjCGNU::GenerateConstantString(const StringLiteral *SL) {
   ObjCStr = llvm::ConstantExpr::getBitCast(ObjCStr, PtrToInt8Ty);
   ObjCStrings[Str] = ObjCStr;
   ConstantStrings.push_back(ObjCStr);
-  return ConstantAddress(ObjCStr, Align);
+  return ConstantAddress(ObjCStr, Int8Ty, Align);
 }
 
 ///Generates a message send where the super is the receiver.  This is a message
diff --git a/clang/lib/CodeGen/CGObjCMac.cpp b/clang/lib/CodeGen/CGObjCMac.cpp
index 5b925359ac25..425d1a793439 100644
--- a/clang/lib/CodeGen/CGObjCMac.cpp
+++ b/clang/lib/CodeGen/CGObjCMac.cpp
@@ -1983,7 +1983,8 @@ CGObjCCommonMac::GenerateConstantNSString(const StringLiteral *Literal) {
     GetConstantStringEntry(NSConstantStringMap, Literal, StringLength);
 
   if (auto *C = Entry.second)
-    return ConstantAddress(C, CharUnits::fromQuantity(C->getAlignment()));
+    return ConstantAddress(
+        C, C->getValueType(), CharUnits::fromQuantity(C->getAlignment()));
 
   // If we don't already have it, get _NSConstantStringClassReference.
   llvm::Constant *Class = getNSConstantStringClassRef();
@@ -2036,7 +2037,7 @@ CGObjCCommonMac::GenerateConstantNSString(const StringLiteral *Literal) {
                      : NSStringSection);
   Entry.second = GV;
 
-  return ConstantAddress(GV, Alignment);
+  return ConstantAddress(GV, GV->getValueType(), Alignment);
 }
 
 enum {
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index 75709b3c7e78..e35c15421520 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -15,6 +15,7 @@
 #include "CGCleanup.h"
 #include "CGRecordLayout.h"
 #include "CodeGenFunction.h"
+#include "TargetInfo.h"
 #include "clang/AST/APValue.h"
 #include "clang/AST/Attr.h"
 #include "clang/AST/Decl.h"
@@ -687,8 +688,6 @@ static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr,
   // Drill down to the base element type on both arrays.
   const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
   llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, DestAddr);
-  DestAddr =
-      CGF.Builder.CreateElementBitCast(DestAddr, DestAddr.getElementType());
   if (DRD)
     SrcAddr =
         CGF.Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
@@ -775,7 +774,7 @@ LValue ReductionCodeGen::emitSharedLValueUB(CodeGenFunction &CGF,
 }
 
 void ReductionCodeGen::emitAggregateInitialization(
-    CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal,
+    CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr,
     const OMPDeclareReductionDecl *DRD) {
   // Emit VarDecl with copy init for arrays.
   // Get the address of the original variable captured in current
@@ -788,7 +787,7 @@ void ReductionCodeGen::emitAggregateInitialization(
                        EmitDeclareReductionInit,
                        EmitDeclareReductionInit ? ClausesData[N].ReductionOp
                                                 : PrivateVD->getInit(),
-                       DRD, SharedLVal.getAddress(CGF));
+                       DRD, SharedAddr);
 }
 
 ReductionCodeGen::ReductionCodeGen(ArrayRef<const Expr *> Shareds,
@@ -882,7 +881,7 @@ void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N,
 }
 
 void ReductionCodeGen::emitInitialization(
-    CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal,
+    CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr,
     llvm::function_ref<bool(CodeGenFunction &)> DefaultInit) {
   assert(SharedAddresses.size() > N && "No variable was generated");
   const auto *PrivateVD =
@@ -892,21 +891,15 @@ void ReductionCodeGen::emitInitialization(
   QualType PrivateType = PrivateVD->getType();
   PrivateAddr = CGF.Builder.CreateElementBitCast(
       PrivateAddr, CGF.ConvertTypeForMem(PrivateType));
-  QualType SharedType = SharedAddresses[N].first.getType();
-  SharedLVal = CGF.MakeAddrLValue(
-      CGF.Builder.CreateElementBitCast(SharedLVal.getAddress(CGF),
-                                       CGF.ConvertTypeForMem(SharedType)),
-      SharedType, SharedAddresses[N].first.getBaseInfo(),
-      CGF.CGM.getTBAAInfoForSubobject(SharedAddresses[N].first, SharedType));
   if (CGF.getContext().getAsArrayType(PrivateVD->getType())) {
     if (DRD && DRD->getInitializer())
       (void)DefaultInit(CGF);
-    emitAggregateInitialization(CGF, N, PrivateAddr, SharedLVal, DRD);
+    emitAggregateInitialization(CGF, N, PrivateAddr, SharedAddr, DRD);
   } else if (DRD && (DRD->getInitializer() || !PrivateVD->hasInit())) {
     (void)DefaultInit(CGF);
+    QualType SharedType = SharedAddresses[N].first.getType();
     emitInitWithReductionInitializer(CGF, DRD, ClausesData[N].ReductionOp,
-                                     PrivateAddr, SharedLVal.getAddress(CGF),
-                                     SharedLVal.getType());
+                                     PrivateAddr, SharedAddr, SharedType);
   } else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
              !CGF.isTrivialInitializer(PrivateVD->getInit())) {
     CGF.EmitAnyExprToMem(PrivateVD->getInit(), PrivateAddr,
@@ -2016,12 +2009,13 @@ Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
                                                           StringRef Name) {
   std::string Suffix = getName({"artificial", ""});
   llvm::Type *VarLVType = CGF.ConvertTypeForMem(VarType);
-  llvm::Value *GAddr =
+  llvm::GlobalVariable *GAddr =
       getOrCreateInternalVariable(VarLVType, Twine(Name).concat(Suffix));
   if (CGM.getLangOpts().OpenMP && CGM.getLangOpts().OpenMPUseTLS &&
       CGM.getTarget().isTLSSupported()) {
-    cast<llvm::GlobalVariable>(GAddr)->setThreadLocal(/*Val=*/true);
-    return Address(GAddr, CGM.getContext().getTypeAlignInChars(VarType));
+    GAddr->setThreadLocal(/*Val=*/true);
+    return Address(GAddr, GAddr->getValueType(),
+                   CGM.getContext().getTypeAlignInChars(VarType));
   }
   std::string CacheSuffix = getName({"cache", ""});
   llvm::Value *Args[] = {
@@ -2084,7 +2078,8 @@ void CGOpenMPRuntime::emitIfClause(CodeGenFunction &CGF, const Expr *Cond,
 void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
                                        llvm::Function *OutlinedFn,
                                        ArrayRef<llvm::Value *> CapturedVars,
-                                       const Expr *IfCond) {
+                                       const Expr *IfCond,
+                                       llvm::Value *NumThreads) {
   if (!CGF.HaveInsertPoint())
     return;
   llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
@@ -2175,7 +2170,7 @@ Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF,
   return ThreadIDTemp;
 }
 
-llvm::Constant *CGOpenMPRuntime::getOrCreateInternalVariable(
+llvm::GlobalVariable *CGOpenMPRuntime::getOrCreateInternalVariable(
     llvm::Type *Ty, const llvm::Twine &Name, unsigned AddressSpace) {
   SmallString<256> Buffer;
   llvm::raw_svector_ostream Out(Buffer);
@@ -2183,7 +2178,7 @@ llvm::Constant *CGOpenMPRuntime::getOrCreateInternalVariable(
   StringRef RuntimeName = Out.str();
   auto &Elem = *InternalVars.try_emplace(RuntimeName, nullptr).first;
   if (Elem.second) {
-    assert(Elem.second->getType()->getPointerElementType() == Ty &&
+    assert(Elem.second->getType()->isOpaqueOrPointeeTypeMatches(Ty) &&
            "OMP internal variable has different type than requested");
     return &*Elem.second;
   }
@@ -4498,10 +4493,7 @@ CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
         std::tie(Addr, Size) = getPointerAndSize(CGF, E);
         llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());
         LValue Base = CGF.MakeAddrLValue(
-            Address(CGF.Builder.CreateGEP(AffinitiesArray.getElementType(),
-                                          AffinitiesArray.getPointer(), Idx),
-                    AffinitiesArray.getAlignment()),
-            KmpTaskAffinityInfoTy);
+            CGF.Builder.CreateGEP(AffinitiesArray, Idx), KmpTaskAffinityInfoTy);
         // affs[i].base_addr = &<Affinities[i].second>;
         LValue BaseAddrLVal = CGF.EmitLValueForField(
             Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));
@@ -4665,12 +4657,10 @@ CGOpenMPRuntime::getDepobjElements(CodeGenFunction &CGF, LValue DepobjLVal,
           Base.getAddress(CGF), CGF.ConvertTypeForMem(KmpDependInfoPtrTy));
   Base = CGF.MakeAddrLValue(Addr, KmpDependInfoTy, Base.getBaseInfo(),
                             Base.getTBAAInfo());
-  llvm::Value *DepObjAddr = CGF.Builder.CreateGEP(
-      Addr.getElementType(), Addr.getPointer(),
-      llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
+  Address DepObjAddr = CGF.Builder.CreateGEP(
+      Addr, llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
   LValue NumDepsBase = CGF.MakeAddrLValue(
-      Address(DepObjAddr, Addr.getAlignment()), KmpDependInfoTy,
-      Base.getBaseInfo(), Base.getTBAAInfo());
+      DepObjAddr, KmpDependInfoTy, Base.getBaseInfo(), Base.getTBAAInfo());
   // NumDeps = deps[i].base_addr;
   LValue BaseAddrLVal = CGF.EmitLValueForField(
       NumDepsBase, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
@@ -4706,10 +4696,7 @@ static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
       LValue &PosLVal = *Pos.get<LValue *>();
       llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());
       Base = CGF.MakeAddrLValue(
-          Address(CGF.Builder.CreateGEP(DependenciesArray.getElementType(),
-                                        DependenciesArray.getPointer(), Idx),
-                  DependenciesArray.getAlignment()),
-          KmpDependInfoTy);
+          CGF.Builder.CreateGEP(DependenciesArray, Idx), KmpDependInfoTy);
     }
     // deps[i].base_addr = &<Dependencies[i].second>;
     LValue BaseAddrLVal = CGF.EmitLValueForField(
@@ -4766,12 +4753,10 @@ emitDepobjElementsSizes(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
           Base.getAddress(CGF), KmpDependInfoPtrT);
       Base = CGF.MakeAddrLValue(Addr, KmpDependInfoTy, Base.getBaseInfo(),
                                 Base.getTBAAInfo());
-      llvm::Value *DepObjAddr = CGF.Builder.CreateGEP(
-          Addr.getElementType(), Addr.getPointer(),
-          llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
+      Address DepObjAddr = CGF.Builder.CreateGEP(
+          Addr, llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
       LValue NumDepsBase = CGF.MakeAddrLValue(
-          Address(DepObjAddr, Addr.getAlignment()), KmpDependInfoTy,
-          Base.getBaseInfo(), Base.getTBAAInfo());
+          DepObjAddr, KmpDependInfoTy, Base.getBaseInfo(), Base.getTBAAInfo());
       // NumDeps = deps[i].base_addr;
       LValue BaseAddrLVal = CGF.EmitLValueForField(
           NumDepsBase, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
@@ -4827,12 +4812,10 @@ static void emitDepobjElements(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
                                 Base.getTBAAInfo());
 
       // Get number of elements in a single depobj.
-      llvm::Value *DepObjAddr = CGF.Builder.CreateGEP(
-          Addr.getElementType(), Addr.getPointer(),
-          llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
+      Address DepObjAddr = CGF.Builder.CreateGEP(
+          Addr, llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
       LValue NumDepsBase = CGF.MakeAddrLValue(
-          Address(DepObjAddr, Addr.getAlignment()), KmpDependInfoTy,
-          Base.getBaseInfo(), Base.getTBAAInfo());
+          DepObjAddr, KmpDependInfoTy, Base.getBaseInfo(), Base.getTBAAInfo());
       // NumDeps = deps[i].base_addr;
       LValue BaseAddrLVal = CGF.EmitLValueForField(
           NumDepsBase, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
@@ -4844,10 +4827,7 @@ static void emitDepobjElements(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
           ElSize,
           CGF.Builder.CreateIntCast(NumDeps, CGF.SizeTy, /*isSigned=*/false));
       llvm::Value *Pos = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());
-      Address DepAddr =
-          Address(CGF.Builder.CreateGEP(DependenciesArray.getElementType(),
-                                        DependenciesArray.getPointer(), Pos),
-                  DependenciesArray.getAlignment());
+      Address DepAddr = CGF.Builder.CreateGEP(DependenciesArray, Pos);
       CGF.Builder.CreateMemCpy(DepAddr, Base.getAddress(CGF), Size);
 
       // Increase pos.
@@ -5929,25 +5909,20 @@ static llvm::Value *emitReduceInitFunction(CodeGenModule &CGM,
                                 CGM.getContext().getSizeType(), Loc);
   }
   RCG.emitAggregateType(CGF, N, Size);
-  LValue OrigLVal;
+  Address OrigAddr = Address::invalid();
   // If initializer uses initializer from declare reduction construct, emit a
   // pointer to the address of the original reduction item (reuired by reduction
   // initializer)
   if (RCG.usesReductionInitializer(N)) {
     Address SharedAddr = CGF.GetAddrOfLocalVar(&ParamOrig);
-    SharedAddr = CGF.EmitLoadOfPointer(
+    OrigAddr = CGF.EmitLoadOfPointer(
         SharedAddr,
         CGM.getContext().VoidPtrTy.castAs<PointerType>()->getTypePtr());
-    OrigLVal = CGF.MakeAddrLValue(SharedAddr, CGM.getContext().VoidPtrTy);
-  } else {
-    OrigLVal = CGF.MakeNaturalAlignAddrLValue(
-        llvm::ConstantPointerNull::get(CGM.VoidPtrTy),
-        CGM.getContext().VoidPtrTy);
   }
   // Emit the initializer:
   // %0 = bitcast void* %arg to <type>*
   // store <type> <init>, <type>* %0
-  RCG.emitInitialization(CGF, N, PrivateAddr, OrigLVal,
+  RCG.emitInitialization(CGF, N, PrivateAddr, OrigAddr,
                          [](CodeGenFunction &) { return false; });
   CGF.FinishFunction();
   return Fn;
@@ -6122,7 +6097,7 @@ llvm::Value *CGOpenMPRuntime::emitTaskReductionInit(
     llvm::Value *Idxs[] = {llvm::ConstantInt::get(CGM.SizeTy, /*V=*/0),
                            llvm::ConstantInt::get(CGM.SizeTy, Cnt)};
     llvm::Value *GEP = CGF.EmitCheckedInBoundsGEP(
-        TaskRedInput.getPointer(), Idxs,
+        TaskRedInput.getElementType(), TaskRedInput.getPointer(), Idxs,
         /*SignedIndices=*/false, /*IsSubtraction=*/false, Loc,
         ".rd_input.gep.");
     LValue ElemLVal = CGF.MakeNaturalAlignAddrLValue(GEP, RDType);
@@ -6620,6 +6595,8 @@ void CGOpenMPRuntime::emitTargetOutlinedFunctionHelper(
     OutlinedFn->addFnAttr("omp_target_thread_limit",
                           std::to_string(DefaultValThreads));
   }
+
+  CGM.getTargetCodeGenInfo().setTargetAttributes(nullptr, OutlinedFn, CGM);
 }
 
 /// Checks if the expression is constant or does not have non-trivial function
@@ -12680,12 +12657,11 @@ void CGOpenMPRuntime::emitLastprivateConditionalUpdate(CodeGenFunction &CGF,
 
   // Last value of the lastprivate conditional.
   // decltype(priv_a) last_a;
-  llvm::Constant *Last = getOrCreateInternalVariable(
+  llvm::GlobalVariable *Last = getOrCreateInternalVariable(
       CGF.ConvertTypeForMem(LVal.getType()), UniqueDeclName);
-  cast<llvm::GlobalVariable>(Last)->setAlignment(
-      LVal.getAlignment().getAsAlign());
-  LValue LastLVal =
-      CGF.MakeAddrLValue(Last, LVal.getType(), LVal.getAlignment());
+  Last->setAlignment(LVal.getAlignment().getAsAlign());
+  LValue LastLVal = CGF.MakeAddrLValue(
+      Address(Last, Last->getValueType(), LVal.getAlignment()), LVal.getType());
 
   // Global loop counter. Required to handle inner parallel-for regions.
   // iv
@@ -12812,7 +12788,7 @@ void CGOpenMPRuntime::checkAndEmitSharedLastprivateConditional(
   const CapturedStmt *CS = D.getCapturedStmt(CaptureRegions.back());
   for (const auto &Pair : It->DeclToUniqueName) {
     const auto *VD = cast<VarDecl>(Pair.first->getCanonicalDecl());
-    if (!CS->capturesVariable(VD) || IgnoredDecls.count(VD) > 0)
+    if (!CS->capturesVariable(VD) || IgnoredDecls.contains(VD))
       continue;
     auto I = LPCI->getSecond().find(Pair.first);
     assert(I != LPCI->getSecond().end() &&
@@ -12858,7 +12834,8 @@ void CGOpenMPRuntime::emitLastprivateConditionalFinalUpdate(
   if (!GV)
     return;
   LValue LPLVal = CGF.MakeAddrLValue(
-      GV, PrivLVal.getType().getNonReferenceType(), PrivLVal.getAlignment());
+      Address(GV, GV->getValueType(), PrivLVal.getAlignment()),
+      PrivLVal.getType().getNonReferenceType());
   llvm::Value *Res = CGF.EmitLoadOfScalar(LPLVal, Loc);
   CGF.EmitStoreOfScalar(Res, PrivLVal);
 }
@@ -12887,7 +12864,8 @@ void CGOpenMPSIMDRuntime::emitParallelCall(CodeGenFunction &CGF,
                                            SourceLocation Loc,
                                            llvm::Function *OutlinedFn,
                                            ArrayRef<llvm::Value *> CapturedVars,
-                                           const Expr *IfCond) {
+                                           const Expr *IfCond,
+                                           llvm::Value *NumThreads) {
   llvm_unreachable("Not supported in SIMD-only mode");
 }
 
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.h b/clang/lib/CodeGen/CGOpenMPRuntime.h
index 527a23a8af6a..b83ec78696d1 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.h
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.h
@@ -162,10 +162,10 @@ private:
   /// Performs aggregate initialization.
   /// \param N Number of reduction item in the common list.
   /// \param PrivateAddr Address of the corresponding private item.
-  /// \param SharedLVal Address of the original shared variable.
+  /// \param SharedAddr Address of the original shared variable.
   /// \param DRD Declare reduction construct used for reduction item.
   void emitAggregateInitialization(CodeGenFunction &CGF, unsigned N,
-                                   Address PrivateAddr, LValue SharedLVal,
+                                   Address PrivateAddr, Address SharedAddr,
                                    const OMPDeclareReductionDecl *DRD);
 
 public:
@@ -187,10 +187,10 @@ public:
   /// \param PrivateAddr Address of the corresponding private item.
   /// \param DefaultInit Default initialization sequence that should be
   /// performed if no reduction specific initialization is found.
-  /// \param SharedLVal Address of the original shared variable.
+  /// \param SharedAddr Address of the original shared variable.
   void
   emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr,
-                     LValue SharedLVal,
+                     Address SharedAddr,
                      llvm::function_ref<bool(CodeGenFunction &)> DefaultInit);
   /// Returns true if the private copy requires cleanups.
   bool needCleanups(unsigned N);
@@ -471,8 +471,8 @@ private:
   /// <critical_section_name> + ".var" for "omp critical" directives; 2)
   /// <mangled_name_for_global_var> + ".cache." for cache for threadprivate
   /// variables.
-  llvm::StringMap<llvm::AssertingVH<llvm::Constant>, llvm::BumpPtrAllocator>
-      InternalVars;
+  llvm::StringMap<llvm::AssertingVH<llvm::GlobalVariable>,
+                  llvm::BumpPtrAllocator> InternalVars;
   /// Type typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *);
   llvm::Type *KmpRoutineEntryPtrTy = nullptr;
   QualType KmpRoutineEntryPtrQTy;
@@ -829,9 +829,9 @@ private:
   /// \param Ty Type of the global variable. If it is exist already the type
   /// must be the same.
   /// \param Name Name of the variable.
-  llvm::Constant *getOrCreateInternalVariable(llvm::Type *Ty,
-                                              const llvm::Twine &Name,
-                                              unsigned AddressSpace = 0);
+  llvm::GlobalVariable *getOrCreateInternalVariable(llvm::Type *Ty,
+                                                    const llvm::Twine &Name,
+                                                    unsigned AddressSpace = 0);
 
   /// Set of threadprivate variables with the generated initializer.
   llvm::StringSet<> ThreadPrivateWithDefinition;
@@ -1015,11 +1015,13 @@ public:
   /// variables used in \a OutlinedFn function.
   /// \param IfCond Condition in the associated 'if' clause, if it was
   /// specified, nullptr otherwise.
+  /// \param NumThreads The value corresponding to the num_threads clause, if
+  /// any, or nullptr.
   ///
   virtual void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
                                 llvm::Function *OutlinedFn,
                                 ArrayRef<llvm::Value *> CapturedVars,
-                                const Expr *IfCond);
+                                const Expr *IfCond, llvm::Value *NumThreads);
 
   /// Emits a critical region.
   /// \param CriticalName Name of the critical region.
@@ -1991,11 +1993,13 @@ public:
   /// variables used in \a OutlinedFn function.
   /// \param IfCond Condition in the associated 'if' clause, if it was
   /// specified, nullptr otherwise.
+  /// \param NumThreads The value corresponding to the num_threads clause, if
+  /// any, or nullptr.
   ///
   void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
                         llvm::Function *OutlinedFn,
                         ArrayRef<llvm::Value *> CapturedVars,
-                        const Expr *IfCond) override;
+                        const Expr *IfCond, llvm::Value *NumThreads) override;
 
   /// Emits a critical region.
   /// \param CriticalName Name of the critical region.
diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
index dcb224f33156..866454ddeaed 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
@@ -1221,11 +1221,7 @@ void CGOpenMPRuntimeGPU::emitProcBindClause(CodeGenFunction &CGF,
 void CGOpenMPRuntimeGPU::emitNumThreadsClause(CodeGenFunction &CGF,
                                                 llvm::Value *NumThreads,
                                                 SourceLocation Loc) {
-  // Do nothing in case of SPMD mode and L0 parallel.
-  if (getExecutionMode() == CGOpenMPRuntimeGPU::EM_SPMD)
-    return;
-
-  CGOpenMPRuntime::emitNumThreadsClause(CGF, NumThreads, Loc);
+  // Nothing to do.
 }
 
 void CGOpenMPRuntimeGPU::emitNumTeamsClause(CodeGenFunction &CGF,
@@ -1510,13 +1506,16 @@ void CGOpenMPRuntimeGPU::emitParallelCall(CodeGenFunction &CGF,
                                           SourceLocation Loc,
                                           llvm::Function *OutlinedFn,
                                           ArrayRef<llvm::Value *> CapturedVars,
-                                          const Expr *IfCond) {
+                                          const Expr *IfCond,
+                                          llvm::Value *NumThreads) {
   if (!CGF.HaveInsertPoint())
     return;
 
-  auto &&ParallelGen = [this, Loc, OutlinedFn, CapturedVars,
-                        IfCond](CodeGenFunction &CGF, PrePostActionTy &Action) {
+  auto &&ParallelGen = [this, Loc, OutlinedFn, CapturedVars, IfCond,
+                        NumThreads](CodeGenFunction &CGF,
+                                    PrePostActionTy &Action) {
     CGBuilderTy &Bld = CGF.Builder;
+    llvm::Value *NumThreadsVal = NumThreads;
     llvm::Function *WFn = WrapperFunctionsMap[OutlinedFn];
     llvm::Value *ID = llvm::ConstantPointerNull::get(CGM.Int8PtrTy);
     if (WFn)
@@ -1556,13 +1555,18 @@ void CGOpenMPRuntimeGPU::emitParallelCall(CodeGenFunction &CGF,
     else
       IfCondVal = llvm::ConstantInt::get(CGF.Int32Ty, 1);
 
-    assert(IfCondVal && "Expected a value");
+    if (!NumThreadsVal)
+      NumThreadsVal = llvm::ConstantInt::get(CGF.Int32Ty, -1);
+    else
+      NumThreadsVal = Bld.CreateZExtOrTrunc(NumThreadsVal, CGF.Int32Ty),
+
+      assert(IfCondVal && "Expected a value");
     llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
     llvm::Value *Args[] = {
         RTLoc,
         getThreadID(CGF, Loc),
         IfCondVal,
-        llvm::ConstantInt::get(CGF.Int32Ty, -1),
+        NumThreadsVal,
         llvm::ConstantInt::get(CGF.Int32Ty, -1),
         FnPtr,
         ID,
@@ -2186,11 +2190,8 @@ static llvm::Value *emitInterWarpCopyFunction(CodeGenModule &CGM,
       // elemptr = ((CopyType*)(elemptrptr)) + I
       Address ElemPtr = Address(ElemPtrPtr, Align);
       ElemPtr = Bld.CreateElementBitCast(ElemPtr, CopyType);
-      if (NumIters > 1) {
-        ElemPtr = Address(Bld.CreateGEP(ElemPtr.getElementType(),
-                                        ElemPtr.getPointer(), Cnt),
-                          ElemPtr.getAlignment());
-      }
+      if (NumIters > 1)
+        ElemPtr = Bld.CreateGEP(ElemPtr, Cnt);
 
       // Get pointer to location in transfer medium.
       // MediumPtr = &medium[warp_id]
@@ -2256,11 +2257,8 @@ static llvm::Value *emitInterWarpCopyFunction(CodeGenModule &CGM,
           TargetElemPtrPtr, /*Volatile=*/false, C.VoidPtrTy, Loc);
       Address TargetElemPtr = Address(TargetElemPtrVal, Align);
       TargetElemPtr = Bld.CreateElementBitCast(TargetElemPtr, CopyType);
-      if (NumIters > 1) {
-        TargetElemPtr = Address(Bld.CreateGEP(TargetElemPtr.getElementType(),
-                                              TargetElemPtr.getPointer(), Cnt),
-                                TargetElemPtr.getAlignment());
-      }
+      if (NumIters > 1)
+        TargetElemPtr = Bld.CreateGEP(TargetElemPtr, Cnt);
 
       // *TargetElemPtr = SrcMediumVal;
       llvm::Value *SrcMediumValue =
@@ -3899,6 +3897,7 @@ void CGOpenMPRuntimeGPU::processRequiresDirective(
       case CudaArch::GFX1033:
       case CudaArch::GFX1034:
       case CudaArch::GFX1035:
+      case CudaArch::Generic:
       case CudaArch::UNUSED:
       case CudaArch::UNKNOWN:
         break;
diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.h b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.h
index ac51264d7685..1d30c5061743 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.h
+++ b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.h
@@ -257,10 +257,13 @@ public:
   /// variables used in \a OutlinedFn function.
   /// \param IfCond Condition in the associated 'if' clause, if it was
   /// specified, nullptr otherwise.
+  /// \param NumThreads The value corresponding to the num_threads clause, if
+  /// any,
+  ///                   or nullptr.
   void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
                         llvm::Function *OutlinedFn,
                         ArrayRef<llvm::Value *> CapturedVars,
-                        const Expr *IfCond) override;
+                        const Expr *IfCond, llvm::Value *NumThreads) override;
 
   /// Emit an implicit/explicit barrier for OpenMP threads.
   /// \param Kind Directive for which this implicit barrier call must be
diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp
index d399ff919cc3..ef0068cd3b0c 100644
--- a/clang/lib/CodeGen/CGStmt.cpp
+++ b/clang/lib/CodeGen/CGStmt.cpp
@@ -2454,7 +2454,7 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
     const ABIArgInfo &RetAI = CurFnInfo->getReturnInfo();
     if (RetAI.isDirect() || RetAI.isExtend()) {
       // Make a fake lvalue for the return value slot.
-      LValue ReturnSlot = MakeAddrLValue(ReturnValue, FnRetTy);
+      LValue ReturnSlot = MakeAddrLValueWithoutTBAA(ReturnValue, FnRetTy);
       CGM.getTargetCodeGenInfo().addReturnRegisterOutputs(
           *this, ReturnSlot, Constraints, ResultRegTypes, ResultTruncRegTypes,
           ResultRegDests, AsmString, S.getNumOutputs());
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index f6853a22cd36..4c11f7d67534 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -24,10 +24,13 @@
 #include "clang/AST/StmtVisitor.h"
 #include "clang/Basic/OpenMPKinds.h"
 #include "clang/Basic/PrettyStackTrace.h"
+#include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/Frontend/OpenMP/OMPConstants.h"
 #include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
 #include "llvm/IR/Constants.h"
+#include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/IR/Instructions.h"
+#include "llvm/IR/Metadata.h"
 #include "llvm/Support/AtomicOrdering.h"
 using namespace clang;
 using namespace CodeGen;
@@ -375,8 +378,7 @@ static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc,
       AddrLV.getAddress(CGF).getPointer(), Ctx.getUIntPtrType(),
       Ctx.getPointerType(DstType), Loc);
   Address TmpAddr =
-      CGF.MakeNaturalAlignAddrLValue(CastedPtr, Ctx.getPointerType(DstType))
-          .getAddress(CGF);
+      CGF.MakeNaturalAlignAddrLValue(CastedPtr, DstType).getAddress(CGF);
   return TmpAddr;
 }
 
@@ -1245,7 +1247,7 @@ void CodeGenFunction::EmitOMPReductionClauseInit(
     RedCG.emitAggregateType(*this, Count);
     AutoVarEmission Emission = EmitAutoVarAlloca(*PrivateVD);
     RedCG.emitInitialization(*this, Count, Emission.getAllocatedAddress(),
-                             RedCG.getSharedLValue(Count),
+                             RedCG.getSharedLValue(Count).getAddress(*this),
                              [&Emission](CodeGenFunction &CGF) {
                                CGF.EmitAutoVarInit(Emission);
                                return true;
@@ -1557,14 +1559,14 @@ static void emitCommonOMPParallelDirective(
     OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
     const CodeGenBoundParametersTy &CodeGenBoundParameters) {
   const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
+  llvm::Value *NumThreads = nullptr;
   llvm::Function *OutlinedFn =
       CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction(
           S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
   if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) {
     CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
-    llvm::Value *NumThreads =
-        CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
-                           /*IgnoreResultAssign=*/true);
+    NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
+                                    /*IgnoreResultAssign=*/true);
     CGF.CGM.getOpenMPRuntime().emitNumThreadsClause(
         CGF, NumThreads, NumThreadsClause->getBeginLoc());
   }
@@ -1591,7 +1593,7 @@ static void emitCommonOMPParallelDirective(
   CodeGenBoundParameters(CGF, S, CapturedVars);
   CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
   CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getBeginLoc(), OutlinedFn,
-                                              CapturedVars, IfCond);
+                                              CapturedVars, IfCond, NumThreads);
 }
 
 static bool isAllocatableDecl(const VarDecl *VD) {
@@ -1972,7 +1974,7 @@ CodeGenFunction::EmitOMPCollapsedCanonicalLoopNest(const Stmt *S, int Depth) {
 
   // Pop the \p Depth loops requested by the call from that stack and restore
   // the previous context.
-  OMPLoopNestStack.set_size(OMPLoopNestStack.size() - Depth);
+  OMPLoopNestStack.pop_back_n(Depth);
   ExpectedOMPLoopDepth = ParentExpectedOMPLoopDepth;
 
   return Result;
@@ -4299,10 +4301,10 @@ public:
           PrivateDecls.push_back(VD);
     }
   }
-  void VisitOMPExecutableDirective(const OMPExecutableDirective *) { return; }
-  void VisitCapturedStmt(const CapturedStmt *) { return; }
-  void VisitLambdaExpr(const LambdaExpr *) { return; }
-  void VisitBlockExpr(const BlockExpr *) { return; }
+  void VisitOMPExecutableDirective(const OMPExecutableDirective *) {}
+  void VisitCapturedStmt(const CapturedStmt *) {}
+  void VisitLambdaExpr(const LambdaExpr *) {}
+  void VisitBlockExpr(const BlockExpr *) {}
   void VisitStmt(const Stmt *S) {
     if (!S)
       return;
@@ -4431,6 +4433,53 @@ void CodeGenFunction::EmitOMPTaskBasedDirective(
         UntiedLocalVars;
     // Set proper addresses for generated private copies.
     OMPPrivateScope Scope(CGF);
+    // Generate debug info for variables present in shared clause.
+    if (auto *DI = CGF.getDebugInfo()) {
+      llvm::SmallDenseMap<const VarDecl *, FieldDecl *> CaptureFields =
+          CGF.CapturedStmtInfo->getCaptureFields();
+      llvm::Value *ContextValue = CGF.CapturedStmtInfo->getContextValue();
+      if (CaptureFields.size() && ContextValue) {
+        unsigned CharWidth = CGF.getContext().getCharWidth();
+        // The shared variables are packed together as members of structure.
+        // So the address of each shared variable can be computed by adding
+        // offset of it (within record) to the base address of record. For each
+        // shared variable, debug intrinsic llvm.dbg.declare is generated with
+        // appropriate expressions (DIExpression).
+        // Ex:
+        //  %12 = load %struct.anon*, %struct.anon** %__context.addr.i
+        //  call void @llvm.dbg.declare(metadata %struct.anon* %12,
+        //            metadata !svar1,
+        //            metadata !DIExpression(DW_OP_deref))
+        //  call void @llvm.dbg.declare(metadata %struct.anon* %12,
+        //            metadata !svar2,
+        //            metadata !DIExpression(DW_OP_plus_uconst, 8, DW_OP_deref))
+        for (auto It = CaptureFields.begin(); It != CaptureFields.end(); ++It) {
+          const VarDecl *SharedVar = It->first;
+          RecordDecl *CaptureRecord = It->second->getParent();
+          const ASTRecordLayout &Layout =
+              CGF.getContext().getASTRecordLayout(CaptureRecord);
+          unsigned Offset =
+              Layout.getFieldOffset(It->second->getFieldIndex()) / CharWidth;
+          (void)DI->EmitDeclareOfAutoVariable(SharedVar, ContextValue,
+                                              CGF.Builder, false);
+          llvm::Instruction &Last = CGF.Builder.GetInsertBlock()->back();
+          // Get the call dbg.declare instruction we just created and update
+          // its DIExpression to add offset to base address.
+          if (auto DDI = dyn_cast<llvm::DbgVariableIntrinsic>(&Last)) {
+            SmallVector<uint64_t, 8> Ops;
+            // Add offset to the base address if non zero.
+            if (Offset) {
+              Ops.push_back(llvm::dwarf::DW_OP_plus_uconst);
+              Ops.push_back(Offset);
+            }
+            Ops.push_back(llvm::dwarf::DW_OP_deref);
+            auto &Ctx = DDI->getContext();
+            llvm::DIExpression *DIExpr = llvm::DIExpression::get(Ctx, Ops);
+            Last.setOperand(2, llvm::MetadataAsValue::get(Ctx, DIExpr));
+          }
+        }
+      }
+    }
     llvm::SmallVector<std::pair<const VarDecl *, Address>, 16> FirstprivatePtrs;
     if (!Data.PrivateVars.empty() || !Data.FirstprivateVars.empty() ||
         !Data.LastprivateVars.empty() || !Data.PrivateLocals.empty()) {
@@ -5918,6 +5967,9 @@ static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind,
     emitOMPAtomicCaptureExpr(CGF, AO, IsPostfixUpdate, V, X, E, UE,
                              IsXLHSInRHSPart, Loc);
     break;
+  case OMPC_compare:
+    // Do nothing here as we already emit an error.
+    break;
   case OMPC_if:
   case OMPC_final:
   case OMPC_num_threads:
diff --git a/clang/lib/CodeGen/CGValue.h b/clang/lib/CodeGen/CGValue.h
index 4b39a0520833..f01eece042f8 100644
--- a/clang/lib/CodeGen/CGValue.h
+++ b/clang/lib/CodeGen/CGValue.h
@@ -47,6 +47,8 @@ class RValue {
   llvm::PointerIntPair<llvm::Value *, 2, Flavor> V1;
   // Stores second value and volatility.
   llvm::PointerIntPair<llvm::Value *, 1, bool> V2;
+  // Stores element type for aggregate values.
+  llvm::Type *ElementType;
 
 public:
   bool isScalar() const { return V1.getInt() == Scalar; }
@@ -71,7 +73,8 @@ public:
   Address getAggregateAddress() const {
     assert(isAggregate() && "Not an aggregate!");
     auto align = reinterpret_cast<uintptr_t>(V2.getPointer()) >> AggAlignShift;
-    return Address(V1.getPointer(), CharUnits::fromQuantity(align));
+    return Address(
+        V1.getPointer(), ElementType, CharUnits::fromQuantity(align));
   }
   llvm::Value *getAggregatePointer() const {
     assert(isAggregate() && "Not an aggregate!");
@@ -108,6 +111,7 @@ public:
     RValue ER;
     ER.V1.setPointer(addr.getPointer());
     ER.V1.setInt(Aggregate);
+    ER.ElementType = addr.getElementType();
 
     auto align = static_cast<uintptr_t>(addr.getAlignment().getQuantity());
     ER.V2.setPointer(reinterpret_cast<llvm::Value*>(align << AggAlignShift));
@@ -175,6 +179,7 @@ class LValue {
   } LVType;
 
   llvm::Value *V;
+  llvm::Type *ElementType;
 
   union {
     // Index into a vector subscript: V[i]
@@ -230,6 +235,13 @@ private:
                   LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo) {
     assert((!Alignment.isZero() || Type->isIncompleteType()) &&
            "initializing l-value with zero alignment!");
+    if (isGlobalReg())
+      assert(ElementType == nullptr && "Global reg does not store elem type");
+    else
+      assert(llvm::cast<llvm::PointerType>(V->getType())
+                 ->isOpaqueOrPointeeTypeMatches(ElementType) &&
+             "Pointer element type mismatch");
+
     this->Type = Type;
     this->Quals = Quals;
     const unsigned MaxAlign = 1U << 31;
@@ -327,17 +339,18 @@ public:
     return V;
   }
   Address getAddress(CodeGenFunction &CGF) const {
-    return Address(getPointer(CGF), getAlignment());
+    return Address(getPointer(CGF), ElementType, getAlignment());
   }
   void setAddress(Address address) {
     assert(isSimple());
     V = address.getPointer();
+    ElementType = address.getElementType();
     Alignment = address.getAlignment().getQuantity();
   }
 
   // vector elt lvalue
   Address getVectorAddress() const {
-    return Address(getVectorPointer(), getAlignment());
+    return Address(getVectorPointer(), ElementType, getAlignment());
   }
   llvm::Value *getVectorPointer() const {
     assert(isVectorElt());
@@ -349,7 +362,7 @@ public:
   }
 
   Address getMatrixAddress() const {
-    return Address(getMatrixPointer(), getAlignment());
+    return Address(getMatrixPointer(), ElementType, getAlignment());
   }
   llvm::Value *getMatrixPointer() const {
     assert(isMatrixElt());
@@ -362,7 +375,7 @@ public:
 
   // extended vector elements.
   Address getExtVectorAddress() const {
-    return Address(getExtVectorPointer(), getAlignment());
+    return Address(getExtVectorPointer(), ElementType, getAlignment());
   }
   llvm::Value *getExtVectorPointer() const {
     assert(isExtVectorElt());
@@ -375,7 +388,7 @@ public:
 
   // bitfield lvalue
   Address getBitFieldAddress() const {
-    return Address(getBitFieldPointer(), getAlignment());
+    return Address(getBitFieldPointer(), ElementType, getAlignment());
   }
   llvm::Value *getBitFieldPointer() const { assert(isBitField()); return V; }
   const CGBitFieldInfo &getBitFieldInfo() const {
@@ -395,6 +408,7 @@ public:
     R.LVType = Simple;
     assert(address.getPointer()->getType()->isPointerTy());
     R.V = address.getPointer();
+    R.ElementType = address.getElementType();
     R.Initialize(type, qs, address.getAlignment(), BaseInfo, TBAAInfo);
     return R;
   }
@@ -405,6 +419,7 @@ public:
     LValue R;
     R.LVType = VectorElt;
     R.V = vecAddress.getPointer();
+    R.ElementType = vecAddress.getElementType();
     R.VectorIdx = Idx;
     R.Initialize(type, type.getQualifiers(), vecAddress.getAlignment(),
                  BaseInfo, TBAAInfo);
@@ -417,6 +432,7 @@ public:
     LValue R;
     R.LVType = ExtVectorElt;
     R.V = vecAddress.getPointer();
+    R.ElementType = vecAddress.getElementType();
     R.VectorElts = Elts;
     R.Initialize(type, type.getQualifiers(), vecAddress.getAlignment(),
                  BaseInfo, TBAAInfo);
@@ -435,17 +451,20 @@ public:
     LValue R;
     R.LVType = BitField;
     R.V = Addr.getPointer();
+    R.ElementType = Addr.getElementType();
     R.BitFieldInfo = &Info;
     R.Initialize(type, type.getQualifiers(), Addr.getAlignment(), BaseInfo,
                  TBAAInfo);
     return R;
   }
 
-  static LValue MakeGlobalReg(Address Reg, QualType type) {
+  static LValue MakeGlobalReg(llvm::Value *V, CharUnits alignment,
+                              QualType type) {
     LValue R;
     R.LVType = GlobalReg;
-    R.V = Reg.getPointer();
-    R.Initialize(type, type.getQualifiers(), Reg.getAlignment(),
+    R.V = V;
+    R.ElementType = nullptr;
+    R.Initialize(type, type.getQualifiers(), alignment,
                  LValueBaseInfo(AlignmentSource::Decl), TBAAAccessInfo());
     return R;
   }
@@ -456,6 +475,7 @@ public:
     LValue R;
     R.LVType = MatrixElt;
     R.V = matAddress.getPointer();
+    R.ElementType = matAddress.getElementType();
     R.VectorIdx = Idx;
     R.Initialize(type, type.getQualifiers(), matAddress.getAlignment(),
                  BaseInfo, TBAAInfo);
@@ -470,13 +490,11 @@ public:
 /// An aggregate value slot.
 class AggValueSlot {
   /// The address.
-  llvm::Value *Addr;
+  Address Addr;
 
   // Qualifiers
   Qualifiers Quals;
 
-  unsigned Alignment;
-
   /// DestructedFlag - This is set to true if some external code is
   /// responsible for setting up a destructor for the slot.  Otherwise
   /// the code which constructs it should push the appropriate cleanup.
@@ -520,6 +538,14 @@ class AggValueSlot {
   /// them.
   bool SanitizerCheckedFlag : 1;
 
+  AggValueSlot(Address Addr, Qualifiers Quals, bool DestructedFlag,
+               bool ObjCGCFlag, bool ZeroedFlag, bool AliasedFlag,
+               bool OverlapFlag, bool SanitizerCheckedFlag)
+      : Addr(Addr), Quals(Quals), DestructedFlag(DestructedFlag),
+        ObjCGCFlag(ObjCGCFlag), ZeroedFlag(ZeroedFlag),
+        AliasedFlag(AliasedFlag), OverlapFlag(OverlapFlag),
+        SanitizerCheckedFlag(SanitizerCheckedFlag) {}
+
 public:
   enum IsAliased_t { IsNotAliased, IsAliased };
   enum IsDestructed_t { IsNotDestructed, IsDestructed };
@@ -553,22 +579,8 @@ public:
                               Overlap_t mayOverlap,
                               IsZeroed_t isZeroed = IsNotZeroed,
                        IsSanitizerChecked_t isChecked = IsNotSanitizerChecked) {
-    AggValueSlot AV;
-    if (addr.isValid()) {
-      AV.Addr = addr.getPointer();
-      AV.Alignment = addr.getAlignment().getQuantity();
-    } else {
-      AV.Addr = nullptr;
-      AV.Alignment = 0;
-    }
-    AV.Quals = quals;
-    AV.DestructedFlag = isDestructed;
-    AV.ObjCGCFlag = needsGC;
-    AV.ZeroedFlag = isZeroed;
-    AV.AliasedFlag = isAliased;
-    AV.OverlapFlag = mayOverlap;
-    AV.SanitizerCheckedFlag = isChecked;
-    return AV;
+    return AggValueSlot(addr, quals, isDestructed, needsGC, isZeroed, isAliased,
+                        mayOverlap, isChecked);
   }
 
   static AggValueSlot
@@ -609,19 +621,19 @@ public:
   }
 
   llvm::Value *getPointer() const {
-    return Addr;
+    return Addr.getPointer();
   }
 
   Address getAddress() const {
-    return Address(Addr, getAlignment());
+    return Addr;
   }
 
   bool isIgnored() const {
-    return Addr == nullptr;
+    return !Addr.isValid();
   }
 
   CharUnits getAlignment() const {
-    return CharUnits::fromQuantity(Alignment);
+    return Addr.getAlignment();
   }
 
   IsAliased_t isPotentiallyAliased() const {
diff --git a/clang/lib/CodeGen/CodeGenAction.cpp b/clang/lib/CodeGen/CodeGenAction.cpp
index 52c54d3c7a72..b72b16cf2b5f 100644
--- a/clang/lib/CodeGen/CodeGenAction.cpp
+++ b/clang/lib/CodeGen/CodeGenAction.cpp
@@ -571,7 +571,6 @@ void BackendConsumer::SrcMgrDiagHandler(const llvm::DiagnosticInfoSrcMgr &DI) {
   // If Loc is invalid, we still need to report the issue, it just gets no
   // location info.
   Diags.Report(Loc, DiagID).AddString(Message);
-  return;
 }
 
 bool
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index d87cf2d49720..e6adec6948af 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -188,8 +188,8 @@ LValue CodeGenFunction::MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T) {
   LValueBaseInfo BaseInfo;
   TBAAAccessInfo TBAAInfo;
   CharUnits Alignment = CGM.getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo);
-  return LValue::MakeAddr(Address(V, Alignment), T, getContext(), BaseInfo,
-                          TBAAInfo);
+  Address Addr(V, ConvertTypeForMem(T), Alignment);
+  return LValue::MakeAddr(Addr, T, getContext(), BaseInfo, TBAAInfo);
 }
 
 /// Given a value of type T* that may not be to a complete object,
@@ -200,7 +200,8 @@ CodeGenFunction::MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T) {
   TBAAAccessInfo TBAAInfo;
   CharUnits Align = CGM.getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo,
                                                 /* forPointeeType= */ true);
-  return MakeAddrLValue(Address(V, Align), T, BaseInfo, TBAAInfo);
+  Address Addr(V, ConvertTypeForMem(T), Align);
+  return MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo);
 }
 
 
@@ -243,7 +244,7 @@ TypeEvaluationKind CodeGenFunction::getEvaluationKind(QualType type) {
     case Type::Enum:
     case Type::ObjCObjectPointer:
     case Type::Pipe:
-    case Type::ExtInt:
+    case Type::BitInt:
       return TEK_Scalar;
 
     // Complexes.
@@ -1070,7 +1071,8 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
     auto AI = CurFn->arg_begin();
     if (CurFnInfo->getReturnInfo().isSRetAfterThis())
       ++AI;
-    ReturnValue = Address(&*AI, CurFnInfo->getReturnInfo().getIndirectAlign());
+    ReturnValue = Address(&*AI, ConvertType(RetTy),
+                          CurFnInfo->getReturnInfo().getIndirectAlign());
     if (!CurFnInfo->getReturnInfo().getIndirectByVal()) {
       ReturnValuePointer =
           CreateDefaultAlignTempAlloca(Int8PtrTy, "result.ptr");
@@ -1298,47 +1300,44 @@ QualType CodeGenFunction::BuildFunctionArgList(GlobalDecl GD,
 
 void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn,
                                    const CGFunctionInfo &FnInfo) {
+  assert(Fn && "generating code for null Function");
   const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
   CurGD = GD;
 
   FunctionArgList Args;
   QualType ResTy = BuildFunctionArgList(GD, Args);
 
-  // When generating code for a builtin with an inline declaration, use a
-  // mangled name to hold the actual body, while keeping an external definition
-  // in case the function pointer is referenced somewhere.
-  if (Fn) {
-    if (FD->isInlineBuiltinDeclaration()) {
-      std::string FDInlineName = (Fn->getName() + ".inline").str();
-      llvm::Module *M = Fn->getParent();
-      llvm::Function *Clone = M->getFunction(FDInlineName);
-      if (!Clone) {
-        Clone = llvm::Function::Create(Fn->getFunctionType(),
-                                       llvm::GlobalValue::InternalLinkage,
-                                       Fn->getAddressSpace(), FDInlineName, M);
-        Clone->addFnAttr(llvm::Attribute::AlwaysInline);
-      }
-      Fn->setLinkage(llvm::GlobalValue::ExternalLinkage);
-      Fn = Clone;
+  if (FD->isInlineBuiltinDeclaration()) {
+    // When generating code for a builtin with an inline declaration, use a
+    // mangled name to hold the actual body, while keeping an external
+    // definition in case the function pointer is referenced somewhere.
+    std::string FDInlineName = (Fn->getName() + ".inline").str();
+    llvm::Module *M = Fn->getParent();
+    llvm::Function *Clone = M->getFunction(FDInlineName);
+    if (!Clone) {
+      Clone = llvm::Function::Create(Fn->getFunctionType(),
+                                     llvm::GlobalValue::InternalLinkage,
+                                     Fn->getAddressSpace(), FDInlineName, M);
+      Clone->addFnAttr(llvm::Attribute::AlwaysInline);
     }
-
+    Fn->setLinkage(llvm::GlobalValue::ExternalLinkage);
+    Fn = Clone;
+  } else {
     // Detect the unusual situation where an inline version is shadowed by a
     // non-inline version. In that case we should pick the external one
     // everywhere. That's GCC behavior too. Unfortunately, I cannot find a way
     // to detect that situation before we reach codegen, so do some late
     // replacement.
-    else {
-      for (const FunctionDecl *PD = FD->getPreviousDecl(); PD;
-           PD = PD->getPreviousDecl()) {
-        if (LLVM_UNLIKELY(PD->isInlineBuiltinDeclaration())) {
-          std::string FDInlineName = (Fn->getName() + ".inline").str();
-          llvm::Module *M = Fn->getParent();
-          if (llvm::Function *Clone = M->getFunction(FDInlineName)) {
-            Clone->replaceAllUsesWith(Fn);
-            Clone->eraseFromParent();
-          }
-          break;
+    for (const FunctionDecl *PD = FD->getPreviousDecl(); PD;
+         PD = PD->getPreviousDecl()) {
+      if (LLVM_UNLIKELY(PD->isInlineBuiltinDeclaration())) {
+        std::string FDInlineName = (Fn->getName() + ".inline").str();
+        llvm::Module *M = Fn->getParent();
+        if (llvm::Function *Clone = M->getFunction(FDInlineName)) {
+          Clone->replaceAllUsesWith(Fn);
+          Clone->eraseFromParent();
         }
+        break;
       }
     }
   }
@@ -1347,8 +1346,7 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn,
   if (FD->hasAttr<NoDebugAttr>()) {
     // Clear non-distinct debug info that was possibly attached to the function
     // due to an earlier declaration without the nodebug attribute
-    if (Fn)
-      Fn->setSubprogram(nullptr);
+    Fn->setSubprogram(nullptr);
     // Disable debug info indefinitely for this function
     DebugInfo = nullptr;
   }
@@ -2202,12 +2200,13 @@ void CodeGenFunction::EmitVariablyModifiedType(QualType type) {
     case Type::Record:
     case Type::Enum:
     case Type::Elaborated:
+    case Type::Using:
     case Type::TemplateSpecialization:
     case Type::ObjCTypeParam:
     case Type::ObjCObject:
     case Type::ObjCInterface:
     case Type::ObjCObjectPointer:
-    case Type::ExtInt:
+    case Type::BitInt:
       llvm_unreachable("type class is never variably-modified!");
 
     case Type::Adjusted:
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index ff5b6634da1c..f76ce8a6400d 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -459,6 +459,11 @@ public:
     /// Get the name of the capture helper.
     virtual StringRef getHelperName() const { return "__captured_stmt"; }
 
+    /// Get the CaptureFields
+    llvm::SmallDenseMap<const VarDecl *, FieldDecl *> getCaptureFields() {
+      return CaptureFields;
+    }
+
   private:
     /// The kind of captured statement being generated.
     CapturedRegionKind Kind;
@@ -2494,14 +2499,16 @@ public:
 
   LValue MakeAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment,
                         AlignmentSource Source = AlignmentSource::Type) {
-    return LValue::MakeAddr(Address(V, Alignment), T, getContext(),
-                            LValueBaseInfo(Source), CGM.getTBAAAccessInfo(T));
+    Address Addr(V, ConvertTypeForMem(T), Alignment);
+    return LValue::MakeAddr(Addr, T, getContext(), LValueBaseInfo(Source),
+                            CGM.getTBAAAccessInfo(T));
   }
 
-  LValue MakeAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment,
-                        LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo) {
-    return LValue::MakeAddr(Address(V, Alignment), T, getContext(),
-                            BaseInfo, TBAAInfo);
+  LValue
+  MakeAddrLValueWithoutTBAA(Address Addr, QualType T,
+                            AlignmentSource Source = AlignmentSource::Type) {
+    return LValue::MakeAddr(Addr, T, getContext(), LValueBaseInfo(Source),
+                            TBAAAccessInfo());
   }
 
   LValue MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T);
@@ -3128,15 +3135,18 @@ public:
 
   class ParamValue {
     llvm::Value *Value;
+    llvm::Type *ElementType;
     unsigned Alignment;
-    ParamValue(llvm::Value *V, unsigned A) : Value(V), Alignment(A) {}
+    ParamValue(llvm::Value *V, llvm::Type *T, unsigned A)
+        : Value(V), ElementType(T), Alignment(A) {}
   public:
     static ParamValue forDirect(llvm::Value *value) {
-      return ParamValue(value, 0);
+      return ParamValue(value, nullptr, 0);
     }
     static ParamValue forIndirect(Address addr) {
       assert(!addr.getAlignment().isZero());
-      return ParamValue(addr.getPointer(), addr.getAlignment().getQuantity());
+      return ParamValue(addr.getPointer(), addr.getElementType(),
+                        addr.getAlignment().getQuantity());
     }
 
     bool isIndirect() const { return Alignment != 0; }
@@ -3149,7 +3159,7 @@ public:
 
     Address getIndirectAddress() const {
       assert(isIndirect());
-      return Address(Value, CharUnits::fromQuantity(Alignment));
+      return Address(Value, ElementType, CharUnits::fromQuantity(Alignment));
     }
   };
 
@@ -4405,7 +4415,7 @@ public:
 
   /// EmitCXXGlobalVarDeclInit - Create the initializer for a C++
   /// variable with global storage.
-  void EmitCXXGlobalVarDeclInit(const VarDecl &D, llvm::Constant *DeclPtr,
+  void EmitCXXGlobalVarDeclInit(const VarDecl &D, llvm::GlobalVariable *GV,
                                 bool PerformInit);
 
   llvm::Function *createAtExitStub(const VarDecl &VD, llvm::FunctionCallee Dtor,
@@ -4556,7 +4566,7 @@ public:
   /// \p SignedIndices indicates whether any of the GEP indices are signed.
   /// \p IsSubtraction indicates whether the expression used to form the GEP
   /// is a subtraction.
-  llvm::Value *EmitCheckedInBoundsGEP(llvm::Value *Ptr,
+  llvm::Value *EmitCheckedInBoundsGEP(llvm::Type *ElemTy, llvm::Value *Ptr,
                                       ArrayRef<llvm::Value *> IdxList,
                                       bool SignedIndices,
                                       bool IsSubtraction,
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index 9ba1a5c25e81..36b7ce87336c 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -2832,7 +2832,7 @@ ConstantAddress CodeGenModule::GetAddrOfMSGuidDecl(const MSGuidDecl *GD) {
 
   // Look for an existing global.
   if (llvm::GlobalVariable *GV = getModule().getNamedGlobal(Name))
-    return ConstantAddress(GV, Alignment);
+    return ConstantAddress(GV, GV->getValueType(), Alignment);
 
   ConstantEmitter Emitter(*this);
   llvm::Constant *Init;
@@ -2866,15 +2866,15 @@ ConstantAddress CodeGenModule::GetAddrOfMSGuidDecl(const MSGuidDecl *GD) {
     GV->setComdat(TheModule.getOrInsertComdat(GV->getName()));
   setDSOLocal(GV);
 
-  llvm::Constant *Addr = GV;
   if (!V.isAbsent()) {
     Emitter.finalize(GV);
-  } else {
-    llvm::Type *Ty = getTypes().ConvertTypeForMem(GD->getType());
-    Addr = llvm::ConstantExpr::getBitCast(
-        GV, Ty->getPointerTo(GV->getAddressSpace()));
+    return ConstantAddress(GV, GV->getValueType(), Alignment);
   }
-  return ConstantAddress(Addr, Alignment);
+
+  llvm::Type *Ty = getTypes().ConvertTypeForMem(GD->getType());
+  llvm::Constant *Addr = llvm::ConstantExpr::getBitCast(
+      GV, Ty->getPointerTo(GV->getAddressSpace()));
+  return ConstantAddress(Addr, Ty, Alignment);
 }
 
 ConstantAddress CodeGenModule::GetAddrOfTemplateParamObject(
@@ -2883,7 +2883,7 @@ ConstantAddress CodeGenModule::GetAddrOfTemplateParamObject(
   CharUnits Alignment = getNaturalTypeAlignment(TPO->getType());
 
   if (llvm::GlobalVariable *GV = getModule().getNamedGlobal(Name))
-    return ConstantAddress(GV, Alignment);
+    return ConstantAddress(GV, GV->getValueType(), Alignment);
 
   ConstantEmitter Emitter(*this);
   llvm::Constant *Init = Emitter.emitForInitializer(
@@ -2901,7 +2901,7 @@ ConstantAddress CodeGenModule::GetAddrOfTemplateParamObject(
     GV->setComdat(TheModule.getOrInsertComdat(GV->getName()));
   Emitter.finalize(GV);
 
-  return ConstantAddress(GV, Alignment);
+  return ConstantAddress(GV, GV->getValueType(), Alignment);
 }
 
 ConstantAddress CodeGenModule::GetWeakRefReference(const ValueDecl *VD) {
@@ -2916,7 +2916,7 @@ ConstantAddress CodeGenModule::GetWeakRefReference(const ValueDecl *VD) {
   if (Entry) {
     unsigned AS = getContext().getTargetAddressSpace(VD->getType());
     auto Ptr = llvm::ConstantExpr::getBitCast(Entry, DeclTy->getPointerTo(AS));
-    return ConstantAddress(Ptr, Alignment);
+    return ConstantAddress(Ptr, DeclTy, Alignment);
   }
 
   llvm::Constant *Aliasee;
@@ -2932,7 +2932,7 @@ ConstantAddress CodeGenModule::GetWeakRefReference(const ValueDecl *VD) {
   F->setLinkage(llvm::Function::ExternalWeakLinkage);
   WeakRefReferences.insert(F);
 
-  return ConstantAddress(Aliasee, Alignment);
+  return ConstantAddress(Aliasee, DeclTy, Alignment);
 }
 
 void CodeGenModule::EmitGlobal(GlobalDecl GD) {
@@ -3886,6 +3886,14 @@ llvm::Constant *CodeGenModule::GetAddrOfFunction(GlobalDecl GD,
   return F;
 }
 
+llvm::Constant *CodeGenModule::GetFunctionStart(const ValueDecl *Decl) {
+  llvm::GlobalValue *F =
+      cast<llvm::GlobalValue>(GetAddrOfFunction(Decl)->stripPointerCasts());
+
+  return llvm::ConstantExpr::getBitCast(llvm::NoCFIValue::get(F),
+                                        llvm::Type::getInt8PtrTy(VMContext));
+}
+
 static const FunctionDecl *
 GetRuntimeFunctionDecl(ASTContext &C, StringRef Name) {
   TranslationUnitDecl *TUDecl = C.getTranslationUnitDecl();
@@ -5228,7 +5236,8 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) {
                                StringLength);
 
   if (auto *C = Entry.second)
-    return ConstantAddress(C, CharUnits::fromQuantity(C->getAlignment()));
+    return ConstantAddress(
+        C, C->getValueType(), CharUnits::fromQuantity(C->getAlignment()));
 
   llvm::Constant *Zero = llvm::Constant::getNullValue(Int32Ty);
   llvm::Constant *Zeros[] = { Zero, Zero };
@@ -5409,7 +5418,7 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) {
   }
   Entry.second = GV;
 
-  return ConstantAddress(GV, Alignment);
+  return ConstantAddress(GV, GV->getValueType(), Alignment);
 }
 
 bool CodeGenModule::getExpressionLocationsEnabled() const {
@@ -5527,7 +5536,7 @@ CodeGenModule::GetAddrOfConstantStringFromLiteral(const StringLiteral *S,
       if (uint64_t(Alignment.getQuantity()) > GV->getAlignment())
         GV->setAlignment(Alignment.getAsAlign());
       return ConstantAddress(castStringLiteralToDefaultAddressSpace(*this, GV),
-                             Alignment);
+                             GV->getValueType(), Alignment);
     }
   }
 
@@ -5557,7 +5566,7 @@ CodeGenModule::GetAddrOfConstantStringFromLiteral(const StringLiteral *S,
                                   QualType());
 
   return ConstantAddress(castStringLiteralToDefaultAddressSpace(*this, GV),
-                         Alignment);
+                         GV->getValueType(), Alignment);
 }
 
 /// GetAddrOfConstantStringFromObjCEncode - Return a pointer to a constant
@@ -5590,7 +5599,7 @@ ConstantAddress CodeGenModule::GetAddrOfConstantCString(
       if (uint64_t(Alignment.getQuantity()) > GV->getAlignment())
         GV->setAlignment(Alignment.getAsAlign());
       return ConstantAddress(castStringLiteralToDefaultAddressSpace(*this, GV),
-                             Alignment);
+                             GV->getValueType(), Alignment);
     }
   }
 
@@ -5604,7 +5613,7 @@ ConstantAddress CodeGenModule::GetAddrOfConstantCString(
     *Entry = GV;
 
   return ConstantAddress(castStringLiteralToDefaultAddressSpace(*this, GV),
-                         Alignment);
+                         GV->getValueType(), Alignment);
 }
 
 ConstantAddress CodeGenModule::GetAddrOfGlobalTemporary(
@@ -5634,7 +5643,9 @@ ConstantAddress CodeGenModule::GetAddrOfGlobalTemporary(
           getModule(), Type, false, llvm::GlobalVariable::InternalLinkage,
           nullptr);
     }
-    return ConstantAddress(InsertResult.first->second, Align);
+    return ConstantAddress(
+        InsertResult.first->second,
+        InsertResult.first->second->getType()->getPointerElementType(), Align);
   }
 
   // FIXME: If an externally-visible declaration extends multiple temporaries,
@@ -5725,7 +5736,7 @@ ConstantAddress CodeGenModule::GetAddrOfGlobalTemporary(
   }
   Entry = CV;
 
-  return ConstantAddress(CV, Align);
+  return ConstantAddress(CV, Type, Align);
 }
 
 /// EmitObjCPropertyImplementations - Emit information for synthesized
@@ -6398,6 +6409,11 @@ void CodeGenModule::EmitOMPThreadPrivateDecl(const OMPThreadPrivateDecl *D) {
 llvm::Metadata *
 CodeGenModule::CreateMetadataIdentifierImpl(QualType T, MetadataTypeMap &Map,
                                             StringRef Suffix) {
+  if (auto *FnType = T->getAs<FunctionProtoType>())
+    T = getContext().getFunctionType(
+        FnType->getReturnType(), FnType->getParamTypes(),
+        FnType->getExtProtoInfo().withExceptionSpec(EST_None));
+
   llvm::Metadata *&InternalId = Map[T.getCanonicalType()];
   if (InternalId)
     return InternalId;
diff --git a/clang/lib/CodeGen/CodeGenModule.h b/clang/lib/CodeGen/CodeGenModule.h
index e1c7f486d334..f1565511f98a 100644
--- a/clang/lib/CodeGen/CodeGenModule.h
+++ b/clang/lib/CodeGen/CodeGenModule.h
@@ -881,6 +881,9 @@ public:
                                     ForDefinition_t IsForDefinition
                                       = NotForDefinition);
 
+  // Return the function body address of the given function.
+  llvm::Constant *GetFunctionStart(const ValueDecl *Decl);
+
   /// Get the address of the RTTI descriptor for the given type.
   llvm::Constant *GetAddrOfRTTIDescriptor(QualType Ty, bool ForEH = false);
 
diff --git a/clang/lib/CodeGen/CodeGenTBAA.cpp b/clang/lib/CodeGen/CodeGenTBAA.cpp
index f4ebe6885675..95763d8e18b7 100644
--- a/clang/lib/CodeGen/CodeGenTBAA.cpp
+++ b/clang/lib/CodeGen/CodeGenTBAA.cpp
@@ -209,12 +209,12 @@ llvm::MDNode *CodeGenTBAA::getTypeInfoHelper(const Type *Ty) {
     return createScalarTypeNode(OutName, getChar(), Size);
   }
 
-  if (const auto *EIT = dyn_cast<ExtIntType>(Ty)) {
+  if (const auto *EIT = dyn_cast<BitIntType>(Ty)) {
     SmallString<256> OutName;
     llvm::raw_svector_ostream Out(OutName);
     // Don't specify signed/unsigned since integer types can alias despite sign
     // differences.
-    Out << "_ExtInt(" << EIT->getNumBits() << ')';
+    Out << "_BitInt(" << EIT->getNumBits() << ')';
     return createScalarTypeNode(OutName, getChar(), Size);
   }
 
diff --git a/clang/lib/CodeGen/CodeGenTypes.cpp b/clang/lib/CodeGen/CodeGenTypes.cpp
index fb05475a4e8c..77721510dfd0 100644
--- a/clang/lib/CodeGen/CodeGenTypes.cpp
+++ b/clang/lib/CodeGen/CodeGenTypes.cpp
@@ -97,10 +97,10 @@ llvm::Type *CodeGenTypes::ConvertTypeForMem(QualType T, bool ForBitField) {
 
   llvm::Type *R = ConvertType(T);
 
-  // If this is a bool type, or an ExtIntType in a bitfield representation,
-  // map this integer to the target-specified size.
-  if ((ForBitField && T->isExtIntType()) ||
-      (!T->isExtIntType() && R->isIntegerTy(1)))
+  // If this is a bool type, or a bit-precise integer type in a bitfield
+  // representation, map this integer to the target-specified size.
+  if ((ForBitField && T->isBitIntType()) ||
+      (!T->isBitIntType() && R->isIntegerTy(1)))
     return llvm::IntegerType::get(getLLVMContext(),
                                   (unsigned)Context.getTypeSize(T));
 
@@ -786,8 +786,8 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) {
     ResultType = CGM.getOpenCLRuntime().getPipeType(cast<PipeType>(Ty));
     break;
   }
-  case Type::ExtInt: {
-    const auto &EIT = cast<ExtIntType>(Ty);
+  case Type::BitInt: {
+    const auto &EIT = cast<BitIntType>(Ty);
     ResultType = llvm::Type::getIntNTy(getLLVMContext(), EIT->getNumBits());
     break;
   }
diff --git a/clang/lib/CodeGen/ItaniumCXXABI.cpp b/clang/lib/CodeGen/ItaniumCXXABI.cpp
index 04163aeaddc5..1a15b09c7b2b 100644
--- a/clang/lib/CodeGen/ItaniumCXXABI.cpp
+++ b/clang/lib/CodeGen/ItaniumCXXABI.cpp
@@ -1345,7 +1345,8 @@ void ItaniumCXXABI::emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) {
       AllocExceptionFn, llvm::ConstantInt::get(SizeTy, TypeSize), "exception");
 
   CharUnits ExnAlign = CGF.getContext().getExnObjectAlignment();
-  CGF.EmitAnyExprToExn(E->getSubExpr(), Address(ExceptionPtr, ExnAlign));
+  CGF.EmitAnyExprToExn(
+      E->getSubExpr(), Address(ExceptionPtr, CGM.Int8Ty, ExnAlign));
 
   // Now throw the exception.
   llvm::Constant *TypeInfo = CGM.GetAddrOfRTTIDescriptor(ThrowType,
@@ -2465,7 +2466,7 @@ void ItaniumCXXABI::EmitGuardedInit(CodeGenFunction &CGF,
     CGM.setStaticLocalDeclGuardAddress(&D, guard);
   }
 
-  Address guardAddr = Address(guard, guardAlignment);
+  Address guardAddr = Address(guard, guard->getValueType(), guardAlignment);
 
   // Test whether the variable has completed initialization.
   //
@@ -2880,7 +2881,7 @@ void ItaniumCXXABI::EmitThreadLocalInitFuncs(
     Guard->setAlignment(GuardAlign.getAsAlign());
 
     CodeGenFunction(CGM).GenerateCXXGlobalInitFunc(
-        InitFunc, OrderedInits, ConstantAddress(Guard, GuardAlign));
+        InitFunc, OrderedInits, ConstantAddress(Guard, CGM.Int8Ty, GuardAlign));
     // On Darwin platforms, use CXX_FAST_TLS calling convention.
     if (CGM.getTarget().getTriple().isOSDarwin()) {
       InitFunc->setCallingConv(llvm::CallingConv::CXX_FAST_TLS);
@@ -3529,7 +3530,7 @@ void ItaniumRTTIBuilder::BuildVTablePointer(const Type *Ty) {
     llvm_unreachable("Pipe types shouldn't get here");
 
   case Type::Builtin:
-  case Type::ExtInt:
+  case Type::BitInt:
   // GCC treats vector and complex types as fundamental types.
   case Type::Vector:
   case Type::ExtVector:
@@ -3802,7 +3803,7 @@ llvm::Constant *ItaniumRTTIBuilder::BuildTypeInfo(
   case Type::Pipe:
     break;
 
-  case Type::ExtInt:
+  case Type::BitInt:
     break;
 
   case Type::ConstantArray:
diff --git a/clang/lib/CodeGen/MicrosoftCXXABI.cpp b/clang/lib/CodeGen/MicrosoftCXXABI.cpp
index 0fd5a0ffe06c..5971a7709304 100644
--- a/clang/lib/CodeGen/MicrosoftCXXABI.cpp
+++ b/clang/lib/CodeGen/MicrosoftCXXABI.cpp
@@ -917,7 +917,7 @@ void MicrosoftCXXABI::emitBeginCatch(CodeGenFunction &CGF,
 std::tuple<Address, llvm::Value *, const CXXRecordDecl *>
 MicrosoftCXXABI::performBaseAdjustment(CodeGenFunction &CGF, Address Value,
                                        QualType SrcRecordTy) {
-  Value = CGF.Builder.CreateBitCast(Value, CGF.Int8PtrTy);
+  Value = CGF.Builder.CreateElementBitCast(Value, CGF.Int8Ty);
   const CXXRecordDecl *SrcDecl = SrcRecordTy->getAsCXXRecordDecl();
   const ASTContext &Context = getContext();
 
@@ -2408,14 +2408,14 @@ static ConstantAddress getInitThreadEpochPtr(CodeGenModule &CGM) {
   StringRef VarName("_Init_thread_epoch");
   CharUnits Align = CGM.getIntAlign();
   if (auto *GV = CGM.getModule().getNamedGlobal(VarName))
-    return ConstantAddress(GV, Align);
+    return ConstantAddress(GV, GV->getValueType(), Align);
   auto *GV = new llvm::GlobalVariable(
       CGM.getModule(), CGM.IntTy,
       /*isConstant=*/false, llvm::GlobalVariable::ExternalLinkage,
       /*Initializer=*/nullptr, VarName,
       /*InsertBefore=*/nullptr, llvm::GlobalVariable::GeneralDynamicTLSModel);
   GV->setAlignment(Align.getAsAlign());
-  return ConstantAddress(GV, Align);
+  return ConstantAddress(GV, GV->getValueType(), Align);
 }
 
 static llvm::FunctionCallee getInitThreadHeaderFn(CodeGenModule &CGM) {
@@ -2567,7 +2567,7 @@ void MicrosoftCXXABI::EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D,
       GI->Guard = GuardVar;
   }
 
-  ConstantAddress GuardAddr(GuardVar, GuardAlign);
+  ConstantAddress GuardAddr(GuardVar, GuardTy, GuardAlign);
 
   assert(GuardVar->getLinkage() == GV->getLinkage() &&
          "static local from the same function had different linkage");
diff --git a/clang/lib/CodeGen/TargetInfo.cpp b/clang/lib/CodeGen/TargetInfo.cpp
index 36e0319c8ab9..85089cdb2200 100644
--- a/clang/lib/CodeGen/TargetInfo.cpp
+++ b/clang/lib/CodeGen/TargetInfo.cpp
@@ -104,7 +104,7 @@ bool ABIInfo::isPromotableIntegerTypeForABI(QualType Ty) const {
   if (Ty->isPromotableIntegerType())
     return true;
 
-  if (const auto *EIT = Ty->getAs<ExtIntType>())
+  if (const auto *EIT = Ty->getAs<BitIntType>())
     if (EIT->getNumBits() < getContext().getTypeSize(getContext().IntTy))
       return true;
 
@@ -431,7 +431,7 @@ static Address emitMergePHI(CodeGenFunction &CGF,
   PHI->addIncoming(Addr1.getPointer(), Block1);
   PHI->addIncoming(Addr2.getPointer(), Block2);
   CharUnits Align = std::min(Addr1.getAlignment(), Addr2.getAlignment());
-  return Address(PHI, Align);
+  return Address(PHI, Addr1.getElementType(), Align);
 }
 
 TargetCodeGenInfo::~TargetCodeGenInfo() = default;
@@ -762,7 +762,7 @@ ABIArgInfo DefaultABIInfo::classifyArgumentType(QualType Ty) const {
     Ty = EnumTy->getDecl()->getIntegerType();
 
   ASTContext &Context = getContext();
-  if (const auto *EIT = Ty->getAs<ExtIntType>())
+  if (const auto *EIT = Ty->getAs<BitIntType>())
     if (EIT->getNumBits() >
         Context.getTypeSize(Context.getTargetInfo().hasInt128Type()
                                 ? Context.Int128Ty
@@ -784,7 +784,7 @@ ABIArgInfo DefaultABIInfo::classifyReturnType(QualType RetTy) const {
   if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
     RetTy = EnumTy->getDecl()->getIntegerType();
 
-  if (const auto *EIT = RetTy->getAs<ExtIntType>())
+  if (const auto *EIT = RetTy->getAs<BitIntType>())
     if (EIT->getNumBits() >
         getContext().getTypeSize(getContext().getTargetInfo().hasInt128Type()
                                      ? getContext().Int128Ty
@@ -1008,8 +1008,9 @@ ABIArgInfo PNaClABIInfo::classifyArgumentType(QualType Ty) const {
   } else if (Ty->isFloatingType()) {
     // Floating-point types don't go inreg.
     return ABIArgInfo::getDirect();
-  } else if (const auto *EIT = Ty->getAs<ExtIntType>()) {
-    // Treat extended integers as integers if <=64, otherwise pass indirectly.
+  } else if (const auto *EIT = Ty->getAs<BitIntType>()) {
+    // Treat bit-precise integers as integers if <= 64, otherwise pass
+    // indirectly.
     if (EIT->getNumBits() > 64)
       return getNaturalAlignIndirect(Ty);
     return ABIArgInfo::getDirect();
@@ -1027,8 +1028,8 @@ ABIArgInfo PNaClABIInfo::classifyReturnType(QualType RetTy) const {
   if (isAggregateTypeForABI(RetTy))
     return getNaturalAlignIndirect(RetTy);
 
-  // Treat extended integers as integers if <=64, otherwise pass indirectly.
-  if (const auto *EIT = RetTy->getAs<ExtIntType>()) {
+  // Treat bit-precise integers as integers if <= 64, otherwise pass indirectly.
+  if (const auto *EIT = RetTy->getAs<BitIntType>()) {
     if (EIT->getNumBits() > 64)
       return getNaturalAlignIndirect(RetTy);
     return ABIArgInfo::getDirect();
@@ -1590,7 +1591,7 @@ ABIArgInfo X86_32ABIInfo::classifyReturnType(QualType RetTy,
   if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
     RetTy = EnumTy->getDecl()->getIntegerType();
 
-  if (const auto *EIT = RetTy->getAs<ExtIntType>())
+  if (const auto *EIT = RetTy->getAs<BitIntType>())
     if (EIT->getNumBits() > 64)
       return getIndirectReturnResult(RetTy, State);
 
@@ -1926,7 +1927,7 @@ ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty,
     return ABIArgInfo::getExtend(Ty);
   }
 
-  if (const auto * EIT = Ty->getAs<ExtIntType>()) {
+  if (const auto *EIT = Ty->getAs<BitIntType>()) {
     if (EIT->getNumBits() <= 64) {
       if (InReg)
         return ABIArgInfo::getDirectInReg();
@@ -3009,7 +3010,7 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase,
     return;
   }
 
-  if (const auto *EITy = Ty->getAs<ExtIntType>()) {
+  if (const auto *EITy = Ty->getAs<BitIntType>()) {
     if (EITy->getNumBits() <= 64)
       Current = Integer;
     else if (EITy->getNumBits() <= 128)
@@ -3200,7 +3201,7 @@ ABIArgInfo X86_64ABIInfo::getIndirectReturnResult(QualType Ty) const {
     if (const EnumType *EnumTy = Ty->getAs<EnumType>())
       Ty = EnumTy->getDecl()->getIntegerType();
 
-    if (Ty->isExtIntType())
+    if (Ty->isBitIntType())
       return getNaturalAlignIndirect(Ty);
 
     return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
@@ -3237,7 +3238,7 @@ ABIArgInfo X86_64ABIInfo::getIndirectResult(QualType Ty,
   // but this code would be much safer if we could mark the argument with
   // 'onstack'. See PR12193.
   if (!isAggregateTypeForABI(Ty) && !IsIllegalVectorType(Ty) &&
-      !Ty->isExtIntType()) {
+      !Ty->isBitIntType()) {
     // Treat an enum type as its underlying type.
     if (const EnumType *EnumTy = Ty->getAs<EnumType>())
       Ty = EnumTy->getDecl()->getIntegerType();
@@ -4033,7 +4034,7 @@ static Address EmitX86_64VAArgFromMemory(CodeGenFunction &CGF,
   CGF.Builder.CreateStore(overflow_arg_area, overflow_arg_area_p);
 
   // AMD64-ABI 3.5.7p5: Step 11. Return the fetched type.
-  return Address(Res, Align);
+  return Address(Res, LTy, Align);
 }
 
 Address X86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
@@ -4146,7 +4147,7 @@ Address X86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
     RegAddr = CGF.Builder.CreateElementBitCast(Tmp, LTy);
   } else if (neededInt) {
     RegAddr = Address(CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, gp_offset),
-                      CharUnits::fromQuantity(8));
+                      CGF.Int8Ty, CharUnits::fromQuantity(8));
     RegAddr = CGF.Builder.CreateElementBitCast(RegAddr, LTy);
 
     // Copy to a temporary if necessary to ensure the appropriate alignment.
@@ -4164,7 +4165,7 @@ Address X86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
 
   } else if (neededSSE == 1) {
     RegAddr = Address(CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, fp_offset),
-                      CharUnits::fromQuantity(16));
+                      CGF.Int8Ty, CharUnits::fromQuantity(16));
     RegAddr = CGF.Builder.CreateElementBitCast(RegAddr, LTy);
   } else {
     assert(neededSSE == 2 && "Invalid number of needed registers!");
@@ -4176,7 +4177,7 @@ Address X86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
     // all the SSE registers to the RSA.
     Address RegAddrLo = Address(CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea,
                                                       fp_offset),
-                                CharUnits::fromQuantity(16));
+                                CGF.Int8Ty, CharUnits::fromQuantity(16));
     Address RegAddrHi =
       CGF.Builder.CreateConstInBoundsByteGEP(RegAddrLo,
                                              CharUnits::fromQuantity(16));
@@ -4357,12 +4358,12 @@ ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, unsigned &FreeSSERegs,
     }
   }
 
-  if (Ty->isExtIntType()) {
+  if (Ty->isBitIntType()) {
     // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
     // not 1, 2, 4, or 8 bytes, must be passed by reference."
-    // However, non-power-of-two _ExtInts will be passed as 1,2,4 or 8 bytes
-    // anyway as long is it fits in them, so we don't have to check the power of
-    // 2.
+    // However, non-power-of-two bit-precise integers will be passed as 1, 2, 4,
+    // or 8 bytes anyway as long is it fits in them, so we don't have to check
+    // the power of 2.
     if (Width <= 64)
       return ABIArgInfo::getDirect();
     return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
@@ -5069,7 +5070,7 @@ PPC64_SVR4_ABIInfo::isPromotableTypeForABI(QualType Ty) const {
       break;
     }
 
-  if (const auto *EIT = Ty->getAs<ExtIntType>())
+  if (const auto *EIT = Ty->getAs<BitIntType>())
     if (EIT->getNumBits() < 64)
       return true;
 
@@ -5083,13 +5084,16 @@ CharUnits PPC64_SVR4_ABIInfo::getParamTypeAlignment(QualType Ty) const {
   if (const ComplexType *CTy = Ty->getAs<ComplexType>())
     Ty = CTy->getElementType();
 
+  auto FloatUsesVector = [this](QualType Ty){
+    return Ty->isRealFloatingType() && &getContext().getFloatTypeSemantics(
+                                           Ty) == &llvm::APFloat::IEEEquad();
+  };
+
   // Only vector types of size 16 bytes need alignment (larger types are
   // passed via reference, smaller types are not aligned).
   if (Ty->isVectorType()) {
     return CharUnits::fromQuantity(getContext().getTypeSize(Ty) == 128 ? 16 : 8);
-  } else if (Ty->isRealFloatingType() &&
-             &getContext().getFloatTypeSemantics(Ty) ==
-                 &llvm::APFloat::IEEEquad()) {
+  } else if (FloatUsesVector(Ty)) {
     // According to ABI document section 'Optional Save Areas': If extended
     // precision floating-point values in IEEE BINARY 128 QUADRUPLE PRECISION
     // format are supported, map them to a single quadword, quadword aligned.
@@ -5116,7 +5120,9 @@ CharUnits PPC64_SVR4_ABIInfo::getParamTypeAlignment(QualType Ty) const {
 
   // With special case aggregates, only vector base types need alignment.
   if (AlignAsType) {
-    return CharUnits::fromQuantity(AlignAsType->isVectorType() ? 16 : 8);
+    bool UsesVector = AlignAsType->isVectorType() ||
+                      FloatUsesVector(QualType(AlignAsType, 0));
+    return CharUnits::fromQuantity(UsesVector ? 16 : 8);
   }
 
   // Otherwise, we only need alignment for any aggregate type that
@@ -5289,7 +5295,7 @@ PPC64_SVR4_ABIInfo::classifyArgumentType(QualType Ty) const {
     }
   }
 
-  if (const auto *EIT = Ty->getAs<ExtIntType>())
+  if (const auto *EIT = Ty->getAs<BitIntType>())
     if (EIT->getNumBits() > 128)
       return getNaturalAlignIndirect(Ty, /*ByVal=*/true);
 
@@ -5365,7 +5371,7 @@ PPC64_SVR4_ABIInfo::classifyReturnType(QualType RetTy) const {
     }
   }
 
-  if (const auto *EIT = RetTy->getAs<ExtIntType>())
+  if (const auto *EIT = RetTy->getAs<BitIntType>())
     if (EIT->getNumBits() > 128)
       return getNaturalAlignIndirect(RetTy, /*ByVal=*/false);
 
@@ -5717,7 +5723,7 @@ AArch64ABIInfo::classifyArgumentType(QualType Ty, bool IsVariadic,
     if (const EnumType *EnumTy = Ty->getAs<EnumType>())
       Ty = EnumTy->getDecl()->getIntegerType();
 
-    if (const auto *EIT = Ty->getAs<ExtIntType>())
+    if (const auto *EIT = Ty->getAs<BitIntType>())
       if (EIT->getNumBits() > 128)
         return getNaturalAlignIndirect(Ty);
 
@@ -5819,7 +5825,7 @@ ABIArgInfo AArch64ABIInfo::classifyReturnType(QualType RetTy,
     if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
       RetTy = EnumTy->getDecl()->getIntegerType();
 
-    if (const auto *EIT = RetTy->getAs<ExtIntType>())
+    if (const auto *EIT = RetTy->getAs<BitIntType>())
       if (EIT->getNumBits() > 128)
         return getNaturalAlignIndirect(RetTy);
 
@@ -6561,7 +6567,7 @@ ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty, bool isVariadic,
       Ty = EnumTy->getDecl()->getIntegerType();
     }
 
-    if (const auto *EIT = Ty->getAs<ExtIntType>())
+    if (const auto *EIT = Ty->getAs<BitIntType>())
       if (EIT->getNumBits() > 64)
         return getNaturalAlignIndirect(Ty, /*ByVal=*/true);
 
@@ -6763,7 +6769,7 @@ ABIArgInfo ARMABIInfo::classifyReturnType(QualType RetTy, bool isVariadic,
     if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
       RetTy = EnumTy->getDecl()->getIntegerType();
 
-    if (const auto *EIT = RetTy->getAs<ExtIntType>())
+    if (const auto *EIT = RetTy->getAs<BitIntType>())
       if (EIT->getNumBits() > 64)
         return getNaturalAlignIndirect(RetTy, /*ByVal=*/false);
 
@@ -7100,7 +7106,7 @@ bool NVPTXABIInfo::isUnsupportedType(QualType T) const {
       (T->isFloat128Type() ||
        (T->isRealFloatingType() && Context.getTypeSize(T) == 128)))
     return true;
-  if (const auto *EIT = T->getAs<ExtIntType>())
+  if (const auto *EIT = T->getAs<BitIntType>())
     return EIT->getNumBits() >
            (Context.getTargetInfo().hasInt128Type() ? 128U : 64U);
   if (!Context.getTargetInfo().hasInt128Type() && T->isIntegerType() &&
@@ -7177,7 +7183,7 @@ ABIArgInfo NVPTXABIInfo::classifyArgumentType(QualType Ty) const {
     return getNaturalAlignIndirect(Ty, /* byval */ true);
   }
 
-  if (const auto *EIT = Ty->getAs<ExtIntType>()) {
+  if (const auto *EIT = Ty->getAs<BitIntType>()) {
     if ((EIT->getNumBits() > 128) ||
         (!getContext().getTargetInfo().hasInt128Type() &&
          EIT->getNumBits() > 64))
@@ -7391,7 +7397,7 @@ bool SystemZABIInfo::isPromotableIntegerTypeForABI(QualType Ty) const {
   if (ABIInfo::isPromotableIntegerTypeForABI(Ty))
     return true;
 
-  if (const auto *EIT = Ty->getAs<ExtIntType>())
+  if (const auto *EIT = Ty->getAs<BitIntType>())
     if (EIT->getNumBits() < 64)
       return true;
 
@@ -7994,7 +8000,7 @@ MipsABIInfo::classifyArgumentType(QualType Ty, uint64_t &Offset) const {
     Ty = EnumTy->getDecl()->getIntegerType();
 
   // Make sure we pass indirectly things that are too large.
-  if (const auto *EIT = Ty->getAs<ExtIntType>())
+  if (const auto *EIT = Ty->getAs<BitIntType>())
     if (EIT->getNumBits() > 128 ||
         (EIT->getNumBits() > 64 &&
          !getContext().getTargetInfo().hasInt128Type()))
@@ -8085,7 +8091,7 @@ ABIArgInfo MipsABIInfo::classifyReturnType(QualType RetTy) const {
     RetTy = EnumTy->getDecl()->getIntegerType();
 
   // Make sure we pass indirectly things that are too large.
-  if (const auto *EIT = RetTy->getAs<ExtIntType>())
+  if (const auto *EIT = RetTy->getAs<BitIntType>())
     if (EIT->getNumBits() > 128 ||
         (EIT->getNumBits() > 64 &&
          !getContext().getTargetInfo().hasInt128Type()))
@@ -8460,7 +8466,7 @@ ABIArgInfo HexagonABIInfo::classifyArgumentType(QualType Ty,
     if (Size <= 64)
       HexagonAdjustRegsLeft(Size, RegsLeft);
 
-    if (Size > 64 && Ty->isExtIntType())
+    if (Size > 64 && Ty->isBitIntType())
       return getNaturalAlignIndirect(Ty, /*ByVal=*/true);
 
     return isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
@@ -8516,7 +8522,7 @@ ABIArgInfo HexagonABIInfo::classifyReturnType(QualType RetTy) const {
     if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
       RetTy = EnumTy->getDecl()->getIntegerType();
 
-    if (Size > 64 && RetTy->isExtIntType())
+    if (Size > 64 && RetTy->isBitIntType())
       return getNaturalAlignIndirect(RetTy, /*ByVal=*/false);
 
     return isPromotableIntegerTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy)
@@ -8887,7 +8893,7 @@ ABIArgInfo LanaiABIInfo::classifyArgumentType(QualType Ty,
   bool InReg = shouldUseInReg(Ty, State);
 
   // Don't pass >64 bit integers in registers.
-  if (const auto *EIT = Ty->getAs<ExtIntType>())
+  if (const auto *EIT = Ty->getAs<BitIntType>())
     if (EIT->getNumBits() > 64)
       return getIndirectResult(Ty, /*ByVal=*/true, State);
 
@@ -9161,6 +9167,10 @@ class AMDGPUTargetCodeGenInfo : public TargetCodeGenInfo {
 public:
   AMDGPUTargetCodeGenInfo(CodeGenTypes &CGT)
       : TargetCodeGenInfo(std::make_unique<AMDGPUABIInfo>(CGT)) {}
+
+  void setFunctionDeclAttributes(const FunctionDecl *FD, llvm::Function *F,
+                                 CodeGenModule &CGM) const;
+
   void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
                            CodeGen::CodeGenModule &M) const override;
   unsigned getOpenCLKernelCallingConv() const override;
@@ -9200,36 +9210,13 @@ static bool requiresAMDGPUProtectedVisibility(const Decl *D,
            cast<VarDecl>(D)->getType()->isCUDADeviceBuiltinTextureType()));
 }
 
-void AMDGPUTargetCodeGenInfo::setTargetAttributes(
-    const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const {
-  if (requiresAMDGPUProtectedVisibility(D, GV)) {
-    GV->setVisibility(llvm::GlobalValue::ProtectedVisibility);
-    GV->setDSOLocal(true);
-  }
-
-  if (GV->isDeclaration())
-    return;
-  const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D);
-  if (!FD)
-    return;
-
-  llvm::Function *F = cast<llvm::Function>(GV);
-
-  const auto *ReqdWGS = M.getLangOpts().OpenCL ?
-    FD->getAttr<ReqdWorkGroupSizeAttr>() : nullptr;
-
-
-  const bool IsOpenCLKernel = M.getLangOpts().OpenCL &&
-                              FD->hasAttr<OpenCLKernelAttr>();
-  const bool IsHIPKernel = M.getLangOpts().HIP &&
-                           FD->hasAttr<CUDAGlobalAttr>();
-  if ((IsOpenCLKernel || IsHIPKernel) &&
-      (M.getTriple().getOS() == llvm::Triple::AMDHSA))
-    F->addFnAttr("amdgpu-implicitarg-num-bytes", "56");
-
-  if (IsHIPKernel)
-    F->addFnAttr("uniform-work-group-size", "true");
-
+void AMDGPUTargetCodeGenInfo::setFunctionDeclAttributes(
+    const FunctionDecl *FD, llvm::Function *F, CodeGenModule &M) const {
+  const auto *ReqdWGS =
+      M.getLangOpts().OpenCL ? FD->getAttr<ReqdWorkGroupSizeAttr>() : nullptr;
+  const bool IsOpenCLKernel =
+      M.getLangOpts().OpenCL && FD->hasAttr<OpenCLKernelAttr>();
+  const bool IsHIPKernel = M.getLangOpts().HIP && FD->hasAttr<CUDAGlobalAttr>();
 
   const auto *FlatWGS = FD->getAttr<AMDGPUFlatWorkGroupSizeAttr>();
   if (ReqdWGS || FlatWGS) {
@@ -9297,6 +9284,38 @@ void AMDGPUTargetCodeGenInfo::setTargetAttributes(
     if (NumVGPR != 0)
       F->addFnAttr("amdgpu-num-vgpr", llvm::utostr(NumVGPR));
   }
+}
+
+void AMDGPUTargetCodeGenInfo::setTargetAttributes(
+    const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const {
+  if (requiresAMDGPUProtectedVisibility(D, GV)) {
+    GV->setVisibility(llvm::GlobalValue::ProtectedVisibility);
+    GV->setDSOLocal(true);
+  }
+
+  if (GV->isDeclaration())
+    return;
+
+  llvm::Function *F = dyn_cast<llvm::Function>(GV);
+  if (!F)
+    return;
+
+  const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D);
+  if (FD)
+    setFunctionDeclAttributes(FD, F, M);
+
+  const bool IsOpenCLKernel =
+      M.getLangOpts().OpenCL && FD && FD->hasAttr<OpenCLKernelAttr>();
+  const bool IsHIPKernel =
+      M.getLangOpts().HIP && FD && FD->hasAttr<CUDAGlobalAttr>();
+
+  const bool IsOpenMP = M.getLangOpts().OpenMP && !FD;
+  if ((IsOpenCLKernel || IsHIPKernel || IsOpenMP) &&
+      (M.getTriple().getOS() == llvm::Triple::AMDHSA))
+    F->addFnAttr("amdgpu-implicitarg-num-bytes", "56");
+
+  if (IsHIPKernel)
+    F->addFnAttr("uniform-work-group-size", "true");
 
   if (M.getContext().getTargetInfo().allowAMDGPUUnsafeFPAtomics())
     F->addFnAttr("amdgpu-unsafe-fp-atomics", "true");
@@ -9343,7 +9362,9 @@ AMDGPUTargetCodeGenInfo::getGlobalVarAddressSpace(CodeGenModule &CGM,
   if (AddrSpace != LangAS::Default)
     return AddrSpace;
 
-  if (CGM.isTypeConstant(D->getType(), false)) {
+  // Only promote to address space 4 if VarDecl has constant initialization.
+  if (CGM.isTypeConstant(D->getType(), false) &&
+      D->hasConstantInitialization()) {
     if (auto ConstAS = CGM.getTarget().getConstantAddressSpace())
       return ConstAS.getValue();
   }
@@ -9606,7 +9627,7 @@ SparcV9ABIInfo::classifyType(QualType Ty, unsigned SizeLimit) const {
   if (Size < 64 && Ty->isIntegerType())
     return ABIArgInfo::getExtend(Ty);
 
-  if (const auto *EIT = Ty->getAs<ExtIntType>())
+  if (const auto *EIT = Ty->getAs<BitIntType>())
     if (EIT->getNumBits() < 64)
       return ABIArgInfo::getExtend(Ty);
 
@@ -9860,7 +9881,7 @@ ABIArgInfo ARCABIInfo::classifyArgumentType(QualType Ty,
         ABIArgInfo::getDirect(Result, 0, nullptr, false);
   }
 
-  if (const auto *EIT = Ty->getAs<ExtIntType>())
+  if (const auto *EIT = Ty->getAs<BitIntType>())
     if (EIT->getNumBits() > 64)
       return getIndirectByValue(Ty);
 
@@ -10209,12 +10230,23 @@ public:
 private:
   void setCCs();
 };
+
+class SPIRVABIInfo : public CommonSPIRABIInfo {
+public:
+  SPIRVABIInfo(CodeGenTypes &CGT) : CommonSPIRABIInfo(CGT) {}
+  void computeInfo(CGFunctionInfo &FI) const override;
+
+private:
+  ABIArgInfo classifyKernelArgumentType(QualType Ty) const;
+};
 } // end anonymous namespace
 namespace {
 class CommonSPIRTargetCodeGenInfo : public TargetCodeGenInfo {
 public:
   CommonSPIRTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
       : TargetCodeGenInfo(std::make_unique<CommonSPIRABIInfo>(CGT)) {}
+  CommonSPIRTargetCodeGenInfo(std::unique_ptr<ABIInfo> ABIInfo)
+      : TargetCodeGenInfo(std::move(ABIInfo)) {}
 
   LangAS getASTAllocaAddressSpace() const override {
     return getLangASFromTargetAS(
@@ -10223,18 +10255,60 @@ public:
 
   unsigned getOpenCLKernelCallingConv() const override;
 };
-
+class SPIRVTargetCodeGenInfo : public CommonSPIRTargetCodeGenInfo {
+public:
+  SPIRVTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
+      : CommonSPIRTargetCodeGenInfo(std::make_unique<SPIRVABIInfo>(CGT)) {}
+  void setCUDAKernelCallingConvention(const FunctionType *&FT) const override;
+};
 } // End anonymous namespace.
+
 void CommonSPIRABIInfo::setCCs() {
   assert(getRuntimeCC() == llvm::CallingConv::C);
   RuntimeCC = llvm::CallingConv::SPIR_FUNC;
 }
 
+ABIArgInfo SPIRVABIInfo::classifyKernelArgumentType(QualType Ty) const {
+  if (getContext().getLangOpts().HIP) {
+    // Coerce pointer arguments with default address space to CrossWorkGroup
+    // pointers for HIPSPV. When the language mode is HIP, the SPIRTargetInfo
+    // maps cuda_device to SPIR-V's CrossWorkGroup address space.
+    llvm::Type *LTy = CGT.ConvertType(Ty);
+    auto DefaultAS = getContext().getTargetAddressSpace(LangAS::Default);
+    auto GlobalAS = getContext().getTargetAddressSpace(LangAS::cuda_device);
+    if (LTy->isPointerTy() && LTy->getPointerAddressSpace() == DefaultAS) {
+      LTy = llvm::PointerType::get(
+          cast<llvm::PointerType>(LTy)->getElementType(), GlobalAS);
+      return ABIArgInfo::getDirect(LTy, 0, nullptr, false);
+    }
+  }
+  return classifyArgumentType(Ty);
+}
+
+void SPIRVABIInfo::computeInfo(CGFunctionInfo &FI) const {
+  // The logic is same as in DefaultABIInfo with an exception on the kernel
+  // arguments handling.
+  llvm::CallingConv::ID CC = FI.getCallingConvention();
+
+  if (!getCXXABI().classifyReturnType(FI))
+    FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
+
+  for (auto &I : FI.arguments()) {
+    if (CC == llvm::CallingConv::SPIR_KERNEL) {
+      I.info = classifyKernelArgumentType(I.type);
+    } else {
+      I.info = classifyArgumentType(I.type);
+    }
+  }
+}
+
 namespace clang {
 namespace CodeGen {
 void computeSPIRKernelABIInfo(CodeGenModule &CGM, CGFunctionInfo &FI) {
-  DefaultABIInfo SPIRABI(CGM.getTypes());
-  SPIRABI.computeInfo(FI);
+  if (CGM.getTarget().getTriple().isSPIRV())
+    SPIRVABIInfo(CGM.getTypes()).computeInfo(FI);
+  else
+    CommonSPIRABIInfo(CGM.getTypes()).computeInfo(FI);
 }
 }
 }
@@ -10243,6 +10317,16 @@ unsigned CommonSPIRTargetCodeGenInfo::getOpenCLKernelCallingConv() const {
   return llvm::CallingConv::SPIR_KERNEL;
 }
 
+void SPIRVTargetCodeGenInfo::setCUDAKernelCallingConvention(
+    const FunctionType *&FT) const {
+  // Convert HIP kernels to SPIR-V kernels.
+  if (getABIInfo().getContext().getLangOpts().HIP) {
+    FT = getABIInfo().getContext().adjustFunctionType(
+        FT, FT->getExtInfo().withCallingConv(CC_OpenCLKernel));
+    return;
+  }
+}
+
 static bool appendType(SmallStringEnc &Enc, QualType QType,
                        const CodeGen::CodeGenModule &CGM,
                        TypeStringCache &TSC);
@@ -10943,7 +11027,7 @@ ABIArgInfo RISCVABIInfo::classifyArgumentType(QualType Ty, bool IsFixed,
       return extendType(Ty);
     }
 
-    if (const auto *EIT = Ty->getAs<ExtIntType>()) {
+    if (const auto *EIT = Ty->getAs<BitIntType>()) {
       if (EIT->getNumBits() < XLen && !MustUseStack)
         return extendType(Ty);
       if (EIT->getNumBits() > 128 ||
@@ -11308,9 +11392,10 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() {
     return SetCGInfo(new ARCTargetCodeGenInfo(Types));
   case llvm::Triple::spir:
   case llvm::Triple::spir64:
+    return SetCGInfo(new CommonSPIRTargetCodeGenInfo(Types));
   case llvm::Triple::spirv32:
   case llvm::Triple::spirv64:
-    return SetCGInfo(new CommonSPIRTargetCodeGenInfo(Types));
+    return SetCGInfo(new SPIRVTargetCodeGenInfo(Types));
   case llvm::Triple::ve:
     return SetCGInfo(new VETargetCodeGenInfo(Types));
   }
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index d501bd026219..3b551ea94cc2 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -23,7 +23,8 @@
 #include "ToolChains/FreeBSD.h"
 #include "ToolChains/Fuchsia.h"
 #include "ToolChains/Gnu.h"
-#include "ToolChains/HIP.h"
+#include "ToolChains/HIPAMD.h"
+#include "ToolChains/HIPSPV.h"
 #include "ToolChains/Haiku.h"
 #include "ToolChains/Hexagon.h"
 #include "ToolChains/Hurd.h"
@@ -42,6 +43,7 @@
 #include "ToolChains/PPCLinux.h"
 #include "ToolChains/PS4CPU.h"
 #include "ToolChains/RISCVToolchain.h"
+#include "ToolChains/SPIRV.h"
 #include "ToolChains/Solaris.h"
 #include "ToolChains/TCE.h"
 #include "ToolChains/VEToolchain.h"
@@ -99,8 +101,39 @@ using namespace clang::driver;
 using namespace clang;
 using namespace llvm::opt;
 
-static llvm::Triple getHIPOffloadTargetTriple() {
-  static const llvm::Triple T("amdgcn-amd-amdhsa");
+static llvm::Optional<llvm::Triple>
+getHIPOffloadTargetTriple(const Driver &D, const ArgList &Args) {
+  if (Args.hasArg(options::OPT_offload_EQ)) {
+    auto HIPOffloadTargets = Args.getAllArgValues(options::OPT_offload_EQ);
+
+    // HIP compilation flow does not support multiple targets for now. We need
+    // the HIPActionBuilder (and possibly the CudaActionBuilder{,Base}too) to
+    // support multiple tool chains first.
+    switch (HIPOffloadTargets.size()) {
+    default:
+      D.Diag(diag::err_drv_only_one_offload_target_supported_in) << "HIP";
+      return llvm::None;
+    case 0:
+      D.Diag(diag::err_drv_invalid_or_unsupported_offload_target) << "";
+      return llvm::None;
+    case 1:
+      break;
+    }
+    llvm::Triple TT(HIPOffloadTargets[0]);
+    if (TT.getArch() == llvm::Triple::amdgcn &&
+        TT.getVendor() == llvm::Triple::AMD &&
+        TT.getOS() == llvm::Triple::AMDHSA)
+      return TT;
+    if (TT.getArch() == llvm::Triple::spirv64 &&
+        TT.getVendor() == llvm::Triple::UnknownVendor &&
+        TT.getOS() == llvm::Triple::UnknownOS)
+      return TT;
+    D.Diag(diag::err_drv_invalid_or_unsupported_offload_target)
+        << HIPOffloadTargets[0];
+    return llvm::None;
+  }
+
+  static const llvm::Triple T("amdgcn-amd-amdhsa"); // Default HIP triple.
   return T;
 }
 
@@ -694,17 +727,14 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
       return;
     }
     const ToolChain *HostTC = C.getSingleOffloadToolChain<Action::OFK_Host>();
-    const llvm::Triple &HostTriple = HostTC->getTriple();
     auto OFK = Action::OFK_HIP;
-    llvm::Triple HIPTriple = getHIPOffloadTargetTriple();
-    // Use the HIP and host triples as the key into the ToolChains map,
-    // because the device toolchain we create depends on both.
-    auto &HIPTC = ToolChains[HIPTriple.str() + "/" + HostTriple.str()];
-    if (!HIPTC) {
-      HIPTC = std::make_unique<toolchains::HIPToolChain>(
-          *this, HIPTriple, *HostTC, C.getInputArgs());
-    }
-    C.addOffloadDeviceToolChain(HIPTC.get(), OFK);
+    auto HIPTriple = getHIPOffloadTargetTriple(*this, C.getInputArgs());
+    if (!HIPTriple)
+      return;
+    auto *HIPTC = &getOffloadingDeviceToolChain(C.getInputArgs(), *HIPTriple,
+                                                *HostTC, OFK);
+    assert(HIPTC && "Could not create offloading device tool chain.");
+    C.addOffloadDeviceToolChain(HIPTC, OFK);
   }
 
   //
@@ -1545,7 +1575,7 @@ int Driver::ExecuteCompilation(
   if (Diags.hasErrorOccurred())
     return 1;
 
-  // Set up response file names for each command, if necessary
+  // Set up response file names for each command, if necessary.
   for (auto &Job : C.getJobs())
     setUpResponseFiles(C, Job);
 
@@ -2040,11 +2070,7 @@ static bool ContainsCompileOrAssembleAction(const Action *A) {
       isa<AssembleJobAction>(A))
     return true;
 
-  for (const Action *Input : A->inputs())
-    if (ContainsCompileOrAssembleAction(Input))
-      return true;
-
-  return false;
+  return llvm::any_of(A->inputs(), ContainsCompileOrAssembleAction);
 }
 
 void Driver::BuildUniversalActions(Compilation &C, const ToolChain &TC,
@@ -2733,6 +2759,14 @@ class OffloadingActionBuilder final {
         }
       }
 
+      // --offload and --offload-arch options are mutually exclusive.
+      if (Args.hasArgNoClaim(options::OPT_offload_EQ) &&
+          Args.hasArgNoClaim(options::OPT_offload_arch_EQ,
+                             options::OPT_no_offload_arch_EQ)) {
+        C.getDriver().Diag(diag::err_opt_not_valid_with_opt) << "--offload-arch"
+                                                             << "--offload";
+      }
+
       // Collect all cuda_gpu_arch parameters, removing duplicates.
       std::set<StringRef> GpuArchs;
       bool Error = false;
@@ -2775,8 +2809,12 @@ class OffloadingActionBuilder final {
       // Default to sm_20 which is the lowest common denominator for
       // supported GPUs.  sm_20 code should work correctly, if
       // suboptimally, on all newer GPUs.
-      if (GpuArchList.empty())
-        GpuArchList.push_back(DefaultCudaArch);
+      if (GpuArchList.empty()) {
+        if (ToolChains.front()->getTriple().isSPIRV())
+          GpuArchList.push_back(CudaArch::Generic);
+        else
+          GpuArchList.push_back(DefaultCudaArch);
+      }
 
       return Error;
     }
@@ -2937,8 +2975,11 @@ class OffloadingActionBuilder final {
 
     StringRef getCanonicalOffloadArch(StringRef IdStr) override {
       llvm::StringMap<bool> Features;
-      auto ArchStr =
-          parseTargetID(getHIPOffloadTargetTriple(), IdStr, &Features);
+      // getHIPOffloadTargetTriple() is known to return valid value as it has
+      // been called successfully in the CreateOffloadingDeviceToolChains().
+      auto ArchStr = parseTargetID(
+          *getHIPOffloadTargetTriple(C.getDriver(), C.getInputArgs()), IdStr,
+          &Features);
       if (!ArchStr) {
         C.getDriver().Diag(clang::diag::err_drv_bad_target_id) << IdStr;
         C.setContainsError();
@@ -2992,9 +3033,19 @@ class OffloadingActionBuilder final {
             // When LTO is not enabled, we follow the conventional
             // compiler phases, including backend and assemble phases.
             ActionList AL;
-            auto BackendAction = C.getDriver().ConstructPhaseAction(
-                C, Args, phases::Backend, CudaDeviceActions[I],
-                AssociatedOffloadKind);
+            Action *BackendAction = nullptr;
+            if (ToolChains.front()->getTriple().isSPIRV()) {
+              // Emit LLVM bitcode for SPIR-V targets. SPIR-V device tool chain
+              // (HIPSPVToolChain) runs post-link LLVM IR passes.
+              types::ID Output = Args.hasArg(options::OPT_S)
+                                     ? types::TY_LLVM_IR
+                                     : types::TY_LLVM_BC;
+              BackendAction =
+                  C.MakeAction<BackendJobAction>(CudaDeviceActions[I], Output);
+            } else
+              BackendAction = C.getDriver().ConstructPhaseAction(
+                  C, Args, phases::Backend, CudaDeviceActions[I],
+                  AssociatedOffloadKind);
             auto AssembleAction = C.getDriver().ConstructPhaseAction(
                 C, Args, phases::Assemble, BackendAction,
                 AssociatedOffloadKind);
@@ -3724,6 +3775,14 @@ void Driver::BuildActions(Compilation &C, DerivedArgList &Args,
     }
   }
 
+  // FIXME: Linking separate translation units for SPIR-V is not supported yet.
+  // It can be done either by LLVM IR linking before conversion of the final
+  // linked module to SPIR-V or external SPIR-V linkers can be used e.g.
+  // spirv-link.
+  if (C.getDefaultToolChain().getTriple().isSPIRV() && Inputs.size() > 1) {
+    Diag(clang::diag::warn_drv_spirv_linking_multiple_inputs_unsupported);
+  }
+
   handleArguments(C, Args, Inputs, Actions);
 
   // Builder to be used to build offloading actions.
@@ -3763,8 +3822,15 @@ void Driver::BuildActions(Compilation &C, DerivedArgList &Args,
       // Queue linker inputs.
       if (Phase == phases::Link) {
         assert(Phase == PL.back() && "linking must be final compilation step.");
-        LinkerInputs.push_back(Current);
-        Current = nullptr;
+        // Compilation phases are setup per language, however for SPIR-V the
+        // final linking phase is meaningless since the compilation phase
+        // produces the final binary.
+        // FIXME: OpenCL - we could strip linking phase out from OpenCL
+        // compilation phases if we could verify it is not needed by any target.
+        if (!C.getDefaultToolChain().getTriple().isSPIRV()) {
+          LinkerInputs.push_back(Current);
+          Current = nullptr;
+        }
         break;
       }
 
@@ -4337,6 +4403,12 @@ class ToolSelector final {
     if (!T)
       return nullptr;
 
+    // Can't collapse if we don't have codegen support unless we are
+    // emitting LLVM IR.
+    bool OutputIsLLVM = types::isLLVMIR(ActionInfo[0].JA->getType());
+    if (!T->hasIntegratedBackend() && !(OutputIsLLVM && T->canEmitIR()))
+      return nullptr;
+
     // When using -fembed-bitcode, it is required to have the same tool (clang)
     // for both CompilerJA and BackendJA. Otherwise, combine two stages.
     if (EmbedBitcode) {
@@ -4406,6 +4478,12 @@ class ToolSelector final {
     if (!T)
       return nullptr;
 
+    // Can't collapse if we don't have codegen support unless we are
+    // emitting LLVM IR.
+    bool OutputIsLLVM = types::isLLVMIR(ActionInfo[0].JA->getType());
+    if (!T->hasIntegratedBackend() && !(OutputIsLLVM && T->canEmitIR()))
+      return nullptr;
+
     if (T->canEmitIR() && ((SaveTemps && !InputIsBitcode) || EmbedBitcode))
       return nullptr;
 
@@ -5429,6 +5507,10 @@ const ToolChain &Driver::getToolChain(const ArgList &Args,
       case llvm::Triple::ve:
         TC = std::make_unique<toolchains::VEToolChain>(*this, Target, Args);
         break;
+      case llvm::Triple::spirv32:
+      case llvm::Triple::spirv64:
+        TC = std::make_unique<toolchains::SPIRVToolChain>(*this, Target, Args);
+        break;
       default:
         if (Target.getVendor() == llvm::Triple::Myriad)
           TC = std::make_unique<toolchains::MyriadToolChain>(*this, Target,
@@ -5453,6 +5535,38 @@ const ToolChain &Driver::getToolChain(const ArgList &Args,
   return *TC;
 }
 
+const ToolChain &Driver::getOffloadingDeviceToolChain(
+    const ArgList &Args, const llvm::Triple &Target, const ToolChain &HostTC,
+    const Action::OffloadKind &TargetDeviceOffloadKind) const {
+  // Use device / host triples as the key into the ToolChains map because the
+  // device ToolChain we create depends on both.
+  auto &TC = ToolChains[Target.str() + "/" + HostTC.getTriple().str()];
+  if (!TC) {
+    // Categorized by offload kind > arch rather than OS > arch like
+    // the normal getToolChain call, as it seems a reasonable way to categorize
+    // things.
+    switch (TargetDeviceOffloadKind) {
+    case Action::OFK_HIP: {
+      if (Target.getArch() == llvm::Triple::amdgcn &&
+          Target.getVendor() == llvm::Triple::AMD &&
+          Target.getOS() == llvm::Triple::AMDHSA)
+        TC = std::make_unique<toolchains::HIPAMDToolChain>(*this, Target,
+                                                           HostTC, Args);
+      else if (Target.getArch() == llvm::Triple::spirv64 &&
+               Target.getVendor() == llvm::Triple::UnknownVendor &&
+               Target.getOS() == llvm::Triple::UnknownOS)
+        TC = std::make_unique<toolchains::HIPSPVToolChain>(*this, Target,
+                                                           HostTC, Args);
+      break;
+    }
+    default:
+      break;
+    }
+  }
+
+  return *TC;
+}
+
 bool Driver::ShouldUseClangCompiler(const JobAction &JA) const {
   // Say "no" if there is not exactly one input of a type clang understands.
   if (JA.size() != 1 ||
diff --git a/clang/lib/Driver/Job.cpp b/clang/lib/Driver/Job.cpp
index 5b87106b6565..f63763effaff 100644
--- a/clang/lib/Driver/Job.cpp
+++ b/clang/lib/Driver/Job.cpp
@@ -388,6 +388,8 @@ int CC1Command::Execute(ArrayRef<llvm::Optional<StringRef>> Redirects,
   Argv.push_back(getExecutable());
   Argv.append(getArguments().begin(), getArguments().end());
   Argv.push_back(nullptr);
+  Argv.pop_back(); // The terminating null element shall not be part of the
+                   // slice (main() behavior).
 
   // This flag simply indicates that the program couldn't start, which isn't
   // applicable here.
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 6588cdf9fecd..50c89aaadc18 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -260,7 +260,7 @@ bool ToolChain::IsUnwindTablesDefault(const ArgList &Args) const {
 
 Tool *ToolChain::getClang() const {
   if (!Clang)
-    Clang.reset(new tools::Clang(*this));
+    Clang.reset(new tools::Clang(*this, useIntegratedBackend()));
   return Clang.get();
 }
 
@@ -541,12 +541,9 @@ std::string ToolChain::GetProgramPath(const char *Name) const {
   return D.GetProgramPath(Name, *this);
 }
 
-std::string ToolChain::GetLinkerPath(bool *LinkerIsLLD,
-                                     bool *LinkerIsLLDDarwinNew) const {
+std::string ToolChain::GetLinkerPath(bool *LinkerIsLLD) const {
   if (LinkerIsLLD)
     *LinkerIsLLD = false;
-  if (LinkerIsLLDDarwinNew)
-    *LinkerIsLLDDarwinNew = false;
 
   // Get -fuse-ld= first to prevent -Wunused-command-line-argument. -fuse-ld= is
   // considered as the linker flavor, e.g. "bfd", "gold", or "lld".
@@ -599,11 +596,8 @@ std::string ToolChain::GetLinkerPath(bool *LinkerIsLLD,
 
     std::string LinkerPath(GetProgramPath(LinkerName.c_str()));
     if (llvm::sys::fs::can_execute(LinkerPath)) {
-      // FIXME: Remove LinkerIsLLDDarwinNew once there's only one MachO lld.
       if (LinkerIsLLD)
-        *LinkerIsLLD = UseLinker == "lld" || UseLinker == "lld.darwinold";
-      if (LinkerIsLLDDarwinNew)
-        *LinkerIsLLDDarwinNew = UseLinker == "lld";
+        *LinkerIsLLD = UseLinker == "lld";
       return LinkerPath;
     }
   }
diff --git a/clang/lib/Driver/ToolChains/AMDGPU.cpp b/clang/lib/Driver/ToolChains/AMDGPU.cpp
index b5eaf1adca6b..43ce33750eba 100644
--- a/clang/lib/Driver/ToolChains/AMDGPU.cpp
+++ b/clang/lib/Driver/ToolChains/AMDGPU.cpp
@@ -478,7 +478,8 @@ void RocmInstallationDetector::print(raw_ostream &OS) const {
 
 void RocmInstallationDetector::AddHIPIncludeArgs(const ArgList &DriverArgs,
                                                  ArgStringList &CC1Args) const {
-  bool UsesRuntimeWrapper = VersionMajorMinor > llvm::VersionTuple(3, 5);
+  bool UsesRuntimeWrapper = VersionMajorMinor > llvm::VersionTuple(3, 5) &&
+                            !DriverArgs.hasArg(options::OPT_nohipwrapperinc);
 
   if (!DriverArgs.hasArg(options::OPT_nobuiltininc)) {
     // HIP header includes standard library wrapper headers under clang
diff --git a/clang/lib/Driver/ToolChains/AMDGPUOpenMP.cpp b/clang/lib/Driver/ToolChains/AMDGPUOpenMP.cpp
index 863e2c597d53..f282f04b7931 100644
--- a/clang/lib/Driver/ToolChains/AMDGPUOpenMP.cpp
+++ b/clang/lib/Driver/ToolChains/AMDGPUOpenMP.cpp
@@ -267,7 +267,7 @@ void AMDGPUOpenMPToolChain::addClangTargetOptions(
 
   std::string BitcodeSuffix;
   if (DriverArgs.hasFlag(options::OPT_fopenmp_target_new_runtime,
-                         options::OPT_fno_openmp_target_new_runtime, false))
+                         options::OPT_fno_openmp_target_new_runtime, true))
     BitcodeSuffix = "new-amdgpu-" + GPUArch;
   else
     BitcodeSuffix = "amdgcn-" + GPUArch;
diff --git a/clang/lib/Driver/ToolChains/Arch/AArch64.cpp b/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
index abc32f22d2a1..be13d6d583ce 100644
--- a/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
@@ -568,4 +568,15 @@ fp16_fml_fallthrough:
 
   if (Args.hasArg(options::OPT_mno_neg_immediates))
     Features.push_back("+no-neg-immediates");
+
+  if (Arg *A = Args.getLastArg(options::OPT_mfix_cortex_a53_835769,
+                               options::OPT_mno_fix_cortex_a53_835769)) {
+    if (A->getOption().matches(options::OPT_mfix_cortex_a53_835769))
+      Features.push_back("+fix-cortex-a53-835769");
+    else
+      Features.push_back("-fix-cortex-a53-835769");
+  } else if (Triple.isAndroid()) {
+    // Enabled A53 errata (835769) workaround by default on android
+    Features.push_back("+fix-cortex-a53-835769");
+  }
 }
diff --git a/clang/lib/Driver/ToolChains/Arch/ARM.cpp b/clang/lib/Driver/ToolChains/Arch/ARM.cpp
index 21c091e1a0ba..4013cf230026 100644
--- a/clang/lib/Driver/ToolChains/Arch/ARM.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/ARM.cpp
@@ -147,19 +147,27 @@ bool arm::useAAPCSForMachO(const llvm::Triple &T) {
          T.getOS() == llvm::Triple::UnknownOS || isARMMProfile(T);
 }
 
+// We follow GCC and support when the backend has support for the MRC/MCR
+// instructions that are used to set the hard thread pointer ("CP15 C13
+// Thread id").
+bool arm::isHardTPSupported(const llvm::Triple &Triple) {
+  int Ver = getARMSubArchVersionNumber(Triple);
+  llvm::ARM::ArchKind AK = llvm::ARM::parseArch(Triple.getArchName());
+  return Triple.isARM() || AK == llvm::ARM::ArchKind::ARMV6T2 ||
+         (Ver >= 7 && AK != llvm::ARM::ArchKind::ARMV8MBaseline);
+}
+
 // Select mode for reading thread pointer (-mtp=soft/cp15).
 arm::ReadTPMode arm::getReadTPMode(const Driver &D, const ArgList &Args,
-                                   const llvm::Triple &Triple) {
+                                   const llvm::Triple &Triple, bool ForAS) {
   if (Arg *A = Args.getLastArg(options::OPT_mtp_mode_EQ)) {
     arm::ReadTPMode ThreadPointer =
         llvm::StringSwitch<arm::ReadTPMode>(A->getValue())
             .Case("cp15", ReadTPMode::Cp15)
             .Case("soft", ReadTPMode::Soft)
             .Default(ReadTPMode::Invalid);
-    if (ThreadPointer == ReadTPMode::Cp15 &&
-        getARMSubArchVersionNumber(Triple) < 7 &&
-        llvm::ARM::parseArch(Triple.getArchName()) !=
-            llvm::ARM::ArchKind::ARMV6T2) {
+    if (ThreadPointer == ReadTPMode::Cp15 && !isHardTPSupported(Triple) &&
+        !ForAS) {
       D.Diag(diag::err_target_unsupported_tp_hard) << Triple.getArchName();
       return ReadTPMode::Invalid;
     }
@@ -430,7 +438,6 @@ void arm::getARMTargetFeatures(const Driver &D, const llvm::Triple &Triple,
   bool KernelOrKext =
       Args.hasArg(options::OPT_mkernel, options::OPT_fapple_kext);
   arm::FloatABI ABI = arm::getARMFloatABI(D, Triple, Args);
-  arm::ReadTPMode ThreadPointer = arm::getReadTPMode(D, Args, Triple);
   llvm::Optional<std::pair<const Arg *, StringRef>> WaCPU, WaFPU, WaHDiv,
       WaArch;
 
@@ -482,7 +489,7 @@ void arm::getARMTargetFeatures(const Driver &D, const llvm::Triple &Triple,
     }
   }
 
-  if (ThreadPointer == arm::ReadTPMode::Cp15)
+  if (getReadTPMode(D, Args, Triple, ForAS) == ReadTPMode::Cp15)
     Features.push_back("+read-tp-hard");
 
   const Arg *ArchArg = Args.getLastArg(options::OPT_march_EQ);
@@ -869,6 +876,8 @@ fp16_fml_fallthrough:
     }
   }
 
+  if (Args.getLastArg(options::OPT_mno_bti_at_return_twice))
+    Features.push_back("+no-bti-at-return-twice");
 }
 
 std::string arm::getARMArch(StringRef Arch, const llvm::Triple &Triple) {
diff --git a/clang/lib/Driver/ToolChains/Arch/ARM.h b/clang/lib/Driver/ToolChains/Arch/ARM.h
index b6fd68fbb9c6..881b63bd36b9 100644
--- a/clang/lib/Driver/ToolChains/Arch/ARM.h
+++ b/clang/lib/Driver/ToolChains/Arch/ARM.h
@@ -53,8 +53,9 @@ FloatABI getARMFloatABI(const Driver &D, const llvm::Triple &Triple,
                         const llvm::opt::ArgList &Args);
 void setFloatABIInTriple(const Driver &D, const llvm::opt::ArgList &Args,
                          llvm::Triple &triple);
+bool isHardTPSupported(const llvm::Triple &Triple);
 ReadTPMode getReadTPMode(const Driver &D, const llvm::opt::ArgList &Args,
-                         const llvm::Triple &Triple);
+                         const llvm::Triple &Triple, bool ForAS);
 void setArchNameInTriple(const Driver &D, const llvm::opt::ArgList &Args,
                          types::ID InputType, llvm::Triple &Triple);
 
diff --git a/clang/lib/Driver/ToolChains/Arch/RISCV.cpp b/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
index 323f588c8269..7ad8ca69bed5 100644
--- a/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
@@ -194,27 +194,11 @@ StringRef riscv::getRISCVABI(const ArgList &Args, const llvm::Triple &Triple) {
 
   auto ParseResult = llvm::RISCVISAInfo::parseArchString(
       Arch, /* EnableExperimentalExtension */ true);
-  if (!ParseResult) {
+  if (!ParseResult)
     // Ignore parsing error, just go 3rd step.
     consumeError(ParseResult.takeError());
-  } else {
-    auto &ISAInfo = *ParseResult;
-    bool HasD = ISAInfo->hasExtension("d");
-    unsigned XLen = ISAInfo->getXLen();
-    if (XLen == 32) {
-      bool HasE = ISAInfo->hasExtension("e");
-      if (HasD)
-        return "ilp32d";
-      if (HasE)
-        return "ilp32e";
-      return "ilp32";
-    } else if (XLen == 64) {
-      if (HasD)
-        return "lp64d";
-      return "lp64";
-    }
-    llvm_unreachable("unhandled XLen");
-  }
+  else
+    return llvm::RISCV::computeDefaultABIFromArch(**ParseResult);
 
   // 3. Choose a default based on the triple
   //
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index c5aaa067c4f5..65347a38490e 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -625,8 +625,9 @@ getFramePointerKind(const ArgList &Args, const llvm::Triple &Triple) {
 }
 
 /// Add a CC1 option to specify the debug compilation directory.
-static void addDebugCompDirArg(const ArgList &Args, ArgStringList &CmdArgs,
-                               const llvm::vfs::FileSystem &VFS) {
+static const char *addDebugCompDirArg(const ArgList &Args,
+                                      ArgStringList &CmdArgs,
+                                      const llvm::vfs::FileSystem &VFS) {
   if (Arg *A = Args.getLastArg(options::OPT_ffile_compilation_dir_EQ,
                                options::OPT_fdebug_compilation_dir_EQ)) {
     if (A->getOption().matches(options::OPT_ffile_compilation_dir_EQ))
@@ -638,6 +639,31 @@ static void addDebugCompDirArg(const ArgList &Args, ArgStringList &CmdArgs,
                  VFS.getCurrentWorkingDirectory()) {
     CmdArgs.push_back(Args.MakeArgString("-fdebug-compilation-dir=" + *CWD));
   }
+  StringRef Path(CmdArgs.back());
+  return Path.substr(Path.find('=') + 1).data();
+}
+
+static void addDebugObjectName(const ArgList &Args, ArgStringList &CmdArgs,
+                               const char *DebugCompilationDir,
+                               const char *OutputFileName) {
+  // No need to generate a value for -object-file-name if it was provided.
+  for (auto *Arg : Args.filtered(options::OPT_Xclang))
+    if (StringRef(Arg->getValue()).startswith("-object-file-name"))
+      return;
+
+  if (Args.hasArg(options::OPT_object_file_name_EQ))
+    return;
+
+  SmallString<128> ObjFileNameForDebug(OutputFileName);
+  if (ObjFileNameForDebug != "-" &&
+      !llvm::sys::path::is_absolute(ObjFileNameForDebug) &&
+      (!DebugCompilationDir ||
+       llvm::sys::path::is_absolute(DebugCompilationDir))) {
+    // Make the path absolute in the debug infos like MSVC does.
+    llvm::sys::fs::make_absolute(ObjFileNameForDebug);
+  }
+  CmdArgs.push_back(
+      Args.MakeArgString(Twine("-object-file-name=") + ObjFileNameForDebug));
 }
 
 /// Add a CC1 and CC1AS option to specify the debug file path prefix map.
@@ -976,11 +1002,7 @@ static bool ContainsCompileAction(const Action *A) {
   if (isa<CompileJobAction>(A) || isa<BackendJobAction>(A))
     return true;
 
-  for (const auto &AI : A->inputs())
-    if (ContainsCompileAction(AI))
-      return true;
-
-  return false;
+  return llvm::any_of(A->inputs(), ContainsCompileAction);
 }
 
 /// Check if -relax-all should be passed to the internal assembler.
@@ -1806,19 +1828,6 @@ void Clang::AddAArch64TargetArgs(const ArgList &Args,
 
   RenderAArch64ABI(Triple, Args, CmdArgs);
 
-  if (Arg *A = Args.getLastArg(options::OPT_mfix_cortex_a53_835769,
-                               options::OPT_mno_fix_cortex_a53_835769)) {
-    CmdArgs.push_back("-mllvm");
-    if (A->getOption().matches(options::OPT_mfix_cortex_a53_835769))
-      CmdArgs.push_back("-aarch64-fix-cortex-a53-835769=1");
-    else
-      CmdArgs.push_back("-aarch64-fix-cortex-a53-835769=0");
-  } else if (Triple.isAndroid()) {
-    // Enabled A53 errata (835769) workaround by default on android
-    CmdArgs.push_back("-mllvm");
-    CmdArgs.push_back("-aarch64-fix-cortex-a53-835769=1");
-  }
-
   // Forward the -mglobal-merge option for explicit control over the pass.
   if (Arg *A = Args.getLastArg(options::OPT_mglobal_merge,
                                options::OPT_mno_global_merge)) {
@@ -5666,7 +5675,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back("-fno-autolink");
 
   // Add in -fdebug-compilation-dir if necessary.
-  addDebugCompDirArg(Args, CmdArgs, D.getVFS());
+  const char *DebugCompilationDir =
+      addDebugCompDirArg(Args, CmdArgs, D.getVFS());
 
   addDebugPrefixMapArg(D, Args, CmdArgs);
 
@@ -5904,7 +5914,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
       // runtime.
       if (Args.hasFlag(options::OPT_fopenmp_target_new_runtime,
                        options::OPT_fno_openmp_target_new_runtime,
-                       /*Default=*/!getToolChain().getTriple().isAMDGCN()))
+                       /*Default=*/true))
         CmdArgs.push_back("-fopenmp-target-new-runtime");
 
       // When in OpenMP offloading mode, enable debugging on the device.
@@ -6997,18 +7007,18 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
   if (Arg *A = Args.getLastArg(options::OPT_moutline_atomics,
                                options::OPT_mno_outline_atomics)) {
-    if (A->getOption().matches(options::OPT_moutline_atomics)) {
-      // Option -moutline-atomics supported for AArch64 target only.
-      if (!Triple.isAArch64()) {
-        D.Diag(diag::warn_drv_moutline_atomics_unsupported_opt)
-            << Triple.getArchName();
-      } else {
+    // Option -moutline-atomics supported for AArch64 target only.
+    if (!Triple.isAArch64()) {
+      D.Diag(diag::warn_drv_moutline_atomics_unsupported_opt)
+          << Triple.getArchName() << A->getOption().getName();
+    } else {
+      if (A->getOption().matches(options::OPT_moutline_atomics)) {
         CmdArgs.push_back("-target-feature");
         CmdArgs.push_back("+outline-atomics");
+      } else {
+        CmdArgs.push_back("-target-feature");
+        CmdArgs.push_back("-outline-atomics");
       }
-    } else {
-      CmdArgs.push_back("-target-feature");
-      CmdArgs.push_back("-outline-atomics");
     }
   } else if (Triple.isAArch64() &&
              getToolChain().IsAArch64OutlineAtomicsDefault(Args)) {
@@ -7038,6 +7048,11 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back(Args.MakeArgString(Str));
   }
 
+  // Add the output path to the object file for CodeView debug infos.
+  if (EmitCodeView && Output.isFilename())
+    addDebugObjectName(Args, CmdArgs, DebugCompilationDir,
+                       Output.getFilename());
+
   // Add the "-o out -x type src.c" flags last. This is done primarily to make
   // the -cc1 command easier to edit when reproducing compiler crashes.
   if (Output.getType() == types::TY_Dependencies) {
@@ -7111,11 +7126,11 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   Args.ClaimAllArgs(options::OPT_emit_llvm);
 }
 
-Clang::Clang(const ToolChain &TC)
+Clang::Clang(const ToolChain &TC, bool HasIntegratedBackend)
     // CAUTION! The first constructor argument ("clang") is not arbitrary,
     // as it is for other tools. Some operations on a Tool actually test
     // whether that tool is Clang based on the Tool's Name as a string.
-    : Tool("clang", "clang frontend", TC) {}
+    : Tool("clang", "clang frontend", TC), HasBackend(HasIntegratedBackend) {}
 
 Clang::~Clang() {}
 
@@ -7655,11 +7670,14 @@ void ClangAs::ConstructJob(Compilation &C, const JobAction &JA,
   Args.AddAllArgs(CmdArgs, options::OPT_I_Group);
 
   // Determine the original source input.
-  const Action *SourceAction = &JA;
-  while (SourceAction->getKind() != Action::InputClass) {
-    assert(!SourceAction->getInputs().empty() && "unexpected root action!");
-    SourceAction = SourceAction->getInputs()[0];
-  }
+  auto FindSource = [](const Action *S) -> const Action * {
+    while (S->getKind() != Action::InputClass) {
+      assert(!S->getInputs().empty() && "unexpected root action!");
+      S = S->getInputs()[0];
+    }
+    return S;
+  };
+  const Action *SourceAction = FindSource(&JA);
 
   // Forward -g and handle debug info related flags, assuming we are dealing
   // with an actual assembly file.
@@ -7678,6 +7696,10 @@ void ClangAs::ConstructJob(Compilation &C, const JobAction &JA,
 
   codegenoptions::DebugInfoKind DebugInfoKind = codegenoptions::NoDebugInfo;
 
+  // Add the -fdebug-compilation-dir flag if needed.
+  const char *DebugCompilationDir =
+      addDebugCompDirArg(Args, CmdArgs, C.getDriver().getVFS());
+
   if (SourceAction->getType() == types::TY_Asm ||
       SourceAction->getType() == types::TY_PP_Asm) {
     // You might think that it would be ok to set DebugInfoKind outside of
@@ -7686,8 +7708,6 @@ void ClangAs::ConstructJob(Compilation &C, const JobAction &JA,
     // and it's not clear whether that test is just overly restrictive.
     DebugInfoKind = (WantDebug ? codegenoptions::DebugInfoConstructor
                                : codegenoptions::NoDebugInfo);
-    // Add the -fdebug-compilation-dir flag if needed.
-    addDebugCompDirArg(Args, CmdArgs, C.getDriver().getVFS());
 
     addDebugPrefixMapArg(getToolChain().getDriver(), Args, CmdArgs);
 
@@ -7798,6 +7818,29 @@ void ClangAs::ConstructJob(Compilation &C, const JobAction &JA,
 
   Args.AddAllArgs(CmdArgs, options::OPT_mllvm);
 
+  if (DebugInfoKind > codegenoptions::NoDebugInfo && Output.isFilename())
+    addDebugObjectName(Args, CmdArgs, DebugCompilationDir,
+                       Output.getFilename());
+
+  // Fixup any previous commands that use -object-file-name because when we
+  // generated them, the final .obj name wasn't yet known.
+  for (Command &J : C.getJobs()) {
+    if (SourceAction != FindSource(&J.getSource()))
+      continue;
+    auto &JArgs = J.getArguments();
+    for (unsigned I = 0; I < JArgs.size(); ++I) {
+      if (StringRef(JArgs[I]).startswith("-object-file-name=") &&
+          Output.isFilename()) {
+        ArgStringList NewArgs(JArgs.begin(), JArgs.begin() + I);
+        addDebugObjectName(Args, NewArgs, DebugCompilationDir,
+                           Output.getFilename());
+        NewArgs.append(JArgs.begin() + I + 1, JArgs.end());
+        J.replaceArguments(NewArgs);
+        break;
+      }
+    }
+  }
+
   assert(Output.isFilename() && "Unexpected lipo output.");
   CmdArgs.push_back("-o");
   CmdArgs.push_back(Output.getFilename());
@@ -7878,7 +7921,7 @@ void OffloadBundler::ConstructJob(Compilation &C, const JobAction &JA,
     Triples += '-';
     Triples += CurTC->getTriple().normalize();
     if ((CurKind == Action::OFK_HIP || CurKind == Action::OFK_Cuda) &&
-        CurDep->getOffloadingArch()) {
+        !StringRef(CurDep->getOffloadingArch()).empty()) {
       Triples += '-';
       Triples += CurDep->getOffloadingArch();
     }
diff --git a/clang/lib/Driver/ToolChains/Clang.h b/clang/lib/Driver/ToolChains/Clang.h
index d4b4988b4a8c..00e0490e069b 100644
--- a/clang/lib/Driver/ToolChains/Clang.h
+++ b/clang/lib/Driver/ToolChains/Clang.h
@@ -26,6 +26,10 @@ namespace tools {
 
 /// Clang compiler tool.
 class LLVM_LIBRARY_VISIBILITY Clang : public Tool {
+  // Indicates whether this instance has integrated backend using
+  // internal LLVM infrastructure.
+  bool HasBackend;
+
 public:
   static const char *getBaseInputName(const llvm::opt::ArgList &Args,
                                       const InputInfo &Input);
@@ -99,11 +103,12 @@ private:
       const InputInfo &Input, const llvm::opt::ArgList &Args) const;
 
 public:
-  Clang(const ToolChain &TC);
+  Clang(const ToolChain &TC, bool HasIntegratedBackend = true);
   ~Clang() override;
 
   bool hasGoodDiagnostics() const override { return true; }
   bool hasIntegratedAssembler() const override { return true; }
+  bool hasIntegratedBackend() const override { return HasBackend; }
   bool hasIntegratedCPP() const override { return true; }
   bool canEmitIR() const override { return true; }
 
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index 630baf9d6ae6..407f81a2ae09 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -15,7 +15,7 @@
 #include "Arch/SystemZ.h"
 #include "Arch/VE.h"
 #include "Arch/X86.h"
-#include "HIP.h"
+#include "HIPAMD.h"
 #include "Hexagon.h"
 #include "clang/Basic/CharInfo.h"
 #include "clang/Basic/LangOptions.h"
diff --git a/clang/lib/Driver/ToolChains/Darwin.cpp b/clang/lib/Driver/ToolChains/Darwin.cpp
index 06d3edc70e45..f7da3f187814 100644
--- a/clang/lib/Driver/ToolChains/Darwin.cpp
+++ b/clang/lib/Driver/ToolChains/Darwin.cpp
@@ -209,8 +209,7 @@ static bool shouldLinkerNotDedup(bool IsLinkerOnlyAction, const ArgList &Args) {
 void darwin::Linker::AddLinkArgs(Compilation &C, const ArgList &Args,
                                  ArgStringList &CmdArgs,
                                  const InputInfoList &Inputs,
-                                 unsigned Version[5], bool LinkerIsLLD,
-                                 bool LinkerIsLLDDarwinNew) const {
+                                 unsigned Version[5], bool LinkerIsLLD) const {
   const Driver &D = getToolChain().getDriver();
   const toolchains::MachO &MachOTC = getMachOToolChain();
 
@@ -343,7 +342,7 @@ void darwin::Linker::AddLinkArgs(Compilation &C, const ArgList &Args,
   Args.AddAllArgs(CmdArgs, options::OPT_init);
 
   // Add the deployment target.
-  if (Version[0] >= 520 || LinkerIsLLDDarwinNew)
+  if (Version[0] >= 520 || LinkerIsLLD)
     MachOTC.addPlatformVersionArgs(Args, CmdArgs);
   else
     MachOTC.addMinVersionArgs(Args, CmdArgs);
@@ -556,14 +555,13 @@ void darwin::Linker::ConstructJob(Compilation &C, const JobAction &JA,
           << A->getAsString(Args);
   }
 
-  bool LinkerIsLLD, LinkerIsLLDDarwinNew;
-  const char *Exec = Args.MakeArgString(
-      getToolChain().GetLinkerPath(&LinkerIsLLD, &LinkerIsLLDDarwinNew));
+  bool LinkerIsLLD;
+  const char *Exec =
+      Args.MakeArgString(getToolChain().GetLinkerPath(&LinkerIsLLD));
 
   // I'm not sure why this particular decomposition exists in gcc, but
   // we follow suite for ease of comparison.
-  AddLinkArgs(C, Args, CmdArgs, Inputs, Version, LinkerIsLLD,
-              LinkerIsLLDDarwinNew);
+  AddLinkArgs(C, Args, CmdArgs, Inputs, Version, LinkerIsLLD);
 
   if (willEmitRemarks(Args) &&
       checkRemarksOptions(getToolChain().getDriver(), Args,
@@ -715,7 +713,7 @@ void darwin::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   }
 
   ResponseFileSupport ResponseSupport;
-  if (Version[0] >= 705 || LinkerIsLLDDarwinNew) {
+  if (Version[0] >= 705 || LinkerIsLLD) {
     ResponseSupport = ResponseFileSupport::AtFileUTF8();
   } else {
     // For older versions of the linker, use the legacy filelist method instead.
@@ -1412,8 +1410,8 @@ static std::string getSystemOrSDKMacOSVersion(StringRef MacOSSDKVersion) {
   llvm::Triple SystemTriple(llvm::sys::getProcessTriple());
   if (!SystemTriple.isMacOSX())
     return std::string(MacOSSDKVersion);
-  SystemTriple.getMacOSXVersion(Major, Minor, Micro);
-  VersionTuple SystemVersion(Major, Minor, Micro);
+  VersionTuple SystemVersion;
+  SystemTriple.getMacOSXVersion(SystemVersion);
   bool HadExtra;
   if (!Driver::GetReleaseVersion(MacOSSDKVersion, Major, Minor, Micro,
                                  HadExtra))
@@ -1554,12 +1552,10 @@ struct DarwinPlatform {
                    const Optional<DarwinSDKInfo> &SDKInfo) {
     DarwinPlatform Result(TargetArg, getPlatformFromOS(TT.getOS()), OSVersion,
                           A);
-    unsigned Major, Minor, Micro;
-    TT.getOSVersion(Major, Minor, Micro);
-    if (Major == 0)
+    VersionTuple OsVersion = TT.getOSVersion();
+    if (OsVersion.getMajor() == 0)
       Result.HasOSVersion = false;
-    Result.setEnvironment(TT.getEnvironment(),
-                          VersionTuple(Major, Minor, Micro), SDKInfo);
+    Result.setEnvironment(TT.getEnvironment(), OsVersion, SDKInfo);
     return Result;
   }
   static DarwinPlatform
@@ -1805,7 +1801,7 @@ inferDeploymentTargetFromSDK(DerivedArgList &Args,
 
 std::string getOSVersion(llvm::Triple::OSType OS, const llvm::Triple &Triple,
                          const Driver &TheDriver) {
-  unsigned Major, Minor, Micro;
+  VersionTuple OsVersion;
   llvm::Triple SystemTriple(llvm::sys::getProcessTriple());
   switch (OS) {
   case llvm::Triple::Darwin:
@@ -1814,24 +1810,22 @@ std::string getOSVersion(llvm::Triple::OSType OS, const llvm::Triple &Triple,
     // macos, use the host triple to infer OS version.
     if (Triple.isMacOSX() && SystemTriple.isMacOSX() &&
         !Triple.getOSMajorVersion())
-      SystemTriple.getMacOSXVersion(Major, Minor, Micro);
-    else if (!Triple.getMacOSXVersion(Major, Minor, Micro))
+      SystemTriple.getMacOSXVersion(OsVersion);
+    else if (!Triple.getMacOSXVersion(OsVersion))
       TheDriver.Diag(diag::err_drv_invalid_darwin_version)
           << Triple.getOSName();
     break;
   case llvm::Triple::IOS:
     if (Triple.isMacCatalystEnvironment() && !Triple.getOSMajorVersion()) {
-      Major = 13;
-      Minor = 1;
-      Micro = 0;
+      OsVersion = VersionTuple(13, 1);
     } else
-      Triple.getiOSVersion(Major, Minor, Micro);
+      OsVersion = Triple.getiOSVersion();
     break;
   case llvm::Triple::TvOS:
-    Triple.getOSVersion(Major, Minor, Micro);
+    OsVersion = Triple.getOSVersion();
     break;
   case llvm::Triple::WatchOS:
-    Triple.getWatchOSVersion(Major, Minor, Micro);
+    OsVersion = Triple.getWatchOSVersion();
     break;
   default:
     llvm_unreachable("Unexpected OS type");
@@ -1839,7 +1833,9 @@ std::string getOSVersion(llvm::Triple::OSType OS, const llvm::Triple &Triple,
   }
 
   std::string OSVersion;
-  llvm::raw_string_ostream(OSVersion) << Major << '.' << Minor << '.' << Micro;
+  llvm::raw_string_ostream(OSVersion)
+      << OsVersion.getMajor() << '.' << OsVersion.getMinor().getValueOr(0)
+      << '.' << OsVersion.getSubminor().getValueOr(0);
   return OSVersion;
 }
 
@@ -1909,15 +1905,13 @@ getDeploymentTargetFromMTargetOSArg(DerivedArgList &Args,
     return None;
   }
 
-  unsigned Major, Minor, Micro;
-  TT.getOSVersion(Major, Minor, Micro);
-  if (!Major) {
+  VersionTuple Version = TT.getOSVersion();
+  if (!Version.getMajor()) {
     TheDriver.Diag(diag::err_drv_invalid_version_number)
         << A->getAsString(Args);
     return None;
   }
-  return DarwinPlatform::createFromMTargetOS(TT.getOS(),
-                                             VersionTuple(Major, Minor, Micro),
+  return DarwinPlatform::createFromMTargetOS(TT.getOS(), Version,
                                              TT.getEnvironment(), A, SDKInfo);
 }
 
diff --git a/clang/lib/Driver/ToolChains/Darwin.h b/clang/lib/Driver/ToolChains/Darwin.h
index a307cd317ac3..5e23047a5512 100644
--- a/clang/lib/Driver/ToolChains/Darwin.h
+++ b/clang/lib/Driver/ToolChains/Darwin.h
@@ -64,7 +64,7 @@ class LLVM_LIBRARY_VISIBILITY Linker : public MachOTool {
   void AddLinkArgs(Compilation &C, const llvm::opt::ArgList &Args,
                    llvm::opt::ArgStringList &CmdArgs,
                    const InputInfoList &Inputs, unsigned Version[5],
-                   bool LinkerIsLLD, bool LinkerIsLLDDarwinNew) const;
+                   bool LinkerIsLLD) const;
 
 public:
   Linker(const ToolChain &TC) : MachOTool("darwin::Linker", "linker", TC) {}
diff --git a/clang/lib/Driver/ToolChains/FreeBSD.cpp b/clang/lib/Driver/ToolChains/FreeBSD.cpp
index d08ea282f6df..de635f5816cf 100644
--- a/clang/lib/Driver/ToolChains/FreeBSD.cpp
+++ b/clang/lib/Driver/ToolChains/FreeBSD.cpp
@@ -293,8 +293,8 @@ void freebsd::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   addLinkerCompressDebugSectionsOption(ToolChain, Args, CmdArgs);
   AddLinkerInputs(ToolChain, Inputs, Args, CmdArgs, JA);
 
-  bool Profiling = Args.hasArg(options::OPT_pg) &&
-                   ToolChain.getTriple().getOSMajorVersion() < 14;
+  unsigned Major = ToolChain.getTriple().getOSMajorVersion();
+  bool Profiling = Args.hasArg(options::OPT_pg) && Major != 0 && Major < 14;
   if (!Args.hasArg(options::OPT_nostdlib, options::OPT_nodefaultlibs)) {
     // Use the static OpenMP runtime with -static-openmp
     bool StaticOpenMP = Args.hasArg(options::OPT_static_openmp) &&
@@ -419,8 +419,8 @@ void FreeBSD::addLibStdCxxIncludePaths(
 void FreeBSD::AddCXXStdlibLibArgs(const ArgList &Args,
                                   ArgStringList &CmdArgs) const {
   CXXStdlibType Type = GetCXXStdlibType(Args);
-  bool Profiling =
-      Args.hasArg(options::OPT_pg) && getTriple().getOSMajorVersion() < 14;
+  unsigned Major = getTriple().getOSMajorVersion();
+  bool Profiling = Args.hasArg(options::OPT_pg) && Major != 0 && Major < 14;
 
   switch (Type) {
   case ToolChain::CST_Libcxx:
diff --git a/clang/lib/Driver/ToolChains/HIP.cpp b/clang/lib/Driver/ToolChains/HIPAMD.cpp
index 07af1a0457c7..6d553791b394 100644
--- a/clang/lib/Driver/ToolChains/HIP.cpp
+++ b/clang/lib/Driver/ToolChains/HIPAMD.cpp
@@ -1,4 +1,4 @@
-//===--- HIP.cpp - HIP Tool and ToolChain Implementations -------*- C++ -*-===//
+//===--- HIPAMD.cpp - HIP Tool and ToolChain Implementations ----*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,9 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "HIP.h"
+#include "HIPAMD.h"
 #include "AMDGPU.h"
 #include "CommonArgs.h"
+#include "HIPUtility.h"
 #include "clang/Basic/Cuda.h"
 #include "clang/Basic/TargetID.h"
 #include "clang/Driver/Compilation.h"
@@ -34,10 +35,6 @@ using namespace llvm::opt;
 #define NULL_FILE "/dev/null"
 #endif
 
-namespace {
-const unsigned HIPCodeObjectAlign = 4096;
-} // namespace
-
 static bool shouldSkipSanitizeOption(const ToolChain &TC,
                                      const llvm::opt::ArgList &DriverArgs,
                                      StringRef TargetID,
@@ -76,9 +73,9 @@ static bool shouldSkipSanitizeOption(const ToolChain &TC,
 }
 
 void AMDGCN::Linker::constructLldCommand(Compilation &C, const JobAction &JA,
-                                          const InputInfoList &Inputs,
-                                          const InputInfo &Output,
-                                          const llvm::opt::ArgList &Args) const {
+                                         const InputInfoList &Inputs,
+                                         const InputInfo &Output,
+                                         const llvm::opt::ArgList &Args) const {
   // Construct lld command.
   // The output from ld.lld is an HSA code object file.
   ArgStringList LldArgs{"-flavor", "gnu", "--no-undefined", "-shared",
@@ -129,142 +126,28 @@ void AMDGCN::Linker::constructLldCommand(Compilation &C, const JobAction &JA,
                                          Lld, LldArgs, Inputs, Output));
 }
 
-// Construct a clang-offload-bundler command to bundle code objects for
-// different GPU's into a HIP fat binary.
-void AMDGCN::constructHIPFatbinCommand(Compilation &C, const JobAction &JA,
-                  StringRef OutputFileName, const InputInfoList &Inputs,
-                  const llvm::opt::ArgList &Args, const Tool& T) {
-  // Construct clang-offload-bundler command to bundle object files for
-  // for different GPU archs.
-  ArgStringList BundlerArgs;
-  BundlerArgs.push_back(Args.MakeArgString("-type=o"));
-  BundlerArgs.push_back(
-      Args.MakeArgString("-bundle-align=" + Twine(HIPCodeObjectAlign)));
-
-  // ToDo: Remove the dummy host binary entry which is required by
-  // clang-offload-bundler.
-  std::string BundlerTargetArg = "-targets=host-x86_64-unknown-linux";
-  std::string BundlerInputArg = "-inputs=" NULL_FILE;
-
-  // For code object version 2 and 3, the offload kind in bundle ID is 'hip'
-  // for backward compatibility. For code object version 4 and greater, the
-  // offload kind in bundle ID is 'hipv4'.
-  std::string OffloadKind = "hip";
-  if (getAMDGPUCodeObjectVersion(C.getDriver(), Args) >= 4)
-    OffloadKind = OffloadKind + "v4";
-  for (const auto &II : Inputs) {
-    const auto* A = II.getAction();
-    BundlerTargetArg = BundlerTargetArg + "," + OffloadKind +
-                       "-amdgcn-amd-amdhsa--" +
-                       StringRef(A->getOffloadingArch()).str();
-    BundlerInputArg = BundlerInputArg + "," + II.getFilename();
-  }
-  BundlerArgs.push_back(Args.MakeArgString(BundlerTargetArg));
-  BundlerArgs.push_back(Args.MakeArgString(BundlerInputArg));
-
-  std::string Output = std::string(OutputFileName);
-  auto BundlerOutputArg =
-      Args.MakeArgString(std::string("-outputs=").append(Output));
-  BundlerArgs.push_back(BundlerOutputArg);
-
-  const char *Bundler = Args.MakeArgString(
-      T.getToolChain().GetProgramPath("clang-offload-bundler"));
-  C.addCommand(std::make_unique<Command>(
-      JA, T, ResponseFileSupport::None(), Bundler, BundlerArgs, Inputs,
-      InputInfo(&JA, Args.MakeArgString(Output))));
-}
-
-/// Add Generated HIP Object File which has device images embedded into the
-/// host to the argument list for linking. Using MC directives, embed the
-/// device code and also define symbols required by the code generation so that
-/// the image can be retrieved at runtime.
-void AMDGCN::Linker::constructGenerateObjFileFromHIPFatBinary(
-    Compilation &C, const InputInfo &Output,
-    const InputInfoList &Inputs, const ArgList &Args,
-    const JobAction &JA) const {
-  const ToolChain &TC = getToolChain();
-  std::string Name =
-      std::string(llvm::sys::path::stem(Output.getFilename()));
-
-  // Create Temp Object File Generator,
-  // Offload Bundled file and Bundled Object file.
-  // Keep them if save-temps is enabled.
-  const char *McinFile;
-  const char *BundleFile;
-  if (C.getDriver().isSaveTempsEnabled()) {
-    McinFile = C.getArgs().MakeArgString(Name + ".mcin");
-    BundleFile = C.getArgs().MakeArgString(Name + ".hipfb");
-  } else {
-    auto TmpNameMcin = C.getDriver().GetTemporaryPath(Name, "mcin");
-    McinFile = C.addTempFile(C.getArgs().MakeArgString(TmpNameMcin));
-    auto TmpNameFb = C.getDriver().GetTemporaryPath(Name, "hipfb");
-    BundleFile = C.addTempFile(C.getArgs().MakeArgString(TmpNameFb));
-  }
-  constructHIPFatbinCommand(C, JA, BundleFile, Inputs, Args, *this);
-
-  // Create a buffer to write the contents of the temp obj generator.
-  std::string ObjBuffer;
-  llvm::raw_string_ostream ObjStream(ObjBuffer);
-
-  // Add MC directives to embed target binaries. We ensure that each
-  // section and image is 16-byte aligned. This is not mandatory, but
-  // increases the likelihood of data to be aligned with a cache block
-  // in several main host machines.
-  ObjStream << "#       HIP Object Generator\n";
-  ObjStream << "# *** Automatically generated by Clang ***\n";
-  ObjStream << "  .protected __hip_fatbin\n";
-  ObjStream << "  .type __hip_fatbin,@object\n";
-  ObjStream << "  .section .hip_fatbin,\"a\",@progbits\n";
-  ObjStream << "  .globl __hip_fatbin\n";
-  ObjStream << "  .p2align " << llvm::Log2(llvm::Align(HIPCodeObjectAlign))
-            << "\n";
-  ObjStream << "__hip_fatbin:\n";
-  ObjStream << "  .incbin \"" << BundleFile << "\"\n";
-  ObjStream.flush();
-
-  // Dump the contents of the temp object file gen if the user requested that.
-  // We support this option to enable testing of behavior with -###.
-  if (C.getArgs().hasArg(options::OPT_fhip_dump_offload_linker_script))
-    llvm::errs() << ObjBuffer;
-
-  // Open script file and write the contents.
-  std::error_code EC;
-  llvm::raw_fd_ostream Objf(McinFile, EC, llvm::sys::fs::OF_None);
-
-  if (EC) {
-    C.getDriver().Diag(clang::diag::err_unable_to_make_temp) << EC.message();
-    return;
-  }
-
-  Objf << ObjBuffer;
-
-  ArgStringList McArgs{"-o",      Output.getFilename(),
-                       McinFile,  "--filetype=obj"};
-  const char *Mc = Args.MakeArgString(TC.GetProgramPath("llvm-mc"));
-  C.addCommand(std::make_unique<Command>(JA, *this, ResponseFileSupport::None(),
-                                         Mc, McArgs, Inputs, Output));
-}
-
 // For amdgcn the inputs of the linker job are device bitcode and output is
 // object file. It calls llvm-link, opt, llc, then lld steps.
 void AMDGCN::Linker::ConstructJob(Compilation &C, const JobAction &JA,
-                                   const InputInfo &Output,
-                                   const InputInfoList &Inputs,
-                                   const ArgList &Args,
-                                   const char *LinkingOutput) const {
+                                  const InputInfo &Output,
+                                  const InputInfoList &Inputs,
+                                  const ArgList &Args,
+                                  const char *LinkingOutput) const {
   if (Inputs.size() > 0 &&
       Inputs[0].getType() == types::TY_Image &&
       JA.getType() == types::TY_Object)
-    return constructGenerateObjFileFromHIPFatBinary(C, Output, Inputs, Args, JA);
+    return HIP::constructGenerateObjFileFromHIPFatBinary(C, Output, Inputs,
+                                                         Args, JA, *this);
 
   if (JA.getType() == types::TY_HIP_FATBIN)
-    return constructHIPFatbinCommand(C, JA, Output.getFilename(), Inputs, Args, *this);
+    return HIP::constructHIPFatbinCommand(C, JA, Output.getFilename(), Inputs,
+                                          Args, *this);
 
   return constructLldCommand(C, JA, Inputs, Output, Args);
 }
 
-HIPToolChain::HIPToolChain(const Driver &D, const llvm::Triple &Triple,
-                             const ToolChain &HostTC, const ArgList &Args)
+HIPAMDToolChain::HIPAMDToolChain(const Driver &D, const llvm::Triple &Triple,
+                                 const ToolChain &HostTC, const ArgList &Args)
     : ROCMToolChain(D, Triple, Args), HostTC(HostTC) {
   // Lookup binaries into the driver directory, this is used to
   // discover the clang-offload-bundler executable.
@@ -279,9 +162,8 @@ HIPToolChain::HIPToolChain(const Driver &D, const llvm::Triple &Triple,
   }
 }
 
-void HIPToolChain::addClangTargetOptions(
-    const llvm::opt::ArgList &DriverArgs,
-    llvm::opt::ArgStringList &CC1Args,
+void HIPAMDToolChain::addClangTargetOptions(
+    const llvm::opt::ArgList &DriverArgs, llvm::opt::ArgStringList &CC1Args,
     Action::OffloadKind DeviceOffloadingKind) const {
   HostTC.addClangTargetOptions(DriverArgs, CC1Args, DeviceOffloadingKind);
 
@@ -324,9 +206,9 @@ void HIPToolChain::addClangTargetOptions(
 }
 
 llvm::opt::DerivedArgList *
-HIPToolChain::TranslateArgs(const llvm::opt::DerivedArgList &Args,
-                             StringRef BoundArch,
-                             Action::OffloadKind DeviceOffloadKind) const {
+HIPAMDToolChain::TranslateArgs(const llvm::opt::DerivedArgList &Args,
+                               StringRef BoundArch,
+                               Action::OffloadKind DeviceOffloadKind) const {
   DerivedArgList *DAL =
       HostTC.TranslateArgs(Args, BoundArch, DeviceOffloadKind);
   if (!DAL)
@@ -349,44 +231,44 @@ HIPToolChain::TranslateArgs(const llvm::opt::DerivedArgList &Args,
   return DAL;
 }
 
-Tool *HIPToolChain::buildLinker() const {
+Tool *HIPAMDToolChain::buildLinker() const {
   assert(getTriple().getArch() == llvm::Triple::amdgcn);
   return new tools::AMDGCN::Linker(*this);
 }
 
-void HIPToolChain::addClangWarningOptions(ArgStringList &CC1Args) const {
+void HIPAMDToolChain::addClangWarningOptions(ArgStringList &CC1Args) const {
   HostTC.addClangWarningOptions(CC1Args);
 }
 
 ToolChain::CXXStdlibType
-HIPToolChain::GetCXXStdlibType(const ArgList &Args) const {
+HIPAMDToolChain::GetCXXStdlibType(const ArgList &Args) const {
   return HostTC.GetCXXStdlibType(Args);
 }
 
-void HIPToolChain::AddClangSystemIncludeArgs(const ArgList &DriverArgs,
-                                              ArgStringList &CC1Args) const {
+void HIPAMDToolChain::AddClangSystemIncludeArgs(const ArgList &DriverArgs,
+                                                ArgStringList &CC1Args) const {
   HostTC.AddClangSystemIncludeArgs(DriverArgs, CC1Args);
 }
 
-void HIPToolChain::AddClangCXXStdlibIncludeArgs(const ArgList &Args,
-                                                 ArgStringList &CC1Args) const {
+void HIPAMDToolChain::AddClangCXXStdlibIncludeArgs(
+    const ArgList &Args, ArgStringList &CC1Args) const {
   HostTC.AddClangCXXStdlibIncludeArgs(Args, CC1Args);
 }
 
-void HIPToolChain::AddIAMCUIncludeArgs(const ArgList &Args,
-                                        ArgStringList &CC1Args) const {
+void HIPAMDToolChain::AddIAMCUIncludeArgs(const ArgList &Args,
+                                          ArgStringList &CC1Args) const {
   HostTC.AddIAMCUIncludeArgs(Args, CC1Args);
 }
 
-void HIPToolChain::AddHIPIncludeArgs(const ArgList &DriverArgs,
-                                     ArgStringList &CC1Args) const {
+void HIPAMDToolChain::AddHIPIncludeArgs(const ArgList &DriverArgs,
+                                        ArgStringList &CC1Args) const {
   RocmInstallation.AddHIPIncludeArgs(DriverArgs, CC1Args);
 }
 
-SanitizerMask HIPToolChain::getSupportedSanitizers() const {
-  // The HIPToolChain only supports sanitizers in the sense that it allows
+SanitizerMask HIPAMDToolChain::getSupportedSanitizers() const {
+  // The HIPAMDToolChain only supports sanitizers in the sense that it allows
   // sanitizer arguments on the command line if they are supported by the host
-  // toolchain. The HIPToolChain will actually ignore any command line
+  // toolchain. The HIPAMDToolChain will actually ignore any command line
   // arguments for any of these "supported" sanitizers. That means that no
   // sanitization of device code is actually supported at this time.
   //
@@ -396,13 +278,13 @@ SanitizerMask HIPToolChain::getSupportedSanitizers() const {
   return HostTC.getSupportedSanitizers();
 }
 
-VersionTuple HIPToolChain::computeMSVCVersion(const Driver *D,
-                                               const ArgList &Args) const {
+VersionTuple HIPAMDToolChain::computeMSVCVersion(const Driver *D,
+                                                 const ArgList &Args) const {
   return HostTC.computeMSVCVersion(D, Args);
 }
 
 llvm::SmallVector<ToolChain::BitCodeLibraryInfo, 12>
-HIPToolChain::getHIPDeviceLibs(const llvm::opt::ArgList &DriverArgs) const {
+HIPAMDToolChain::getHIPDeviceLibs(const llvm::opt::ArgList &DriverArgs) const {
   llvm::SmallVector<BitCodeLibraryInfo, 12> BCLibs;
   if (DriverArgs.hasArg(options::OPT_nogpulib))
     return {};
@@ -476,11 +358,11 @@ HIPToolChain::getHIPDeviceLibs(const llvm::opt::ArgList &DriverArgs) const {
   return BCLibs;
 }
 
-void HIPToolChain::checkTargetID(const llvm::opt::ArgList &DriverArgs) const {
+void HIPAMDToolChain::checkTargetID(
+    const llvm::opt::ArgList &DriverArgs) const {
   auto PTID = getParsedTargetID(DriverArgs);
   if (PTID.OptionalTargetID && !PTID.OptionalGPUArch) {
     getDriver().Diag(clang::diag::err_drv_bad_target_id)
         << PTID.OptionalTargetID.getValue();
   }
-  return;
 }
diff --git a/clang/lib/Driver/ToolChains/HIP.h b/clang/lib/Driver/ToolChains/HIPAMD.h
index 60b3d69b3f52..cc472a595db9 100644
--- a/clang/lib/Driver/ToolChains/HIP.h
+++ b/clang/lib/Driver/ToolChains/HIPAMD.h
@@ -1,4 +1,4 @@
-//===--- HIP.h - HIP ToolChain Implementations ------------------*- C++ -*-===//
+//===--- HIPAMD.h - HIP ToolChain Implementations ---------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,12 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_HIP_H
-#define LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_HIP_H
+#ifndef LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_HIPAMD_H
+#define LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_HIPAMD_H
 
-#include "clang/Driver/ToolChain.h"
-#include "clang/Driver/Tool.h"
 #include "AMDGPU.h"
+#include "clang/Driver/Tool.h"
+#include "clang/Driver/ToolChain.h"
 
 namespace clang {
 namespace driver {
@@ -19,11 +19,6 @@ namespace driver {
 namespace tools {
 
 namespace AMDGCN {
-  // Construct command for creating HIP fatbin.
-  void constructHIPFatbinCommand(Compilation &C, const JobAction &JA,
-                  StringRef OutputFileName, const InputInfoList &Inputs,
-                  const llvm::opt::ArgList &TCArgs, const Tool& T);
-
 // Runs llvm-link/opt/llc/lld, which links multiple LLVM bitcode, together with
 // device library, then compiles it to ISA in a shared object.
 class LLVM_LIBRARY_VISIBILITY Linker : public Tool {
@@ -38,17 +33,9 @@ public:
                     const char *LinkingOutput) const override;
 
 private:
-
   void constructLldCommand(Compilation &C, const JobAction &JA,
                            const InputInfoList &Inputs, const InputInfo &Output,
                            const llvm::opt::ArgList &Args) const;
-
-  // Construct command for creating Object from HIP fatbin.
-  void constructGenerateObjFileFromHIPFatBinary(Compilation &C,
-                                                const InputInfo &Output,
-                                                const InputInfoList &Inputs,
-                                                const llvm::opt::ArgList &Args,
-                                                const JobAction &JA) const;
 };
 
 } // end namespace AMDGCN
@@ -56,10 +43,10 @@ private:
 
 namespace toolchains {
 
-class LLVM_LIBRARY_VISIBILITY HIPToolChain final : public ROCMToolChain {
+class LLVM_LIBRARY_VISIBILITY HIPAMDToolChain final : public ROCMToolChain {
 public:
-  HIPToolChain(const Driver &D, const llvm::Triple &Triple,
-                const ToolChain &HostTC, const llvm::opt::ArgList &Args);
+  HIPAMDToolChain(const Driver &D, const llvm::Triple &Triple,
+                  const ToolChain &HostTC, const llvm::opt::ArgList &Args);
 
   const llvm::Triple *getAuxTriple() const override {
     return &HostTC.getTriple();
@@ -68,9 +55,10 @@ public:
   llvm::opt::DerivedArgList *
   TranslateArgs(const llvm::opt::DerivedArgList &Args, StringRef BoundArch,
                 Action::OffloadKind DeviceOffloadKind) const override;
-  void addClangTargetOptions(const llvm::opt::ArgList &DriverArgs,
-                             llvm::opt::ArgStringList &CC1Args,
-                             Action::OffloadKind DeviceOffloadKind) const override;
+  void
+  addClangTargetOptions(const llvm::opt::ArgList &DriverArgs,
+                        llvm::opt::ArgStringList &CC1Args,
+                        Action::OffloadKind DeviceOffloadKind) const override;
   void addClangWarningOptions(llvm::opt::ArgStringList &CC1Args) const override;
   CXXStdlibType GetCXXStdlibType(const llvm::opt::ArgList &Args) const override;
   void
@@ -105,4 +93,4 @@ protected:
 } // end namespace driver
 } // end namespace clang
 
-#endif // LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_HIP_H
+#endif // LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_HIPAMD_H
diff --git a/clang/lib/Driver/ToolChains/HIPSPV.cpp b/clang/lib/Driver/ToolChains/HIPSPV.cpp
new file mode 100644
index 000000000000..d68c87e9b3e7
--- /dev/null
+++ b/clang/lib/Driver/ToolChains/HIPSPV.cpp
@@ -0,0 +1,292 @@
+//===--- HIPSPV.cpp - HIPSPV ToolChain Implementation -----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "HIPSPV.h"
+#include "CommonArgs.h"
+#include "HIPUtility.h"
+#include "clang/Driver/Compilation.h"
+#include "clang/Driver/Driver.h"
+#include "clang/Driver/DriverDiagnostic.h"
+#include "clang/Driver/InputInfo.h"
+#include "clang/Driver/Options.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+
+using namespace clang::driver;
+using namespace clang::driver::toolchains;
+using namespace clang::driver::tools;
+using namespace clang;
+using namespace llvm::opt;
+
+// Convenience function for creating temporary file for both modes of
+// isSaveTempsEnabled().
+static const char *getTempFile(Compilation &C, StringRef Prefix,
+                               StringRef Extension) {
+  if (C.getDriver().isSaveTempsEnabled()) {
+    return C.getArgs().MakeArgString(Prefix + "." + Extension);
+  }
+  auto TmpFile = C.getDriver().GetTemporaryPath(Prefix, Extension);
+  return C.addTempFile(C.getArgs().MakeArgString(TmpFile));
+}
+
+// Locates HIP pass plugin.
+static std::string findPassPlugin(const Driver &D,
+                                  const llvm::opt::ArgList &Args) {
+  StringRef Path = Args.getLastArgValue(options::OPT_hipspv_pass_plugin_EQ);
+  if (!Path.empty()) {
+    if (llvm::sys::fs::exists(Path))
+      return Path.str();
+    D.Diag(diag::err_drv_no_such_file) << Path;
+  }
+
+  StringRef hipPath = Args.getLastArgValue(options::OPT_hip_path_EQ);
+  if (!hipPath.empty()) {
+    SmallString<128> PluginPath(hipPath);
+    llvm::sys::path::append(PluginPath, "lib", "libLLVMHipSpvPasses.so");
+    if (llvm::sys::fs::exists(PluginPath))
+      return PluginPath.str().str();
+    PluginPath.assign(hipPath);
+    llvm::sys::path::append(PluginPath, "lib", "llvm",
+                            "libLLVMHipSpvPasses.so");
+    if (llvm::sys::fs::exists(PluginPath))
+      return PluginPath.str().str();
+  }
+
+  return std::string();
+}
+
+void HIPSPV::Linker::constructLinkAndEmitSpirvCommand(
+    Compilation &C, const JobAction &JA, const InputInfoList &Inputs,
+    const InputInfo &Output, const llvm::opt::ArgList &Args) const {
+
+  assert(!Inputs.empty() && "Must have at least one input.");
+  std::string Name = std::string(llvm::sys::path::stem(Output.getFilename()));
+  const char *TempFile = getTempFile(C, Name + "-link", "bc");
+
+  // Link LLVM bitcode.
+  ArgStringList LinkArgs{};
+  for (auto Input : Inputs)
+    LinkArgs.push_back(Input.getFilename());
+  LinkArgs.append({"-o", TempFile});
+  const char *LlvmLink =
+      Args.MakeArgString(getToolChain().GetProgramPath("llvm-link"));
+  C.addCommand(std::make_unique<Command>(JA, *this, ResponseFileSupport::None(),
+                                         LlvmLink, LinkArgs, Inputs, Output));
+
+  // Post-link HIP lowering.
+
+  // Run LLVM IR passes to lower/expand/emulate HIP code that does not translate
+  // to SPIR-V (E.g. dynamic shared memory).
+  auto PassPluginPath = findPassPlugin(C.getDriver(), Args);
+  if (!PassPluginPath.empty()) {
+    const char *PassPathCStr = C.getArgs().MakeArgString(PassPluginPath);
+    const char *OptOutput = getTempFile(C, Name + "-lower", "bc");
+    ArgStringList OptArgs{TempFile,     "-load-pass-plugin",
+                          PassPathCStr, "-passes=hip-post-link-passes",
+                          "-o",         OptOutput};
+    const char *Opt = Args.MakeArgString(getToolChain().GetProgramPath("opt"));
+    C.addCommand(std::make_unique<Command>(
+        JA, *this, ResponseFileSupport::None(), Opt, OptArgs, Inputs, Output));
+    TempFile = OptOutput;
+  }
+
+  // Emit SPIR-V binary.
+
+  llvm::opt::ArgStringList TrArgs{"--spirv-max-version=1.1",
+                                  "--spirv-ext=+all"};
+  InputInfo TrInput = InputInfo(types::TY_LLVM_BC, TempFile, "");
+  SPIRV::constructTranslateCommand(C, *this, JA, Output, TrInput, TrArgs);
+}
+
+void HIPSPV::Linker::ConstructJob(Compilation &C, const JobAction &JA,
+                                  const InputInfo &Output,
+                                  const InputInfoList &Inputs,
+                                  const ArgList &Args,
+                                  const char *LinkingOutput) const {
+  if (Inputs.size() > 0 && Inputs[0].getType() == types::TY_Image &&
+      JA.getType() == types::TY_Object)
+    return HIP::constructGenerateObjFileFromHIPFatBinary(C, Output, Inputs,
+                                                         Args, JA, *this);
+
+  if (JA.getType() == types::TY_HIP_FATBIN)
+    return HIP::constructHIPFatbinCommand(C, JA, Output.getFilename(), Inputs,
+                                          Args, *this);
+
+  constructLinkAndEmitSpirvCommand(C, JA, Inputs, Output, Args);
+}
+
+HIPSPVToolChain::HIPSPVToolChain(const Driver &D, const llvm::Triple &Triple,
+                                 const ToolChain &HostTC, const ArgList &Args)
+    : ToolChain(D, Triple, Args), HostTC(HostTC) {
+  // Lookup binaries into the driver directory, this is used to
+  // discover the clang-offload-bundler executable.
+  getProgramPaths().push_back(getDriver().Dir);
+}
+
+void HIPSPVToolChain::addClangTargetOptions(
+    const llvm::opt::ArgList &DriverArgs, llvm::opt::ArgStringList &CC1Args,
+    Action::OffloadKind DeviceOffloadingKind) const {
+  HostTC.addClangTargetOptions(DriverArgs, CC1Args, DeviceOffloadingKind);
+
+  assert(DeviceOffloadingKind == Action::OFK_HIP &&
+         "Only HIP offloading kinds are supported for GPUs.");
+
+  CC1Args.append(
+      {"-fcuda-is-device", "-fcuda-allow-variadic-functions",
+       // A crude workaround for llvm-spirv which does not handle the
+       // autovectorized code well (vector reductions, non-i{8,16,32,64} types).
+       // TODO: Allow autovectorization when SPIR-V backend arrives.
+       "-mllvm", "-vectorize-loops=false", "-mllvm", "-vectorize-slp=false"});
+
+  if (DriverArgs.hasFlag(options::OPT_fcuda_approx_transcendentals,
+                         options::OPT_fno_cuda_approx_transcendentals, false))
+    CC1Args.push_back("-fcuda-approx-transcendentals");
+
+  // Default to "hidden" visibility, as object level linking will not be
+  // supported for the foreseeable future.
+  if (!DriverArgs.hasArg(options::OPT_fvisibility_EQ,
+                         options::OPT_fvisibility_ms_compat))
+    CC1Args.append(
+        {"-fvisibility", "hidden", "-fapply-global-visibility-to-externs"});
+
+  llvm::for_each(getHIPDeviceLibs(DriverArgs),
+                 [&](const BitCodeLibraryInfo &BCFile) {
+                   CC1Args.append({"-mlink-builtin-bitcode",
+                                   DriverArgs.MakeArgString(BCFile.Path)});
+                 });
+}
+
+Tool *HIPSPVToolChain::buildLinker() const {
+  assert(getTriple().getArch() == llvm::Triple::spirv64);
+  return new tools::HIPSPV::Linker(*this);
+}
+
+void HIPSPVToolChain::addClangWarningOptions(ArgStringList &CC1Args) const {
+  HostTC.addClangWarningOptions(CC1Args);
+}
+
+ToolChain::CXXStdlibType
+HIPSPVToolChain::GetCXXStdlibType(const ArgList &Args) const {
+  return HostTC.GetCXXStdlibType(Args);
+}
+
+void HIPSPVToolChain::AddClangSystemIncludeArgs(const ArgList &DriverArgs,
+                                                ArgStringList &CC1Args) const {
+  HostTC.AddClangSystemIncludeArgs(DriverArgs, CC1Args);
+}
+
+void HIPSPVToolChain::AddClangCXXStdlibIncludeArgs(
+    const ArgList &Args, ArgStringList &CC1Args) const {
+  HostTC.AddClangCXXStdlibIncludeArgs(Args, CC1Args);
+}
+
+void HIPSPVToolChain::AddIAMCUIncludeArgs(const ArgList &Args,
+                                          ArgStringList &CC1Args) const {
+  HostTC.AddIAMCUIncludeArgs(Args, CC1Args);
+}
+
+void HIPSPVToolChain::AddHIPIncludeArgs(const ArgList &DriverArgs,
+                                        ArgStringList &CC1Args) const {
+  if (DriverArgs.hasArg(options::OPT_nogpuinc))
+    return;
+
+  StringRef hipPath = DriverArgs.getLastArgValue(options::OPT_hip_path_EQ);
+  if (hipPath.empty()) {
+    getDriver().Diag(diag::err_drv_hipspv_no_hip_path) << 1 << "'-nogpuinc'";
+    return;
+  }
+  SmallString<128> P(hipPath);
+  llvm::sys::path::append(P, "include");
+  CC1Args.append({"-isystem", DriverArgs.MakeArgString(P)});
+}
+
+llvm::SmallVector<ToolChain::BitCodeLibraryInfo, 12>
+HIPSPVToolChain::getHIPDeviceLibs(const llvm::opt::ArgList &DriverArgs) const {
+  llvm::SmallVector<ToolChain::BitCodeLibraryInfo, 12> BCLibs;
+  if (DriverArgs.hasArg(options::OPT_nogpulib))
+    return {};
+
+  ArgStringList LibraryPaths;
+  // Find device libraries in --hip-device-lib-path and HIP_DEVICE_LIB_PATH.
+  auto HipDeviceLibPathArgs = DriverArgs.getAllArgValues(
+      // --hip-device-lib-path is alias to this option.
+      clang::driver::options::OPT_rocm_device_lib_path_EQ);
+  for (auto Path : HipDeviceLibPathArgs)
+    LibraryPaths.push_back(DriverArgs.MakeArgString(Path));
+
+  StringRef HipPath = DriverArgs.getLastArgValue(options::OPT_hip_path_EQ);
+  if (!HipPath.empty()) {
+    SmallString<128> Path(HipPath);
+    llvm::sys::path::append(Path, "lib", "hip-device-lib");
+    LibraryPaths.push_back(DriverArgs.MakeArgString(Path));
+  }
+
+  addDirectoryList(DriverArgs, LibraryPaths, "", "HIP_DEVICE_LIB_PATH");
+
+  // Maintain compatability with --hip-device-lib.
+  auto BCLibArgs = DriverArgs.getAllArgValues(options::OPT_hip_device_lib_EQ);
+  if (!BCLibArgs.empty()) {
+    llvm::for_each(BCLibArgs, [&](StringRef BCName) {
+      StringRef FullName;
+      for (std::string LibraryPath : LibraryPaths) {
+        SmallString<128> Path(LibraryPath);
+        llvm::sys::path::append(Path, BCName);
+        FullName = Path;
+        if (llvm::sys::fs::exists(FullName)) {
+          BCLibs.emplace_back(FullName.str());
+          return;
+        }
+      }
+      getDriver().Diag(diag::err_drv_no_such_file) << BCName;
+    });
+  } else {
+    // Search device library named as 'hipspv-<triple>.bc'.
+    auto TT = getTriple().normalize();
+    std::string BCName = "hipspv-" + TT + ".bc";
+    for (auto *LibPath : LibraryPaths) {
+      SmallString<128> Path(LibPath);
+      llvm::sys::path::append(Path, BCName);
+      if (llvm::sys::fs::exists(Path)) {
+        BCLibs.emplace_back(Path.str().str());
+        return BCLibs;
+      }
+    }
+    getDriver().Diag(diag::err_drv_no_hipspv_device_lib)
+        << 1 << ("'" + TT + "' target");
+    return {};
+  }
+
+  return BCLibs;
+}
+
+SanitizerMask HIPSPVToolChain::getSupportedSanitizers() const {
+  // The HIPSPVToolChain only supports sanitizers in the sense that it allows
+  // sanitizer arguments on the command line if they are supported by the host
+  // toolchain. The HIPSPVToolChain will actually ignore any command line
+  // arguments for any of these "supported" sanitizers. That means that no
+  // sanitization of device code is actually supported at this time.
+  //
+  // This behavior is necessary because the host and device toolchains
+  // invocations often share the command line, so the device toolchain must
+  // tolerate flags meant only for the host toolchain.
+  return HostTC.getSupportedSanitizers();
+}
+
+VersionTuple HIPSPVToolChain::computeMSVCVersion(const Driver *D,
+                                                 const ArgList &Args) const {
+  return HostTC.computeMSVCVersion(D, Args);
+}
+
+void HIPSPVToolChain::adjustDebugInfoKind(
+    codegenoptions::DebugInfoKind &DebugInfoKind,
+    const llvm::opt::ArgList &Args) const {
+  // Debug info generation is disabled for SPIRV-LLVM-Translator
+  // which currently aborts on the presence of DW_OP_LLVM_convert.
+  // TODO: Enable debug info when the SPIR-V backend arrives.
+  DebugInfoKind = codegenoptions::NoDebugInfo;
+}
diff --git a/clang/lib/Driver/ToolChains/HIPSPV.h b/clang/lib/Driver/ToolChains/HIPSPV.h
new file mode 100644
index 000000000000..79520f77c742
--- /dev/null
+++ b/clang/lib/Driver/ToolChains/HIPSPV.h
@@ -0,0 +1,103 @@
+//===--- HIPSPV.h - HIP ToolChain Implementations ---------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_HIPSPV_H
+#define LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_HIPSPV_H
+
+#include "SPIRV.h"
+#include "clang/Driver/Tool.h"
+#include "clang/Driver/ToolChain.h"
+
+namespace clang {
+namespace driver {
+namespace tools {
+namespace HIPSPV {
+
+// Runs llvm-link/opt/llc/lld, which links multiple LLVM bitcode, together with
+// device library, then compiles it to SPIR-V in a shared object.
+class LLVM_LIBRARY_VISIBILITY Linker : public Tool {
+public:
+  Linker(const ToolChain &TC) : Tool("HIPSPV::Linker", "hipspv-link", TC) {}
+
+  bool hasIntegratedCPP() const override { return false; }
+
+  void ConstructJob(Compilation &C, const JobAction &JA,
+                    const InputInfo &Output, const InputInfoList &Inputs,
+                    const llvm::opt::ArgList &TCArgs,
+                    const char *LinkingOutput) const override;
+
+private:
+  void constructLinkAndEmitSpirvCommand(Compilation &C, const JobAction &JA,
+                                        const InputInfoList &Inputs,
+                                        const InputInfo &Output,
+                                        const llvm::opt::ArgList &Args) const;
+};
+
+} // namespace HIPSPV
+} // namespace tools
+
+namespace toolchains {
+
+class LLVM_LIBRARY_VISIBILITY HIPSPVToolChain final : public ToolChain {
+public:
+  HIPSPVToolChain(const Driver &D, const llvm::Triple &Triple,
+                  const ToolChain &HostTC, const llvm::opt::ArgList &Args);
+
+  const llvm::Triple *getAuxTriple() const override {
+    return &HostTC.getTriple();
+  }
+
+  void
+  addClangTargetOptions(const llvm::opt::ArgList &DriverArgs,
+                        llvm::opt::ArgStringList &CC1Args,
+                        Action::OffloadKind DeviceOffloadKind) const override;
+  void addClangWarningOptions(llvm::opt::ArgStringList &CC1Args) const override;
+  CXXStdlibType GetCXXStdlibType(const llvm::opt::ArgList &Args) const override;
+  void
+  AddClangSystemIncludeArgs(const llvm::opt::ArgList &DriverArgs,
+                            llvm::opt::ArgStringList &CC1Args) const override;
+  void AddClangCXXStdlibIncludeArgs(
+      const llvm::opt::ArgList &Args,
+      llvm::opt::ArgStringList &CC1Args) const override;
+  void AddIAMCUIncludeArgs(const llvm::opt::ArgList &DriverArgs,
+                           llvm::opt::ArgStringList &CC1Args) const override;
+  void AddHIPIncludeArgs(const llvm::opt::ArgList &DriverArgs,
+                         llvm::opt::ArgStringList &CC1Args) const override;
+  llvm::SmallVector<BitCodeLibraryInfo, 12>
+  getHIPDeviceLibs(const llvm::opt::ArgList &Args) const override;
+
+  SanitizerMask getSupportedSanitizers() const override;
+
+  VersionTuple
+  computeMSVCVersion(const Driver *D,
+                     const llvm::opt::ArgList &Args) const override;
+
+  void adjustDebugInfoKind(codegenoptions::DebugInfoKind &DebugInfoKind,
+                           const llvm::opt::ArgList &Args) const override;
+  bool IsIntegratedAssemblerDefault() const override { return true; }
+  bool IsMathErrnoDefault() const override { return false; }
+  bool useIntegratedAs() const override { return true; }
+  bool isCrossCompiling() const override { return true; }
+  bool isPICDefault() const override { return false; }
+  bool isPIEDefault(const llvm::opt::ArgList &Args) const override {
+    return false;
+  }
+  bool isPICDefaultForced() const override { return false; }
+  bool SupportsProfiling() const override { return false; }
+
+  const ToolChain &HostTC;
+
+protected:
+  Tool *buildLinker() const override;
+};
+
+} // end namespace toolchains
+} // end namespace driver
+} // end namespace clang
+
+#endif // LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_HIPSPV_H
diff --git a/clang/lib/Driver/ToolChains/HIPUtility.cpp b/clang/lib/Driver/ToolChains/HIPUtility.cpp
new file mode 100644
index 000000000000..1b04a20bacbf
--- /dev/null
+++ b/clang/lib/Driver/ToolChains/HIPUtility.cpp
@@ -0,0 +1,167 @@
+//===--- HIPUtility.cpp - Common HIP Tool Chain Utilities -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "HIPUtility.h"
+#include "CommonArgs.h"
+#include "clang/Driver/Compilation.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/Triple.h"
+#include "llvm/Support/Path.h"
+
+using namespace clang::driver;
+using namespace clang::driver::tools;
+using namespace llvm::opt;
+
+#if defined(_WIN32) || defined(_WIN64)
+#define NULL_FILE "nul"
+#else
+#define NULL_FILE "/dev/null"
+#endif
+
+namespace {
+const unsigned HIPCodeObjectAlign = 4096;
+} // namespace
+
+// Constructs a triple string for clang offload bundler.
+static std::string normalizeForBundler(const llvm::Triple &T,
+                                       bool HasTargetID) {
+  return HasTargetID ? (T.getArchName() + "-" + T.getVendorName() + "-" +
+                        T.getOSName() + "-" + T.getEnvironmentName())
+                           .str()
+                     : T.normalize();
+}
+
+// Construct a clang-offload-bundler command to bundle code objects for
+// different devices into a HIP fat binary.
+void HIP::constructHIPFatbinCommand(Compilation &C, const JobAction &JA,
+                                    llvm::StringRef OutputFileName,
+                                    const InputInfoList &Inputs,
+                                    const llvm::opt::ArgList &Args,
+                                    const Tool &T) {
+  // Construct clang-offload-bundler command to bundle object files for
+  // for different GPU archs.
+  ArgStringList BundlerArgs;
+  BundlerArgs.push_back(Args.MakeArgString("-type=o"));
+  BundlerArgs.push_back(
+      Args.MakeArgString("-bundle-align=" + Twine(HIPCodeObjectAlign)));
+
+  // ToDo: Remove the dummy host binary entry which is required by
+  // clang-offload-bundler.
+  std::string BundlerTargetArg = "-targets=host-x86_64-unknown-linux";
+  std::string BundlerInputArg = "-inputs=" NULL_FILE;
+
+  // AMDGCN:
+  // For code object version 2 and 3, the offload kind in bundle ID is 'hip'
+  // for backward compatibility. For code object version 4 and greater, the
+  // offload kind in bundle ID is 'hipv4'.
+  std::string OffloadKind = "hip";
+  auto &TT = T.getToolChain().getTriple();
+  if (TT.isAMDGCN() && getAMDGPUCodeObjectVersion(C.getDriver(), Args) >= 4)
+    OffloadKind = OffloadKind + "v4";
+  for (const auto &II : Inputs) {
+    const auto *A = II.getAction();
+    auto ArchStr = llvm::StringRef(A->getOffloadingArch());
+    BundlerTargetArg +=
+        "," + OffloadKind + "-" + normalizeForBundler(TT, !ArchStr.empty());
+    if (!ArchStr.empty())
+      BundlerTargetArg += "-" + ArchStr.str();
+    BundlerInputArg = BundlerInputArg + "," + II.getFilename();
+  }
+  BundlerArgs.push_back(Args.MakeArgString(BundlerTargetArg));
+  BundlerArgs.push_back(Args.MakeArgString(BundlerInputArg));
+
+  std::string Output = std::string(OutputFileName);
+  auto *BundlerOutputArg =
+      Args.MakeArgString(std::string("-outputs=").append(Output));
+  BundlerArgs.push_back(BundlerOutputArg);
+
+  const char *Bundler = Args.MakeArgString(
+      T.getToolChain().GetProgramPath("clang-offload-bundler"));
+  C.addCommand(std::make_unique<Command>(
+      JA, T, ResponseFileSupport::None(), Bundler, BundlerArgs, Inputs,
+      InputInfo(&JA, Args.MakeArgString(Output))));
+}
+
+/// Add Generated HIP Object File which has device images embedded into the
+/// host to the argument list for linking. Using MC directives, embed the
+/// device code and also define symbols required by the code generation so that
+/// the image can be retrieved at runtime.
+void HIP::constructGenerateObjFileFromHIPFatBinary(
+    Compilation &C, const InputInfo &Output, const InputInfoList &Inputs,
+    const ArgList &Args, const JobAction &JA, const Tool &T) {
+  const ToolChain &TC = T.getToolChain();
+  std::string Name = std::string(llvm::sys::path::stem(Output.getFilename()));
+
+  // Create Temp Object File Generator,
+  // Offload Bundled file and Bundled Object file.
+  // Keep them if save-temps is enabled.
+  const char *McinFile;
+  const char *BundleFile;
+  if (C.getDriver().isSaveTempsEnabled()) {
+    McinFile = C.getArgs().MakeArgString(Name + ".mcin");
+    BundleFile = C.getArgs().MakeArgString(Name + ".hipfb");
+  } else {
+    auto TmpNameMcin = C.getDriver().GetTemporaryPath(Name, "mcin");
+    McinFile = C.addTempFile(C.getArgs().MakeArgString(TmpNameMcin));
+    auto TmpNameFb = C.getDriver().GetTemporaryPath(Name, "hipfb");
+    BundleFile = C.addTempFile(C.getArgs().MakeArgString(TmpNameFb));
+  }
+  HIP::constructHIPFatbinCommand(C, JA, BundleFile, Inputs, Args, T);
+
+  // Create a buffer to write the contents of the temp obj generator.
+  std::string ObjBuffer;
+  llvm::raw_string_ostream ObjStream(ObjBuffer);
+
+  auto HostTriple =
+      C.getSingleOffloadToolChain<Action::OFK_Host>()->getTriple();
+
+  // Add MC directives to embed target binaries. We ensure that each
+  // section and image is 16-byte aligned. This is not mandatory, but
+  // increases the likelihood of data to be aligned with a cache block
+  // in several main host machines.
+  ObjStream << "#       HIP Object Generator\n";
+  ObjStream << "# *** Automatically generated by Clang ***\n";
+  if (HostTriple.isWindowsMSVCEnvironment()) {
+    ObjStream << "  .section .hip_fatbin, \"dw\"\n";
+  } else {
+    ObjStream << "  .protected __hip_fatbin\n";
+    ObjStream << "  .type __hip_fatbin,@object\n";
+    ObjStream << "  .section .hip_fatbin,\"a\",@progbits\n";
+  }
+  ObjStream << "  .globl __hip_fatbin\n";
+  ObjStream << "  .p2align " << llvm::Log2(llvm::Align(HIPCodeObjectAlign))
+            << "\n";
+  ObjStream << "__hip_fatbin:\n";
+  ObjStream << "  .incbin ";
+  llvm::sys::printArg(ObjStream, BundleFile, /*Quote=*/true);
+  ObjStream << "\n";
+  ObjStream.flush();
+
+  // Dump the contents of the temp object file gen if the user requested that.
+  // We support this option to enable testing of behavior with -###.
+  if (C.getArgs().hasArg(options::OPT_fhip_dump_offload_linker_script))
+    llvm::errs() << ObjBuffer;
+
+  // Open script file and write the contents.
+  std::error_code EC;
+  llvm::raw_fd_ostream Objf(McinFile, EC, llvm::sys::fs::OF_None);
+
+  if (EC) {
+    C.getDriver().Diag(clang::diag::err_unable_to_make_temp) << EC.message();
+    return;
+  }
+
+  Objf << ObjBuffer;
+
+  ArgStringList McArgs{"-triple", Args.MakeArgString(HostTriple.normalize()),
+                       "-o",      Output.getFilename(),
+                       McinFile,  "--filetype=obj"};
+  const char *Mc = Args.MakeArgString(TC.GetProgramPath("llvm-mc"));
+  C.addCommand(std::make_unique<Command>(JA, T, ResponseFileSupport::None(), Mc,
+                                         McArgs, Inputs, Output));
+}
diff --git a/clang/lib/Driver/ToolChains/HIPUtility.h b/clang/lib/Driver/ToolChains/HIPUtility.h
new file mode 100644
index 000000000000..29e5a922024a
--- /dev/null
+++ b/clang/lib/Driver/ToolChains/HIPUtility.h
@@ -0,0 +1,35 @@
+//===--- HIPUtility.h - Common HIP Tool Chain Utilities ---------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_HIPUTILITY_H
+#define LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_HIPUTILITY_H
+
+#include "clang/Driver/Tool.h"
+
+namespace clang {
+namespace driver {
+namespace tools {
+namespace HIP {
+
+// Construct command for creating HIP fatbin.
+void constructHIPFatbinCommand(Compilation &C, const JobAction &JA,
+                               StringRef OutputFileName,
+                               const InputInfoList &Inputs,
+                               const llvm::opt::ArgList &TCArgs, const Tool &T);
+
+// Construct command for creating Object from HIP fatbin.
+void constructGenerateObjFileFromHIPFatBinary(
+    Compilation &C, const InputInfo &Output, const InputInfoList &Inputs,
+    const llvm::opt::ArgList &Args, const JobAction &JA, const Tool &T);
+
+} // namespace HIP
+} // namespace tools
+} // namespace driver
+} // namespace clang
+
+#endif // LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_HIPUTILITY_H
diff --git a/clang/lib/Driver/ToolChains/Hexagon.cpp b/clang/lib/Driver/ToolChains/Hexagon.cpp
index 18270818d158..ba3040636604 100644
--- a/clang/lib/Driver/ToolChains/Hexagon.cpp
+++ b/clang/lib/Driver/ToolChains/Hexagon.cpp
@@ -26,8 +26,8 @@ using namespace clang;
 using namespace llvm::opt;
 
 // Default hvx-length for various versions.
-static StringRef getDefaultHvxLength(StringRef Cpu) {
-  return llvm::StringSwitch<StringRef>(Cpu)
+static StringRef getDefaultHvxLength(StringRef HvxVer) {
+  return llvm::StringSwitch<StringRef>(HvxVer)
       .Case("v60", "64b")
       .Case("v62", "64b")
       .Case("v65", "64b")
@@ -51,42 +51,107 @@ static void handleHVXTargetFeatures(const Driver &D, const ArgList &Args,
   // Handle HVX warnings.
   handleHVXWarnings(D, Args);
 
-  // Add the +hvx* features based on commandline flags.
-  StringRef HVXFeature, HVXLength;
-
-  // Handle -mhvx, -mhvx=, -mno-hvx.
-  if (Arg *A = Args.getLastArg(options::OPT_mno_hexagon_hvx,
-                               options::OPT_mhexagon_hvx,
-                               options::OPT_mhexagon_hvx_EQ)) {
-    if (A->getOption().matches(options::OPT_mno_hexagon_hvx))
-      return;
-    if (A->getOption().matches(options::OPT_mhexagon_hvx_EQ)) {
-      HasHVX = true;
-      HVXFeature = Cpu = A->getValue();
-      HVXFeature = Args.MakeArgString(llvm::Twine("+hvx") + HVXFeature.lower());
-    } else if (A->getOption().matches(options::OPT_mhexagon_hvx)) {
-      HasHVX = true;
-      HVXFeature = Args.MakeArgString(llvm::Twine("+hvx") + Cpu);
+  auto makeFeature = [&Args](Twine T, bool Enable) -> StringRef {
+    const std::string &S = T.str();
+    StringRef Opt(S);
+    if (Opt.endswith("="))
+      Opt = Opt.drop_back(1);
+    if (Opt.startswith("mno-"))
+      Opt = Opt.drop_front(4);
+    else if (Opt.startswith("m"))
+      Opt = Opt.drop_front(1);
+    return Args.MakeArgString(Twine(Enable ? "+" : "-") + Twine(Opt));
+  };
+
+  auto withMinus = [](StringRef S) -> std::string {
+    return "-" + S.str();
+  };
+
+  // Drop tiny core suffix for HVX version.
+  std::string HvxVer =
+      (Cpu.back() == 'T' || Cpu.back() == 't' ? Cpu.drop_back(1) : Cpu).str();
+  HasHVX = false;
+
+  // Handle -mhvx, -mhvx=, -mno-hvx. If both present, -mhvx= wins over -mhvx.
+  auto argOrNull = [&Args](auto FlagOn, auto FlagOff) -> Arg* {
+    if (Arg *A = Args.getLastArg(FlagOn, FlagOff)) {
+      if (A->getOption().matches(FlagOn))
+        return A;
     }
-    Features.push_back(HVXFeature);
+    return nullptr;
+  };
+
+  Arg *HvxBareA =
+      argOrNull(options::OPT_mhexagon_hvx, options::OPT_mno_hexagon_hvx);
+  Arg *HvxVerA =
+      argOrNull(options::OPT_mhexagon_hvx_EQ, options::OPT_mno_hexagon_hvx);
+
+  if (Arg *A = HvxVerA ? HvxVerA : HvxBareA) {
+    if (A->getOption().matches(options::OPT_mhexagon_hvx_EQ))
+      HvxVer = StringRef(A->getValue()).lower(); // lower produces std:string
+    HasHVX = true;
+    Features.push_back(makeFeature(Twine("hvx") + HvxVer, true));
+  } else if (Arg *A = Args.getLastArg(options::OPT_mno_hexagon_hvx)) {
+    // If there was an explicit -mno-hvx, add -hvx to target features.
+    Features.push_back(makeFeature(A->getOption().getName(), false));
   }
 
+  StringRef HvxLen = getDefaultHvxLength(HvxVer);
+
   // Handle -mhvx-length=.
   if (Arg *A = Args.getLastArg(options::OPT_mhexagon_hvx_length_EQ)) {
     // These flags are valid only if HVX in enabled.
     if (!HasHVX)
-      D.Diag(diag::err_drv_invalid_hvx_length);
+      D.Diag(diag::err_drv_needs_hvx) << withMinus(A->getOption().getName());
     else if (A->getOption().matches(options::OPT_mhexagon_hvx_length_EQ))
-      HVXLength = A->getValue();
+      HvxLen = A->getValue();
+  }
+
+  if (HasHVX) {
+    StringRef L = makeFeature(Twine("hvx-length") + HvxLen.lower(), true);
+    Features.push_back(L);
   }
-  // Default hvx-length based on Cpu.
-  else if (HasHVX)
-    HVXLength = getDefaultHvxLength(Cpu);
-
-  if (!HVXLength.empty()) {
-    HVXFeature =
-        Args.MakeArgString(llvm::Twine("+hvx-length") + HVXLength.lower());
-    Features.push_back(HVXFeature);
+
+  unsigned HvxVerNum;
+  // getAsInteger returns 'true' on error.
+  if (StringRef(HvxVer).drop_front(1).getAsInteger(10, HvxVerNum))
+    HvxVerNum = 0;
+
+  // Handle HVX floating point flags.
+  auto checkFlagHvxVersion = [&](auto FlagOn, auto FlagOff,
+                                 unsigned MinVerNum) -> Optional<StringRef> {
+    // Return an Optional<StringRef>:
+    // - None indicates a verification failure, or that the flag was not
+    //   present in Args.
+    // - Otherwise the returned value is that name of the feature to add
+    //   to Features.
+    Arg *A = Args.getLastArg(FlagOn, FlagOff);
+    if (!A)
+      return None;
+
+    StringRef OptName = A->getOption().getName();
+    if (A->getOption().matches(FlagOff))
+      return makeFeature(OptName, false);
+
+    if (!HasHVX) {
+      D.Diag(diag::err_drv_needs_hvx) << withMinus(OptName);
+      return None;
+    }
+    if (HvxVerNum < MinVerNum) {
+      D.Diag(diag::err_drv_needs_hvx_version)
+          << withMinus(OptName) << ("v" + std::to_string(HvxVerNum));
+      return None;
+    }
+    return makeFeature(OptName, true);
+  };
+
+  if (auto F = checkFlagHvxVersion(options::OPT_mhexagon_hvx_qfloat,
+                                   options::OPT_mno_hexagon_hvx_qfloat, 68)) {
+    Features.push_back(*F);
+  }
+  if (auto F = checkFlagHvxVersion(options::OPT_mhexagon_hvx_ieee_fp,
+                                   options::OPT_mno_hexagon_hvx_ieee_fp, 68)) {
+    Features.push_back(*F);
   }
 }
 
@@ -117,7 +182,7 @@ void hexagon::getHexagonTargetFeatures(const Driver &D, const ArgList &Args,
   handleHVXTargetFeatures(D, Args, Features, Cpu, HasHVX);
 
   if (HexagonToolChain::isAutoHVXEnabled(Args) && !HasHVX)
-    D.Diag(diag::warn_drv_vectorize_needs_hvx);
+    D.Diag(diag::warn_drv_needs_hvx) << "auto-vectorization";
 }
 
 // Hexagon tools start.
@@ -156,6 +221,12 @@ void hexagon::Assembler::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back("-fsyntax-only");
   }
 
+  if (Arg *A = Args.getLastArg(options::OPT_mhexagon_hvx_ieee_fp,
+                               options::OPT_mno_hexagon_hvx_ieee_fp)) {
+    if (A->getOption().matches(options::OPT_mhexagon_hvx_ieee_fp))
+      CmdArgs.push_back("-mhvx-ieee-fp");
+  }
+
   if (auto G = toolchains::HexagonToolChain::getSmallDataThreshold(Args)) {
     CmdArgs.push_back(Args.MakeArgString("-gpsize=" + Twine(G.getValue())));
   }
@@ -226,6 +297,7 @@ constructHexagonLinkArgs(Compilation &C, const JobAction &JA,
   StringRef CpuVer = toolchains::HexagonToolChain::GetTargetCPUVersion(Args);
 
   bool NeedsSanitizerDeps = addSanitizerRuntimes(HTC, Args, CmdArgs);
+  bool NeedsXRayDeps = addXRayRuntime(HTC, Args, CmdArgs);
 
   //----------------------------------------------------------------------------
   // Silence warnings for various options
@@ -297,6 +369,8 @@ constructHexagonLinkArgs(Compilation &C, const JobAction &JA,
 
         CmdArgs.push_back("-lunwind");
       }
+      if (NeedsXRayDeps)
+        linkXRayRuntimeDeps(HTC, CmdArgs);
 
       CmdArgs.push_back("-lclang_rt.builtins-hexagon");
       CmdArgs.push_back("-lc");
diff --git a/clang/lib/Driver/ToolChains/Linux.cpp b/clang/lib/Driver/ToolChains/Linux.cpp
index 198774506e5e..e413640abad3 100644
--- a/clang/lib/Driver/ToolChains/Linux.cpp
+++ b/clang/lib/Driver/ToolChains/Linux.cpp
@@ -277,14 +277,11 @@ Linux::Linux(const Driver &D, const llvm::Triple &Triple, const ArgList &Args)
     // Android sysroots contain a library directory for each supported OS
     // version as well as some unversioned libraries in the usual multiarch
     // directory.
-    unsigned Major;
-    unsigned Minor;
-    unsigned Micro;
-    Triple.getEnvironmentVersion(Major, Minor, Micro);
-    addPathIfExists(D,
-                    SysRoot + "/usr/lib/" + MultiarchTriple + "/" +
-                        llvm::to_string(Major),
-                    Paths);
+    addPathIfExists(
+        D,
+        SysRoot + "/usr/lib/" + MultiarchTriple + "/" +
+            llvm::to_string(Triple.getEnvironmentVersion().getMajor()),
+        Paths);
   }
 
   addPathIfExists(D, SysRoot + "/usr/lib/" + MultiarchTriple, Paths);
@@ -666,8 +663,8 @@ void Linux::AddIAMCUIncludeArgs(const ArgList &DriverArgs,
 }
 
 bool Linux::isPIEDefault(const llvm::opt::ArgList &Args) const {
-  return getTriple().isAndroid() || getTriple().isMusl() ||
-         getSanitizerArgs(Args).requiresPIE();
+  return CLANG_DEFAULT_PIE_ON_LINUX || getTriple().isAndroid() ||
+         getTriple().isMusl() || getSanitizerArgs(Args).requiresPIE();
 }
 
 bool Linux::IsAArch64OutlineAtomicsDefault(const ArgList &Args) const {
diff --git a/clang/lib/Driver/ToolChains/MSVC.cpp b/clang/lib/Driver/ToolChains/MSVC.cpp
index 792b0a51fea0..66e9d8ab525a 100644
--- a/clang/lib/Driver/ToolChains/MSVC.cpp
+++ b/clang/lib/Driver/ToolChains/MSVC.cpp
@@ -1194,14 +1194,6 @@ bool MSVCToolChain::getUniversalCRTLibraryPath(const ArgList &Args,
   return true;
 }
 
-static VersionTuple getMSVCVersionFromTriple(const llvm::Triple &Triple) {
-  unsigned Major, Minor, Micro;
-  Triple.getEnvironmentVersion(Major, Minor, Micro);
-  if (Major || Minor || Micro)
-    return VersionTuple(Major, Minor, Micro);
-  return VersionTuple();
-}
-
 static VersionTuple getMSVCVersionFromExe(const std::string &BinDir) {
   VersionTuple Version;
 #ifdef _WIN32
@@ -1374,7 +1366,7 @@ VersionTuple MSVCToolChain::computeMSVCVersion(const Driver *D,
   bool IsWindowsMSVC = getTriple().isWindowsMSVCEnvironment();
   VersionTuple MSVT = ToolChain::computeMSVCVersion(D, Args);
   if (MSVT.empty())
-    MSVT = getMSVCVersionFromTriple(getTriple());
+    MSVT = getTriple().getEnvironmentVersion();
   if (MSVT.empty() && IsWindowsMSVC)
     MSVT = getMSVCVersionFromExe(getSubDirectoryPath(SubDirectoryType::Bin));
   if (MSVT.empty() &&
diff --git a/clang/lib/Driver/ToolChains/NetBSD.cpp b/clang/lib/Driver/ToolChains/NetBSD.cpp
index 7571398b7cc6..37b1fc5215ff 100644
--- a/clang/lib/Driver/ToolChains/NetBSD.cpp
+++ b/clang/lib/Driver/ToolChains/NetBSD.cpp
@@ -270,10 +270,9 @@ void netbsd::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back(Args.MakeArgString(ToolChain.getCompilerRTPath()));
   }
 
-  unsigned Major, Minor, Micro;
-  Triple.getOSVersion(Major, Minor, Micro);
+  VersionTuple OsVersion = Triple.getOSVersion();
   bool useLibgcc = true;
-  if (Major >= 7 || Major == 0) {
+  if (OsVersion >= VersionTuple(7) || OsVersion.getMajor() == 0) {
     switch (ToolChain.getArch()) {
     case llvm::Triple::aarch64:
     case llvm::Triple::aarch64_be:
@@ -409,9 +408,8 @@ Tool *NetBSD::buildAssembler() const {
 Tool *NetBSD::buildLinker() const { return new tools::netbsd::Linker(*this); }
 
 ToolChain::CXXStdlibType NetBSD::GetDefaultCXXStdlibType() const {
-  unsigned Major, Minor, Micro;
-  getTriple().getOSVersion(Major, Minor, Micro);
-  if (Major >= 7 || Major == 0) {
+  VersionTuple OsVersion = getTriple().getOSVersion();
+  if (OsVersion >= VersionTuple(7) || OsVersion.getMajor() == 0) {
     switch (getArch()) {
     case llvm::Triple::aarch64:
     case llvm::Triple::aarch64_be:
@@ -505,14 +503,13 @@ void NetBSD::addClangTargetOptions(const ArgList &DriverArgs,
   if (SanArgs.hasAnySanitizer())
     CC1Args.push_back("-D_REENTRANT");
 
-  unsigned Major, Minor, Micro;
-  getTriple().getOSVersion(Major, Minor, Micro);
+  VersionTuple OsVersion = getTriple().getOSVersion();
   bool UseInitArrayDefault =
-    Major >= 9 || Major == 0 ||
-    getTriple().getArch() == llvm::Triple::aarch64 ||
-    getTriple().getArch() == llvm::Triple::aarch64_be ||
-    getTriple().getArch() == llvm::Triple::arm ||
-    getTriple().getArch() == llvm::Triple::armeb;
+      OsVersion >= VersionTuple(9) || OsVersion.getMajor() == 0 ||
+      getTriple().getArch() == llvm::Triple::aarch64 ||
+      getTriple().getArch() == llvm::Triple::aarch64_be ||
+      getTriple().getArch() == llvm::Triple::arm ||
+      getTriple().getArch() == llvm::Triple::armeb;
 
   if (!DriverArgs.hasFlag(options::OPT_fuse_init_array,
                           options::OPT_fno_use_init_array, UseInitArrayDefault))
diff --git a/clang/lib/Driver/ToolChains/SPIRV.cpp b/clang/lib/Driver/ToolChains/SPIRV.cpp
index 16e72d3c733f..50d03e79bbb0 100644
--- a/clang/lib/Driver/ToolChains/SPIRV.cpp
+++ b/clang/lib/Driver/ToolChains/SPIRV.cpp
@@ -13,6 +13,7 @@
 #include "clang/Driver/Options.h"
 
 using namespace clang::driver;
+using namespace clang::driver::toolchains;
 using namespace clang::driver::tools;
 using namespace llvm::opt;
 
@@ -27,7 +28,7 @@ void SPIRV::constructTranslateCommand(Compilation &C, const Tool &T,
   if (Input.getType() == types::TY_PP_Asm)
     CmdArgs.push_back("-to-binary");
   if (Output.getType() == types::TY_PP_Asm)
-    CmdArgs.push_back("-spirv-text");
+    CmdArgs.push_back("--spirv-tools-dis");
 
   CmdArgs.append({"-o", Output.getFilename()});
 
@@ -47,3 +48,25 @@ void SPIRV::Translator::ConstructJob(Compilation &C, const JobAction &JA,
     llvm_unreachable("Invalid number of input files.");
   constructTranslateCommand(C, *this, JA, Output, Inputs[0], {});
 }
+
+clang::driver::Tool *SPIRVToolChain::getTranslator() const {
+  if (!Translator)
+    Translator = std::make_unique<SPIRV::Translator>(*this);
+  return Translator.get();
+}
+
+clang::driver::Tool *SPIRVToolChain::SelectTool(const JobAction &JA) const {
+  Action::ActionClass AC = JA.getKind();
+  return SPIRVToolChain::getTool(AC);
+}
+
+clang::driver::Tool *SPIRVToolChain::getTool(Action::ActionClass AC) const {
+  switch (AC) {
+  default:
+    break;
+  case Action::BackendJobClass:
+  case Action::AssembleJobClass:
+    return SPIRVToolChain::getTranslator();
+  }
+  return ToolChain::getTool(AC);
+}
diff --git a/clang/lib/Driver/ToolChains/SPIRV.h b/clang/lib/Driver/ToolChains/SPIRV.h
index 35d0446bd8b8..229f7018e3b5 100644
--- a/clang/lib/Driver/ToolChains/SPIRV.h
+++ b/clang/lib/Driver/ToolChains/SPIRV.h
@@ -41,6 +41,39 @@ public:
 
 } // namespace SPIRV
 } // namespace tools
+
+namespace toolchains {
+
+class LLVM_LIBRARY_VISIBILITY SPIRVToolChain final : public ToolChain {
+  mutable std::unique_ptr<Tool> Translator;
+
+public:
+  SPIRVToolChain(const Driver &D, const llvm::Triple &Triple,
+                 const llvm::opt::ArgList &Args)
+      : ToolChain(D, Triple, Args) {}
+
+  bool useIntegratedAs() const override { return true; }
+  bool useIntegratedBackend() const override { return false; }
+
+  bool IsMathErrnoDefault() const override { return false; }
+  bool isCrossCompiling() const override { return true; }
+  bool isPICDefault() const override { return false; }
+  bool isPIEDefault(const llvm::opt::ArgList &Args) const override {
+    return false;
+  }
+  bool isPICDefaultForced() const override { return false; }
+  bool SupportsProfiling() const override { return false; }
+
+  clang::driver::Tool *SelectTool(const JobAction &JA) const override;
+
+protected:
+  clang::driver::Tool *getTool(Action::ActionClass AC) const override;
+
+private:
+  clang::driver::Tool *getTranslator() const;
+};
+
+} // namespace toolchains
 } // namespace driver
 } // namespace clang
 #endif
diff --git a/clang/lib/Driver/ToolChains/VEToolchain.cpp b/clang/lib/Driver/ToolChains/VEToolchain.cpp
index 1fcc52684baa..4cdeec7f9d8a 100644
--- a/clang/lib/Driver/ToolChains/VEToolchain.cpp
+++ b/clang/lib/Driver/ToolChains/VEToolchain.cpp
@@ -28,17 +28,27 @@ VEToolChain::VEToolChain(const Driver &D, const llvm::Triple &Triple,
   getProgramPaths().push_back("/opt/nec/ve/bin");
   // ProgramPaths are found via 'PATH' environment variable.
 
-  // default file paths are:
-  //   ${RESOURCEDIR}/lib/linux/ve (== getArchSpecificLibPath)
-  //   /lib/../lib64
-  //   /usr/lib/../lib64
-  //   ${BINPATH}/../lib
-  //   /lib
-  //   /usr/lib
-  //
-  // These are OK for host, but no go for VE.  So, defines them all
-  // from scratch here.
+  // Default library paths are following:
+  //   ${RESOURCEDIR}/lib/ve-unknown-linux-gnu,
+  // These are OK.
+
+  // Default file paths are following:
+  //   ${RESOURCEDIR}/lib/linux/ve, (== getArchSpecificLibPath)
+  //   /lib/../lib64,
+  //   /usr/lib/../lib64,
+  //   ${BINPATH}/../lib,
+  //   /lib,
+  //   /usr/lib,
+  // These are OK for host, but no go for VE.
+
+  // Define file paths from scratch here.
   getFilePaths().clear();
+
+  // Add library directories:
+  //   ${BINPATH}/../lib/ve-unknown-linux-gnu, (== getStdlibPath)
+  //   ${RESOURCEDIR}/lib/linux/ve, (== getArchSpecificLibPath)
+  //   ${SYSROOT}/opt/nec/ve/lib,
+  getFilePaths().push_back(getStdlibPath());
   getFilePaths().push_back(getArchSpecificLibPath());
   getFilePaths().push_back(computeSysRoot() + "/opt/nec/ve/lib");
 }
@@ -115,9 +125,10 @@ void VEToolChain::AddClangCXXStdlibIncludeArgs(const ArgList &DriverArgs,
     ArrayRef<StringRef> DirVec(Dirs);
     addSystemIncludes(DriverArgs, CC1Args, DirVec);
   } else {
-    SmallString<128> P(getDriver().ResourceDir);
-    llvm::sys::path::append(P, "include/c++/v1");
-    addSystemInclude(DriverArgs, CC1Args, P);
+    // Add following paths for multiple target installation.
+    //   ${INSTALLDIR}/include/ve-unknown-linux-gnu/c++/v1,
+    //   ${INSTALLDIR}/include/c++/v1,
+    addLibCxxIncludePaths(DriverArgs, CC1Args);
   }
 }
 
diff --git a/clang/lib/Driver/XRayArgs.cpp b/clang/lib/Driver/XRayArgs.cpp
index b44509ad3b88..63b575178bd1 100644
--- a/clang/lib/Driver/XRayArgs.cpp
+++ b/clang/lib/Driver/XRayArgs.cpp
@@ -40,6 +40,7 @@ XRayArgs::XRayArgs(const ToolChain &TC, const ArgList &Args) {
     case llvm::Triple::x86_64:
     case llvm::Triple::arm:
     case llvm::Triple::aarch64:
+    case llvm::Triple::hexagon:
     case llvm::Triple::ppc64le:
     case llvm::Triple::mips:
     case llvm::Triple::mipsel:
diff --git a/clang/lib/Format/BreakableToken.cpp b/clang/lib/Format/BreakableToken.cpp
index 968b35bfda23..5d03c9811e1b 100644
--- a/clang/lib/Format/BreakableToken.cpp
+++ b/clang/lib/Format/BreakableToken.cpp
@@ -91,7 +91,7 @@ getCommentSplit(StringRef Text, unsigned ContentStartColumn,
   // In JavaScript, some @tags can be followed by {, and machinery that parses
   // these comments will fail to understand the comment if followed by a line
   // break. So avoid ever breaking before a {.
-  if (Style.Language == FormatStyle::LK_JavaScript) {
+  if (Style.isJavaScript()) {
     StringRef::size_type SpaceOffset =
         Text.find_first_of(Blanks, MaxSplitBytes);
     if (SpaceOffset != StringRef::npos && SpaceOffset + 1 < Text.size() &&
@@ -127,8 +127,7 @@ getCommentSplit(StringRef Text, unsigned ContentStartColumn,
     }
 
     // Avoid ever breaking before a @tag or a { in JavaScript.
-    if (Style.Language == FormatStyle::LK_JavaScript &&
-        SpaceOffset + 1 < Text.size() &&
+    if (Style.isJavaScript() && SpaceOffset + 1 < Text.size() &&
         (Text[SpaceOffset + 1] == '{' || Text[SpaceOffset + 1] == '@')) {
       SpaceOffset = Text.find_last_of(Blanks, SpaceOffset);
       continue;
@@ -460,8 +459,7 @@ BreakableBlockComment::BreakableBlockComment(
   IndentAtLineBreak = std::max<unsigned>(IndentAtLineBreak, Decoration.size());
 
   // Detect a multiline jsdoc comment and set DelimitersOnNewline in that case.
-  if (Style.Language == FormatStyle::LK_JavaScript ||
-      Style.Language == FormatStyle::LK_Java) {
+  if (Style.isJavaScript() || Style.Language == FormatStyle::LK_Java) {
     if ((Lines[0] == "*" || Lines[0].startswith("* ")) && Lines.size() > 1) {
       // This is a multiline jsdoc comment.
       DelimitersOnNewline = true;
@@ -580,8 +578,7 @@ const llvm::StringSet<>
 };
 
 unsigned BreakableBlockComment::getContentIndent(unsigned LineIndex) const {
-  if (Style.Language != FormatStyle::LK_Java &&
-      Style.Language != FormatStyle::LK_JavaScript)
+  if (Style.Language != FormatStyle::LK_Java && !Style.isJavaScript())
     return 0;
   // The content at LineIndex 0 of a comment like:
   // /** line 0 */
diff --git a/clang/lib/Format/ContinuationIndenter.cpp b/clang/lib/Format/ContinuationIndenter.cpp
index 5073f5105d05..4225d6b67b0e 100644
--- a/clang/lib/Format/ContinuationIndenter.cpp
+++ b/clang/lib/Format/ContinuationIndenter.cpp
@@ -422,8 +422,7 @@ bool ContinuationIndenter::mustBreak(const LineState &State) {
       //     ...
       //   }.bind(...));
       // FIXME: We should find a more generic solution to this problem.
-      !(State.Column <= NewLineColumn &&
-        Style.Language == FormatStyle::LK_JavaScript) &&
+      !(State.Column <= NewLineColumn && Style.isJavaScript()) &&
       !(Previous.closesScopeAfterBlock() && State.Column <= NewLineColumn))
     return true;
 
@@ -493,14 +492,14 @@ bool ContinuationIndenter::mustBreak(const LineState &State) {
       return true;
   }
 
-  // Break after the closing parenthesis of TypeScript decorators before
-  // functions, getters and setters.
-  static const llvm::StringSet<> BreakBeforeDecoratedTokens = {"get", "set",
-                                                               "function"};
-  if (Style.Language == FormatStyle::LK_JavaScript &&
-      BreakBeforeDecoratedTokens.contains(Current.TokenText) &&
-      Previous.is(tok::r_paren) && Previous.is(TT_JavaAnnotation)) {
-    return true;
+  if (Style.isJavaScript() && Previous.is(tok::r_paren) &&
+      Previous.is(TT_JavaAnnotation)) {
+    // Break after the closing parenthesis of TypeScript decorators before
+    // functions, getters and setters.
+    static const llvm::StringSet<> BreakBeforeDecoratedTokens = {"get", "set",
+                                                                 "function"};
+    if (BreakBeforeDecoratedTokens.contains(Current.TokenText))
+      return true;
   }
 
   // If the return type spans multiple lines, wrap before the function name.
@@ -510,7 +509,7 @@ bool ContinuationIndenter::mustBreak(const LineState &State) {
          Style.AlwaysBreakAfterReturnType != FormatStyle::RTBS_None) &&
         // Don't always break between a JavaScript `function` and the function
         // name.
-        Style.Language != FormatStyle::LK_JavaScript) ||
+        !Style.isJavaScript()) ||
        (Current.is(tok::kw_operator) && !Previous.is(tok::coloncolon))) &&
       !Previous.is(tok::kw_template) && State.Stack.back().BreakBeforeParameter)
     return true;
@@ -827,9 +826,8 @@ unsigned ContinuationIndenter::addTokenOnNewLine(LineState &State,
   // is common and should be formatted like a free-standing function. The same
   // goes for wrapping before the lambda return type arrow.
   if (!Current.is(TT_LambdaArrow) &&
-      (Style.Language != FormatStyle::LK_JavaScript ||
-       Current.NestingLevel != 0 || !PreviousNonComment ||
-       !PreviousNonComment->is(tok::equal) ||
+      (!Style.isJavaScript() || Current.NestingLevel != 0 ||
+       !PreviousNonComment || !PreviousNonComment->is(tok::equal) ||
        !Current.isOneOf(Keywords.kw_async, Keywords.kw_function)))
     State.Stack.back().NestedBlockIndent = State.Column;
 
@@ -1337,6 +1335,9 @@ unsigned ContinuationIndenter::moveStateToNextToken(LineState &State,
 void ContinuationIndenter::moveStatePastFakeLParens(LineState &State,
                                                     bool Newline) {
   const FormatToken &Current = *State.NextToken;
+  if (Current.FakeLParens.empty())
+    return;
+
   const FormatToken *Previous = Current.getPreviousNonComment();
 
   // Don't add extra indentation for the first fake parenthesis after
@@ -1348,10 +1349,7 @@ void ContinuationIndenter::moveStatePastFakeLParens(LineState &State,
                     (Previous->getPrecedence() == prec::Assignment &&
                      Style.AlignOperands != FormatStyle::OAS_DontAlign) ||
                     Previous->is(TT_ObjCMethodExpr)));
-  for (SmallVectorImpl<prec::Level>::const_reverse_iterator
-           I = Current.FakeLParens.rbegin(),
-           E = Current.FakeLParens.rend();
-       I != E; ++I) {
+  for (const auto &PrecedenceLevel : llvm::reverse(Current.FakeLParens)) {
     ParenState NewParenState = State.Stack.back();
     NewParenState.Tok = nullptr;
     NewParenState.ContainsLineBreak = false;
@@ -1363,7 +1361,7 @@ void ContinuationIndenter::moveStatePastFakeLParens(LineState &State,
         NewParenState.NoLineBreak || State.Stack.back().NoLineBreakInOperand;
 
     // Don't propagate AvoidBinPacking into subexpressions of arg/param lists.
-    if (*I > prec::Comma)
+    if (PrecedenceLevel > prec::Comma)
       NewParenState.AvoidBinPacking = false;
 
     // Indent from 'LastSpace' unless these are fake parentheses encapsulating
@@ -1371,11 +1369,11 @@ void ContinuationIndenter::moveStatePastFakeLParens(LineState &State,
     // brackets is disabled.
     if (!Current.isTrailingComment() &&
         (Style.AlignOperands != FormatStyle::OAS_DontAlign ||
-         *I < prec::Assignment) &&
+         PrecedenceLevel < prec::Assignment) &&
         (!Previous || Previous->isNot(tok::kw_return) ||
-         (Style.Language != FormatStyle::LK_Java && *I > 0)) &&
+         (Style.Language != FormatStyle::LK_Java && PrecedenceLevel > 0)) &&
         (Style.AlignAfterOpenBracket != FormatStyle::BAS_DontAlign ||
-         *I != prec::Comma || Current.NestingLevel == 0)) {
+         PrecedenceLevel != prec::Comma || Current.NestingLevel == 0)) {
       NewParenState.Indent =
           std::max(std::max(State.Column, NewParenState.Indent),
                    State.Stack.back().LastSpace);
@@ -1384,7 +1382,7 @@ void ContinuationIndenter::moveStatePastFakeLParens(LineState &State,
     if (Previous &&
         (Previous->getPrecedence() == prec::Assignment ||
          Previous->is(tok::kw_return) ||
-         (*I == prec::Conditional && Previous->is(tok::question) &&
+         (PrecedenceLevel == prec::Conditional && Previous->is(tok::question) &&
           Previous->is(TT_ConditionalExpr))) &&
         !Newline) {
       // If BreakBeforeBinaryOperators is set, un-indent a bit to account for
@@ -1402,9 +1400,9 @@ void ContinuationIndenter::moveStatePastFakeLParens(LineState &State,
     //       ParameterToInnerFunction));
     //   OuterFunction(SomeObject.InnerFunctionCall( // break
     //       ParameterToInnerFunction));
-    if (*I > prec::Unknown)
+    if (PrecedenceLevel > prec::Unknown)
       NewParenState.LastSpace = std::max(NewParenState.LastSpace, State.Column);
-    if (*I != prec::Conditional && !Current.is(TT_UnaryOperator) &&
+    if (PrecedenceLevel != prec::Conditional && !Current.is(TT_UnaryOperator) &&
         Style.AlignAfterOpenBracket != FormatStyle::BAS_DontAlign)
       NewParenState.StartOfFunctionCall = State.Column;
 
@@ -1413,17 +1411,18 @@ void ContinuationIndenter::moveStatePastFakeLParens(LineState &State,
     // an assignment (i.e. *I <= prec::Assignment) as those have different
     // indentation rules. Indent other expression, unless the indentation needs
     // to be skipped.
-    if (*I == prec::Conditional && Previous && Previous->is(tok::colon) &&
-        Previous->is(TT_ConditionalExpr) && I == Current.FakeLParens.rbegin() &&
+    if (PrecedenceLevel == prec::Conditional && Previous &&
+        Previous->is(tok::colon) && Previous->is(TT_ConditionalExpr) &&
+        &PrecedenceLevel == &Current.FakeLParens.back() &&
         !State.Stack.back().IsWrappedConditional) {
       NewParenState.IsChainedConditional = true;
       NewParenState.UnindentOperator = State.Stack.back().UnindentOperator;
-    } else if (*I == prec::Conditional ||
-               (!SkipFirstExtraIndent && *I > prec::Assignment &&
+    } else if (PrecedenceLevel == prec::Conditional ||
+               (!SkipFirstExtraIndent && PrecedenceLevel > prec::Assignment &&
                 !Current.isTrailingComment())) {
       NewParenState.Indent += Style.ContinuationIndentWidth;
     }
-    if ((Previous && !Previous->opensScope()) || *I != prec::Comma)
+    if ((Previous && !Previous->opensScope()) || PrecedenceLevel != prec::Comma)
       NewParenState.BreakBeforeParameter = false;
     State.Stack.push_back(NewParenState);
     SkipFirstExtraIndent = false;
@@ -1518,7 +1517,7 @@ void ContinuationIndenter::moveStatePastScopeOpener(LineState &State,
 
     AvoidBinPacking =
         (State.Stack.back().IsCSharpGenericTypeConstraint) ||
-        (Style.Language == FormatStyle::LK_JavaScript && EndsInComma) ||
+        (Style.isJavaScript() && EndsInComma) ||
         (State.Line->MustBeDeclaration && !BinPackDeclaration) ||
         (!State.Line->MustBeDeclaration && !Style.BinPackArguments) ||
         (Style.ExperimentalAutoDetectBinPacking &&
@@ -1547,7 +1546,7 @@ void ContinuationIndenter::moveStatePastScopeOpener(LineState &State,
       }
     }
 
-    if (Style.Language == FormatStyle::LK_JavaScript && EndsInComma)
+    if (Style.isJavaScript() && EndsInComma)
       BreakBeforeParameter = true;
   }
   // Generally inherit NoLineBreak from the current scope to nested scope.
@@ -1924,9 +1923,9 @@ ContinuationIndenter::createBreakableToken(const FormatToken &Current,
     // FIXME: String literal breaking is currently disabled for C#, Java, Json
     // and JavaScript, as it requires strings to be merged using "+" which we
     // don't support.
-    if (Style.Language == FormatStyle::LK_Java ||
-        Style.Language == FormatStyle::LK_JavaScript || Style.isCSharp() ||
-        Style.isJson() || !Style.BreakStringLiterals || !AllowBreak)
+    if (Style.Language == FormatStyle::LK_Java || Style.isJavaScript() ||
+        Style.isCSharp() || Style.isJson() || !Style.BreakStringLiterals ||
+        !AllowBreak)
       return nullptr;
 
     // Don't break string literals inside preprocessor directives (except for
diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp
index 17de1075aeaa..be01daa38929 100644
--- a/clang/lib/Format/Format.cpp
+++ b/clang/lib/Format/Format.cpp
@@ -2586,12 +2586,31 @@ tooling::Replacements sortCppIncludes(const FormatStyle &Style, StringRef Code,
   bool MainIncludeFound = false;
   bool FormattingOff = false;
 
+  llvm::Regex RawStringRegex(
+      "R\"(([\\[A-Za-z0-9_{}#<>%:;.?*+/^&\\$|~!=,'\\-]|])*)\\(");
+  SmallVector<StringRef, 2> RawStringMatches;
+  std::string RawStringTermination = ")\"";
+
   for (;;) {
     auto Pos = Code.find('\n', SearchFrom);
     StringRef Line =
         Code.substr(Prev, (Pos != StringRef::npos ? Pos : Code.size()) - Prev);
 
     StringRef Trimmed = Line.trim();
+
+    // #includes inside raw string literals need to be ignored.
+    // or we will sort the contents of the string.
+    // Skip past until we think we are at the rawstring literal close.
+    if (RawStringRegex.match(Trimmed, &RawStringMatches)) {
+      std::string CharSequence = RawStringMatches[1].str();
+      RawStringTermination = ")" + CharSequence + "\"";
+      FormattingOff = true;
+    }
+
+    if (Trimmed.contains(RawStringTermination)) {
+      FormattingOff = false;
+    }
+
     if (Trimmed == "// clang-format off" || Trimmed == "/* clang-format off */")
       FormattingOff = true;
     else if (Trimmed == "// clang-format on" ||
@@ -3031,8 +3050,7 @@ reformat(const FormatStyle &Style, StringRef Code,
       });
   }
 
-  if (Style.Language == FormatStyle::LK_JavaScript &&
-      Style.JavaScriptQuotes != FormatStyle::JSQS_Leave)
+  if (Style.isJavaScript() && Style.JavaScriptQuotes != FormatStyle::JSQS_Leave)
     Passes.emplace_back([&](const Environment &Env) {
       return JavaScriptRequoter(Env, Expanded).process();
     });
@@ -3041,7 +3059,7 @@ reformat(const FormatStyle &Style, StringRef Code,
     return Formatter(Env, Expanded, Status).process();
   });
 
-  if (Style.Language == FormatStyle::LK_JavaScript &&
+  if (Style.isJavaScript() &&
       Style.InsertTrailingCommas == FormatStyle::TCS_Wrapped)
     Passes.emplace_back([&](const Environment &Env) {
       return TrailingCommaInserter(Env, Expanded).process();
diff --git a/clang/lib/Format/FormatToken.cpp b/clang/lib/Format/FormatToken.cpp
index 6768f041135c..57f8a5a45cbb 100644
--- a/clang/lib/Format/FormatToken.cpp
+++ b/clang/lib/Format/FormatToken.cpp
@@ -70,6 +70,10 @@ bool FormatToken::isSimpleTypeSpecifier() const {
   }
 }
 
+bool FormatToken::isTypeOrIdentifier() const {
+  return isSimpleTypeSpecifier() || Tok.isOneOf(tok::kw_auto, tok::identifier);
+}
+
 TokenRole::~TokenRole() {}
 
 void TokenRole::precomputeFormattingInfos(const FormatToken *Token) {}
diff --git a/clang/lib/Format/FormatToken.h b/clang/lib/Format/FormatToken.h
index 1a2858018fde..d410ede32240 100644
--- a/clang/lib/Format/FormatToken.h
+++ b/clang/lib/Format/FormatToken.h
@@ -521,7 +521,9 @@ public:
   }
 
   /// Determine whether the token is a simple-type-specifier.
-  bool isSimpleTypeSpecifier() const;
+  LLVM_NODISCARD bool isSimpleTypeSpecifier() const;
+
+  LLVM_NODISCARD bool isTypeOrIdentifier() const;
 
   bool isObjCAccessSpecifier() const {
     return is(tok::at) && Next &&
diff --git a/clang/lib/Format/FormatTokenLexer.cpp b/clang/lib/Format/FormatTokenLexer.cpp
index 64fbd2d5d45b..7736a7042f86 100644
--- a/clang/lib/Format/FormatTokenLexer.cpp
+++ b/clang/lib/Format/FormatTokenLexer.cpp
@@ -78,7 +78,7 @@ ArrayRef<FormatToken *> FormatTokenLexer::lex() {
   assert(FirstInLineIndex == 0);
   do {
     Tokens.push_back(getNextToken());
-    if (Style.Language == FormatStyle::LK_JavaScript) {
+    if (Style.isJavaScript()) {
       tryParseJSRegexLiteral();
       handleTemplateStrings();
     }
@@ -107,7 +107,7 @@ void FormatTokenLexer::tryMergePreviousTokens() {
   if (Style.isCpp() && tryTransformTryUsageForC())
     return;
 
-  if (Style.Language == FormatStyle::LK_JavaScript || Style.isCSharp()) {
+  if (Style.isJavaScript() || Style.isCSharp()) {
     static const tok::TokenKind NullishCoalescingOperator[] = {tok::question,
                                                                tok::question};
     static const tok::TokenKind NullPropagatingOperator[] = {tok::question,
@@ -152,7 +152,7 @@ void FormatTokenLexer::tryMergePreviousTokens() {
   if (tryMergeNSStringLiteral())
     return;
 
-  if (Style.Language == FormatStyle::LK_JavaScript) {
+  if (Style.isJavaScript()) {
     static const tok::TokenKind JSIdentity[] = {tok::equalequal, tok::equal};
     static const tok::TokenKind JSNotIdentity[] = {tok::exclaimequal,
                                                    tok::equal};
@@ -920,8 +920,7 @@ FormatToken *FormatTokenLexer::getNextToken() {
   // finds comments that contain a backslash followed by a line break, truncates
   // the comment token at the backslash, and resets the lexer to restart behind
   // the backslash.
-  if ((Style.Language == FormatStyle::LK_JavaScript ||
-       Style.Language == FormatStyle::LK_Java) &&
+  if ((Style.isJavaScript() || Style.Language == FormatStyle::LK_Java) &&
       FormatTok->is(tok::comment) && FormatTok->TokenText.startswith("//")) {
     size_t BackslashPos = FormatTok->TokenText.find('\\');
     while (BackslashPos != StringRef::npos) {
@@ -982,7 +981,7 @@ FormatToken *FormatTokenLexer::getNextToken() {
                            tok::kw_operator)) {
       FormatTok->Tok.setKind(tok::identifier);
       FormatTok->Tok.setIdentifierInfo(nullptr);
-    } else if (Style.Language == FormatStyle::LK_JavaScript &&
+    } else if (Style.isJavaScript() &&
                FormatTok->isOneOf(tok::kw_struct, tok::kw_union,
                                   tok::kw_operator)) {
       FormatTok->Tok.setKind(tok::identifier);
@@ -1060,14 +1059,12 @@ void FormatTokenLexer::readRawToken(FormatToken &Tok) {
     if (!Tok.TokenText.empty() && Tok.TokenText[0] == '"') {
       Tok.Tok.setKind(tok::string_literal);
       Tok.IsUnterminatedLiteral = true;
-    } else if (Style.Language == FormatStyle::LK_JavaScript &&
-               Tok.TokenText == "''") {
+    } else if (Style.isJavaScript() && Tok.TokenText == "''") {
       Tok.Tok.setKind(tok::string_literal);
     }
   }
 
-  if ((Style.Language == FormatStyle::LK_JavaScript ||
-       Style.Language == FormatStyle::LK_Proto ||
+  if ((Style.isJavaScript() || Style.Language == FormatStyle::LK_Proto ||
        Style.Language == FormatStyle::LK_TextProto) &&
       Tok.is(tok::char_constant)) {
     Tok.Tok.setKind(tok::string_literal);
diff --git a/clang/lib/Format/NamespaceEndCommentsFixer.cpp b/clang/lib/Format/NamespaceEndCommentsFixer.cpp
index def551f863cd..38ab5b9df76d 100644
--- a/clang/lib/Format/NamespaceEndCommentsFixer.cpp
+++ b/clang/lib/Format/NamespaceEndCommentsFixer.cpp
@@ -180,9 +180,13 @@ getNamespaceToken(const AnnotatedLine *Line,
   if (NamespaceTok->is(tok::l_brace)) {
     // "namespace" keyword can be on the line preceding '{', e.g. in styles
     // where BraceWrapping.AfterNamespace is true.
-    if (StartLineIndex > 0)
+    if (StartLineIndex > 0) {
       NamespaceTok = AnnotatedLines[StartLineIndex - 1]->First;
+      if (AnnotatedLines[StartLineIndex - 1]->endsWith(tok::semi))
+        return nullptr;
+    }
   }
+
   return NamespaceTok->getNamespaceToken();
 }
 
diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp
index a94d8cdc3b04..505a7250572b 100644
--- a/clang/lib/Format/TokenAnnotator.cpp
+++ b/clang/lib/Format/TokenAnnotator.cpp
@@ -242,7 +242,7 @@ private:
       bool OperatorCalledAsMemberFunction =
           Prev->Previous && Prev->Previous->isOneOf(tok::period, tok::arrow);
       Contexts.back().IsExpression = OperatorCalledAsMemberFunction;
-    } else if (Style.Language == FormatStyle::LK_JavaScript &&
+    } else if (Style.isJavaScript() &&
                (Line.startsWith(Keywords.kw_type, tok::identifier) ||
                 Line.startsWith(tok::kw_export, Keywords.kw_type,
                                 tok::identifier))) {
@@ -256,13 +256,13 @@ private:
                 Left->Previous->is(TT_BinaryOperator))) {
       // static_assert, if and while usually contain expressions.
       Contexts.back().IsExpression = true;
-    } else if (Style.Language == FormatStyle::LK_JavaScript && Left->Previous &&
+    } else if (Style.isJavaScript() && Left->Previous &&
                (Left->Previous->is(Keywords.kw_function) ||
                 (Left->Previous->endsSequence(tok::identifier,
                                               Keywords.kw_function)))) {
       // function(...) or function f(...)
       Contexts.back().IsExpression = false;
-    } else if (Style.Language == FormatStyle::LK_JavaScript && Left->Previous &&
+    } else if (Style.isJavaScript() && Left->Previous &&
                Left->Previous->is(TT_JsTypeColon)) {
       // let x: (SomeType);
       Contexts.back().IsExpression = false;
@@ -582,7 +582,7 @@ private:
         Left->setType(TT_InlineASMSymbolicNameLSquare);
       } else if (IsCpp11AttributeSpecifier) {
         Left->setType(TT_AttributeSquare);
-      } else if (Style.Language == FormatStyle::LK_JavaScript && Parent &&
+      } else if (Style.isJavaScript() && Parent &&
                  Contexts.back().ContextKind == tok::l_brace &&
                  Parent->isOneOf(tok::l_brace, tok::comma)) {
         Left->setType(TT_JsComputedPropertyName);
@@ -646,8 +646,7 @@ private:
 
     ScopedContextCreator ContextCreator(*this, tok::l_square, BindingIncrease);
     Contexts.back().IsExpression = true;
-    if (Style.Language == FormatStyle::LK_JavaScript && Parent &&
-        Parent->is(TT_JsTypeColon))
+    if (Style.isJavaScript() && Parent && Parent->is(TT_JsTypeColon))
       Contexts.back().IsExpression = false;
 
     Contexts.back().ColonIsObjCMethodExpr = StartsObjCMethodExpr;
@@ -774,7 +773,7 @@ private:
       Contexts.back().ColonIsDictLiteral = true;
       if (Left->is(BK_BracedInit))
         Contexts.back().IsExpression = true;
-      if (Style.Language == FormatStyle::LK_JavaScript && Left->Previous &&
+      if (Style.isJavaScript() && Left->Previous &&
           Left->Previous->is(TT_JsTypeColon))
         Contexts.back().IsExpression = false;
 
@@ -808,12 +807,11 @@ private:
                 Previous->is(tok::string_literal))
               Previous->setType(TT_SelectorName);
           }
-          if (CurrentToken->is(tok::colon) ||
-              Style.Language == FormatStyle::LK_JavaScript)
+          if (CurrentToken->is(tok::colon) || Style.isJavaScript())
             Left->setType(TT_DictLiteral);
         }
         if (CurrentToken->is(tok::comma)) {
-          if (Style.Language == FormatStyle::LK_JavaScript)
+          if (Style.isJavaScript())
             Left->setType(TT_DictLiteral);
           ++CommaCount;
         }
@@ -879,7 +877,7 @@ private:
       if (!Tok->Previous)
         return false;
       // Colons from ?: are handled in parseConditional().
-      if (Style.Language == FormatStyle::LK_JavaScript) {
+      if (Style.isJavaScript()) {
         if (Contexts.back().ColonIsForRangeExpr || // colon in for loop
             (Contexts.size() == 1 &&               // switch/case labels
              !Line.First->isOneOf(tok::kw_enum, tok::kw_case)) ||
@@ -979,8 +977,7 @@ private:
     case tok::amp:
       // | and & in declarations/type expressions represent union and
       // intersection types, respectively.
-      if (Style.Language == FormatStyle::LK_JavaScript &&
-          !Contexts.back().IsExpression)
+      if (Style.isJavaScript() && !Contexts.back().IsExpression)
         Tok->setType(TT_JsTypeOperator);
       break;
     case tok::kw_if:
@@ -995,7 +992,7 @@ private:
       }
       break;
     case tok::kw_for:
-      if (Style.Language == FormatStyle::LK_JavaScript) {
+      if (Style.isJavaScript()) {
         // x.for and {for: ...}
         if ((Tok->Previous && Tok->Previous->is(tok::period)) ||
             (Tok->Next && Tok->Next->is(tok::colon)))
@@ -1106,7 +1103,7 @@ private:
         CurrentToken->Previous->setType(TT_OverloadedOperator);
       break;
     case tok::question:
-      if (Style.Language == FormatStyle::LK_JavaScript && Tok->Next &&
+      if (Style.isJavaScript() && Tok->Next &&
           Tok->Next->isOneOf(tok::semi, tok::comma, tok::colon, tok::r_paren,
                              tok::r_brace)) {
         // Question marks before semicolons, colons, etc. indicate optional
@@ -1119,7 +1116,7 @@ private:
       // Declarations cannot be conditional expressions, this can only be part
       // of a type declaration.
       if (Line.MustBeDeclaration && !Contexts.back().IsExpression &&
-          Style.Language == FormatStyle::LK_JavaScript)
+          Style.isJavaScript())
         break;
       if (Style.isCSharp()) {
         // `Type?)`, `Type?>`, `Type? name;` and `Type? name =` can only be
@@ -1252,7 +1249,7 @@ private:
     if (!CurrentToken)
       return Type;
 
-    if (Style.Language == FormatStyle::LK_JavaScript && IsFirstToken) {
+    if (Style.isJavaScript() && IsFirstToken) {
       // JavaScript files can contain shebang lines of the form:
       // #!/usr/bin/env node
       // Treat these like C++ #include directives.
@@ -1354,14 +1351,13 @@ public:
     bool ImportStatement = false;
 
     // import {...} from '...';
-    if (Style.Language == FormatStyle::LK_JavaScript &&
-        CurrentToken->is(Keywords.kw_import))
+    if (Style.isJavaScript() && CurrentToken->is(Keywords.kw_import))
       ImportStatement = true;
 
     while (CurrentToken) {
       if (CurrentToken->is(tok::kw_virtual))
         KeywordVirtualFound = true;
-      if (Style.Language == FormatStyle::LK_JavaScript) {
+      if (Style.isJavaScript()) {
         // export {...} from '...';
         // An export followed by "from 'some string';" is a re-export from
         // another module identified by a URI and is treated as a
@@ -1504,7 +1500,7 @@ private:
         !Line.First->isOneOf(tok::kw_template, tok::kw_using, tok::kw_return) &&
         // Type aliases use `type X = ...;` in TypeScript and can be exported
         // using `export type ...`.
-        !(Style.Language == FormatStyle::LK_JavaScript &&
+        !(Style.isJavaScript() &&
           (Line.startsWith(Keywords.kw_type, tok::identifier) ||
            Line.startsWith(tok::kw_export, Keywords.kw_type,
                            tok::identifier))) &&
@@ -1633,11 +1629,11 @@ private:
       // The token type is already known.
       return;
 
-    if ((Style.Language == FormatStyle::LK_JavaScript || Style.isCSharp()) &&
+    if ((Style.isJavaScript() || Style.isCSharp()) &&
         Current.is(tok::exclaim)) {
       if (Current.Previous) {
         bool IsIdentifier =
-            Style.Language == FormatStyle::LK_JavaScript
+            Style.isJavaScript()
                 ? Keywords.IsJavaScriptIdentifier(
                       *Current.Previous, /* AcceptIdentifierName= */ true)
                 : Current.Previous->is(tok::identifier);
@@ -1679,7 +1675,7 @@ private:
       Current.setType(TT_LambdaArrow);
     } else if (Current.is(tok::arrow) && AutoFound && Line.MustBeDeclaration &&
                Current.NestingLevel == 0 &&
-               !Current.Previous->is(tok::kw_operator)) {
+               !Current.Previous->isOneOf(tok::kw_operator, tok::identifier)) {
       // not auto operator->() -> xxx;
       Current.setType(TT_TrailingReturnArrow);
     } else if (Current.is(tok::arrow) && Current.Previous &&
@@ -1704,8 +1700,8 @@ private:
     } else if (Current.isOneOf(tok::exclaim, tok::tilde)) {
       Current.setType(TT_UnaryOperator);
     } else if (Current.is(tok::question)) {
-      if (Style.Language == FormatStyle::LK_JavaScript &&
-          Line.MustBeDeclaration && !Contexts.back().IsExpression) {
+      if (Style.isJavaScript() && Line.MustBeDeclaration &&
+          !Contexts.back().IsExpression) {
         // In JavaScript, `interface X { foo?(): bar; }` is an optional method
         // on the interface, not a ternary expression.
         Current.setType(TT_JsTypeOptionalQuestion);
@@ -1748,8 +1744,7 @@ private:
                 Current.setType(TT_FunctionAnnotationRParen);
           }
         }
-    } else if (Current.is(tok::at) && Current.Next &&
-               Style.Language != FormatStyle::LK_JavaScript &&
+    } else if (Current.is(tok::at) && Current.Next && !Style.isJavaScript() &&
                Style.Language != FormatStyle::LK_Java) {
       // In Java & JavaScript, "@..." is a decorator or annotation. In ObjC, it
       // marks declarations and properties that need special formatting.
@@ -1796,7 +1791,7 @@ private:
       // function declaration have been found.
       Current.setType(TT_TrailingAnnotation);
     } else if ((Style.Language == FormatStyle::LK_Java ||
-                Style.Language == FormatStyle::LK_JavaScript) &&
+                Style.isJavaScript()) &&
                Current.Previous) {
       if (Current.Previous->is(tok::at) &&
           Current.isNot(Keywords.kw_interface)) {
@@ -1826,14 +1821,16 @@ private:
     if (Tok.Previous->isOneOf(TT_LeadingJavaAnnotation, Keywords.kw_instanceof,
                               Keywords.kw_as))
       return false;
-    if (Style.Language == FormatStyle::LK_JavaScript &&
-        Tok.Previous->is(Keywords.kw_in))
+    if (Style.isJavaScript() && Tok.Previous->is(Keywords.kw_in))
       return false;
 
     // Skip "const" as it does not have an influence on whether this is a name.
     FormatToken *PreviousNotConst = Tok.getPreviousNonComment();
-    while (PreviousNotConst && PreviousNotConst->is(tok::kw_const))
-      PreviousNotConst = PreviousNotConst->getPreviousNonComment();
+
+    // For javascript const can be like "let" or "var"
+    if (!Style.isJavaScript())
+      while (PreviousNotConst && PreviousNotConst->is(tok::kw_const))
+        PreviousNotConst = PreviousNotConst->getPreviousNonComment();
 
     if (!PreviousNotConst)
       return false;
@@ -1852,10 +1849,24 @@ private:
         PreviousNotConst->is(TT_TypeDeclarationParen))
       return true;
 
-    return (!IsPPKeyword &&
-            PreviousNotConst->isOneOf(tok::identifier, tok::kw_auto)) ||
-           PreviousNotConst->is(TT_PointerOrReference) ||
-           PreviousNotConst->isSimpleTypeSpecifier();
+    // If is a preprocess keyword like #define.
+    if (IsPPKeyword)
+      return false;
+
+    // int a or auto a.
+    if (PreviousNotConst->isOneOf(tok::identifier, tok::kw_auto))
+      return true;
+
+    // *a or &a or &&a.
+    if (PreviousNotConst->is(TT_PointerOrReference))
+      return true;
+
+    // MyClass a;
+    if (PreviousNotConst->isSimpleTypeSpecifier())
+      return true;
+
+    // const a = in JavaScript.
+    return (Style.isJavaScript() && PreviousNotConst->is(tok::kw_const));
   }
 
   /// Determine whether ')' is ending a cast.
@@ -2006,7 +2017,7 @@ private:
   /// Return the type of the given token assuming it is * or &.
   TokenType determineStarAmpUsage(const FormatToken &Tok, bool IsExpression,
                                   bool InTemplateArgument) {
-    if (Style.Language == FormatStyle::LK_JavaScript)
+    if (Style.isJavaScript())
       return TT_BinaryOperator;
 
     // && in C# must be a binary operator.
@@ -2034,9 +2045,8 @@ private:
                            tok::comma, tok::semi, tok::kw_return, tok::colon,
                            tok::kw_co_return, tok::kw_co_await,
                            tok::kw_co_yield, tok::equal, tok::kw_delete,
-                           tok::kw_sizeof, tok::kw_throw) ||
-        PrevToken->isOneOf(TT_BinaryOperator, TT_ConditionalExpr,
-                           TT_UnaryOperator, TT_CastRParen))
+                           tok::kw_sizeof, tok::kw_throw, TT_BinaryOperator,
+                           TT_ConditionalExpr, TT_UnaryOperator, TT_CastRParen))
       return TT_UnaryOperator;
 
     if (NextToken->is(tok::l_square) && NextToken->isNot(TT_LambdaLSquare))
@@ -2174,8 +2184,8 @@ public:
 
       int CurrentPrecedence = getCurrentPrecedence();
 
-      if (Current && Current->is(TT_SelectorName) &&
-          Precedence == CurrentPrecedence) {
+      if (Precedence == CurrentPrecedence && Current &&
+          Current->is(TT_SelectorName)) {
         if (LatestOperator)
           addFakeParenthesis(Start, prec::Level(Precedence));
         Start = Current;
@@ -2251,19 +2261,17 @@ private:
         return 0;
       if (Current->is(TT_RangeBasedForLoopColon))
         return prec::Comma;
-      if ((Style.Language == FormatStyle::LK_Java ||
-           Style.Language == FormatStyle::LK_JavaScript) &&
+      if ((Style.Language == FormatStyle::LK_Java || Style.isJavaScript()) &&
           Current->is(Keywords.kw_instanceof))
         return prec::Relational;
-      if (Style.Language == FormatStyle::LK_JavaScript &&
+      if (Style.isJavaScript() &&
           Current->isOneOf(Keywords.kw_in, Keywords.kw_as))
         return prec::Relational;
       if (Current->is(TT_BinaryOperator) || Current->is(tok::comma))
         return Current->getPrecedence();
       if (Current->isOneOf(tok::period, tok::arrow))
         return PrecedenceArrowAndPeriod;
-      if ((Style.Language == FormatStyle::LK_Java ||
-           Style.Language == FormatStyle::LK_JavaScript) &&
+      if ((Style.Language == FormatStyle::LK_Java || Style.isJavaScript()) &&
           Current->isOneOf(Keywords.kw_extends, Keywords.kw_implements,
                            Keywords.kw_throws))
         return 0;
@@ -2374,11 +2382,9 @@ static unsigned maxNestingDepth(const AnnotatedLine &Line) {
 }
 
 void TokenAnnotator::annotate(AnnotatedLine &Line) {
-  for (SmallVectorImpl<AnnotatedLine *>::iterator I = Line.Children.begin(),
-                                                  E = Line.Children.end();
-       I != E; ++I) {
-    annotate(**I);
-  }
+  for (auto &Child : Line.Children)
+    annotate(*Child);
+
   AnnotatingParser Parser(Style, Line, Keywords);
   Line.Type = Parser.parseLine();
 
@@ -2734,7 +2740,7 @@ unsigned TokenAnnotator::splitPenalty(const AnnotatedLine &Line,
       return 2;
     if (Left.is(tok::comma) && Left.NestingLevel == 0)
       return 3;
-  } else if (Style.Language == FormatStyle::LK_JavaScript) {
+  } else if (Style.isJavaScript()) {
     if (Right.is(Keywords.kw_function) && Left.isNot(tok::comma))
       return 100;
     if (Left.is(TT_JsTypeColon))
@@ -3024,8 +3030,14 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
          Style.SpaceAroundPointerQualifiers == FormatStyle::SAPQ_Both) &&
         (Left.is(TT_AttributeParen) || Left.canBePointerOrReferenceQualifier()))
       return true;
+    if (Left.Tok.isLiteral())
+      return true;
+    // for (auto a = 0, b = 0; const auto & c : {1, 2, 3})
+    if (Left.isTypeOrIdentifier() && Right.Next && Right.Next->Next &&
+        Right.Next->Next->is(TT_RangeBasedForLoopColon))
+      return getTokenPointerOrReferenceAlignment(Right) !=
+             FormatStyle::PAS_Left;
     return (
-        Left.Tok.isLiteral() ||
         (!Left.isOneOf(TT_PointerOrReference, tok::l_paren) &&
          (getTokenPointerOrReferenceAlignment(Right) != FormatStyle::PAS_Left ||
           (Line.IsMultiVariableDeclStmt &&
@@ -3044,18 +3056,32 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
          Style.SpaceAroundPointerQualifiers == FormatStyle::SAPQ_Both) &&
         Right.canBePointerOrReferenceQualifier())
       return true;
-    return Right.Tok.isLiteral() || Right.is(TT_BlockComment) ||
-           (Right.isOneOf(Keywords.kw_override, Keywords.kw_final) &&
-            !Right.is(TT_StartOfName)) ||
-           (Right.is(tok::l_brace) && Right.is(BK_Block)) ||
-           (!Right.isOneOf(TT_PointerOrReference, TT_ArraySubscriptLSquare,
-                           tok::l_paren) &&
-            (getTokenPointerOrReferenceAlignment(Left) !=
-                 FormatStyle::PAS_Right &&
-             !Line.IsMultiVariableDeclStmt) &&
-            Left.Previous &&
-            !Left.Previous->isOneOf(tok::l_paren, tok::coloncolon,
-                                    tok::l_square));
+    // & 1
+    if (Right.Tok.isLiteral())
+      return true;
+    // & /* comment
+    if (Right.is(TT_BlockComment))
+      return true;
+    // foo() -> const Bar * override/final
+    if (Right.isOneOf(Keywords.kw_override, Keywords.kw_final) &&
+        !Right.is(TT_StartOfName))
+      return true;
+    // & {
+    if (Right.is(tok::l_brace) && Right.is(BK_Block))
+      return true;
+    // for (auto a = 0, b = 0; const auto& c : {1, 2, 3})
+    if (Left.Previous && Left.Previous->isTypeOrIdentifier() && Right.Next &&
+        Right.Next->is(TT_RangeBasedForLoopColon))
+      return getTokenPointerOrReferenceAlignment(Left) !=
+             FormatStyle::PAS_Right;
+    return !Right.isOneOf(TT_PointerOrReference, TT_ArraySubscriptLSquare,
+                          tok::l_paren) &&
+           (getTokenPointerOrReferenceAlignment(Left) !=
+                FormatStyle::PAS_Right &&
+            !Line.IsMultiVariableDeclStmt) &&
+           Left.Previous &&
+           !Left.Previous->isOneOf(tok::l_paren, tok::coloncolon,
+                                   tok::l_square);
   }
   // Ensure right pointer alignment with ellipsis e.g. int *...P
   if (Left.is(tok::ellipsis) && Left.Previous &&
@@ -3146,8 +3172,7 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
     return Style.Cpp11BracedListStyle ? Style.SpacesInParentheses : true;
   if (Left.is(TT_BlockComment))
     // No whitespace in x(/*foo=*/1), except for JavaScript.
-    return Style.Language == FormatStyle::LK_JavaScript ||
-           !Left.TokenText.endswith("=*/");
+    return Style.isJavaScript() || !Left.TokenText.endswith("=*/");
 
   // Space between template and attribute.
   // e.g. template <typename T> [[nodiscard]] ...
@@ -3225,7 +3250,13 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
     return false;
   if (Left.is(tok::period) || Right.is(tok::period))
     return false;
-  if (Right.is(tok::hash) && Left.is(tok::identifier) && Left.TokenText == "L")
+  // u#str, U#str, L#str, u8#str
+  // uR#str, UR#str, LR#str, u8R#str
+  if (Right.is(tok::hash) && Left.is(tok::identifier) &&
+      (Left.TokenText == "L" || Left.TokenText == "u" ||
+       Left.TokenText == "U" || Left.TokenText == "u8" ||
+       Left.TokenText == "LR" || Left.TokenText == "uR" ||
+       Left.TokenText == "UR" || Left.TokenText == "u8R"))
     return false;
   if (Left.is(TT_TemplateCloser) && Left.MatchingParen &&
       Left.MatchingParen->Previous &&
@@ -3396,7 +3427,7 @@ bool TokenAnnotator::spaceRequiredBefore(const AnnotatedLine &Line,
                      Keywords.kw_async, Keywords.kw_unsafe) &&
         Right.is(tok::l_paren))
       return true;
-  } else if (Style.Language == FormatStyle::LK_JavaScript) {
+  } else if (Style.isJavaScript()) {
     if (Left.is(TT_FatArrow))
       return true;
     // for await ( ...
@@ -3694,11 +3725,18 @@ bool TokenAnnotator::mustBreakBefore(const AnnotatedLine &Line,
     return true;
 
   if (Style.isCSharp()) {
+    if (Left.is(TT_FatArrow) && Right.is(tok::l_brace) &&
+        Style.BraceWrapping.AfterFunction)
+      return true;
     if (Right.is(TT_CSharpNamedArgumentColon) ||
         Left.is(TT_CSharpNamedArgumentColon))
       return false;
     if (Right.is(TT_CSharpGenericTypeConstraint))
       return true;
+    if (Right.Next && Right.Next->is(TT_FatArrow) &&
+        (Right.is(tok::numeric_constant) ||
+         (Right.is(tok::identifier) && Right.TokenText == "_")))
+      return true;
 
     // Break after C# [...] and before public/protected/private/internal.
     if (Left.is(TT_AttributeSquare) && Left.is(tok::r_square) &&
@@ -3710,7 +3748,7 @@ bool TokenAnnotator::mustBreakBefore(const AnnotatedLine &Line,
         Left.is(tok::r_square) && Right.is(tok::l_square))
       return true;
 
-  } else if (Style.Language == FormatStyle::LK_JavaScript) {
+  } else if (Style.isJavaScript()) {
     // FIXME: This might apply to other languages and token kinds.
     if (Right.is(tok::string_literal) && Left.is(tok::plus) && Left.Previous &&
         Left.Previous->is(tok::string_literal))
@@ -3800,15 +3838,13 @@ bool TokenAnnotator::mustBreakBefore(const AnnotatedLine &Line,
   if (Style.JavaScriptWrapImports || Line.Type != LT_ImportStatement) {
     const FormatToken *BeforeClosingBrace = nullptr;
     if ((Left.isOneOf(tok::l_brace, TT_ArrayInitializerLSquare) ||
-         (Style.Language == FormatStyle::LK_JavaScript &&
-          Left.is(tok::l_paren))) &&
+         (Style.isJavaScript() && Left.is(tok::l_paren))) &&
         Left.isNot(BK_Block) && Left.MatchingParen)
       BeforeClosingBrace = Left.MatchingParen->Previous;
     else if (Right.MatchingParen &&
              (Right.MatchingParen->isOneOf(tok::l_brace,
                                            TT_ArrayInitializerLSquare) ||
-              (Style.Language == FormatStyle::LK_JavaScript &&
-               Right.MatchingParen->is(tok::l_paren))))
+              (Style.isJavaScript() && Right.MatchingParen->is(tok::l_paren))))
       BeforeClosingBrace = &Left;
     if (BeforeClosingBrace && (BeforeClosingBrace->is(tok::comma) ||
                                BeforeClosingBrace->isTrailingComment()))
@@ -3927,8 +3963,7 @@ bool TokenAnnotator::mustBreakBefore(const AnnotatedLine &Line,
   }
 
   // Put multiple Java annotation on a new line.
-  if ((Style.Language == FormatStyle::LK_Java ||
-       Style.Language == FormatStyle::LK_JavaScript) &&
+  if ((Style.Language == FormatStyle::LK_Java || Style.isJavaScript()) &&
       Left.is(TT_LeadingJavaAnnotation) &&
       Right.isNot(TT_LeadingJavaAnnotation) && Right.isNot(tok::l_paren) &&
       (Line.Last->is(tok::l_brace) || Style.BreakAfterJavaFieldAnnotations))
@@ -4071,7 +4106,7 @@ bool TokenAnnotator::canBreakBefore(const AnnotatedLine &Line,
     if (Right.isOneOf(Keywords.kw_throws, Keywords.kw_extends,
                       Keywords.kw_implements))
       return true;
-  } else if (Style.Language == FormatStyle::LK_JavaScript) {
+  } else if (Style.isJavaScript()) {
     const FormatToken *NonComment = Right.getPreviousNonComment();
     if (NonComment &&
         NonComment->isOneOf(
diff --git a/clang/lib/Format/UnwrappedLineFormatter.cpp b/clang/lib/Format/UnwrappedLineFormatter.cpp
index d099cfee9dea..f652a4e7088f 100644
--- a/clang/lib/Format/UnwrappedLineFormatter.cpp
+++ b/clang/lib/Format/UnwrappedLineFormatter.cpp
@@ -62,7 +62,7 @@ public:
       Indent = Line.Level * IndentWidth + AdditionalIndent;
     } else {
       IndentForLevel.resize(Line.Level + 1);
-      Indent = getIndent(IndentForLevel, Line.Level);
+      Indent = getIndent(Line.Level);
     }
     if (static_cast<int>(Indent) + Offset >= 0)
       Indent += Offset;
@@ -97,8 +97,8 @@ private:
   /// For example, 'public:' labels in classes are offset by 1 or 2
   /// characters to the left from their level.
   int getIndentOffset(const FormatToken &RootToken) {
-    if (Style.Language == FormatStyle::LK_Java ||
-        Style.Language == FormatStyle::LK_JavaScript || Style.isCSharp())
+    if (Style.Language == FormatStyle::LK_Java || Style.isJavaScript() ||
+        Style.isCSharp())
       return 0;
     if (RootToken.isAccessSpecifier(false) ||
         RootToken.isObjCAccessSpecifier() ||
@@ -118,12 +118,12 @@ private:
   /// \p IndentForLevel must contain the indent for the level \c l
   /// at \p IndentForLevel[l], or a value < 0 if the indent for
   /// that level is unknown.
-  unsigned getIndent(ArrayRef<int> IndentForLevel, unsigned Level) {
+  unsigned getIndent(unsigned Level) const {
     if (IndentForLevel[Level] != -1)
       return IndentForLevel[Level];
     if (Level == 0)
       return 0;
-    return getIndent(IndentForLevel, Level - 1) + Style.IndentWidth;
+    return getIndent(Level - 1) + Style.IndentWidth;
   }
 
   const FormatStyle &Style;
@@ -393,11 +393,24 @@ private:
 
     // Try to merge a block with left brace wrapped that wasn't yet covered
     if (TheLine->Last->is(tok::l_brace)) {
-      return !Style.BraceWrapping.AfterFunction ||
-                     (I[1]->First->is(tok::r_brace) &&
-                      !Style.BraceWrapping.SplitEmptyRecord)
-                 ? tryMergeSimpleBlock(I, E, Limit)
-                 : 0;
+      const FormatToken *Tok = TheLine->First;
+      bool ShouldMerge = false;
+      if (Tok->is(tok::kw_typedef)) {
+        Tok = Tok->getNextNonComment();
+        assert(Tok);
+      }
+      if (Tok->isOneOf(tok::kw_class, tok::kw_struct)) {
+        ShouldMerge = !Style.BraceWrapping.AfterClass ||
+                      (I[1]->First->is(tok::r_brace) &&
+                       !Style.BraceWrapping.SplitEmptyRecord);
+      } else if (Tok->is(tok::kw_enum)) {
+        ShouldMerge = Style.AllowShortEnumsOnASingleLine;
+      } else {
+        ShouldMerge = !Style.BraceWrapping.AfterFunction ||
+                      (I[1]->First->is(tok::r_brace) &&
+                       !Style.BraceWrapping.SplitEmptyFunction);
+      }
+      return ShouldMerge ? tryMergeSimpleBlock(I, E, Limit) : 0;
     }
     // Try to merge a function block with left brace wrapped
     if (I[1]->First->is(TT_FunctionLBrace) &&
@@ -584,6 +597,9 @@ private:
                             Keywords.kw___except)) {
       if (Style.AllowShortBlocksOnASingleLine == FormatStyle::SBS_Never)
         return 0;
+      if (Style.AllowShortBlocksOnASingleLine == FormatStyle::SBS_Empty &&
+          !I[1]->First->is(tok::r_brace))
+        return 0;
       // Don't merge when we can't except the case when
       // the control statement block is empty
       if (!Style.AllowShortIfStatementsOnASingleLine &&
@@ -1015,9 +1031,9 @@ private:
     QueueType Queue;
 
     // Insert start element into queue.
-    StateNode *Node =
+    StateNode *RootNode =
         new (Allocator.Allocate()) StateNode(InitialState, false, nullptr);
-    Queue.push(QueueItem(OrderedPenalty(0, Count), Node));
+    Queue.push(QueueItem(OrderedPenalty(0, Count), RootNode));
     ++Count;
 
     unsigned Penalty = 0;
@@ -1044,9 +1060,9 @@ private:
 
       FormatDecision LastFormat = Node->State.NextToken->getDecision();
       if (LastFormat == FD_Unformatted || LastFormat == FD_Continue)
-        addNextStateToQueue(Penalty, Node, /*NewLine=*/false, &Count, &Queue);
+        addNextStateToQueue(Penalty, Node, /*NewLine=*/false, Count, Queue);
       if (LastFormat == FD_Unformatted || LastFormat == FD_Break)
-        addNextStateToQueue(Penalty, Node, /*NewLine=*/true, &Count, &Queue);
+        addNextStateToQueue(Penalty, Node, /*NewLine=*/true, Count, Queue);
     }
 
     if (Queue.empty()) {
@@ -1072,7 +1088,7 @@ private:
   /// Assume the current state is \p PreviousNode and has been reached with a
   /// penalty of \p Penalty. Insert a line break if \p NewLine is \c true.
   void addNextStateToQueue(unsigned Penalty, StateNode *PreviousNode,
-                           bool NewLine, unsigned *Count, QueueType *Queue) {
+                           bool NewLine, unsigned &Count, QueueType &Queue) {
     if (NewLine && !Indenter->canBreak(PreviousNode->State))
       return;
     if (!NewLine && Indenter->mustBreak(PreviousNode->State))
@@ -1085,8 +1101,8 @@ private:
 
     Penalty += Indenter->addTokenToState(Node->State, NewLine, true);
 
-    Queue->push(QueueItem(OrderedPenalty(Penalty, *Count), Node));
-    ++(*Count);
+    Queue.push(QueueItem(OrderedPenalty(Penalty, Count), Node));
+    ++Count;
   }
 
   /// Applies the best formatting by reconstructing the path in the
@@ -1184,8 +1200,7 @@ unsigned UnwrappedLineFormatter::format(
       bool FitsIntoOneLine =
           TheLine.Last->TotalLength + Indent <= ColumnLimit ||
           (TheLine.Type == LT_ImportStatement &&
-           (Style.Language != FormatStyle::LK_JavaScript ||
-            !Style.JavaScriptWrapImports)) ||
+           (!Style.isJavaScript() || !Style.JavaScriptWrapImports)) ||
           (Style.isCSharp() &&
            TheLine.InPPDirective); // don't split #regions in C#
       if (Style.ColumnLimit == 0)
diff --git a/clang/lib/Format/UnwrappedLineParser.cpp b/clang/lib/Format/UnwrappedLineParser.cpp
index 5b9fe267aae6..b6e55aab708f 100644
--- a/clang/lib/Format/UnwrappedLineParser.cpp
+++ b/clang/lib/Format/UnwrappedLineParser.cpp
@@ -366,8 +366,7 @@ void UnwrappedLineParser::parse() {
 void UnwrappedLineParser::parseFile() {
   // The top-level context in a file always has declarations, except for pre-
   // processor directives and JavaScript files.
-  bool MustBeDeclaration =
-      !Line->InPPDirective && Style.Language != FormatStyle::LK_JavaScript;
+  bool MustBeDeclaration = !Line->InPPDirective && !Style.isJavaScript();
   ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
                                           MustBeDeclaration);
   if (Style.Language == FormatStyle::LK_TextProto)
@@ -478,8 +477,7 @@ void UnwrappedLineParser::parseLevel(bool HasOpeningBrace) {
       LLVM_FALLTHROUGH;
     }
     case tok::kw_case:
-      if (Style.Language == FormatStyle::LK_JavaScript &&
-          Line->MustBeDeclaration) {
+      if (Style.isJavaScript() && Line->MustBeDeclaration) {
         // A 'case: string' style field declaration.
         parseStructuralElement();
         break;
@@ -528,7 +526,7 @@ void UnwrappedLineParser::calculateBraceTypes(bool ExpectClassBody) {
 
     switch (Tok->Tok.getKind()) {
     case tok::l_brace:
-      if (Style.Language == FormatStyle::LK_JavaScript && PrevTok) {
+      if (Style.isJavaScript() && PrevTok) {
         if (PrevTok->isOneOf(tok::colon, tok::less))
           // A ':' indicates this code is in a type, or a braced list
           // following a label in an object literal ({a: {b: 1}}).
@@ -581,7 +579,7 @@ void UnwrappedLineParser::calculateBraceTypes(bool ExpectClassBody) {
           // FIXME: Some of these do not apply to JS, e.g. "} {" can never be a
           // braced list in JS.
           ProbablyBracedList =
-              (Style.Language == FormatStyle::LK_JavaScript &&
+              (Style.isJavaScript() &&
                NextTok->isOneOf(Keywords.kw_of, Keywords.kw_in,
                                 Keywords.kw_as)) ||
               (Style.isCpp() && NextTok->is(tok::l_paren)) ||
@@ -791,7 +789,7 @@ void UnwrappedLineParser::parseChildBlock() {
   FormatTok->setBlockKind(BK_Block);
   nextToken();
   {
-    bool SkipIndent = (Style.Language == FormatStyle::LK_JavaScript &&
+    bool SkipIndent = (Style.isJavaScript() &&
                        (isGoogScope(*Line) || isIIFE(*Line, Keywords)));
     ScopedLineState LineState(*this);
     ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
@@ -1140,7 +1138,6 @@ void UnwrappedLineParser::parseModuleImport() {
   }
 
   addUnwrappedLine();
-  return;
 }
 
 // readTokenWithJavaScriptASI reads the next token and terminates the current
@@ -1222,39 +1219,39 @@ void UnwrappedLineParser::parseStructuralElement(bool IsTopLevel) {
   case tok::kw_public:
   case tok::kw_protected:
   case tok::kw_private:
-    if (Style.Language == FormatStyle::LK_Java ||
-        Style.Language == FormatStyle::LK_JavaScript || Style.isCSharp())
+    if (Style.Language == FormatStyle::LK_Java || Style.isJavaScript() ||
+        Style.isCSharp())
       nextToken();
     else
       parseAccessSpecifier();
     return;
   case tok::kw_if:
-    if (Style.Language == FormatStyle::LK_JavaScript && Line->MustBeDeclaration)
+    if (Style.isJavaScript() && Line->MustBeDeclaration)
       // field/method declaration.
       break;
     parseIfThenElse();
     return;
   case tok::kw_for:
   case tok::kw_while:
-    if (Style.Language == FormatStyle::LK_JavaScript && Line->MustBeDeclaration)
+    if (Style.isJavaScript() && Line->MustBeDeclaration)
       // field/method declaration.
       break;
     parseForOrWhileLoop();
     return;
   case tok::kw_do:
-    if (Style.Language == FormatStyle::LK_JavaScript && Line->MustBeDeclaration)
+    if (Style.isJavaScript() && Line->MustBeDeclaration)
       // field/method declaration.
       break;
     parseDoWhile();
     return;
   case tok::kw_switch:
-    if (Style.Language == FormatStyle::LK_JavaScript && Line->MustBeDeclaration)
+    if (Style.isJavaScript() && Line->MustBeDeclaration)
       // 'switch: string' field declaration.
       break;
     parseSwitch();
     return;
   case tok::kw_default:
-    if (Style.Language == FormatStyle::LK_JavaScript && Line->MustBeDeclaration)
+    if (Style.isJavaScript() && Line->MustBeDeclaration)
       // 'default: string' field declaration.
       break;
     nextToken();
@@ -1265,14 +1262,14 @@ void UnwrappedLineParser::parseStructuralElement(bool IsTopLevel) {
     // e.g. "default void f() {}" in a Java interface.
     break;
   case tok::kw_case:
-    if (Style.Language == FormatStyle::LK_JavaScript && Line->MustBeDeclaration)
+    if (Style.isJavaScript() && Line->MustBeDeclaration)
       // 'case: string' field declaration.
       break;
     parseCaseLabel();
     return;
   case tok::kw_try:
   case tok::kw___try:
-    if (Style.Language == FormatStyle::LK_JavaScript && Line->MustBeDeclaration)
+    if (Style.isJavaScript() && Line->MustBeDeclaration)
       // field/method declaration.
       break;
     parseTryCatch();
@@ -1282,24 +1279,25 @@ void UnwrappedLineParser::parseStructuralElement(bool IsTopLevel) {
     if (FormatTok->Tok.is(tok::string_literal)) {
       nextToken();
       if (FormatTok->Tok.is(tok::l_brace)) {
-        if (!Style.IndentExternBlock) {
-          if (Style.BraceWrapping.AfterExternBlock) {
-            addUnwrappedLine();
-          }
-          unsigned AddLevels = Style.BraceWrapping.AfterExternBlock ? 1u : 0u;
-          parseBlock(/*MustBeDeclaration=*/true, AddLevels);
-        } else {
-          unsigned AddLevels =
-              Style.IndentExternBlock == FormatStyle::IEBS_Indent ? 1u : 0u;
-          parseBlock(/*MustBeDeclaration=*/true, AddLevels);
-        }
+        if (Style.BraceWrapping.AfterExternBlock)
+          addUnwrappedLine();
+        // Either we indent or for backwards compatibility we follow the
+        // AfterExternBlock style.
+        unsigned AddLevels =
+            (Style.IndentExternBlock == FormatStyle::IEBS_Indent) ||
+                    (Style.BraceWrapping.AfterExternBlock &&
+                     Style.IndentExternBlock ==
+                         FormatStyle::IEBS_AfterExternBlock)
+                ? 1u
+                : 0u;
+        parseBlock(/*MustBeDeclaration=*/true, AddLevels);
         addUnwrappedLine();
         return;
       }
     }
     break;
   case tok::kw_export:
-    if (Style.Language == FormatStyle::LK_JavaScript) {
+    if (Style.isJavaScript()) {
       parseJavaScriptEs6ImportExport();
       return;
     }
@@ -1325,7 +1323,7 @@ void UnwrappedLineParser::parseStructuralElement(bool IsTopLevel) {
       return;
     }
     if (FormatTok->is(Keywords.kw_import)) {
-      if (Style.Language == FormatStyle::LK_JavaScript) {
+      if (Style.isJavaScript()) {
         parseJavaScriptEs6ImportExport();
         return;
       }
@@ -1437,10 +1435,10 @@ void UnwrappedLineParser::parseStructuralElement(bool IsTopLevel) {
       break;
     case tok::kw_concept:
       parseConcept();
-      break;
+      return;
     case tok::kw_requires:
       parseRequires();
-      break;
+      return;
     case tok::kw_enum:
       // Ignore if this is part of "template <enum ...".
       if (Previous && Previous->is(tok::less)) {
@@ -1479,7 +1477,7 @@ void UnwrappedLineParser::parseStructuralElement(bool IsTopLevel) {
       if (Style.Language == FormatStyle::LK_Java && FormatTok &&
           FormatTok->is(tok::kw_class))
         nextToken();
-      if (Style.Language == FormatStyle::LK_JavaScript && FormatTok &&
+      if (Style.isJavaScript() && FormatTok &&
           FormatTok->Tok.getIdentifierInfo())
         // JavaScript only has pseudo keywords, all keywords are allowed to
         // appear in "IdentifierName" positions. See http://es5.github.io/#x7.6
@@ -1536,8 +1534,7 @@ void UnwrappedLineParser::parseStructuralElement(bool IsTopLevel) {
       // element continues.
       break;
     case tok::kw_try:
-      if (Style.Language == FormatStyle::LK_JavaScript &&
-          Line->MustBeDeclaration) {
+      if (Style.isJavaScript() && Line->MustBeDeclaration) {
         // field/method declaration.
         nextToken();
         break;
@@ -1564,17 +1561,15 @@ void UnwrappedLineParser::parseStructuralElement(bool IsTopLevel) {
       // expressions (functions that are not on their own line) must not create
       // a new unwrapped line, so they are special cased below.
       size_t TokenCount = Line->Tokens.size();
-      if (Style.Language == FormatStyle::LK_JavaScript &&
-          FormatTok->is(Keywords.kw_function) &&
+      if (Style.isJavaScript() && FormatTok->is(Keywords.kw_function) &&
           (TokenCount > 1 || (TokenCount == 1 && !Line->Tokens.front().Tok->is(
                                                      Keywords.kw_async)))) {
         tryToParseJSFunction();
         break;
       }
-      if ((Style.Language == FormatStyle::LK_JavaScript ||
-           Style.Language == FormatStyle::LK_Java) &&
+      if ((Style.isJavaScript() || Style.Language == FormatStyle::LK_Java) &&
           FormatTok->is(Keywords.kw_interface)) {
-        if (Style.Language == FormatStyle::LK_JavaScript) {
+        if (Style.isJavaScript()) {
           // In JavaScript/TypeScript, "interface" can be used as a standalone
           // identifier, e.g. in `var interface = 1;`. If "interface" is
           // followed by another identifier, it is very like to be an actual
@@ -1610,7 +1605,7 @@ void UnwrappedLineParser::parseStructuralElement(bool IsTopLevel) {
 
       // JS doesn't have macros, and within classes colons indicate fields, not
       // labels.
-      if (Style.Language == FormatStyle::LK_JavaScript)
+      if (Style.isJavaScript())
         break;
 
       TokenCount = Line->Tokens.size();
@@ -1641,19 +1636,9 @@ void UnwrappedLineParser::parseStructuralElement(bool IsTopLevel) {
       break;
     }
     case tok::equal:
-      // Fat arrows (=>) have tok::TokenKind tok::equal but TokenType
-      // TT_FatArrow. They always start an expression or a child block if
-      // followed by a curly brace.
-      if (FormatTok->is(TT_FatArrow)) {
-        nextToken();
-        if (FormatTok->is(tok::l_brace)) {
-          // C# may break after => if the next character is a newline.
-          if (Style.isCSharp() && Style.BraceWrapping.AfterFunction == true) {
-            // calling `addUnwrappedLine()` here causes odd parsing errors.
-            FormatTok->MustBreakBefore = true;
-          }
-          parseChildBlock();
-        }
+      if ((Style.isJavaScript() || Style.isCSharp()) &&
+          FormatTok->is(TT_FatArrow)) {
+        tryToParseChildBlock();
         break;
       }
 
@@ -1729,7 +1714,7 @@ bool UnwrappedLineParser::tryToParsePropertyAccessor() {
   // Try to parse the property accessor:
   // https://docs.microsoft.com/en-us/dotnet/csharp/programming-guide/classes-and-structs/properties
   Tokens->setPosition(StoredPosition);
-  if (!IsTrivialPropertyAccessor && Style.BraceWrapping.AfterFunction == true)
+  if (!IsTrivialPropertyAccessor && Style.BraceWrapping.AfterFunction)
     addUnwrappedLine();
   nextToken();
   do {
@@ -1944,6 +1929,19 @@ bool UnwrappedLineParser::tryToParseBracedList() {
   return true;
 }
 
+bool UnwrappedLineParser::tryToParseChildBlock() {
+  assert(Style.isJavaScript() || Style.isCSharp());
+  assert(FormatTok->is(TT_FatArrow));
+  // Fat arrows (=>) have tok::TokenKind tok::equal but TokenType TT_FatArrow.
+  // They always start an expression or a child block if followed by a curly
+  // brace.
+  nextToken();
+  if (FormatTok->isNot(tok::l_brace))
+    return false;
+  parseChildBlock();
+  return true;
+}
+
 bool UnwrappedLineParser::parseBracedList(bool ContinueOnSemicolons,
                                           bool IsEnum,
                                           tok::TokenKind ClosingBraceKind) {
@@ -1952,38 +1950,15 @@ bool UnwrappedLineParser::parseBracedList(bool ContinueOnSemicolons,
   // FIXME: Once we have an expression parser in the UnwrappedLineParser,
   // replace this by using parseAssignmentExpression() inside.
   do {
-    if (Style.isCSharp()) {
-      // Fat arrows (=>) have tok::TokenKind tok::equal but TokenType
-      // TT_FatArrow. They always start an expression or a child block if
-      // followed by a curly brace.
-      if (FormatTok->is(TT_FatArrow)) {
-        nextToken();
-        if (FormatTok->is(tok::l_brace)) {
-          // C# may break after => if the next character is a newline.
-          if (Style.isCSharp() && Style.BraceWrapping.AfterFunction == true) {
-            // calling `addUnwrappedLine()` here causes odd parsing errors.
-            FormatTok->MustBreakBefore = true;
-          }
-          parseChildBlock();
-          continue;
-        }
-      }
-    }
-    if (Style.Language == FormatStyle::LK_JavaScript) {
+    if (Style.isCSharp() && FormatTok->is(TT_FatArrow) &&
+        tryToParseChildBlock())
+      continue;
+    if (Style.isJavaScript()) {
       if (FormatTok->is(Keywords.kw_function) ||
           FormatTok->startsSequence(Keywords.kw_async, Keywords.kw_function)) {
         tryToParseJSFunction();
         continue;
       }
-      if (FormatTok->is(TT_FatArrow)) {
-        nextToken();
-        // Fat arrows can be followed by simple expressions or by child blocks
-        // in curly braces.
-        if (FormatTok->is(tok::l_brace)) {
-          parseChildBlock();
-          continue;
-        }
-      }
       if (FormatTok->is(tok::l_brace)) {
         // Could be a method inside of a braced list `{a() { return 1; }}`.
         if (tryToParseBracedList())
@@ -1998,12 +1973,6 @@ bool UnwrappedLineParser::parseBracedList(bool ContinueOnSemicolons,
       return !HasError;
     }
     switch (FormatTok->Tok.getKind()) {
-    case tok::caret:
-      nextToken();
-      if (FormatTok->is(tok::l_brace)) {
-        parseChildBlock();
-      }
-      break;
     case tok::l_square:
       if (Style.isCSharp())
         parseSquare();
@@ -2014,7 +1983,7 @@ bool UnwrappedLineParser::parseBracedList(bool ContinueOnSemicolons,
       parseParens();
       // JavaScript can just have free standing methods and getters/setters in
       // object literals. Detect them by a "{" following ")".
-      if (Style.Language == FormatStyle::LK_JavaScript) {
+      if (Style.isJavaScript()) {
         if (FormatTok->is(tok::l_brace))
           parseChildBlock();
         break;
@@ -2041,7 +2010,7 @@ bool UnwrappedLineParser::parseBracedList(bool ContinueOnSemicolons,
       // lists (in so-called TypeMemberLists). Thus, the semicolon cannot be
       // used for error recovery if we have otherwise determined that this is
       // a braced list.
-      if (Style.Language == FormatStyle::LK_JavaScript) {
+      if (Style.isJavaScript()) {
         nextToken();
         break;
       }
@@ -2095,18 +2064,18 @@ void UnwrappedLineParser::parseParens() {
       break;
     case tok::equal:
       if (Style.isCSharp() && FormatTok->is(TT_FatArrow))
-        parseStructuralElement();
+        tryToParseChildBlock();
       else
         nextToken();
       break;
     case tok::kw_class:
-      if (Style.Language == FormatStyle::LK_JavaScript)
+      if (Style.isJavaScript())
         parseRecord(/*ParseAsExpr=*/true);
       else
         nextToken();
       break;
     case tok::identifier:
-      if (Style.Language == FormatStyle::LK_JavaScript &&
+      if (Style.isJavaScript() &&
           (FormatTok->is(Keywords.kw_function) ||
            FormatTok->startsSequence(Keywords.kw_async, Keywords.kw_function)))
         tryToParseJSFunction();
@@ -2160,15 +2129,22 @@ void UnwrappedLineParser::parseSquare(bool LambdaIntroducer) {
 }
 
 void UnwrappedLineParser::parseIfThenElse() {
+  auto HandleAttributes = [this]() {
+    // Handle AttributeMacro, e.g. `if (x) UNLIKELY`.
+    if (FormatTok->is(TT_AttributeMacro))
+      nextToken();
+    // Handle [[likely]] / [[unlikely]] attributes.
+    if (FormatTok->is(tok::l_square) && tryToParseSimpleAttribute())
+      parseSquare();
+  };
+
   assert(FormatTok->Tok.is(tok::kw_if) && "'if' expected");
   nextToken();
   if (FormatTok->Tok.isOneOf(tok::kw_constexpr, tok::identifier))
     nextToken();
   if (FormatTok->Tok.is(tok::l_paren))
     parseParens();
-  // handle [[likely]] / [[unlikely]]
-  if (FormatTok->is(tok::l_square) && tryToParseSimpleAttribute())
-    parseSquare();
+  HandleAttributes();
   bool NeedsUnwrappedLine = false;
   if (FormatTok->Tok.is(tok::l_brace)) {
     CompoundStatementIndenter Indenter(this, Style, Line->Level);
@@ -2185,9 +2161,7 @@ void UnwrappedLineParser::parseIfThenElse() {
   }
   if (FormatTok->Tok.is(tok::kw_else)) {
     nextToken();
-    // handle [[likely]] / [[unlikely]]
-    if (FormatTok->Tok.is(tok::l_square) && tryToParseSimpleAttribute())
-      parseSquare();
+    HandleAttributes();
     if (FormatTok->Tok.is(tok::l_brace)) {
       CompoundStatementIndenter Indenter(this, Style, Line->Level);
       parseBlock();
@@ -2272,8 +2246,7 @@ void UnwrappedLineParser::parseTryCatch() {
       nextToken();
     if (!(FormatTok->isOneOf(tok::kw_catch, Keywords.kw___except,
                              tok::kw___finally) ||
-          ((Style.Language == FormatStyle::LK_Java ||
-            Style.Language == FormatStyle::LK_JavaScript) &&
+          ((Style.Language == FormatStyle::LK_Java || Style.isJavaScript()) &&
            FormatTok->is(Keywords.kw_finally)) ||
           (FormatTok->Tok.isObjCAtKeyword(tok::objc_catch) ||
            FormatTok->Tok.isObjCAtKeyword(tok::objc_finally))))
@@ -2396,8 +2369,7 @@ void UnwrappedLineParser::parseForOrWhileLoop() {
          "'for', 'while' or foreach macro expected");
   nextToken();
   // JS' for await ( ...
-  if (Style.Language == FormatStyle::LK_JavaScript &&
-      FormatTok->is(Keywords.kw_await))
+  if (Style.isJavaScript() && FormatTok->is(Keywords.kw_await))
     nextToken();
   if (Style.isCpp() && FormatTok->is(tok::kw_co_await))
     nextToken();
@@ -2643,8 +2615,7 @@ bool UnwrappedLineParser::parseEnum() {
   // In TypeScript, "enum" can also be used as property name, e.g. in interface
   // declarations. An "enum" keyword followed by a colon would be a syntax
   // error and thus assume it is just an identifier.
-  if (Style.Language == FormatStyle::LK_JavaScript &&
-      FormatTok->isOneOf(tok::colon, tok::question))
+  if (Style.isJavaScript() && FormatTok->isOneOf(tok::colon, tok::question))
     return false;
 
   // In protobuf, "enum" can be used as a field name.
@@ -2716,8 +2687,8 @@ bool UnwrappedLineParser::parseStructLike() {
   // record declaration or definition can start a structural element.
   parseRecord();
   // This does not apply to Java, JavaScript and C#.
-  if (Style.Language == FormatStyle::LK_Java ||
-      Style.Language == FormatStyle::LK_JavaScript || Style.isCSharp()) {
+  if (Style.Language == FormatStyle::LK_Java || Style.isJavaScript() ||
+      Style.isCSharp()) {
     if (FormatTok->is(tok::semi))
       nextToken();
     addUnwrappedLine();
@@ -2846,10 +2817,9 @@ void UnwrappedLineParser::parseRecord(bool ParseAsExpr) {
   while (FormatTok->isOneOf(tok::identifier, tok::coloncolon, tok::hashhash,
                             tok::kw___attribute, tok::kw___declspec,
                             tok::kw_alignas, tok::l_square, tok::r_square) ||
-         ((Style.Language == FormatStyle::LK_Java ||
-           Style.Language == FormatStyle::LK_JavaScript) &&
+         ((Style.Language == FormatStyle::LK_Java || Style.isJavaScript()) &&
           FormatTok->isOneOf(tok::period, tok::comma))) {
-    if (Style.Language == FormatStyle::LK_JavaScript &&
+    if (Style.isJavaScript() &&
         FormatTok->isOneOf(Keywords.kw_extends, Keywords.kw_implements)) {
       // JavaScript/TypeScript supports inline object types in
       // extends/implements positions:
@@ -3323,7 +3293,7 @@ void UnwrappedLineParser::nextToken(int LevelDifference) {
   flushComments(isOnNewLine(*FormatTok));
   pushToken(FormatTok);
   FormatToken *Previous = FormatTok;
-  if (Style.Language != FormatStyle::LK_JavaScript)
+  if (!Style.isJavaScript())
     readToken(LevelDifference);
   else
     readTokenWithJavaScriptASI();
diff --git a/clang/lib/Format/UnwrappedLineParser.h b/clang/lib/Format/UnwrappedLineParser.h
index b4c082654597..0c79723d50fc 100644
--- a/clang/lib/Format/UnwrappedLineParser.h
+++ b/clang/lib/Format/UnwrappedLineParser.h
@@ -138,6 +138,7 @@ private:
   // https://docs.microsoft.com/en-us/dotnet/csharp/language-reference/keywords/where-generic-type-constraint
   void parseCSharpGenericTypeConstraint();
   bool tryToParseLambda();
+  bool tryToParseChildBlock();
   bool tryToParseLambdaIntroducer();
   bool tryToParsePropertyAccessor();
   void tryToParseJSFunction();
diff --git a/clang/lib/Format/WhitespaceManager.cpp b/clang/lib/Format/WhitespaceManager.cpp
index fae8a1c3fdc6..96a66da0f82b 100644
--- a/clang/lib/Format/WhitespaceManager.cpp
+++ b/clang/lib/Format/WhitespaceManager.cpp
@@ -1282,9 +1282,12 @@ void WhitespaceManager::generateChanges() {
                                  C.EscapedNewlineColumn);
       else
         appendNewlineText(ReplacementText, C.NewlinesBefore);
+      // FIXME: This assert should hold if we computed the column correctly.
+      // assert((int)C.StartOfTokenColumn >= C.Spaces);
       appendIndentText(
           ReplacementText, C.Tok->IndentLevel, std::max(0, C.Spaces),
-          C.StartOfTokenColumn - std::max(0, C.Spaces), C.IsAligned);
+          std::max((int)C.StartOfTokenColumn, C.Spaces) - std::max(0, C.Spaces),
+          C.IsAligned);
       ReplacementText.append(C.CurrentLinePrefix);
       storeReplacement(C.OriginalWhitespaceRange, ReplacementText);
     }
diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp
index 1432607204bd..31e7ea3d243d 100644
--- a/clang/lib/Frontend/CompilerInstance.cpp
+++ b/clang/lib/Frontend/CompilerInstance.cpp
@@ -1154,12 +1154,12 @@ compileModuleImpl(CompilerInstance &ImportingInstance, SourceLocation ImportLoc,
   // Remove any macro definitions that are explicitly ignored by the module.
   // They aren't supposed to affect how the module is built anyway.
   HeaderSearchOptions &HSOpts = Invocation->getHeaderSearchOpts();
-  llvm::erase_if(
-      PPOpts.Macros, [&HSOpts](const std::pair<std::string, bool> &def) {
-        StringRef MacroDef = def.first;
-        return HSOpts.ModulesIgnoreMacros.count(
-                   llvm::CachedHashString(MacroDef.split('=').first)) > 0;
-      });
+  llvm::erase_if(PPOpts.Macros,
+                 [&HSOpts](const std::pair<std::string, bool> &def) {
+                   StringRef MacroDef = def.first;
+                   return HSOpts.ModulesIgnoreMacros.contains(
+                       llvm::CachedHashString(MacroDef.split('=').first));
+                 });
 
   // If the original compiler invocation had -fmodule-name, pass it through.
   Invocation->getLangOpts()->ModuleName =
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index c104a6f40e20..b71addd84bfd 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -770,9 +770,7 @@ static void parseAnalyzerConfigs(AnalyzerOptions &AnOpts,
 static void getAllNoBuiltinFuncValues(ArgList &Args,
                                       std::vector<std::string> &Funcs) {
   std::vector<std::string> Values = Args.getAllArgValues(OPT_fno_builtin_);
-  auto BuiltinEnd = llvm::partition(Values, [](const std::string FuncName) {
-    return Builtin::Context::isBuiltinFunc(FuncName);
-  });
+  auto BuiltinEnd = llvm::partition(Values, Builtin::Context::isBuiltinFunc);
   Funcs.insert(Funcs.end(), Values.begin(), BuiltinEnd);
 }
 
@@ -1285,7 +1283,7 @@ static std::string serializeXRayInstrumentationBundle(const XRayInstrSet &S) {
   std::string Buffer;
   llvm::raw_string_ostream OS(Buffer);
   llvm::interleave(BundleParts, OS, [&OS](StringRef Part) { OS << Part; }, ",");
-  return OS.str();
+  return Buffer;
 }
 
 // Set the profile kind using fprofile-instrument-use-path.
@@ -4123,6 +4121,13 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
         {std::string(Split.first), std::string(Split.second)});
   }
 
+  // Error if -mvscale-min is unbounded.
+  if (Arg *A = Args.getLastArg(options::OPT_mvscale_min_EQ)) {
+    unsigned VScaleMin;
+    if (StringRef(A->getValue()).getAsInteger(10, VScaleMin) || VScaleMin == 0)
+      Diags.Report(diag::err_cc1_unbounded_vscale_min);
+  }
+
   return Diags.getNumErrors() == NumErrorsBefore;
 }
 
@@ -4513,7 +4518,7 @@ bool CompilerInvocation::CreateFromArgsImpl(
 
   // Store the command-line for using in the CodeView backend.
   Res.getCodeGenOpts().Argv0 = Argv0;
-  Res.getCodeGenOpts().CommandLineArgs = CommandLineArgs;
+  append_range(Res.getCodeGenOpts().CommandLineArgs, CommandLineArgs);
 
   FixupInvocation(Res, Diags, Args, DashX);
 
diff --git a/clang/lib/Frontend/InitPreprocessor.cpp b/clang/lib/Frontend/InitPreprocessor.cpp
index 0c153446142e..629f99110661 100644
--- a/clang/lib/Frontend/InitPreprocessor.cpp
+++ b/clang/lib/Frontend/InitPreprocessor.cpp
@@ -500,8 +500,12 @@ static void InitializeStandardPredefinedMacros(const TargetInfo &TI,
   // Not "standard" per se, but available even with the -undef flag.
   if (LangOpts.AsmPreprocessor)
     Builder.defineMacro("__ASSEMBLER__");
-  if (LangOpts.CUDA && !LangOpts.HIP)
-    Builder.defineMacro("__CUDA__");
+  if (LangOpts.CUDA) {
+    if (LangOpts.GPURelocatableDeviceCode)
+      Builder.defineMacro("__CLANG_RDC__");
+    if (!LangOpts.HIP)
+      Builder.defineMacro("__CUDA__");
+  }
   if (LangOpts.HIP) {
     Builder.defineMacro("__HIP__");
     Builder.defineMacro("__HIPCC__");
diff --git a/clang/lib/Frontend/TestModuleFileExtension.cpp b/clang/lib/Frontend/TestModuleFileExtension.cpp
index ea737e6891bf..2d5145d0c54c 100644
--- a/clang/lib/Frontend/TestModuleFileExtension.cpp
+++ b/clang/lib/Frontend/TestModuleFileExtension.cpp
@@ -133,5 +133,5 @@ std::string TestModuleFileExtension::str() const {
   llvm::raw_string_ostream OS(Buffer);
   OS << BlockName << ":" << MajorVersion << ":" << MinorVersion << ":" << Hashed
      << ":" << UserInfo;
-  return OS.str();
+  return Buffer;
 }
diff --git a/clang/lib/Headers/arm_neon_sve_bridge.h b/clang/lib/Headers/arm_neon_sve_bridge.h
new file mode 100644
index 000000000000..17699d8d11dd
--- /dev/null
+++ b/clang/lib/Headers/arm_neon_sve_bridge.h
@@ -0,0 +1,184 @@
+/*===---- arm_neon_sve_bridge.h - ARM NEON SVE Bridge intrinsics -----------===
+ *
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===-----------------------------------------------------------------------===
+ */
+
+#ifndef __ARM_NEON_SVE_BRIDGE_H
+#define __ARM_NEON_SVE_BRIDGE_H
+
+#include <arm_neon.h>
+#include <arm_sve.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Function attributes */
+#define __ai static __inline__ __attribute__((__always_inline__, __nodebug__))
+#define __aio                                                                  \
+  static __inline__                                                            \
+      __attribute__((__always_inline__, __nodebug__, __overloadable__))
+
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_s8)))
+svint8_t svset_neonq(svint8_t, int8x16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_s16)))
+svint16_t svset_neonq(svint16_t, int16x8_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_s32)))
+svint32_t svset_neonq(svint32_t, int32x4_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_s64)))
+svint64_t svset_neonq(svint64_t, int64x2_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_u8)))
+svuint8_t svset_neonq(svuint8_t, uint8x16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_u16)))
+svuint16_t svset_neonq(svuint16_t, uint16x8_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_u32)))
+svuint32_t svset_neonq(svuint32_t, uint32x4_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_u64)))
+svuint64_t svset_neonq(svuint64_t, uint64x2_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_f16)))
+svfloat16_t svset_neonq(svfloat16_t, float16x8_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_f32)))
+svfloat32_t svset_neonq(svfloat32_t, float32x4_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_f64)))
+svfloat64_t svset_neonq(svfloat64_t, float64x2_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_s8)))
+svint8_t svset_neonq_s8(svint8_t, int8x16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_s16)))
+svint16_t svset_neonq_s16(svint16_t, int16x8_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_s32)))
+svint32_t svset_neonq_s32(svint32_t, int32x4_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_s64)))
+svint64_t svset_neonq_s64(svint64_t, int64x2_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_u8)))
+svuint8_t svset_neonq_u8(svuint8_t, uint8x16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_u16)))
+svuint16_t svset_neonq_u16(svuint16_t, uint16x8_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_u32)))
+svuint32_t svset_neonq_u32(svuint32_t, uint32x4_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_u64)))
+svuint64_t svset_neonq_u64(svuint64_t, uint64x2_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_f16)))
+svfloat16_t svset_neonq_f16(svfloat16_t, float16x8_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_f32)))
+svfloat32_t svset_neonq_f32(svfloat32_t, float32x4_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_f64)))
+svfloat64_t svset_neonq_f64(svfloat64_t, float64x2_t);
+
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_s8)))
+int8x16_t svget_neonq(svint8_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_s16)))
+int16x8_t svget_neonq(svint16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_s32)))
+int32x4_t svget_neonq(svint32_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_s64)))
+int64x2_t svget_neonq(svint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_u8)))
+uint8x16_t svget_neonq(svuint8_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_u16)))
+uint16x8_t svget_neonq(svuint16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_u32)))
+uint32x4_t svget_neonq(svuint32_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_u64)))
+uint64x2_t svget_neonq(svuint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_f16)))
+float16x8_t svget_neonq(svfloat16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_f32)))
+float32x4_t svget_neonq(svfloat32_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_f64)))
+float64x2_t svget_neonq(svfloat64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_s8)))
+int8x16_t svget_neonq_s8(svint8_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_s16)))
+int16x8_t svget_neonq_s16(svint16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_s32)))
+int32x4_t svget_neonq_s32(svint32_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_s64)))
+int64x2_t svget_neonq_s64(svint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_u8)))
+uint8x16_t svget_neonq_u8(svuint8_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_u16)))
+uint16x8_t svget_neonq_u16(svuint16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_u32)))
+uint32x4_t svget_neonq_u32(svuint32_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_u64)))
+uint64x2_t svget_neonq_u64(svuint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_f16)))
+float16x8_t svget_neonq_f16(svfloat16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_f32)))
+float32x4_t svget_neonq_f32(svfloat32_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_f64)))
+float64x2_t svget_neonq_f64(svfloat64_t);
+
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_s8)))
+svint8_t svdup_neonq(int8x16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_s16)))
+svint16_t svdup_neonq(int16x8_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_s32)))
+svint32_t svdup_neonq(int32x4_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_s64)))
+svint64_t svdup_neonq(int64x2_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_u8)))
+svuint8_t svdup_neonq(uint8x16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_u16)))
+svuint16_t svdup_neonq(uint16x8_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_u32)))
+svuint32_t svdup_neonq(uint32x4_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_u64)))
+svuint64_t svdup_neonq(uint64x2_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_f16)))
+svfloat16_t svdup_neonq(float16x8_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_f32)))
+svfloat32_t svdup_neonq(float32x4_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_f64)))
+svfloat64_t svdup_neonq(float64x2_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_s8)))
+svint8_t svdup_neonq_s8(int8x16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_s16)))
+svint16_t svdup_neonq_s16(int16x8_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_s32)))
+svint32_t svdup_neonq_s32(int32x4_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_s64)))
+svint64_t svdup_neonq_s64(int64x2_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_u8)))
+svuint8_t svdup_neonq_u8(uint8x16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_u16)))
+svuint16_t svdup_neonq_u16(uint16x8_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_u32)))
+svuint32_t svdup_neonq_u32(uint32x4_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_u64)))
+svuint64_t svdup_neonq_u64(uint64x2_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_f16)))
+svfloat16_t svdup_neonq_f16(float16x8_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_f32)))
+svfloat32_t svdup_neonq_f32(float32x4_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_f64)))
+svfloat64_t svdup_neonq_f64(float64x2_t);
+
+#if defined(__ARM_FEATURE_SVE_BF16)
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_bf16)))
+svbfloat16_t svset_neonq(svbfloat16_t, bfloat16x8_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svset_neonq_bf16)))
+svbfloat16_t svset_neonq_bf16(svbfloat16_t, bfloat16x8_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_bf16)))
+bfloat16x8_t svget_neonq(svbfloat16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svget_neonq_bf16)))
+bfloat16x8_t svget_neonq_bf16(svbfloat16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_bf16)))
+svbfloat16_t svdup_neonq(bfloat16x8_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_neonq_bf16)))
+svbfloat16_t svdup_neonq_bf16(bfloat16x8_t);
+#endif // defined(__ARM_FEATURE_SVE_BF16)
+
+#undef __ai
+#undef __aio
+
+#ifdef __cplusplus
+} // extern "C"
+#endif
+
+#endif //__ARM_NEON_SVE_BRIDGE_H
diff --git a/clang/lib/Headers/hexagon_protos.h b/clang/lib/Headers/hexagon_protos.h
index cdffd93bb859..2642f3c8428d 100644
--- a/clang/lib/Headers/hexagon_protos.h
+++ b/clang/lib/Headers/hexagon_protos.h
@@ -8003,17 +8003,6 @@
 #define Q6_P_vtrunohb_PP __builtin_HEXAGON_S6_vtrunohb_ppp
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
-#if __HEXAGON_ARCH__ >= 62
-/* ==========================================================================
-   Assembly Syntax:       Vd32=vmem(Rt32):nt
-   C Intrinsic Prototype: HVX_Vector Q6_V_vmem_R_nt(Word32 Rt)
-   Instruction Type:      MAPPING
-   Execution Slots:       SLOT0123
-   ========================================================================== */
-
-#define Q6_V_vmem_R_nt __builtin_HEXAGON_V6_ldntnt0
-#endif /* __HEXAGON_ARCH___ >= 62 */
-
 #if __HEXAGON_ARCH__ >= 65
 /* ==========================================================================
    Assembly Syntax:       Pd4=!any8(vcmpb.eq(Rss32,Rtt32))
diff --git a/clang/lib/Headers/hexagon_types.h b/clang/lib/Headers/hexagon_types.h
index 6958809418d8..029727cc4817 100644
--- a/clang/lib/Headers/hexagon_types.h
+++ b/clang/lib/Headers/hexagon_types.h
@@ -1177,37 +1177,6 @@ private:
 
 #endif /* __cplusplus */
 
-// V65 Silver types
-#if __Q6S_ARCH__ >= 65
-  // Silver vector types are 128 bytes, and pairs are 256. The vector predicate
-  // types are 16 bytes and 32 bytes for pairs.
-  typedef long HEXAGON_VecPred128 __attribute__((__vector_size__(16)))
-    __attribute__((aligned(128)));
-
-  typedef long HEXAGON_VecPred256 __attribute__((__vector_size__(32)))
-    __attribute__((aligned(128)));
-
-  typedef long HEXAGON_Vect1024 __attribute__((__vector_size__(128)))
-    __attribute__((aligned(128)));
-
-  typedef long HEXAGON_Vect2048 __attribute__((__vector_size__(256)))
-    __attribute__((aligned(256)));
-
-  typedef long HEXAGON_UVect1024 __attribute__((__vector_size__(128)))
-    __attribute__((aligned(4)));
-
-  typedef long HEXAGON_UVect2048 __attribute__((__vector_size__(256)))
-    __attribute__((aligned(4)));
-
-  #define Q6S_VectorPredPair HEXAGON_VecPred256
-  #define Q6S_VectorPred     HEXAGON_VecPred128
-  #define Q6S_Vector         HEXAGON_Vect1024
-  #define Q6S_VectorPair     HEXAGON_Vect2048
-  #define Q6S_UVector        HEXAGON_UVect1024
-  #define Q6S_UVectorPair    HEXAGON_UVect2048
-
-#else /* __Q6S_ARCH__ >= 65 */
-
 // V65 Vector types
 #if __HVX_ARCH__ >= 65
 #if defined __HVX__ && (__HVX_LENGTH__ == 128)
@@ -1256,7 +1225,6 @@ private:
 #endif /* defined __HVX__ &&  (__HVX_LENGTH__ == 64) */
 #endif /* defined __HVX__ && (__HVX_LENGTH__ == 128) */
 #endif /* __HVX_ARCH__ >= 65 */
-#endif /* __Q6S_ARCH__ >= 65 */
 
 /* Predicates */
 
diff --git a/clang/lib/Headers/hvx_hexagon_protos.h b/clang/lib/Headers/hvx_hexagon_protos.h
index 41ce7a6b93e9..7e3679a38b2c 100644
--- a/clang/lib/Headers/hvx_hexagon_protos.h
+++ b/clang/lib/Headers/hvx_hexagon_protos.h
@@ -9,7 +9,6 @@
 //===----------------------------------------------------------------------===//
 
 
-
 #ifndef _HVX_HEXAGON_PROTOS_H_
 #define _HVX_HEXAGON_PROTOS_H_ 1
 
@@ -28,7 +27,7 @@
    Execution Slots:       SLOT0
    ========================================================================== */
 
-#define Q6_R_vextract_VR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_extractw)
+#define Q6_R_vextract_VR(Vu,Rs) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_extractw)(Vu,Rs)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -39,7 +38,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_V_hi_W __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_hi)
+#define Q6_V_hi_W(Vss) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_hi)(Vss)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -50,7 +49,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_V_lo_W __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_lo)
+#define Q6_V_lo_W(Vss) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_lo)(Vss)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -61,7 +60,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_V_vsplat_R __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_lvsplatw)
+#define Q6_V_vsplat_R(Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_lvsplatw)(Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -72,7 +71,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_and_QQ __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_and)
+#define Q6_Q_and_QQ(Qs,Qt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_and)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qt),-1))),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -83,7 +82,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_and_QQn __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_and_n)
+#define Q6_Q_and_QQn(Qs,Qt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_and_n)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qt),-1))),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -94,7 +93,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_not_Q __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_not)
+#define Q6_Q_not_Q(Qs) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_not)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1))),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -105,7 +104,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_or_QQ __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_or)
+#define Q6_Q_or_QQ(Qs,Qt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_or)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qt),-1))),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -116,7 +115,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_or_QQn __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_or_n)
+#define Q6_Q_or_QQn(Qs,Qt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_or_n)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qt),-1))),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -127,7 +126,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_vsetq_R __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_scalar2)
+#define Q6_Q_vsetq_R(Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_scalar2)(Rt)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -138,7 +137,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_xor_QQ __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_xor)
+#define Q6_Q_xor_QQ(Qs,Qt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_xor)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qt),-1))),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -149,7 +148,7 @@
    Execution Slots:       SLOT0
    ========================================================================== */
 
-#define Q6_vmem_QnRIV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vS32b_nqpred_ai)
+#define Q6_vmem_QnRIV(Qv,Rt,Vs) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vS32b_nqpred_ai)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Rt,Vs)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -160,7 +159,7 @@
    Execution Slots:       SLOT0
    ========================================================================== */
 
-#define Q6_vmem_QnRIV_nt __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vS32b_nt_nqpred_ai)
+#define Q6_vmem_QnRIV_nt(Qv,Rt,Vs) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vS32b_nt_nqpred_ai)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Rt,Vs)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -171,7 +170,7 @@
    Execution Slots:       SLOT0
    ========================================================================== */
 
-#define Q6_vmem_QRIV_nt __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vS32b_nt_qpred_ai)
+#define Q6_vmem_QRIV_nt(Qv,Rt,Vs) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vS32b_nt_qpred_ai)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Rt,Vs)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -182,7 +181,7 @@
    Execution Slots:       SLOT0
    ========================================================================== */
 
-#define Q6_vmem_QRIV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vS32b_qpred_ai)
+#define Q6_vmem_QRIV(Qv,Rt,Vs) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vS32b_qpred_ai)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Rt,Vs)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -193,7 +192,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vuh_vabsdiff_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsdiffh)
+#define Q6_Vuh_vabsdiff_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsdiffh)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -204,7 +203,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vub_vabsdiff_VubVub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsdiffub)
+#define Q6_Vub_vabsdiff_VubVub(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsdiffub)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -215,7 +214,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vuh_vabsdiff_VuhVuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsdiffuh)
+#define Q6_Vuh_vabsdiff_VuhVuh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsdiffuh)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -226,7 +225,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vuw_vabsdiff_VwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsdiffw)
+#define Q6_Vuw_vabsdiff_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsdiffw)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -237,7 +236,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vh_vabs_Vh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsh)
+#define Q6_Vh_vabs_Vh(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsh)(Vu)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -248,7 +247,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vh_vabs_Vh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsh_sat)
+#define Q6_Vh_vabs_Vh_sat(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsh_sat)(Vu)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -259,7 +258,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vw_vabs_Vw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsw)
+#define Q6_Vw_vabs_Vw(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsw)(Vu)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -270,7 +269,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vw_vabs_Vw_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsw_sat)
+#define Q6_Vw_vabs_Vw_sat(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsw_sat)(Vu)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -281,7 +280,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vb_vadd_VbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddb)
+#define Q6_Vb_vadd_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddb)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -292,7 +291,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Wb_vadd_WbWb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddb_dv)
+#define Q6_Wb_vadd_WbWb(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddb_dv)(Vuu,Vvv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -303,7 +302,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vb_condacc_QnVbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddbnq)
+#define Q6_Vb_condacc_QnVbVb(Qv,Vx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddbnq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vx,Vu)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -314,7 +313,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vb_condacc_QVbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddbq)
+#define Q6_Vb_condacc_QVbVb(Qv,Vx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddbq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vx,Vu)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -325,7 +324,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vh_vadd_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddh)
+#define Q6_Vh_vadd_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddh)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -336,7 +335,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Wh_vadd_WhWh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddh_dv)
+#define Q6_Wh_vadd_WhWh(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddh_dv)(Vuu,Vvv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -347,7 +346,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vh_condacc_QnVhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddhnq)
+#define Q6_Vh_condacc_QnVhVh(Qv,Vx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddhnq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vx,Vu)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -358,7 +357,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vh_condacc_QVhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddhq)
+#define Q6_Vh_condacc_QVhVh(Qv,Vx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddhq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vx,Vu)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -369,7 +368,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vh_vadd_VhVh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddhsat)
+#define Q6_Vh_vadd_VhVh_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddhsat)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -380,7 +379,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Wh_vadd_WhWh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddhsat_dv)
+#define Q6_Wh_vadd_WhWh_sat(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddhsat_dv)(Vuu,Vvv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -391,7 +390,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Ww_vadd_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddhw)
+#define Q6_Ww_vadd_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddhw)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -402,7 +401,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Wh_vadd_VubVub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddubh)
+#define Q6_Wh_vadd_VubVub(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddubh)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -413,7 +412,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vub_vadd_VubVub_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddubsat)
+#define Q6_Vub_vadd_VubVub_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddubsat)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -424,7 +423,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Wub_vadd_WubWub_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddubsat_dv)
+#define Q6_Wub_vadd_WubWub_sat(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddubsat_dv)(Vuu,Vvv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -435,7 +434,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vuh_vadd_VuhVuh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadduhsat)
+#define Q6_Vuh_vadd_VuhVuh_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadduhsat)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -446,7 +445,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Wuh_vadd_WuhWuh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadduhsat_dv)
+#define Q6_Wuh_vadd_WuhWuh_sat(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadduhsat_dv)(Vuu,Vvv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -457,7 +456,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Ww_vadd_VuhVuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadduhw)
+#define Q6_Ww_vadd_VuhVuh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadduhw)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -468,7 +467,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vw_vadd_VwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddw)
+#define Q6_Vw_vadd_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddw)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -479,7 +478,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Ww_vadd_WwWw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddw_dv)
+#define Q6_Ww_vadd_WwWw(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddw_dv)(Vuu,Vvv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -490,7 +489,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vw_condacc_QnVwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddwnq)
+#define Q6_Vw_condacc_QnVwVw(Qv,Vx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddwnq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vx,Vu)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -501,7 +500,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vw_condacc_QVwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddwq)
+#define Q6_Vw_condacc_QVwVw(Qv,Vx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddwq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vx,Vu)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -512,7 +511,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vw_vadd_VwVw_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddwsat)
+#define Q6_Vw_vadd_VwVw_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddwsat)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -523,7 +522,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Ww_vadd_WwWw_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddwsat_dv)
+#define Q6_Ww_vadd_WwWw_sat(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddwsat_dv)(Vuu,Vvv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -534,7 +533,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_V_valign_VVR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_valignb)
+#define Q6_V_valign_VVR(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_valignb)(Vu,Vv,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -545,7 +544,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_V_valign_VVI __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_valignbi)
+#define Q6_V_valign_VVI(Vu,Vv,Iu3) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_valignbi)(Vu,Vv,Iu3)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -556,7 +555,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_V_vand_VV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vand)
+#define Q6_V_vand_VV(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vand)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -567,7 +566,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_V_vand_QR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)
+#define Q6_V_vand_QR(Qu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qu),-1),Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -578,7 +577,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_V_vandor_VQR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt_acc)
+#define Q6_V_vandor_VQR(Vx,Qu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt_acc)(Vx,__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qu),-1),Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -589,7 +588,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Q_vand_VR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)
+#define Q6_Q_vand_VR(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)(Vu,Rt)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -600,7 +599,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Q_vandor_QVR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt_acc)
+#define Q6_Q_vandor_QVR(Qx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt_acc)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Rt)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -611,7 +610,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vh_vasl_VhR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaslh)
+#define Q6_Vh_vasl_VhR(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaslh)(Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -622,7 +621,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vh_vasl_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaslhv)
+#define Q6_Vh_vasl_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaslhv)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -633,7 +632,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vw_vasl_VwR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaslw)
+#define Q6_Vw_vasl_VwR(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaslw)(Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -644,7 +643,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vw_vaslacc_VwVwR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaslw_acc)
+#define Q6_Vw_vaslacc_VwVwR(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaslw_acc)(Vx,Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -655,7 +654,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vw_vasl_VwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaslwv)
+#define Q6_Vw_vasl_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaslwv)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -666,7 +665,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vh_vasr_VhR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrh)
+#define Q6_Vh_vasr_VhR(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrh)(Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -677,7 +676,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vb_vasr_VhVhR_rnd_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrhbrndsat)
+#define Q6_Vb_vasr_VhVhR_rnd_sat(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrhbrndsat)(Vu,Vv,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -688,7 +687,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vub_vasr_VhVhR_rnd_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrhubrndsat)
+#define Q6_Vub_vasr_VhVhR_rnd_sat(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrhubrndsat)(Vu,Vv,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -699,7 +698,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vub_vasr_VhVhR_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrhubsat)
+#define Q6_Vub_vasr_VhVhR_sat(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrhubsat)(Vu,Vv,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -710,7 +709,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vh_vasr_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrhv)
+#define Q6_Vh_vasr_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrhv)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -721,7 +720,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vw_vasr_VwR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrw)
+#define Q6_Vw_vasr_VwR(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrw)(Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -732,7 +731,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vw_vasracc_VwVwR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrw_acc)
+#define Q6_Vw_vasracc_VwVwR(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrw_acc)(Vx,Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -743,7 +742,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vh_vasr_VwVwR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrwh)
+#define Q6_Vh_vasr_VwVwR(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrwh)(Vu,Vv,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -754,7 +753,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vh_vasr_VwVwR_rnd_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrwhrndsat)
+#define Q6_Vh_vasr_VwVwR_rnd_sat(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrwhrndsat)(Vu,Vv,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -765,7 +764,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vh_vasr_VwVwR_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrwhsat)
+#define Q6_Vh_vasr_VwVwR_sat(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrwhsat)(Vu,Vv,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -776,7 +775,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vuh_vasr_VwVwR_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrwuhsat)
+#define Q6_Vuh_vasr_VwVwR_sat(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrwuhsat)(Vu,Vv,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -787,7 +786,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vw_vasr_VwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrwv)
+#define Q6_Vw_vasr_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrwv)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -798,7 +797,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_V_equals_V __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vassign)
+#define Q6_V_equals_V(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vassign)(Vu)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -809,7 +808,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_W_equals_W __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vassignp)
+#define Q6_W_equals_W(Vuu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vassignp)(Vuu)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -820,7 +819,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vh_vavg_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavgh)
+#define Q6_Vh_vavg_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavgh)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -831,7 +830,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vh_vavg_VhVh_rnd __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavghrnd)
+#define Q6_Vh_vavg_VhVh_rnd(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavghrnd)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -842,7 +841,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vub_vavg_VubVub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavgub)
+#define Q6_Vub_vavg_VubVub(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavgub)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -853,7 +852,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vub_vavg_VubVub_rnd __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavgubrnd)
+#define Q6_Vub_vavg_VubVub_rnd(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavgubrnd)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -864,7 +863,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vuh_vavg_VuhVuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavguh)
+#define Q6_Vuh_vavg_VuhVuh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavguh)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -875,7 +874,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vuh_vavg_VuhVuh_rnd __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavguhrnd)
+#define Q6_Vuh_vavg_VuhVuh_rnd(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavguhrnd)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -886,7 +885,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vw_vavg_VwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavgw)
+#define Q6_Vw_vavg_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavgw)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -897,7 +896,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vw_vavg_VwVw_rnd __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavgwrnd)
+#define Q6_Vw_vavg_VwVw_rnd(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavgwrnd)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -908,7 +907,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vuh_vcl0_Vuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcl0h)
+#define Q6_Vuh_vcl0_Vuh(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcl0h)(Vu)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -919,7 +918,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vuw_vcl0_Vuw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcl0w)
+#define Q6_Vuw_vcl0_Vuw(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcl0w)(Vu)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -930,7 +929,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_W_vcombine_VV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcombine)
+#define Q6_W_vcombine_VV(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcombine)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -941,7 +940,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_V_vzero __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vd0)
+#define Q6_V_vzero() __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vd0)()
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -952,7 +951,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vb_vdeal_Vb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdealb)
+#define Q6_Vb_vdeal_Vb(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdealb)(Vu)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -963,7 +962,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vb_vdeale_VbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdealb4w)
+#define Q6_Vb_vdeale_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdealb4w)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -974,7 +973,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vh_vdeal_Vh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdealh)
+#define Q6_Vh_vdeal_Vh(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdealh)(Vu)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -985,7 +984,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_W_vdeal_VVR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdealvdd)
+#define Q6_W_vdeal_VVR(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdealvdd)(Vu,Vv,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -996,7 +995,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_V_vdelta_VV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdelta)
+#define Q6_V_vdelta_VV(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdelta)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1007,7 +1006,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vh_vdmpy_VubRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpybus)
+#define Q6_Vh_vdmpy_VubRb(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpybus)(Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1018,7 +1017,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vh_vdmpyacc_VhVubRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpybus_acc)
+#define Q6_Vh_vdmpyacc_VhVubRb(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpybus_acc)(Vx,Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1029,7 +1028,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Wh_vdmpy_WubRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpybus_dv)
+#define Q6_Wh_vdmpy_WubRb(Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpybus_dv)(Vuu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1040,7 +1039,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Wh_vdmpyacc_WhWubRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpybus_dv_acc)
+#define Q6_Wh_vdmpyacc_WhWubRb(Vxx,Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpybus_dv_acc)(Vxx,Vuu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1051,7 +1050,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vw_vdmpy_VhRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhb)
+#define Q6_Vw_vdmpy_VhRb(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhb)(Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1062,7 +1061,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vw_vdmpyacc_VwVhRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhb_acc)
+#define Q6_Vw_vdmpyacc_VwVhRb(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhb_acc)(Vx,Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1073,7 +1072,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Ww_vdmpy_WhRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhb_dv)
+#define Q6_Ww_vdmpy_WhRb(Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhb_dv)(Vuu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1084,7 +1083,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Ww_vdmpyacc_WwWhRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhb_dv_acc)
+#define Q6_Ww_vdmpyacc_WwWhRb(Vxx,Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhb_dv_acc)(Vxx,Vuu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1095,7 +1094,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vw_vdmpy_WhRh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhisat)
+#define Q6_Vw_vdmpy_WhRh_sat(Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhisat)(Vuu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1106,29 +1105,29 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vw_vdmpyacc_VwWhRh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhisat_acc)
+#define Q6_Vw_vdmpyacc_VwWhRh_sat(Vx,Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhisat_acc)(Vx,Vuu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
 /* ==========================================================================
    Assembly Syntax:       Vd32.w=vdmpy(Vu32.h,Rt32.h):sat
    C Intrinsic Prototype: HVX_Vector Q6_Vw_vdmpy_VhRh_sat(HVX_Vector Vu, Word32 Rt)
-   Instruction Type:      CVI_VX_DV
+   Instruction Type:      CVI_VX
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vw_vdmpy_VhRh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhsat)
+#define Q6_Vw_vdmpy_VhRh_sat(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhsat)(Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
 /* ==========================================================================
    Assembly Syntax:       Vx32.w+=vdmpy(Vu32.h,Rt32.h):sat
    C Intrinsic Prototype: HVX_Vector Q6_Vw_vdmpyacc_VwVhRh_sat(HVX_Vector Vx, HVX_Vector Vu, Word32 Rt)
-   Instruction Type:      CVI_VX_DV
+   Instruction Type:      CVI_VX
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vw_vdmpyacc_VwVhRh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhsat_acc)
+#define Q6_Vw_vdmpyacc_VwVhRh_sat(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhsat_acc)(Vx,Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1139,7 +1138,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vw_vdmpy_WhRuh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhsuisat)
+#define Q6_Vw_vdmpy_WhRuh_sat(Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhsuisat)(Vuu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1150,40 +1149,40 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vw_vdmpyacc_VwWhRuh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhsuisat_acc)
+#define Q6_Vw_vdmpyacc_VwWhRuh_sat(Vx,Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhsuisat_acc)(Vx,Vuu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
 /* ==========================================================================
    Assembly Syntax:       Vd32.w=vdmpy(Vu32.h,Rt32.uh):sat
    C Intrinsic Prototype: HVX_Vector Q6_Vw_vdmpy_VhRuh_sat(HVX_Vector Vu, Word32 Rt)
-   Instruction Type:      CVI_VX_DV
+   Instruction Type:      CVI_VX
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vw_vdmpy_VhRuh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhsusat)
+#define Q6_Vw_vdmpy_VhRuh_sat(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhsusat)(Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
 /* ==========================================================================
    Assembly Syntax:       Vx32.w+=vdmpy(Vu32.h,Rt32.uh):sat
    C Intrinsic Prototype: HVX_Vector Q6_Vw_vdmpyacc_VwVhRuh_sat(HVX_Vector Vx, HVX_Vector Vu, Word32 Rt)
-   Instruction Type:      CVI_VX_DV
+   Instruction Type:      CVI_VX
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vw_vdmpyacc_VwVhRuh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhsusat_acc)
+#define Q6_Vw_vdmpyacc_VwVhRuh_sat(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhsusat_acc)(Vx,Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
 /* ==========================================================================
    Assembly Syntax:       Vd32.w=vdmpy(Vu32.h,Vv32.h):sat
    C Intrinsic Prototype: HVX_Vector Q6_Vw_vdmpy_VhVh_sat(HVX_Vector Vu, HVX_Vector Vv)
-   Instruction Type:      CVI_VX_DV
+   Instruction Type:      CVI_VX
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vw_vdmpy_VhVh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhvsat)
+#define Q6_Vw_vdmpy_VhVh_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhvsat)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1194,7 +1193,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vw_vdmpyacc_VwVhVh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhvsat_acc)
+#define Q6_Vw_vdmpyacc_VwVhVh_sat(Vx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhvsat_acc)(Vx,Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1205,7 +1204,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Wuw_vdsad_WuhRuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdsaduh)
+#define Q6_Wuw_vdsad_WuhRuh(Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdsaduh)(Vuu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1216,7 +1215,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Wuw_vdsadacc_WuwWuhRuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdsaduh_acc)
+#define Q6_Wuw_vdsadacc_WuwWuhRuh(Vxx,Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdsaduh_acc)(Vxx,Vuu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1227,7 +1226,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_vcmp_eq_VbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqb)
+#define Q6_Q_vcmp_eq_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqb)(Vu,Vv)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1238,7 +1237,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_vcmp_eqand_QVbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqb_and)
+#define Q6_Q_vcmp_eqand_QVbVb(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqb_and)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1249,7 +1248,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_vcmp_eqor_QVbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqb_or)
+#define Q6_Q_vcmp_eqor_QVbVb(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqb_or)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1260,7 +1259,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_vcmp_eqxacc_QVbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqb_xor)
+#define Q6_Q_vcmp_eqxacc_QVbVb(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqb_xor)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1271,7 +1270,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_vcmp_eq_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqh)
+#define Q6_Q_vcmp_eq_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqh)(Vu,Vv)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1282,7 +1281,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_vcmp_eqand_QVhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqh_and)
+#define Q6_Q_vcmp_eqand_QVhVh(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqh_and)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1293,7 +1292,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_vcmp_eqor_QVhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqh_or)
+#define Q6_Q_vcmp_eqor_QVhVh(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqh_or)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1304,7 +1303,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_vcmp_eqxacc_QVhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqh_xor)
+#define Q6_Q_vcmp_eqxacc_QVhVh(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqh_xor)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1315,7 +1314,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_vcmp_eq_VwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqw)
+#define Q6_Q_vcmp_eq_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqw)(Vu,Vv)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1326,7 +1325,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_vcmp_eqand_QVwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqw_and)
+#define Q6_Q_vcmp_eqand_QVwVw(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqw_and)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1337,7 +1336,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_vcmp_eqor_QVwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqw_or)
+#define Q6_Q_vcmp_eqor_QVwVw(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqw_or)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1348,7 +1347,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_vcmp_eqxacc_QVwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqw_xor)
+#define Q6_Q_vcmp_eqxacc_QVwVw(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqw_xor)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1359,7 +1358,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_vcmp_gt_VbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtb)
+#define Q6_Q_vcmp_gt_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtb)(Vu,Vv)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1370,7 +1369,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_vcmp_gtand_QVbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtb_and)
+#define Q6_Q_vcmp_gtand_QVbVb(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtb_and)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1381,7 +1380,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_vcmp_gtor_QVbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtb_or)
+#define Q6_Q_vcmp_gtor_QVbVb(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtb_or)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1392,7 +1391,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_vcmp_gtxacc_QVbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtb_xor)
+#define Q6_Q_vcmp_gtxacc_QVbVb(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtb_xor)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1403,7 +1402,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_vcmp_gt_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgth)
+#define Q6_Q_vcmp_gt_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgth)(Vu,Vv)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1414,7 +1413,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_vcmp_gtand_QVhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgth_and)
+#define Q6_Q_vcmp_gtand_QVhVh(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgth_and)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1425,7 +1424,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_vcmp_gtor_QVhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgth_or)
+#define Q6_Q_vcmp_gtor_QVhVh(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgth_or)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1436,7 +1435,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_vcmp_gtxacc_QVhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgth_xor)
+#define Q6_Q_vcmp_gtxacc_QVhVh(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgth_xor)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1447,7 +1446,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_vcmp_gt_VubVub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtub)
+#define Q6_Q_vcmp_gt_VubVub(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtub)(Vu,Vv)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1458,7 +1457,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_vcmp_gtand_QVubVub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtub_and)
+#define Q6_Q_vcmp_gtand_QVubVub(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtub_and)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1469,7 +1468,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_vcmp_gtor_QVubVub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtub_or)
+#define Q6_Q_vcmp_gtor_QVubVub(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtub_or)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1480,7 +1479,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_vcmp_gtxacc_QVubVub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtub_xor)
+#define Q6_Q_vcmp_gtxacc_QVubVub(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtub_xor)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1491,7 +1490,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_vcmp_gt_VuhVuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuh)
+#define Q6_Q_vcmp_gt_VuhVuh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuh)(Vu,Vv)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1502,7 +1501,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_vcmp_gtand_QVuhVuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuh_and)
+#define Q6_Q_vcmp_gtand_QVuhVuh(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuh_and)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1513,7 +1512,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_vcmp_gtor_QVuhVuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuh_or)
+#define Q6_Q_vcmp_gtor_QVuhVuh(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuh_or)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1524,7 +1523,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_vcmp_gtxacc_QVuhVuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuh_xor)
+#define Q6_Q_vcmp_gtxacc_QVuhVuh(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuh_xor)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1535,7 +1534,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_vcmp_gt_VuwVuw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuw)
+#define Q6_Q_vcmp_gt_VuwVuw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuw)(Vu,Vv)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1546,7 +1545,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_vcmp_gtand_QVuwVuw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuw_and)
+#define Q6_Q_vcmp_gtand_QVuwVuw(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuw_and)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1557,7 +1556,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_vcmp_gtor_QVuwVuw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuw_or)
+#define Q6_Q_vcmp_gtor_QVuwVuw(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuw_or)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1568,7 +1567,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_vcmp_gtxacc_QVuwVuw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuw_xor)
+#define Q6_Q_vcmp_gtxacc_QVuwVuw(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuw_xor)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1579,7 +1578,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_vcmp_gt_VwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtw)
+#define Q6_Q_vcmp_gt_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtw)(Vu,Vv)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1590,7 +1589,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_vcmp_gtand_QVwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtw_and)
+#define Q6_Q_vcmp_gtand_QVwVw(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtw_and)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1601,7 +1600,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_vcmp_gtor_QVwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtw_or)
+#define Q6_Q_vcmp_gtor_QVwVw(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtw_or)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1612,7 +1611,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_vcmp_gtxacc_QVwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtw_xor)
+#define Q6_Q_vcmp_gtxacc_QVwVw(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtw_xor)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1623,7 +1622,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vw_vinsert_VwR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vinsertwr)
+#define Q6_Vw_vinsert_VwR(Vx,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vinsertwr)(Vx,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1634,7 +1633,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_V_vlalign_VVR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlalignb)
+#define Q6_V_vlalign_VVR(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlalignb)(Vu,Vv,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1645,7 +1644,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_V_vlalign_VVI __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlalignbi)
+#define Q6_V_vlalign_VVI(Vu,Vv,Iu3) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlalignbi)(Vu,Vv,Iu3)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1656,7 +1655,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vuh_vlsr_VuhR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlsrh)
+#define Q6_Vuh_vlsr_VuhR(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlsrh)(Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1667,7 +1666,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vh_vlsr_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlsrhv)
+#define Q6_Vh_vlsr_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlsrhv)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1678,7 +1677,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vuw_vlsr_VuwR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlsrw)
+#define Q6_Vuw_vlsr_VuwR(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlsrw)(Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1689,7 +1688,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vw_vlsr_VwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlsrwv)
+#define Q6_Vw_vlsr_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlsrwv)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1700,7 +1699,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vb_vlut32_VbVbR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvvb)
+#define Q6_Vb_vlut32_VbVbR(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvvb)(Vu,Vv,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1711,7 +1710,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vb_vlut32or_VbVbVbR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvvb_oracc)
+#define Q6_Vb_vlut32or_VbVbVbR(Vx,Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvvb_oracc)(Vx,Vu,Vv,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1722,7 +1721,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Wh_vlut16_VbVhR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvwh)
+#define Q6_Wh_vlut16_VbVhR(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvwh)(Vu,Vv,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1733,7 +1732,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Wh_vlut16or_WhVbVhR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvwh_oracc)
+#define Q6_Wh_vlut16or_WhVbVhR(Vxx,Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvwh_oracc)(Vxx,Vu,Vv,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1744,7 +1743,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vh_vmax_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmaxh)
+#define Q6_Vh_vmax_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmaxh)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1755,7 +1754,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vub_vmax_VubVub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmaxub)
+#define Q6_Vub_vmax_VubVub(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmaxub)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1766,7 +1765,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vuh_vmax_VuhVuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmaxuh)
+#define Q6_Vuh_vmax_VuhVuh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmaxuh)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1777,7 +1776,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vw_vmax_VwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmaxw)
+#define Q6_Vw_vmax_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmaxw)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1788,7 +1787,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vh_vmin_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vminh)
+#define Q6_Vh_vmin_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vminh)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1799,7 +1798,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vub_vmin_VubVub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vminub)
+#define Q6_Vub_vmin_VubVub(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vminub)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1810,7 +1809,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vuh_vmin_VuhVuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vminuh)
+#define Q6_Vuh_vmin_VuhVuh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vminuh)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1821,7 +1820,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vw_vmin_VwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vminw)
+#define Q6_Vw_vmin_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vminw)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1832,7 +1831,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Wh_vmpa_WubRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpabus)
+#define Q6_Wh_vmpa_WubRb(Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpabus)(Vuu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1843,7 +1842,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Wh_vmpaacc_WhWubRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpabus_acc)
+#define Q6_Wh_vmpaacc_WhWubRb(Vxx,Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpabus_acc)(Vxx,Vuu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1854,7 +1853,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Wh_vmpa_WubWb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpabusv)
+#define Q6_Wh_vmpa_WubWb(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpabusv)(Vuu,Vvv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1865,7 +1864,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Wh_vmpa_WubWub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpabuuv)
+#define Q6_Wh_vmpa_WubWub(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpabuuv)(Vuu,Vvv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1876,7 +1875,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Ww_vmpa_WhRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpahb)
+#define Q6_Ww_vmpa_WhRb(Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpahb)(Vuu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1887,7 +1886,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Ww_vmpaacc_WwWhRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpahb_acc)
+#define Q6_Ww_vmpaacc_WwWhRb(Vxx,Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpahb_acc)(Vxx,Vuu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1898,7 +1897,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Wh_vmpy_VubRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpybus)
+#define Q6_Wh_vmpy_VubRb(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpybus)(Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1909,7 +1908,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Wh_vmpyacc_WhVubRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpybus_acc)
+#define Q6_Wh_vmpyacc_WhVubRb(Vxx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpybus_acc)(Vxx,Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1920,7 +1919,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Wh_vmpy_VubVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpybusv)
+#define Q6_Wh_vmpy_VubVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpybusv)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1931,7 +1930,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Wh_vmpyacc_WhVubVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpybusv_acc)
+#define Q6_Wh_vmpyacc_WhVubVb(Vxx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpybusv_acc)(Vxx,Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1942,7 +1941,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Wh_vmpy_VbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpybv)
+#define Q6_Wh_vmpy_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpybv)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1953,7 +1952,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Wh_vmpyacc_WhVbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpybv_acc)
+#define Q6_Wh_vmpyacc_WhVbVb(Vxx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpybv_acc)(Vxx,Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1964,7 +1963,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vw_vmpye_VwVuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyewuh)
+#define Q6_Vw_vmpye_VwVuh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyewuh)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1975,7 +1974,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Ww_vmpy_VhRh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyh)
+#define Q6_Ww_vmpy_VhRh(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyh)(Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -1986,29 +1985,29 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Ww_vmpyacc_WwVhRh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhsat_acc)
+#define Q6_Ww_vmpyacc_WwVhRh_sat(Vxx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhsat_acc)(Vxx,Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
 /* ==========================================================================
    Assembly Syntax:       Vd32.h=vmpy(Vu32.h,Rt32.h):<<1:rnd:sat
    C Intrinsic Prototype: HVX_Vector Q6_Vh_vmpy_VhRh_s1_rnd_sat(HVX_Vector Vu, Word32 Rt)
-   Instruction Type:      CVI_VX_DV
+   Instruction Type:      CVI_VX
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vh_vmpy_VhRh_s1_rnd_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhsrs)
+#define Q6_Vh_vmpy_VhRh_s1_rnd_sat(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhsrs)(Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
 /* ==========================================================================
    Assembly Syntax:       Vd32.h=vmpy(Vu32.h,Rt32.h):<<1:sat
    C Intrinsic Prototype: HVX_Vector Q6_Vh_vmpy_VhRh_s1_sat(HVX_Vector Vu, Word32 Rt)
-   Instruction Type:      CVI_VX_DV
+   Instruction Type:      CVI_VX
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vh_vmpy_VhRh_s1_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhss)
+#define Q6_Vh_vmpy_VhRh_s1_sat(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhss)(Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2019,7 +2018,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Ww_vmpy_VhVuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhus)
+#define Q6_Ww_vmpy_VhVuh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhus)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2030,7 +2029,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Ww_vmpyacc_WwVhVuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhus_acc)
+#define Q6_Ww_vmpyacc_WwVhVuh(Vxx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhus_acc)(Vxx,Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2041,7 +2040,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Ww_vmpy_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhv)
+#define Q6_Ww_vmpy_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhv)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2052,18 +2051,18 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Ww_vmpyacc_WwVhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhv_acc)
+#define Q6_Ww_vmpyacc_WwVhVh(Vxx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhv_acc)(Vxx,Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
 /* ==========================================================================
    Assembly Syntax:       Vd32.h=vmpy(Vu32.h,Vv32.h):<<1:rnd:sat
    C Intrinsic Prototype: HVX_Vector Q6_Vh_vmpy_VhVh_s1_rnd_sat(HVX_Vector Vu, HVX_Vector Vv)
-   Instruction Type:      CVI_VX_DV
+   Instruction Type:      CVI_VX
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vh_vmpy_VhVh_s1_rnd_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhvsrs)
+#define Q6_Vh_vmpy_VhVh_s1_rnd_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhvsrs)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2074,7 +2073,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vw_vmpyieo_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyieoh)
+#define Q6_Vw_vmpyieo_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyieoh)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2085,7 +2084,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vw_vmpyieacc_VwVwVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiewh_acc)
+#define Q6_Vw_vmpyieacc_VwVwVh(Vx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiewh_acc)(Vx,Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2096,7 +2095,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vw_vmpyie_VwVuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiewuh)
+#define Q6_Vw_vmpyie_VwVuh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiewuh)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2107,7 +2106,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vw_vmpyieacc_VwVwVuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiewuh_acc)
+#define Q6_Vw_vmpyieacc_VwVwVuh(Vx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiewuh_acc)(Vx,Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2118,7 +2117,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vh_vmpyi_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyih)
+#define Q6_Vh_vmpyi_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyih)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2129,7 +2128,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vh_vmpyiacc_VhVhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyih_acc)
+#define Q6_Vh_vmpyiacc_VhVhVh(Vx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyih_acc)(Vx,Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2140,7 +2139,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vh_vmpyi_VhRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyihb)
+#define Q6_Vh_vmpyi_VhRb(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyihb)(Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2151,7 +2150,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vh_vmpyiacc_VhVhRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyihb_acc)
+#define Q6_Vh_vmpyiacc_VhVhRb(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyihb_acc)(Vx,Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2162,7 +2161,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vw_vmpyio_VwVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiowh)
+#define Q6_Vw_vmpyio_VwVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiowh)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2173,7 +2172,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vw_vmpyi_VwRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiwb)
+#define Q6_Vw_vmpyi_VwRb(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiwb)(Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2184,7 +2183,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vw_vmpyiacc_VwVwRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiwb_acc)
+#define Q6_Vw_vmpyiacc_VwVwRb(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiwb_acc)(Vx,Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2195,7 +2194,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vw_vmpyi_VwRh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiwh)
+#define Q6_Vw_vmpyi_VwRh(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiwh)(Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2206,7 +2205,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vw_vmpyiacc_VwVwRh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiwh_acc)
+#define Q6_Vw_vmpyiacc_VwVwRh(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiwh_acc)(Vx,Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2217,7 +2216,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vw_vmpyo_VwVh_s1_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyowh)
+#define Q6_Vw_vmpyo_VwVh_s1_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyowh)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2228,7 +2227,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vw_vmpyo_VwVh_s1_rnd_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyowh_rnd)
+#define Q6_Vw_vmpyo_VwVh_s1_rnd_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyowh_rnd)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2239,7 +2238,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vw_vmpyoacc_VwVwVh_s1_rnd_sat_shift __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyowh_rnd_sacc)
+#define Q6_Vw_vmpyoacc_VwVwVh_s1_rnd_sat_shift(Vx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyowh_rnd_sacc)(Vx,Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2250,7 +2249,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vw_vmpyoacc_VwVwVh_s1_sat_shift __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyowh_sacc)
+#define Q6_Vw_vmpyoacc_VwVwVh_s1_sat_shift(Vx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyowh_sacc)(Vx,Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2261,7 +2260,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Wuh_vmpy_VubRub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyub)
+#define Q6_Wuh_vmpy_VubRub(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyub)(Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2272,7 +2271,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Wuh_vmpyacc_WuhVubRub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyub_acc)
+#define Q6_Wuh_vmpyacc_WuhVubRub(Vxx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyub_acc)(Vxx,Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2283,7 +2282,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Wuh_vmpy_VubVub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyubv)
+#define Q6_Wuh_vmpy_VubVub(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyubv)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2294,7 +2293,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Wuh_vmpyacc_WuhVubVub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyubv_acc)
+#define Q6_Wuh_vmpyacc_WuhVubVub(Vxx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyubv_acc)(Vxx,Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2305,7 +2304,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Wuw_vmpy_VuhRuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyuh)
+#define Q6_Wuw_vmpy_VuhRuh(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyuh)(Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2316,7 +2315,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Wuw_vmpyacc_WuwVuhRuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyuh_acc)
+#define Q6_Wuw_vmpyacc_WuwVuhRuh(Vxx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyuh_acc)(Vxx,Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2327,7 +2326,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Wuw_vmpy_VuhVuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyuhv)
+#define Q6_Wuw_vmpy_VuhVuh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyuhv)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2338,7 +2337,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Wuw_vmpyacc_WuwVuhVuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyuhv_acc)
+#define Q6_Wuw_vmpyacc_WuwVuhVuh(Vxx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyuhv_acc)(Vxx,Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2349,7 +2348,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_V_vmux_QVV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmux)
+#define Q6_V_vmux_QVV(Qt,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmux)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qt),-1),Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2360,7 +2359,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vh_vnavg_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vnavgh)
+#define Q6_Vh_vnavg_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vnavgh)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2371,7 +2370,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vb_vnavg_VubVub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vnavgub)
+#define Q6_Vb_vnavg_VubVub(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vnavgub)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2382,7 +2381,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vw_vnavg_VwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vnavgw)
+#define Q6_Vw_vnavg_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vnavgw)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2393,7 +2392,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vh_vnormamt_Vh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vnormamth)
+#define Q6_Vh_vnormamt_Vh(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vnormamth)(Vu)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2404,7 +2403,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vw_vnormamt_Vw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vnormamtw)
+#define Q6_Vw_vnormamt_Vw(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vnormamtw)(Vu)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2415,7 +2414,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_V_vnot_V __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vnot)
+#define Q6_V_vnot_V(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vnot)(Vu)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2426,7 +2425,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_V_vor_VV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vor)
+#define Q6_V_vor_VV(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vor)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2437,7 +2436,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vb_vpacke_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackeb)
+#define Q6_Vb_vpacke_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackeb)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2448,7 +2447,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vh_vpacke_VwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackeh)
+#define Q6_Vh_vpacke_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackeh)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2459,7 +2458,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vb_vpack_VhVh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackhb_sat)
+#define Q6_Vb_vpack_VhVh_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackhb_sat)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2470,7 +2469,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vub_vpack_VhVh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackhub_sat)
+#define Q6_Vub_vpack_VhVh_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackhub_sat)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2481,7 +2480,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vb_vpacko_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackob)
+#define Q6_Vb_vpacko_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackob)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2492,7 +2491,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vh_vpacko_VwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackoh)
+#define Q6_Vh_vpacko_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackoh)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2503,7 +2502,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vh_vpack_VwVw_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackwh_sat)
+#define Q6_Vh_vpack_VwVw_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackwh_sat)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2514,7 +2513,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vuh_vpack_VwVw_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackwuh_sat)
+#define Q6_Vuh_vpack_VwVw_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackwuh_sat)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2525,7 +2524,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vh_vpopcount_Vh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpopcounth)
+#define Q6_Vh_vpopcount_Vh(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpopcounth)(Vu)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2536,7 +2535,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_V_vrdelta_VV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrdelta)
+#define Q6_V_vrdelta_VV(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrdelta)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2547,7 +2546,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vw_vrmpy_VubRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybus)
+#define Q6_Vw_vrmpy_VubRb(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybus)(Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2558,7 +2557,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vw_vrmpyacc_VwVubRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybus_acc)
+#define Q6_Vw_vrmpyacc_VwVubRb(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybus_acc)(Vx,Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2569,7 +2568,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Ww_vrmpy_WubRbI __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybusi)
+#define Q6_Ww_vrmpy_WubRbI(Vuu,Rt,Iu1) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybusi)(Vuu,Rt,Iu1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2580,7 +2579,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Ww_vrmpyacc_WwWubRbI __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybusi_acc)
+#define Q6_Ww_vrmpyacc_WwWubRbI(Vxx,Vuu,Rt,Iu1) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybusi_acc)(Vxx,Vuu,Rt,Iu1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2591,18 +2590,18 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vw_vrmpy_VubVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybusv)
+#define Q6_Vw_vrmpy_VubVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybusv)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
 /* ==========================================================================
    Assembly Syntax:       Vx32.w+=vrmpy(Vu32.ub,Vv32.b)
    C Intrinsic Prototype: HVX_Vector Q6_Vw_vrmpyacc_VwVubVb(HVX_Vector Vx, HVX_Vector Vu, HVX_Vector Vv)
-   Instruction Type:      CVI_VX_DV
+   Instruction Type:      CVI_VX
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vw_vrmpyacc_VwVubVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybusv_acc)
+#define Q6_Vw_vrmpyacc_VwVubVb(Vx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybusv_acc)(Vx,Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2613,18 +2612,18 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vw_vrmpy_VbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybv)
+#define Q6_Vw_vrmpy_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybv)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
 /* ==========================================================================
    Assembly Syntax:       Vx32.w+=vrmpy(Vu32.b,Vv32.b)
    C Intrinsic Prototype: HVX_Vector Q6_Vw_vrmpyacc_VwVbVb(HVX_Vector Vx, HVX_Vector Vu, HVX_Vector Vv)
-   Instruction Type:      CVI_VX_DV
+   Instruction Type:      CVI_VX
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vw_vrmpyacc_VwVbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybv_acc)
+#define Q6_Vw_vrmpyacc_VwVbVb(Vx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybv_acc)(Vx,Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2635,7 +2634,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vuw_vrmpy_VubRub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpyub)
+#define Q6_Vuw_vrmpy_VubRub(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpyub)(Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2646,7 +2645,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vuw_vrmpyacc_VuwVubRub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpyub_acc)
+#define Q6_Vuw_vrmpyacc_VuwVubRub(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpyub_acc)(Vx,Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2657,7 +2656,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Wuw_vrmpy_WubRubI __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpyubi)
+#define Q6_Wuw_vrmpy_WubRubI(Vuu,Rt,Iu1) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpyubi)(Vuu,Rt,Iu1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2668,7 +2667,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Wuw_vrmpyacc_WuwWubRubI __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpyubi_acc)
+#define Q6_Wuw_vrmpyacc_WuwWubRubI(Vxx,Vuu,Rt,Iu1) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpyubi_acc)(Vxx,Vuu,Rt,Iu1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2679,18 +2678,18 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vuw_vrmpy_VubVub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpyubv)
+#define Q6_Vuw_vrmpy_VubVub(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpyubv)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
 /* ==========================================================================
    Assembly Syntax:       Vx32.uw+=vrmpy(Vu32.ub,Vv32.ub)
    C Intrinsic Prototype: HVX_Vector Q6_Vuw_vrmpyacc_VuwVubVub(HVX_Vector Vx, HVX_Vector Vu, HVX_Vector Vv)
-   Instruction Type:      CVI_VX_DV
+   Instruction Type:      CVI_VX
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vuw_vrmpyacc_VuwVubVub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpyubv_acc)
+#define Q6_Vuw_vrmpyacc_VuwVubVub(Vx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpyubv_acc)(Vx,Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2701,7 +2700,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_V_vror_VR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vror)
+#define Q6_V_vror_VR(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vror)(Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2712,7 +2711,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vb_vround_VhVh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vroundhb)
+#define Q6_Vb_vround_VhVh_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vroundhb)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2723,7 +2722,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vub_vround_VhVh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vroundhub)
+#define Q6_Vub_vround_VhVh_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vroundhub)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2734,7 +2733,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vh_vround_VwVw_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vroundwh)
+#define Q6_Vh_vround_VwVw_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vroundwh)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2745,7 +2744,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vuh_vround_VwVw_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vroundwuh)
+#define Q6_Vuh_vround_VwVw_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vroundwuh)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2756,7 +2755,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Wuw_vrsad_WubRubI __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrsadubi)
+#define Q6_Wuw_vrsad_WubRubI(Vuu,Rt,Iu1) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrsadubi)(Vuu,Rt,Iu1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2767,7 +2766,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Wuw_vrsadacc_WuwWubRubI __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrsadubi_acc)
+#define Q6_Wuw_vrsadacc_WuwWubRubI(Vxx,Vuu,Rt,Iu1) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrsadubi_acc)(Vxx,Vuu,Rt,Iu1)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2778,7 +2777,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vub_vsat_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsathub)
+#define Q6_Vub_vsat_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsathub)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2789,7 +2788,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vh_vsat_VwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsatwh)
+#define Q6_Vh_vsat_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsatwh)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2800,7 +2799,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Wh_vsxt_Vb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsb)
+#define Q6_Wh_vsxt_Vb(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsb)(Vu)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2811,7 +2810,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Ww_vsxt_Vh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsh)
+#define Q6_Ww_vsxt_Vh(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsh)(Vu)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2822,7 +2821,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vh_vshuffe_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshufeh)
+#define Q6_Vh_vshuffe_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshufeh)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2833,7 +2832,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vb_vshuff_Vb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshuffb)
+#define Q6_Vb_vshuff_Vb(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshuffb)(Vu)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2844,7 +2843,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vb_vshuffe_VbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshuffeb)
+#define Q6_Vb_vshuffe_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshuffeb)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2855,7 +2854,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vh_vshuff_Vh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshuffh)
+#define Q6_Vh_vshuff_Vh(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshuffh)(Vu)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2866,7 +2865,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vb_vshuffo_VbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshuffob)
+#define Q6_Vb_vshuffo_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshuffob)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2877,7 +2876,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_W_vshuff_VVR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshuffvdd)
+#define Q6_W_vshuff_VVR(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshuffvdd)(Vu,Vv,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2888,7 +2887,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Wb_vshuffoe_VbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshufoeb)
+#define Q6_Wb_vshuffoe_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshufoeb)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2899,7 +2898,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Wh_vshuffoe_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshufoeh)
+#define Q6_Wh_vshuffoe_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshufoeh)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2910,7 +2909,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vh_vshuffo_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshufoh)
+#define Q6_Vh_vshuffo_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshufoh)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2921,7 +2920,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vb_vsub_VbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubb)
+#define Q6_Vb_vsub_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubb)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2932,7 +2931,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Wb_vsub_WbWb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubb_dv)
+#define Q6_Wb_vsub_WbWb(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubb_dv)(Vuu,Vvv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2943,7 +2942,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vb_condnac_QnVbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubbnq)
+#define Q6_Vb_condnac_QnVbVb(Qv,Vx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubbnq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vx,Vu)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2954,7 +2953,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vb_condnac_QVbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubbq)
+#define Q6_Vb_condnac_QVbVb(Qv,Vx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubbq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vx,Vu)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2965,7 +2964,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vh_vsub_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubh)
+#define Q6_Vh_vsub_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubh)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2976,7 +2975,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Wh_vsub_WhWh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubh_dv)
+#define Q6_Wh_vsub_WhWh(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubh_dv)(Vuu,Vvv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2987,7 +2986,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vh_condnac_QnVhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubhnq)
+#define Q6_Vh_condnac_QnVhVh(Qv,Vx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubhnq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vx,Vu)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -2998,7 +2997,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vh_condnac_QVhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubhq)
+#define Q6_Vh_condnac_QVhVh(Qv,Vx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubhq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vx,Vu)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -3009,7 +3008,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vh_vsub_VhVh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubhsat)
+#define Q6_Vh_vsub_VhVh_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubhsat)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -3020,7 +3019,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Wh_vsub_WhWh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubhsat_dv)
+#define Q6_Wh_vsub_WhWh_sat(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubhsat_dv)(Vuu,Vvv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -3031,7 +3030,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Ww_vsub_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubhw)
+#define Q6_Ww_vsub_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubhw)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -3042,7 +3041,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Wh_vsub_VubVub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsububh)
+#define Q6_Wh_vsub_VubVub(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsububh)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -3053,7 +3052,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vub_vsub_VubVub_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsububsat)
+#define Q6_Vub_vsub_VubVub_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsububsat)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -3064,7 +3063,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Wub_vsub_WubWub_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsububsat_dv)
+#define Q6_Wub_vsub_WubWub_sat(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsububsat_dv)(Vuu,Vvv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -3075,7 +3074,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vuh_vsub_VuhVuh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubuhsat)
+#define Q6_Vuh_vsub_VuhVuh_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubuhsat)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -3086,7 +3085,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Wuh_vsub_WuhWuh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubuhsat_dv)
+#define Q6_Wuh_vsub_WuhWuh_sat(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubuhsat_dv)(Vuu,Vvv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -3097,7 +3096,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Ww_vsub_VuhVuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubuhw)
+#define Q6_Ww_vsub_VuhVuh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubuhw)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -3108,7 +3107,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vw_vsub_VwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubw)
+#define Q6_Vw_vsub_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubw)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -3119,7 +3118,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Ww_vsub_WwWw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubw_dv)
+#define Q6_Ww_vsub_WwWw(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubw_dv)(Vuu,Vvv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -3130,7 +3129,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vw_condnac_QnVwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubwnq)
+#define Q6_Vw_condnac_QnVwVw(Qv,Vx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubwnq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vx,Vu)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -3141,7 +3140,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vw_condnac_QVwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubwq)
+#define Q6_Vw_condnac_QVwVw(Qv,Vx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubwq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vx,Vu)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -3152,7 +3151,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vw_vsub_VwVw_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubwsat)
+#define Q6_Vw_vsub_VwVw_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubwsat)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -3163,7 +3162,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Ww_vsub_WwWw_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubwsat_dv)
+#define Q6_Ww_vsub_WwWw_sat(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubwsat_dv)(Vuu,Vvv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -3174,7 +3173,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_W_vswap_QVV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vswap)
+#define Q6_W_vswap_QVV(Qt,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vswap)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qt),-1),Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -3185,7 +3184,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Wh_vtmpy_WbRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vtmpyb)
+#define Q6_Wh_vtmpy_WbRb(Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vtmpyb)(Vuu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -3196,7 +3195,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Wh_vtmpyacc_WhWbRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vtmpyb_acc)
+#define Q6_Wh_vtmpyacc_WhWbRb(Vxx,Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vtmpyb_acc)(Vxx,Vuu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -3207,7 +3206,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Wh_vtmpy_WubRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vtmpybus)
+#define Q6_Wh_vtmpy_WubRb(Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vtmpybus)(Vuu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -3218,7 +3217,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Wh_vtmpyacc_WhWubRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vtmpybus_acc)
+#define Q6_Wh_vtmpyacc_WhWubRb(Vxx,Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vtmpybus_acc)(Vxx,Vuu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -3229,7 +3228,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Ww_vtmpy_WhRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vtmpyhb)
+#define Q6_Ww_vtmpy_WhRb(Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vtmpyhb)(Vuu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -3240,7 +3239,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Ww_vtmpyacc_WwWhRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vtmpyhb_acc)
+#define Q6_Ww_vtmpyacc_WwWhRb(Vxx,Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vtmpyhb_acc)(Vxx,Vuu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -3251,7 +3250,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Wh_vunpack_Vb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vunpackb)
+#define Q6_Wh_vunpack_Vb(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vunpackb)(Vu)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -3262,7 +3261,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Ww_vunpack_Vh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vunpackh)
+#define Q6_Ww_vunpack_Vh(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vunpackh)(Vu)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -3273,7 +3272,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Wh_vunpackoor_WhVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vunpackob)
+#define Q6_Wh_vunpackoor_WhVb(Vxx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vunpackob)(Vxx,Vu)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -3284,7 +3283,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Ww_vunpackoor_WwVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vunpackoh)
+#define Q6_Ww_vunpackoor_WwVh(Vxx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vunpackoh)(Vxx,Vu)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -3295,7 +3294,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Wuh_vunpack_Vub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vunpackub)
+#define Q6_Wuh_vunpack_Vub(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vunpackub)(Vu)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -3306,7 +3305,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Wuw_vunpack_Vuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vunpackuh)
+#define Q6_Wuw_vunpack_Vuh(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vunpackuh)(Vu)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -3317,7 +3316,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_V_vxor_VV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vxor)
+#define Q6_V_vxor_VV(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vxor)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -3328,7 +3327,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Wuh_vzxt_Vub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vzb)
+#define Q6_Wuh_vzxt_Vub(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vzb)(Vu)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 60
@@ -3339,7 +3338,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Wuw_vzxt_Vuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vzh)
+#define Q6_Wuw_vzxt_Vuh(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vzh)(Vu)
 #endif /* __HEXAGON_ARCH___ >= 60 */
 
 #if __HVX_ARCH__ >= 62
@@ -3350,7 +3349,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vb_vsplat_R __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_lvsplatb)
+#define Q6_Vb_vsplat_R(Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_lvsplatb)(Rt)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3361,7 +3360,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vh_vsplat_R __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_lvsplath)
+#define Q6_Vh_vsplat_R(Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_lvsplath)(Rt)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3372,7 +3371,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Q_vsetq2_R __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_scalar2v2)
+#define Q6_Q_vsetq2_R(Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_scalar2v2)(Rt)),-1)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3383,7 +3382,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Qb_vshuffe_QhQh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_shuffeqh)
+#define Q6_Qb_vshuffe_QhQh(Qs,Qt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_shuffeqh)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qt),-1))),-1)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3394,7 +3393,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Qh_vshuffe_QwQw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_shuffeqw)
+#define Q6_Qh_vshuffe_QwQw(Qs,Qt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_shuffeqw)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qt),-1))),-1)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3405,7 +3404,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vb_vadd_VbVb_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddbsat)
+#define Q6_Vb_vadd_VbVb_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddbsat)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3416,7 +3415,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Wb_vadd_WbWb_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddbsat_dv)
+#define Q6_Wb_vadd_WbWb_sat(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddbsat_dv)(Vuu,Vvv)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3427,7 +3426,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vw_vadd_VwVwQ_carry __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddcarry)
+#define Q6_Vw_vadd_VwVwQ_carry(Vu,Vv,Qx) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddcarry)(Vu,Vv,Qx)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3438,7 +3437,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vh_vadd_vclb_VhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddclbh)
+#define Q6_Vh_vadd_vclb_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddclbh)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3449,7 +3448,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vw_vadd_vclb_VwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddclbw)
+#define Q6_Vw_vadd_vclb_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddclbw)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3460,7 +3459,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Ww_vaddacc_WwVhVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddhw_acc)
+#define Q6_Ww_vaddacc_WwVhVh(Vxx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddhw_acc)(Vxx,Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3471,7 +3470,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Wh_vaddacc_WhVubVub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddubh_acc)
+#define Q6_Wh_vaddacc_WhVubVub(Vxx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddubh_acc)(Vxx,Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3482,7 +3481,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vub_vadd_VubVb_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddububb_sat)
+#define Q6_Vub_vadd_VubVb_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddububb_sat)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3493,7 +3492,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Ww_vaddacc_WwVuhVuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadduhw_acc)
+#define Q6_Ww_vaddacc_WwVuhVuh(Vxx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadduhw_acc)(Vxx,Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3504,7 +3503,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vuw_vadd_VuwVuw_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadduwsat)
+#define Q6_Vuw_vadd_VuwVuw_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadduwsat)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3515,7 +3514,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Wuw_vadd_WuwWuw_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadduwsat_dv)
+#define Q6_Wuw_vadd_WuwWuw_sat(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadduwsat_dv)(Vuu,Vvv)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3526,7 +3525,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_V_vand_QnR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandnqrt)
+#define Q6_V_vand_QnR(Qu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandnqrt)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qu),-1),Rt)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3537,7 +3536,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_V_vandor_VQnR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandnqrt_acc)
+#define Q6_V_vandor_VQnR(Vx,Qu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandnqrt_acc)(Vx,__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qu),-1),Rt)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3548,7 +3547,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_V_vand_QnV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvnqv)
+#define Q6_V_vand_QnV(Qv,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvnqv)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vu)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3559,7 +3558,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_V_vand_QV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvqv)
+#define Q6_V_vand_QV(Qv,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvqv)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vu)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3570,7 +3569,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vb_vasr_VhVhR_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrhbsat)
+#define Q6_Vb_vasr_VhVhR_sat(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrhbsat)(Vu,Vv,Rt)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3581,7 +3580,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vuh_vasr_VuwVuwR_rnd_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasruwuhrndsat)
+#define Q6_Vuh_vasr_VuwVuwR_rnd_sat(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasruwuhrndsat)(Vu,Vv,Rt)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3592,7 +3591,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vuh_vasr_VwVwR_rnd_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrwuhrndsat)
+#define Q6_Vuh_vasr_VwVwR_rnd_sat(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrwuhrndsat)(Vu,Vv,Rt)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3603,7 +3602,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vub_vlsr_VubR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlsrb)
+#define Q6_Vub_vlsr_VubR(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlsrb)(Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3614,7 +3613,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vb_vlut32_VbVbR_nomatch __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvvb_nm)
+#define Q6_Vb_vlut32_VbVbR_nomatch(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvvb_nm)(Vu,Vv,Rt)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3625,7 +3624,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vb_vlut32or_VbVbVbI __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvvb_oracci)
+#define Q6_Vb_vlut32or_VbVbVbI(Vx,Vu,Vv,Iu3) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvvb_oracci)(Vx,Vu,Vv,Iu3)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3636,7 +3635,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vb_vlut32_VbVbI __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvvbi)
+#define Q6_Vb_vlut32_VbVbI(Vu,Vv,Iu3) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvvbi)(Vu,Vv,Iu3)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3647,7 +3646,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Wh_vlut16_VbVhR_nomatch __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvwh_nm)
+#define Q6_Wh_vlut16_VbVhR_nomatch(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvwh_nm)(Vu,Vv,Rt)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3658,7 +3657,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Wh_vlut16or_WhVbVhI __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvwh_oracci)
+#define Q6_Wh_vlut16or_WhVbVhI(Vxx,Vu,Vv,Iu3) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvwh_oracci)(Vxx,Vu,Vv,Iu3)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3669,7 +3668,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Wh_vlut16_VbVhI __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvwhi)
+#define Q6_Wh_vlut16_VbVhI(Vu,Vv,Iu3) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvwhi)(Vu,Vv,Iu3)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3680,7 +3679,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vb_vmax_VbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmaxb)
+#define Q6_Vb_vmax_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmaxb)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3691,7 +3690,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vb_vmin_VbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vminb)
+#define Q6_Vb_vmin_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vminb)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3702,7 +3701,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Ww_vmpa_WuhRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpauhb)
+#define Q6_Ww_vmpa_WuhRb(Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpauhb)(Vuu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3713,7 +3712,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Ww_vmpaacc_WwWuhRb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpauhb_acc)
+#define Q6_Ww_vmpaacc_WwWuhRb(Vxx,Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpauhb_acc)(Vxx,Vuu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3724,7 +3723,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_W_vmpye_VwVuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyewuh_64)
+#define Q6_W_vmpye_VwVuh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyewuh_64)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3735,7 +3734,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vw_vmpyi_VwRub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiwub)
+#define Q6_Vw_vmpyi_VwRub(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiwub)(Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3746,7 +3745,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vw_vmpyiacc_VwVwRub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiwub_acc)
+#define Q6_Vw_vmpyiacc_VwVwRub(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiwub_acc)(Vx,Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3757,7 +3756,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_W_vmpyoacc_WVwVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyowh_64_acc)
+#define Q6_W_vmpyoacc_WVwVh(Vxx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyowh_64_acc)(Vxx,Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3768,7 +3767,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vub_vround_VuhVuh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrounduhub)
+#define Q6_Vub_vround_VuhVuh_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrounduhub)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3779,7 +3778,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vuh_vround_VuwVuw_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrounduwuh)
+#define Q6_Vuh_vround_VuwVuw_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrounduwuh)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3790,7 +3789,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vuh_vsat_VuwVuw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsatuwuh)
+#define Q6_Vuh_vsat_VuwVuw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsatuwuh)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3801,7 +3800,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vb_vsub_VbVb_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubbsat)
+#define Q6_Vb_vsub_VbVb_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubbsat)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3812,7 +3811,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Wb_vsub_WbWb_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubbsat_dv)
+#define Q6_Wb_vsub_WbWb_sat(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubbsat_dv)(Vuu,Vvv)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3823,7 +3822,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vw_vsub_VwVwQ_carry __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubcarry)
+#define Q6_Vw_vsub_VwVwQ_carry(Vu,Vv,Qx) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubcarry)(Vu,Vv,Qx)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3834,7 +3833,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vub_vsub_VubVb_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubububb_sat)
+#define Q6_Vub_vsub_VubVb_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubububb_sat)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3845,7 +3844,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vuw_vsub_VuwVuw_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubuwsat)
+#define Q6_Vuw_vsub_VuwVuw_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubuwsat)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 62
@@ -3856,7 +3855,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Wuw_vsub_WuwWuw_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubuwsat_dv)
+#define Q6_Wuw_vsub_WuwWuw_sat(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubuwsat_dv)(Vuu,Vvv)
 #endif /* __HEXAGON_ARCH___ >= 62 */
 
 #if __HVX_ARCH__ >= 65
@@ -3867,7 +3866,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vb_vabs_Vb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsb)
+#define Q6_Vb_vabs_Vb(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsb)(Vu)
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -3878,7 +3877,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vb_vabs_Vb_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsb_sat)
+#define Q6_Vb_vabs_Vb_sat(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsb_sat)(Vu)
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -3889,7 +3888,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vh_vaslacc_VhVhR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaslh_acc)
+#define Q6_Vh_vaslacc_VhVhR(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaslh_acc)(Vx,Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -3900,7 +3899,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vh_vasracc_VhVhR __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrh_acc)
+#define Q6_Vh_vasracc_VhVhR(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrh_acc)(Vx,Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -3911,7 +3910,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vub_vasr_VuhVuhR_rnd_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasruhubrndsat)
+#define Q6_Vub_vasr_VuhVuhR_rnd_sat(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasruhubrndsat)(Vu,Vv,Rt)
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -3922,7 +3921,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vub_vasr_VuhVuhR_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasruhubsat)
+#define Q6_Vub_vasr_VuhVuhR_sat(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasruhubsat)(Vu,Vv,Rt)
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -3933,7 +3932,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vuh_vasr_VuwVuwR_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasruwuhsat)
+#define Q6_Vuh_vasr_VuwVuwR_sat(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasruwuhsat)(Vu,Vv,Rt)
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -3944,7 +3943,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vb_vavg_VbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavgb)
+#define Q6_Vb_vavg_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavgb)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -3955,7 +3954,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vb_vavg_VbVb_rnd __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavgbrnd)
+#define Q6_Vb_vavg_VbVb_rnd(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavgbrnd)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -3966,7 +3965,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vuw_vavg_VuwVuw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavguw)
+#define Q6_Vuw_vavg_VuwVuw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavguw)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -3977,7 +3976,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vuw_vavg_VuwVuw_rnd __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavguwrnd)
+#define Q6_Vuw_vavg_VuwVuw_rnd(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavguwrnd)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -3988,7 +3987,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_W_vzero __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdd0)
+#define Q6_W_vzero() __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdd0)()
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -3999,7 +3998,7 @@
    Execution Slots:       SLOT01
    ========================================================================== */
 
-#define Q6_vgather_ARMVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgathermh)
+#define Q6_vgather_ARMVh(Rs,Rt,Mu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgathermh)(Rs,Rt,Mu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -4010,7 +4009,7 @@
    Execution Slots:       SLOT01
    ========================================================================== */
 
-#define Q6_vgather_AQRMVh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgathermhq)
+#define Q6_vgather_AQRMVh(Rs,Qs,Rt,Mu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgathermhq)(Rs,__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),Rt,Mu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -4021,7 +4020,7 @@
    Execution Slots:       SLOT01
    ========================================================================== */
 
-#define Q6_vgather_ARMWw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgathermhw)
+#define Q6_vgather_ARMWw(Rs,Rt,Mu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgathermhw)(Rs,Rt,Mu,Vvv)
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -4032,7 +4031,7 @@
    Execution Slots:       SLOT01
    ========================================================================== */
 
-#define Q6_vgather_AQRMWw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgathermhwq)
+#define Q6_vgather_AQRMWw(Rs,Qs,Rt,Mu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgathermhwq)(Rs,__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),Rt,Mu,Vvv)
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -4043,7 +4042,7 @@
    Execution Slots:       SLOT01
    ========================================================================== */
 
-#define Q6_vgather_ARMVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgathermw)
+#define Q6_vgather_ARMVw(Rs,Rt,Mu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgathermw)(Rs,Rt,Mu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -4054,7 +4053,7 @@
    Execution Slots:       SLOT01
    ========================================================================== */
 
-#define Q6_vgather_AQRMVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgathermwq)
+#define Q6_vgather_AQRMVw(Rs,Qs,Rt,Mu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgathermwq)(Rs,__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),Rt,Mu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -4065,7 +4064,7 @@
    Execution Slots:       SLOT2
    ========================================================================== */
 
-#define Q6_Vh_vlut4_VuhPh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlut4)
+#define Q6_Vh_vlut4_VuhPh(Vu,Rtt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlut4)(Vu,Rtt)
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -4076,7 +4075,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Wh_vmpa_WubRub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpabuu)
+#define Q6_Wh_vmpa_WubRub(Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpabuu)(Vuu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -4087,7 +4086,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Wh_vmpaacc_WhWubRub __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpabuu_acc)
+#define Q6_Wh_vmpaacc_WhWubRub(Vxx,Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpabuu_acc)(Vxx,Vuu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -4098,7 +4097,7 @@
    Execution Slots:       SLOT2
    ========================================================================== */
 
-#define Q6_Vh_vmpa_VhVhVhPh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpahhsat)
+#define Q6_Vh_vmpa_VhVhVhPh_sat(Vx,Vu,Rtt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpahhsat)(Vx,Vu,Rtt)
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -4109,7 +4108,7 @@
    Execution Slots:       SLOT2
    ========================================================================== */
 
-#define Q6_Vh_vmpa_VhVhVuhPuh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpauhuhsat)
+#define Q6_Vh_vmpa_VhVhVuhPuh_sat(Vx,Vu,Rtt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpauhuhsat)(Vx,Vu,Rtt)
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -4120,7 +4119,7 @@
    Execution Slots:       SLOT2
    ========================================================================== */
 
-#define Q6_Vh_vmps_VhVhVuhPuh_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpsuhuhsat)
+#define Q6_Vh_vmps_VhVhVuhPuh_sat(Vx,Vu,Rtt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpsuhuhsat)(Vx,Vu,Rtt)
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -4131,7 +4130,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Ww_vmpyacc_WwVhRh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyh_acc)
+#define Q6_Ww_vmpyacc_WwVhRh(Vxx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyh_acc)(Vxx,Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -4142,7 +4141,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vuw_vmpye_VuhRuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyuhe)
+#define Q6_Vuw_vmpye_VuhRuh(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyuhe)(Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -4153,7 +4152,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Vuw_vmpyeacc_VuwVuhRuh __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyuhe_acc)
+#define Q6_Vuw_vmpyeacc_VuwVuhRuh(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyuhe_acc)(Vx,Vu,Rt)
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -4164,7 +4163,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vb_vnavg_VbVb __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vnavgb)
+#define Q6_Vb_vnavg_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vnavgb)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -4175,7 +4174,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vb_prefixsum_Q __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vprefixqb)
+#define Q6_Vb_prefixsum_Q(Qv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vprefixqb)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1))
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -4186,7 +4185,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vh_prefixsum_Q __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vprefixqh)
+#define Q6_Vh_prefixsum_Q(Qv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vprefixqh)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1))
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -4197,7 +4196,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vw_prefixsum_Q __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vprefixqw)
+#define Q6_Vw_prefixsum_Q(Qv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vprefixqw)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1))
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -4208,7 +4207,7 @@
    Execution Slots:       SLOT0
    ========================================================================== */
 
-#define Q6_vscatter_RMVhV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermh)
+#define Q6_vscatter_RMVhV(Rt,Mu,Vv,Vw) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermh)(Rt,Mu,Vv,Vw)
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -4219,7 +4218,7 @@
    Execution Slots:       SLOT0
    ========================================================================== */
 
-#define Q6_vscatteracc_RMVhV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermh_add)
+#define Q6_vscatteracc_RMVhV(Rt,Mu,Vv,Vw) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermh_add)(Rt,Mu,Vv,Vw)
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -4230,7 +4229,7 @@
    Execution Slots:       SLOT0
    ========================================================================== */
 
-#define Q6_vscatter_QRMVhV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermhq)
+#define Q6_vscatter_QRMVhV(Qs,Rt,Mu,Vv,Vw) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermhq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),Rt,Mu,Vv,Vw)
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -4241,7 +4240,7 @@
    Execution Slots:       SLOT0
    ========================================================================== */
 
-#define Q6_vscatter_RMWwV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermhw)
+#define Q6_vscatter_RMWwV(Rt,Mu,Vvv,Vw) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermhw)(Rt,Mu,Vvv,Vw)
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -4252,7 +4251,7 @@
    Execution Slots:       SLOT0
    ========================================================================== */
 
-#define Q6_vscatteracc_RMWwV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermhw_add)
+#define Q6_vscatteracc_RMWwV(Rt,Mu,Vvv,Vw) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermhw_add)(Rt,Mu,Vvv,Vw)
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -4263,7 +4262,7 @@
    Execution Slots:       SLOT0
    ========================================================================== */
 
-#define Q6_vscatter_QRMWwV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermhwq)
+#define Q6_vscatter_QRMWwV(Qs,Rt,Mu,Vvv,Vw) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermhwq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),Rt,Mu,Vvv,Vw)
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -4274,7 +4273,7 @@
    Execution Slots:       SLOT0
    ========================================================================== */
 
-#define Q6_vscatter_RMVwV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermw)
+#define Q6_vscatter_RMVwV(Rt,Mu,Vv,Vw) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermw)(Rt,Mu,Vv,Vw)
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -4285,7 +4284,7 @@
    Execution Slots:       SLOT0
    ========================================================================== */
 
-#define Q6_vscatteracc_RMVwV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermw_add)
+#define Q6_vscatteracc_RMVwV(Rt,Mu,Vv,Vw) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermw_add)(Rt,Mu,Vv,Vw)
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 65
@@ -4296,7 +4295,7 @@
    Execution Slots:       SLOT0
    ========================================================================== */
 
-#define Q6_vscatter_QRMVwV __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermwq)
+#define Q6_vscatter_QRMVwV(Qs,Rt,Mu,Vv,Vw) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermwq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),Rt,Mu,Vv,Vw)
 #endif /* __HEXAGON_ARCH___ >= 65 */
 
 #if __HVX_ARCH__ >= 66
@@ -4307,7 +4306,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vw_vadd_VwVwQ_carry_sat __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddcarrysat)
+#define Q6_Vw_vadd_VwVwQ_carry_sat(Vu,Vv,Qs) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddcarrysat)(Vu,Vv,__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1))
 #endif /* __HEXAGON_ARCH___ >= 66 */
 
 #if __HVX_ARCH__ >= 66
@@ -4318,7 +4317,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Ww_vasrinto_WwVwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasr_into)
+#define Q6_Ww_vasrinto_WwVwVw(Vxx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasr_into)(Vxx,Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 66 */
 
 #if __HVX_ARCH__ >= 66
@@ -4329,7 +4328,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vuw_vrotr_VuwVuw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrotr)
+#define Q6_Vuw_vrotr_VuwVuw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrotr)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 66 */
 
 #if __HVX_ARCH__ >= 66
@@ -4340,7 +4339,7 @@
    Execution Slots:       SLOT0123
    ========================================================================== */
 
-#define Q6_Vw_vsatdw_VwVw __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsatdw)
+#define Q6_Vw_vsatdw_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsatdw)(Vu,Vv)
 #endif /* __HEXAGON_ARCH___ >= 66 */
 
 #if __HVX_ARCH__ >= 68
@@ -4351,7 +4350,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Ww_v6mpy_WubWbI_h __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_v6mpyhubs10)
+#define Q6_Ww_v6mpy_WubWbI_h(Vuu,Vvv,Iu2) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_v6mpyhubs10)(Vuu,Vvv,Iu2)
 #endif /* __HEXAGON_ARCH___ >= 68 */
 
 #if __HVX_ARCH__ >= 68
@@ -4362,7 +4361,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Ww_v6mpyacc_WwWubWbI_h __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_v6mpyhubs10_vxx)
+#define Q6_Ww_v6mpyacc_WwWubWbI_h(Vxx,Vuu,Vvv,Iu2) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_v6mpyhubs10_vxx)(Vxx,Vuu,Vvv,Iu2)
 #endif /* __HEXAGON_ARCH___ >= 68 */
 
 #if __HVX_ARCH__ >= 68
@@ -4373,7 +4372,7 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Ww_v6mpy_WubWbI_v __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_v6mpyvubs10)
+#define Q6_Ww_v6mpy_WubWbI_v(Vuu,Vvv,Iu2) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_v6mpyvubs10)(Vuu,Vvv,Iu2)
 #endif /* __HEXAGON_ARCH___ >= 68 */
 
 #if __HVX_ARCH__ >= 68
@@ -4384,9 +4383,801 @@
    Execution Slots:       SLOT23
    ========================================================================== */
 
-#define Q6_Ww_v6mpyacc_WwWubWbI_v __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_v6mpyvubs10_vxx)
+#define Q6_Ww_v6mpyacc_WwWubWbI_v(Vxx,Vuu,Vvv,Iu2) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_v6mpyvubs10_vxx)(Vxx,Vuu,Vvv,Iu2)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.hf=vabs(Vu32.hf)
+   C Intrinsic Prototype: HVX_Vector Q6_Vhf_vabs_Vhf(HVX_Vector Vu)
+   Instruction Type:      CVI_VX_LATE
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Vhf_vabs_Vhf(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabs_hf)(Vu)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.sf=vabs(Vu32.sf)
+   C Intrinsic Prototype: HVX_Vector Q6_Vsf_vabs_Vsf(HVX_Vector Vu)
+   Instruction Type:      CVI_VX_LATE
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Vsf_vabs_Vsf(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabs_sf)(Vu)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.qf16=vadd(Vu32.hf,Vv32.hf)
+   C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vadd_VhfVhf(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VS
+   Execution Slots:       SLOT0123
+   ========================================================================== */
+
+#define Q6_Vqf16_vadd_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadd_hf)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.hf=vadd(Vu32.hf,Vv32.hf)
+   C Intrinsic Prototype: HVX_Vector Q6_Vhf_vadd_VhfVhf(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VX
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Vhf_vadd_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadd_hf_hf)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.qf16=vadd(Vu32.qf16,Vv32.qf16)
+   C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vadd_Vqf16Vqf16(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VS
+   Execution Slots:       SLOT0123
+   ========================================================================== */
+
+#define Q6_Vqf16_vadd_Vqf16Vqf16(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadd_qf16)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.qf16=vadd(Vu32.qf16,Vv32.hf)
+   C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vadd_Vqf16Vhf(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VS
+   Execution Slots:       SLOT0123
+   ========================================================================== */
+
+#define Q6_Vqf16_vadd_Vqf16Vhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadd_qf16_mix)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.qf32=vadd(Vu32.qf32,Vv32.qf32)
+   C Intrinsic Prototype: HVX_Vector Q6_Vqf32_vadd_Vqf32Vqf32(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VS
+   Execution Slots:       SLOT0123
+   ========================================================================== */
+
+#define Q6_Vqf32_vadd_Vqf32Vqf32(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadd_qf32)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.qf32=vadd(Vu32.qf32,Vv32.sf)
+   C Intrinsic Prototype: HVX_Vector Q6_Vqf32_vadd_Vqf32Vsf(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VS
+   Execution Slots:       SLOT0123
+   ========================================================================== */
+
+#define Q6_Vqf32_vadd_Vqf32Vsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadd_qf32_mix)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.qf32=vadd(Vu32.sf,Vv32.sf)
+   C Intrinsic Prototype: HVX_Vector Q6_Vqf32_vadd_VsfVsf(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VS
+   Execution Slots:       SLOT0123
+   ========================================================================== */
+
+#define Q6_Vqf32_vadd_VsfVsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadd_sf)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vdd32.sf=vadd(Vu32.hf,Vv32.hf)
+   C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vadd_VhfVhf(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VX_DV
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Wsf_vadd_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadd_sf_hf)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.sf=vadd(Vu32.sf,Vv32.sf)
+   C Intrinsic Prototype: HVX_Vector Q6_Vsf_vadd_VsfVsf(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VX
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Vsf_vadd_VsfVsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadd_sf_sf)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.w=vfmv(Vu32.w)
+   C Intrinsic Prototype: HVX_Vector Q6_Vw_vfmv_Vw(HVX_Vector Vu)
+   Instruction Type:      CVI_VX_LATE
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Vw_vfmv_Vw(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vassign_fp)(Vu)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.hf=Vu32.qf16
+   C Intrinsic Prototype: HVX_Vector Q6_Vhf_equals_Vqf16(HVX_Vector Vu)
+   Instruction Type:      CVI_VS
+   Execution Slots:       SLOT0123
+   ========================================================================== */
+
+#define Q6_Vhf_equals_Vqf16(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vconv_hf_qf16)(Vu)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.hf=Vuu32.qf32
+   C Intrinsic Prototype: HVX_Vector Q6_Vhf_equals_Wqf32(HVX_VectorPair Vuu)
+   Instruction Type:      CVI_VS
+   Execution Slots:       SLOT0123
+   ========================================================================== */
+
+#define Q6_Vhf_equals_Wqf32(Vuu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vconv_hf_qf32)(Vuu)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.sf=Vu32.qf32
+   C Intrinsic Prototype: HVX_Vector Q6_Vsf_equals_Vqf32(HVX_Vector Vu)
+   Instruction Type:      CVI_VS
+   Execution Slots:       SLOT0123
+   ========================================================================== */
+
+#define Q6_Vsf_equals_Vqf32(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vconv_sf_qf32)(Vu)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.b=vcvt(Vu32.hf,Vv32.hf)
+   C Intrinsic Prototype: HVX_Vector Q6_Vb_vcvt_VhfVhf(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VX
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Vb_vcvt_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_b_hf)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.h=vcvt(Vu32.hf)
+   C Intrinsic Prototype: HVX_Vector Q6_Vh_vcvt_Vhf(HVX_Vector Vu)
+   Instruction Type:      CVI_VX
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Vh_vcvt_Vhf(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_h_hf)(Vu)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vdd32.hf=vcvt(Vu32.b)
+   C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vcvt_Vb(HVX_Vector Vu)
+   Instruction Type:      CVI_VX_DV
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Whf_vcvt_Vb(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_hf_b)(Vu)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.hf=vcvt(Vu32.h)
+   C Intrinsic Prototype: HVX_Vector Q6_Vhf_vcvt_Vh(HVX_Vector Vu)
+   Instruction Type:      CVI_VX
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Vhf_vcvt_Vh(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_hf_h)(Vu)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.hf=vcvt(Vu32.sf,Vv32.sf)
+   C Intrinsic Prototype: HVX_Vector Q6_Vhf_vcvt_VsfVsf(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VX
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Vhf_vcvt_VsfVsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_hf_sf)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vdd32.hf=vcvt(Vu32.ub)
+   C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vcvt_Vub(HVX_Vector Vu)
+   Instruction Type:      CVI_VX_DV
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Whf_vcvt_Vub(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_hf_ub)(Vu)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.hf=vcvt(Vu32.uh)
+   C Intrinsic Prototype: HVX_Vector Q6_Vhf_vcvt_Vuh(HVX_Vector Vu)
+   Instruction Type:      CVI_VX
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Vhf_vcvt_Vuh(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_hf_uh)(Vu)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vdd32.sf=vcvt(Vu32.hf)
+   C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vcvt_Vhf(HVX_Vector Vu)
+   Instruction Type:      CVI_VX_DV
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Wsf_vcvt_Vhf(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_sf_hf)(Vu)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.ub=vcvt(Vu32.hf,Vv32.hf)
+   C Intrinsic Prototype: HVX_Vector Q6_Vub_vcvt_VhfVhf(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VX
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Vub_vcvt_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_ub_hf)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.uh=vcvt(Vu32.hf)
+   C Intrinsic Prototype: HVX_Vector Q6_Vuh_vcvt_Vhf(HVX_Vector Vu)
+   Instruction Type:      CVI_VX
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Vuh_vcvt_Vhf(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_uh_hf)(Vu)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.sf=vdmpy(Vu32.hf,Vv32.hf)
+   C Intrinsic Prototype: HVX_Vector Q6_Vsf_vdmpy_VhfVhf(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VX
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Vsf_vdmpy_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpy_sf_hf)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vx32.sf+=vdmpy(Vu32.hf,Vv32.hf)
+   C Intrinsic Prototype: HVX_Vector Q6_Vsf_vdmpyacc_VsfVhfVhf(HVX_Vector Vx, HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VX
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Vsf_vdmpyacc_VsfVhfVhf(Vx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpy_sf_hf_acc)(Vx,Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.hf=vfmax(Vu32.hf,Vv32.hf)
+   C Intrinsic Prototype: HVX_Vector Q6_Vhf_vfmax_VhfVhf(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VX_LATE
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Vhf_vfmax_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vfmax_hf)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.sf=vfmax(Vu32.sf,Vv32.sf)
+   C Intrinsic Prototype: HVX_Vector Q6_Vsf_vfmax_VsfVsf(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VX_LATE
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Vsf_vfmax_VsfVsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vfmax_sf)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.hf=vfmin(Vu32.hf,Vv32.hf)
+   C Intrinsic Prototype: HVX_Vector Q6_Vhf_vfmin_VhfVhf(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VX_LATE
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Vhf_vfmin_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vfmin_hf)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.sf=vfmin(Vu32.sf,Vv32.sf)
+   C Intrinsic Prototype: HVX_Vector Q6_Vsf_vfmin_VsfVsf(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VX_LATE
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Vsf_vfmin_VsfVsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vfmin_sf)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.hf=vfneg(Vu32.hf)
+   C Intrinsic Prototype: HVX_Vector Q6_Vhf_vfneg_Vhf(HVX_Vector Vu)
+   Instruction Type:      CVI_VX_LATE
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Vhf_vfneg_Vhf(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vfneg_hf)(Vu)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.sf=vfneg(Vu32.sf)
+   C Intrinsic Prototype: HVX_Vector Q6_Vsf_vfneg_Vsf(HVX_Vector Vu)
+   Instruction Type:      CVI_VX_LATE
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Vsf_vfneg_Vsf(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vfneg_sf)(Vu)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Qd4=vcmp.gt(Vu32.hf,Vv32.hf)
+   C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gt_VhfVhf(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VA
+   Execution Slots:       SLOT0123
+   ========================================================================== */
+
+#define Q6_Q_vcmp_gt_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgthf)(Vu,Vv)),-1)
 #endif /* __HEXAGON_ARCH___ >= 68 */
 
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Qx4&=vcmp.gt(Vu32.hf,Vv32.hf)
+   C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtand_QVhfVhf(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VA
+   Execution Slots:       SLOT0123
+   ========================================================================== */
+
+#define Q6_Q_vcmp_gtand_QVhfVhf(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgthf_and)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Qx4|=vcmp.gt(Vu32.hf,Vv32.hf)
+   C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtor_QVhfVhf(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VA
+   Execution Slots:       SLOT0123
+   ========================================================================== */
+
+#define Q6_Q_vcmp_gtor_QVhfVhf(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgthf_or)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Qx4^=vcmp.gt(Vu32.hf,Vv32.hf)
+   C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtxacc_QVhfVhf(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VA
+   Execution Slots:       SLOT0123
+   ========================================================================== */
+
+#define Q6_Q_vcmp_gtxacc_QVhfVhf(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgthf_xor)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Qd4=vcmp.gt(Vu32.sf,Vv32.sf)
+   C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gt_VsfVsf(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VA
+   Execution Slots:       SLOT0123
+   ========================================================================== */
+
+#define Q6_Q_vcmp_gt_VsfVsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtsf)(Vu,Vv)),-1)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Qx4&=vcmp.gt(Vu32.sf,Vv32.sf)
+   C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtand_QVsfVsf(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VA
+   Execution Slots:       SLOT0123
+   ========================================================================== */
+
+#define Q6_Q_vcmp_gtand_QVsfVsf(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtsf_and)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Qx4|=vcmp.gt(Vu32.sf,Vv32.sf)
+   C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtor_QVsfVsf(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VA
+   Execution Slots:       SLOT0123
+   ========================================================================== */
+
+#define Q6_Q_vcmp_gtor_QVsfVsf(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtsf_or)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Qx4^=vcmp.gt(Vu32.sf,Vv32.sf)
+   C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtxacc_QVsfVsf(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VA
+   Execution Slots:       SLOT0123
+   ========================================================================== */
+
+#define Q6_Q_vcmp_gtxacc_QVsfVsf(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtsf_xor)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.hf=vmax(Vu32.hf,Vv32.hf)
+   C Intrinsic Prototype: HVX_Vector Q6_Vhf_vmax_VhfVhf(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VA
+   Execution Slots:       SLOT0123
+   ========================================================================== */
+
+#define Q6_Vhf_vmax_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmax_hf)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.sf=vmax(Vu32.sf,Vv32.sf)
+   C Intrinsic Prototype: HVX_Vector Q6_Vsf_vmax_VsfVsf(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VA
+   Execution Slots:       SLOT0123
+   ========================================================================== */
+
+#define Q6_Vsf_vmax_VsfVsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmax_sf)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.hf=vmin(Vu32.hf,Vv32.hf)
+   C Intrinsic Prototype: HVX_Vector Q6_Vhf_vmin_VhfVhf(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VA
+   Execution Slots:       SLOT0123
+   ========================================================================== */
+
+#define Q6_Vhf_vmin_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmin_hf)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.sf=vmin(Vu32.sf,Vv32.sf)
+   C Intrinsic Prototype: HVX_Vector Q6_Vsf_vmin_VsfVsf(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VA
+   Execution Slots:       SLOT0123
+   ========================================================================== */
+
+#define Q6_Vsf_vmin_VsfVsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmin_sf)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.hf=vmpy(Vu32.hf,Vv32.hf)
+   C Intrinsic Prototype: HVX_Vector Q6_Vhf_vmpy_VhfVhf(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VX
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Vhf_vmpy_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_hf_hf)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vx32.hf+=vmpy(Vu32.hf,Vv32.hf)
+   C Intrinsic Prototype: HVX_Vector Q6_Vhf_vmpyacc_VhfVhfVhf(HVX_Vector Vx, HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VX
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Vhf_vmpyacc_VhfVhfVhf(Vx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_hf_hf_acc)(Vx,Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.qf16=vmpy(Vu32.qf16,Vv32.qf16)
+   C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vmpy_Vqf16Vqf16(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VX_DV
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Vqf16_vmpy_Vqf16Vqf16(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_qf16)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.qf16=vmpy(Vu32.hf,Vv32.hf)
+   C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vmpy_VhfVhf(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VX_DV
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Vqf16_vmpy_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_qf16_hf)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.qf16=vmpy(Vu32.qf16,Vv32.hf)
+   C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vmpy_Vqf16Vhf(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VX_DV
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Vqf16_vmpy_Vqf16Vhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_qf16_mix_hf)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.qf32=vmpy(Vu32.qf32,Vv32.qf32)
+   C Intrinsic Prototype: HVX_Vector Q6_Vqf32_vmpy_Vqf32Vqf32(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VX_DV
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Vqf32_vmpy_Vqf32Vqf32(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_qf32)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vdd32.qf32=vmpy(Vu32.hf,Vv32.hf)
+   C Intrinsic Prototype: HVX_VectorPair Q6_Wqf32_vmpy_VhfVhf(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VX_DV
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Wqf32_vmpy_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_qf32_hf)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vdd32.qf32=vmpy(Vu32.qf16,Vv32.hf)
+   C Intrinsic Prototype: HVX_VectorPair Q6_Wqf32_vmpy_Vqf16Vhf(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VX_DV
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Wqf32_vmpy_Vqf16Vhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_qf32_mix_hf)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vdd32.qf32=vmpy(Vu32.qf16,Vv32.qf16)
+   C Intrinsic Prototype: HVX_VectorPair Q6_Wqf32_vmpy_Vqf16Vqf16(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VX_DV
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Wqf32_vmpy_Vqf16Vqf16(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_qf32_qf16)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.qf32=vmpy(Vu32.sf,Vv32.sf)
+   C Intrinsic Prototype: HVX_Vector Q6_Vqf32_vmpy_VsfVsf(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VX_DV
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Vqf32_vmpy_VsfVsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_qf32_sf)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vdd32.sf=vmpy(Vu32.hf,Vv32.hf)
+   C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vmpy_VhfVhf(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VX_DV
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Wsf_vmpy_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_sf_hf)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vxx32.sf+=vmpy(Vu32.hf,Vv32.hf)
+   C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vmpyacc_WsfVhfVhf(HVX_VectorPair Vxx, HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VX_DV
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Wsf_vmpyacc_WsfVhfVhf(Vxx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_sf_hf_acc)(Vxx,Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.sf=vmpy(Vu32.sf,Vv32.sf)
+   C Intrinsic Prototype: HVX_Vector Q6_Vsf_vmpy_VsfVsf(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VX_DV
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Vsf_vmpy_VsfVsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_sf_sf)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.qf16=vsub(Vu32.hf,Vv32.hf)
+   C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vsub_VhfVhf(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VS
+   Execution Slots:       SLOT0123
+   ========================================================================== */
+
+#define Q6_Vqf16_vsub_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsub_hf)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.hf=vsub(Vu32.hf,Vv32.hf)
+   C Intrinsic Prototype: HVX_Vector Q6_Vhf_vsub_VhfVhf(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VX
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Vhf_vsub_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsub_hf_hf)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.qf16=vsub(Vu32.qf16,Vv32.qf16)
+   C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vsub_Vqf16Vqf16(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VS
+   Execution Slots:       SLOT0123
+   ========================================================================== */
+
+#define Q6_Vqf16_vsub_Vqf16Vqf16(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsub_qf16)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.qf16=vsub(Vu32.qf16,Vv32.hf)
+   C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vsub_Vqf16Vhf(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VS
+   Execution Slots:       SLOT0123
+   ========================================================================== */
+
+#define Q6_Vqf16_vsub_Vqf16Vhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsub_qf16_mix)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.qf32=vsub(Vu32.qf32,Vv32.qf32)
+   C Intrinsic Prototype: HVX_Vector Q6_Vqf32_vsub_Vqf32Vqf32(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VS
+   Execution Slots:       SLOT0123
+   ========================================================================== */
+
+#define Q6_Vqf32_vsub_Vqf32Vqf32(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsub_qf32)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.qf32=vsub(Vu32.qf32,Vv32.sf)
+   C Intrinsic Prototype: HVX_Vector Q6_Vqf32_vsub_Vqf32Vsf(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VS
+   Execution Slots:       SLOT0123
+   ========================================================================== */
+
+#define Q6_Vqf32_vsub_Vqf32Vsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsub_qf32_mix)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.qf32=vsub(Vu32.sf,Vv32.sf)
+   C Intrinsic Prototype: HVX_Vector Q6_Vqf32_vsub_VsfVsf(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VS
+   Execution Slots:       SLOT0123
+   ========================================================================== */
+
+#define Q6_Vqf32_vsub_VsfVsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsub_sf)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vdd32.sf=vsub(Vu32.hf,Vv32.hf)
+   C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vsub_VhfVhf(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VX_DV
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Wsf_vsub_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsub_sf_hf)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 68
+/* ==========================================================================
+   Assembly Syntax:       Vd32.sf=vsub(Vu32.sf,Vv32.sf)
+   C Intrinsic Prototype: HVX_Vector Q6_Vsf_vsub_VsfVsf(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VX
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Vsf_vsub_VsfVsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsub_sf_sf)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 68 */
+
+#if __HVX_ARCH__ >= 69
+/* ==========================================================================
+   Assembly Syntax:       Vd32.ub=vasr(Vuu32.uh,Vv32.ub):rnd:sat
+   C Intrinsic Prototype: HVX_Vector Q6_Vub_vasr_WuhVub_rnd_sat(HVX_VectorPair Vuu, HVX_Vector Vv)
+   Instruction Type:      CVI_VS
+   Execution Slots:       SLOT0123
+   ========================================================================== */
+
+#define Q6_Vub_vasr_WuhVub_rnd_sat(Vuu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrvuhubrndsat)(Vuu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 69 */
+
+#if __HVX_ARCH__ >= 69
+/* ==========================================================================
+   Assembly Syntax:       Vd32.ub=vasr(Vuu32.uh,Vv32.ub):sat
+   C Intrinsic Prototype: HVX_Vector Q6_Vub_vasr_WuhVub_sat(HVX_VectorPair Vuu, HVX_Vector Vv)
+   Instruction Type:      CVI_VS
+   Execution Slots:       SLOT0123
+   ========================================================================== */
+
+#define Q6_Vub_vasr_WuhVub_sat(Vuu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrvuhubsat)(Vuu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 69 */
+
+#if __HVX_ARCH__ >= 69
+/* ==========================================================================
+   Assembly Syntax:       Vd32.uh=vasr(Vuu32.w,Vv32.uh):rnd:sat
+   C Intrinsic Prototype: HVX_Vector Q6_Vuh_vasr_WwVuh_rnd_sat(HVX_VectorPair Vuu, HVX_Vector Vv)
+   Instruction Type:      CVI_VS
+   Execution Slots:       SLOT0123
+   ========================================================================== */
+
+#define Q6_Vuh_vasr_WwVuh_rnd_sat(Vuu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrvwuhrndsat)(Vuu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 69 */
+
+#if __HVX_ARCH__ >= 69
+/* ==========================================================================
+   Assembly Syntax:       Vd32.uh=vasr(Vuu32.w,Vv32.uh):sat
+   C Intrinsic Prototype: HVX_Vector Q6_Vuh_vasr_WwVuh_sat(HVX_VectorPair Vuu, HVX_Vector Vv)
+   Instruction Type:      CVI_VS
+   Execution Slots:       SLOT0123
+   ========================================================================== */
+
+#define Q6_Vuh_vasr_WwVuh_sat(Vuu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrvwuhsat)(Vuu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 69 */
+
+#if __HVX_ARCH__ >= 69
+/* ==========================================================================
+   Assembly Syntax:       Vd32.uh=vmpy(Vu32.uh,Vv32.uh):>>16
+   C Intrinsic Prototype: HVX_Vector Q6_Vuh_vmpy_VuhVuh_rs16(HVX_Vector Vu, HVX_Vector Vv)
+   Instruction Type:      CVI_VX
+   Execution Slots:       SLOT23
+   ========================================================================== */
+
+#define Q6_Vuh_vmpy_VuhVuh_rs16(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyuhvs)(Vu,Vv)
+#endif /* __HEXAGON_ARCH___ >= 69 */
+
 #endif /* __HVX__ */
 
 #endif
diff --git a/clang/lib/Headers/opencl-c.h b/clang/lib/Headers/opencl-c.h
index 32af848a94c4..77a7a8b9bb3a 100644
--- a/clang/lib/Headers/opencl-c.h
+++ b/clang/lib/Headers/opencl-c.h
@@ -11190,305 +11190,305 @@ half16 __ovld __cnfn select(half16 a, half16 b, ushort16 c);
  * 64-bit aligned if gentype is long, ulong, double.
  */
 
-char2 __ovld vload2(size_t offset, const __constant char *p);
-uchar2 __ovld vload2(size_t offset, const __constant uchar *p);
-short2 __ovld vload2(size_t offset, const __constant short *p);
-ushort2 __ovld vload2(size_t offset, const __constant ushort *p);
-int2 __ovld vload2(size_t offset, const __constant int *p);
-uint2 __ovld vload2(size_t offset, const __constant uint *p);
-long2 __ovld vload2(size_t offset, const __constant long *p);
-ulong2 __ovld vload2(size_t offset, const __constant ulong *p);
-float2 __ovld vload2(size_t offset, const __constant float *p);
-char3 __ovld vload3(size_t offset, const __constant char *p);
-uchar3 __ovld vload3(size_t offset, const __constant uchar *p);
-short3 __ovld vload3(size_t offset, const __constant short *p);
-ushort3 __ovld vload3(size_t offset, const __constant ushort *p);
-int3 __ovld vload3(size_t offset, const __constant int *p);
-uint3 __ovld vload3(size_t offset, const __constant uint *p);
-long3 __ovld vload3(size_t offset, const __constant long *p);
-ulong3 __ovld vload3(size_t offset, const __constant ulong *p);
-float3 __ovld vload3(size_t offset, const __constant float *p);
-char4 __ovld vload4(size_t offset, const __constant char *p);
-uchar4 __ovld vload4(size_t offset, const __constant uchar *p);
-short4 __ovld vload4(size_t offset, const __constant short *p);
-ushort4 __ovld vload4(size_t offset, const __constant ushort *p);
-int4 __ovld vload4(size_t offset, const __constant int *p);
-uint4 __ovld vload4(size_t offset, const __constant uint *p);
-long4 __ovld vload4(size_t offset, const __constant long *p);
-ulong4 __ovld vload4(size_t offset, const __constant ulong *p);
-float4 __ovld vload4(size_t offset, const __constant float *p);
-char8 __ovld vload8(size_t offset, const __constant char *p);
-uchar8 __ovld vload8(size_t offset, const __constant uchar *p);
-short8 __ovld vload8(size_t offset, const __constant short *p);
-ushort8 __ovld vload8(size_t offset, const __constant ushort *p);
-int8 __ovld vload8(size_t offset, const __constant int *p);
-uint8 __ovld vload8(size_t offset, const __constant uint *p);
-long8 __ovld vload8(size_t offset, const __constant long *p);
-ulong8 __ovld vload8(size_t offset, const __constant ulong *p);
-float8 __ovld vload8(size_t offset, const __constant float *p);
-char16 __ovld vload16(size_t offset, const __constant char *p);
-uchar16 __ovld vload16(size_t offset, const __constant uchar *p);
-short16 __ovld vload16(size_t offset, const __constant short *p);
-ushort16 __ovld vload16(size_t offset, const __constant ushort *p);
-int16 __ovld vload16(size_t offset, const __constant int *p);
-uint16 __ovld vload16(size_t offset, const __constant uint *p);
-long16 __ovld vload16(size_t offset, const __constant long *p);
-ulong16 __ovld vload16(size_t offset, const __constant ulong *p);
-float16 __ovld vload16(size_t offset, const __constant float *p);
+char2 __ovld __purefn vload2(size_t offset, const __constant char *p);
+uchar2 __ovld __purefn vload2(size_t offset, const __constant uchar *p);
+short2 __ovld __purefn vload2(size_t offset, const __constant short *p);
+ushort2 __ovld __purefn vload2(size_t offset, const __constant ushort *p);
+int2 __ovld __purefn vload2(size_t offset, const __constant int *p);
+uint2 __ovld __purefn vload2(size_t offset, const __constant uint *p);
+long2 __ovld __purefn vload2(size_t offset, const __constant long *p);
+ulong2 __ovld __purefn vload2(size_t offset, const __constant ulong *p);
+float2 __ovld __purefn vload2(size_t offset, const __constant float *p);
+char3 __ovld __purefn vload3(size_t offset, const __constant char *p);
+uchar3 __ovld __purefn vload3(size_t offset, const __constant uchar *p);
+short3 __ovld __purefn vload3(size_t offset, const __constant short *p);
+ushort3 __ovld __purefn vload3(size_t offset, const __constant ushort *p);
+int3 __ovld __purefn vload3(size_t offset, const __constant int *p);
+uint3 __ovld __purefn vload3(size_t offset, const __constant uint *p);
+long3 __ovld __purefn vload3(size_t offset, const __constant long *p);
+ulong3 __ovld __purefn vload3(size_t offset, const __constant ulong *p);
+float3 __ovld __purefn vload3(size_t offset, const __constant float *p);
+char4 __ovld __purefn vload4(size_t offset, const __constant char *p);
+uchar4 __ovld __purefn vload4(size_t offset, const __constant uchar *p);
+short4 __ovld __purefn vload4(size_t offset, const __constant short *p);
+ushort4 __ovld __purefn vload4(size_t offset, const __constant ushort *p);
+int4 __ovld __purefn vload4(size_t offset, const __constant int *p);
+uint4 __ovld __purefn vload4(size_t offset, const __constant uint *p);
+long4 __ovld __purefn vload4(size_t offset, const __constant long *p);
+ulong4 __ovld __purefn vload4(size_t offset, const __constant ulong *p);
+float4 __ovld __purefn vload4(size_t offset, const __constant float *p);
+char8 __ovld __purefn vload8(size_t offset, const __constant char *p);
+uchar8 __ovld __purefn vload8(size_t offset, const __constant uchar *p);
+short8 __ovld __purefn vload8(size_t offset, const __constant short *p);
+ushort8 __ovld __purefn vload8(size_t offset, const __constant ushort *p);
+int8 __ovld __purefn vload8(size_t offset, const __constant int *p);
+uint8 __ovld __purefn vload8(size_t offset, const __constant uint *p);
+long8 __ovld __purefn vload8(size_t offset, const __constant long *p);
+ulong8 __ovld __purefn vload8(size_t offset, const __constant ulong *p);
+float8 __ovld __purefn vload8(size_t offset, const __constant float *p);
+char16 __ovld __purefn vload16(size_t offset, const __constant char *p);
+uchar16 __ovld __purefn vload16(size_t offset, const __constant uchar *p);
+short16 __ovld __purefn vload16(size_t offset, const __constant short *p);
+ushort16 __ovld __purefn vload16(size_t offset, const __constant ushort *p);
+int16 __ovld __purefn vload16(size_t offset, const __constant int *p);
+uint16 __ovld __purefn vload16(size_t offset, const __constant uint *p);
+long16 __ovld __purefn vload16(size_t offset, const __constant long *p);
+ulong16 __ovld __purefn vload16(size_t offset, const __constant ulong *p);
+float16 __ovld __purefn vload16(size_t offset, const __constant float *p);
 #ifdef cl_khr_fp64
-double2 __ovld vload2(size_t offset, const __constant double *p);
-double3 __ovld vload3(size_t offset, const __constant double *p);
-double4 __ovld vload4(size_t offset, const __constant double *p);
-double8 __ovld vload8(size_t offset, const __constant double *p);
-double16 __ovld vload16(size_t offset, const __constant double *p);
+double2 __ovld __purefn vload2(size_t offset, const __constant double *p);
+double3 __ovld __purefn vload3(size_t offset, const __constant double *p);
+double4 __ovld __purefn vload4(size_t offset, const __constant double *p);
+double8 __ovld __purefn vload8(size_t offset, const __constant double *p);
+double16 __ovld __purefn vload16(size_t offset, const __constant double *p);
 #endif //cl_khr_fp64
 
 #ifdef cl_khr_fp16
-half __ovld vload(size_t offset, const __constant half *p);
-half2 __ovld vload2(size_t offset, const __constant half *p);
-half3 __ovld vload3(size_t offset, const __constant half *p);
-half4 __ovld vload4(size_t offset, const __constant half *p);
-half8 __ovld vload8(size_t offset, const __constant half *p);
-half16 __ovld vload16(size_t offset, const __constant half *p);
+half __ovld __purefn vload(size_t offset, const __constant half *p);
+half2 __ovld __purefn vload2(size_t offset, const __constant half *p);
+half3 __ovld __purefn vload3(size_t offset, const __constant half *p);
+half4 __ovld __purefn vload4(size_t offset, const __constant half *p);
+half8 __ovld __purefn vload8(size_t offset, const __constant half *p);
+half16 __ovld __purefn vload16(size_t offset, const __constant half *p);
 #endif //cl_khr_fp16
 
 #if defined(__opencl_c_generic_address_space)
-char2 __ovld vload2(size_t offset, const char *p);
-uchar2 __ovld vload2(size_t offset, const uchar *p);
-short2 __ovld vload2(size_t offset, const short *p);
-ushort2 __ovld vload2(size_t offset, const ushort *p);
-int2 __ovld vload2(size_t offset, const int *p);
-uint2 __ovld vload2(size_t offset, const uint *p);
-long2 __ovld vload2(size_t offset, const long *p);
-ulong2 __ovld vload2(size_t offset, const ulong *p);
-float2 __ovld vload2(size_t offset, const float *p);
-char3 __ovld vload3(size_t offset, const char *p);
-uchar3 __ovld vload3(size_t offset, const uchar *p);
-short3 __ovld vload3(size_t offset, const short *p);
-ushort3 __ovld vload3(size_t offset, const ushort *p);
-int3 __ovld vload3(size_t offset, const int *p);
-uint3 __ovld vload3(size_t offset, const uint *p);
-long3 __ovld vload3(size_t offset, const long *p);
-ulong3 __ovld vload3(size_t offset, const ulong *p);
-float3 __ovld vload3(size_t offset, const float *p);
-char4 __ovld vload4(size_t offset, const char *p);
-uchar4 __ovld vload4(size_t offset, const uchar *p);
-short4 __ovld vload4(size_t offset, const short *p);
-ushort4 __ovld vload4(size_t offset, const ushort *p);
-int4 __ovld vload4(size_t offset, const int *p);
-uint4 __ovld vload4(size_t offset, const uint *p);
-long4 __ovld vload4(size_t offset, const long *p);
-ulong4 __ovld vload4(size_t offset, const ulong *p);
-float4 __ovld vload4(size_t offset, const float *p);
-char8 __ovld vload8(size_t offset, const char *p);
-uchar8 __ovld vload8(size_t offset, const uchar *p);
-short8 __ovld vload8(size_t offset, const short *p);
-ushort8 __ovld vload8(size_t offset, const ushort *p);
-int8 __ovld vload8(size_t offset, const int *p);
-uint8 __ovld vload8(size_t offset, const uint *p);
-long8 __ovld vload8(size_t offset, const long *p);
-ulong8 __ovld vload8(size_t offset, const ulong *p);
-float8 __ovld vload8(size_t offset, const float *p);
-char16 __ovld vload16(size_t offset, const char *p);
-uchar16 __ovld vload16(size_t offset, const uchar *p);
-short16 __ovld vload16(size_t offset, const short *p);
-ushort16 __ovld vload16(size_t offset, const ushort *p);
-int16 __ovld vload16(size_t offset, const int *p);
-uint16 __ovld vload16(size_t offset, const uint *p);
-long16 __ovld vload16(size_t offset, const long *p);
-ulong16 __ovld vload16(size_t offset, const ulong *p);
-float16 __ovld vload16(size_t offset, const float *p);
+char2 __ovld __purefn vload2(size_t offset, const char *p);
+uchar2 __ovld __purefn vload2(size_t offset, const uchar *p);
+short2 __ovld __purefn vload2(size_t offset, const short *p);
+ushort2 __ovld __purefn vload2(size_t offset, const ushort *p);
+int2 __ovld __purefn vload2(size_t offset, const int *p);
+uint2 __ovld __purefn vload2(size_t offset, const uint *p);
+long2 __ovld __purefn vload2(size_t offset, const long *p);
+ulong2 __ovld __purefn vload2(size_t offset, const ulong *p);
+float2 __ovld __purefn vload2(size_t offset, const float *p);
+char3 __ovld __purefn vload3(size_t offset, const char *p);
+uchar3 __ovld __purefn vload3(size_t offset, const uchar *p);
+short3 __ovld __purefn vload3(size_t offset, const short *p);
+ushort3 __ovld __purefn vload3(size_t offset, const ushort *p);
+int3 __ovld __purefn vload3(size_t offset, const int *p);
+uint3 __ovld __purefn vload3(size_t offset, const uint *p);
+long3 __ovld __purefn vload3(size_t offset, const long *p);
+ulong3 __ovld __purefn vload3(size_t offset, const ulong *p);
+float3 __ovld __purefn vload3(size_t offset, const float *p);
+char4 __ovld __purefn vload4(size_t offset, const char *p);
+uchar4 __ovld __purefn vload4(size_t offset, const uchar *p);
+short4 __ovld __purefn vload4(size_t offset, const short *p);
+ushort4 __ovld __purefn vload4(size_t offset, const ushort *p);
+int4 __ovld __purefn vload4(size_t offset, const int *p);
+uint4 __ovld __purefn vload4(size_t offset, const uint *p);
+long4 __ovld __purefn vload4(size_t offset, const long *p);
+ulong4 __ovld __purefn vload4(size_t offset, const ulong *p);
+float4 __ovld __purefn vload4(size_t offset, const float *p);
+char8 __ovld __purefn vload8(size_t offset, const char *p);
+uchar8 __ovld __purefn vload8(size_t offset, const uchar *p);
+short8 __ovld __purefn vload8(size_t offset, const short *p);
+ushort8 __ovld __purefn vload8(size_t offset, const ushort *p);
+int8 __ovld __purefn vload8(size_t offset, const int *p);
+uint8 __ovld __purefn vload8(size_t offset, const uint *p);
+long8 __ovld __purefn vload8(size_t offset, const long *p);
+ulong8 __ovld __purefn vload8(size_t offset, const ulong *p);
+float8 __ovld __purefn vload8(size_t offset, const float *p);
+char16 __ovld __purefn vload16(size_t offset, const char *p);
+uchar16 __ovld __purefn vload16(size_t offset, const uchar *p);
+short16 __ovld __purefn vload16(size_t offset, const short *p);
+ushort16 __ovld __purefn vload16(size_t offset, const ushort *p);
+int16 __ovld __purefn vload16(size_t offset, const int *p);
+uint16 __ovld __purefn vload16(size_t offset, const uint *p);
+long16 __ovld __purefn vload16(size_t offset, const long *p);
+ulong16 __ovld __purefn vload16(size_t offset, const ulong *p);
+float16 __ovld __purefn vload16(size_t offset, const float *p);
 
 #ifdef cl_khr_fp64
-double2 __ovld vload2(size_t offset, const double *p);
-double3 __ovld vload3(size_t offset, const double *p);
-double4 __ovld vload4(size_t offset, const double *p);
-double8 __ovld vload8(size_t offset, const double *p);
-double16 __ovld vload16(size_t offset, const double *p);
+double2 __ovld __purefn vload2(size_t offset, const double *p);
+double3 __ovld __purefn vload3(size_t offset, const double *p);
+double4 __ovld __purefn vload4(size_t offset, const double *p);
+double8 __ovld __purefn vload8(size_t offset, const double *p);
+double16 __ovld __purefn vload16(size_t offset, const double *p);
 #endif //cl_khr_fp64
 
 #ifdef cl_khr_fp16
-half __ovld vload(size_t offset, const half *p);
-half2 __ovld vload2(size_t offset, const half *p);
-half3 __ovld vload3(size_t offset, const half *p);
-half4 __ovld vload4(size_t offset, const half *p);
-half8 __ovld vload8(size_t offset, const half *p);
-half16 __ovld vload16(size_t offset, const half *p);
+half __ovld __purefn vload(size_t offset, const half *p);
+half2 __ovld __purefn vload2(size_t offset, const half *p);
+half3 __ovld __purefn vload3(size_t offset, const half *p);
+half4 __ovld __purefn vload4(size_t offset, const half *p);
+half8 __ovld __purefn vload8(size_t offset, const half *p);
+half16 __ovld __purefn vload16(size_t offset, const half *p);
 #endif //cl_khr_fp16
 #else
-char2 __ovld vload2(size_t offset, const __global char *p);
-uchar2 __ovld vload2(size_t offset, const __global uchar *p);
-short2 __ovld vload2(size_t offset, const __global short *p);
-ushort2 __ovld vload2(size_t offset, const __global ushort *p);
-int2 __ovld vload2(size_t offset, const __global int *p);
-uint2 __ovld vload2(size_t offset, const __global uint *p);
-long2 __ovld vload2(size_t offset, const __global long *p);
-ulong2 __ovld vload2(size_t offset, const __global ulong *p);
-float2 __ovld vload2(size_t offset, const __global float *p);
-char3 __ovld vload3(size_t offset, const __global char *p);
-uchar3 __ovld vload3(size_t offset, const __global uchar *p);
-short3 __ovld vload3(size_t offset, const __global short *p);
-ushort3 __ovld vload3(size_t offset, const __global ushort *p);
-int3 __ovld vload3(size_t offset, const __global int *p);
-uint3 __ovld vload3(size_t offset, const __global uint *p);
-long3 __ovld vload3(size_t offset, const __global long *p);
-ulong3 __ovld vload3(size_t offset, const __global ulong *p);
-float3 __ovld vload3(size_t offset, const __global float *p);
-char4 __ovld vload4(size_t offset, const __global char *p);
-uchar4 __ovld vload4(size_t offset, const __global uchar *p);
-short4 __ovld vload4(size_t offset, const __global short *p);
-ushort4 __ovld vload4(size_t offset, const __global ushort *p);
-int4 __ovld vload4(size_t offset, const __global int *p);
-uint4 __ovld vload4(size_t offset, const __global uint *p);
-long4 __ovld vload4(size_t offset, const __global long *p);
-ulong4 __ovld vload4(size_t offset, const __global ulong *p);
-float4 __ovld vload4(size_t offset, const __global float *p);
-char8 __ovld vload8(size_t offset, const __global char *p);
-uchar8 __ovld vload8(size_t offset, const __global uchar *p);
-short8 __ovld vload8(size_t offset, const __global short *p);
-ushort8 __ovld vload8(size_t offset, const __global ushort *p);
-int8 __ovld vload8(size_t offset, const __global int *p);
-uint8 __ovld vload8(size_t offset, const __global uint *p);
-long8 __ovld vload8(size_t offset, const __global long *p);
-ulong8 __ovld vload8(size_t offset, const __global ulong *p);
-float8 __ovld vload8(size_t offset, const __global float *p);
-char16 __ovld vload16(size_t offset, const __global char *p);
-uchar16 __ovld vload16(size_t offset, const __global uchar *p);
-short16 __ovld vload16(size_t offset, const __global short *p);
-ushort16 __ovld vload16(size_t offset, const __global ushort *p);
-int16 __ovld vload16(size_t offset, const __global int *p);
-uint16 __ovld vload16(size_t offset, const __global uint *p);
-long16 __ovld vload16(size_t offset, const __global long *p);
-ulong16 __ovld vload16(size_t offset, const __global ulong *p);
-float16 __ovld vload16(size_t offset, const __global float *p);
-char2 __ovld vload2(size_t offset, const __local char *p);
-uchar2 __ovld vload2(size_t offset, const __local uchar *p);
-short2 __ovld vload2(size_t offset, const __local short *p);
-ushort2 __ovld vload2(size_t offset, const __local ushort *p);
-int2 __ovld vload2(size_t offset, const __local int *p);
-uint2 __ovld vload2(size_t offset, const __local uint *p);
-long2 __ovld vload2(size_t offset, const __local long *p);
-ulong2 __ovld vload2(size_t offset, const __local ulong *p);
-float2 __ovld vload2(size_t offset, const __local float *p);
-char3 __ovld vload3(size_t offset, const __local char *p);
-uchar3 __ovld vload3(size_t offset, const __local uchar *p);
-short3 __ovld vload3(size_t offset, const __local short *p);
-ushort3 __ovld vload3(size_t offset, const __local ushort *p);
-int3 __ovld vload3(size_t offset, const __local int *p);
-uint3 __ovld vload3(size_t offset, const __local uint *p);
-long3 __ovld vload3(size_t offset, const __local long *p);
-ulong3 __ovld vload3(size_t offset, const __local ulong *p);
-float3 __ovld vload3(size_t offset, const __local float *p);
-char4 __ovld vload4(size_t offset, const __local char *p);
-uchar4 __ovld vload4(size_t offset, const __local uchar *p);
-short4 __ovld vload4(size_t offset, const __local short *p);
-ushort4 __ovld vload4(size_t offset, const __local ushort *p);
-int4 __ovld vload4(size_t offset, const __local int *p);
-uint4 __ovld vload4(size_t offset, const __local uint *p);
-long4 __ovld vload4(size_t offset, const __local long *p);
-ulong4 __ovld vload4(size_t offset, const __local ulong *p);
-float4 __ovld vload4(size_t offset, const __local float *p);
-char8 __ovld vload8(size_t offset, const __local char *p);
-uchar8 __ovld vload8(size_t offset, const __local uchar *p);
-short8 __ovld vload8(size_t offset, const __local short *p);
-ushort8 __ovld vload8(size_t offset, const __local ushort *p);
-int8 __ovld vload8(size_t offset, const __local int *p);
-uint8 __ovld vload8(size_t offset, const __local uint *p);
-long8 __ovld vload8(size_t offset, const __local long *p);
-ulong8 __ovld vload8(size_t offset, const __local ulong *p);
-float8 __ovld vload8(size_t offset, const __local float *p);
-char16 __ovld vload16(size_t offset, const __local char *p);
-uchar16 __ovld vload16(size_t offset, const __local uchar *p);
-short16 __ovld vload16(size_t offset, const __local short *p);
-ushort16 __ovld vload16(size_t offset, const __local ushort *p);
-int16 __ovld vload16(size_t offset, const __local int *p);
-uint16 __ovld vload16(size_t offset, const __local uint *p);
-long16 __ovld vload16(size_t offset, const __local long *p);
-ulong16 __ovld vload16(size_t offset, const __local ulong *p);
-float16 __ovld vload16(size_t offset, const __local float *p);
-char2 __ovld vload2(size_t offset, const __private char *p);
-uchar2 __ovld vload2(size_t offset, const __private uchar *p);
-short2 __ovld vload2(size_t offset, const __private short *p);
-ushort2 __ovld vload2(size_t offset, const __private ushort *p);
-int2 __ovld vload2(size_t offset, const __private int *p);
-uint2 __ovld vload2(size_t offset, const __private uint *p);
-long2 __ovld vload2(size_t offset, const __private long *p);
-ulong2 __ovld vload2(size_t offset, const __private ulong *p);
-float2 __ovld vload2(size_t offset, const __private float *p);
-char3 __ovld vload3(size_t offset, const __private char *p);
-uchar3 __ovld vload3(size_t offset, const __private uchar *p);
-short3 __ovld vload3(size_t offset, const __private short *p);
-ushort3 __ovld vload3(size_t offset, const __private ushort *p);
-int3 __ovld vload3(size_t offset, const __private int *p);
-uint3 __ovld vload3(size_t offset, const __private uint *p);
-long3 __ovld vload3(size_t offset, const __private long *p);
-ulong3 __ovld vload3(size_t offset, const __private ulong *p);
-float3 __ovld vload3(size_t offset, const __private float *p);
-char4 __ovld vload4(size_t offset, const __private char *p);
-uchar4 __ovld vload4(size_t offset, const __private uchar *p);
-short4 __ovld vload4(size_t offset, const __private short *p);
-ushort4 __ovld vload4(size_t offset, const __private ushort *p);
-int4 __ovld vload4(size_t offset, const __private int *p);
-uint4 __ovld vload4(size_t offset, const __private uint *p);
-long4 __ovld vload4(size_t offset, const __private long *p);
-ulong4 __ovld vload4(size_t offset, const __private ulong *p);
-float4 __ovld vload4(size_t offset, const __private float *p);
-char8 __ovld vload8(size_t offset, const __private char *p);
-uchar8 __ovld vload8(size_t offset, const __private uchar *p);
-short8 __ovld vload8(size_t offset, const __private short *p);
-ushort8 __ovld vload8(size_t offset, const __private ushort *p);
-int8 __ovld vload8(size_t offset, const __private int *p);
-uint8 __ovld vload8(size_t offset, const __private uint *p);
-long8 __ovld vload8(size_t offset, const __private long *p);
-ulong8 __ovld vload8(size_t offset, const __private ulong *p);
-float8 __ovld vload8(size_t offset, const __private float *p);
-char16 __ovld vload16(size_t offset, const __private char *p);
-uchar16 __ovld vload16(size_t offset, const __private uchar *p);
-short16 __ovld vload16(size_t offset, const __private short *p);
-ushort16 __ovld vload16(size_t offset, const __private ushort *p);
-int16 __ovld vload16(size_t offset, const __private int *p);
-uint16 __ovld vload16(size_t offset, const __private uint *p);
-long16 __ovld vload16(size_t offset, const __private long *p);
-ulong16 __ovld vload16(size_t offset, const __private ulong *p);
-float16 __ovld vload16(size_t offset, const __private float *p);
+char2 __ovld __purefn vload2(size_t offset, const __global char *p);
+uchar2 __ovld __purefn vload2(size_t offset, const __global uchar *p);
+short2 __ovld __purefn vload2(size_t offset, const __global short *p);
+ushort2 __ovld __purefn vload2(size_t offset, const __global ushort *p);
+int2 __ovld __purefn vload2(size_t offset, const __global int *p);
+uint2 __ovld __purefn vload2(size_t offset, const __global uint *p);
+long2 __ovld __purefn vload2(size_t offset, const __global long *p);
+ulong2 __ovld __purefn vload2(size_t offset, const __global ulong *p);
+float2 __ovld __purefn vload2(size_t offset, const __global float *p);
+char3 __ovld __purefn vload3(size_t offset, const __global char *p);
+uchar3 __ovld __purefn vload3(size_t offset, const __global uchar *p);
+short3 __ovld __purefn vload3(size_t offset, const __global short *p);
+ushort3 __ovld __purefn vload3(size_t offset, const __global ushort *p);
+int3 __ovld __purefn vload3(size_t offset, const __global int *p);
+uint3 __ovld __purefn vload3(size_t offset, const __global uint *p);
+long3 __ovld __purefn vload3(size_t offset, const __global long *p);
+ulong3 __ovld __purefn vload3(size_t offset, const __global ulong *p);
+float3 __ovld __purefn vload3(size_t offset, const __global float *p);
+char4 __ovld __purefn vload4(size_t offset, const __global char *p);
+uchar4 __ovld __purefn vload4(size_t offset, const __global uchar *p);
+short4 __ovld __purefn vload4(size_t offset, const __global short *p);
+ushort4 __ovld __purefn vload4(size_t offset, const __global ushort *p);
+int4 __ovld __purefn vload4(size_t offset, const __global int *p);
+uint4 __ovld __purefn vload4(size_t offset, const __global uint *p);
+long4 __ovld __purefn vload4(size_t offset, const __global long *p);
+ulong4 __ovld __purefn vload4(size_t offset, const __global ulong *p);
+float4 __ovld __purefn vload4(size_t offset, const __global float *p);
+char8 __ovld __purefn vload8(size_t offset, const __global char *p);
+uchar8 __ovld __purefn vload8(size_t offset, const __global uchar *p);
+short8 __ovld __purefn vload8(size_t offset, const __global short *p);
+ushort8 __ovld __purefn vload8(size_t offset, const __global ushort *p);
+int8 __ovld __purefn vload8(size_t offset, const __global int *p);
+uint8 __ovld __purefn vload8(size_t offset, const __global uint *p);
+long8 __ovld __purefn vload8(size_t offset, const __global long *p);
+ulong8 __ovld __purefn vload8(size_t offset, const __global ulong *p);
+float8 __ovld __purefn vload8(size_t offset, const __global float *p);
+char16 __ovld __purefn vload16(size_t offset, const __global char *p);
+uchar16 __ovld __purefn vload16(size_t offset, const __global uchar *p);
+short16 __ovld __purefn vload16(size_t offset, const __global short *p);
+ushort16 __ovld __purefn vload16(size_t offset, const __global ushort *p);
+int16 __ovld __purefn vload16(size_t offset, const __global int *p);
+uint16 __ovld __purefn vload16(size_t offset, const __global uint *p);
+long16 __ovld __purefn vload16(size_t offset, const __global long *p);
+ulong16 __ovld __purefn vload16(size_t offset, const __global ulong *p);
+float16 __ovld __purefn vload16(size_t offset, const __global float *p);
+char2 __ovld __purefn vload2(size_t offset, const __local char *p);
+uchar2 __ovld __purefn vload2(size_t offset, const __local uchar *p);
+short2 __ovld __purefn vload2(size_t offset, const __local short *p);
+ushort2 __ovld __purefn vload2(size_t offset, const __local ushort *p);
+int2 __ovld __purefn vload2(size_t offset, const __local int *p);
+uint2 __ovld __purefn vload2(size_t offset, const __local uint *p);
+long2 __ovld __purefn vload2(size_t offset, const __local long *p);
+ulong2 __ovld __purefn vload2(size_t offset, const __local ulong *p);
+float2 __ovld __purefn vload2(size_t offset, const __local float *p);
+char3 __ovld __purefn vload3(size_t offset, const __local char *p);
+uchar3 __ovld __purefn vload3(size_t offset, const __local uchar *p);
+short3 __ovld __purefn vload3(size_t offset, const __local short *p);
+ushort3 __ovld __purefn vload3(size_t offset, const __local ushort *p);
+int3 __ovld __purefn vload3(size_t offset, const __local int *p);
+uint3 __ovld __purefn vload3(size_t offset, const __local uint *p);
+long3 __ovld __purefn vload3(size_t offset, const __local long *p);
+ulong3 __ovld __purefn vload3(size_t offset, const __local ulong *p);
+float3 __ovld __purefn vload3(size_t offset, const __local float *p);
+char4 __ovld __purefn vload4(size_t offset, const __local char *p);
+uchar4 __ovld __purefn vload4(size_t offset, const __local uchar *p);
+short4 __ovld __purefn vload4(size_t offset, const __local short *p);
+ushort4 __ovld __purefn vload4(size_t offset, const __local ushort *p);
+int4 __ovld __purefn vload4(size_t offset, const __local int *p);
+uint4 __ovld __purefn vload4(size_t offset, const __local uint *p);
+long4 __ovld __purefn vload4(size_t offset, const __local long *p);
+ulong4 __ovld __purefn vload4(size_t offset, const __local ulong *p);
+float4 __ovld __purefn vload4(size_t offset, const __local float *p);
+char8 __ovld __purefn vload8(size_t offset, const __local char *p);
+uchar8 __ovld __purefn vload8(size_t offset, const __local uchar *p);
+short8 __ovld __purefn vload8(size_t offset, const __local short *p);
+ushort8 __ovld __purefn vload8(size_t offset, const __local ushort *p);
+int8 __ovld __purefn vload8(size_t offset, const __local int *p);
+uint8 __ovld __purefn vload8(size_t offset, const __local uint *p);
+long8 __ovld __purefn vload8(size_t offset, const __local long *p);
+ulong8 __ovld __purefn vload8(size_t offset, const __local ulong *p);
+float8 __ovld __purefn vload8(size_t offset, const __local float *p);
+char16 __ovld __purefn vload16(size_t offset, const __local char *p);
+uchar16 __ovld __purefn vload16(size_t offset, const __local uchar *p);
+short16 __ovld __purefn vload16(size_t offset, const __local short *p);
+ushort16 __ovld __purefn vload16(size_t offset, const __local ushort *p);
+int16 __ovld __purefn vload16(size_t offset, const __local int *p);
+uint16 __ovld __purefn vload16(size_t offset, const __local uint *p);
+long16 __ovld __purefn vload16(size_t offset, const __local long *p);
+ulong16 __ovld __purefn vload16(size_t offset, const __local ulong *p);
+float16 __ovld __purefn vload16(size_t offset, const __local float *p);
+char2 __ovld __purefn vload2(size_t offset, const __private char *p);
+uchar2 __ovld __purefn vload2(size_t offset, const __private uchar *p);
+short2 __ovld __purefn vload2(size_t offset, const __private short *p);
+ushort2 __ovld __purefn vload2(size_t offset, const __private ushort *p);
+int2 __ovld __purefn vload2(size_t offset, const __private int *p);
+uint2 __ovld __purefn vload2(size_t offset, const __private uint *p);
+long2 __ovld __purefn vload2(size_t offset, const __private long *p);
+ulong2 __ovld __purefn vload2(size_t offset, const __private ulong *p);
+float2 __ovld __purefn vload2(size_t offset, const __private float *p);
+char3 __ovld __purefn vload3(size_t offset, const __private char *p);
+uchar3 __ovld __purefn vload3(size_t offset, const __private uchar *p);
+short3 __ovld __purefn vload3(size_t offset, const __private short *p);
+ushort3 __ovld __purefn vload3(size_t offset, const __private ushort *p);
+int3 __ovld __purefn vload3(size_t offset, const __private int *p);
+uint3 __ovld __purefn vload3(size_t offset, const __private uint *p);
+long3 __ovld __purefn vload3(size_t offset, const __private long *p);
+ulong3 __ovld __purefn vload3(size_t offset, const __private ulong *p);
+float3 __ovld __purefn vload3(size_t offset, const __private float *p);
+char4 __ovld __purefn vload4(size_t offset, const __private char *p);
+uchar4 __ovld __purefn vload4(size_t offset, const __private uchar *p);
+short4 __ovld __purefn vload4(size_t offset, const __private short *p);
+ushort4 __ovld __purefn vload4(size_t offset, const __private ushort *p);
+int4 __ovld __purefn vload4(size_t offset, const __private int *p);
+uint4 __ovld __purefn vload4(size_t offset, const __private uint *p);
+long4 __ovld __purefn vload4(size_t offset, const __private long *p);
+ulong4 __ovld __purefn vload4(size_t offset, const __private ulong *p);
+float4 __ovld __purefn vload4(size_t offset, const __private float *p);
+char8 __ovld __purefn vload8(size_t offset, const __private char *p);
+uchar8 __ovld __purefn vload8(size_t offset, const __private uchar *p);
+short8 __ovld __purefn vload8(size_t offset, const __private short *p);
+ushort8 __ovld __purefn vload8(size_t offset, const __private ushort *p);
+int8 __ovld __purefn vload8(size_t offset, const __private int *p);
+uint8 __ovld __purefn vload8(size_t offset, const __private uint *p);
+long8 __ovld __purefn vload8(size_t offset, const __private long *p);
+ulong8 __ovld __purefn vload8(size_t offset, const __private ulong *p);
+float8 __ovld __purefn vload8(size_t offset, const __private float *p);
+char16 __ovld __purefn vload16(size_t offset, const __private char *p);
+uchar16 __ovld __purefn vload16(size_t offset, const __private uchar *p);
+short16 __ovld __purefn vload16(size_t offset, const __private short *p);
+ushort16 __ovld __purefn vload16(size_t offset, const __private ushort *p);
+int16 __ovld __purefn vload16(size_t offset, const __private int *p);
+uint16 __ovld __purefn vload16(size_t offset, const __private uint *p);
+long16 __ovld __purefn vload16(size_t offset, const __private long *p);
+ulong16 __ovld __purefn vload16(size_t offset, const __private ulong *p);
+float16 __ovld __purefn vload16(size_t offset, const __private float *p);
 
 #ifdef cl_khr_fp64
-double2 __ovld vload2(size_t offset, const __global double *p);
-double3 __ovld vload3(size_t offset, const __global double *p);
-double4 __ovld vload4(size_t offset, const __global double *p);
-double8 __ovld vload8(size_t offset, const __global double *p);
-double16 __ovld vload16(size_t offset, const __global double *p);
-double2 __ovld vload2(size_t offset, const __local double *p);
-double3 __ovld vload3(size_t offset, const __local double *p);
-double4 __ovld vload4(size_t offset, const __local double *p);
-double8 __ovld vload8(size_t offset, const __local double *p);
-double16 __ovld vload16(size_t offset, const __local double *p);
-double2 __ovld vload2(size_t offset, const __private double *p);
-double3 __ovld vload3(size_t offset, const __private double *p);
-double4 __ovld vload4(size_t offset, const __private double *p);
-double8 __ovld vload8(size_t offset, const __private double *p);
-double16 __ovld vload16(size_t offset, const __private double *p);
+double2 __ovld __purefn vload2(size_t offset, const __global double *p);
+double3 __ovld __purefn vload3(size_t offset, const __global double *p);
+double4 __ovld __purefn vload4(size_t offset, const __global double *p);
+double8 __ovld __purefn vload8(size_t offset, const __global double *p);
+double16 __ovld __purefn vload16(size_t offset, const __global double *p);
+double2 __ovld __purefn vload2(size_t offset, const __local double *p);
+double3 __ovld __purefn vload3(size_t offset, const __local double *p);
+double4 __ovld __purefn vload4(size_t offset, const __local double *p);
+double8 __ovld __purefn vload8(size_t offset, const __local double *p);
+double16 __ovld __purefn vload16(size_t offset, const __local double *p);
+double2 __ovld __purefn vload2(size_t offset, const __private double *p);
+double3 __ovld __purefn vload3(size_t offset, const __private double *p);
+double4 __ovld __purefn vload4(size_t offset, const __private double *p);
+double8 __ovld __purefn vload8(size_t offset, const __private double *p);
+double16 __ovld __purefn vload16(size_t offset, const __private double *p);
 #endif //cl_khr_fp64
 
 #ifdef cl_khr_fp16
-half __ovld vload(size_t offset, const __global half *p);
-half2 __ovld vload2(size_t offset, const __global half *p);
-half3 __ovld vload3(size_t offset, const __global half *p);
-half4 __ovld vload4(size_t offset, const __global half *p);
-half8 __ovld vload8(size_t offset, const __global half *p);
-half16 __ovld vload16(size_t offset, const __global half *p);
-half __ovld vload(size_t offset, const __local half *p);
-half2 __ovld vload2(size_t offset, const __local half *p);
-half3 __ovld vload3(size_t offset, const __local half *p);
-half4 __ovld vload4(size_t offset, const __local half *p);
-half8 __ovld vload8(size_t offset, const __local half *p);
-half16 __ovld vload16(size_t offset, const __local half *p);
-half __ovld vload(size_t offset, const __private half *p);
-half2 __ovld vload2(size_t offset, const __private half *p);
-half3 __ovld vload3(size_t offset, const __private half *p);
-half4 __ovld vload4(size_t offset, const __private half *p);
-half8 __ovld vload8(size_t offset, const __private half *p);
-half16 __ovld vload16(size_t offset, const __private half *p);
+half __ovld __purefn vload(size_t offset, const __global half *p);
+half2 __ovld __purefn vload2(size_t offset, const __global half *p);
+half3 __ovld __purefn vload3(size_t offset, const __global half *p);
+half4 __ovld __purefn vload4(size_t offset, const __global half *p);
+half8 __ovld __purefn vload8(size_t offset, const __global half *p);
+half16 __ovld __purefn vload16(size_t offset, const __global half *p);
+half __ovld __purefn vload(size_t offset, const __local half *p);
+half2 __ovld __purefn vload2(size_t offset, const __local half *p);
+half3 __ovld __purefn vload3(size_t offset, const __local half *p);
+half4 __ovld __purefn vload4(size_t offset, const __local half *p);
+half8 __ovld __purefn vload8(size_t offset, const __local half *p);
+half16 __ovld __purefn vload16(size_t offset, const __local half *p);
+half __ovld __purefn vload(size_t offset, const __private half *p);
+half2 __ovld __purefn vload2(size_t offset, const __private half *p);
+half3 __ovld __purefn vload3(size_t offset, const __private half *p);
+half4 __ovld __purefn vload4(size_t offset, const __private half *p);
+half8 __ovld __purefn vload8(size_t offset, const __private half *p);
+half16 __ovld __purefn vload16(size_t offset, const __private half *p);
 #endif //cl_khr_fp16
 #endif //defined(__opencl_c_generic_address_space)
 
@@ -11736,13 +11736,13 @@ void __ovld vstore16(half16 data, size_t offset, __private half *p);
  * The read address computed as (p + offset)
  * must be 16-bit aligned.
  */
-float __ovld vload_half(size_t offset, const __constant half *p);
+float __ovld __purefn vload_half(size_t offset, const __constant half *p);
 #if defined(__opencl_c_generic_address_space)
-float __ovld vload_half(size_t offset, const half *p);
+float __ovld __purefn vload_half(size_t offset, const half *p);
 #else
-float __ovld vload_half(size_t offset, const __global half *p);
-float __ovld vload_half(size_t offset, const __local half *p);
-float __ovld vload_half(size_t offset, const __private half *p);
+float __ovld __purefn vload_half(size_t offset, const __global half *p);
+float __ovld __purefn vload_half(size_t offset, const __local half *p);
+float __ovld __purefn vload_half(size_t offset, const __private half *p);
 #endif //defined(__opencl_c_generic_address_space)
 
 /**
@@ -11753,33 +11753,33 @@ float __ovld vload_half(size_t offset, const __private half *p);
  * value is returned. The read address computed
  * as (p + (offset * n)) must be 16-bit aligned.
  */
-float2 __ovld vload_half2(size_t offset, const __constant half *p);
-float3 __ovld vload_half3(size_t offset, const __constant half *p);
-float4 __ovld vload_half4(size_t offset, const __constant half *p);
-float8 __ovld vload_half8(size_t offset, const __constant half *p);
-float16 __ovld vload_half16(size_t offset, const __constant half *p);
+float2 __ovld __purefn vload_half2(size_t offset, const __constant half *p);
+float3 __ovld __purefn vload_half3(size_t offset, const __constant half *p);
+float4 __ovld __purefn vload_half4(size_t offset, const __constant half *p);
+float8 __ovld __purefn vload_half8(size_t offset, const __constant half *p);
+float16 __ovld __purefn vload_half16(size_t offset, const __constant half *p);
 #if defined(__opencl_c_generic_address_space)
-float2 __ovld vload_half2(size_t offset, const half *p);
-float3 __ovld vload_half3(size_t offset, const half *p);
-float4 __ovld vload_half4(size_t offset, const half *p);
-float8 __ovld vload_half8(size_t offset, const half *p);
-float16 __ovld vload_half16(size_t offset, const half *p);
+float2 __ovld __purefn vload_half2(size_t offset, const half *p);
+float3 __ovld __purefn vload_half3(size_t offset, const half *p);
+float4 __ovld __purefn vload_half4(size_t offset, const half *p);
+float8 __ovld __purefn vload_half8(size_t offset, const half *p);
+float16 __ovld __purefn vload_half16(size_t offset, const half *p);
 #else
-float2 __ovld vload_half2(size_t offset, const __global half *p);
-float3 __ovld vload_half3(size_t offset, const __global half *p);
-float4 __ovld vload_half4(size_t offset, const __global half *p);
-float8 __ovld vload_half8(size_t offset, const __global half *p);
-float16 __ovld vload_half16(size_t offset, const __global half *p);
-float2 __ovld vload_half2(size_t offset, const __local half *p);
-float3 __ovld vload_half3(size_t offset, const __local half *p);
-float4 __ovld vload_half4(size_t offset, const __local half *p);
-float8 __ovld vload_half8(size_t offset, const __local half *p);
-float16 __ovld vload_half16(size_t offset, const __local half *p);
-float2 __ovld vload_half2(size_t offset, const __private half *p);
-float3 __ovld vload_half3(size_t offset, const __private half *p);
-float4 __ovld vload_half4(size_t offset, const __private half *p);
-float8 __ovld vload_half8(size_t offset, const __private half *p);
-float16 __ovld vload_half16(size_t offset, const __private half *p);
+float2 __ovld __purefn vload_half2(size_t offset, const __global half *p);
+float3 __ovld __purefn vload_half3(size_t offset, const __global half *p);
+float4 __ovld __purefn vload_half4(size_t offset, const __global half *p);
+float8 __ovld __purefn vload_half8(size_t offset, const __global half *p);
+float16 __ovld __purefn vload_half16(size_t offset, const __global half *p);
+float2 __ovld __purefn vload_half2(size_t offset, const __local half *p);
+float3 __ovld __purefn vload_half3(size_t offset, const __local half *p);
+float4 __ovld __purefn vload_half4(size_t offset, const __local half *p);
+float8 __ovld __purefn vload_half8(size_t offset, const __local half *p);
+float16 __ovld __purefn vload_half16(size_t offset, const __local half *p);
+float2 __ovld __purefn vload_half2(size_t offset, const __private half *p);
+float3 __ovld __purefn vload_half3(size_t offset, const __private half *p);
+float4 __ovld __purefn vload_half4(size_t offset, const __private half *p);
+float8 __ovld __purefn vload_half8(size_t offset, const __private half *p);
+float16 __ovld __purefn vload_half16(size_t offset, const __private half *p);
 #endif //defined(__opencl_c_generic_address_space)
 
 /**
@@ -12073,33 +12073,33 @@ void __ovld vstore_half16_rtn(double16 data, size_t offset, __private half *p);
  * The address computed as (p + (offset * 4))
  * must be aligned to sizeof (half) * 4 bytes.
  */
-float2 __ovld vloada_half2(size_t offset, const __constant half *p);
-float3 __ovld vloada_half3(size_t offset, const __constant half *p);
-float4 __ovld vloada_half4(size_t offset, const __constant half *p);
-float8 __ovld vloada_half8(size_t offset, const __constant half *p);
-float16 __ovld vloada_half16(size_t offset, const __constant half *p);
+float2 __ovld __purefn vloada_half2(size_t offset, const __constant half *p);
+float3 __ovld __purefn vloada_half3(size_t offset, const __constant half *p);
+float4 __ovld __purefn vloada_half4(size_t offset, const __constant half *p);
+float8 __ovld __purefn vloada_half8(size_t offset, const __constant half *p);
+float16 __ovld __purefn vloada_half16(size_t offset, const __constant half *p);
 #if defined(__opencl_c_generic_address_space)
-float2 __ovld vloada_half2(size_t offset, const half *p);
-float3 __ovld vloada_half3(size_t offset, const half *p);
-float4 __ovld vloada_half4(size_t offset, const half *p);
-float8 __ovld vloada_half8(size_t offset, const half *p);
-float16 __ovld vloada_half16(size_t offset, const half *p);
+float2 __ovld __purefn vloada_half2(size_t offset, const half *p);
+float3 __ovld __purefn vloada_half3(size_t offset, const half *p);
+float4 __ovld __purefn vloada_half4(size_t offset, const half *p);
+float8 __ovld __purefn vloada_half8(size_t offset, const half *p);
+float16 __ovld __purefn vloada_half16(size_t offset, const half *p);
 #else
-float2 __ovld vloada_half2(size_t offset, const __global half *p);
-float3 __ovld vloada_half3(size_t offset, const __global half *p);
-float4 __ovld vloada_half4(size_t offset, const __global half *p);
-float8 __ovld vloada_half8(size_t offset, const __global half *p);
-float16 __ovld vloada_half16(size_t offset, const __global half *p);
-float2 __ovld vloada_half2(size_t offset, const __local half *p);
-float3 __ovld vloada_half3(size_t offset, const __local half *p);
-float4 __ovld vloada_half4(size_t offset, const __local half *p);
-float8 __ovld vloada_half8(size_t offset, const __local half *p);
-float16 __ovld vloada_half16(size_t offset, const __local half *p);
-float2 __ovld vloada_half2(size_t offset, const __private half *p);
-float3 __ovld vloada_half3(size_t offset, const __private half *p);
-float4 __ovld vloada_half4(size_t offset, const __private half *p);
-float8 __ovld vloada_half8(size_t offset, const __private half *p);
-float16 __ovld vloada_half16(size_t offset, const __private half *p);
+float2 __ovld __purefn vloada_half2(size_t offset, const __global half *p);
+float3 __ovld __purefn vloada_half3(size_t offset, const __global half *p);
+float4 __ovld __purefn vloada_half4(size_t offset, const __global half *p);
+float8 __ovld __purefn vloada_half8(size_t offset, const __global half *p);
+float16 __ovld __purefn vloada_half16(size_t offset, const __global half *p);
+float2 __ovld __purefn vloada_half2(size_t offset, const __local half *p);
+float3 __ovld __purefn vloada_half3(size_t offset, const __local half *p);
+float4 __ovld __purefn vloada_half4(size_t offset, const __local half *p);
+float8 __ovld __purefn vloada_half8(size_t offset, const __local half *p);
+float16 __ovld __purefn vloada_half16(size_t offset, const __local half *p);
+float2 __ovld __purefn vloada_half2(size_t offset, const __private half *p);
+float3 __ovld __purefn vloada_half3(size_t offset, const __private half *p);
+float4 __ovld __purefn vloada_half4(size_t offset, const __private half *p);
+float8 __ovld __purefn vloada_half8(size_t offset, const __private half *p);
+float16 __ovld __purefn vloada_half16(size_t offset, const __private half *p);
 #endif //defined(__opencl_c_generic_address_space)
 
 /**
diff --git a/clang/lib/Headers/unwind.h b/clang/lib/Headers/unwind.h
index 029524b7bc84..6e069798f02d 100644
--- a/clang/lib/Headers/unwind.h
+++ b/clang/lib/Headers/unwind.h
@@ -172,7 +172,8 @@ typedef enum {
   _UVRSC_CORE = 0,        /* integer register */
   _UVRSC_VFP = 1,         /* vfp */
   _UVRSC_WMMXD = 3,       /* Intel WMMX data register */
-  _UVRSC_WMMXC = 4        /* Intel WMMX control register */
+  _UVRSC_WMMXC = 4,       /* Intel WMMX control register */
+  _UVRSC_PSEUDO = 5       /* Special purpose pseudo register */
 } _Unwind_VRS_RegClass;
 
 typedef enum {
diff --git a/clang/lib/Lex/ModuleMap.cpp b/clang/lib/Lex/ModuleMap.cpp
index 9fa170410da3..0b136aeb580f 100644
--- a/clang/lib/Lex/ModuleMap.cpp
+++ b/clang/lib/Lex/ModuleMap.cpp
@@ -832,12 +832,16 @@ std::pair<Module *, bool> ModuleMap::findOrCreateModule(StringRef Name,
   return std::make_pair(Result, true);
 }
 
-Module *ModuleMap::createGlobalModuleFragmentForModuleUnit(SourceLocation Loc) {
-  PendingSubmodules.emplace_back(
-      new Module("<global>", Loc, nullptr, /*IsFramework*/ false,
-                 /*IsExplicit*/ true, NumCreatedModules++));
-  PendingSubmodules.back()->Kind = Module::GlobalModuleFragment;
-  return PendingSubmodules.back().get();
+Module *ModuleMap::createGlobalModuleFragmentForModuleUnit(SourceLocation Loc,
+                                                           Module *Parent) {
+  auto *Result = new Module("<global>", Loc, Parent, /*IsFramework*/ false,
+                            /*IsExplicit*/ true, NumCreatedModules++);
+  Result->Kind = Module::GlobalModuleFragment;
+  // If the created module isn't owned by a parent, send it to PendingSubmodules
+  // to wait for its parent.
+  if (!Result->Parent)
+    PendingSubmodules.emplace_back(Result);
+  return Result;
 }
 
 Module *
diff --git a/clang/lib/Lex/TokenLexer.cpp b/clang/lib/Lex/TokenLexer.cpp
index 41e7f3f1dccb..e71a65f031e4 100644
--- a/clang/lib/Lex/TokenLexer.cpp
+++ b/clang/lib/Lex/TokenLexer.cpp
@@ -472,11 +472,9 @@ void TokenLexer::ExpandFunctionArguments() {
 
         // If the '##' came from expanding an argument, turn it into 'unknown'
         // to avoid pasting.
-        for (Token &Tok : llvm::make_range(ResultToks.begin() + FirstResult,
-                                           ResultToks.end())) {
+        for (Token &Tok : llvm::drop_begin(ResultToks, FirstResult))
           if (Tok.is(tok::hashhash))
             Tok.setKind(tok::unknown);
-        }
 
         if(ExpandLocStart.isValid()) {
           updateLocForMacroArgTokens(CurTok.getLocation(),
diff --git a/clang/lib/Parse/ParseCXXInlineMethods.cpp b/clang/lib/Parse/ParseCXXInlineMethods.cpp
index 116724a0d50b..19cddc69ebfc 100644
--- a/clang/lib/Parse/ParseCXXInlineMethods.cpp
+++ b/clang/lib/Parse/ParseCXXInlineMethods.cpp
@@ -452,13 +452,14 @@ void Parser::ParseLexedMethodDeclaration(LateParsedMethodDeclaration &LM) {
     CXXMethodDecl *Method;
     if (FunctionTemplateDecl *FunTmpl
           = dyn_cast<FunctionTemplateDecl>(LM.Method))
-      Method = cast<CXXMethodDecl>(FunTmpl->getTemplatedDecl());
+      Method = dyn_cast<CXXMethodDecl>(FunTmpl->getTemplatedDecl());
     else
-      Method = cast<CXXMethodDecl>(LM.Method);
+      Method = dyn_cast<CXXMethodDecl>(LM.Method);
 
-    Sema::CXXThisScopeRAII ThisScope(Actions, Method->getParent(),
-                                     Method->getMethodQualifiers(),
-                                     getLangOpts().CPlusPlus11);
+    Sema::CXXThisScopeRAII ThisScope(
+        Actions, Method ? Method->getParent() : nullptr,
+        Method ? Method->getMethodQualifiers() : Qualifiers{},
+        Method && getLangOpts().CPlusPlus11);
 
     // Parse the exception-specification.
     SourceRange SpecificationRange;
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index 1bdeccc4cbf5..0c1f88bc51d1 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -2891,7 +2891,8 @@ void Parser::ParseAlignmentSpecifier(ParsedAttributes &Attrs,
 }
 
 ExprResult Parser::ParseExtIntegerArgument() {
-  assert(Tok.is(tok::kw__ExtInt) && "Not an extended int type");
+  assert(Tok.isOneOf(tok::kw__ExtInt, tok::kw__BitInt) &&
+         "Not an extended int type");
   ConsumeToken();
 
   BalancedDelimiterTracker T(*this, tok::l_paren);
@@ -3882,11 +3883,13 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
       isInvalid = DS.SetTypeSpecType(DeclSpec::TST_int, Loc, PrevSpec,
                                      DiagID, Policy);
       break;
-    case tok::kw__ExtInt: {
+    case tok::kw__ExtInt:
+    case tok::kw__BitInt: {
+      DiagnoseBitIntUse(Tok);
       ExprResult ER = ParseExtIntegerArgument();
       if (ER.isInvalid())
         continue;
-      isInvalid = DS.SetExtIntType(Loc, ER.get(), PrevSpec, DiagID, Policy);
+      isInvalid = DS.SetBitIntType(Loc, ER.get(), PrevSpec, DiagID, Policy);
       ConsumedEnd = PrevTokLocation;
       break;
     }
@@ -5015,6 +5018,7 @@ bool Parser::isKnownToBeTypeSpecifier(const Token &Tok) const {
   case tok::kw_char32_t:
   case tok::kw_int:
   case tok::kw__ExtInt:
+  case tok::kw__BitInt:
   case tok::kw___bf16:
   case tok::kw_half:
   case tok::kw_float:
@@ -5097,6 +5101,7 @@ bool Parser::isTypeSpecifierQualifier() {
   case tok::kw_char32_t:
   case tok::kw_int:
   case tok::kw__ExtInt:
+  case tok::kw__BitInt:
   case tok::kw_half:
   case tok::kw___bf16:
   case tok::kw_float:
@@ -5268,6 +5273,7 @@ bool Parser::isDeclarationSpecifier(bool DisambiguatingWithExpression) {
 
   case tok::kw_int:
   case tok::kw__ExtInt:
+  case tok::kw__BitInt:
   case tok::kw_half:
   case tok::kw___bf16:
   case tok::kw_float:
@@ -7476,3 +7482,24 @@ bool Parser::TryAltiVecTokenOutOfLine(DeclSpec &DS, SourceLocation Loc,
   }
   return false;
 }
+
+void Parser::DiagnoseBitIntUse(const Token &Tok) {
+  // If the token is for _ExtInt, diagnose it as being deprecated. Otherwise,
+  // the token is about _BitInt and gets (potentially) diagnosed as use of an
+  // extension.
+  assert(Tok.isOneOf(tok::kw__ExtInt, tok::kw__BitInt) &&
+         "expected either an _ExtInt or _BitInt token!");
+
+  SourceLocation Loc = Tok.getLocation();
+  if (Tok.is(tok::kw__ExtInt)) {
+    Diag(Loc, diag::warn_ext_int_deprecated)
+        << FixItHint::CreateReplacement(Loc, "_BitInt");
+  } else {
+    // In C2x mode, diagnose that the use is not compatible with pre-C2x modes.
+    // Otherwise, diagnose that the use is a Clang extension.
+    if (getLangOpts().C2x)
+      Diag(Loc, diag::warn_c17_compat_bit_int);
+    else
+      Diag(Loc, diag::ext_bit_int) << getLangOpts().CPlusPlus;
+  }
+}
diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp
index 2c8b4f9f441f..09a3842f5809 100644
--- a/clang/lib/Parse/ParseExpr.cpp
+++ b/clang/lib/Parse/ParseExpr.cpp
@@ -1513,6 +1513,7 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
   case tok::kw___int64:
   case tok::kw___int128:
   case tok::kw__ExtInt:
+  case tok::kw__BitInt:
   case tok::kw_signed:
   case tok::kw_unsigned:
   case tok::kw_half:
diff --git a/clang/lib/Parse/ParseExprCXX.cpp b/clang/lib/Parse/ParseExprCXX.cpp
index 4e5c0ac6c1c1..76c510ddd36c 100644
--- a/clang/lib/Parse/ParseExprCXX.cpp
+++ b/clang/lib/Parse/ParseExprCXX.cpp
@@ -2191,12 +2191,14 @@ void Parser::ParseCXXSimpleTypeSpecifier(DeclSpec &DS) {
     return;
   }
 
-  case tok::kw__ExtInt: {
+  case tok::kw__ExtInt:
+  case tok::kw__BitInt: {
+    DiagnoseBitIntUse(Tok);
     ExprResult ER = ParseExtIntegerArgument();
     if (ER.isInvalid())
       DS.SetTypeSpecError();
     else
-      DS.SetExtIntType(Loc, ER.get(), PrevSpec, DiagID, Policy);
+      DS.SetBitIntType(Loc, ER.get(), PrevSpec, DiagID, Policy);
 
     // Do this here because we have already consumed the close paren.
     DS.SetRangeEnd(PrevTokLocation);
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index 613ad742c93f..300b022d83b9 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -3192,6 +3192,7 @@ OMPClause *Parser::ParseOpenMPClause(OpenMPDirectiveKind DKind,
   case OMPC_read:
   case OMPC_write:
   case OMPC_capture:
+  case OMPC_compare:
   case OMPC_seq_cst:
   case OMPC_acq_rel:
   case OMPC_acquire:
diff --git a/clang/lib/Parse/ParseTentative.cpp b/clang/lib/Parse/ParseTentative.cpp
index be3823ecda01..35c9036fb27e 100644
--- a/clang/lib/Parse/ParseTentative.cpp
+++ b/clang/lib/Parse/ParseTentative.cpp
@@ -1690,6 +1690,7 @@ Parser::isCXXDeclarationSpecifier(Parser::TPResult BracedCastResult,
   case tok::kw__Atomic:
     return TPResult::True;
 
+  case tok::kw__BitInt:
   case tok::kw__ExtInt: {
     if (NextToken().isNot(tok::l_paren))
       return TPResult::Error;
@@ -1741,6 +1742,7 @@ bool Parser::isCXXDeclarationSpecifierAType() {
   case tok::kw_short:
   case tok::kw_int:
   case tok::kw__ExtInt:
+  case tok::kw__BitInt:
   case tok::kw_long:
   case tok::kw___int64:
   case tok::kw___int128:
diff --git a/clang/lib/Rewrite/HTMLRewrite.cpp b/clang/lib/Rewrite/HTMLRewrite.cpp
index 371557a624c9..e9b678b69594 100644
--- a/clang/lib/Rewrite/HTMLRewrite.cpp
+++ b/clang/lib/Rewrite/HTMLRewrite.cpp
@@ -203,7 +203,7 @@ std::string html::EscapeText(StringRef s, bool EscapeSpaces, bool ReplaceTabs) {
     }
   }
 
-  return os.str();
+  return Str;
 }
 
 static void AddLineNumber(RewriteBuffer &RB, unsigned LineNo,
diff --git a/clang/lib/Sema/CodeCompleteConsumer.cpp b/clang/lib/Sema/CodeCompleteConsumer.cpp
index 6c47cedfccf3..0a2ca54e244a 100644
--- a/clang/lib/Sema/CodeCompleteConsumer.cpp
+++ b/clang/lib/Sema/CodeCompleteConsumer.cpp
@@ -335,7 +335,7 @@ std::string CodeCompletionString::getAsString() const {
       break;
     }
   }
-  return OS.str();
+  return Result;
 }
 
 const char *CodeCompletionString::getTypedText() const {
@@ -640,7 +640,7 @@ static std::string getOverloadAsString(const CodeCompletionString &CCS) {
       break;
     }
   }
-  return OS.str();
+  return Result;
 }
 
 void PrintingCodeCompleteConsumer::ProcessOverloadCandidates(
diff --git a/clang/lib/Sema/DeclSpec.cpp b/clang/lib/Sema/DeclSpec.cpp
index 4405f29f3d99..d4dc790c008a 100644
--- a/clang/lib/Sema/DeclSpec.cpp
+++ b/clang/lib/Sema/DeclSpec.cpp
@@ -365,7 +365,7 @@ bool Declarator::isDeclarationOfFunction() const {
     case TST_half:
     case TST_int:
     case TST_int128:
-    case TST_extint:
+    case TST_bitint:
     case TST_struct:
     case TST_interface:
     case TST_union:
@@ -551,7 +551,7 @@ const char *DeclSpec::getSpecifierName(DeclSpec::TST T,
   case DeclSpec::TST_char32:      return "char32_t";
   case DeclSpec::TST_int:         return "int";
   case DeclSpec::TST_int128:      return "__int128";
-  case DeclSpec::TST_extint:      return "_ExtInt";
+  case DeclSpec::TST_bitint:      return "_BitInt";
   case DeclSpec::TST_half:        return "half";
   case DeclSpec::TST_float:       return "float";
   case DeclSpec::TST_double:      return "double";
@@ -932,7 +932,7 @@ bool DeclSpec::SetTypeSpecError() {
   return false;
 }
 
-bool DeclSpec::SetExtIntType(SourceLocation KWLoc, Expr *BitsExpr,
+bool DeclSpec::SetBitIntType(SourceLocation KWLoc, Expr *BitsExpr,
                              const char *&PrevSpec, unsigned &DiagID,
                              const PrintingPolicy &Policy) {
   assert(BitsExpr && "no expression provided!");
@@ -945,7 +945,7 @@ bool DeclSpec::SetExtIntType(SourceLocation KWLoc, Expr *BitsExpr,
     return true;
   }
 
-  TypeSpecType = TST_extint;
+  TypeSpecType = TST_bitint;
   ExprRep = BitsExpr;
   TSTLoc = KWLoc;
   TSTNameLoc = KWLoc;
@@ -1252,7 +1252,7 @@ void DeclSpec::Finish(Sema &S, const PrintingPolicy &Policy) {
       TypeSpecType = TST_int; // unsigned -> unsigned int, signed -> signed int.
     else if (TypeSpecType != TST_int && TypeSpecType != TST_int128 &&
              TypeSpecType != TST_char && TypeSpecType != TST_wchar &&
-             !IsFixedPointType && TypeSpecType != TST_extint) {
+             !IsFixedPointType && TypeSpecType != TST_bitint) {
       S.Diag(TSSLoc, diag::err_invalid_sign_spec)
         << getSpecifierName((TST)TypeSpecType, Policy);
       // signed double -> double.
@@ -1302,7 +1302,7 @@ void DeclSpec::Finish(Sema &S, const PrintingPolicy &Policy) {
                                                  " double");
       TypeSpecType = TST_double;   // _Complex -> _Complex double.
     } else if (TypeSpecType == TST_int || TypeSpecType == TST_char ||
-               TypeSpecType == TST_extint) {
+               TypeSpecType == TST_bitint) {
       // Note that this intentionally doesn't include _Complex _Bool.
       if (!S.getLangOpts().CPlusPlus)
         S.Diag(TSTLoc, diag::ext_integer_complex);
diff --git a/clang/lib/Sema/OpenCLBuiltins.td b/clang/lib/Sema/OpenCLBuiltins.td
index 8cf7ec58eff5..38debc5aa9fc 100644
--- a/clang/lib/Sema/OpenCLBuiltins.td
+++ b/clang/lib/Sema/OpenCLBuiltins.td
@@ -806,17 +806,17 @@ multiclass VloadVstore<list<AddressSpace> addrspaces, bit defStores> {
   foreach AS = addrspaces in {
     foreach VSize = [2, 3, 4, 8, 16] in {
       foreach name = ["vload" # VSize] in {
-        def : Builtin<name, [VectorType<Char, VSize>, Size, PointerType<ConstType<Char>, AS>]>;
-        def : Builtin<name, [VectorType<UChar, VSize>, Size, PointerType<ConstType<UChar>, AS>]>;
-        def : Builtin<name, [VectorType<Short, VSize>, Size, PointerType<ConstType<Short>, AS>]>;
-        def : Builtin<name, [VectorType<UShort, VSize>, Size, PointerType<ConstType<UShort>, AS>]>;
-        def : Builtin<name, [VectorType<Int, VSize>, Size, PointerType<ConstType<Int>, AS>]>;
-        def : Builtin<name, [VectorType<UInt, VSize>, Size, PointerType<ConstType<UInt>, AS>]>;
-        def : Builtin<name, [VectorType<Long, VSize>, Size, PointerType<ConstType<Long>, AS>]>;
-        def : Builtin<name, [VectorType<ULong, VSize>, Size, PointerType<ConstType<ULong>, AS>]>;
-        def : Builtin<name, [VectorType<Float, VSize>, Size, PointerType<ConstType<Float>, AS>]>;
-        def : Builtin<name, [VectorType<Double, VSize>, Size, PointerType<ConstType<Double>, AS>]>;
-        def : Builtin<name, [VectorType<Half, VSize>, Size, PointerType<ConstType<Half>, AS>]>;
+        def : Builtin<name, [VectorType<Char, VSize>, Size, PointerType<ConstType<Char>, AS>], Attr.Pure>;
+        def : Builtin<name, [VectorType<UChar, VSize>, Size, PointerType<ConstType<UChar>, AS>], Attr.Pure>;
+        def : Builtin<name, [VectorType<Short, VSize>, Size, PointerType<ConstType<Short>, AS>], Attr.Pure>;
+        def : Builtin<name, [VectorType<UShort, VSize>, Size, PointerType<ConstType<UShort>, AS>], Attr.Pure>;
+        def : Builtin<name, [VectorType<Int, VSize>, Size, PointerType<ConstType<Int>, AS>], Attr.Pure>;
+        def : Builtin<name, [VectorType<UInt, VSize>, Size, PointerType<ConstType<UInt>, AS>], Attr.Pure>;
+        def : Builtin<name, [VectorType<Long, VSize>, Size, PointerType<ConstType<Long>, AS>], Attr.Pure>;
+        def : Builtin<name, [VectorType<ULong, VSize>, Size, PointerType<ConstType<ULong>, AS>], Attr.Pure>;
+        def : Builtin<name, [VectorType<Float, VSize>, Size, PointerType<ConstType<Float>, AS>], Attr.Pure>;
+        def : Builtin<name, [VectorType<Double, VSize>, Size, PointerType<ConstType<Double>, AS>], Attr.Pure>;
+        def : Builtin<name, [VectorType<Half, VSize>, Size, PointerType<ConstType<Half>, AS>], Attr.Pure>;
       }
       if defStores then {
         foreach name = ["vstore" # VSize] in {
@@ -848,10 +848,10 @@ defm : VloadVstore<[ConstantAS], 0>;
 
 multiclass VloadVstoreHalf<list<AddressSpace> addrspaces, bit defStores> {
   foreach AS = addrspaces in {
-    def : Builtin<"vload_half", [Float, Size, PointerType<ConstType<Half>, AS>]>;
+    def : Builtin<"vload_half", [Float, Size, PointerType<ConstType<Half>, AS>], Attr.Pure>;
     foreach VSize = [2, 3, 4, 8, 16] in {
       foreach name = ["vload_half" # VSize, "vloada_half" # VSize] in {
-        def : Builtin<name, [VectorType<Float, VSize>, Size, PointerType<ConstType<Half>, AS>]>;
+        def : Builtin<name, [VectorType<Float, VSize>, Size, PointerType<ConstType<Half>, AS>], Attr.Pure>;
       }
     }
     if defStores then {
@@ -877,7 +877,7 @@ let MaxVersion = CL20 in {
 let MinVersion = CL20 in {
   defm : VloadVstoreHalf<[GenericAS], 1>;
 }
-// vload with constant address space is available regardless of version.
+// vload_half and vloada_half with constant address space are available regardless of version.
 defm : VloadVstoreHalf<[ConstantAS], 0>;
 
 // OpenCL v3.0 s6.15.8 - Synchronization Functions.
diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp
index a2b8f475aa8c..734ed0f62ec6 100644
--- a/clang/lib/Sema/Sema.cpp
+++ b/clang/lib/Sema/Sema.cpp
@@ -1881,8 +1881,8 @@ void Sema::checkTypeSupport(QualType Ty, SourceLocation Loc, ValueDecl *D) {
     if (Ty->isDependentType())
       return;
 
-    if (Ty->isExtIntType()) {
-      if (!Context.getTargetInfo().hasExtIntType()) {
+    if (Ty->isBitIntType()) {
+      if (!Context.getTargetInfo().hasBitIntType()) {
         PartialDiagnostic PD = PDiag(diag::err_target_unsupported_type);
         if (D)
           PD << D;
diff --git a/clang/lib/Sema/SemaAttr.cpp b/clang/lib/Sema/SemaAttr.cpp
index 100f8e36a9b8..b69492768848 100644
--- a/clang/lib/Sema/SemaAttr.cpp
+++ b/clang/lib/Sema/SemaAttr.cpp
@@ -792,7 +792,7 @@ attrMatcherRuleListToString(ArrayRef<attr::SubjectMatchRule> Rules) {
       OS << (I.index() == Rules.size() - 1 ? ", and " : ", ");
     OS << "'" << attr::getSubjectMatchRuleSpelling(I.value()) << "'";
   }
-  return OS.str();
+  return Result;
 }
 
 } // end anonymous namespace
diff --git a/clang/lib/Sema/SemaCUDA.cpp b/clang/lib/Sema/SemaCUDA.cpp
index 840b3daae63c..59601c5ce79d 100644
--- a/clang/lib/Sema/SemaCUDA.cpp
+++ b/clang/lib/Sema/SemaCUDA.cpp
@@ -886,7 +886,6 @@ void Sema::CUDACheckLambdaCapture(CXXMethodDecl *Callee,
                           diag::warn_maybe_capture_bad_target_this_ptr, Callee,
                           *this);
   }
-  return;
 }
 
 void Sema::CUDASetLambdaAttrs(CXXMethodDecl *Method) {
diff --git a/clang/lib/Sema/SemaCXXScopeSpec.cpp b/clang/lib/Sema/SemaCXXScopeSpec.cpp
index c4826b5a6e8f..8cecf6c6ab4f 100644
--- a/clang/lib/Sema/SemaCXXScopeSpec.cpp
+++ b/clang/lib/Sema/SemaCXXScopeSpec.cpp
@@ -736,8 +736,15 @@ bool Sema::BuildCXXNestedNameSpecifier(Scope *S, NestedNameSpecInfo &IdInfo,
 
     QualType T =
         Context.getTypeDeclType(cast<TypeDecl>(SD->getUnderlyingDecl()));
+
+    if (T->isEnumeralType())
+      Diag(IdInfo.IdentifierLoc, diag::warn_cxx98_compat_enum_nested_name_spec);
+
     TypeLocBuilder TLB;
-    if (isa<InjectedClassNameType>(T)) {
+    if (const auto *USD = dyn_cast<UsingShadowDecl>(SD)) {
+      T = Context.getUsingType(USD, T);
+      TLB.pushTypeSpec(T).setNameLoc(IdInfo.IdentifierLoc);
+    } else if (isa<InjectedClassNameType>(T)) {
       InjectedClassNameTypeLoc InjectedTL
         = TLB.push<InjectedClassNameTypeLoc>(T);
       InjectedTL.setNameLoc(IdInfo.IdentifierLoc);
@@ -770,9 +777,6 @@ bool Sema::BuildCXXNestedNameSpecifier(Scope *S, NestedNameSpecInfo &IdInfo,
       llvm_unreachable("Unhandled TypeDecl node in nested-name-specifier");
     }
 
-    if (T->isEnumeralType())
-      Diag(IdInfo.IdentifierLoc, diag::warn_cxx98_compat_enum_nested_name_spec);
-
     SS.Extend(Context, SourceLocation(), TLB.getTypeLocInContext(Context, T),
               IdInfo.CCLoc);
     return false;
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 33e2b3b5027d..4e83fa1fffca 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -195,6 +195,29 @@ static bool SemaBuiltinAddressof(Sema &S, CallExpr *TheCall) {
   return false;
 }
 
+/// Check that the argument to __builtin_function_start is a function.
+static bool SemaBuiltinFunctionStart(Sema &S, CallExpr *TheCall) {
+  if (checkArgCount(S, TheCall, 1))
+    return true;
+
+  ExprResult Arg = S.DefaultFunctionArrayLvalueConversion(TheCall->getArg(0));
+  if (Arg.isInvalid())
+    return true;
+
+  TheCall->setArg(0, Arg.get());
+  const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(
+      Arg.get()->getAsBuiltinConstantDeclRef(S.getASTContext()));
+
+  if (!FD) {
+    S.Diag(TheCall->getBeginLoc(), diag::err_function_start_invalid_type)
+        << TheCall->getSourceRange();
+    return true;
+  }
+
+  return !S.checkAddressOfFunctionIsAvailable(FD, /*Complain=*/true,
+                                              TheCall->getBeginLoc());
+}
+
 /// Check the number of arguments and set the result type to
 /// the argument type.
 static bool SemaBuiltinPreserveAI(Sema &S, CallExpr *TheCall) {
@@ -325,17 +348,17 @@ static bool SemaBuiltinOverflow(Sema &S, CallExpr *TheCall,
     }
   }
 
-  // Disallow signed ExtIntType args larger than 128 bits to mul function until
-  // we improve backend support.
+  // Disallow signed bit-precise integer args larger than 128 bits to mul
+  // function until we improve backend support.
   if (BuiltinID == Builtin::BI__builtin_mul_overflow) {
     for (unsigned I = 0; I < 3; ++I) {
       const auto Arg = TheCall->getArg(I);
       // Third argument will be a pointer.
       auto Ty = I < 2 ? Arg->getType() : Arg->getType()->getPointeeType();
-      if (Ty->isExtIntType() && Ty->isSignedIntegerType() &&
+      if (Ty->isBitIntType() && Ty->isSignedIntegerType() &&
           S.getASTContext().getIntWidth(Ty) > 128)
         return S.Diag(Arg->getBeginLoc(),
-                      diag::err_overflow_builtin_ext_int_max_size)
+                      diag::err_overflow_builtin_bit_int_max_size)
                << 128;
     }
   }
@@ -446,14 +469,14 @@ public:
       break;
     }
 
-    auto OptionalFW = FS.getFieldWidth();
-    if (OptionalFW.getHowSpecified() !=
+    analyze_format_string::OptionalAmount FW = FS.getFieldWidth();
+    if (FW.getHowSpecified() !=
         analyze_format_string::OptionalAmount::HowSpecified::Constant)
       return true;
 
-    unsigned SourceSize = OptionalFW.getConstantAmount() + NulByte;
+    unsigned SourceSize = FW.getConstantAmount() + NulByte;
 
-    auto DestSizeAPS = ComputeSizeArgument(FS.getArgIndex());
+    Optional<llvm::APSInt> DestSizeAPS = ComputeSizeArgument(FS.getArgIndex());
     if (!DestSizeAPS)
       return true;
 
@@ -652,20 +675,53 @@ void Sema::checkFortifiedBuiltinMemoryFunction(FunctionDecl *FD,
       isConstantEvaluated())
     return;
 
-  unsigned BuiltinID = FD->getBuiltinID(/*ConsiderWrappers=*/true);
+  bool UseDABAttr = false;
+  const FunctionDecl *UseDecl = FD;
+
+  const auto *DABAttr = FD->getAttr<DiagnoseAsBuiltinAttr>();
+  if (DABAttr) {
+    UseDecl = DABAttr->getFunction();
+    assert(UseDecl && "Missing FunctionDecl in DiagnoseAsBuiltin attribute!");
+    UseDABAttr = true;
+  }
+
+  unsigned BuiltinID = UseDecl->getBuiltinID(/*ConsiderWrappers=*/true);
+
   if (!BuiltinID)
     return;
 
   const TargetInfo &TI = getASTContext().getTargetInfo();
   unsigned SizeTypeWidth = TI.getTypeWidth(TI.getSizeType());
 
+  auto TranslateIndex = [&](unsigned Index) -> Optional<unsigned> {
+    // If we refer to a diagnose_as_builtin attribute, we need to change the
+    // argument index to refer to the arguments of the called function. Unless
+    // the index is out of bounds, which presumably means it's a variadic
+    // function.
+    if (!UseDABAttr)
+      return Index;
+    unsigned DABIndices = DABAttr->argIndices_size();
+    unsigned NewIndex = Index < DABIndices
+                            ? DABAttr->argIndices_begin()[Index]
+                            : Index - DABIndices + FD->getNumParams();
+    if (NewIndex >= TheCall->getNumArgs())
+      return llvm::None;
+    return NewIndex;
+  };
+
   auto ComputeExplicitObjectSizeArgument =
       [&](unsigned Index) -> Optional<llvm::APSInt> {
+    Optional<unsigned> IndexOptional = TranslateIndex(Index);
+    if (!IndexOptional)
+      return llvm::None;
+    unsigned NewIndex = IndexOptional.getValue();
     Expr::EvalResult Result;
-    Expr *SizeArg = TheCall->getArg(Index);
+    Expr *SizeArg = TheCall->getArg(NewIndex);
     if (!SizeArg->EvaluateAsInt(Result, getASTContext()))
       return llvm::None;
-    return Result.Val.getInt();
+    llvm::APSInt Integer = Result.Val.getInt();
+    Integer.setIsUnsigned(true);
+    return Integer;
   };
 
   auto ComputeSizeArgument = [&](unsigned Index) -> Optional<llvm::APSInt> {
@@ -680,7 +736,12 @@ void Sema::checkFortifiedBuiltinMemoryFunction(FunctionDecl *FD,
         BOSType = POS->getType();
     }
 
-    const Expr *ObjArg = TheCall->getArg(Index);
+    Optional<unsigned> IndexOptional = TranslateIndex(Index);
+    if (!IndexOptional)
+      return llvm::None;
+    unsigned NewIndex = IndexOptional.getValue();
+
+    const Expr *ObjArg = TheCall->getArg(NewIndex);
     uint64_t Result;
     if (!ObjArg->tryEvaluateObjectSize(Result, getASTContext(), BOSType))
       return llvm::None;
@@ -690,7 +751,12 @@ void Sema::checkFortifiedBuiltinMemoryFunction(FunctionDecl *FD,
   };
 
   auto ComputeStrLenArgument = [&](unsigned Index) -> Optional<llvm::APSInt> {
-    Expr *ObjArg = TheCall->getArg(Index);
+    Optional<unsigned> IndexOptional = TranslateIndex(Index);
+    if (!IndexOptional)
+      return llvm::None;
+    unsigned NewIndex = IndexOptional.getValue();
+
+    const Expr *ObjArg = TheCall->getArg(NewIndex);
     uint64_t Result;
     if (!ObjArg->tryEvaluateStrLen(Result, getASTContext()))
       return llvm::None;
@@ -898,7 +964,8 @@ void Sema::checkFortifiedBuiltinMemoryFunction(FunctionDecl *FD,
   }
 
   if (!SourceSize || !DestinationSize ||
-      SourceSize.getValue().ule(DestinationSize.getValue()))
+      llvm::APSInt::compareValues(SourceSize.getValue(),
+                                  DestinationSize.getValue()) <= 0)
     return;
 
   StringRef FunctionName = GetFunctionName();
@@ -1874,6 +1941,10 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
     if (SemaBuiltinAddressof(*this, TheCall))
       return ExprError();
     break;
+  case Builtin::BI__builtin_function_start:
+    if (SemaBuiltinFunctionStart(*this, TheCall))
+      return ExprError();
+    break;
   case Builtin::BI__builtin_is_aligned:
   case Builtin::BI__builtin_align_up:
   case Builtin::BI__builtin_align_down:
@@ -2098,20 +2169,85 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
     break;
   }
 
-  case Builtin::BI__builtin_elementwise_abs:
-    if (SemaBuiltinElementwiseMathOneArg(TheCall))
+  // __builtin_elementwise_abs restricts the element type to signed integers or
+  // floating point types only.
+  case Builtin::BI__builtin_elementwise_abs: {
+    if (PrepareBuiltinElementwiseMathOneArgCall(TheCall))
+      return ExprError();
+
+    QualType ArgTy = TheCall->getArg(0)->getType();
+    QualType EltTy = ArgTy;
+
+    if (auto *VecTy = EltTy->getAs<VectorType>())
+      EltTy = VecTy->getElementType();
+    if (EltTy->isUnsignedIntegerType()) {
+      Diag(TheCall->getArg(0)->getBeginLoc(),
+           diag::err_builtin_invalid_arg_type)
+          << 1 << /* signed integer or float ty*/ 3 << ArgTy;
+      return ExprError();
+    }
+    break;
+  }
+
+  // __builtin_elementwise_ceil restricts the element type to floating point
+  // types only.
+  case Builtin::BI__builtin_elementwise_ceil: {
+    if (PrepareBuiltinElementwiseMathOneArgCall(TheCall))
+      return ExprError();
+
+    QualType ArgTy = TheCall->getArg(0)->getType();
+    QualType EltTy = ArgTy;
+
+    if (auto *VecTy = EltTy->getAs<VectorType>())
+      EltTy = VecTy->getElementType();
+    if (!EltTy->isFloatingType()) {
+      Diag(TheCall->getArg(0)->getBeginLoc(),
+           diag::err_builtin_invalid_arg_type)
+          << 1 << /* float ty*/ 5 << ArgTy;
+
       return ExprError();
+    }
     break;
+  }
+
   case Builtin::BI__builtin_elementwise_min:
   case Builtin::BI__builtin_elementwise_max:
     if (SemaBuiltinElementwiseMath(TheCall))
       return ExprError();
     break;
   case Builtin::BI__builtin_reduce_max:
-  case Builtin::BI__builtin_reduce_min:
-    if (SemaBuiltinReduceMath(TheCall))
+  case Builtin::BI__builtin_reduce_min: {
+    if (PrepareBuiltinReduceMathOneArgCall(TheCall))
+      return ExprError();
+
+    const Expr *Arg = TheCall->getArg(0);
+    const auto *TyA = Arg->getType()->getAs<VectorType>();
+    if (!TyA) {
+      Diag(Arg->getBeginLoc(), diag::err_builtin_invalid_arg_type)
+          << 1 << /* vector ty*/ 4 << Arg->getType();
+      return ExprError();
+    }
+
+    TheCall->setType(TyA->getElementType());
+    break;
+  }
+
+  // __builtin_reduce_xor supports vector of integers only.
+  case Builtin::BI__builtin_reduce_xor: {
+    if (PrepareBuiltinReduceMathOneArgCall(TheCall))
+      return ExprError();
+
+    const Expr *Arg = TheCall->getArg(0);
+    const auto *TyA = Arg->getType()->getAs<VectorType>();
+    if (!TyA || !TyA->getElementType()->isIntegerType()) {
+      Diag(Arg->getBeginLoc(), diag::err_builtin_invalid_arg_type)
+          << 1  << /* vector of integers */ 6 << Arg->getType();
       return ExprError();
+    }
+    TheCall->setType(TyA->getElementType());
     break;
+  }
+
   case Builtin::BI__builtin_matrix_transpose:
     return SemaBuiltinMatrixTranspose(TheCall, TheCallResult);
 
@@ -3496,14 +3632,43 @@ bool Sema::CheckPPCBuiltinFunctionCall(const TargetInfo &TI, unsigned BuiltinID,
   case PPC::BI__builtin_altivec_dss:
     return SemaBuiltinConstantArgRange(TheCall, 0, 0, 3);
   case PPC::BI__builtin_tbegin:
-  case PPC::BI__builtin_tend: i = 0; l = 0; u = 1; break;
-  case PPC::BI__builtin_tsr: i = 0; l = 0; u = 7; break;
+  case PPC::BI__builtin_tend:
+    return SemaBuiltinConstantArgRange(TheCall, 0, 0, 1) ||
+           SemaFeatureCheck(*this, TheCall, "htm",
+                            diag::err_ppc_builtin_requires_htm);
+  case PPC::BI__builtin_tsr:
+    return SemaBuiltinConstantArgRange(TheCall, 0, 0, 7) ||
+           SemaFeatureCheck(*this, TheCall, "htm",
+                            diag::err_ppc_builtin_requires_htm);
   case PPC::BI__builtin_tabortwc:
-  case PPC::BI__builtin_tabortdc: i = 0; l = 0; u = 31; break;
+  case PPC::BI__builtin_tabortdc:
+    return SemaBuiltinConstantArgRange(TheCall, 0, 0, 31) ||
+           SemaFeatureCheck(*this, TheCall, "htm",
+                            diag::err_ppc_builtin_requires_htm);
   case PPC::BI__builtin_tabortwci:
   case PPC::BI__builtin_tabortdci:
-    return SemaBuiltinConstantArgRange(TheCall, 0, 0, 31) ||
-           SemaBuiltinConstantArgRange(TheCall, 2, 0, 31);
+    return SemaFeatureCheck(*this, TheCall, "htm",
+                            diag::err_ppc_builtin_requires_htm) ||
+           (SemaBuiltinConstantArgRange(TheCall, 0, 0, 31) ||
+            SemaBuiltinConstantArgRange(TheCall, 2, 0, 31));
+  case PPC::BI__builtin_tabort:
+  case PPC::BI__builtin_tcheck:
+  case PPC::BI__builtin_treclaim:
+  case PPC::BI__builtin_trechkpt:
+  case PPC::BI__builtin_tendall:
+  case PPC::BI__builtin_tresume:
+  case PPC::BI__builtin_tsuspend:
+  case PPC::BI__builtin_get_texasr:
+  case PPC::BI__builtin_get_texasru:
+  case PPC::BI__builtin_get_tfhar:
+  case PPC::BI__builtin_get_tfiar:
+  case PPC::BI__builtin_set_texasr:
+  case PPC::BI__builtin_set_texasru:
+  case PPC::BI__builtin_set_tfhar:
+  case PPC::BI__builtin_set_tfiar:
+  case PPC::BI__builtin_ttest:
+    return SemaFeatureCheck(*this, TheCall, "htm",
+                            diag::err_ppc_builtin_requires_htm);
   // According to GCC 'Basic PowerPC Built-in Functions Available on ISA 2.05',
   // __builtin_(un)pack_longdouble are available only if long double uses IBM
   // extended double representation.
@@ -5819,8 +5984,8 @@ ExprResult Sema::BuildAtomicExpr(SourceRange CallRange, SourceRange ExprRange,
                 ? 0
                 : 1);
 
-  if (ValType->isExtIntType()) {
-    Diag(Ptr->getExprLoc(), diag::err_atomic_builtin_ext_int_prohibit);
+  if (ValType->isBitIntType()) {
+    Diag(Ptr->getExprLoc(), diag::err_atomic_builtin_bit_int_prohibit);
     return ExprError();
   }
 
@@ -6217,11 +6382,11 @@ Sema::SemaBuiltinAtomicOverloaded(ExprResult TheCallResult) {
   // gracefully.
   TheCall->setType(ResultType);
 
-  // Prohibit use of _ExtInt with atomic builtins.
-  // The arguments would have already been converted to the first argument's
-  // type, so only need to check the first argument.
-  const auto *ExtIntValType = ValType->getAs<ExtIntType>();
-  if (ExtIntValType && !llvm::isPowerOf2_64(ExtIntValType->getNumBits())) {
+  // Prohibit problematic uses of bit-precise integer types with atomic
+  // builtins. The arguments would have already been converted to the first
+  // argument's type, so only need to check the first argument.
+  const auto *BitIntValType = ValType->getAs<BitIntType>();
+  if (BitIntValType && !llvm::isPowerOf2_64(BitIntValType->getNumBits())) {
     Diag(FirstArg->getExprLoc(), diag::err_atomic_builtin_ext_int_size);
     return ExprError();
   }
@@ -11249,7 +11414,7 @@ struct IntRange {
                         false/*NonNegative*/);
     }
 
-    if (const auto *EIT = dyn_cast<ExtIntType>(T))
+    if (const auto *EIT = dyn_cast<BitIntType>(T))
       return IntRange(EIT->getNumBits(), EIT->isUnsigned());
 
     const BuiltinType *BT = cast<BuiltinType>(T);
@@ -11275,7 +11440,7 @@ struct IntRange {
     if (const EnumType *ET = dyn_cast<EnumType>(T))
       T = C.getCanonicalType(ET->getDecl()->getIntegerType()).getTypePtr();
 
-    if (const auto *EIT = dyn_cast<ExtIntType>(T))
+    if (const auto *EIT = dyn_cast<BitIntType>(T))
       return IntRange(EIT->getNumBits(), EIT->isUnsigned());
 
     const BuiltinType *BT = cast<BuiltinType>(T);
@@ -16697,26 +16862,19 @@ static bool checkMathBuiltinElementType(Sema &S, SourceLocation Loc,
   return false;
 }
 
-bool Sema::SemaBuiltinElementwiseMathOneArg(CallExpr *TheCall) {
+bool Sema::PrepareBuiltinElementwiseMathOneArgCall(CallExpr *TheCall) {
   if (checkArgCount(*this, TheCall, 1))
     return true;
 
   ExprResult A = UsualUnaryConversions(TheCall->getArg(0));
-  SourceLocation ArgLoc = TheCall->getArg(0)->getBeginLoc();
   if (A.isInvalid())
     return true;
 
   TheCall->setArg(0, A.get());
   QualType TyA = A.get()->getType();
-  if (checkMathBuiltinElementType(*this, ArgLoc, TyA))
-    return true;
 
-  QualType EltTy = TyA;
-  if (auto *VecTy = EltTy->getAs<VectorType>())
-    EltTy = VecTy->getElementType();
-  if (EltTy->isUnsignedIntegerType())
-    return Diag(ArgLoc, diag::err_builtin_invalid_arg_type)
-           << 1 << /*signed integer or float ty*/ 3 << TyA;
+  if (checkMathBuiltinElementType(*this, A.get()->getBeginLoc(), TyA))
+    return true;
 
   TheCall->setType(TyA);
   return false;
@@ -16752,7 +16910,7 @@ bool Sema::SemaBuiltinElementwiseMath(CallExpr *TheCall) {
   return false;
 }
 
-bool Sema::SemaBuiltinReduceMath(CallExpr *TheCall) {
+bool Sema::PrepareBuiltinReduceMathOneArgCall(CallExpr *TheCall) {
   if (checkArgCount(*this, TheCall, 1))
     return true;
 
@@ -16761,14 +16919,6 @@ bool Sema::SemaBuiltinReduceMath(CallExpr *TheCall) {
     return true;
 
   TheCall->setArg(0, A.get());
-  const VectorType *TyA = A.get()->getType()->getAs<VectorType>();
-  if (!TyA) {
-    SourceLocation ArgLoc = TheCall->getArg(0)->getBeginLoc();
-    return Diag(ArgLoc, diag::err_builtin_invalid_arg_type)
-           << 1 << /* vector ty*/ 4 << A.get()->getType();
-  }
-
-  TheCall->setType(TyA->getElementType());
   return false;
 }
 
diff --git a/clang/lib/Sema/SemaCodeComplete.cpp b/clang/lib/Sema/SemaCodeComplete.cpp
index 083a67db7a91..93c07ccc891f 100644
--- a/clang/lib/Sema/SemaCodeComplete.cpp
+++ b/clang/lib/Sema/SemaCodeComplete.cpp
@@ -569,7 +569,6 @@ void PreferredTypeBuilder::enterMemAccess(Sema &S, SourceLocation Tok,
     return;
   // Keep the expected type, only update the location.
   ExpectedLoc = Tok;
-  return;
 }
 
 void PreferredTypeBuilder::enterUnary(Sema &S, SourceLocation Tok,
diff --git a/clang/lib/Sema/SemaCoroutine.cpp b/clang/lib/Sema/SemaCoroutine.cpp
index b999b08d1662..e89cecd08cca 100644
--- a/clang/lib/Sema/SemaCoroutine.cpp
+++ b/clang/lib/Sema/SemaCoroutine.cpp
@@ -237,9 +237,9 @@ static bool isValidCoroutineContext(Sema &S, SourceLocation Loc,
   // placeholder type shall not be a coroutine."
   if (FD->getReturnType()->isUndeducedType())
     DiagInvalid(DiagAutoRet);
-  // [dcl.fct.def.coroutine]p1: "The parameter-declaration-clause of the
-  // coroutine shall not terminate with an ellipsis that is not part of a
-  // parameter-declaration."
+  // [dcl.fct.def.coroutine]p1
+  // The parameter-declaration-clause of the coroutine shall not terminate with
+  // an ellipsis that is not part of a parameter-declaration.
   if (FD->isVariadic())
     DiagInvalid(DiagVarargs);
 
@@ -579,8 +579,12 @@ VarDecl *Sema::buildCoroutinePromise(SourceLocation Loc) {
                                    /*TopLevelOfInitList=*/false,
                                    /*TreatUnavailableAsInvalid=*/false);
 
-    // Attempt to initialize the promise type with the arguments.
-    // If that fails, fall back to the promise type's default constructor.
+    // [dcl.fct.def.coroutine]5.7
+    // promise-constructor-arguments is determined as follows: overload
+    // resolution is performed on a promise constructor call created by
+    // assembling an argument list  q_1 ... q_n . If a viable constructor is
+    // found ([over.match.viable]), then promise-constructor-arguments is ( q_1
+    // , ...,  q_n ), otherwise promise-constructor-arguments is empty.
     if (InitSeq) {
       ExprResult Result = InitSeq.Perform(*this, Entity, Kind, CtorArgExprs);
       if (Result.isInvalid()) {
@@ -648,6 +652,10 @@ static void checkNoThrow(Sema &S, const Stmt *E,
           return;
       }
       if (ThrowingDecls.empty()) {
+        // [dcl.fct.def.coroutine]p15
+        //   The expression co_­await promise.final_­suspend() shall not be
+        //   potentially-throwing ([except.spec]).
+        //
         // First time seeing an error, emit the error message.
         S.Diag(cast<FunctionDecl>(S.CurContext)->getLocation(),
                diag::err_coroutine_promise_final_suspend_requires_nothrow);
@@ -995,9 +1003,8 @@ static Expr *buildStdNoThrowDeclRef(Sema &S, SourceLocation Loc) {
   LookupResult Result(S, &S.PP.getIdentifierTable().get("nothrow"), Loc,
                       Sema::LookupOrdinaryName);
   if (!S.LookupQualifiedName(Result, Std)) {
-    // FIXME: <coroutine> should have been included already.
-    // If we require it to include <new> then this diagnostic is no longer
-    // needed.
+    // <coroutine> is not requred to include <new>, so we couldn't omit
+    // the check here.
     S.Diag(Loc, diag::err_implicit_coroutine_std_nothrow_type_not_found);
     return nullptr;
   }
@@ -1029,9 +1036,21 @@ static FunctionDecl *findDeleteForPromise(Sema &S, SourceLocation Loc,
   auto *PointeeRD = PromiseType->getAsCXXRecordDecl();
   assert(PointeeRD && "PromiseType must be a CxxRecordDecl type");
 
+  // [dcl.fct.def.coroutine]p12
+  // The deallocation function's name is looked up by searching for it in the
+  // scope of the promise type. If nothing is found, a search is performed in
+  // the global scope.
   if (S.FindDeallocationFunction(Loc, PointeeRD, DeleteName, OperatorDelete))
     return nullptr;
 
+  // FIXME: We didn't implement following selection:
+  // [dcl.fct.def.coroutine]p12
+  //   If both a usual deallocation function with only a pointer parameter and a
+  //   usual deallocation function with both a pointer parameter and a size
+  //   parameter are found, then the selected deallocation function shall be the
+  //   one with two parameters. Otherwise, the selected deallocation function
+  //   shall be the function with one parameter.
+
   if (!OperatorDelete) {
     // Look for a global declaration.
     const bool CanProvideSize = S.isCompleteType(Loc, PromiseType);
@@ -1062,8 +1081,8 @@ void Sema::CheckCompletedCoroutineBody(FunctionDecl *FD, Stmt *&Body) {
     return;
   }
 
-  // Coroutines [stmt.return]p1:
-  //   A return statement shall not appear in a coroutine.
+  // [stmt.return.coroutine]p1:
+  //   A coroutine shall not enclose a return statement ([stmt.return]).
   if (Fn->FirstReturnLoc.isValid()) {
     assert(Fn->FirstCoroutineStmtLoc.isValid() &&
                    "first coroutine location not set");
@@ -1164,12 +1183,15 @@ bool CoroutineStmtBuilder::makeReturnOnAllocFailure() {
   assert(!IsPromiseDependentType &&
          "cannot make statement while the promise type is dependent");
 
-  // [dcl.fct.def.coroutine]/8
-  // The unqualified-id get_return_object_on_allocation_failure is looked up in
-  // the scope of class P by class member access lookup (3.4.5). ...
-  // If an allocation function returns nullptr, ... the coroutine return value
-  // is obtained by a call to ... get_return_object_on_allocation_failure().
-
+  // [dcl.fct.def.coroutine]p10
+  //   If a search for the name get_­return_­object_­on_­allocation_­failure in
+  // the scope of the promise type ([class.member.lookup]) finds any
+  // declarations, then the result of a call to an allocation function used to
+  // obtain storage for the coroutine state is assumed to return nullptr if it
+  // fails to obtain storage, ... If the allocation function returns nullptr,
+  // ... and the return value is obtained by a call to
+  // T::get_­return_­object_­on_­allocation_­failure(), where T is the
+  // promise type.
   DeclarationName DN =
       S.PP.getIdentifierInfo("get_return_object_on_allocation_failure");
   LookupResult Found(S, DN, Loc, Sema::LookupMemberName);
@@ -1215,12 +1237,11 @@ bool CoroutineStmtBuilder::makeNewAndDeleteExpr() {
 
   const bool RequiresNoThrowAlloc = ReturnStmtOnAllocFailure != nullptr;
 
-  // [dcl.fct.def.coroutine]/7
-  // Lookup allocation functions using a parameter list composed of the
-  // requested size of the coroutine state being allocated, followed by
-  // the coroutine function's arguments. If a matching allocation function
-  // exists, use it. Otherwise, use an allocation function that just takes
-  // the requested size.
+  // According to [dcl.fct.def.coroutine]p9, Lookup allocation functions using a
+  // parameter list composed of the requested size of the coroutine state being
+  // allocated, followed by the coroutine function's arguments. If a matching
+  // allocation function exists, use it. Otherwise, use an allocation function
+  // that just takes the requested size.
 
   FunctionDecl *OperatorNew = nullptr;
   FunctionDecl *OperatorDelete = nullptr;
@@ -1228,21 +1249,32 @@ bool CoroutineStmtBuilder::makeNewAndDeleteExpr() {
   bool PassAlignment = false;
   SmallVector<Expr *, 1> PlacementArgs;
 
-  // [dcl.fct.def.coroutine]/7
-  // "The allocation function’s name is looked up in the scope of P.
-  // [...] If the lookup finds an allocation function in the scope of P,
-  // overload resolution is performed on a function call created by assembling
-  // an argument list. The first argument is the amount of space requested,
-  // and has type std::size_t. The lvalues p1 ... pn are the succeeding
-  // arguments."
+  // [dcl.fct.def.coroutine]p9
+  //   An implementation may need to allocate additional storage for a
+  //   coroutine.
+  // This storage is known as the coroutine state and is obtained by calling a
+  // non-array allocation function ([basic.stc.dynamic.allocation]). The
+  // allocation function's name is looked up by searching for it in the scope of
+  // the promise type.
+  // - If any declarations are found, overload resolution is performed on a
+  // function call created by assembling an argument list. The first argument is
+  // the amount of space requested, and has type std::size_t. The
+  // lvalues p1 ... pn are the succeeding arguments.
   //
   // ...where "p1 ... pn" are defined earlier as:
   //
-  // [dcl.fct.def.coroutine]/3
-  // "For a coroutine f that is a non-static member function, let P1 denote the
-  // type of the implicit object parameter (13.3.1) and P2 ... Pn be the types
-  // of the function parameters; otherwise let P1 ... Pn be the types of the
-  // function parameters. Let p1 ... pn be lvalues denoting those objects."
+  // [dcl.fct.def.coroutine]p3
+  //   The promise type of a coroutine is `std::coroutine_traits<R, P1, ...,
+  //   Pn>`
+  // , where R is the return type of the function, and `P1, ..., Pn` are the
+  // sequence of types of the non-object function parameters, preceded by the
+  // type of the object parameter ([dcl.fct]) if the coroutine is a non-static
+  // member function. [dcl.fct.def.coroutine]p4 In the following, p_i is an
+  // lvalue of type P_i, where p1 denotes the object parameter and p_i+1 denotes
+  // the i-th non-object function parameter for a non-static member function,
+  // and p_i denotes the i-th function parameter otherwise. For a non-static
+  // member function, q_1 is an lvalue that denotes *this; any other q_i is an
+  // lvalue that denotes the parameter copy corresponding to p_i.
   if (auto *MD = dyn_cast<CXXMethodDecl>(&FD)) {
     if (MD->isInstance() && !isLambdaCallOperator(MD)) {
       ExprResult ThisExpr = S.ActOnCXXThis(Loc);
@@ -1273,10 +1305,10 @@ bool CoroutineStmtBuilder::makeNewAndDeleteExpr() {
                             /*isArray*/ false, PassAlignment, PlacementArgs,
                             OperatorNew, UnusedResult, /*Diagnose*/ false);
 
-  // [dcl.fct.def.coroutine]/7
-  // "If no matching function is found, overload resolution is performed again
-  // on a function call created by passing just the amount of space required as
-  // an argument of type std::size_t."
+  // [dcl.fct.def.coroutine]p9
+  //   If no viable function is found ([over.match.viable]), overload resolution
+  // is performed again on a function call created by passing just the amount of
+  // space required as an argument of type std::size_t.
   if (!OperatorNew && !PlacementArgs.empty()) {
     PlacementArgs.clear();
     S.FindAllocationFunctions(Loc, SourceRange(), /*NewScope*/ Sema::AFS_Class,
@@ -1285,10 +1317,11 @@ bool CoroutineStmtBuilder::makeNewAndDeleteExpr() {
                               OperatorNew, UnusedResult, /*Diagnose*/ false);
   }
 
-  // [dcl.fct.def.coroutine]/7
-  // "The allocation function’s name is looked up in the scope of P. If this
-  // lookup fails, the allocation function’s name is looked up in the global
-  // scope."
+  // [dcl.fct.def.coroutine]p9
+  //   The allocation function's name is looked up by searching for it in the
+  // scope of the promise type.
+  // - If any declarations are found, ...
+  // - Otherwise, a search is performed in the global scope.
   if (!OperatorNew) {
     S.FindAllocationFunctions(Loc, SourceRange(), /*NewScope*/ Sema::AFS_Global,
                               /*DeleteScope*/ Sema::AFS_Both, PromiseType,
@@ -1328,8 +1361,12 @@ bool CoroutineStmtBuilder::makeNewAndDeleteExpr() {
     }
   }
 
-  if ((OperatorDelete = findDeleteForPromise(S, Loc, PromiseType)) == nullptr)
+  if ((OperatorDelete = findDeleteForPromise(S, Loc, PromiseType)) == nullptr) {
+    // FIXME: We should add an error here. According to:
+    // [dcl.fct.def.coroutine]p12
+    //   If no usual deallocation function is found, the program is ill-formed.
     return false;
+  }
 
   Expr *FramePtr =
       S.BuildBuiltinCallExpr(Loc, Builtin::BI__builtin_coro_frame, {});
@@ -1368,7 +1405,11 @@ bool CoroutineStmtBuilder::makeNewAndDeleteExpr() {
 
   SmallVector<Expr *, 2> DeleteArgs{CoroFree};
 
-  // Check if we need to pass the size.
+  // [dcl.fct.def.coroutine]p12
+  //   The selected deallocation function shall be called with the address of
+  //   the block of storage to be reclaimed as its first argument. If a
+  //   deallocation function with a parameter of type std::size_t is
+  //   used, the size of the block is passed as the corresponding argument.
   const auto *OpDeleteType =
       OpDeleteQualType.getTypePtr()->castAs<FunctionProtoType>();
   if (OpDeleteType->getNumParams() > 1)
@@ -1391,9 +1432,13 @@ bool CoroutineStmtBuilder::makeOnFallthrough() {
   assert(!IsPromiseDependentType &&
          "cannot make statement while the promise type is dependent");
 
-  // [dcl.fct.def.coroutine]/4
-  // The unqualified-ids 'return_void' and 'return_value' are looked up in
-  // the scope of class P. If both are found, the program is ill-formed.
+  // [dcl.fct.def.coroutine]/p6
+  // If searches for the names return_­void and return_­value in the scope of
+  // the promise type each find any declarations, the program is ill-formed.
+  // [Note 1: If return_­void is found, flowing off the end of a coroutine is
+  // equivalent to a co_­return with no operand. Otherwise, flowing off the end
+  // of a coroutine results in undefined behavior ([stmt.return.coroutine]). —
+  // end note]
   bool HasRVoid, HasRValue;
   LookupResult LRVoid =
       lookupMember(S, "return_void", PromiseRecordDecl, Loc, HasRVoid);
@@ -1414,18 +1459,20 @@ bool CoroutineStmtBuilder::makeOnFallthrough() {
         << LRValue.getLookupName();
     return false;
   } else if (!HasRVoid && !HasRValue) {
-    // FIXME: The PDTS currently specifies this case as UB, not ill-formed.
-    // However we still diagnose this as an error since until the PDTS is fixed.
-    S.Diag(FD.getLocation(),
-           diag::err_coroutine_promise_requires_return_function)
-        << PromiseRecordDecl;
-    S.Diag(PromiseRecordDecl->getLocation(), diag::note_defined_here)
-        << PromiseRecordDecl;
-    return false;
+    // We need to set 'Fallthrough'. Otherwise the other analysis part might
+    // think the coroutine has defined a return_value method. So it might emit
+    // **false** positive warning. e.g.,
+    //
+    //    promise_without_return_func foo() {
+    //        co_await something();
+    //    }
+    //
+    // Then AnalysisBasedWarning would emit a warning about `foo()` lacking a
+    // co_return statements, which isn't correct.
+    Fallthrough = S.ActOnNullStmt(PromiseRecordDecl->getLocation());
+    if (Fallthrough.isInvalid())
+      return false;
   } else if (HasRVoid) {
-    // If the unqualified-id return_void is found, flowing off the end of a
-    // coroutine is equivalent to a co_return with no operand. Otherwise,
-    // flowing off the end of a coroutine results in undefined behavior.
     Fallthrough = S.BuildCoreturnStmt(FD.getLocation(), nullptr,
                                       /*IsImplicit*/false);
     Fallthrough = S.ActOnFinishFullStmt(Fallthrough.get());
@@ -1481,8 +1528,9 @@ bool CoroutineStmtBuilder::makeOnException() {
 }
 
 bool CoroutineStmtBuilder::makeReturnObject() {
-  // Build implicit 'p.get_return_object()' expression and form initialization
-  // of return type from it.
+  // [dcl.fct.def.coroutine]p7
+  // The expression promise.get_­return_­object() is used to initialize the
+  // returned reference or prvalue result object of a call to a coroutine.
   ExprResult ReturnObject =
       buildPromiseCall(S, Fn.CoroutinePromise, Loc, "get_return_object", None);
   if (ReturnObject.isInvalid())
@@ -1620,6 +1668,12 @@ bool Sema::buildCoroutineParameterMoves(SourceLocation Loc) {
   if (!ScopeInfo->CoroutineParameterMoves.empty())
     return false;
 
+  // [dcl.fct.def.coroutine]p13
+  //   When a coroutine is invoked, after initializing its parameters
+  //   ([expr.call]), a copy is created for each coroutine parameter. For a
+  //   parameter of type cv T, the copy is a variable of type cv T with
+  //   automatic storage duration that is direct-initialized from an xvalue of
+  //   type T referring to the parameter.
   for (auto *PD : FD->parameters()) {
     if (PD->getType()->isDependentType())
       continue;
@@ -1636,7 +1690,9 @@ bool Sema::buildCoroutineParameterMoves(SourceLocation Loc) {
       CExpr = castForMoving(*this, PDRefExpr.get());
     else
       CExpr = PDRefExpr.get();
-
+    // [dcl.fct.def.coroutine]p13
+    //   The initialization and destruction of each parameter copy occurs in the
+    //   context of the called coroutine.
     auto D = buildVarDecl(*this, Loc, PD->getType(), PD->getIdentifier());
     AddInitializerToDecl(D, CExpr, /*DirectInit=*/true);
 
@@ -1661,43 +1717,53 @@ ClassTemplateDecl *Sema::lookupCoroutineTraits(SourceLocation KwLoc,
                                                SourceLocation FuncLoc,
                                                NamespaceDecl *&Namespace) {
   if (!StdCoroutineTraitsCache) {
-    NamespaceDecl *CoroNamespace = getStdNamespace();
-    LookupResult Result(*this, &PP.getIdentifierTable().get("coroutine_traits"),
-                        FuncLoc, LookupOrdinaryName);
-
-    if (!CoroNamespace || !LookupQualifiedName(Result, CoroNamespace)) {
-      /// Look up in namespace std::experimental, for compatibility.
-      /// TODO: Remove this extra lookup when <experimental/coroutine> is
-      /// removed.
-      CoroNamespace = lookupStdExperimentalNamespace();
-      if (!CoroNamespace || !LookupQualifiedName(Result, CoroNamespace)) {
-        Diag(KwLoc, diag::err_implied_coroutine_type_not_found)
-            << "std::coroutine_traits";
-        return nullptr;
-      }
+    // Because coroutines moved from std::experimental in the TS to std in
+    // C++20, we look in both places to give users time to transition their
+    // TS-specific code to C++20.  Diagnostics are given when the TS usage is
+    // discovered.
+    // TODO: Become stricter when <experimental/coroutine> is removed.
+
+    auto const &TraitIdent = PP.getIdentifierTable().get("coroutine_traits");
+
+    NamespaceDecl *StdSpace = getStdNamespace();
+    LookupResult ResStd(*this, &TraitIdent, FuncLoc, LookupOrdinaryName);
+    bool InStd = StdSpace && LookupQualifiedName(ResStd, StdSpace);
+
+    NamespaceDecl *ExpSpace = lookupStdExperimentalNamespace();
+    LookupResult ResExp(*this, &TraitIdent, FuncLoc, LookupOrdinaryName);
+    bool InExp = ExpSpace && LookupQualifiedName(ResExp, ExpSpace);
+
+    if (!InStd && !InExp) {
+      // The goggles, they found nothing!
+      Diag(KwLoc, diag::err_implied_coroutine_type_not_found)
+          << "std::coroutine_traits";
+      return nullptr;
+    }
+
+    if (!InStd) {
+      // Found only in std::experimental.
       Diag(KwLoc, diag::warn_deprecated_coroutine_namespace)
           << "coroutine_traits";
-    } else {
-      /// When we found coroutine_traits in std namespace. Make sure there is no
-      /// misleading definition in std::experimental namespace.
-      NamespaceDecl *ExpNamespace = lookupStdExperimentalNamespace();
-      LookupResult ExpResult(*this,
-                             &PP.getIdentifierTable().get("coroutine_traits"),
-                             FuncLoc, LookupOrdinaryName);
-      if (ExpNamespace && LookupQualifiedName(ExpResult, ExpNamespace)) {
-        Diag(KwLoc,
-             diag::err_mixed_use_std_and_experimental_namespace_for_coroutine);
-        return nullptr;
-      }
+    } else if (InExp) {
+      // Found in std and std::experimental.
+      Diag(KwLoc,
+           diag::err_mixed_use_std_and_experimental_namespace_for_coroutine);
+      Diag(KwLoc, diag::warn_deprecated_coroutine_namespace)
+          << "coroutine_traits";
+      return nullptr;
     }
 
+    // Prefer ::std to std::experimental.
+    auto &Result = InStd ? ResStd : ResExp;
+    CoroTraitsNamespaceCache = InStd ? StdSpace : ExpSpace;
+
+    // coroutine_traits is required to be a class template.
     if (!(StdCoroutineTraitsCache = Result.getAsSingle<ClassTemplateDecl>())) {
       Result.suppressDiagnostics();
       NamedDecl *Found = *Result.begin();
       Diag(Found->getLocation(), diag::err_malformed_std_coroutine_traits);
       return nullptr;
     }
-    CoroTraitsNamespaceCache = CoroNamespace;
   }
   Namespace = CoroTraitsNamespaceCache;
   return StdCoroutineTraitsCache;
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 7be71ca49ea2..3c58f1d19c04 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -372,6 +372,7 @@ ParsedType Sema::getTypeName(const IdentifierInfo &II, SourceLocation NameLoc,
   }
 
   NamedDecl *IIDecl = nullptr;
+  UsingShadowDecl *FoundUsingShadow = nullptr;
   switch (Result.getResultKind()) {
   case LookupResult::NotFound:
   case LookupResult::NotFoundInCurrentInstantiation:
@@ -441,8 +442,10 @@ ParsedType Sema::getTypeName(const IdentifierInfo &II, SourceLocation NameLoc,
           (AllowDeducedTemplate && getAsTypeTemplateDecl(RealRes))) {
         if (!IIDecl ||
             // Make the selection of the recovery decl deterministic.
-            RealRes->getLocation() < IIDecl->getLocation())
+            RealRes->getLocation() < IIDecl->getLocation()) {
           IIDecl = RealRes;
+          FoundUsingShadow = dyn_cast<UsingShadowDecl>(*Res);
+        }
       }
     }
 
@@ -465,6 +468,7 @@ ParsedType Sema::getTypeName(const IdentifierInfo &II, SourceLocation NameLoc,
 
   case LookupResult::Found:
     IIDecl = Result.getFoundDecl();
+    FoundUsingShadow = dyn_cast<UsingShadowDecl>(*Result.begin());
     break;
   }
 
@@ -491,14 +495,20 @@ ParsedType Sema::getTypeName(const IdentifierInfo &II, SourceLocation NameLoc,
     (void)DiagnoseUseOfDecl(IDecl, NameLoc);
     if (!HasTrailingDot)
       T = Context.getObjCInterfaceType(IDecl);
+    FoundUsingShadow = nullptr; // FIXME: Target must be a TypeDecl.
   } else if (auto *UD = dyn_cast<UnresolvedUsingIfExistsDecl>(IIDecl)) {
     (void)DiagnoseUseOfDecl(UD, NameLoc);
     // Recover with 'int'
     T = Context.IntTy;
+    FoundUsingShadow = nullptr;
   } else if (AllowDeducedTemplate) {
-    if (auto *TD = getAsTypeTemplateDecl(IIDecl))
+    if (auto *TD = getAsTypeTemplateDecl(IIDecl)) {
+      // FIXME: TemplateName should include FoundUsingShadow sugar.
       T = Context.getDeducedTemplateSpecializationType(TemplateName(TD),
                                                        QualType(), false);
+      // Don't wrap in a further UsingType.
+      FoundUsingShadow = nullptr;
+    }
   }
 
   if (T.isNull()) {
@@ -507,6 +517,9 @@ ParsedType Sema::getTypeName(const IdentifierInfo &II, SourceLocation NameLoc,
     return nullptr;
   }
 
+  if (FoundUsingShadow)
+    T = Context.getUsingType(FoundUsingShadow, T);
+
   // NOTE: avoid constructing an ElaboratedType(Loc) if this is a
   // constructor or destructor name (in such a case, the scope specifier
   // will be attached to the enclosing Expr or Decl node).
@@ -843,21 +856,6 @@ static bool isTagTypeWithMissingTag(Sema &SemaRef, LookupResult &Result,
   return false;
 }
 
-/// Build a ParsedType for a simple-type-specifier with a nested-name-specifier.
-static ParsedType buildNestedType(Sema &S, CXXScopeSpec &SS,
-                                  QualType T, SourceLocation NameLoc) {
-  ASTContext &Context = S.Context;
-
-  TypeLocBuilder Builder;
-  Builder.pushTypeSpec(T).setNameLoc(NameLoc);
-
-  T = S.getElaboratedType(ETK_None, SS, T);
-  ElaboratedTypeLoc ElabTL = Builder.push<ElaboratedTypeLoc>(T);
-  ElabTL.setElaboratedKeywordLoc(SourceLocation());
-  ElabTL.setQualifierLoc(SS.getWithLocInContext(Context));
-  return S.CreateParsedType(T, Builder.getTypeSourceInfo(Context, T));
-}
-
 Sema::NameClassification Sema::ClassifyName(Scope *S, CXXScopeSpec &SS,
                                             IdentifierInfo *&Name,
                                             SourceLocation NameLoc,
@@ -1134,14 +1132,28 @@ Corrected:
                          : NameClassification::TypeTemplate(Template);
   }
 
+  auto BuildTypeFor = [&](TypeDecl *Type, NamedDecl *Found) {
+    QualType T = Context.getTypeDeclType(Type);
+    if (const auto *USD = dyn_cast<UsingShadowDecl>(Found))
+      T = Context.getUsingType(USD, T);
+
+    if (SS.isEmpty()) // No elaborated type, trivial location info
+      return ParsedType::make(T);
+
+    TypeLocBuilder Builder;
+    Builder.pushTypeSpec(T).setNameLoc(NameLoc);
+    T = getElaboratedType(ETK_None, SS, T);
+    ElaboratedTypeLoc ElabTL = Builder.push<ElaboratedTypeLoc>(T);
+    ElabTL.setElaboratedKeywordLoc(SourceLocation());
+    ElabTL.setQualifierLoc(SS.getWithLocInContext(Context));
+    return CreateParsedType(T, Builder.getTypeSourceInfo(Context, T));
+  };
+
   NamedDecl *FirstDecl = (*Result.begin())->getUnderlyingDecl();
   if (TypeDecl *Type = dyn_cast<TypeDecl>(FirstDecl)) {
     DiagnoseUseOfDecl(Type, NameLoc);
     MarkAnyDeclReferenced(Type->getLocation(), Type, /*OdrUse=*/false);
-    QualType T = Context.getTypeDeclType(Type);
-    if (SS.isNotEmpty())
-      return buildNestedType(*this, SS, T, NameLoc);
-    return ParsedType::make(T);
+    return BuildTypeFor(Type, *Result.begin());
   }
 
   ObjCInterfaceDecl *Class = dyn_cast<ObjCInterfaceDecl>(FirstDecl);
@@ -1190,10 +1202,7 @@ Corrected:
       isTagTypeWithMissingTag(*this, Result, S, SS, Name, NameLoc)) {
     TypeDecl *Type = Result.getAsSingle<TypeDecl>();
     DiagnoseUseOfDecl(Type, NameLoc);
-    QualType T = Context.getTypeDeclType(Type);
-    if (SS.isNotEmpty())
-      return buildNestedType(*this, SS, T, NameLoc);
-    return ParsedType::make(T);
+    return BuildTypeFor(Type, *Result.begin());
   }
 
   // If we already know which single declaration is referenced, just annotate
@@ -8504,7 +8513,14 @@ static NamedDecl *DiagnoseInvalidRedeclaration(
         << NewFD->getParamDecl(Idx - 1)->getType();
     } else if (FDisConst != NewFDisConst) {
       SemaRef.Diag(FD->getLocation(), diag::note_member_def_close_const_match)
-          << NewFDisConst << FD->getSourceRange().getEnd();
+          << NewFDisConst << FD->getSourceRange().getEnd()
+          << (NewFDisConst
+                  ? FixItHint::CreateRemoval(ExtraArgs.D.getFunctionTypeInfo()
+                                                 .getConstQualifierLoc())
+                  : FixItHint::CreateInsertion(ExtraArgs.D.getFunctionTypeInfo()
+                                                   .getRParenLoc()
+                                                   .getLocWithOffset(1),
+                                               " const"));
     } else
       SemaRef.Diag(FD->getLocation(),
                    IsMember ? diag::note_member_def_close_match
@@ -9195,6 +9211,7 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
             << Name << RemoveRange
             << FixItHint::CreateRemoval(RemoveRange)
             << FixItHint::CreateInsertion(InsertLoc, "<>");
+          Invalid = true;
         }
       }
     } else {
@@ -15359,7 +15376,7 @@ bool Sema::CheckEnumUnderlyingType(TypeSourceInfo *TI) {
     if (BT->isInteger())
       return false;
 
-  if (T->isExtIntType())
+  if (T->isBitIntType())
     return false;
 
   return Diag(UnderlyingLoc, diag::err_enum_invalid_underlying) << T;
@@ -18277,7 +18294,7 @@ static void CheckForDuplicateEnumValues(Sema &S, ArrayRef<Decl *> Elements,
 
     // Emit one note for each of the remaining enum constants with
     // the same value.
-    for (auto *ECD : llvm::make_range(Vec->begin() + 1, Vec->end()))
+    for (auto *ECD : llvm::drop_begin(*Vec))
       S.Diag(ECD->getLocation(), diag::note_duplicate_element)
         << ECD << toString(ECD->getInitVal(), 10)
         << ECD->getSourceRange();
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index 4df8687aff89..b6bd2e69629d 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -1001,6 +1001,84 @@ public:
 };
 }
 
+static void handleDiagnoseAsBuiltinAttr(Sema &S, Decl *D,
+                                        const ParsedAttr &AL) {
+  const auto *DeclFD = cast<FunctionDecl>(D);
+
+  if (const auto *MethodDecl = dyn_cast<CXXMethodDecl>(DeclFD))
+    if (!MethodDecl->isStatic()) {
+      S.Diag(AL.getLoc(), diag::err_attribute_no_member_function) << AL;
+      return;
+    }
+
+  auto DiagnoseType = [&](unsigned Index, AttributeArgumentNType T) {
+    SourceLocation Loc = [&]() {
+      auto Union = AL.getArg(Index - 1);
+      if (Union.is<Expr *>())
+        return Union.get<Expr *>()->getBeginLoc();
+      return Union.get<IdentifierLoc *>()->Loc;
+    }();
+
+    S.Diag(Loc, diag::err_attribute_argument_n_type) << AL << Index << T;
+  };
+
+  FunctionDecl *AttrFD = [&]() -> FunctionDecl * {
+    if (!AL.isArgExpr(0))
+      return nullptr;
+    auto *F = dyn_cast_or_null<DeclRefExpr>(AL.getArgAsExpr(0));
+    if (!F)
+      return nullptr;
+    return dyn_cast_or_null<FunctionDecl>(F->getFoundDecl());
+  }();
+
+  if (!AttrFD || !AttrFD->getBuiltinID(true)) {
+    DiagnoseType(1, AANT_ArgumentBuiltinFunction);
+    return;
+  }
+
+  if (AttrFD->getNumParams() != AL.getNumArgs() - 1) {
+    S.Diag(AL.getLoc(), diag::err_attribute_wrong_number_arguments_for)
+        << AL << AttrFD << AttrFD->getNumParams();
+    return;
+  }
+
+  SmallVector<unsigned, 8> Indices;
+
+  for (unsigned I = 1; I < AL.getNumArgs(); ++I) {
+    if (!AL.isArgExpr(I)) {
+      DiagnoseType(I + 1, AANT_ArgumentIntegerConstant);
+      return;
+    }
+
+    const Expr *IndexExpr = AL.getArgAsExpr(I);
+    uint32_t Index;
+
+    if (!checkUInt32Argument(S, AL, IndexExpr, Index, I + 1, false))
+      return;
+
+    if (Index > DeclFD->getNumParams()) {
+      S.Diag(AL.getLoc(), diag::err_attribute_bounds_for_function)
+          << AL << Index << DeclFD << DeclFD->getNumParams();
+      return;
+    }
+
+    QualType T1 = AttrFD->getParamDecl(I - 1)->getType();
+    QualType T2 = DeclFD->getParamDecl(Index - 1)->getType();
+
+    if (T1.getCanonicalType().getUnqualifiedType() !=
+        T2.getCanonicalType().getUnqualifiedType()) {
+      S.Diag(IndexExpr->getBeginLoc(), diag::err_attribute_parameter_types)
+          << AL << Index << DeclFD << T2 << I << AttrFD << T1;
+      return;
+    }
+
+    Indices.push_back(Index - 1);
+  }
+
+  D->addAttr(::new (S.Context) DiagnoseAsBuiltinAttr(
+      S.Context, AL, AttrFD, Indices.data(), Indices.size()));
+}
+
 static void handleDiagnoseIfAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   S.Diag(AL.getLoc(), diag::ext_clang_diagnose_if);
 
@@ -4502,7 +4580,7 @@ void Sema::AddModeAttr(Decl *D, const AttributeCommonInfo &CI,
     return;
   }
   bool IntegralOrAnyEnumType = (OldElemTy->isIntegralOrEnumerationType() &&
-                                !OldElemTy->isExtIntType()) ||
+                                !OldElemTy->isBitIntType()) ||
                                OldElemTy->getAs<EnumType>();
 
   if (!OldElemTy->getAs<BuiltinType>() && !OldElemTy->isComplexType() &&
@@ -8159,6 +8237,9 @@ static void ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D,
   case ParsedAttr::AT_DiagnoseIf:
     handleDiagnoseIfAttr(S, D, AL);
     break;
+  case ParsedAttr::AT_DiagnoseAsBuiltin:
+    handleDiagnoseAsBuiltinAttr(S, D, AL);
+    break;
   case ParsedAttr::AT_NoBuiltin:
     handleNoBuiltinAttr(S, D, AL);
     break;
diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp
index 2658e9698688..01f0079198c7 100644
--- a/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/clang/lib/Sema/SemaDeclCXX.cpp
@@ -8435,9 +8435,6 @@ bool Sema::CheckExplicitlyDefaultedComparison(Scope *S, FunctionDecl *FD,
                                               DefaultedComparisonKind DCK) {
   assert(DCK != DefaultedComparisonKind::None && "not a defaulted comparison");
 
-  CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(FD->getLexicalDeclContext());
-  assert(RD && "defaulted comparison is not defaulted in a class");
-
   // Perform any unqualified lookups we're going to need to default this
   // function.
   if (S) {
@@ -8455,43 +8452,17 @@ bool Sema::CheckExplicitlyDefaultedComparison(Scope *S, FunctionDecl *FD,
   //       const C&, or
   //    -- a friend of C having two parameters of type const C& or two
   //       parameters of type C.
-  QualType ExpectedParmType1 = Context.getRecordType(RD);
-  QualType ExpectedParmType2 =
-      Context.getLValueReferenceType(ExpectedParmType1.withConst());
-  if (isa<CXXMethodDecl>(FD))
-    ExpectedParmType1 = ExpectedParmType2;
-  for (const ParmVarDecl *Param : FD->parameters()) {
-    if (!Param->getType()->isDependentType() &&
-        !Context.hasSameType(Param->getType(), ExpectedParmType1) &&
-        !Context.hasSameType(Param->getType(), ExpectedParmType2)) {
-      // Don't diagnose an implicit 'operator=='; we will have diagnosed the
-      // corresponding defaulted 'operator<=>' already.
-      if (!FD->isImplicit()) {
-        Diag(FD->getLocation(), diag::err_defaulted_comparison_param)
-            << (int)DCK << Param->getType() << ExpectedParmType1
-            << !isa<CXXMethodDecl>(FD)
-            << ExpectedParmType2 << Param->getSourceRange();
-      }
-      return true;
-    }
-  }
-  if (FD->getNumParams() == 2 &&
-      !Context.hasSameType(FD->getParamDecl(0)->getType(),
-                           FD->getParamDecl(1)->getType())) {
-    if (!FD->isImplicit()) {
-      Diag(FD->getLocation(), diag::err_defaulted_comparison_param_mismatch)
-          << (int)DCK
-          << FD->getParamDecl(0)->getType()
-          << FD->getParamDecl(0)->getSourceRange()
-          << FD->getParamDecl(1)->getType()
-          << FD->getParamDecl(1)->getSourceRange();
-    }
-    return true;
-  }
 
-  // ... non-static const member ...
-  if (auto *MD = dyn_cast<CXXMethodDecl>(FD)) {
+  CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(FD->getLexicalDeclContext());
+  bool IsMethod = isa<CXXMethodDecl>(FD);
+  if (IsMethod) {
+    auto *MD = cast<CXXMethodDecl>(FD);
     assert(!MD->isStatic() && "comparison function cannot be a static member");
+
+    // If we're out-of-class, this is the class we're comparing.
+    if (!RD)
+      RD = MD->getParent();
+
     if (!MD->isConst()) {
       SourceLocation InsertLoc;
       if (FunctionTypeLoc Loc = MD->getFunctionTypeLoc())
@@ -8500,7 +8471,7 @@ bool Sema::CheckExplicitlyDefaultedComparison(Scope *S, FunctionDecl *FD,
       // corresponding defaulted 'operator<=>' already.
       if (!MD->isImplicit()) {
         Diag(MD->getLocation(), diag::err_defaulted_comparison_non_const)
-          << (int)DCK << FixItHint::CreateInsertion(InsertLoc, " const");
+            << (int)DCK << FixItHint::CreateInsertion(InsertLoc, " const");
       }
 
       // Add the 'const' to the type to recover.
@@ -8510,9 +8481,98 @@ bool Sema::CheckExplicitlyDefaultedComparison(Scope *S, FunctionDecl *FD,
       MD->setType(Context.getFunctionType(FPT->getReturnType(),
                                           FPT->getParamTypes(), EPI));
     }
-  } else {
-    // A non-member function declared in a class must be a friend.
+  }
+
+  if (FD->getNumParams() != (IsMethod ? 1 : 2)) {
+    // Let's not worry about using a variadic template pack here -- who would do
+    // such a thing?
+    Diag(FD->getLocation(), diag::err_defaulted_comparison_num_args)
+        << int(IsMethod) << int(DCK);
+    return true;
+  }
+
+  const ParmVarDecl *KnownParm = nullptr;
+  for (const ParmVarDecl *Param : FD->parameters()) {
+    QualType ParmTy = Param->getType();
+    if (ParmTy->isDependentType())
+      continue;
+    if (!KnownParm) {
+      auto CTy = ParmTy;
+      // Is it `T const &`?
+      bool Ok = !IsMethod;
+      QualType ExpectedTy;
+      if (RD)
+        ExpectedTy = Context.getRecordType(RD);
+      if (auto *Ref = CTy->getAs<ReferenceType>()) {
+        CTy = Ref->getPointeeType();
+        if (RD)
+          ExpectedTy.addConst();
+        Ok = true;
+      }
+
+      // Is T a class?
+      if (!Ok) {
+      } else if (RD) {
+        if (!RD->isDependentType() && !Context.hasSameType(CTy, ExpectedTy))
+          Ok = false;
+      } else if (auto *CRD = CTy->getAsRecordDecl()) {
+        RD = cast<CXXRecordDecl>(CRD);
+      } else {
+        Ok = false;
+      }
+
+      if (Ok) {
+        KnownParm = Param;
+      } else {
+        // Don't diagnose an implicit 'operator=='; we will have diagnosed the
+        // corresponding defaulted 'operator<=>' already.
+        if (!FD->isImplicit()) {
+          if (RD) {
+            QualType PlainTy = Context.getRecordType(RD);
+            QualType RefTy =
+                Context.getLValueReferenceType(PlainTy.withConst());
+            Diag(FD->getLocation(), diag::err_defaulted_comparison_param)
+                << int(DCK) << ParmTy << RefTy << int(!IsMethod) << PlainTy
+                << Param->getSourceRange();
+          } else {
+            assert(!IsMethod && "should know expected type for method");
+            Diag(FD->getLocation(),
+                 diag::err_defaulted_comparison_param_unknown)
+                << int(DCK) << ParmTy << Param->getSourceRange();
+          }
+        }
+        return true;
+      }
+    } else if (!Context.hasSameType(KnownParm->getType(), ParmTy)) {
+      Diag(FD->getLocation(), diag::err_defaulted_comparison_param_mismatch)
+          << int(DCK) << KnownParm->getType() << KnownParm->getSourceRange()
+          << ParmTy << Param->getSourceRange();
+      return true;
+    }
+  }
+
+  assert(RD && "must have determined class");
+  if (IsMethod) {
+  } else if (isa<CXXRecordDecl>(FD->getLexicalDeclContext())) {
+    // In-class, must be a friend decl.
     assert(FD->getFriendObjectKind() && "expected a friend declaration");
+  } else {
+    // Out of class, require the defaulted comparison to be a friend (of a
+    // complete type).
+    if (RequireCompleteType(FD->getLocation(), Context.getRecordType(RD),
+                            diag::err_defaulted_comparison_not_friend, int(DCK),
+                            int(1)))
+      return true;
+
+    if (llvm::find_if(RD->friends(), [&](const FriendDecl *F) {
+          return FD->getCanonicalDecl() ==
+                 F->getFriendDecl()->getCanonicalDecl();
+        }) == RD->friends().end()) {
+      Diag(FD->getLocation(), diag::err_defaulted_comparison_not_friend)
+          << int(DCK) << int(0) << RD;
+      Diag(RD->getCanonicalDecl()->getLocation(), diag::note_declared_at);
+      return true;
+    }
   }
 
   // C++2a [class.eq]p1, [class.rel]p1:
@@ -8670,7 +8730,10 @@ void Sema::DefineDefaultedComparison(SourceLocation UseLoc, FunctionDecl *FD,
 
   {
     // Build and set up the function body.
-    CXXRecordDecl *RD = cast<CXXRecordDecl>(FD->getLexicalParent());
+    // The first parameter has type maybe-ref-to maybe-const T, use that to get
+    // the type of the class being compared.
+    auto PT = FD->getParamDecl(0)->getType();
+    CXXRecordDecl *RD = PT.getNonReferenceType()->getAsCXXRecordDecl();
     SourceLocation BodyLoc =
         FD->getEndLoc().isValid() ? FD->getEndLoc() : FD->getLocation();
     StmtResult Body =
@@ -16146,6 +16209,23 @@ Decl *Sema::ActOnStartLinkageSpecification(Scope *S, SourceLocation ExternLoc,
   LinkageSpecDecl *D = LinkageSpecDecl::Create(Context, CurContext, ExternLoc,
                                                LangStr->getExprLoc(), Language,
                                                LBraceLoc.isValid());
+
+  /// C++ [module.unit]p7.2.3
+  /// - Otherwise, if the declaration
+  ///   - ...
+  ///   - ...
+  ///   - appears within a linkage-specification,
+  ///   it is attached to the global module.
+  ///
+  /// If the declaration is already in global module fragment, we don't
+  /// need to attach it again.
+  if (getLangOpts().CPlusPlusModules && isCurrentModulePurview()) {
+    Module *GlobalModule =
+        PushGlobalModuleFragment(ExternLoc, /*IsImplicit=*/true);
+    D->setModuleOwnershipKind(Decl::ModuleOwnershipKind::ModulePrivate);
+    D->setLocalOwningModule(GlobalModule);
+  }
+
   CurContext->addDecl(D);
   PushDeclContext(S, D);
   return D;
@@ -16162,6 +16242,14 @@ Decl *Sema::ActOnFinishLinkageSpecification(Scope *S,
     LinkageSpecDecl* LSDecl = cast<LinkageSpecDecl>(LinkageSpec);
     LSDecl->setRBraceLoc(RBraceLoc);
   }
+
+  // If the current module doesn't has Parent, it implies that the
+  // LinkageSpec isn't in the module created by itself. So we don't
+  // need to pop it.
+  if (getLangOpts().CPlusPlusModules && getCurrentModule() &&
+      getCurrentModule()->isGlobalModule() && getCurrentModule()->Parent)
+    PopGlobalModuleFragment();
+
   PopDeclContext();
   return LinkageSpec;
 }
@@ -17155,13 +17243,6 @@ void Sema::SetDeclDefaulted(Decl *Dcl, SourceLocation DefaultLoc) {
     return;
   }
 
-  if (DefKind.isComparison() &&
-      !isa<CXXRecordDecl>(FD->getLexicalDeclContext())) {
-    Diag(FD->getLocation(), diag::err_defaulted_comparison_out_of_class)
-        << (int)DefKind.asComparison();
-    return;
-  }
-
   // Issue compatibility warning. We already warned if the operator is
   // 'operator<=>' when parsing the '<=>' token.
   if (DefKind.isComparison() &&
@@ -17183,31 +17264,40 @@ void Sema::SetDeclDefaulted(Decl *Dcl, SourceLocation DefaultLoc) {
   // that we've marked it as defaulted.
   FD->setWillHaveBody(false);
 
-  // If this definition appears within the record, do the checking when
-  // the record is complete. This is always the case for a defaulted
-  // comparison.
-  if (DefKind.isComparison())
-    return;
-  auto *MD = cast<CXXMethodDecl>(FD);
-
-  const FunctionDecl *Primary = FD;
-  if (const FunctionDecl *Pattern = FD->getTemplateInstantiationPattern())
-    // Ask the template instantiation pattern that actually had the
-    // '= default' on it.
-    Primary = Pattern;
+  if (DefKind.isComparison()) {
+    // If this comparison's defaulting occurs within the definition of its
+    // lexical class context, we have to do the checking when complete.
+    if (auto const *RD = dyn_cast<CXXRecordDecl>(FD->getLexicalDeclContext()))
+      if (!RD->isCompleteDefinition())
+        return;
+  }
 
-  // If the method was defaulted on its first declaration, we will have
+  // If this member fn was defaulted on its first declaration, we will have
   // already performed the checking in CheckCompletedCXXClass. Such a
   // declaration doesn't trigger an implicit definition.
-  if (Primary->getCanonicalDecl()->isDefaulted())
-    return;
+  if (isa<CXXMethodDecl>(FD)) {
+    const FunctionDecl *Primary = FD;
+    if (const FunctionDecl *Pattern = FD->getTemplateInstantiationPattern())
+      // Ask the template instantiation pattern that actually had the
+      // '= default' on it.
+      Primary = Pattern;
+    if (Primary->getCanonicalDecl()->isDefaulted())
+      return;
+  }
 
-  // FIXME: Once we support defining comparisons out of class, check for a
-  // defaulted comparison here.
-  if (CheckExplicitlyDefaultedSpecialMember(MD, DefKind.asSpecialMember()))
-    MD->setInvalidDecl();
-  else
-    DefineDefaultedFunction(*this, MD, DefaultLoc);
+  if (DefKind.isComparison()) {
+    if (CheckExplicitlyDefaultedComparison(nullptr, FD, DefKind.asComparison()))
+      FD->setInvalidDecl();
+    else
+      DefineDefaultedComparison(DefaultLoc, FD, DefKind.asComparison());
+  } else {
+    auto *MD = cast<CXXMethodDecl>(FD);
+
+    if (CheckExplicitlyDefaultedSpecialMember(MD, DefKind.asSpecialMember()))
+      MD->setInvalidDecl();
+    else
+      DefineDefaultedFunction(*this, MD, DefaultLoc);
+  }
 }
 
 static void SearchForReturnInStmt(Sema &Self, Stmt *S) {
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index b305d4e5b92f..d32b3f217aa0 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -4384,7 +4384,7 @@ static void captureVariablyModifiedType(ASTContext &Context, QualType T,
     case Type::ObjCObjectPointer:
     case Type::ObjCTypeParam:
     case Type::Pipe:
-    case Type::ExtInt:
+    case Type::BitInt:
       llvm_unreachable("type class is never variably-modified!");
     case Type::Adjusted:
       T = cast<AdjustedType>(Ty)->getOriginalType();
@@ -4443,6 +4443,9 @@ static void captureVariablyModifiedType(ASTContext &Context, QualType T,
     case Type::Decltype:
       T = cast<DecltypeType>(Ty)->desugar();
       break;
+    case Type::Using:
+      T = cast<UsingType>(Ty)->desugar();
+      break;
     case Type::Auto:
     case Type::DeducedTemplateSpecialization:
       T = cast<DeducedType>(Ty)->getDeducedType();
@@ -8388,9 +8391,10 @@ QualType Sema::CheckConditionalOperands(ExprResult &Cond, ExprResult &LHS,
   // If both operands have arithmetic type, do the usual arithmetic conversions
   // to find a common type: C99 6.5.15p3,5.
   if (LHSTy->isArithmeticType() && RHSTy->isArithmeticType()) {
-    // Disallow invalid arithmetic conversions, such as those between ExtInts of
-    // different sizes, or between ExtInts and other types.
-    if (ResTy.isNull() && (LHSTy->isExtIntType() || RHSTy->isExtIntType())) {
+    // Disallow invalid arithmetic conversions, such as those between bit-
+    // precise integers types of different sizes, or between a bit-precise
+    // integer and another type.
+    if (ResTy.isNull() && (LHSTy->isBitIntType() || RHSTy->isBitIntType())) {
       Diag(QuestionLoc, diag::err_typecheck_cond_incompatible_operands)
           << LHSTy << RHSTy << LHS.get()->getSourceRange()
           << RHS.get()->getSourceRange();
@@ -10974,7 +10978,7 @@ static void DiagnoseBadShiftValues(Sema& S, ExprResult &LHS, ExprResult &RHS,
 
   QualType LHSExprType = LHS.get()->getType();
   uint64_t LeftSize = S.Context.getTypeSize(LHSExprType);
-  if (LHSExprType->isExtIntType())
+  if (LHSExprType->isBitIntType())
     LeftSize = S.Context.getIntWidth(LHSExprType);
   else if (LHSExprType->isFixedPointType()) {
     auto FXSema = S.Context.getFixedPointSemantics(LHSExprType);
@@ -12260,27 +12264,32 @@ QualType Sema::GetSignedVectorType(QualType V) {
   if (isa<ExtVectorType>(VTy)) {
     if (TypeSize == Context.getTypeSize(Context.CharTy))
       return Context.getExtVectorType(Context.CharTy, VTy->getNumElements());
-    else if (TypeSize == Context.getTypeSize(Context.ShortTy))
+    if (TypeSize == Context.getTypeSize(Context.ShortTy))
       return Context.getExtVectorType(Context.ShortTy, VTy->getNumElements());
-    else if (TypeSize == Context.getTypeSize(Context.IntTy))
+    if (TypeSize == Context.getTypeSize(Context.IntTy))
       return Context.getExtVectorType(Context.IntTy, VTy->getNumElements());
-    else if (TypeSize == Context.getTypeSize(Context.LongTy))
+    if (TypeSize == Context.getTypeSize(Context.Int128Ty))
+      return Context.getExtVectorType(Context.Int128Ty, VTy->getNumElements());
+    if (TypeSize == Context.getTypeSize(Context.LongTy))
       return Context.getExtVectorType(Context.LongTy, VTy->getNumElements());
     assert(TypeSize == Context.getTypeSize(Context.LongLongTy) &&
            "Unhandled vector element size in vector compare");
     return Context.getExtVectorType(Context.LongLongTy, VTy->getNumElements());
   }
 
+  if (TypeSize == Context.getTypeSize(Context.Int128Ty))
+    return Context.getVectorType(Context.Int128Ty, VTy->getNumElements(),
+                                 VectorType::GenericVector);
   if (TypeSize == Context.getTypeSize(Context.LongLongTy))
     return Context.getVectorType(Context.LongLongTy, VTy->getNumElements(),
                                  VectorType::GenericVector);
-  else if (TypeSize == Context.getTypeSize(Context.LongTy))
+  if (TypeSize == Context.getTypeSize(Context.LongTy))
     return Context.getVectorType(Context.LongTy, VTy->getNumElements(),
                                  VectorType::GenericVector);
-  else if (TypeSize == Context.getTypeSize(Context.IntTy))
+  if (TypeSize == Context.getTypeSize(Context.IntTy))
     return Context.getVectorType(Context.IntTy, VTy->getNumElements(),
                                  VectorType::GenericVector);
-  else if (TypeSize == Context.getTypeSize(Context.ShortTy))
+  if (TypeSize == Context.getTypeSize(Context.ShortTy))
     return Context.getVectorType(Context.ShortTy, VTy->getNumElements(),
                                  VectorType::GenericVector);
   assert(TypeSize == Context.getTypeSize(Context.CharTy) &&
diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index d25f329f85e4..54f0242d2ca1 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -1346,7 +1346,7 @@ bool Sema::CheckCXXThisCapture(SourceLocation Loc, const bool Explicit,
   // implicitly capturing the *enclosing  object* by reference (see loop
   // above)).
   assert((!ByCopy ||
-          dyn_cast<LambdaScopeInfo>(FunctionScopes[MaxFunctionScopesIndex])) &&
+          isa<LambdaScopeInfo>(FunctionScopes[MaxFunctionScopesIndex])) &&
          "Only a lambda can capture the enclosing object (referred to by "
          "*this) by copy");
   QualType ThisTy = getCurrentThisType();
diff --git a/clang/lib/Sema/SemaLookup.cpp b/clang/lib/Sema/SemaLookup.cpp
index 0711e6d89383..635e93ba8460 100644
--- a/clang/lib/Sema/SemaLookup.cpp
+++ b/clang/lib/Sema/SemaLookup.cpp
@@ -2935,7 +2935,7 @@ addAssociatedClassesAndNamespaces(AssociatedLookup &Result, QualType Ty) {
     case Type::ExtVector:
     case Type::ConstantMatrix:
     case Type::Complex:
-    case Type::ExtInt:
+    case Type::BitInt:
       break;
 
     // Non-deduced auto types only get here for error cases.
diff --git a/clang/lib/Sema/SemaModule.cpp b/clang/lib/Sema/SemaModule.cpp
index af95b1a93cc4..a4b9f3c242c1 100644
--- a/clang/lib/Sema/SemaModule.cpp
+++ b/clang/lib/Sema/SemaModule.cpp
@@ -68,15 +68,8 @@ Sema::ActOnGlobalModuleFragmentDecl(SourceLocation ModuleLoc) {
 
   // We start in the global module; all those declarations are implicitly
   // module-private (though they do not have module linkage).
-  auto &Map = PP.getHeaderSearchInfo().getModuleMap();
-  auto *GlobalModule = Map.createGlobalModuleFragmentForModuleUnit(ModuleLoc);
-  assert(GlobalModule && "module creation should not fail");
-
-  // Enter the scope of the global module.
-  ModuleScopes.push_back({});
-  ModuleScopes.back().BeginLoc = ModuleLoc;
-  ModuleScopes.back().Module = GlobalModule;
-  VisibleModules.setVisible(GlobalModule, ModuleLoc);
+  Module *GlobalModule =
+      PushGlobalModuleFragment(ModuleLoc, /*IsImplicit=*/false);
 
   // All declarations created from now on are owned by the global module.
   auto *TU = Context.getTranslationUnitDecl();
@@ -390,11 +383,18 @@ DeclResult Sema::ActOnModuleImport(SourceLocation StartLoc,
   if (!ModuleScopes.empty())
     Context.addModuleInitializer(ModuleScopes.back().Module, Import);
 
-  // Re-export the module if needed.
   if (!ModuleScopes.empty() && ModuleScopes.back().ModuleInterface) {
+    // Re-export the module if the imported module is exported.
+    // Note that we don't need to add re-exported module to Imports field
+    // since `Exports` implies the module is imported already.
     if (ExportLoc.isValid() || getEnclosingExportDecl(Import))
       getCurrentModule()->Exports.emplace_back(Mod, false);
+    else
+      getCurrentModule()->Imports.insert(Mod);
   } else if (ExportLoc.isValid()) {
+    // [module.interface]p1:
+    // An export-declaration shall inhabit a namespace scope and appear in the
+    // purview of a module interface unit.
     Diag(ExportLoc, diag::err_export_not_in_module_interface);
   }
 
@@ -708,3 +708,26 @@ Decl *Sema::ActOnFinishExportDecl(Scope *S, Decl *D, SourceLocation RBraceLoc) {
 
   return D;
 }
+
+Module *Sema::PushGlobalModuleFragment(SourceLocation BeginLoc,
+                                       bool IsImplicit) {
+  ModuleMap &Map = PP.getHeaderSearchInfo().getModuleMap();
+  Module *GlobalModule =
+      Map.createGlobalModuleFragmentForModuleUnit(BeginLoc, getCurrentModule());
+  assert(GlobalModule && "module creation should not fail");
+
+  // Enter the scope of the global module.
+  ModuleScopes.push_back({BeginLoc, GlobalModule,
+                          /*ModuleInterface=*/false,
+                          /*ImplicitGlobalModuleFragment=*/IsImplicit,
+                          /*VisibleModuleSet*/{}});
+  VisibleModules.setVisible(GlobalModule, BeginLoc);
+
+  return GlobalModule;
+}
+
+void Sema::PopGlobalModuleFragment() {
+  assert(!ModuleScopes.empty() && getCurrentModule()->isGlobalModule() &&
+         "left the wrong module scope, which is not global module fragment");
+  ModuleScopes.pop_back();
+}
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 22ae5f59d41b..ba0481874577 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -255,14 +255,14 @@ private:
     return &Stack.back().first[Size - 1];
   }
   const SharingMapTy *getTopOfStackOrNull() const {
-    return const_cast<DSAStackTy&>(*this).getTopOfStackOrNull();
+    return const_cast<DSAStackTy &>(*this).getTopOfStackOrNull();
   }
   SharingMapTy &getTopOfStack() {
     assert(!isStackEmpty() && "no current directive");
     return *getTopOfStackOrNull();
   }
   const SharingMapTy &getTopOfStack() const {
-    return const_cast<DSAStackTy&>(*this).getTopOfStack();
+    return const_cast<DSAStackTy &>(*this).getTopOfStack();
   }
 
   SharingMapTy *getSecondOnStackOrNull() {
@@ -272,7 +272,7 @@ private:
     return &Stack.back().first[Size - 2];
   }
   const SharingMapTy *getSecondOnStackOrNull() const {
-    return const_cast<DSAStackTy&>(*this).getSecondOnStackOrNull();
+    return const_cast<DSAStackTy &>(*this).getSecondOnStackOrNull();
   }
 
   /// Get the stack element at a certain level (previously returned by
@@ -286,7 +286,7 @@ private:
     return Stack.back().first[Level];
   }
   const SharingMapTy &getStackElemAtLevel(unsigned Level) const {
-    return const_cast<DSAStackTy&>(*this).getStackElemAtLevel(Level);
+    return const_cast<DSAStackTy &>(*this).getStackElemAtLevel(Level);
   }
 
   DSAVarData getDSA(const_iterator &Iter, ValueDecl *D) const;
@@ -354,9 +354,7 @@ public:
     const SharingMapTy *Top = getTopOfStackOrNull();
     return Top && Top->BodyComplete;
   }
-  void setBodyComplete() {
-    getTopOfStack().BodyComplete = true;
-  }
+  void setBodyComplete() { getTopOfStack().BodyComplete = true; }
 
   bool isForceVarCapturing() const { return ForceCapturing; }
   void setForceVarCapturing(bool V) { ForceCapturing = V; }
@@ -392,6 +390,7 @@ public:
   class ParentDirectiveScope {
     DSAStackTy &Self;
     bool Active;
+
   public:
     ParentDirectiveScope(DSAStackTy &Self, bool Activate)
         : Self(Self), Active(false) {
@@ -433,8 +432,7 @@ public:
   }
   /// Marks (or clears) declaration as possibly loop counter.
   void resetPossibleLoopCounter(const Decl *D = nullptr) {
-    getTopOfStack().PossiblyLoopCounter =
-        D ? D->getCanonicalDecl() : D;
+    getTopOfStack().PossiblyLoopCounter = D ? D->getCanonicalDecl() : D;
   }
   /// Gets the possible loop counter decl.
   const Decl *getPossiblyLoopCunter() const {
@@ -631,13 +629,10 @@ public:
   }
 
   /// Add requires decl to internal vector
-  void addRequiresDecl(OMPRequiresDecl *RD) {
-    RequiresDecls.push_back(RD);
-  }
+  void addRequiresDecl(OMPRequiresDecl *RD) { RequiresDecls.push_back(RD); }
 
   /// Checks if the defined 'requires' directive has specified type of clause.
-  template <typename ClauseType>
-  bool hasRequiresDeclWithClause() const {
+  template <typename ClauseType> bool hasRequiresDeclWithClause() const {
     return llvm::any_of(RequiresDecls, [](const OMPRequiresDecl *D) {
       return llvm::any_of(D->clauselists(), [](const OMPClause *C) {
         return isa<ClauseType>(C);
@@ -680,9 +675,7 @@ public:
 
   /// Returns the location of the first encountered atomic directive in the
   /// module.
-  SourceLocation getAtomicDirectiveLoc() const {
-    return AtomicLocation;
-  }
+  SourceLocation getAtomicDirectiveLoc() const { return AtomicLocation; }
 
   // Return previously encountered target region locations.
   ArrayRef<SourceLocation> getEncounteredTargetLocs() const {
@@ -706,8 +699,7 @@ public:
   }
   /// Set default data mapping attribute to Modifier:Kind
   void setDefaultDMAAttr(OpenMPDefaultmapClauseModifier M,
-                         OpenMPDefaultmapClauseKind Kind,
-                         SourceLocation Loc) {
+                         OpenMPDefaultmapClauseKind Kind, SourceLocation Loc) {
     DefaultmapInfo &DMI = getTopOfStack().DefaultmapMap[Kind];
     DMI.ImplicitBehavior = M;
     DMI.SLoc = Loc;
@@ -749,12 +741,10 @@ public:
                                    : getStackElemAtLevel(Level).DefaultAttr;
   }
   DefaultDataSharingAttributes getDefaultDSA() const {
-    return isStackEmpty() ? DSA_unspecified
-                          : getTopOfStack().DefaultAttr;
+    return isStackEmpty() ? DSA_unspecified : getTopOfStack().DefaultAttr;
   }
   SourceLocation getDefaultDSALocation() const {
-    return isStackEmpty() ? SourceLocation()
-                          : getTopOfStack().DefaultAttrLoc;
+    return isStackEmpty() ? SourceLocation() : getTopOfStack().DefaultAttrLoc;
   }
   OpenMPDefaultmapClauseModifier
   getDefaultmapModifier(OpenMPDefaultmapClauseKind Kind) const {
@@ -1457,8 +1447,7 @@ void DSAStackTy::addTaskgroupReductionData(const ValueDecl *D, SourceRange SR,
          "Additional reduction info may be specified only once for reduction "
          "items.");
   ReductionData.set(BOK, SR);
-  Expr *&TaskgroupReductionRef =
-      getTopOfStack().TaskgroupReductionRef;
+  Expr *&TaskgroupReductionRef = getTopOfStack().TaskgroupReductionRef;
   if (!TaskgroupReductionRef) {
     VarDecl *VD = buildVarDecl(SemaRef, SR.getBegin(),
                                SemaRef.Context.VoidPtrTy, ".task_red.");
@@ -1483,8 +1472,7 @@ void DSAStackTy::addTaskgroupReductionData(const ValueDecl *D, SourceRange SR,
          "Additional reduction info may be specified only once for reduction "
          "items.");
   ReductionData.set(ReductionRef, SR);
-  Expr *&TaskgroupReductionRef =
-      getTopOfStack().TaskgroupReductionRef;
+  Expr *&TaskgroupReductionRef = getTopOfStack().TaskgroupReductionRef;
   if (!TaskgroupReductionRef) {
     VarDecl *VD = buildVarDecl(SemaRef, SR.getBegin(),
                                SemaRef.Context.VoidPtrTy, ".task_red.");
@@ -1595,10 +1583,9 @@ static bool rejectConstNotMutableType(Sema &SemaRef, const ValueDecl *D,
   ASTContext &Context = SemaRef.getASTContext();
   bool IsClassType;
   if (isConstNotMutableType(SemaRef, Type, AcceptIfMutable, &IsClassType)) {
-    unsigned Diag = ListItemNotVar
-                        ? diag::err_omp_const_list_item
-                        : IsClassType ? diag::err_omp_const_not_mutable_variable
-                                      : diag::err_omp_const_variable;
+    unsigned Diag = ListItemNotVar ? diag::err_omp_const_list_item
+                    : IsClassType  ? diag::err_omp_const_not_mutable_variable
+                                   : diag::err_omp_const_variable;
     SemaRef.Diag(ELoc, Diag) << getOpenMPClauseName(CKind);
     if (!ListItemNotVar && D) {
       const VarDecl *VD = dyn_cast<VarDecl>(D);
@@ -1658,8 +1645,7 @@ const DSAStackTy::DSAVarData DSAStackTy::getTopDSA(ValueDecl *D,
         });
     if (IterTarget != end()) {
       const_iterator ParentIterTarget = IterTarget + 1;
-      for (const_iterator Iter = begin();
-           Iter != ParentIterTarget; ++Iter) {
+      for (const_iterator Iter = begin(); Iter != ParentIterTarget; ++Iter) {
         if (isOpenMPLocal(VD, Iter)) {
           DVar.RefExpr =
               buildDeclRefExpr(SemaRef, VD, D->getType().getNonReferenceType(),
@@ -1677,9 +1663,9 @@ const DSAStackTy::DSAVarData DSAStackTy::getTopDSA(ValueDecl *D,
           return DVar;
         }
         const_iterator End = end();
-        if (!SemaRef.isOpenMPCapturedByRef(
-                D, std::distance(ParentIterTarget, End),
-                /*OpenMPCaptureLevel=*/0)) {
+        if (!SemaRef.isOpenMPCapturedByRef(D,
+                                           std::distance(ParentIterTarget, End),
+                                           /*OpenMPCaptureLevel=*/0)) {
           DVar.RefExpr =
               buildDeclRefExpr(SemaRef, VD, D->getType().getNonReferenceType(),
                                IterTarget->ConstructLoc);
@@ -1891,9 +1877,7 @@ void Sema::InitDataSharingAttributesStack() {
 
 #define DSAStack static_cast<DSAStackTy *>(VarDataSharingAttributesStack)
 
-void Sema::pushOpenMPFunctionRegion() {
-  DSAStack->pushFunction();
-}
+void Sema::pushOpenMPFunctionRegion() { DSAStack->pushFunction(); }
 
 void Sema::popOpenMPFunctionRegion(const FunctionScopeInfo *OldFSI) {
   DSAStack->popFunction(OldFSI);
@@ -2070,8 +2054,8 @@ bool Sema::isOpenMPCapturedByRef(const ValueDecl *D, unsigned Level,
 
     DSAStack->checkMappableExprComponentListsForDeclAtLevel(
         D, Level,
-        [&IsVariableUsedInMapClause, &IsVariableAssociatedWithSection, D](
-            OMPClauseMappableExprCommon::MappableExprComponentListRef
+        [&IsVariableUsedInMapClause, &IsVariableAssociatedWithSection,
+         D](OMPClauseMappableExprCommon::MappableExprComponentListRef
                 MapExprComponents,
             OpenMPClauseKind WhereFoundClauseKind) {
           // Only the map clause information influences how a variable is
@@ -2248,7 +2232,7 @@ VarDecl *Sema::isOpenMPCapturedDecl(ValueDecl *D, bool CheckScopeInfo,
     bool OpenMPFound = false;
     for (unsigned I = StopAt + 1; I > 0; --I) {
       FunctionScopeInfo *FSI = FunctionScopes[I - 1];
-      if(!isa<CapturingScopeInfo>(FSI))
+      if (!isa<CapturingScopeInfo>(FSI))
         return nullptr;
       if (auto *RSI = dyn_cast<CapturedRegionScopeInfo>(FSI))
         if (RSI->CapRegionKind == CR_OpenMP) {
@@ -2322,9 +2306,7 @@ void Sema::startOpenMPCXXRangeFor() {
 OpenMPClauseKind Sema::isOpenMPPrivateDecl(ValueDecl *D, unsigned Level,
                                            unsigned CapLevel) const {
   assert(LangOpts.OpenMP && "OpenMP is not allowed");
-  if (DSAStack->hasExplicitDirective(
-          [](OpenMPDirectiveKind K) { return isOpenMPTaskingDirective(K); },
-          Level)) {
+  if (DSAStack->hasExplicitDirective(isOpenMPTaskingDirective, Level)) {
     bool IsTriviallyCopyable =
         D->getType().getNonReferenceType().isTriviallyCopyableType(Context) &&
         !D->getType()
@@ -2351,8 +2333,7 @@ OpenMPClauseKind Sema::isOpenMPPrivateDecl(ValueDecl *D, unsigned Level,
     }
   }
   if (isOpenMPLoopDirective(DSAStack->getCurrentDirective())) {
-    if (DSAStack->getAssociatedLoops() > 0 &&
-        !DSAStack->isLoopStarted()) {
+    if (DSAStack->getAssociatedLoops() > 0 && !DSAStack->isLoopStarted()) {
       DSAStack->resetPossibleLoopCounter(D);
       DSAStack->loopStart();
       return OMPC_private;
@@ -2519,16 +2500,16 @@ void Sema::finalizeOpenMPDelayedAnalysis(const FunctionDecl *Caller,
         << HostDevTy;
     return;
   }
-      if (!LangOpts.OpenMPIsDevice && DevTy &&
-          *DevTy == OMPDeclareTargetDeclAttr::DT_NoHost) {
-        // Diagnose nohost function called during host codegen.
-        StringRef NoHostDevTy = getOpenMPSimpleClauseTypeName(
-            OMPC_device_type, OMPC_DEVICE_TYPE_nohost);
-        Diag(Loc, diag::err_omp_wrong_device_function_call) << NoHostDevTy << 1;
-        Diag(*OMPDeclareTargetDeclAttr::getLocation(FD),
-             diag::note_omp_marked_device_type_here)
-            << NoHostDevTy;
-      }
+  if (!LangOpts.OpenMPIsDevice && DevTy &&
+      *DevTy == OMPDeclareTargetDeclAttr::DT_NoHost) {
+    // Diagnose nohost function called during host codegen.
+    StringRef NoHostDevTy = getOpenMPSimpleClauseTypeName(
+        OMPC_device_type, OMPC_DEVICE_TYPE_nohost);
+    Diag(Loc, diag::err_omp_wrong_device_function_call) << NoHostDevTy << 1;
+    Diag(*OMPDeclareTargetDeclAttr::getLocation(FD),
+         diag::note_omp_marked_device_type_here)
+        << NoHostDevTy;
+  }
 }
 
 void Sema::StartOpenMPDSABlock(OpenMPDirectiveKind DKind,
@@ -2779,7 +2760,6 @@ public:
   std::unique_ptr<CorrectionCandidateCallback> clone() override {
     return std::make_unique<VarDeclFilterCCC>(*this);
   }
-
 };
 
 class VarOrFuncDeclFilterCCC final : public CorrectionCandidateCallback {
@@ -3151,9 +3131,10 @@ applyOMPAllocateAttribute(Sema &S, VarDecl *VD,
     ML->DeclarationMarkedOpenMPAllocate(VD, A);
 }
 
-Sema::DeclGroupPtrTy Sema::ActOnOpenMPAllocateDirective(
-    SourceLocation Loc, ArrayRef<Expr *> VarList,
-    ArrayRef<OMPClause *> Clauses, DeclContext *Owner) {
+Sema::DeclGroupPtrTy
+Sema::ActOnOpenMPAllocateDirective(SourceLocation Loc, ArrayRef<Expr *> VarList,
+                                   ArrayRef<OMPClause *> Clauses,
+                                   DeclContext *Owner) {
   assert(Clauses.size() <= 2 && "Expected at most two clauses.");
   Expr *Alignment = nullptr;
   Expr *Allocator = nullptr;
@@ -3500,7 +3481,8 @@ public:
       return;
     if (auto *VD = dyn_cast<VarDecl>(E->getDecl())) {
       // Check the datasharing rules for the expressions in the clauses.
-      if (!CS) {
+      if (!CS || (isa<OMPCapturedExprDecl>(VD) && !CS->capturesVariable(VD) &&
+                  !Stack->getTopDSA(VD, /*FromParent=*/false).RefExpr)) {
         if (auto *CED = dyn_cast<OMPCapturedExprDecl>(VD))
           if (!CED->hasAttr<OMPCaptureNoInitAttr>()) {
             Visit(CED->getInit());
@@ -3819,6 +3801,10 @@ public:
   }
   void VisitOMPExecutableDirective(OMPExecutableDirective *S) {
     for (OMPClause *C : S->clauses()) {
+      // Skip analysis of arguments of private clauses for task|target
+      // directives.
+      if (isa_and_nonnull<OMPPrivateClause>(C))
+        continue;
       // Skip analysis of arguments of implicitly defined firstprivate clause
       // for task|target directives.
       // Skip analysis of arguments of implicitly defined map clause for target
@@ -3841,6 +3827,18 @@ public:
     VisitStmt(S);
   }
 
+  void VisitCallExpr(CallExpr *S) {
+    for (Stmt *C : S->arguments()) {
+      if (C) {
+        // Check implicitly captured variables in the task-based directives to
+        // check if they must be firstprivatized.
+        Visit(C);
+      }
+    }
+    if (Expr *Callee = S->getCallee())
+      if (auto *CE = dyn_cast<MemberExpr>(Callee->IgnoreParenImpCasts()))
+        Visit(CE->getBase());
+  }
   void VisitStmt(Stmt *S) {
     for (Stmt *C : S->children()) {
       if (C) {
@@ -5089,8 +5087,8 @@ static std::pair<ValueDecl *, bool> getPrivateItem(Sema &S, Expr *&RefExpr,
        !isa<CXXThisExpr>(ME->getBase()->IgnoreParenImpCasts()) ||
        !isa<FieldDecl>(ME->getMemberDecl()))) {
     if (IsArrayExpr != NoArrayExpr) {
-      S.Diag(ELoc, diag::err_omp_expected_base_var_name) << IsArrayExpr
-                                                         << ERange;
+      S.Diag(ELoc, diag::err_omp_expected_base_var_name)
+          << IsArrayExpr << ERange;
     } else {
       S.Diag(ELoc,
              AllowArraySection
@@ -5134,8 +5132,7 @@ static void checkAllocateClauses(Sema &S, DSAStackTy *Stack,
          "Expected non-dependent context.");
   auto AllocateRange =
       llvm::make_filter_range(Clauses, OMPAllocateClause::classof);
-  llvm::DenseMap<CanonicalDeclPtr<Decl>, CanonicalDeclPtr<VarDecl>>
-      DeclToCopy;
+  llvm::DenseMap<CanonicalDeclPtr<Decl>, CanonicalDeclPtr<VarDecl>> DeclToCopy;
   auto PrivateRange = llvm::make_filter_range(Clauses, [](const OMPClause *C) {
     return isOpenMPPrivate(C->getClauseKind());
   });
@@ -6018,7 +6015,7 @@ StmtResult Sema::ActOnOpenMPExecutableDirective(
     break;
   case OMPD_parallel_master:
     Res = ActOnOpenMPParallelMasterDirective(ClausesWithImplicit, AStmt,
-                                               StartLoc, EndLoc);
+                                             StartLoc, EndLoc);
     AllowedNameModifiers.push_back(OMPD_parallel);
     break;
   case OMPD_parallel_sections:
@@ -6357,6 +6354,7 @@ StmtResult Sema::ActOnOpenMPExecutableDirective(
       case OMPC_write:
       case OMPC_update:
       case OMPC_capture:
+      case OMPC_compare:
       case OMPC_seq_cst:
       case OMPC_acq_rel:
       case OMPC_acquire:
@@ -7626,10 +7624,10 @@ bool OpenMPIterationSpaceChecker::setStep(Expr *NewStep, bool Subtract) {
     // != with increment is treated as <; != with decrement is treated as >
     if (!TestIsLessOp.hasValue())
       TestIsLessOp = IsConstPos || (IsUnsigned && !Subtract);
-    if (UB && (IsConstZero ||
-               (TestIsLessOp.getValue() ?
-                  (IsConstNeg || (IsUnsigned && Subtract)) :
-                  (IsConstPos || (IsUnsigned && !Subtract))))) {
+    if (UB &&
+        (IsConstZero || (TestIsLessOp.getValue()
+                             ? (IsConstNeg || (IsUnsigned && Subtract))
+                             : (IsConstPos || (IsUnsigned && !Subtract))))) {
       SemaRef.Diag(NewStep->getExprLoc(),
                    diag::err_omp_loop_incr_not_compatible)
           << LCDecl << TestIsLessOp.getValue() << NewStep->getSourceRange();
@@ -8068,11 +8066,13 @@ calculateNumIters(Sema &SemaRef, Scope *S, SourceLocation DefaultLoc,
     return nullptr;
   llvm::APSInt LRes, SRes;
   bool IsLowerConst = false, IsStepConst = false;
-  if (Optional<llvm::APSInt> Res = Lower->getIntegerConstantExpr(SemaRef.Context)) {
+  if (Optional<llvm::APSInt> Res =
+          Lower->getIntegerConstantExpr(SemaRef.Context)) {
     LRes = *Res;
     IsLowerConst = true;
   }
-  if (Optional<llvm::APSInt> Res = Step->getIntegerConstantExpr(SemaRef.Context)) {
+  if (Optional<llvm::APSInt> Res =
+          Step->getIntegerConstantExpr(SemaRef.Context)) {
     SRes = *Res;
     IsStepConst = true;
   }
@@ -8110,7 +8110,8 @@ calculateNumIters(Sema &SemaRef, Scope *S, SourceLocation DefaultLoc,
   }
   llvm::APSInt URes;
   bool IsUpperConst = false;
-  if (Optional<llvm::APSInt> Res = Upper->getIntegerConstantExpr(SemaRef.Context)) {
+  if (Optional<llvm::APSInt> Res =
+          Upper->getIntegerConstantExpr(SemaRef.Context)) {
     URes = *Res;
     IsUpperConst = true;
   }
@@ -8566,10 +8567,12 @@ Expr *OpenMPIterationSpaceChecker::buildPreCond(
   // TODO: this can be improved by calculating min/max values but not sure that
   // it will be very effective.
   if (CondDependOnLC || InitDependOnLC)
-    return SemaRef.PerformImplicitConversion(
-        SemaRef.ActOnIntegerConstant(SourceLocation(), 1).get(),
-        SemaRef.Context.BoolTy, /*Action=*/Sema::AA_Casting,
-        /*AllowExplicit=*/true).get();
+    return SemaRef
+        .PerformImplicitConversion(
+            SemaRef.ActOnIntegerConstant(SourceLocation(), 1).get(),
+            SemaRef.Context.BoolTy, /*Action=*/Sema::AA_Casting,
+            /*AllowExplicit=*/true)
+        .get();
 
   // Try to build LB <op> UB, where <op> is <, >, <=, or >=.
   Sema::TentativeAnalysisScope Trap(SemaRef);
@@ -8579,12 +8582,11 @@ Expr *OpenMPIterationSpaceChecker::buildPreCond(
   if (!NewLB.isUsable() || !NewUB.isUsable())
     return nullptr;
 
-  ExprResult CondExpr =
-      SemaRef.BuildBinOp(S, DefaultLoc,
-                         TestIsLessOp.getValue() ?
-                           (TestIsStrictOp ? BO_LT : BO_LE) :
-                           (TestIsStrictOp ? BO_GT : BO_GE),
-                         NewLB.get(), NewUB.get());
+  ExprResult CondExpr = SemaRef.BuildBinOp(
+      S, DefaultLoc,
+      TestIsLessOp.getValue() ? (TestIsStrictOp ? BO_LT : BO_LE)
+                              : (TestIsStrictOp ? BO_GT : BO_GE),
+      NewLB.get(), NewUB.get());
   if (CondExpr.isUsable()) {
     if (!SemaRef.Context.hasSameUnqualifiedType(CondExpr.get()->getType(),
                                                 SemaRef.Context.BoolTy))
@@ -8804,6 +8806,9 @@ static bool checkOpenMPIterationSpace(
   }
   assert(((For && For->getBody()) || (CXXFor && CXXFor->getBody())) &&
          "No loop body.");
+  // Postpone analysis in dependent contexts for ranged for loops.
+  if (CXXFor && SemaRef.CurContext->isDependentContext())
+    return false;
 
   OpenMPIterationSpaceChecker ISC(SemaRef, SupportsNonRectangular, DSA,
                                   For ? For->getForLoc() : CXXFor->getForLoc());
@@ -9684,9 +9689,8 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
       ExprResult Iter;
 
       // Compute prod
-      ExprResult Prod =
-          SemaRef.ActOnIntegerConstant(SourceLocation(), 1).get();
-      for (unsigned int K = Cnt+1; K < NestedLoopCount; ++K)
+      ExprResult Prod = SemaRef.ActOnIntegerConstant(SourceLocation(), 1).get();
+      for (unsigned int K = Cnt + 1; K < NestedLoopCount; ++K)
         Prod = SemaRef.BuildBinOp(CurScope, UpdLoc, BO_Mul, Prod.get(),
                                   IterSpaces[K].NumIterations);
 
@@ -9694,8 +9698,8 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
       // If there is at least one more inner loop to avoid
       // multiplication by 1.
       if (Cnt + 1 < NestedLoopCount)
-        Iter = SemaRef.BuildBinOp(CurScope, UpdLoc, BO_Div,
-                                  Acc.get(), Prod.get());
+        Iter =
+            SemaRef.BuildBinOp(CurScope, UpdLoc, BO_Div, Acc.get(), Prod.get());
       else
         Iter = Acc;
       if (!Iter.isUsable()) {
@@ -9708,12 +9712,11 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
       // Check if there is at least one more inner loop to avoid
       // multiplication by 1.
       if (Cnt + 1 < NestedLoopCount)
-        Prod = SemaRef.BuildBinOp(CurScope, UpdLoc, BO_Mul,
-                                  Iter.get(), Prod.get());
+        Prod = SemaRef.BuildBinOp(CurScope, UpdLoc, BO_Mul, Iter.get(),
+                                  Prod.get());
       else
         Prod = Iter;
-      Acc = SemaRef.BuildBinOp(CurScope, UpdLoc, BO_Sub,
-                               Acc.get(), Prod.get());
+      Acc = SemaRef.BuildBinOp(CurScope, UpdLoc, BO_Sub, Acc.get(), Prod.get());
 
       // Build update: IS.CounterVar(Private) = IS.Start + Iter * IS.Step
       auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IS.CounterVar)->getDecl());
@@ -10000,7 +10003,7 @@ StmtResult Sema::ActOnOpenMPSectionsDirective(ArrayRef<OMPClause *> Clauses,
       return StmtError();
     // All associated statements must be '#pragma omp section' except for
     // the first one.
-    for (Stmt *SectionStmt : llvm::make_range(std::next(S.begin()), S.end())) {
+    for (Stmt *SectionStmt : llvm::drop_begin(S)) {
       if (!SectionStmt || !isa<OMPSectionDirective>(SectionStmt)) {
         if (SectionStmt)
           Diag(SectionStmt->getBeginLoc(),
@@ -10382,7 +10385,7 @@ Sema::ActOnOpenMPParallelSectionsDirective(ArrayRef<OMPClause *> Clauses,
       return StmtError();
     // All associated statements must be '#pragma omp section' except for
     // the first one.
-    for (Stmt *SectionStmt : llvm::make_range(std::next(S.begin()), S.end())) {
+    for (Stmt *SectionStmt : llvm::drop_begin(S)) {
       if (!SectionStmt || !isa<OMPSectionDirective>(SectionStmt)) {
         if (SectionStmt)
           Diag(SectionStmt->getBeginLoc(),
@@ -10752,7 +10755,6 @@ private:
   bool checkBinaryOperation(BinaryOperator *AtomicBinOp, unsigned DiagId = 0,
                             unsigned NoteId = 0);
 };
-} // namespace
 
 bool OpenMPAtomicUpdateChecker::checkBinaryOperation(
     BinaryOperator *AtomicBinOp, unsigned DiagId, unsigned NoteId) {
@@ -10913,6 +10915,7 @@ bool OpenMPAtomicUpdateChecker::checkStatement(Stmt *S, unsigned DiagId,
   }
   return ErrorFound != NoError;
 }
+} // namespace
 
 StmtResult Sema::ActOnOpenMPAtomicDirective(ArrayRef<OMPClause *> Clauses,
                                             Stmt *AStmt,
@@ -10933,9 +10936,12 @@ StmtResult Sema::ActOnOpenMPAtomicDirective(ArrayRef<OMPClause *> Clauses,
   OpenMPClauseKind MemOrderKind = OMPC_unknown;
   SourceLocation MemOrderLoc;
   for (const OMPClause *C : Clauses) {
-    if (C->getClauseKind() == OMPC_read || C->getClauseKind() == OMPC_write ||
-        C->getClauseKind() == OMPC_update ||
-        C->getClauseKind() == OMPC_capture) {
+    switch (C->getClauseKind()) {
+    case OMPC_read:
+    case OMPC_write:
+    case OMPC_update:
+    case OMPC_capture:
+    case OMPC_compare: {
       if (AtomicKind != OMPC_unknown) {
         Diag(C->getBeginLoc(), diag::err_omp_atomic_several_clauses)
             << SourceRange(C->getBeginLoc(), C->getEndLoc());
@@ -10945,12 +10951,13 @@ StmtResult Sema::ActOnOpenMPAtomicDirective(ArrayRef<OMPClause *> Clauses,
         AtomicKind = C->getClauseKind();
         AtomicKindLoc = C->getBeginLoc();
       }
+      break;
     }
-    if (C->getClauseKind() == OMPC_seq_cst ||
-        C->getClauseKind() == OMPC_acq_rel ||
-        C->getClauseKind() == OMPC_acquire ||
-        C->getClauseKind() == OMPC_release ||
-        C->getClauseKind() == OMPC_relaxed) {
+    case OMPC_seq_cst:
+    case OMPC_acq_rel:
+    case OMPC_acquire:
+    case OMPC_release:
+    case OMPC_relaxed: {
       if (MemOrderKind != OMPC_unknown) {
         Diag(C->getBeginLoc(), diag::err_omp_several_mem_order_clauses)
             << getOpenMPDirectiveName(OMPD_atomic) << 0
@@ -10961,6 +10968,13 @@ StmtResult Sema::ActOnOpenMPAtomicDirective(ArrayRef<OMPClause *> Clauses,
         MemOrderKind = C->getClauseKind();
         MemOrderLoc = C->getBeginLoc();
       }
+      break;
+    }
+    // The following clauses are allowed, but we don't need to do anything here.
+    case OMPC_hint:
+      break;
+    default:
+      llvm_unreachable("unknown clause is encountered");
     }
   }
   // OpenMP 5.0, 2.17.7 atomic Construct, Restrictions
@@ -11075,8 +11089,8 @@ StmtResult Sema::ActOnOpenMPAtomicDirective(ArrayRef<OMPClause *> Clauses,
     if (ErrorFound != NoError) {
       Diag(ErrorLoc, diag::err_omp_atomic_read_not_expression_statement)
           << ErrorRange;
-      Diag(NoteLoc, diag::note_omp_atomic_read_write) << ErrorFound
-                                                      << NoteRange;
+      Diag(NoteLoc, diag::note_omp_atomic_read_write)
+          << ErrorFound << NoteRange;
       return StmtError();
     }
     if (CurContext->isDependentContext())
@@ -11137,8 +11151,8 @@ StmtResult Sema::ActOnOpenMPAtomicDirective(ArrayRef<OMPClause *> Clauses,
     if (ErrorFound != NoError) {
       Diag(ErrorLoc, diag::err_omp_atomic_write_not_expression_statement)
           << ErrorRange;
-      Diag(NoteLoc, diag::note_omp_atomic_read_write) << ErrorFound
-                                                      << NoteRange;
+      Diag(NoteLoc, diag::note_omp_atomic_read_write)
+          << ErrorFound << NoteRange;
       return StmtError();
     }
     if (CurContext->isDependentContext())
@@ -11154,9 +11168,10 @@ StmtResult Sema::ActOnOpenMPAtomicDirective(ArrayRef<OMPClause *> Clauses,
     //  x = expr binop x;
     OpenMPAtomicUpdateChecker Checker(*this);
     if (Checker.checkStatement(
-            Body, (AtomicKind == OMPC_update)
-                      ? diag::err_omp_atomic_update_not_expression_statement
-                      : diag::err_omp_atomic_not_expression_statement,
+            Body,
+            (AtomicKind == OMPC_update)
+                ? diag::err_omp_atomic_update_not_expression_statement
+                : diag::err_omp_atomic_not_expression_statement,
             diag::note_omp_atomic_update))
       return StmtError();
     if (!CurContext->isDependentContext()) {
@@ -11370,15 +11385,21 @@ StmtResult Sema::ActOnOpenMPAtomicDirective(ArrayRef<OMPClause *> Clauses,
             SourceRange(Body->getBeginLoc(), Body->getBeginLoc());
         ErrorFound = NotACompoundStatement;
       }
-      if (ErrorFound != NoError) {
-        Diag(ErrorLoc, diag::err_omp_atomic_capture_not_compound_statement)
-            << ErrorRange;
-        Diag(NoteLoc, diag::note_omp_atomic_capture) << ErrorFound << NoteRange;
-        return StmtError();
-      }
-      if (CurContext->isDependentContext())
-        UE = V = E = X = nullptr;
     }
+    if (ErrorFound != NoError) {
+      Diag(ErrorLoc, diag::err_omp_atomic_capture_not_compound_statement)
+          << ErrorRange;
+      Diag(NoteLoc, diag::note_omp_atomic_capture) << ErrorFound << NoteRange;
+      return StmtError();
+    }
+    if (CurContext->isDependentContext())
+      UE = V = E = X = nullptr;
+  } else if (AtomicKind == OMPC_compare) {
+    // TODO: For now we emit an error here and in emitOMPAtomicExpr we ignore
+    // code gen.
+    unsigned DiagID = Diags.getCustomDiagID(
+        DiagnosticsEngine::Error, "atomic compare is not supported for now");
+    Diag(AtomicKindLoc, DiagID);
   }
 
   setFunctionHasBranchProtectedScope();
@@ -12283,8 +12304,8 @@ StmtResult Sema::ActOnOpenMPTargetParallelForSimdDirective(
   // define the nested loops number.
   unsigned NestedLoopCount = checkOpenMPLoop(
       OMPD_target_parallel_for_simd, getCollapseNumberExpr(Clauses),
-      getOrderedNumberExpr(Clauses), CS, *this, *DSAStack,
-      VarsWithImplicitDSA, B);
+      getOrderedNumberExpr(Clauses), CS, *this, *DSAStack, VarsWithImplicitDSA,
+      B);
   if (NestedLoopCount == 0)
     return StmtError();
 
@@ -13459,6 +13480,7 @@ OMPClause *Sema::ActOnOpenMPSingleExprClause(OpenMPClauseKind Kind, Expr *Expr,
   case OMPC_write:
   case OMPC_update:
   case OMPC_capture:
+  case OMPC_compare:
   case OMPC_seq_cst:
   case OMPC_acq_rel:
   case OMPC_acquire:
@@ -14290,6 +14312,7 @@ static OpenMPDirectiveKind getOpenMPCaptureRegionForClause(
   case OMPC_write:
   case OMPC_update:
   case OMPC_capture:
+  case OMPC_compare:
   case OMPC_seq_cst:
   case OMPC_acq_rel:
   case OMPC_acquire:
@@ -14751,6 +14774,7 @@ OMPClause *Sema::ActOnOpenMPSimpleClause(
   case OMPC_read:
   case OMPC_write:
   case OMPC_capture:
+  case OMPC_compare:
   case OMPC_seq_cst:
   case OMPC_acq_rel:
   case OMPC_acquire:
@@ -15056,6 +15080,7 @@ OMPClause *Sema::ActOnOpenMPSingleExprWithArgClause(
   case OMPC_write:
   case OMPC_update:
   case OMPC_capture:
+  case OMPC_compare:
   case OMPC_seq_cst:
   case OMPC_acq_rel:
   case OMPC_acquire:
@@ -15244,6 +15269,9 @@ OMPClause *Sema::ActOnOpenMPClause(OpenMPClauseKind Kind,
   case OMPC_capture:
     Res = ActOnOpenMPCaptureClause(StartLoc, EndLoc);
     break;
+  case OMPC_compare:
+    Res = ActOnOpenMPCompareClause(StartLoc, EndLoc);
+    break;
   case OMPC_seq_cst:
     Res = ActOnOpenMPSeqCstClause(StartLoc, EndLoc);
     break;
@@ -15390,6 +15418,11 @@ OMPClause *Sema::ActOnOpenMPCaptureClause(SourceLocation StartLoc,
   return new (Context) OMPCaptureClause(StartLoc, EndLoc);
 }
 
+OMPClause *Sema::ActOnOpenMPCompareClause(SourceLocation StartLoc,
+                                          SourceLocation EndLoc) {
+  return new (Context) OMPCompareClause(StartLoc, EndLoc);
+}
+
 OMPClause *Sema::ActOnOpenMPSeqCstClause(SourceLocation StartLoc,
                                          SourceLocation EndLoc) {
   return new (Context) OMPSeqCstClause(StartLoc, EndLoc);
@@ -15858,6 +15891,7 @@ OMPClause *Sema::ActOnOpenMPVarListClause(
   case OMPC_write:
   case OMPC_update:
   case OMPC_capture:
+  case OMPC_compare:
   case OMPC_seq_cst:
   case OMPC_acq_rel:
   case OMPC_acquire:
@@ -15981,9 +16015,8 @@ OMPClause *Sema::ActOnOpenMPPrivateClause(ArrayRef<Expr *> VarList,
       Diag(ELoc, diag::err_omp_variably_modified_type_not_supported)
           << getOpenMPClauseName(OMPC_private) << Type
           << getOpenMPDirectiveName(CurrDir);
-      bool IsDecl =
-          !VD ||
-          VD->isThisDeclarationADefinition(Context) == VarDecl::DeclarationOnly;
+      bool IsDecl = !VD || VD->isThisDeclarationADefinition(Context) ==
+                               VarDecl::DeclarationOnly;
       Diag(D->getLocation(),
            IsDecl ? diag::note_previous_decl : diag::note_defined_here)
           << D;
@@ -16247,9 +16280,8 @@ OMPClause *Sema::ActOnOpenMPFirstprivateClause(ArrayRef<Expr *> VarList,
       Diag(ELoc, diag::err_omp_variably_modified_type_not_supported)
           << getOpenMPClauseName(OMPC_firstprivate) << Type
           << getOpenMPDirectiveName(DSAStack->getCurrentDirective());
-      bool IsDecl =
-          !VD ||
-          VD->isThisDeclarationADefinition(Context) == VarDecl::DeclarationOnly;
+      bool IsDecl = !VD || VD->isThisDeclarationADefinition(Context) ==
+                               VarDecl::DeclarationOnly;
       Diag(D->getLocation(),
            IsDecl ? diag::note_previous_decl : diag::note_defined_here)
           << D;
@@ -17433,8 +17465,8 @@ static bool actOnOMPReductionKindClause(
           llvm::APInt InitValue =
               (BOK != BO_LT) ? IsSigned ? llvm::APInt::getSignedMinValue(Size)
                                         : llvm::APInt::getMinValue(Size)
-                             : IsSigned ? llvm::APInt::getSignedMaxValue(Size)
-                                        : llvm::APInt::getMaxValue(Size);
+              : IsSigned ? llvm::APInt::getSignedMaxValue(Size)
+                             : llvm::APInt::getMaxValue(Size);
           Init = IntegerLiteral::Create(Context, InitValue, IntTy, ELoc);
           if (Type->isPointerType()) {
             // Cast to pointer type.
@@ -17845,9 +17877,8 @@ bool Sema::CheckOpenMPLinearDecl(const ValueDecl *D, SourceLocation ELoc,
               !Ty->isIntegralType(Context) && !Ty->isPointerType())) {
     Diag(ELoc, diag::err_omp_linear_expected_int_or_ptr) << Type;
     if (D) {
-      bool IsDecl =
-          !VD ||
-          VD->isThisDeclarationADefinition(Context) == VarDecl::DeclarationOnly;
+      bool IsDecl = !VD || VD->isThisDeclarationADefinition(Context) ==
+                               VarDecl::DeclarationOnly;
       Diag(D->getLocation(),
            IsDecl ? diag::note_previous_decl : diag::note_defined_here)
           << D;
@@ -18059,13 +18090,12 @@ static bool FinishOpenMPLinearClause(OMPLinearClause &Clause, DeclRefExpr *IV,
     Update = SemaRef.ActOnFinishFullExpr(Update.get(), DE->getBeginLoc(),
                                          /*DiscardedValue*/ false);
 
-    // Build final: Var = InitExpr + NumIterations * Step
+    // Build final: Var = PrivCopy;
     ExprResult Final;
     if (!Info.first)
-      Final =
-          buildCounterUpdate(SemaRef, S, RefExpr->getExprLoc(), CapturedRef,
-                             InitExpr, NumIterations, Step, /*Subtract=*/false,
-                             /*IsNonRectangularLB=*/false);
+      Final = SemaRef.BuildBinOp(
+          S, RefExpr->getExprLoc(), BO_Assign, CapturedRef,
+          SemaRef.DefaultLvalueConversion(*CurPrivate).get());
     else
       Final = *CurPrivate;
     Final = SemaRef.ActOnFinishFullExpr(Final.get(), DE->getBeginLoc(),
@@ -18124,9 +18154,8 @@ OMPClause *Sema::ActOnOpenMPAlignedClause(
     if (!Ty || (!Ty->isArrayType() && !Ty->isPointerType())) {
       Diag(ELoc, diag::err_omp_aligned_expected_array_or_ptr)
           << QType << getLangOpts().CPlusPlus << ERange;
-      bool IsDecl =
-          !VD ||
-          VD->isThisDeclarationADefinition(Context) == VarDecl::DeclarationOnly;
+      bool IsDecl = !VD || VD->isThisDeclarationADefinition(Context) ==
+                               VarDecl::DeclarationOnly;
       Diag(D->getLocation(),
            IsDecl ? diag::note_previous_decl : diag::note_defined_here)
           << D;
@@ -18327,9 +18356,8 @@ OMPClause *Sema::ActOnOpenMPCopyprivateClause(ArrayRef<Expr *> VarList,
       Diag(ELoc, diag::err_omp_variably_modified_type_not_supported)
           << getOpenMPClauseName(OMPC_copyprivate) << Type
           << getOpenMPDirectiveName(DSAStack->getCurrentDirective());
-      bool IsDecl =
-          !VD ||
-          VD->isThisDeclarationADefinition(Context) == VarDecl::DeclarationOnly;
+      bool IsDecl = !VD || VD->isThisDeclarationADefinition(Context) ==
+                               VarDecl::DeclarationOnly;
       Diag(D->getLocation(),
            IsDecl ? diag::note_previous_decl : diag::note_defined_here)
           << D;
@@ -18636,22 +18664,19 @@ Sema::ActOnOpenMPDependClause(Expr *DepModifier, OpenMPDependClauseKind DepKind,
         if (!RefExpr->isValueDependent() && !RefExpr->isTypeDependent() &&
             !RefExpr->isInstantiationDependent() &&
             !RefExpr->containsUnexpandedParameterPack() &&
-            (OMPDependTFound &&
-             DSAStack->getOMPDependT().getTypePtr() == ExprTy.getTypePtr())) {
+            (!RefExpr->IgnoreParenImpCasts()->isLValue() ||
+             (OMPDependTFound &&
+              DSAStack->getOMPDependT().getTypePtr() == ExprTy.getTypePtr()))) {
           Diag(ELoc, diag::err_omp_expected_addressable_lvalue_or_array_item)
-              << (LangOpts.OpenMP >= 50 ? 1 : 0) << 1
-              << RefExpr->getSourceRange();
+              << (LangOpts.OpenMP >= 50 ? 1 : 0)
+              << (LangOpts.OpenMP >= 50 ? 1 : 0) << RefExpr->getSourceRange();
           continue;
         }
 
         auto *ASE = dyn_cast<ArraySubscriptExpr>(SimpleExpr);
-        if (!RefExpr->IgnoreParenImpCasts()->isLValue() ||
-            (ASE && !ASE->getBase()->isTypeDependent() &&
-             !ASE->getBase()
-                  ->getType()
-                  .getNonReferenceType()
-                  ->isPointerType() &&
-             !ASE->getBase()->getType().getNonReferenceType()->isArrayType())) {
+        if (ASE && !ASE->getBase()->isTypeDependent() &&
+            !ASE->getBase()->getType().getNonReferenceType()->isPointerType() &&
+            !ASE->getBase()->getType().getNonReferenceType()->isArrayType()) {
           Diag(ELoc, diag::err_omp_expected_addressable_lvalue_or_array_item)
               << (LangOpts.OpenMP >= 50 ? 1 : 0)
               << (LangOpts.OpenMP >= 50 ? 1 : 0) << RefExpr->getSourceRange();
@@ -18934,7 +18959,7 @@ public:
     if (!isa<FieldDecl>(ME->getMemberDecl())) {
       if (!NoDiagnose) {
         SemaRef.Diag(ELoc, diag::err_omp_expected_access_to_data_field)
-          << ME->getSourceRange();
+            << ME->getSourceRange();
         return false;
       }
       if (RelevantExpr)
@@ -18950,7 +18975,7 @@ public:
     if (FD->isBitField()) {
       if (!NoDiagnose) {
         SemaRef.Diag(ELoc, diag::err_omp_bit_fields_forbidden_in_clause)
-          << ME->getSourceRange() << getOpenMPClauseName(CKind);
+            << ME->getSourceRange() << getOpenMPClauseName(CKind);
         return false;
       }
       if (RelevantExpr)
@@ -18970,7 +18995,7 @@ public:
     if (CurType->isUnionType()) {
       if (!NoDiagnose) {
         SemaRef.Diag(ELoc, diag::err_omp_union_type_not_allowed)
-          << ME->getSourceRange();
+            << ME->getSourceRange();
         return false;
       }
       return RelevantExpr || Visit(E);
@@ -18997,7 +19022,7 @@ public:
     if (!E->getType()->isAnyPointerType() && !E->getType()->isArrayType()) {
       if (!NoDiagnose) {
         SemaRef.Diag(ELoc, diag::err_omp_expected_base_var_name)
-          << 0 << AE->getSourceRange();
+            << 0 << AE->getSourceRange();
         return false;
       }
       return RelevantExpr || Visit(E);
@@ -19006,8 +19031,7 @@ public:
     // If we got an array subscript that express the whole dimension we
     // can have any array expressions before. If it only expressing part of
     // the dimension, we can only have unitary-size array expressions.
-    if (checkArrayExpressionDoesNotReferToWholeSize(SemaRef, AE,
-                                                    E->getType()))
+    if (checkArrayExpressionDoesNotReferToWholeSize(SemaRef, AE, E->getType()))
       AllowWholeSizeArraySection = false;
 
     if (const auto *TE = dyn_cast<CXXThisExpr>(E->IgnoreParenCasts())) {
@@ -19037,7 +19061,7 @@ public:
            "Array sections cannot be implicitly mapped.");
     Expr *E = OASE->getBase()->IgnoreParenImpCasts();
     QualType CurType =
-      OMPArraySectionExpr::getBaseOriginalType(E).getCanonicalType();
+        OMPArraySectionExpr::getBaseOriginalType(E).getCanonicalType();
 
     // OpenMP 4.5 [2.15.5.1, map Clause, Restrictions, C++, p.1]
     //  If the type of a list item is a reference to a type T then the type
@@ -19049,14 +19073,14 @@ public:
 
     if (!IsPointer && !CurType->isArrayType()) {
       SemaRef.Diag(ELoc, diag::err_omp_expected_base_var_name)
-        << 0 << OASE->getSourceRange();
+          << 0 << OASE->getSourceRange();
       return false;
     }
 
     bool NotWhole =
-      checkArrayExpressionDoesNotReferToWholeSize(SemaRef, OASE, CurType);
+        checkArrayExpressionDoesNotReferToWholeSize(SemaRef, OASE, CurType);
     bool NotUnity =
-      checkArrayExpressionDoesNotReferToUnitySize(SemaRef, OASE, CurType);
+        checkArrayExpressionDoesNotReferToUnitySize(SemaRef, OASE, CurType);
 
     if (AllowWholeSizeArraySection) {
       // Any array section is currently allowed. Allowing a whole size array
@@ -19079,9 +19103,9 @@ public:
       // compatible with the properties of the current array section.
       if (NoDiagnose)
         return false;
-      SemaRef.Diag(
-        ELoc, diag::err_array_section_does_not_specify_contiguous_storage)
-        << OASE->getSourceRange();
+      SemaRef.Diag(ELoc,
+                   diag::err_array_section_does_not_specify_contiguous_storage)
+          << OASE->getSourceRange();
       return false;
     }
 
@@ -19180,9 +19204,7 @@ public:
     emitErrorMsg();
     return false;
   }
-  const Expr *getFoundBase() const {
-    return RelevantExpr;
-  }
+  const Expr *getFoundBase() const { return RelevantExpr; }
   explicit MapBaseChecker(
       Sema &SemaRef, OpenMPClauseKind CKind, OpenMPDirectiveKind DKind,
       OMPClauseMappableExprCommon::MappableExprComponentList &Components,
@@ -19193,9 +19215,9 @@ public:
 } // namespace
 
 /// Return the expression of the base of the mappable expression or null if it
-/// cannot be determined and do all the necessary checks to see if the expression
-/// is valid as a standalone mappable expression. In the process, record all the
-/// components of the expression.
+/// cannot be determined and do all the necessary checks to see if the
+/// expression is valid as a standalone mappable expression. In the process,
+/// record all the components of the expression.
 static const Expr *checkMapClauseExpressionBase(
     Sema &SemaRef, Expr *E,
     OMPClauseMappableExprCommon::MappableExprComponentList &CurComponents,
@@ -19385,9 +19407,9 @@ static bool checkMapConflicts(
             return true;
           }
           if (CI->getAssociatedExpression()->getStmtClass() !=
-                         SI->getAssociatedExpression()->getStmtClass() ||
-                     CI->getAssociatedDeclaration()->getCanonicalDecl() ==
-                         SI->getAssociatedDeclaration()->getCanonicalDecl()) {
+                  SI->getAssociatedExpression()->getStmtClass() ||
+              CI->getAssociatedDeclaration()->getCanonicalDecl() ==
+                  SI->getAssociatedDeclaration()->getCanonicalDecl()) {
             assert(CI != CE && SI != SE);
             SemaRef.Diag(DerivedLoc, diag::err_omp_same_pointer_dereferenced)
                 << DerivedLoc;
@@ -19608,7 +19630,7 @@ struct MappableVarListInfo {
     VarBaseDeclarations.reserve(VarList.size());
   }
 };
-}
+} // namespace
 
 // Check the validity of the provided variable list for the provided clause kind
 // \a CKind. In the check process the valid expressions, mappable expression
@@ -21341,8 +21363,7 @@ OMPClause *Sema::ActOnOpenMPInclusiveClause(ArrayRef<Expr *> VarList,
     // A list item that appears in the inclusive or exclusive clause must appear
     // in a reduction clause with the inscan modifier on the enclosing
     // worksharing-loop, worksharing-loop SIMD, or simd construct.
-    if (DVar.CKind != OMPC_reduction ||
-        DVar.Modifier != OMPC_REDUCTION_inscan)
+    if (DVar.CKind != OMPC_reduction || DVar.Modifier != OMPC_REDUCTION_inscan)
       Diag(ELoc, diag::err_omp_inclusive_exclusive_not_reduction)
           << RefExpr->getSourceRange();
 
diff --git a/clang/lib/Sema/SemaStmtAsm.cpp b/clang/lib/Sema/SemaStmtAsm.cpp
index 603611b2d86b..49f7a8d573d5 100644
--- a/clang/lib/Sema/SemaStmtAsm.cpp
+++ b/clang/lib/Sema/SemaStmtAsm.cpp
@@ -296,9 +296,9 @@ StmtResult Sema::ActOnGCCAsmStmt(SourceLocation AsmLoc, bool IsSimple,
         checkExprMemoryConstraintCompat(*this, OutputExpr, Info, false))
       return StmtError();
 
-    // Disallow _ExtInt, since the backends tend to have difficulties with
-    // non-normal sizes.
-    if (OutputExpr->getType()->isExtIntType())
+    // Disallow bit-precise integer types, since the backends tend to have
+    // difficulties with abnormal sizes.
+    if (OutputExpr->getType()->isBitIntType())
       return StmtError(
           Diag(OutputExpr->getBeginLoc(), diag::err_asm_invalid_type)
           << OutputExpr->getType() << 0 /*Input*/
@@ -429,7 +429,7 @@ StmtResult Sema::ActOnGCCAsmStmt(SourceLocation AsmLoc, bool IsSimple,
       }
     }
 
-    if (InputExpr->getType()->isExtIntType())
+    if (InputExpr->getType()->isBitIntType())
       return StmtError(
           Diag(InputExpr->getBeginLoc(), diag::err_asm_invalid_type)
           << InputExpr->getType() << 1 /*Output*/
@@ -924,7 +924,7 @@ StmtResult Sema::ActOnMSAsmStmt(SourceLocation AsmLoc, SourceLocation LBraceLoc,
   setFunctionHasBranchProtectedScope();
 
   for (uint64_t I = 0; I < NumOutputs + NumInputs; ++I) {
-    if (Exprs[I]->getType()->isExtIntType())
+    if (Exprs[I]->getType()->isBitIntType())
       return StmtError(
           Diag(Exprs[I]->getBeginLoc(), diag::err_asm_invalid_type)
           << Exprs[I]->getType() << (I < NumOutputs)
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index f4fd2ea5aa8e..2482f6d404ea 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -6142,12 +6142,12 @@ bool UnnamedLocalNoLinkageFinder::VisitPipeType(const PipeType* T) {
   return false;
 }
 
-bool UnnamedLocalNoLinkageFinder::VisitExtIntType(const ExtIntType *T) {
+bool UnnamedLocalNoLinkageFinder::VisitBitIntType(const BitIntType *T) {
   return false;
 }
 
-bool UnnamedLocalNoLinkageFinder::VisitDependentExtIntType(
-    const DependentExtIntType *T) {
+bool UnnamedLocalNoLinkageFinder::VisitDependentBitIntType(
+    const DependentBitIntType *T) {
   return false;
 }
 
@@ -7089,7 +7089,7 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param,
       QualType IntegerType = ParamType;
       if (const EnumType *Enum = IntegerType->getAs<EnumType>())
         IntegerType = Enum->getDecl()->getIntegerType();
-      Value = Value.extOrTrunc(IntegerType->isExtIntType()
+      Value = Value.extOrTrunc(IntegerType->isBitIntType()
                                    ? Context.getIntWidth(IntegerType)
                                    : Context.getTypeSize(IntegerType));
 
@@ -7184,7 +7184,7 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param,
 
       // Coerce the template argument's value to the value it will have
       // based on the template parameter's type.
-      unsigned AllowedBits = IntegerType->isExtIntType()
+      unsigned AllowedBits = IntegerType->isBitIntType()
                                  ? Context.getIntWidth(IntegerType)
                                  : Context.getTypeSize(IntegerType);
       if (Value.getBitWidth() != AllowedBits)
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index 81edae10335d..e9636d2b942e 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -1597,7 +1597,7 @@ static Sema::TemplateDeductionResult DeduceTemplateArgumentsByTypeMatch(
     case Type::ObjCObject:
     case Type::ObjCInterface:
     case Type::ObjCObjectPointer:
-    case Type::ExtInt:
+    case Type::BitInt:
       return (TDF & TDF_SkipNonDependent) ||
                      ((TDF & TDF_IgnoreQualifiers)
                           ? S.Context.hasSameUnqualifiedType(P, A)
@@ -2144,10 +2144,10 @@ static Sema::TemplateDeductionResult DeduceTemplateArgumentsByTypeMatch(
 
       return Sema::TDK_NonDeducedMismatch;
     }
-    case Type::DependentExtInt: {
-      const auto *IP = P->castAs<DependentExtIntType>();
+    case Type::DependentBitInt: {
+      const auto *IP = P->castAs<DependentBitIntType>();
 
-      if (const auto *IA = A->getAs<ExtIntType>()) {
+      if (const auto *IA = A->getAs<BitIntType>()) {
         if (IP->isUnsigned() != IA->isUnsigned())
           return Sema::TDK_NonDeducedMismatch;
 
@@ -2164,7 +2164,7 @@ static Sema::TemplateDeductionResult DeduceTemplateArgumentsByTypeMatch(
                                              Deduced);
       }
 
-      if (const auto *IA = A->getAs<DependentExtIntType>()) {
+      if (const auto *IA = A->getAs<DependentBitIntType>()) {
         if (IP->isUnsigned() != IA->isUnsigned())
           return Sema::TDK_NonDeducedMismatch;
         return Sema::TDK_Success;
@@ -5949,9 +5949,9 @@ MarkUsedTemplateParameters(ASTContext &Ctx, QualType T,
                                cast<DeducedType>(T)->getDeducedType(),
                                OnlyDeduced, Depth, Used);
     break;
-  case Type::DependentExtInt:
+  case Type::DependentBitInt:
     MarkUsedTemplateParameters(Ctx,
-                               cast<DependentExtIntType>(T)->getNumBitsExpr(),
+                               cast<DependentBitIntType>(T)->getNumBitsExpr(),
                                OnlyDeduced, Depth, Used);
     break;
 
@@ -5966,7 +5966,7 @@ MarkUsedTemplateParameters(ASTContext &Ctx, QualType T,
   case Type::ObjCObjectPointer:
   case Type::UnresolvedUsing:
   case Type::Pipe:
-  case Type::ExtInt:
+  case Type::BitInt:
 #define TYPE(Class, Base)
 #define ABSTRACT_TYPE(Class, Base)
 #define DEPENDENT_TYPE(Class, Base)
diff --git a/clang/lib/Sema/SemaTemplateVariadic.cpp b/clang/lib/Sema/SemaTemplateVariadic.cpp
index c0bb310e64fb..51c79e93ab0a 100644
--- a/clang/lib/Sema/SemaTemplateVariadic.cpp
+++ b/clang/lib/Sema/SemaTemplateVariadic.cpp
@@ -870,7 +870,7 @@ bool Sema::containsUnexpandedParameterPacks(Declarator &D) {
 
   case TST_typeofExpr:
   case TST_decltype:
-  case TST_extint:
+  case TST_bitint:
     if (DS.getRepAsExpr() &&
         DS.getRepAsExpr()->containsUnexpandedParameterPack())
       return true;
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index d2ee669debd0..7a038301a249 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -1435,12 +1435,11 @@ static QualType ConvertDeclSpecToType(TypeProcessingState &state) {
     }
     break;
   }
-  case DeclSpec::TST_extint: {
-    if (!S.Context.getTargetInfo().hasExtIntType())
-      S.Diag(DS.getTypeSpecTypeLoc(), diag::err_type_unsupported)
-        << "_ExtInt";
+  case DeclSpec::TST_bitint: {
+    if (!S.Context.getTargetInfo().hasBitIntType())
+      S.Diag(DS.getTypeSpecTypeLoc(), diag::err_type_unsupported) << "_BitInt";
     Result =
-        S.BuildExtIntType(DS.getTypeSpecSign() == TypeSpecifierSign::Unsigned,
+        S.BuildBitIntType(DS.getTypeSpecSign() == TypeSpecifierSign::Unsigned,
                           DS.getRepAsExpr(), DS.getBeginLoc());
     if (Result.isNull()) {
       Result = Context.IntTy;
@@ -2237,7 +2236,7 @@ QualType Sema::BuildWritePipeType(QualType T, SourceLocation Loc) {
   return Context.getWritePipeType(T);
 }
 
-/// Build a extended int type.
+/// Build a bit-precise integer type.
 ///
 /// \param IsUnsigned Boolean representing the signedness of the type.
 ///
@@ -2245,10 +2244,10 @@ QualType Sema::BuildWritePipeType(QualType T, SourceLocation Loc) {
 /// that.
 ///
 /// \param Loc Location of the keyword.
-QualType Sema::BuildExtIntType(bool IsUnsigned, Expr *BitWidth,
+QualType Sema::BuildBitIntType(bool IsUnsigned, Expr *BitWidth,
                                SourceLocation Loc) {
   if (BitWidth->isInstantiationDependent())
-    return Context.getDependentExtIntType(IsUnsigned, BitWidth);
+    return Context.getDependentBitIntType(IsUnsigned, BitWidth);
 
   llvm::APSInt Bits(32);
   ExprResult ICE =
@@ -2259,22 +2258,22 @@ QualType Sema::BuildExtIntType(bool IsUnsigned, Expr *BitWidth,
 
   int64_t NumBits = Bits.getSExtValue();
   if (!IsUnsigned && NumBits < 2) {
-    Diag(Loc, diag::err_ext_int_bad_size) << 0;
+    Diag(Loc, diag::err_bit_int_bad_size) << 0;
     return QualType();
   }
 
   if (IsUnsigned && NumBits < 1) {
-    Diag(Loc, diag::err_ext_int_bad_size) << 1;
+    Diag(Loc, diag::err_bit_int_bad_size) << 1;
     return QualType();
   }
 
   if (NumBits > llvm::IntegerType::MAX_INT_BITS) {
-    Diag(Loc, diag::err_ext_int_max_size) << IsUnsigned
-                                          << llvm::IntegerType::MAX_INT_BITS;
+    Diag(Loc, diag::err_bit_int_max_size)
+        << IsUnsigned << llvm::IntegerType::MAX_INT_BITS;
     return QualType();
   }
 
-  return Context.getExtIntType(IsUnsigned, NumBits);
+  return Context.getBitIntType(IsUnsigned, NumBits);
 }
 
 /// Check whether the specified array bound can be evaluated using the relevant
@@ -3941,6 +3940,20 @@ static CallingConv getCCForDeclaratorChunk(
         break;
       }
     }
+  } else if (S.getLangOpts().CUDA) {
+    // If we're compiling CUDA/HIP code and targeting SPIR-V we need to make
+    // sure the kernels will be marked with the right calling convention so that
+    // they will be visible by the APIs that ingest SPIR-V.
+    llvm::Triple Triple = S.Context.getTargetInfo().getTriple();
+    if (Triple.getArch() == llvm::Triple::spirv32 ||
+        Triple.getArch() == llvm::Triple::spirv64) {
+      for (const ParsedAttr &AL : D.getDeclSpec().getAttributes()) {
+        if (AL.getKind() == ParsedAttr::AT_CUDAGlobal) {
+          CC = CC_OpenCLKernel;
+          break;
+        }
+      }
+    }
   }
 
   return CC;
@@ -6077,11 +6090,11 @@ namespace {
       TL.getValueLoc().initializeFullCopy(TInfo->getTypeLoc());
     }
 
-    void VisitExtIntTypeLoc(ExtIntTypeLoc TL) {
+    void VisitExtIntTypeLoc(BitIntTypeLoc TL) {
       TL.setNameLoc(DS.getTypeSpecTypeLoc());
     }
 
-    void VisitDependentExtIntTypeLoc(DependentExtIntTypeLoc TL) {
+    void VisitDependentExtIntTypeLoc(DependentBitIntTypeLoc TL) {
       TL.setNameLoc(DS.getTypeSpecTypeLoc());
     }
 
@@ -6211,7 +6224,7 @@ namespace {
       assert(Chunk.Kind == DeclaratorChunk::Pipe);
       TL.setKWLoc(Chunk.Loc);
     }
-    void VisitExtIntTypeLoc(ExtIntTypeLoc TL) {
+    void VisitBitIntTypeLoc(BitIntTypeLoc TL) {
       TL.setNameLoc(Chunk.Loc);
     }
     void VisitMacroQualifiedTypeLoc(MacroQualifiedTypeLoc TL) {
@@ -6528,7 +6541,6 @@ static void HandleBTFTypeTagAttribute(QualType &Type, const ParsedAttr &Attr,
   StringRef BTFTypeTag = StrLiteral->getString();
   Type = State.getAttributedType(
       ::new (Ctx) BTFTypeTagAttr(Ctx, Attr, BTFTypeTag), Type, Type);
-  return;
 }
 
 /// HandleAddressSpaceTypeAttribute - Process an address_space attribute on the
@@ -9079,9 +9091,8 @@ QualType Sema::BuildAtomicType(QualType T, SourceLocation Loc) {
     else if (!T.isTriviallyCopyableType(Context))
       // Some other non-trivially-copyable type (probably a C++ class)
       DisallowedKind = 7;
-    else if (T->isExtIntType()) {
-        DisallowedKind = 8;
-    }
+    else if (T->isBitIntType())
+      DisallowedKind = 8;
 
     if (DisallowedKind != -1) {
       Diag(Loc, diag::err_atomic_specifier_bad_type) << DisallowedKind << T;
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 7f3326c13263..298a3f7a83d8 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -933,6 +933,11 @@ public:
   /// the UnresolvedUsingTypenameDecl was transformed to.
   QualType RebuildUnresolvedUsingType(SourceLocation NameLoc, Decl *D);
 
+  /// Build a new type found via an alias.
+  QualType RebuildUsingType(UsingShadowDecl *Found, QualType Underlying) {
+    return SemaRef.Context.getUsingType(Found, Underlying);
+  }
+
   /// Build a new typedef type.
   QualType RebuildTypedefType(TypedefNameDecl *Typedef) {
     return SemaRef.Context.getTypeDeclType(Typedef);
@@ -1195,12 +1200,12 @@ public:
   QualType RebuildPipeType(QualType ValueType, SourceLocation KWLoc,
                            bool isReadPipe);
 
-   /// Build an extended int given its value type.
-  QualType RebuildExtIntType(bool IsUnsigned, unsigned NumBits,
+  /// Build a bit-precise int given its value type.
+  QualType RebuildBitIntType(bool IsUnsigned, unsigned NumBits,
                              SourceLocation Loc);
 
-  /// Build a dependent extended int given its value type.
-  QualType RebuildDependentExtIntType(bool IsUnsigned, Expr *NumBitsExpr,
+  /// Build a dependent bit-precise int given its value type.
+  QualType RebuildDependentBitIntType(bool IsUnsigned, Expr *NumBitsExpr,
                                       SourceLocation Loc);
 
   /// Build a new template name given a nested name specifier, a flag
@@ -6072,9 +6077,9 @@ QualType TreeTransform<Derived>::TransformFunctionNoProtoType(
   return Result;
 }
 
-template<typename Derived> QualType
-TreeTransform<Derived>::TransformUnresolvedUsingType(TypeLocBuilder &TLB,
-                                                 UnresolvedUsingTypeLoc TL) {
+template <typename Derived>
+QualType TreeTransform<Derived>::TransformUnresolvedUsingType(
+    TypeLocBuilder &TLB, UnresolvedUsingTypeLoc TL) {
   const UnresolvedUsingType *T = TL.getTypePtr();
   Decl *D = getDerived().TransformDecl(TL.getNameLoc(), T->getDecl());
   if (!D)
@@ -6095,6 +6100,32 @@ TreeTransform<Derived>::TransformUnresolvedUsingType(TypeLocBuilder &TLB,
   return Result;
 }
 
+template <typename Derived>
+QualType TreeTransform<Derived>::TransformUsingType(TypeLocBuilder &TLB,
+                                                    UsingTypeLoc TL) {
+  const UsingType *T = TL.getTypePtr();
+
+  auto *Found = cast_or_null<UsingShadowDecl>(getDerived().TransformDecl(
+      TL.getLocalSourceRange().getBegin(), T->getFoundDecl()));
+  if (!Found)
+    return QualType();
+
+  QualType Underlying = getDerived().TransformType(T->desugar());
+  if (Underlying.isNull())
+    return QualType();
+
+  QualType Result = TL.getType();
+  if (getDerived().AlwaysRebuild() || Found != T->getFoundDecl() ||
+      Underlying != T->getUnderlyingType()) {
+    Result = getDerived().RebuildUsingType(Found, Underlying);
+    if (Result.isNull())
+      return QualType();
+  }
+
+  TLB.pushTypeSpec(Result).setNameLoc(TL.getNameLoc());
+  return Result;
+}
+
 template<typename Derived>
 QualType TreeTransform<Derived>::TransformTypedefType(TypeLocBuilder &TLB,
                                                       TypedefTypeLoc TL) {
@@ -6430,27 +6461,27 @@ QualType TreeTransform<Derived>::TransformPipeType(TypeLocBuilder &TLB,
 }
 
 template <typename Derived>
-QualType TreeTransform<Derived>::TransformExtIntType(TypeLocBuilder &TLB,
-                                                     ExtIntTypeLoc TL) {
-  const ExtIntType *EIT = TL.getTypePtr();
+QualType TreeTransform<Derived>::TransformBitIntType(TypeLocBuilder &TLB,
+                                                     BitIntTypeLoc TL) {
+  const BitIntType *EIT = TL.getTypePtr();
   QualType Result = TL.getType();
 
   if (getDerived().AlwaysRebuild()) {
-    Result = getDerived().RebuildExtIntType(EIT->isUnsigned(),
+    Result = getDerived().RebuildBitIntType(EIT->isUnsigned(),
                                             EIT->getNumBits(), TL.getNameLoc());
     if (Result.isNull())
       return QualType();
   }
 
-  ExtIntTypeLoc NewTL = TLB.push<ExtIntTypeLoc>(Result);
+  BitIntTypeLoc NewTL = TLB.push<BitIntTypeLoc>(Result);
   NewTL.setNameLoc(TL.getNameLoc());
   return Result;
 }
 
 template <typename Derived>
-QualType TreeTransform<Derived>::TransformDependentExtIntType(
-    TypeLocBuilder &TLB, DependentExtIntTypeLoc TL) {
-  const DependentExtIntType *EIT = TL.getTypePtr();
+QualType TreeTransform<Derived>::TransformDependentBitIntType(
+    TypeLocBuilder &TLB, DependentBitIntTypeLoc TL) {
+  const DependentBitIntType *EIT = TL.getTypePtr();
 
   EnterExpressionEvaluationContext Unevaluated(
       SemaRef, Sema::ExpressionEvaluationContext::ConstantEvaluated);
@@ -6463,18 +6494,18 @@ QualType TreeTransform<Derived>::TransformDependentExtIntType(
   QualType Result = TL.getType();
 
   if (getDerived().AlwaysRebuild() || BitsExpr.get() != EIT->getNumBitsExpr()) {
-    Result = getDerived().RebuildDependentExtIntType(
+    Result = getDerived().RebuildDependentBitIntType(
         EIT->isUnsigned(), BitsExpr.get(), TL.getNameLoc());
 
     if (Result.isNull())
       return QualType();
   }
 
-  if (isa<DependentExtIntType>(Result)) {
-    DependentExtIntTypeLoc NewTL = TLB.push<DependentExtIntTypeLoc>(Result);
+  if (isa<DependentBitIntType>(Result)) {
+    DependentBitIntTypeLoc NewTL = TLB.push<DependentBitIntTypeLoc>(Result);
     NewTL.setNameLoc(TL.getNameLoc());
   } else {
-    ExtIntTypeLoc NewTL = TLB.push<ExtIntTypeLoc>(Result);
+    BitIntTypeLoc NewTL = TLB.push<BitIntTypeLoc>(Result);
     NewTL.setNameLoc(TL.getNameLoc());
   }
   return Result;
@@ -9431,6 +9462,13 @@ TreeTransform<Derived>::TransformOMPCaptureClause(OMPCaptureClause *C) {
 
 template <typename Derived>
 OMPClause *
+TreeTransform<Derived>::TransformOMPCompareClause(OMPCompareClause *C) {
+  // No need to rebuild this clause, no template-dependent parameters.
+  return C;
+}
+
+template <typename Derived>
+OMPClause *
 TreeTransform<Derived>::TransformOMPSeqCstClause(OMPSeqCstClause *C) {
   // No need to rebuild this clause, no template-dependent parameters.
   return C;
@@ -14462,7 +14500,6 @@ QualType TreeTransform<Derived>::RebuildUnresolvedUsingType(SourceLocation Loc,
   if (D->isInvalidDecl()) return QualType();
 
   // FIXME: Doesn't account for ObjCInterfaceDecl!
-  TypeDecl *Ty;
   if (auto *UPD = dyn_cast<UsingPackDecl>(D)) {
     // A valid resolved using typename pack expansion decl can have multiple
     // UsingDecls, but they must each have exactly one type, and it must be
@@ -14498,17 +14535,18 @@ QualType TreeTransform<Derived>::RebuildUnresolvedUsingType(SourceLocation Loc,
     // A valid resolved using typename decl points to exactly one type decl.
     assert(++Using->shadow_begin() == Using->shadow_end());
 
-    NamedDecl *Target = Using->shadow_begin()->getTargetDecl();
-    if (SemaRef.DiagnoseUseOfDecl(Target, Loc))
+    UsingShadowDecl *Shadow = *Using->shadow_begin();
+    if (SemaRef.DiagnoseUseOfDecl(Shadow->getTargetDecl(), Loc))
       return QualType();
-    Ty = cast<TypeDecl>(Target);
+    return SemaRef.Context.getUsingType(
+        Shadow, SemaRef.Context.getTypeDeclType(
+                    cast<TypeDecl>(Shadow->getTargetDecl())));
   } else {
     assert(isa<UnresolvedUsingTypenameDecl>(D) &&
            "UnresolvedUsingTypenameDecl transformed to non-using decl");
-    Ty = cast<UnresolvedUsingTypenameDecl>(D);
+    return SemaRef.Context.getTypeDeclType(
+        cast<UnresolvedUsingTypenameDecl>(D));
   }
-
-  return SemaRef.Context.getTypeDeclType(Ty);
 }
 
 template <typename Derived>
@@ -14557,20 +14595,20 @@ QualType TreeTransform<Derived>::RebuildPipeType(QualType ValueType,
 }
 
 template <typename Derived>
-QualType TreeTransform<Derived>::RebuildExtIntType(bool IsUnsigned,
+QualType TreeTransform<Derived>::RebuildBitIntType(bool IsUnsigned,
                                                    unsigned NumBits,
                                                    SourceLocation Loc) {
   llvm::APInt NumBitsAP(SemaRef.Context.getIntWidth(SemaRef.Context.IntTy),
                         NumBits, true);
   IntegerLiteral *Bits = IntegerLiteral::Create(SemaRef.Context, NumBitsAP,
                                                 SemaRef.Context.IntTy, Loc);
-  return SemaRef.BuildExtIntType(IsUnsigned, Bits, Loc);
+  return SemaRef.BuildBitIntType(IsUnsigned, Bits, Loc);
 }
 
 template <typename Derived>
-QualType TreeTransform<Derived>::RebuildDependentExtIntType(
+QualType TreeTransform<Derived>::RebuildDependentBitIntType(
     bool IsUnsigned, Expr *NumBitsExpr, SourceLocation Loc) {
-  return SemaRef.BuildExtIntType(IsUnsigned, NumBitsExpr, Loc);
+  return SemaRef.BuildBitIntType(IsUnsigned, NumBitsExpr, Loc);
 }
 
 template<typename Derived>
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index a033bccbe506..f93e0d2ed1c4 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -6607,6 +6607,10 @@ void TypeLocReader::VisitUnresolvedUsingTypeLoc(UnresolvedUsingTypeLoc TL) {
   TL.setNameLoc(readSourceLocation());
 }
 
+void TypeLocReader::VisitUsingTypeLoc(UsingTypeLoc TL) {
+  TL.setNameLoc(readSourceLocation());
+}
+
 void TypeLocReader::VisitTypedefTypeLoc(TypedefTypeLoc TL) {
   TL.setNameLoc(readSourceLocation());
 }
@@ -6772,11 +6776,11 @@ void TypeLocReader::VisitPipeTypeLoc(PipeTypeLoc TL) {
   TL.setKWLoc(readSourceLocation());
 }
 
-void TypeLocReader::VisitExtIntTypeLoc(clang::ExtIntTypeLoc TL) {
+void TypeLocReader::VisitBitIntTypeLoc(clang::BitIntTypeLoc TL) {
   TL.setNameLoc(readSourceLocation());
 }
-void TypeLocReader::VisitDependentExtIntTypeLoc(
-    clang::DependentExtIntTypeLoc TL) {
+void TypeLocReader::VisitDependentBitIntTypeLoc(
+    clang::DependentBitIntTypeLoc TL) {
   TL.setNameLoc(readSourceLocation());
 }
 
@@ -11761,6 +11765,9 @@ OMPClause *OMPClauseReader::readClause() {
   case llvm::omp::OMPC_capture:
     C = new (Context) OMPCaptureClause();
     break;
+  case llvm::omp::OMPC_compare:
+    C = new (Context) OMPCompareClause();
+    break;
   case llvm::omp::OMPC_seq_cst:
     C = new (Context) OMPSeqCstClause();
     break;
@@ -12119,6 +12126,8 @@ void OMPClauseReader::VisitOMPUpdateClause(OMPUpdateClause *C) {
 
 void OMPClauseReader::VisitOMPCaptureClause(OMPCaptureClause *) {}
 
+void OMPClauseReader::VisitOMPCompareClause(OMPCompareClause *) {}
+
 void OMPClauseReader::VisitOMPSeqCstClause(OMPSeqCstClause *) {}
 
 void OMPClauseReader::VisitOMPAcqRelClause(OMPAcqRelClause *) {}
diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp
index 62a31f299d6b..2144befcdb14 100644
--- a/clang/lib/Serialization/ASTReaderDecl.cpp
+++ b/clang/lib/Serialization/ASTReaderDecl.cpp
@@ -2948,6 +2948,7 @@ uint64_t ASTReader::getGlobalBitOffset(ModuleFile &M, uint64_t LocalOffset) {
 static bool isSameTemplateParameterList(const ASTContext &C,
                                         const TemplateParameterList *X,
                                         const TemplateParameterList *Y);
+static bool isSameEntity(NamedDecl *X, NamedDecl *Y);
 
 /// Determine whether two template parameters are similar enough
 /// that they may be used in declarations of the same template.
@@ -2967,7 +2968,9 @@ static bool isSameTemplateParameter(const NamedDecl *X,
     if (!TXTC != !TYTC)
       return false;
     if (TXTC && TYTC) {
-      if (TXTC->getNamedConcept() != TYTC->getNamedConcept())
+      auto *NCX = TXTC->getNamedConcept();
+      auto *NCY = TYTC->getNamedConcept();
+      if (!NCX || !NCY || !isSameEntity(NCX, NCY))
         return false;
       if (TXTC->hasExplicitTemplateArgs() != TYTC->hasExplicitTemplateArgs())
         return false;
@@ -3111,11 +3114,12 @@ static bool hasSameOverloadableAttrs(const FunctionDecl *A,
 
 /// Determine whether the two declarations refer to the same entity.
 static bool isSameEntity(NamedDecl *X, NamedDecl *Y) {
-  assert(X->getDeclName() == Y->getDeclName() && "Declaration name mismatch!");
-
   if (X == Y)
     return true;
 
+  if (X->getDeclName() != Y->getDeclName())
+    return false;
+
   // Must be in the same context.
   //
   // Note that we can't use DeclContext::Equals here, because the DeclContexts
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index a1972f5c6496..65a780e67510 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -396,6 +396,10 @@ void TypeLocWriter::VisitUnresolvedUsingTypeLoc(UnresolvedUsingTypeLoc TL) {
   Record.AddSourceLocation(TL.getNameLoc());
 }
 
+void TypeLocWriter::VisitUsingTypeLoc(UsingTypeLoc TL) {
+  Record.AddSourceLocation(TL.getNameLoc());
+}
+
 void TypeLocWriter::VisitTypedefTypeLoc(TypedefTypeLoc TL) {
   Record.AddSourceLocation(TL.getNameLoc());
 }
@@ -562,11 +566,11 @@ void TypeLocWriter::VisitPipeTypeLoc(PipeTypeLoc TL) {
   Record.AddSourceLocation(TL.getKWLoc());
 }
 
-void TypeLocWriter::VisitExtIntTypeLoc(clang::ExtIntTypeLoc TL) {
+void TypeLocWriter::VisitBitIntTypeLoc(clang::BitIntTypeLoc TL) {
   Record.AddSourceLocation(TL.getNameLoc());
 }
-void TypeLocWriter::VisitDependentExtIntTypeLoc(
-    clang::DependentExtIntTypeLoc TL) {
+void TypeLocWriter::VisitDependentBitIntTypeLoc(
+    clang::DependentBitIntTypeLoc TL) {
   Record.AddSourceLocation(TL.getNameLoc());
 }
 
@@ -6248,6 +6252,8 @@ void OMPClauseWriter::VisitOMPUpdateClause(OMPUpdateClause *C) {
 
 void OMPClauseWriter::VisitOMPCaptureClause(OMPCaptureClause *) {}
 
+void OMPClauseWriter::VisitOMPCompareClause(OMPCompareClause *) {}
+
 void OMPClauseWriter::VisitOMPSeqCstClause(OMPSeqCstClause *) {}
 
 void OMPClauseWriter::VisitOMPAcqRelClause(OMPAcqRelClause *) {}
diff --git a/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp
index 13781b336426..4a56156de4b2 100644
--- a/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp
@@ -66,7 +66,8 @@ bool BuiltinFunctionChecker::evalCall(const CallEvent &Call,
   case Builtin::BI__builtin_expect:
   case Builtin::BI__builtin_expect_with_probability:
   case Builtin::BI__builtin_assume_aligned:
-  case Builtin::BI__builtin_addressof: {
+  case Builtin::BI__builtin_addressof:
+  case Builtin::BI__builtin_function_start: {
     // For __builtin_unpredictable, __builtin_expect,
     // __builtin_expect_with_probability and __builtin_assume_aligned,
     // just return the value of the subexpression.
diff --git a/clang/lib/StaticAnalyzer/Checkers/ConversionChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/ConversionChecker.cpp
index 8da482a2aec9..6ea39fb95e9a 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ConversionChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ConversionChecker.cpp
@@ -56,9 +56,8 @@ private:
 
 void ConversionChecker::checkPreStmt(const ImplicitCastExpr *Cast,
                                      CheckerContext &C) const {
-  // TODO: For now we only warn about DeclRefExpr, to avoid noise. Warn for
-  // calculations also.
-  if (!isa<DeclRefExpr>(Cast->IgnoreParenImpCasts()))
+  // Don't warn for implicit conversions to bool
+  if (Cast->getType()->isBooleanType())
     return;
 
   // Don't warn for loss of sign/precision in macros.
@@ -70,6 +69,9 @@ void ConversionChecker::checkPreStmt(const ImplicitCastExpr *Cast,
   const Stmt *Parent = PM.getParent(Cast);
   if (!Parent)
     return;
+  // Dont warn if this is part of an explicit cast
+  if (isa<ExplicitCastExpr>(Parent))
+    return;
 
   bool LossOfSign = false;
   bool LossOfPrecision = false;
@@ -78,8 +80,10 @@ void ConversionChecker::checkPreStmt(const ImplicitCastExpr *Cast,
   if (const auto *B = dyn_cast<BinaryOperator>(Parent)) {
     BinaryOperator::Opcode Opc = B->getOpcode();
     if (Opc == BO_Assign) {
-      LossOfSign = isLossOfSign(Cast, C);
-      LossOfPrecision = isLossOfPrecision(Cast, Cast->getType(), C);
+      if (!Cast->IgnoreParenImpCasts()->isEvaluatable(C.getASTContext())) {
+        LossOfSign = isLossOfSign(Cast, C);
+        LossOfPrecision = isLossOfPrecision(Cast, Cast->getType(), C);
+      }
     } else if (Opc == BO_AddAssign || Opc == BO_SubAssign) {
       // No loss of sign.
       LossOfPrecision = isLossOfPrecision(Cast, B->getLHS()->getType(), C);
@@ -98,7 +102,12 @@ void ConversionChecker::checkPreStmt(const ImplicitCastExpr *Cast,
     } else if (B->isRelationalOp() || B->isMultiplicativeOp()) {
       LossOfSign = isLossOfSign(Cast, C);
     }
-  } else if (isa<DeclStmt>(Parent)) {
+  } else if (isa<DeclStmt, ReturnStmt>(Parent)) {
+    if (!Cast->IgnoreParenImpCasts()->isEvaluatable(C.getASTContext())) {
+      LossOfSign = isLossOfSign(Cast, C);
+      LossOfPrecision = isLossOfPrecision(Cast, Cast->getType(), C);
+    }
+  } else {
     LossOfSign = isLossOfSign(Cast, C);
     LossOfPrecision = isLossOfPrecision(Cast, Cast->getType(), C);
   }
diff --git a/clang/lib/StaticAnalyzer/Checkers/FuchsiaHandleChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/FuchsiaHandleChecker.cpp
index e3f4be0726c8..6e4801aa8e91 100644
--- a/clang/lib/StaticAnalyzer/Checkers/FuchsiaHandleChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/FuchsiaHandleChecker.cpp
@@ -389,7 +389,7 @@ void FuchsiaHandleChecker::checkPostCall(const CallEvent &Call,
         llvm::raw_string_ostream OS(SBuf);
         OS << "Function '" << FuncDecl->getDeclName()
            << "' returns an open handle";
-        return OS.str();
+        return SBuf;
       } else
         return "";
     });
@@ -405,7 +405,7 @@ void FuchsiaHandleChecker::checkPostCall(const CallEvent &Call,
         llvm::raw_string_ostream OS(SBuf);
         OS << "Function '" << FuncDecl->getDeclName()
            << "' returns an unowned handle";
-        return OS.str();
+        return SBuf;
       } else
         return "";
     });
@@ -439,7 +439,7 @@ void FuchsiaHandleChecker::checkPostCall(const CallEvent &Call,
               llvm::raw_string_ostream OS(SBuf);
               OS << "Handle released through " << ParamDiagIdx
                  << llvm::getOrdinalSuffix(ParamDiagIdx) << " parameter";
-              return OS.str();
+              return SBuf;
             } else
               return "";
           });
@@ -453,7 +453,7 @@ void FuchsiaHandleChecker::checkPostCall(const CallEvent &Call,
             llvm::raw_string_ostream OS(SBuf);
             OS << "Handle allocated through " << ParamDiagIdx
                << llvm::getOrdinalSuffix(ParamDiagIdx) << " parameter";
-            return OS.str();
+            return SBuf;
           } else
             return "";
         });
@@ -467,7 +467,7 @@ void FuchsiaHandleChecker::checkPostCall(const CallEvent &Call,
             llvm::raw_string_ostream OS(SBuf);
             OS << "Unowned handle allocated through " << ParamDiagIdx
                << llvm::getOrdinalSuffix(ParamDiagIdx) << " parameter";
-            return OS.str();
+            return SBuf;
           } else
             return "";
         });
diff --git a/clang/lib/StaticAnalyzer/Checkers/SmartPtr.h b/clang/lib/StaticAnalyzer/Checkers/SmartPtr.h
index 6a40f8eda5fa..b4352b450c7f 100644
--- a/clang/lib/StaticAnalyzer/Checkers/SmartPtr.h
+++ b/clang/lib/StaticAnalyzer/Checkers/SmartPtr.h
@@ -25,8 +25,6 @@ bool isStdSmartPtrCall(const CallEvent &Call);
 bool isStdSmartPtr(const CXXRecordDecl *RD);
 bool isStdSmartPtr(const Expr *E);
 
-bool isStdSmartPtr(const CXXRecordDecl *RD);
-
 /// Returns whether the smart pointer is null or not.
 bool isNullSmartPtr(const ProgramStateRef State, const MemRegion *ThisRegion);
 
diff --git a/clang/lib/StaticAnalyzer/Core/CallEvent.cpp b/clang/lib/StaticAnalyzer/Core/CallEvent.cpp
index 764dad3e7ab4..ae46709340d3 100644
--- a/clang/lib/StaticAnalyzer/Core/CallEvent.cpp
+++ b/clang/lib/StaticAnalyzer/Core/CallEvent.cpp
@@ -762,9 +762,9 @@ void CXXInstanceCall::getInitialStackFrameContents(
       QualType Ty = Ctx.getPointerType(Ctx.getRecordType(Class));
 
       // FIXME: CallEvent maybe shouldn't be directly accessing StoreManager.
-      bool Failed;
-      ThisVal = StateMgr.getStoreManager().attemptDownCast(ThisVal, Ty, Failed);
-      if (Failed) {
+      Optional<SVal> V =
+          StateMgr.getStoreManager().evalBaseToDerived(ThisVal, Ty);
+      if (!V.hasValue()) {
         // We might have suffered some sort of placement new earlier, so
         // we're constructing in a completely unexpected storage.
         // Fall back to a generic pointer cast for this-value.
@@ -772,7 +772,8 @@ void CXXInstanceCall::getInitialStackFrameContents(
         const CXXRecordDecl *StaticClass = StaticMD->getParent();
         QualType StaticTy = Ctx.getPointerType(Ctx.getRecordType(StaticClass));
         ThisVal = SVB.evalCast(ThisVal, Ty, StaticTy);
-      }
+      } else
+        ThisVal = *V;
     }
 
     if (!ThisVal.isUnknown())
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp
index 69d67cf9b465..637e4edfd778 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp
@@ -439,14 +439,15 @@ void ExprEngine::VisitCast(const CastExpr *CastE, const Expr *Ex,
         if (CastE->isGLValue())
           resultType = getContext().getPointerType(resultType);
 
-        bool Failed = false;
-
-        // Check if the value being cast evaluates to 0.
-        if (val.isZeroConstant())
-          Failed = true;
-        // Else, evaluate the cast.
-        else
-          val = getStoreManager().attemptDownCast(val, T, Failed);
+        bool Failed = true;
+
+        // Check if the value being cast does not evaluates to 0.
+        if (!val.isZeroConstant())
+          if (Optional<SVal> V =
+                  StateMgr.getStoreManager().evalBaseToDerived(val, T)) {
+            val = *V;
+            Failed = false;
+          }
 
         if (Failed) {
           if (T->isReferenceType()) {
@@ -478,14 +479,13 @@ void ExprEngine::VisitCast(const CastExpr *CastE, const Expr *Ex,
         if (CastE->isGLValue())
           resultType = getContext().getPointerType(resultType);
 
-        bool Failed = false;
-
         if (!val.isConstant()) {
-          val = getStoreManager().attemptDownCast(val, T, Failed);
+          Optional<SVal> V = getStoreManager().evalBaseToDerived(val, T);
+          val = V ? *V : UnknownVal();
         }
 
         // Failed to cast or the result is unknown, fall back to conservative.
-        if (Failed || val.isUnknown()) {
+        if (val.isUnknown()) {
           val =
             svalBuilder.conjureSymbolVal(nullptr, CastE, LCtx, resultType,
                                          currBldrCtx->blockCount());
diff --git a/clang/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp b/clang/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp
index 3b847d6f0d87..b4578385a147 100644
--- a/clang/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp
+++ b/clang/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp
@@ -410,7 +410,7 @@ std::string HTMLDiagnostics::GenerateHTML(const PathDiagnostic& D, Rewriter &R,
     }
 
     // Append files to the main report file in the order they appear in the path
-    for (auto I : llvm::make_range(FileIDs.begin() + 1, FileIDs.end())) {
+    for (auto I : llvm::drop_begin(FileIDs)) {
       std::string s;
       llvm::raw_string_ostream os(s);
 
@@ -437,7 +437,7 @@ std::string HTMLDiagnostics::GenerateHTML(const PathDiagnostic& D, Rewriter &R,
   for (auto BI : *Buf)
     os << BI;
 
-  return os.str();
+  return file;
 }
 
 void HTMLDiagnostics::dumpCoverageData(
@@ -534,7 +534,7 @@ document.addEventListener("DOMContentLoaded", function() {
 </form>
 )<<<";
 
-  return os.str();
+  return s;
 }
 
 void HTMLDiagnostics::FinalizeHTML(const PathDiagnostic& D, Rewriter &R,
@@ -1202,7 +1202,7 @@ std::string getSpanBeginForControl(const char *ClassName, unsigned Index) {
   std::string Result;
   llvm::raw_string_ostream OS(Result);
   OS << "<span id=\"" << ClassName << Index << "\">";
-  return OS.str();
+  return Result;
 }
 
 std::string getSpanBeginForControlStart(unsigned Index) {
diff --git a/clang/lib/StaticAnalyzer/Core/MemRegion.cpp b/clang/lib/StaticAnalyzer/Core/MemRegion.cpp
index f77fcb030a15..58bea12f8783 100644
--- a/clang/lib/StaticAnalyzer/Core/MemRegion.cpp
+++ b/clang/lib/StaticAnalyzer/Core/MemRegion.cpp
@@ -444,7 +444,7 @@ std::string MemRegion::getString() const {
   std::string s;
   llvm::raw_string_ostream os(s);
   dumpToStream(os);
-  return os.str();
+  return s;
 }
 
 void MemRegion::dumpToStream(raw_ostream &os) const {
diff --git a/clang/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp b/clang/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp
index 23c67c64f975..4b0d4942e528 100644
--- a/clang/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp
+++ b/clang/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp
@@ -110,6 +110,14 @@ public:
 
 RangeSet::ContainerType RangeSet::Factory::EmptySet{};
 
+RangeSet RangeSet::Factory::add(RangeSet LHS, RangeSet RHS) {
+  ContainerType Result;
+  Result.reserve(LHS.size() + RHS.size());
+  std::merge(LHS.begin(), LHS.end(), RHS.begin(), RHS.end(),
+             std::back_inserter(Result));
+  return makePersistent(std::move(Result));
+}
+
 RangeSet RangeSet::Factory::add(RangeSet Original, Range Element) {
   ContainerType Result;
   Result.reserve(Original.size() + 1);
@@ -126,6 +134,186 @@ RangeSet RangeSet::Factory::add(RangeSet Original, const llvm::APSInt &Point) {
   return add(Original, Range(Point));
 }
 
+RangeSet RangeSet::Factory::unite(RangeSet LHS, RangeSet RHS) {
+  ContainerType Result = unite(*LHS.Impl, *RHS.Impl);
+  return makePersistent(std::move(Result));
+}
+
+RangeSet RangeSet::Factory::unite(RangeSet Original, Range R) {
+  ContainerType Result;
+  Result.push_back(R);
+  Result = unite(*Original.Impl, Result);
+  return makePersistent(std::move(Result));
+}
+
+RangeSet RangeSet::Factory::unite(RangeSet Original, llvm::APSInt Point) {
+  return unite(Original, Range(ValueFactory.getValue(Point)));
+}
+
+RangeSet RangeSet::Factory::unite(RangeSet Original, llvm::APSInt From,
+                                  llvm::APSInt To) {
+  return unite(Original,
+               Range(ValueFactory.getValue(From), ValueFactory.getValue(To)));
+}
+
+template <typename T>
+void swapIterators(T &First, T &FirstEnd, T &Second, T &SecondEnd) {
+  std::swap(First, Second);
+  std::swap(FirstEnd, SecondEnd);
+}
+
+RangeSet::ContainerType RangeSet::Factory::unite(const ContainerType &LHS,
+                                                 const ContainerType &RHS) {
+  if (LHS.empty())
+    return RHS;
+  if (RHS.empty())
+    return LHS;
+
+  using llvm::APSInt;
+  using iterator = ContainerType::const_iterator;
+
+  iterator First = LHS.begin();
+  iterator FirstEnd = LHS.end();
+  iterator Second = RHS.begin();
+  iterator SecondEnd = RHS.end();
+  APSIntType Ty = APSIntType(First->From());
+  const APSInt Min = Ty.getMinValue();
+
+  // Handle a corner case first when both range sets start from MIN.
+  // This helps to avoid complicated conditions below. Specifically, this
+  // particular check for `MIN` is not needed in the loop below every time
+  // when we do `Second->From() - One` operation.
+  if (Min == First->From() && Min == Second->From()) {
+    if (First->To() > Second->To()) {
+      //    [ First    ]--->
+      //    [ Second ]----->
+      // MIN^
+      // The Second range is entirely inside the First one.
+
+      // Check if Second is the last in its RangeSet.
+      if (++Second == SecondEnd)
+        //    [ First     ]--[ First + 1 ]--->
+        //    [ Second ]--------------------->
+        // MIN^
+        // The Union is equal to First's RangeSet.
+        return LHS;
+    } else {
+      // case 1: [ First ]----->
+      // case 2: [ First   ]--->
+      //         [ Second  ]--->
+      //      MIN^
+      // The First range is entirely inside or equal to the Second one.
+
+      // Check if First is the last in its RangeSet.
+      if (++First == FirstEnd)
+        //    [ First ]----------------------->
+        //    [ Second  ]--[ Second + 1 ]---->
+        // MIN^
+        // The Union is equal to Second's RangeSet.
+        return RHS;
+    }
+  }
+
+  const APSInt One = Ty.getValue(1);
+  ContainerType Result;
+
+  // This is called when there are no ranges left in one of the ranges.
+  // Append the rest of the ranges from another range set to the Result
+  // and return with that.
+  const auto AppendTheRest = [&Result](iterator I, iterator E) {
+    Result.append(I, E);
+    return Result;
+  };
+
+  while (true) {
+    // We want to keep the following invariant at all times:
+    // ---[ First ------>
+    // -----[ Second --->
+    if (First->From() > Second->From())
+      swapIterators(First, FirstEnd, Second, SecondEnd);
+
+    // The Union definitely starts with First->From().
+    // ----------[ First ------>
+    // ------------[ Second --->
+    // ----------[ Union ------>
+    // UnionStart^
+    const llvm::APSInt &UnionStart = First->From();
+
+    // Loop where the invariant holds.
+    while (true) {
+      // Skip all enclosed ranges.
+      // ---[                  First                     ]--->
+      // -----[ Second ]--[ Second + 1 ]--[ Second + N ]----->
+      while (First->To() >= Second->To()) {
+        // Check if Second is the last in its RangeSet.
+        if (++Second == SecondEnd) {
+          // Append the Union.
+          // ---[ Union      ]--->
+          // -----[ Second ]----->
+          // --------[ First ]--->
+          //         UnionEnd^
+          Result.emplace_back(UnionStart, First->To());
+          // ---[ Union ]----------------->
+          // --------------[ First + 1]--->
+          // Append all remaining ranges from the First's RangeSet.
+          return AppendTheRest(++First, FirstEnd);
+        }
+      }
+
+      // Check if First and Second are disjoint. It means that we find
+      // the end of the Union. Exit the loop and append the Union.
+      // ---[ First ]=------------->
+      // ------------=[ Second ]--->
+      // ----MinusOne^
+      if (First->To() < Second->From() - One)
+        break;
+
+      // First is entirely inside the Union. Go next.
+      // ---[ Union ----------->
+      // ---- [ First ]-------->
+      // -------[ Second ]----->
+      // Check if First is the last in its RangeSet.
+      if (++First == FirstEnd) {
+        // Append the Union.
+        // ---[ Union       ]--->
+        // -----[ First ]------->
+        // --------[ Second ]--->
+        //          UnionEnd^
+        Result.emplace_back(UnionStart, Second->To());
+        // ---[ Union ]------------------>
+        // --------------[ Second + 1]--->
+        // Append all remaining ranges from the Second's RangeSet.
+        return AppendTheRest(++Second, SecondEnd);
+      }
+
+      // We know that we are at one of the two cases:
+      // case 1: --[ First ]--------->
+      // case 2: ----[ First ]------->
+      // --------[ Second ]---------->
+      // In both cases First starts after Second->From().
+      // Make sure that the loop invariant holds.
+      swapIterators(First, FirstEnd, Second, SecondEnd);
+    }
+
+    // Here First and Second are disjoint.
+    // Append the Union.
+    // ---[ Union    ]--------------->
+    // -----------------[ Second ]--->
+    // ------[ First ]--------------->
+    //       UnionEnd^
+    Result.emplace_back(UnionStart, First->To());
+
+    // Check if First is the last in its RangeSet.
+    if (++First == FirstEnd)
+      // ---[ Union ]--------------->
+      // --------------[ Second ]--->
+      // Append all remaining ranges from the Second's RangeSet.
+      return AppendTheRest(Second, SecondEnd);
+  }
+
+  llvm_unreachable("Normally, we should not reach here");
+}
+
 RangeSet RangeSet::Factory::getRangeSet(Range From) {
   ContainerType Result;
   Result.push_back(From);
@@ -155,13 +343,6 @@ RangeSet::ContainerType *RangeSet::Factory::construct(ContainerType &&From) {
   return new (Buffer) ContainerType(std::move(From));
 }
 
-RangeSet RangeSet::Factory::add(RangeSet LHS, RangeSet RHS) {
-  ContainerType Result;
-  std::merge(LHS.begin(), LHS.end(), RHS.begin(), RHS.end(),
-             std::back_inserter(Result));
-  return makePersistent(std::move(Result));
-}
-
 const llvm::APSInt &RangeSet::getMinValue() const {
   assert(!isEmpty());
   return begin()->From();
@@ -325,11 +506,6 @@ RangeSet RangeSet::Factory::intersect(const RangeSet::ContainerType &LHS,
   const_iterator First = LHS.begin(), Second = RHS.begin(),
                  FirstEnd = LHS.end(), SecondEnd = RHS.end();
 
-  const auto SwapIterators = [&First, &FirstEnd, &Second, &SecondEnd]() {
-    std::swap(First, Second);
-    std::swap(FirstEnd, SecondEnd);
-  };
-
   // If we ran out of ranges in one set, but not in the other,
   // it means that those elements are definitely not in the
   // intersection.
@@ -339,7 +515,7 @@ RangeSet RangeSet::Factory::intersect(const RangeSet::ContainerType &LHS,
     //    ----[ First ---------------------->
     //    --------[ Second ----------------->
     if (Second->From() < First->From())
-      SwapIterators();
+      swapIterators(First, FirstEnd, Second, SecondEnd);
 
     // Loop where the invariant holds:
     do {
@@ -373,7 +549,7 @@ RangeSet RangeSet::Factory::intersect(const RangeSet::ContainerType &LHS,
       if (Second->To() > First->To()) {
         // Here we make a decision to keep First as the "longer"
         // range.
-        SwapIterators();
+        swapIterators(First, FirstEnd, Second, SecondEnd);
       }
 
       // At this point, we have the following situation:
@@ -2191,42 +2367,6 @@ LLVM_NODISCARD ProgramStateRef reAssume(ProgramStateRef State,
                                      Constraint->getMaxValue(), true);
 }
 
-// Simplify the given symbol with the help of the SValBuilder. In
-// SValBuilder::symplifySval, we traverse the symbol tree and query the
-// constraint values for the sub-trees and if a value is a constant we do the
-// constant folding. Compound symbols might collapse to simpler symbol tree
-// that is still possible to further simplify. Thus, we do the simplification on
-// a new symbol tree until we reach the simplest form, i.e. the fixpoint.
-//
-// Consider the following symbol `(b * b) * b * b` which has this tree:
-//       *
-//      / \
-//     *   b
-//    /  \
-//   /    b
-// (b * b)
-// Now, if the `b * b == 1` new constraint is added then during the first
-// iteration we have the following transformations:
-//       *                  *
-//      / \                / \
-//     *   b     -->      b   b
-//    /  \
-//   /    b
-//  1
-// We need another iteration to reach the final result `1`.
-LLVM_NODISCARD
-static SVal simplifyUntilFixpoint(SValBuilder &SVB, ProgramStateRef State,
-                                  const SymbolRef Sym) {
-  SVal Val = SVB.makeSymbolVal(Sym);
-  SVal SimplifiedVal = SVB.simplifySVal(State, Val);
-  // Do the simplification until we can.
-  while (SimplifiedVal != Val) {
-    Val = SimplifiedVal;
-    SimplifiedVal = SVB.simplifySVal(State, Val);
-  }
-  return SimplifiedVal;
-}
-
 // Iterate over all symbols and try to simplify them. Once a symbol is
 // simplified then we check if we can merge the simplified symbol's equivalence
 // class to this class. This way, we simplify not just the symbols but the
@@ -2238,8 +2378,7 @@ EquivalenceClass::simplify(SValBuilder &SVB, RangeSet::Factory &F,
   SymbolSet ClassMembers = Class.getClassMembers(State);
   for (const SymbolRef &MemberSym : ClassMembers) {
 
-    const SVal SimplifiedMemberVal =
-        simplifyUntilFixpoint(SVB, State, MemberSym);
+    const SVal SimplifiedMemberVal = simplifyToSVal(State, MemberSym);
     const SymbolRef SimplifiedMemberSym = SimplifiedMemberVal.getAsSymbol();
 
     // The symbol is collapsed to a constant, check if the current State is
diff --git a/clang/lib/StaticAnalyzer/Core/SValBuilder.cpp b/clang/lib/StaticAnalyzer/Core/SValBuilder.cpp
index 71bfc86ab8f7..8edcef319088 100644
--- a/clang/lib/StaticAnalyzer/Core/SValBuilder.cpp
+++ b/clang/lib/StaticAnalyzer/Core/SValBuilder.cpp
@@ -459,13 +459,23 @@ SVal SValBuilder::evalBinOp(ProgramStateRef state, BinaryOperator::Opcode op,
     return evalBinOpLN(state, op, *LV, rhs.castAs<NonLoc>(), type);
   }
 
-  if (Optional<Loc> RV = rhs.getAs<Loc>()) {
-    // Support pointer arithmetic where the addend is on the left
-    // and the pointer on the right.
-    assert(op == BO_Add);
+  if (const Optional<Loc> RV = rhs.getAs<Loc>()) {
+    const auto IsCommutative = [](BinaryOperatorKind Op) {
+      return Op == BO_Mul || Op == BO_Add || Op == BO_And || Op == BO_Xor ||
+             Op == BO_Or;
+    };
+
+    if (IsCommutative(op)) {
+      // Swap operands.
+      return evalBinOpLN(state, op, *RV, lhs.castAs<NonLoc>(), type);
+    }
 
-    // Commute the operands.
-    return evalBinOpLN(state, op, *RV, lhs.castAs<NonLoc>(), type);
+    // If the right operand is a concrete int location then we have nothing
+    // better but to treat it as a simple nonloc.
+    if (auto RV = rhs.getAs<loc::ConcreteInt>()) {
+      const nonloc::ConcreteInt RhsAsLoc = makeIntVal(RV->getValue());
+      return evalBinOpNN(state, op, lhs.castAs<NonLoc>(), RhsAsLoc, type);
+    }
   }
 
   return evalBinOpNN(state, op, lhs.castAs<NonLoc>(), rhs.castAs<NonLoc>(),
diff --git a/clang/lib/StaticAnalyzer/Core/SimpleSValBuilder.cpp b/clang/lib/StaticAnalyzer/Core/SimpleSValBuilder.cpp
index 4ca35dd06ae5..dad8a7b3caae 100644
--- a/clang/lib/StaticAnalyzer/Core/SimpleSValBuilder.cpp
+++ b/clang/lib/StaticAnalyzer/Core/SimpleSValBuilder.cpp
@@ -21,6 +21,35 @@ using namespace ento;
 
 namespace {
 class SimpleSValBuilder : public SValBuilder {
+
+  // With one `simplifySValOnce` call, a compound symbols might collapse to
+  // simpler symbol tree that is still possible to further simplify. Thus, we
+  // do the simplification on a new symbol tree until we reach the simplest
+  // form, i.e. the fixpoint.
+  // Consider the following symbol `(b * b) * b * b` which has this tree:
+  //       *
+  //      / \
+  //     *   b
+  //    /  \
+  //   /    b
+  // (b * b)
+  // Now, if the `b * b == 1` new constraint is added then during the first
+  // iteration we have the following transformations:
+  //       *                  *
+  //      / \                / \
+  //     *   b     -->      b   b
+  //    /  \
+  //   /    b
+  //  1
+  // We need another iteration to reach the final result `1`.
+  SVal simplifyUntilFixpoint(ProgramStateRef State, SVal Val);
+
+  // Recursively descends into symbolic expressions and replaces symbols
+  // with their known values (in the sense of the getKnownValue() method).
+  // We traverse the symbol tree and query the constraint values for the
+  // sub-trees and if a value is a constant we do the constant folding.
+  SVal simplifySValOnce(ProgramStateRef State, SVal V);
+
 public:
   SimpleSValBuilder(llvm::BumpPtrAllocator &alloc, ASTContext &context,
                     ProgramStateManager &stateMgr)
@@ -40,8 +69,6 @@ public:
   ///  (integer) value, that value is returned. Otherwise, returns NULL.
   const llvm::APSInt *getKnownValue(ProgramStateRef state, SVal V) override;
 
-  /// Recursively descends into symbolic expressions and replaces symbols
-  /// with their known values (in the sense of the getKnownValue() method).
   SVal simplifySVal(ProgramStateRef State, SVal V) override;
 
   SVal MakeSymIntVal(const SymExpr *LHS, BinaryOperator::Opcode op,
@@ -1105,7 +1132,20 @@ const llvm::APSInt *SimpleSValBuilder::getKnownValue(ProgramStateRef state,
   return nullptr;
 }
 
+SVal SimpleSValBuilder::simplifyUntilFixpoint(ProgramStateRef State, SVal Val) {
+  SVal SimplifiedVal = simplifySValOnce(State, Val);
+  while (SimplifiedVal != Val) {
+    Val = SimplifiedVal;
+    SimplifiedVal = simplifySValOnce(State, Val);
+  }
+  return SimplifiedVal;
+}
+
 SVal SimpleSValBuilder::simplifySVal(ProgramStateRef State, SVal V) {
+  return simplifyUntilFixpoint(State, V);
+}
+
+SVal SimpleSValBuilder::simplifySValOnce(ProgramStateRef State, SVal V) {
   // For now, this function tries to constant-fold symbols inside a
   // nonloc::SymbolVal, and does nothing else. More simplifications should
   // be possible, such as constant-folding an index in an ElementRegion.
diff --git a/clang/lib/StaticAnalyzer/Core/Store.cpp b/clang/lib/StaticAnalyzer/Core/Store.cpp
index 05feb1325c93..2bcdb0faf5da 100644
--- a/clang/lib/StaticAnalyzer/Core/Store.cpp
+++ b/clang/lib/StaticAnalyzer/Core/Store.cpp
@@ -314,10 +314,7 @@ static const CXXRecordDecl *getCXXRecordType(const MemRegion *MR) {
   return nullptr;
 }
 
-SVal StoreManager::attemptDownCast(SVal Base, QualType TargetType,
-                                   bool &Failed) {
-  Failed = false;
-
+Optional<SVal> StoreManager::evalBaseToDerived(SVal Base, QualType TargetType) {
   const MemRegion *MR = Base.getAsRegion();
   if (!MR)
     return UnknownVal();
@@ -392,7 +389,9 @@ SVal StoreManager::attemptDownCast(SVal Base, QualType TargetType,
   }
 
   // We failed if the region we ended up with has perfect type info.
-  Failed = isa<TypedValueRegion>(MR);
+  if (isa<TypedValueRegion>(MR))
+    return None;
+
   return UnknownVal();
 }
 
diff --git a/clang/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp b/clang/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp
index f692c68045ee..f6ddcb763f9d 100644
--- a/clang/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp
+++ b/clang/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp
@@ -35,6 +35,7 @@
 #include "clang/StaticAnalyzer/Core/PathSensitive/AnalysisManager.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
 #include "llvm/ADT/PostOrderIterator.h"
+#include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
@@ -493,13 +494,11 @@ void AnalysisConsumer::HandleDeclsCallGraph(const unsigned LocalTUDeclsSize) {
   }
 }
 
-static bool isBisonFile(ASTContext &C) {
+static bool fileContainsString(StringRef Substring, ASTContext &C) {
   const SourceManager &SM = C.getSourceManager();
   FileID FID = SM.getMainFileID();
   StringRef Buffer = SM.getBufferOrFake(FID).getBuffer();
-  if (Buffer.startswith("/* A Bison parser, made by"))
-    return true;
-  return false;
+  return Buffer.contains(Substring);
 }
 
 void AnalysisConsumer::runAnalysisOnTranslationUnit(ASTContext &C) {
@@ -546,38 +545,48 @@ void AnalysisConsumer::reportAnalyzerProgress(StringRef S) {
 }
 
 void AnalysisConsumer::HandleTranslationUnit(ASTContext &C) {
-
   // Don't run the actions if an error has occurred with parsing the file.
   DiagnosticsEngine &Diags = PP.getDiagnostics();
   if (Diags.hasErrorOccurred() || Diags.hasFatalErrorOccurred())
     return;
 
-  if (isBisonFile(C)) {
+  // Explicitly destroy the PathDiagnosticConsumer.  This will flush its output.
+  // FIXME: This should be replaced with something that doesn't rely on
+  // side-effects in PathDiagnosticConsumer's destructor. This is required when
+  // used with option -disable-free.
+  const auto DiagFlusherScopeExit =
+      llvm::make_scope_exit([this] { Mgr.reset(); });
+
+  if (Opts->ShouldIgnoreBisonGeneratedFiles &&
+      fileContainsString("/* A Bison parser, made by", C)) {
     reportAnalyzerProgress("Skipping bison-generated file\n");
-  } else if (Opts->DisableAllCheckers) {
+    return;
+  }
 
-    // Don't analyze if the user explicitly asked for no checks to be performed
-    // on this file.
+  if (Opts->ShouldIgnoreFlexGeneratedFiles &&
+      fileContainsString("/* A lexical scanner generated by flex", C)) {
+    reportAnalyzerProgress("Skipping flex-generated file\n");
+    return;
+  }
+
+  // Don't analyze if the user explicitly asked for no checks to be performed
+  // on this file.
+  if (Opts->DisableAllCheckers) {
     reportAnalyzerProgress("All checks are disabled using a supplied option\n");
-  } else {
-    // Otherwise, just run the analysis.
-    runAnalysisOnTranslationUnit(C);
+    return;
   }
 
+  // Otherwise, just run the analysis.
+  runAnalysisOnTranslationUnit(C);
+
   // Count how many basic blocks we have not covered.
   NumBlocksInAnalyzedFunctions = FunctionSummaries.getTotalNumBasicBlocks();
   NumVisitedBlocksInAnalyzedFunctions =
       FunctionSummaries.getTotalNumVisitedBasicBlocks();
   if (NumBlocksInAnalyzedFunctions > 0)
     PercentReachableBlocks =
-      (FunctionSummaries.getTotalNumVisitedBasicBlocks() * 100) /
+        (FunctionSummaries.getTotalNumVisitedBasicBlocks() * 100) /
         NumBlocksInAnalyzedFunctions;
-
-  // Explicitly destroy the PathDiagnosticConsumer.  This will flush its output.
-  // FIXME: This should be replaced with something that doesn't rely on
-  // side-effects in PathDiagnosticConsumer's destructor. This is required when
-  // used with option -disable-free.
-  Mgr.reset();
 }
 
 AnalysisConsumer::AnalysisMode
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
index f7c711690d7e..acceec690c11 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
@@ -9,66 +9,54 @@
 #include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h"
 #include "clang/Lex/DependencyDirectivesSourceMinimizer.h"
 #include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/SmallVectorMemoryBuffer.h"
 #include "llvm/Support/Threading.h"
 
 using namespace clang;
 using namespace tooling;
 using namespace dependencies;
 
-CachedFileSystemEntry CachedFileSystemEntry::createFileEntry(
-    StringRef Filename, llvm::vfs::FileSystem &FS, bool Minimize) {
+llvm::ErrorOr<llvm::vfs::Status>
+CachedFileSystemEntry::initFile(StringRef Filename, llvm::vfs::FileSystem &FS) {
   // Load the file and its content from the file system.
-  llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> MaybeFile =
-      FS.openFileForRead(Filename);
+  auto MaybeFile = FS.openFileForRead(Filename);
   if (!MaybeFile)
     return MaybeFile.getError();
-  llvm::ErrorOr<llvm::vfs::Status> Stat = (*MaybeFile)->status();
-  if (!Stat)
-    return Stat.getError();
+  auto File = std::move(*MaybeFile);
+
+  auto MaybeStat = File->status();
+  if (!MaybeStat)
+    return MaybeStat.getError();
+  auto Stat = std::move(*MaybeStat);
 
-  llvm::vfs::File &F = **MaybeFile;
-  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> MaybeBuffer =
-      F.getBuffer(Stat->getName());
+  auto MaybeBuffer = File->getBuffer(Stat.getName());
   if (!MaybeBuffer)
     return MaybeBuffer.getError();
+  auto Buffer = std::move(*MaybeBuffer);
+
+  OriginalContents = std::move(Buffer);
+  return Stat;
+}
+
+void CachedFileSystemEntry::minimizeFile() {
+  assert(OriginalContents && "minimizing missing contents");
 
   llvm::SmallString<1024> MinimizedFileContents;
   // Minimize the file down to directives that might affect the dependencies.
-  const auto &Buffer = *MaybeBuffer;
   SmallVector<minimize_source_to_dependency_directives::Token, 64> Tokens;
-  if (!Minimize || minimizeSourceToDependencyDirectives(
-                       Buffer->getBuffer(), MinimizedFileContents, Tokens)) {
-    // Use the original file unless requested otherwise, or
-    // if the minimization failed.
-    // FIXME: Propage the diagnostic if desired by the client.
-    CachedFileSystemEntry Result;
-    Result.MaybeStat = std::move(*Stat);
-    Result.Contents.reserve(Buffer->getBufferSize() + 1);
-    Result.Contents.append(Buffer->getBufferStart(), Buffer->getBufferEnd());
-    // Implicitly null terminate the contents for Clang's lexer.
-    Result.Contents.push_back('\0');
-    Result.Contents.pop_back();
-    return Result;
+  if (minimizeSourceToDependencyDirectives(OriginalContents->getBuffer(),
+                                           MinimizedFileContents, Tokens)) {
+    // FIXME: Propagate the diagnostic if desired by the client.
+    // Use the original file if the minimization failed.
+    MinimizedContentsStorage =
+        llvm::MemoryBuffer::getMemBuffer(*OriginalContents);
+    MinimizedContentsAccess.store(MinimizedContentsStorage.get());
+    return;
   }
 
-  CachedFileSystemEntry Result;
-  size_t Size = MinimizedFileContents.size();
-  Result.MaybeStat = llvm::vfs::Status(Stat->getName(), Stat->getUniqueID(),
-                                       Stat->getLastModificationTime(),
-                                       Stat->getUser(), Stat->getGroup(), Size,
-                                       Stat->getType(), Stat->getPermissions());
   // The contents produced by the minimizer must be null terminated.
   assert(MinimizedFileContents.data()[MinimizedFileContents.size()] == '\0' &&
          "not null terminated contents");
-  // Even though there's an implicit null terminator in the minimized contents,
-  // we want to temporarily make it explicit. This will ensure that the
-  // std::move will preserve it even if it needs to do a copy if the
-  // SmallString still has the small capacity.
-  MinimizedFileContents.push_back('\0');
-  Result.Contents = std::move(MinimizedFileContents);
-  // Now make the null terminator implicit again, so that Clang's lexer can find
-  // it right where the buffer ends.
-  Result.Contents.pop_back();
 
   // Compute the skipped PP ranges that speedup skipping over inactive
   // preprocessor blocks.
@@ -86,20 +74,20 @@ CachedFileSystemEntry CachedFileSystemEntry::createFileEntry(
     }
     Mapping[Range.Offset] = Range.Length;
   }
-  Result.PPSkippedRangeMapping = std::move(Mapping);
-
-  return Result;
-}
-
-CachedFileSystemEntry
-CachedFileSystemEntry::createDirectoryEntry(llvm::vfs::Status &&Stat) {
-  assert(Stat.isDirectory() && "not a directory!");
-  auto Result = CachedFileSystemEntry();
-  Result.MaybeStat = std::move(Stat);
-  return Result;
+  PPSkippedRangeMapping = std::move(Mapping);
+
+  MinimizedContentsStorage = std::make_unique<llvm::SmallVectorMemoryBuffer>(
+      std::move(MinimizedFileContents));
+  // The algorithm in `getOrCreateFileSystemEntry` uses the presence of
+  // minimized contents to decide whether an entry is up-to-date or not.
+  // If it is up-to-date, the skipped range mappings must be already computed.
+  // This is why we need to store the minimized contents **after** storing the
+  // skipped range mappings. Failing to do so would lead to a data race.
+  MinimizedContentsAccess.store(MinimizedContentsStorage.get());
 }
 
-DependencyScanningFilesystemSharedCache::SingleCache::SingleCache() {
+DependencyScanningFilesystemSharedCache::
+    DependencyScanningFilesystemSharedCache() {
   // This heuristic was chosen using a empirical testing on a
   // reasonably high core machine (iMacPro 18 cores / 36 threads). The cache
   // sharding gives a performance edge by reducing the lock contention.
@@ -111,19 +99,13 @@ DependencyScanningFilesystemSharedCache::SingleCache::SingleCache() {
 }
 
 DependencyScanningFilesystemSharedCache::SharedFileSystemEntry &
-DependencyScanningFilesystemSharedCache::SingleCache::get(StringRef Key) {
+DependencyScanningFilesystemSharedCache::get(StringRef Key) {
   CacheShard &Shard = CacheShards[llvm::hash_value(Key) % NumShards];
-  std::unique_lock<std::mutex> LockGuard(Shard.CacheLock);
+  std::lock_guard<std::mutex> LockGuard(Shard.CacheLock);
   auto It = Shard.Cache.try_emplace(Key);
   return It.first->getValue();
 }
 
-DependencyScanningFilesystemSharedCache::SharedFileSystemEntry &
-DependencyScanningFilesystemSharedCache::get(StringRef Key, bool Minimized) {
-  SingleCache &Cache = Minimized ? CacheMinimized : CacheOriginal;
-  return Cache.get(Key);
-}
-
 /// Whitelist file extensions that should be minimized, treating no extension as
 /// a source file that should be minimized.
 ///
@@ -134,15 +116,14 @@ static bool shouldMinimizeBasedOnExtension(StringRef Filename) {
   if (Ext.empty())
     return true; // C++ standard library
   return llvm::StringSwitch<bool>(Ext)
-    .CasesLower(".c", ".cc", ".cpp", ".c++", ".cxx", true)
-    .CasesLower(".h", ".hh", ".hpp", ".h++", ".hxx", true)
-    .CasesLower(".m", ".mm", true)
-    .CasesLower(".i", ".ii", ".mi", ".mmi", true)
-    .CasesLower(".def", ".inc", true)
-    .Default(false);
+      .CasesLower(".c", ".cc", ".cpp", ".c++", ".cxx", true)
+      .CasesLower(".h", ".hh", ".hpp", ".h++", ".hxx", true)
+      .CasesLower(".m", ".mm", true)
+      .CasesLower(".i", ".ii", ".mi", ".mmi", true)
+      .CasesLower(".def", ".inc", true)
+      .Default(false);
 }
 
-
 static bool shouldCacheStatFailures(StringRef Filename) {
   StringRef Ext = llvm::sys::path::extension(Filename);
   if (Ext.empty())
@@ -167,38 +148,33 @@ bool DependencyScanningWorkerFilesystem::shouldMinimize(StringRef RawFilename) {
   return !NotToBeMinimized.contains(Filename);
 }
 
-CachedFileSystemEntry DependencyScanningWorkerFilesystem::createFileSystemEntry(
-    llvm::ErrorOr<llvm::vfs::Status> &&MaybeStatus, StringRef Filename,
-    bool ShouldMinimize) {
-  if (!MaybeStatus)
-    return CachedFileSystemEntry(MaybeStatus.getError());
-
-  if (MaybeStatus->isDirectory())
-    return CachedFileSystemEntry::createDirectoryEntry(std::move(*MaybeStatus));
-
-  return CachedFileSystemEntry::createFileEntry(Filename, getUnderlyingFS(),
-                                                ShouldMinimize);
+void CachedFileSystemEntry::init(llvm::ErrorOr<llvm::vfs::Status> &&MaybeStatus,
+                                 StringRef Filename,
+                                 llvm::vfs::FileSystem &FS) {
+  if (!MaybeStatus || MaybeStatus->isDirectory())
+    MaybeStat = std::move(MaybeStatus);
+  else
+    MaybeStat = initFile(Filename, FS);
 }
 
-llvm::ErrorOr<const CachedFileSystemEntry *>
+llvm::ErrorOr<EntryRef>
 DependencyScanningWorkerFilesystem::getOrCreateFileSystemEntry(
-    const StringRef Filename) {
-  bool ShouldMinimize = shouldMinimize(Filename);
+    StringRef Filename) {
+  bool ShouldBeMinimized = shouldMinimize(Filename);
 
-  if (const auto *Entry = Cache.getCachedEntry(Filename, ShouldMinimize))
-    return Entry;
+  const auto *Entry = LocalCache.getCachedEntry(Filename);
+  if (Entry && !Entry->needsUpdate(ShouldBeMinimized))
+    return EntryRef(ShouldBeMinimized, *Entry);
 
   // FIXME: Handle PCM/PCH files.
   // FIXME: Handle module map files.
 
-  DependencyScanningFilesystemSharedCache::SharedFileSystemEntry
-      &SharedCacheEntry = SharedCache.get(Filename, ShouldMinimize);
-  const CachedFileSystemEntry *Result;
+  auto &SharedCacheEntry = SharedCache.get(Filename);
   {
-    std::unique_lock<std::mutex> LockGuard(SharedCacheEntry.ValueLock);
+    std::lock_guard<std::mutex> LockGuard(SharedCacheEntry.ValueLock);
     CachedFileSystemEntry &CacheEntry = SharedCacheEntry.Value;
 
-    if (!CacheEntry.isValid()) {
+    if (!CacheEntry.isInitialized()) {
       auto MaybeStatus = getUnderlyingFS().status(Filename);
       if (!MaybeStatus && !shouldCacheStatFailures(Filename))
         // HACK: We need to always restat non source files if the stat fails.
@@ -206,27 +182,30 @@ DependencyScanningWorkerFilesystem::getOrCreateFileSystemEntry(
         //   files before building them, and then looks for them again. If we
         //   cache the stat failure, it won't see them the second time.
         return MaybeStatus.getError();
-      CacheEntry = createFileSystemEntry(std::move(MaybeStatus), Filename,
-                                         ShouldMinimize);
+      CacheEntry.init(std::move(MaybeStatus), Filename, getUnderlyingFS());
     }
 
-    Result = &CacheEntry;
+    // Checking `needsUpdate` verifies the entry represents an opened file.
+    // Only checking `needsMinimization` could lead to minimization of files
+    // that we failed to load (such files don't have `OriginalContents`).
+    if (CacheEntry.needsUpdate(ShouldBeMinimized))
+      CacheEntry.minimizeFile();
   }
 
   // Store the result in the local cache.
-  Cache.setCachedEntry(Filename, ShouldMinimize, Result);
-  return Result;
+  Entry = &SharedCacheEntry.Value;
+  return EntryRef(ShouldBeMinimized, *Entry);
 }
 
 llvm::ErrorOr<llvm::vfs::Status>
 DependencyScanningWorkerFilesystem::status(const Twine &Path) {
   SmallString<256> OwnedFilename;
   StringRef Filename = Path.toStringRef(OwnedFilename);
-  const llvm::ErrorOr<const CachedFileSystemEntry *> Result =
-      getOrCreateFileSystemEntry(Filename);
+
+  llvm::ErrorOr<EntryRef> Result = getOrCreateFileSystemEntry(Filename);
   if (!Result)
     return Result.getError();
-  return (*Result)->getStatus();
+  return Result->getStatus();
 }
 
 namespace {
@@ -240,7 +219,7 @@ public:
       : Buffer(std::move(Buffer)), Stat(std::move(Stat)) {}
 
   static llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
-  create(const CachedFileSystemEntry *Entry,
+  create(EntryRef Entry,
          ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings);
 
   llvm::ErrorOr<llvm::vfs::Status> status() override { return Stat; }
@@ -261,21 +240,22 @@ private:
 } // end anonymous namespace
 
 llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> MinimizedVFSFile::create(
-    const CachedFileSystemEntry *Entry,
-    ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings) {
-  if (Entry->isDirectory())
-    return llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>(
-        std::make_error_code(std::errc::is_a_directory));
-  llvm::ErrorOr<StringRef> Contents = Entry->getContents();
+    EntryRef Entry, ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings) {
+  if (Entry.isDirectory())
+    return std::make_error_code(std::errc::is_a_directory);
+
+  llvm::ErrorOr<StringRef> Contents = Entry.getContents();
   if (!Contents)
     return Contents.getError();
   auto Result = std::make_unique<MinimizedVFSFile>(
-      llvm::MemoryBuffer::getMemBuffer(*Contents, Entry->getName(),
+      llvm::MemoryBuffer::getMemBuffer(*Contents, Entry.getName(),
                                        /*RequiresNullTerminator=*/false),
-      *Entry->getStatus());
-  if (!Entry->getPPSkippedRangeMapping().empty() && PPSkipMappings)
-    (*PPSkipMappings)[Result->Buffer->getBufferStart()] =
-        &Entry->getPPSkippedRangeMapping();
+      *Entry.getStatus());
+
+  const auto *EntrySkipMappings = Entry.getPPSkippedRangeMapping();
+  if (EntrySkipMappings && !EntrySkipMappings->empty() && PPSkipMappings)
+    (*PPSkipMappings)[Result->Buffer->getBufferStart()] = EntrySkipMappings;
+
   return llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>(
       std::unique_ptr<llvm::vfs::File>(std::move(Result)));
 }
@@ -285,8 +265,7 @@ DependencyScanningWorkerFilesystem::openFileForRead(const Twine &Path) {
   SmallString<256> OwnedFilename;
   StringRef Filename = Path.toStringRef(OwnedFilename);
 
-  const llvm::ErrorOr<const CachedFileSystemEntry *> Result =
-      getOrCreateFileSystemEntry(Filename);
+  llvm::ErrorOr<EntryRef> Result = getOrCreateFileSystemEntry(Filename);
   if (!Result)
     return Result.getError();
   return MinimizedVFSFile::create(Result.get(), PPSkipMappings);
diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
index 383a850301a1..086215e7a573 100644
--- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
+++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
@@ -37,9 +37,13 @@ CompilerInvocation ModuleDepCollector::makeInvocationForModuleBuildWithoutPaths(
   CI.getLangOpts()->resetNonModularOptions();
   CI.getPreprocessorOpts().resetNonModularOptions();
 
-  // Remove options incompatible with explicit module build.
+  // Remove options incompatible with explicit module build or are likely to
+  // differ between identical modules discovered from different translation
+  // units.
   CI.getFrontendOpts().Inputs.clear();
   CI.getFrontendOpts().OutputFile.clear();
+  CI.getCodeGenOpts().MainFileName.clear();
+  CI.getCodeGenOpts().DwarfDebugFlags.clear();
 
   CI.getFrontendOpts().ProgramAction = frontend::GenerateModule;
   CI.getLangOpts()->ModuleName = Deps.ID.ModuleName;
@@ -233,7 +237,13 @@ ModuleID ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
                                    .getHeaderSearchInfo()
                                    .getModuleMap()
                                    .getModuleMapFileForUniquing(M);
-  MD.ClangModuleMapFile = std::string(ModuleMap ? ModuleMap->getName() : "");
+
+  if (ModuleMap) {
+    StringRef Path = ModuleMap->tryGetRealPathName();
+    if (Path.empty())
+      Path = ModuleMap->getName();
+    MD.ClangModuleMapFile = std::string(Path);
+  }
 
   serialization::ModuleFile *MF =
       MDC.ScanInstance.getASTReader()->getModuleManager().lookup(
diff --git a/clang/lib/Tooling/Syntax/Tokens.cpp b/clang/lib/Tooling/Syntax/Tokens.cpp
index 8a31e776d030..e2014f965c90 100644
--- a/clang/lib/Tooling/Syntax/Tokens.cpp
+++ b/clang/lib/Tooling/Syntax/Tokens.cpp
@@ -927,5 +927,5 @@ std::string TokenBuffer::dumpForTests() const {
           M.EndExpanded);
     }
   }
-  return OS.str();
+  return Dump;
 }
diff --git a/clang/lib/Tooling/Syntax/Tree.cpp b/clang/lib/Tooling/Syntax/Tree.cpp
index 1e3a90f3a316..c813865e95cd 100644
--- a/clang/lib/Tooling/Syntax/Tree.cpp
+++ b/clang/lib/Tooling/Syntax/Tree.cpp
@@ -263,7 +263,7 @@ std::string syntax::Node::dumpTokens(const SourceManager &SM) const {
       OS << " ";
     }
   });
-  return OS.str();
+  return Storage;
 }
 
 void syntax::Node::assertInvariants() const {
diff --git a/clang/tools/clang-format/ClangFormat.cpp b/clang/tools/clang-format/ClangFormat.cpp
index 56dc628869a4..893c17d91708 100644
--- a/clang/tools/clang-format/ClangFormat.cpp
+++ b/clang/tools/clang-format/ClangFormat.cpp
@@ -460,7 +460,7 @@ static bool format(StringRef FileName) {
 
   // To format JSON insert a variable to trick the code into thinking its
   // JavaScript.
-  if (FormatStyle->isJson()) {
+  if (FormatStyle->isJson() && !FormatStyle->DisableFormat) {
     auto Err = Replaces.add(tooling::Replacement(
         tooling::Replacement(AssumedFileName, 0, 0, "x = ")));
     if (Err) {
diff --git a/clang/tools/driver/driver.cpp b/clang/tools/driver/driver.cpp
index c9129ee9e502..a7bfb07e002b 100644
--- a/clang/tools/driver/driver.cpp
+++ b/clang/tools/driver/driver.cpp
@@ -120,7 +120,7 @@ static void ApplyOneQAOverride(raw_ostream &OS,
     OS << "### Adding argument " << Str << " at end\n";
     Args.push_back(Str);
   } else if (Edit[0] == 's' && Edit[1] == '/' && Edit.endswith("/") &&
-             Edit.slice(2, Edit.size()-1).find('/') != StringRef::npos) {
+             Edit.slice(2, Edit.size() - 1).contains('/')) {
     StringRef MatchPattern = Edit.substr(2).split('/').first;
     StringRef ReplPattern = Edit.substr(2).split('/').second;
     ReplPattern = ReplPattern.slice(0, ReplPattern.size()-1);
diff --git a/clang/utils/TableGen/MveEmitter.cpp b/clang/utils/TableGen/MveEmitter.cpp
index f5b6f4f01688..1a2532fbf53f 100644
--- a/clang/utils/TableGen/MveEmitter.cpp
+++ b/clang/utils/TableGen/MveEmitter.cpp
@@ -349,13 +349,8 @@ public:
   bool requiresFloat() const override { return false; };
   bool requiresMVE() const override { return true; }
   std::string llvmName() const override {
-    // Use <4 x i1> instead of <2 x i1> for two-lane vector types. See
-    // the comment in llvm/lib/Target/ARM/ARMInstrMVE.td for further
-    // explanation.
-    unsigned ModifiedLanes = (Lanes == 2 ? 4 : Lanes);
-
-    return "llvm::FixedVectorType::get(Builder.getInt1Ty(), " +
-           utostr(ModifiedLanes) + ")";
+    return "llvm::FixedVectorType::get(Builder.getInt1Ty(), " + utostr(Lanes) +
+           ")";
   }
 
   static bool classof(const Type *T) {
diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc
index 008b8dde5820..44719126b596 100644
--- a/compiler-rt/include/profile/InstrProfData.inc
+++ b/compiler-rt/include/profile/InstrProfData.inc
@@ -653,15 +653,17 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
 
 /* Profile version is always of type uint64_t. Reserve the upper 8 bits in the
  * version for other variants of profile. We set the lowest bit of the upper 8
- * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton
+ * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentation
  * generated profile, and 0 if this is a Clang FE generated profile.
  * 1 in bit 57 indicates there are context-sensitive records in the profile.
+ * The 59th bit indicates whether to use debug info to correlate profiles.
  */
 #define VARIANT_MASKS_ALL 0xff00000000000000ULL
 #define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL)
 #define VARIANT_MASK_IR_PROF (0x1ULL << 56)
 #define VARIANT_MASK_CSIR_PROF (0x1ULL << 57)
 #define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58)
+#define VARIANT_MASK_DBG_CORRELATE (0x1ULL << 59)
 #define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version
 #define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime
 #define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias
diff --git a/compiler-rt/include/sanitizer/dfsan_interface.h b/compiler-rt/include/sanitizer/dfsan_interface.h
index d6209a3ea2b2..bc0652c99a14 100644
--- a/compiler-rt/include/sanitizer/dfsan_interface.h
+++ b/compiler-rt/include/sanitizer/dfsan_interface.h
@@ -54,6 +54,10 @@ dfsan_origin dfsan_get_origin(long data);
 /// Retrieves the label associated with the data at the given address.
 dfsan_label dfsan_read_label(const void *addr, size_t size);
 
+/// Return the origin associated with the first taint byte in the size bytes
+/// from the address addr.
+dfsan_origin dfsan_read_origin_of_first_taint(const void *addr, size_t size);
+
 /// Returns whether the given label label contains the label elem.
 int dfsan_has_label(dfsan_label label, dfsan_label elem);
 
@@ -87,6 +91,9 @@ void dfsan_weak_hook_strncmp(void *caller_pc, const char *s1, const char *s2,
 /// prints description at the beginning of the trace. If origin tracking is not
 /// on, or the address is not labeled, it prints nothing.
 void dfsan_print_origin_trace(const void *addr, const char *description);
+/// As above, but use an origin id from dfsan_get_origin() instead of address.
+/// Does not include header line with taint label and address information.
+void dfsan_print_origin_id_trace(dfsan_origin origin);
 
 /// Prints the origin trace of the label at the address \p addr to a
 /// pre-allocated output buffer. If origin tracking is not on, or the address is
@@ -124,6 +131,10 @@ void dfsan_print_origin_trace(const void *addr, const char *description);
 /// return value is not less than \p out_buf_size.
 size_t dfsan_sprint_origin_trace(const void *addr, const char *description,
                                  char *out_buf, size_t out_buf_size);
+/// As above, but use an origin id from dfsan_get_origin() instead of address.
+/// Does not include header line with taint label and address information.
+size_t dfsan_sprint_origin_id_trace(dfsan_origin origin, char *out_buf,
+                                    size_t out_buf_size);
 
 /// Prints the stack trace leading to this call to a pre-allocated output
 /// buffer.
diff --git a/compiler-rt/lib/asan/asan_activation.cpp b/compiler-rt/lib/asan/asan_activation.cpp
index 795df95a5414..1757838600ca 100644
--- a/compiler-rt/lib/asan/asan_activation.cpp
+++ b/compiler-rt/lib/asan/asan_activation.cpp
@@ -112,7 +112,7 @@ void AsanDeactivate() {
   disabled.quarantine_size_mb = 0;
   disabled.thread_local_quarantine_size_kb = 0;
   // Redzone must be at least Max(16, granularity) bytes long.
-  disabled.min_redzone = Max(16, (int)SHADOW_GRANULARITY);
+  disabled.min_redzone = Max(16, (int)ASAN_SHADOW_GRANULARITY);
   disabled.max_redzone = disabled.min_redzone;
   disabled.alloc_dealloc_mismatch = false;
   disabled.may_return_null = true;
diff --git a/compiler-rt/lib/asan/asan_allocator.cpp b/compiler-rt/lib/asan/asan_allocator.cpp
index 3fa36742060b..1ff7091460ad 100644
--- a/compiler-rt/lib/asan/asan_allocator.cpp
+++ b/compiler-rt/lib/asan/asan_allocator.cpp
@@ -210,8 +210,7 @@ struct QuarantineCallback {
       CHECK_EQ(old_chunk_state, CHUNK_QUARANTINE);
     }
 
-    PoisonShadow(m->Beg(),
-                 RoundUpTo(m->UsedSize(), SHADOW_GRANULARITY),
+    PoisonShadow(m->Beg(), RoundUpTo(m->UsedSize(), ASAN_SHADOW_GRANULARITY),
                  kAsanHeapLeftRedzoneMagic);
 
     // Statistics.
@@ -305,7 +304,6 @@ struct Allocator {
   QuarantineCache fallback_quarantine_cache;
 
   uptr max_user_defined_malloc_size;
-  atomic_uint8_t rss_limit_exceeded;
 
   // ------------------- Options --------------------------
   atomic_uint16_t min_redzone;
@@ -345,14 +343,6 @@ struct Allocator {
                                        : kMaxAllowedMallocSize;
   }
 
-  bool RssLimitExceeded() {
-    return atomic_load(&rss_limit_exceeded, memory_order_relaxed);
-  }
-
-  void SetRssLimitExceeded(bool limit_exceeded) {
-    atomic_store(&rss_limit_exceeded, limit_exceeded, memory_order_relaxed);
-  }
-
   void RePoisonChunk(uptr chunk) {
     // This could be a user-facing chunk (with redzones), or some internal
     // housekeeping chunk, like TransferBatch. Start by assuming the former.
@@ -366,7 +356,7 @@ struct Allocator {
       if (chunk < beg && beg < end && end <= chunk_end) {
         // Looks like a valid AsanChunk in use, poison redzones only.
         PoisonShadow(chunk, beg - chunk, kAsanHeapLeftRedzoneMagic);
-        uptr end_aligned_down = RoundDownTo(end, SHADOW_GRANULARITY);
+        uptr end_aligned_down = RoundDownTo(end, ASAN_SHADOW_GRANULARITY);
         FastPoisonShadowPartialRightRedzone(
             end_aligned_down, end - end_aligned_down,
             chunk_end - end_aligned_down, kAsanHeapLeftRedzoneMagic);
@@ -484,14 +474,14 @@ struct Allocator {
                  AllocType alloc_type, bool can_fill) {
     if (UNLIKELY(!asan_inited))
       AsanInitFromRtl();
-    if (RssLimitExceeded()) {
+    if (UNLIKELY(IsRssLimitExceeded())) {
       if (AllocatorMayReturnNull())
         return nullptr;
       ReportRssLimitExceeded(stack);
     }
     Flags &fl = *flags();
     CHECK(stack);
-    const uptr min_alignment = SHADOW_GRANULARITY;
+    const uptr min_alignment = ASAN_SHADOW_GRANULARITY;
     const uptr user_requested_alignment_log =
         ComputeUserRequestedAlignmentLog(alignment);
     if (alignment < min_alignment)
@@ -572,7 +562,7 @@ struct Allocator {
     m->SetAllocContext(t ? t->tid() : kMainTid, StackDepotPut(*stack));
 
     uptr size_rounded_down_to_granularity =
-        RoundDownTo(size, SHADOW_GRANULARITY);
+        RoundDownTo(size, ASAN_SHADOW_GRANULARITY);
     // Unpoison the bulk of the memory region.
     if (size_rounded_down_to_granularity)
       PoisonShadow(user_beg, size_rounded_down_to_granularity, 0);
@@ -580,7 +570,7 @@ struct Allocator {
     if (size != size_rounded_down_to_granularity && CanPoisonMemory()) {
       u8 *shadow =
           (u8 *)MemToShadow(user_beg + size_rounded_down_to_granularity);
-      *shadow = fl.poison_partial ? (size & (SHADOW_GRANULARITY - 1)) : 0;
+      *shadow = fl.poison_partial ? (size & (ASAN_SHADOW_GRANULARITY - 1)) : 0;
     }
 
     AsanStats &thread_stats = GetCurrentThreadStats();
@@ -650,8 +640,7 @@ struct Allocator {
     }
 
     // Poison the region.
-    PoisonShadow(m->Beg(),
-                 RoundUpTo(m->UsedSize(), SHADOW_GRANULARITY),
+    PoisonShadow(m->Beg(), RoundUpTo(m->UsedSize(), ASAN_SHADOW_GRANULARITY),
                  kAsanHeapFreeMagic);
 
     AsanStats &thread_stats = GetCurrentThreadStats();
@@ -1071,10 +1060,6 @@ void asan_mz_force_unlock() NO_THREAD_SAFETY_ANALYSIS {
   instance.ForceUnlock();
 }
 
-void AsanSoftRssLimitExceededCallback(bool limit_exceeded) {
-  instance.SetRssLimitExceeded(limit_exceeded);
-}
-
 }  // namespace __asan
 
 // --- Implementation of LSan-specific functions --- {{{1
diff --git a/compiler-rt/lib/asan/asan_debugging.cpp b/compiler-rt/lib/asan/asan_debugging.cpp
index 0b4bf52f2490..f078f1041a87 100644
--- a/compiler-rt/lib/asan/asan_debugging.cpp
+++ b/compiler-rt/lib/asan/asan_debugging.cpp
@@ -141,7 +141,7 @@ uptr __asan_get_free_stack(uptr addr, uptr *trace, uptr size, u32 *thread_id) {
 SANITIZER_INTERFACE_ATTRIBUTE
 void __asan_get_shadow_mapping(uptr *shadow_scale, uptr *shadow_offset) {
   if (shadow_scale)
-    *shadow_scale = SHADOW_SCALE;
+    *shadow_scale = ASAN_SHADOW_SCALE;
   if (shadow_offset)
-    *shadow_offset = SHADOW_OFFSET;
+    *shadow_offset = ASAN_SHADOW_OFFSET;
 }
diff --git a/compiler-rt/lib/asan/asan_errors.cpp b/compiler-rt/lib/asan/asan_errors.cpp
index 7cd9fe911afa..a22bf130d823 100644
--- a/compiler-rt/lib/asan/asan_errors.cpp
+++ b/compiler-rt/lib/asan/asan_errors.cpp
@@ -329,7 +329,7 @@ void ErrorBadParamsToAnnotateContiguousContainer::Print() {
       "      old_mid : %p\n"
       "      new_mid : %p\n",
       (void *)beg, (void *)end, (void *)old_mid, (void *)new_mid);
-  uptr granularity = SHADOW_GRANULARITY;
+  uptr granularity = ASAN_SHADOW_GRANULARITY;
   if (!IsAligned(beg, granularity))
     Report("ERROR: beg is not aligned by %zu\n", granularity);
   stack->Print();
@@ -410,7 +410,8 @@ ErrorGeneric::ErrorGeneric(u32 tid, uptr pc_, uptr bp_, uptr sp_, uptr addr,
     if (AddrIsInMem(addr)) {
       u8 *shadow_addr = (u8 *)MemToShadow(addr);
       // If we are accessing 16 bytes, look at the second shadow byte.
-      if (*shadow_addr == 0 && access_size > SHADOW_GRANULARITY) shadow_addr++;
+      if (*shadow_addr == 0 && access_size > ASAN_SHADOW_GRANULARITY)
+        shadow_addr++;
       // If we are in the partial right redzone, look at the next shadow byte.
       if (*shadow_addr > 0 && *shadow_addr < 128) shadow_addr++;
       bool far_from_bounds = false;
@@ -501,10 +502,11 @@ static void PrintLegend(InternalScopedString *str) {
   str->append(
       "Shadow byte legend (one shadow byte represents %d "
       "application bytes):\n",
-      (int)SHADOW_GRANULARITY);
+      (int)ASAN_SHADOW_GRANULARITY);
   PrintShadowByte(str, "  Addressable:           ", 0);
   str->append("  Partially addressable: ");
-  for (u8 i = 1; i < SHADOW_GRANULARITY; i++) PrintShadowByte(str, "", i, " ");
+  for (u8 i = 1; i < ASAN_SHADOW_GRANULARITY; i++)
+    PrintShadowByte(str, "", i, " ");
   str->append("\n");
   PrintShadowByte(str, "  Heap left redzone:       ",
                   kAsanHeapLeftRedzoneMagic);
diff --git a/compiler-rt/lib/asan/asan_fake_stack.cpp b/compiler-rt/lib/asan/asan_fake_stack.cpp
index 07681c10de91..08d81c72597c 100644
--- a/compiler-rt/lib/asan/asan_fake_stack.cpp
+++ b/compiler-rt/lib/asan/asan_fake_stack.cpp
@@ -28,8 +28,8 @@ static const u64 kAllocaRedzoneMask = 31UL;
 // For small size classes inline PoisonShadow for better performance.
 ALWAYS_INLINE void SetShadow(uptr ptr, uptr size, uptr class_id, u64 magic) {
   u64 *shadow = reinterpret_cast<u64*>(MemToShadow(ptr));
-  if (SHADOW_SCALE == 3 && class_id <= 6) {
-    // This code expects SHADOW_SCALE=3.
+  if (ASAN_SHADOW_SCALE == 3 && class_id <= 6) {
+    // This code expects ASAN_SHADOW_SCALE=3.
     for (uptr i = 0; i < (((uptr)1) << class_id); i++) {
       shadow[i] = magic;
       // Make sure this does not become memset.
@@ -294,10 +294,10 @@ void __asan_alloca_poison(uptr addr, uptr size) {
   uptr LeftRedzoneAddr = addr - kAllocaRedzoneSize;
   uptr PartialRzAddr = addr + size;
   uptr RightRzAddr = (PartialRzAddr + kAllocaRedzoneMask) & ~kAllocaRedzoneMask;
-  uptr PartialRzAligned = PartialRzAddr & ~(SHADOW_GRANULARITY - 1);
+  uptr PartialRzAligned = PartialRzAddr & ~(ASAN_SHADOW_GRANULARITY - 1);
   FastPoisonShadow(LeftRedzoneAddr, kAllocaRedzoneSize, kAsanAllocaLeftMagic);
   FastPoisonShadowPartialRightRedzone(
-      PartialRzAligned, PartialRzAddr % SHADOW_GRANULARITY,
+      PartialRzAligned, PartialRzAddr % ASAN_SHADOW_GRANULARITY,
       RightRzAddr - PartialRzAligned, kAsanAllocaRightMagic);
   FastPoisonShadow(RightRzAddr, kAllocaRedzoneSize, kAsanAllocaRightMagic);
 }
@@ -305,7 +305,8 @@ void __asan_alloca_poison(uptr addr, uptr size) {
 SANITIZER_INTERFACE_ATTRIBUTE
 void __asan_allocas_unpoison(uptr top, uptr bottom) {
   if ((!top) || (top > bottom)) return;
-  REAL(memset)(reinterpret_cast<void*>(MemToShadow(top)), 0,
-               (bottom - top) / SHADOW_GRANULARITY);
+  REAL(memset)
+  (reinterpret_cast<void *>(MemToShadow(top)), 0,
+   (bottom - top) / ASAN_SHADOW_GRANULARITY);
 }
 } // extern "C"
diff --git a/compiler-rt/lib/asan/asan_flags.cpp b/compiler-rt/lib/asan/asan_flags.cpp
index c64e46470287..9ea899f84b4b 100644
--- a/compiler-rt/lib/asan/asan_flags.cpp
+++ b/compiler-rt/lib/asan/asan_flags.cpp
@@ -140,9 +140,9 @@ void InitializeFlags() {
            SanitizerToolName);
     Die();
   }
-  // Ensure that redzone is at least SHADOW_GRANULARITY.
-  if (f->redzone < (int)SHADOW_GRANULARITY)
-    f->redzone = SHADOW_GRANULARITY;
+  // Ensure that redzone is at least ASAN_SHADOW_GRANULARITY.
+  if (f->redzone < (int)ASAN_SHADOW_GRANULARITY)
+    f->redzone = ASAN_SHADOW_GRANULARITY;
   // Make "strict_init_order" imply "check_initialization_order".
   // TODO(samsonov): Use a single runtime flag for an init-order checker.
   if (f->strict_init_order) {
diff --git a/compiler-rt/lib/asan/asan_globals.cpp b/compiler-rt/lib/asan/asan_globals.cpp
index 5f56fe6f457d..ecc2600f039a 100644
--- a/compiler-rt/lib/asan/asan_globals.cpp
+++ b/compiler-rt/lib/asan/asan_globals.cpp
@@ -61,14 +61,13 @@ ALWAYS_INLINE void PoisonShadowForGlobal(const Global *g, u8 value) {
 }
 
 ALWAYS_INLINE void PoisonRedZones(const Global &g) {
-  uptr aligned_size = RoundUpTo(g.size, SHADOW_GRANULARITY);
+  uptr aligned_size = RoundUpTo(g.size, ASAN_SHADOW_GRANULARITY);
   FastPoisonShadow(g.beg + aligned_size, g.size_with_redzone - aligned_size,
                    kAsanGlobalRedzoneMagic);
   if (g.size != aligned_size) {
     FastPoisonShadowPartialRightRedzone(
-        g.beg + RoundDownTo(g.size, SHADOW_GRANULARITY),
-        g.size % SHADOW_GRANULARITY,
-        SHADOW_GRANULARITY,
+        g.beg + RoundDownTo(g.size, ASAN_SHADOW_GRANULARITY),
+        g.size % ASAN_SHADOW_GRANULARITY, ASAN_SHADOW_GRANULARITY,
         kAsanGlobalRedzoneMagic);
   }
 }
diff --git a/compiler-rt/lib/asan/asan_interface.inc b/compiler-rt/lib/asan/asan_interface.inc
index ea28fc8ae87c..89ef552b7117 100644
--- a/compiler-rt/lib/asan/asan_interface.inc
+++ b/compiler-rt/lib/asan/asan_interface.inc
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 // Asan interface list.
 //===----------------------------------------------------------------------===//
+
 INTERFACE_FUNCTION(__asan_addr_is_in_fake_stack)
 INTERFACE_FUNCTION(__asan_address_is_poisoned)
 INTERFACE_FUNCTION(__asan_after_dynamic_init)
diff --git a/compiler-rt/lib/asan/asan_linux.cpp b/compiler-rt/lib/asan/asan_linux.cpp
index ad3693d5e6a2..1d92c530bd11 100644
--- a/compiler-rt/lib/asan/asan_linux.cpp
+++ b/compiler-rt/lib/asan/asan_linux.cpp
@@ -107,7 +107,7 @@ uptr FindDynamicShadowStart() {
     return FindPremappedShadowStart(shadow_size_bytes);
 #endif
 
-  return MapDynamicShadow(shadow_size_bytes, SHADOW_SCALE,
+  return MapDynamicShadow(shadow_size_bytes, ASAN_SHADOW_SCALE,
                           /*min_shadow_base_alignment*/ 0, kHighMemEnd);
 }
 
diff --git a/compiler-rt/lib/asan/asan_mac.cpp b/compiler-rt/lib/asan/asan_mac.cpp
index c6950547f089..9161f728d44c 100644
--- a/compiler-rt/lib/asan/asan_mac.cpp
+++ b/compiler-rt/lib/asan/asan_mac.cpp
@@ -55,7 +55,7 @@ void *AsanDoesNotSupportStaticLinkage() {
 }
 
 uptr FindDynamicShadowStart() {
-  return MapDynamicShadow(MemToShadowSize(kHighMemEnd), SHADOW_SCALE,
+  return MapDynamicShadow(MemToShadowSize(kHighMemEnd), ASAN_SHADOW_SCALE,
                           /*min_shadow_base_alignment*/ 0, kHighMemEnd);
 }
 
diff --git a/compiler-rt/lib/asan/asan_mapping.h b/compiler-rt/lib/asan/asan_mapping.h
index e5a7f2007aea..6ca6ee00e5c9 100644
--- a/compiler-rt/lib/asan/asan_mapping.h
+++ b/compiler-rt/lib/asan/asan_mapping.h
@@ -13,8 +13,6 @@
 #ifndef ASAN_MAPPING_H
 #define ASAN_MAPPING_H
 
-#include "asan_internal.h"
-
 // The full explanation of the memory mapping could be found here:
 // https://github.com/google/sanitizers/wiki/AddressSanitizerAlgorithm
 //
@@ -151,149 +149,145 @@
 // || `[0x30000000, 0x35ffffff]` || LowShadow  ||
 // || `[0x00000000, 0x2fffffff]` || LowMem     ||
 
-#if defined(ASAN_SHADOW_SCALE)
-static const u64 kDefaultShadowScale = ASAN_SHADOW_SCALE;
-#else
-static const u64 kDefaultShadowScale = 3;
-#endif
-static const u64 kDefaultShadowSentinel = ~(uptr)0;
-static const u64 kDefaultShadowOffset32 = 1ULL << 29;  // 0x20000000
-static const u64 kDefaultShadowOffset64 = 1ULL << 44;
-static const u64 kDefaultShort64bitShadowOffset =
-    0x7FFFFFFF & (~0xFFFULL << kDefaultShadowScale);  // < 2G.
-static const u64 kAArch64_ShadowOffset64 = 1ULL << 36;
-static const u64 kRiscv64_ShadowOffset64 = 0xd55550000;
-static const u64 kMIPS32_ShadowOffset32 = 0x0aaa0000;
-static const u64 kMIPS64_ShadowOffset64 = 1ULL << 37;
-static const u64 kPPC64_ShadowOffset64 = 1ULL << 44;
-static const u64 kSystemZ_ShadowOffset64 = 1ULL << 52;
-static const u64 kSPARC64_ShadowOffset64 = 1ULL << 43;  // 0x80000000000
-static const u64 kFreeBSD_ShadowOffset32 = 1ULL << 30;  // 0x40000000
-static const u64 kFreeBSD_ShadowOffset64 = 1ULL << 46;  // 0x400000000000
-static const u64 kNetBSD_ShadowOffset32 = 1ULL << 30;  // 0x40000000
-static const u64 kNetBSD_ShadowOffset64 = 1ULL << 46;  // 0x400000000000
-static const u64 kWindowsShadowOffset32 = 3ULL << 28;  // 0x30000000
-
-#define SHADOW_SCALE kDefaultShadowScale
+#define ASAN_SHADOW_SCALE 3
 
 #if SANITIZER_FUCHSIA
-#  define SHADOW_OFFSET (0)
+#  define ASAN_SHADOW_OFFSET_CONST (0)
 #elif SANITIZER_WORDSIZE == 32
 #  if SANITIZER_ANDROID
-#    define SHADOW_OFFSET __asan_shadow_memory_dynamic_address
+#    define ASAN_SHADOW_OFFSET_DYNAMIC
 #  elif defined(__mips__)
-#    define SHADOW_OFFSET kMIPS32_ShadowOffset32
+#    define ASAN_SHADOW_OFFSET_CONST 0x0aaa0000
 #  elif SANITIZER_FREEBSD
-#    define SHADOW_OFFSET kFreeBSD_ShadowOffset32
+#    define ASAN_SHADOW_OFFSET_CONST 0x40000000
 #  elif SANITIZER_NETBSD
-#    define SHADOW_OFFSET kNetBSD_ShadowOffset32
+#    define ASAN_SHADOW_OFFSET_CONST 0x40000000
 #  elif SANITIZER_WINDOWS
-#    define SHADOW_OFFSET kWindowsShadowOffset32
+#    define ASAN_SHADOW_OFFSET_CONST 0x30000000
 #  elif SANITIZER_IOS
-#    define SHADOW_OFFSET __asan_shadow_memory_dynamic_address
+#    define ASAN_SHADOW_OFFSET_DYNAMIC
 #  else
-#    define SHADOW_OFFSET kDefaultShadowOffset32
+#    define ASAN_SHADOW_OFFSET_CONST 0x20000000
 #  endif
 #else
 #  if SANITIZER_IOS
-#    define SHADOW_OFFSET __asan_shadow_memory_dynamic_address
+#    define ASAN_SHADOW_OFFSET_DYNAMIC
 #  elif SANITIZER_MAC && defined(__aarch64__)
-#    define SHADOW_OFFSET __asan_shadow_memory_dynamic_address
-#elif SANITIZER_RISCV64
-#define SHADOW_OFFSET kRiscv64_ShadowOffset64
+#    define ASAN_SHADOW_OFFSET_DYNAMIC
+#  elif SANITIZER_RISCV64
+#    define ASAN_SHADOW_OFFSET_CONST 0x0000000d55550000
 #  elif defined(__aarch64__)
-#    define SHADOW_OFFSET kAArch64_ShadowOffset64
+#    define ASAN_SHADOW_OFFSET_CONST 0x0000001000000000
 #  elif defined(__powerpc64__)
-#    define SHADOW_OFFSET kPPC64_ShadowOffset64
+#    define ASAN_SHADOW_OFFSET_CONST 0x0000100000000000
 #  elif defined(__s390x__)
-#    define SHADOW_OFFSET kSystemZ_ShadowOffset64
+#    define ASAN_SHADOW_OFFSET_CONST 0x0010000000000000
 #  elif SANITIZER_FREEBSD
-#    define SHADOW_OFFSET kFreeBSD_ShadowOffset64
+#    define ASAN_SHADOW_OFFSET_CONST 0x0000400000000000
 #  elif SANITIZER_NETBSD
-#    define SHADOW_OFFSET kNetBSD_ShadowOffset64
+#    define ASAN_SHADOW_OFFSET_CONST 0x0000400000000000
 #  elif SANITIZER_MAC
-#   define SHADOW_OFFSET kDefaultShadowOffset64
+#    define ASAN_SHADOW_OFFSET_CONST 0x0000100000000000
 #  elif defined(__mips64)
-#   define SHADOW_OFFSET kMIPS64_ShadowOffset64
-#elif defined(__sparc__)
-#define SHADOW_OFFSET kSPARC64_ShadowOffset64
+#    define ASAN_SHADOW_OFFSET_CONST 0x0000002000000000
+#  elif defined(__sparc__)
+#    define ASAN_SHADOW_OFFSET_CONST 0x0000080000000000
 #  elif SANITIZER_WINDOWS64
-#   define SHADOW_OFFSET __asan_shadow_memory_dynamic_address
+#    define ASAN_SHADOW_OFFSET_DYNAMIC
 #  else
-#   define SHADOW_OFFSET kDefaultShort64bitShadowOffset
+#    if ASAN_SHADOW_SCALE != 3
+#      error "Value below is based on shadow scale = 3."
+#      error "Original formula was: 0x7FFFFFFF & (~0xFFFULL << SHADOW_SCALE)."
+#    endif
+#    define ASAN_SHADOW_OFFSET_CONST 0x000000007fff8000
 #  endif
 #endif
 
-#if SANITIZER_ANDROID && defined(__arm__)
-# define ASAN_PREMAP_SHADOW 1
-#else
-# define ASAN_PREMAP_SHADOW 0
-#endif
+#if defined(__cplusplus)
+#  include "asan_internal.h"
+
+static const u64 kDefaultShadowSentinel = ~(uptr)0;
 
-#define SHADOW_GRANULARITY (1ULL << SHADOW_SCALE)
+#  if defined(ASAN_SHADOW_OFFSET_CONST)
+static const u64 kConstShadowOffset = ASAN_SHADOW_OFFSET_CONST;
+#    define ASAN_SHADOW_OFFSET kConstShadowOffset
+#  elif defined(ASAN_SHADOW_OFFSET_DYNAMIC)
+#    define ASAN_SHADOW_OFFSET __asan_shadow_memory_dynamic_address
+#  else
+#    error "ASAN_SHADOW_OFFSET can't be determined."
+#  endif
 
-#define DO_ASAN_MAPPING_PROFILE 0  // Set to 1 to profile the functions below.
+#  if SANITIZER_ANDROID && defined(__arm__)
+#    define ASAN_PREMAP_SHADOW 1
+#  else
+#    define ASAN_PREMAP_SHADOW 0
+#  endif
 
-#if DO_ASAN_MAPPING_PROFILE
-# define PROFILE_ASAN_MAPPING() AsanMappingProfile[__LINE__]++;
-#else
-# define PROFILE_ASAN_MAPPING()
-#endif
+#  define ASAN_SHADOW_GRANULARITY (1ULL << ASAN_SHADOW_SCALE)
+
+#  define DO_ASAN_MAPPING_PROFILE 0  // Set to 1 to profile the functions below.
+
+#  if DO_ASAN_MAPPING_PROFILE
+#    define PROFILE_ASAN_MAPPING() AsanMappingProfile[__LINE__]++;
+#  else
+#    define PROFILE_ASAN_MAPPING()
+#  endif
 
 // If 1, all shadow boundaries are constants.
 // Don't set to 1 other than for testing.
-#define ASAN_FIXED_MAPPING 0
+#  define ASAN_FIXED_MAPPING 0
 
 namespace __asan {
 
 extern uptr AsanMappingProfile[];
 
-#if ASAN_FIXED_MAPPING
+#  if ASAN_FIXED_MAPPING
 // Fixed mapping for 64-bit Linux. Mostly used for performance comparison
 // with non-fixed mapping. As of r175253 (Feb 2013) the performance
 // difference between fixed and non-fixed mapping is below the noise level.
 static uptr kHighMemEnd = 0x7fffffffffffULL;
-static uptr kMidMemBeg =    0x3000000000ULL;
-static uptr kMidMemEnd =    0x4fffffffffULL;
-#else
+static uptr kMidMemBeg = 0x3000000000ULL;
+static uptr kMidMemEnd = 0x4fffffffffULL;
+#  else
 extern uptr kHighMemEnd, kMidMemBeg, kMidMemEnd;  // Initialized in __asan_init.
-#endif
+#  endif
 
 }  // namespace __asan
 
-#if defined(__sparc__) && SANITIZER_WORDSIZE == 64
-#  include "asan_mapping_sparc64.h"
-#else
-#define MEM_TO_SHADOW(mem) (((mem) >> SHADOW_SCALE) + (SHADOW_OFFSET))
+#  if defined(__sparc__) && SANITIZER_WORDSIZE == 64
+#    include "asan_mapping_sparc64.h"
+#  else
+#    define MEM_TO_SHADOW(mem) \
+      (((mem) >> ASAN_SHADOW_SCALE) + (ASAN_SHADOW_OFFSET))
 
-#define kLowMemBeg      0
-#define kLowMemEnd      (SHADOW_OFFSET ? SHADOW_OFFSET - 1 : 0)
+#    define kLowMemBeg 0
+#    define kLowMemEnd (ASAN_SHADOW_OFFSET ? ASAN_SHADOW_OFFSET - 1 : 0)
 
-#define kLowShadowBeg   SHADOW_OFFSET
-#define kLowShadowEnd   MEM_TO_SHADOW(kLowMemEnd)
+#    define kLowShadowBeg ASAN_SHADOW_OFFSET
+#    define kLowShadowEnd MEM_TO_SHADOW(kLowMemEnd)
 
-#define kHighMemBeg     (MEM_TO_SHADOW(kHighMemEnd) + 1)
+#    define kHighMemBeg (MEM_TO_SHADOW(kHighMemEnd) + 1)
 
-#define kHighShadowBeg  MEM_TO_SHADOW(kHighMemBeg)
-#define kHighShadowEnd  MEM_TO_SHADOW(kHighMemEnd)
+#    define kHighShadowBeg MEM_TO_SHADOW(kHighMemBeg)
+#    define kHighShadowEnd MEM_TO_SHADOW(kHighMemEnd)
 
-# define kMidShadowBeg MEM_TO_SHADOW(kMidMemBeg)
-# define kMidShadowEnd MEM_TO_SHADOW(kMidMemEnd)
+#    define kMidShadowBeg MEM_TO_SHADOW(kMidMemBeg)
+#    define kMidShadowEnd MEM_TO_SHADOW(kMidMemEnd)
 
 // With the zero shadow base we can not actually map pages starting from 0.
 // This constant is somewhat arbitrary.
-#define kZeroBaseShadowStart 0
-#define kZeroBaseMaxShadowStart (1 << 18)
+#    define kZeroBaseShadowStart 0
+#    define kZeroBaseMaxShadowStart (1 << 18)
 
-#define kShadowGapBeg   (kLowShadowEnd ? kLowShadowEnd + 1 \
-                                       : kZeroBaseShadowStart)
-#define kShadowGapEnd   ((kMidMemBeg ? kMidShadowBeg : kHighShadowBeg) - 1)
+#    define kShadowGapBeg \
+      (kLowShadowEnd ? kLowShadowEnd + 1 : kZeroBaseShadowStart)
+#    define kShadowGapEnd ((kMidMemBeg ? kMidShadowBeg : kHighShadowBeg) - 1)
 
-#define kShadowGap2Beg (kMidMemBeg ? kMidShadowEnd + 1 : 0)
-#define kShadowGap2End (kMidMemBeg ? kMidMemBeg - 1 : 0)
+#    define kShadowGap2Beg (kMidMemBeg ? kMidShadowEnd + 1 : 0)
+#    define kShadowGap2End (kMidMemBeg ? kMidMemBeg - 1 : 0)
 
-#define kShadowGap3Beg (kMidMemBeg ? kMidMemEnd + 1 : 0)
-#define kShadowGap3End (kMidMemBeg ? kHighShadowBeg - 1 : 0)
+#    define kShadowGap3Beg (kMidMemBeg ? kMidMemEnd + 1 : 0)
+#    define kShadowGap3End (kMidMemBeg ? kHighShadowBeg - 1 : 0)
 
 namespace __asan {
 
@@ -331,29 +325,31 @@ static inline bool AddrIsInShadowGap(uptr a) {
   PROFILE_ASAN_MAPPING();
   if (kMidMemBeg) {
     if (a <= kShadowGapEnd)
-      return SHADOW_OFFSET == 0 || a >= kShadowGapBeg;
+      return ASAN_SHADOW_OFFSET == 0 || a >= kShadowGapBeg;
     return (a >= kShadowGap2Beg && a <= kShadowGap2End) ||
            (a >= kShadowGap3Beg && a <= kShadowGap3End);
   }
   // In zero-based shadow mode we treat addresses near zero as addresses
   // in shadow gap as well.
-  if (SHADOW_OFFSET == 0)
+  if (ASAN_SHADOW_OFFSET == 0)
     return a <= kShadowGapEnd;
   return a >= kShadowGapBeg && a <= kShadowGapEnd;
 }
 
 }  // namespace __asan
 
-#endif
+#  endif
 
 namespace __asan {
 
-static inline uptr MemToShadowSize(uptr size) { return size >> SHADOW_SCALE; }
+static inline uptr MemToShadowSize(uptr size) {
+  return size >> ASAN_SHADOW_SCALE;
+}
 
 static inline bool AddrIsInMem(uptr a) {
   PROFILE_ASAN_MAPPING();
   return AddrIsInLowMem(a) || AddrIsInMidMem(a) || AddrIsInHighMem(a) ||
-      (flags()->protect_shadow_gap == 0 && AddrIsInShadowGap(a));
+         (flags()->protect_shadow_gap == 0 && AddrIsInShadowGap(a));
 }
 
 static inline uptr MemToShadow(uptr p) {
@@ -369,17 +365,17 @@ static inline bool AddrIsInShadow(uptr a) {
 
 static inline bool AddrIsAlignedByGranularity(uptr a) {
   PROFILE_ASAN_MAPPING();
-  return (a & (SHADOW_GRANULARITY - 1)) == 0;
+  return (a & (ASAN_SHADOW_GRANULARITY - 1)) == 0;
 }
 
 static inline bool AddressIsPoisoned(uptr a) {
   PROFILE_ASAN_MAPPING();
   const uptr kAccessSize = 1;
-  u8 *shadow_address = (u8*)MEM_TO_SHADOW(a);
+  u8 *shadow_address = (u8 *)MEM_TO_SHADOW(a);
   s8 shadow_value = *shadow_address;
   if (shadow_value) {
-    u8 last_accessed_byte = (a & (SHADOW_GRANULARITY - 1))
-                                 + kAccessSize - 1;
+    u8 last_accessed_byte =
+        (a & (ASAN_SHADOW_GRANULARITY - 1)) + kAccessSize - 1;
     return (last_accessed_byte >= shadow_value);
   }
   return false;
@@ -390,4 +386,6 @@ static const uptr kAsanMappingProfileSize = __LINE__;
 
 }  // namespace __asan
 
+#endif  // __cplusplus
+
 #endif  // ASAN_MAPPING_H
diff --git a/compiler-rt/lib/asan/asan_mapping_sparc64.h b/compiler-rt/lib/asan/asan_mapping_sparc64.h
index 432a1816f797..90261d301f7f 100644
--- a/compiler-rt/lib/asan/asan_mapping_sparc64.h
+++ b/compiler-rt/lib/asan/asan_mapping_sparc64.h
@@ -25,13 +25,14 @@
 // The idea is to chop the high bits before doing the scaling, so the two
 // parts become contiguous again and the usual scheme can be applied.
 
-#define MEM_TO_SHADOW(mem) \
-  ((((mem) << HIGH_BITS) >> (HIGH_BITS + (SHADOW_SCALE))) + (SHADOW_OFFSET))
+#define MEM_TO_SHADOW(mem)                                       \
+  ((((mem) << HIGH_BITS) >> (HIGH_BITS + (ASAN_SHADOW_SCALE))) + \
+   (ASAN_SHADOW_OFFSET))
 
 #define kLowMemBeg 0
-#define kLowMemEnd (SHADOW_OFFSET - 1)
+#define kLowMemEnd (ASAN_SHADOW_OFFSET - 1)
 
-#define kLowShadowBeg SHADOW_OFFSET
+#define kLowShadowBeg ASAN_SHADOW_OFFSET
 #define kLowShadowEnd MEM_TO_SHADOW(kLowMemEnd)
 
 // But of course there is the huge hole between the high shadow memory,
diff --git a/compiler-rt/lib/asan/asan_poisoning.cpp b/compiler-rt/lib/asan/asan_poisoning.cpp
index d97af91e692d..bbc7db4709e1 100644
--- a/compiler-rt/lib/asan/asan_poisoning.cpp
+++ b/compiler-rt/lib/asan/asan_poisoning.cpp
@@ -35,7 +35,7 @@ void PoisonShadow(uptr addr, uptr size, u8 value) {
   CHECK(AddrIsAlignedByGranularity(addr));
   CHECK(AddrIsInMem(addr));
   CHECK(AddrIsAlignedByGranularity(addr + size));
-  CHECK(AddrIsInMem(addr + size - SHADOW_GRANULARITY));
+  CHECK(AddrIsInMem(addr + size - ASAN_SHADOW_GRANULARITY));
   CHECK(REAL(memset));
   FastPoisonShadow(addr, size, value);
 }
@@ -52,12 +52,12 @@ void PoisonShadowPartialRightRedzone(uptr addr,
 
 struct ShadowSegmentEndpoint {
   u8 *chunk;
-  s8 offset;  // in [0, SHADOW_GRANULARITY)
+  s8 offset;  // in [0, ASAN_SHADOW_GRANULARITY)
   s8 value;  // = *chunk;
 
   explicit ShadowSegmentEndpoint(uptr address) {
     chunk = (u8*)MemToShadow(address);
-    offset = address & (SHADOW_GRANULARITY - 1);
+    offset = address & (ASAN_SHADOW_GRANULARITY - 1);
     value = *chunk;
   }
 };
@@ -72,14 +72,14 @@ void AsanPoisonOrUnpoisonIntraObjectRedzone(uptr ptr, uptr size, bool poison) {
   }
   CHECK(size);
   CHECK_LE(size, 4096);
-  CHECK(IsAligned(end, SHADOW_GRANULARITY));
-  if (!IsAligned(ptr, SHADOW_GRANULARITY)) {
+  CHECK(IsAligned(end, ASAN_SHADOW_GRANULARITY));
+  if (!IsAligned(ptr, ASAN_SHADOW_GRANULARITY)) {
     *(u8 *)MemToShadow(ptr) =
-        poison ? static_cast<u8>(ptr % SHADOW_GRANULARITY) : 0;
-    ptr |= SHADOW_GRANULARITY - 1;
+        poison ? static_cast<u8>(ptr % ASAN_SHADOW_GRANULARITY) : 0;
+    ptr |= ASAN_SHADOW_GRANULARITY - 1;
     ptr++;
   }
-  for (; ptr < end; ptr += SHADOW_GRANULARITY)
+  for (; ptr < end; ptr += ASAN_SHADOW_GRANULARITY)
     *(u8*)MemToShadow(ptr) = poison ? kAsanIntraObjectRedzone : 0;
 }
 
@@ -181,12 +181,12 @@ uptr __asan_region_is_poisoned(uptr beg, uptr size) {
   if (!AddrIsInMem(end))
     return end;
   CHECK_LT(beg, end);
-  uptr aligned_b = RoundUpTo(beg, SHADOW_GRANULARITY);
-  uptr aligned_e = RoundDownTo(end, SHADOW_GRANULARITY);
+  uptr aligned_b = RoundUpTo(beg, ASAN_SHADOW_GRANULARITY);
+  uptr aligned_e = RoundDownTo(end, ASAN_SHADOW_GRANULARITY);
   uptr shadow_beg = MemToShadow(aligned_b);
   uptr shadow_end = MemToShadow(aligned_e);
   // First check the first and the last application bytes,
-  // then check the SHADOW_GRANULARITY-aligned region by calling
+  // then check the ASAN_SHADOW_GRANULARITY-aligned region by calling
   // mem_is_zero on the corresponding shadow.
   if (!__asan::AddressIsPoisoned(beg) && !__asan::AddressIsPoisoned(end - 1) &&
       (shadow_end <= shadow_beg ||
@@ -285,7 +285,7 @@ uptr __asan_load_cxx_array_cookie(uptr *p) {
 // assumes that left border of region to be poisoned is properly aligned.
 static void PoisonAlignedStackMemory(uptr addr, uptr size, bool do_poison) {
   if (size == 0) return;
-  uptr aligned_size = size & ~(SHADOW_GRANULARITY - 1);
+  uptr aligned_size = size & ~(ASAN_SHADOW_GRANULARITY - 1);
   PoisonShadow(addr, aligned_size,
                do_poison ? kAsanStackUseAfterScopeMagic : 0);
   if (size == aligned_size)
@@ -351,7 +351,7 @@ void __sanitizer_annotate_contiguous_container(const void *beg_p,
   uptr end = reinterpret_cast<uptr>(end_p);
   uptr old_mid = reinterpret_cast<uptr>(old_mid_p);
   uptr new_mid = reinterpret_cast<uptr>(new_mid_p);
-  uptr granularity = SHADOW_GRANULARITY;
+  uptr granularity = ASAN_SHADOW_GRANULARITY;
   if (!(beg <= old_mid && beg <= new_mid && old_mid <= end && new_mid <= end &&
         IsAligned(beg, granularity))) {
     GET_STACK_TRACE_FATAL_HERE;
diff --git a/compiler-rt/lib/asan/asan_poisoning.h b/compiler-rt/lib/asan/asan_poisoning.h
index 3d536f2d3097..600bd011f304 100644
--- a/compiler-rt/lib/asan/asan_poisoning.h
+++ b/compiler-rt/lib/asan/asan_poisoning.h
@@ -44,8 +44,8 @@ ALWAYS_INLINE void FastPoisonShadow(uptr aligned_beg, uptr aligned_size,
                           common_flags()->clear_shadow_mmap_threshold);
 #else
   uptr shadow_beg = MEM_TO_SHADOW(aligned_beg);
-  uptr shadow_end = MEM_TO_SHADOW(
-      aligned_beg + aligned_size - SHADOW_GRANULARITY) + 1;
+  uptr shadow_end =
+      MEM_TO_SHADOW(aligned_beg + aligned_size - ASAN_SHADOW_GRANULARITY) + 1;
   // FIXME: Page states are different on Windows, so using the same interface
   // for mapping shadow and zeroing out pages doesn't "just work", so we should
   // probably provide higher-level interface for these operations.
@@ -78,11 +78,12 @@ ALWAYS_INLINE void FastPoisonShadowPartialRightRedzone(
   DCHECK(CanPoisonMemory());
   bool poison_partial = flags()->poison_partial;
   u8 *shadow = (u8*)MEM_TO_SHADOW(aligned_addr);
-  for (uptr i = 0; i < redzone_size; i += SHADOW_GRANULARITY, shadow++) {
-    if (i + SHADOW_GRANULARITY <= size) {
+  for (uptr i = 0; i < redzone_size; i += ASAN_SHADOW_GRANULARITY, shadow++) {
+    if (i + ASAN_SHADOW_GRANULARITY <= size) {
       *shadow = 0;  // fully addressable
     } else if (i >= size) {
-      *shadow = (SHADOW_GRANULARITY == 128) ? 0xff : value;  // unaddressable
+      *shadow =
+          (ASAN_SHADOW_GRANULARITY == 128) ? 0xff : value;  // unaddressable
     } else {
       // first size-i bytes are addressable
       *shadow = poison_partial ? static_cast<u8>(size - i) : 0;
diff --git a/compiler-rt/lib/asan/asan_premap_shadow.cpp b/compiler-rt/lib/asan/asan_premap_shadow.cpp
index 666bb9b34bd3..bed2f62a2251 100644
--- a/compiler-rt/lib/asan/asan_premap_shadow.cpp
+++ b/compiler-rt/lib/asan/asan_premap_shadow.cpp
@@ -26,7 +26,7 @@ namespace __asan {
 // Conservative upper limit.
 uptr PremapShadowSize() {
   uptr granularity = GetMmapGranularity();
-  return RoundUpTo(GetMaxVirtualAddress() >> SHADOW_SCALE, granularity);
+  return RoundUpTo(GetMaxVirtualAddress() >> ASAN_SHADOW_SCALE, granularity);
 }
 
 // Returns an address aligned to 8 pages, such that one page on the left and
diff --git a/compiler-rt/lib/asan/asan_rtl.cpp b/compiler-rt/lib/asan/asan_rtl.cpp
index 5be8ef0f6d1c..f0bbbf32e6a6 100644
--- a/compiler-rt/lib/asan/asan_rtl.cpp
+++ b/compiler-rt/lib/asan/asan_rtl.cpp
@@ -146,11 +146,11 @@ ASAN_REPORT_ERROR_N(store, true)
 
 #define ASAN_MEMORY_ACCESS_CALLBACK_BODY(type, is_write, size, exp_arg, fatal) \
   uptr sp = MEM_TO_SHADOW(addr);                                               \
-  uptr s = size <= SHADOW_GRANULARITY ? *reinterpret_cast<u8 *>(sp)            \
-                                      : *reinterpret_cast<u16 *>(sp);          \
+  uptr s = size <= ASAN_SHADOW_GRANULARITY ? *reinterpret_cast<u8 *>(sp)       \
+                                           : *reinterpret_cast<u16 *>(sp);     \
   if (UNLIKELY(s)) {                                                           \
-    if (UNLIKELY(size >= SHADOW_GRANULARITY ||                                 \
-                 ((s8)((addr & (SHADOW_GRANULARITY - 1)) + size - 1)) >=       \
+    if (UNLIKELY(size >= ASAN_SHADOW_GRANULARITY ||                            \
+                 ((s8)((addr & (ASAN_SHADOW_GRANULARITY - 1)) + size - 1)) >=  \
                      (s8)s)) {                                                 \
       ReportGenericErrorWrapper(addr, is_write, size, exp_arg, fatal);         \
     }                                                                          \
@@ -309,7 +309,7 @@ static void InitializeHighMemEnd() {
   kHighMemEnd = GetMaxUserVirtualAddress();
   // Increase kHighMemEnd to make sure it's properly
   // aligned together with kHighMemBeg:
-  kHighMemEnd |= (GetMmapGranularity() << SHADOW_SCALE) - 1;
+  kHighMemEnd |= (GetMmapGranularity() << ASAN_SHADOW_SCALE) - 1;
 #endif  // !ASAN_FIXED_MAPPING
   CHECK_EQ((kHighMemBeg % GetMmapGranularity()), 0);
 }
@@ -361,29 +361,16 @@ void PrintAddressSpaceLayout() {
   Printf("malloc_context_size=%zu\n",
          (uptr)common_flags()->malloc_context_size);
 
-  Printf("SHADOW_SCALE: %d\n", (int)SHADOW_SCALE);
-  Printf("SHADOW_GRANULARITY: %d\n", (int)SHADOW_GRANULARITY);
-  Printf("SHADOW_OFFSET: 0x%zx\n", (uptr)SHADOW_OFFSET);
-  CHECK(SHADOW_SCALE >= 3 && SHADOW_SCALE <= 7);
+  Printf("SHADOW_SCALE: %d\n", (int)ASAN_SHADOW_SCALE);
+  Printf("SHADOW_GRANULARITY: %d\n", (int)ASAN_SHADOW_GRANULARITY);
+  Printf("SHADOW_OFFSET: 0x%zx\n", (uptr)ASAN_SHADOW_OFFSET);
+  CHECK(ASAN_SHADOW_SCALE >= 3 && ASAN_SHADOW_SCALE <= 7);
   if (kMidMemBeg)
     CHECK(kMidShadowBeg > kLowShadowEnd &&
           kMidMemBeg > kMidShadowEnd &&
           kHighShadowBeg > kMidMemEnd);
 }
 
-#if defined(__thumb__) && defined(__linux__)
-#define START_BACKGROUND_THREAD_IN_ASAN_INTERNAL
-#endif
-
-#ifndef START_BACKGROUND_THREAD_IN_ASAN_INTERNAL
-static bool UNUSED __local_asan_dyninit = [] {
-  MaybeStartBackgroudThread();
-  SetSoftRssLimitExceededCallback(AsanSoftRssLimitExceededCallback);
-
-  return false;
-}();
-#endif
-
 static void AsanInitInternal() {
   if (LIKELY(asan_inited)) return;
   SanitizerToolName = "AddressSanitizer";
@@ -434,7 +421,7 @@ static void AsanInitInternal() {
   MaybeReexec();
 
   // Setup internal allocator callback.
-  SetLowLevelAllocateMinAlignment(SHADOW_GRANULARITY);
+  SetLowLevelAllocateMinAlignment(ASAN_SHADOW_GRANULARITY);
   SetLowLevelAllocateCallback(OnLowLevelAllocate);
 
   InitializeAsanInterceptors();
@@ -458,10 +445,8 @@ static void AsanInitInternal() {
   allocator_options.SetFrom(flags(), common_flags());
   InitializeAllocator(allocator_options);
 
-#ifdef START_BACKGROUND_THREAD_IN_ASAN_INTERNAL
-  MaybeStartBackgroudThread();
-  SetSoftRssLimitExceededCallback(AsanSoftRssLimitExceededCallback);
-#endif
+  if (SANITIZER_START_BACKGROUND_THREAD_IN_ASAN_INTERNAL)
+    MaybeStartBackgroudThread();
 
   // On Linux AsanThread::ThreadStart() calls malloc() that's why asan_inited
   // should be set to 1 prior to initializing the threads.
@@ -557,7 +542,7 @@ void UnpoisonStack(uptr bottom, uptr top, const char *type) {
         top - bottom);
     return;
   }
-  PoisonShadow(bottom, RoundUpTo(top - bottom, SHADOW_GRANULARITY), 0);
+  PoisonShadow(bottom, RoundUpTo(top - bottom, ASAN_SHADOW_GRANULARITY), 0);
 }
 
 static void UnpoisonDefaultStack() {
diff --git a/compiler-rt/lib/asan/asan_rtl_x86_64.S b/compiler-rt/lib/asan/asan_rtl_x86_64.S
new file mode 100644
index 000000000000..d27db745ed67
--- /dev/null
+++ b/compiler-rt/lib/asan/asan_rtl_x86_64.S
@@ -0,0 +1,146 @@
+#include "asan_mapping.h"
+#include "sanitizer_common/sanitizer_asm.h"
+
+#if defined(__x86_64__)
+#include "sanitizer_common/sanitizer_platform.h"
+
+.section .text
+.file "asan_rtl_x86_64.S"
+
+#define NAME(n, reg, op, s, i) n##_##op##_##i##_##s##_##reg
+
+#define FNAME(reg, op, s, i) NAME(__asan_check, reg, op, s, i)
+#define RLABEL(reg, op, s, i) NAME(.return, reg, op, s, i)
+#define CLABEL(reg, op, s, i) NAME(.check, reg, op, s, i)
+#define FLABEL(reg, op, s, i) NAME(.fail, reg, op, s, i)
+
+#define BEGINF(reg, op, s, i) \
+.globl  FNAME(reg, op, s, i) ;\
+.hidden  FNAME(reg, op, s, i) ;\
+ASM_TYPE_FUNCTION(FNAME(reg, op, s, i)) ;\
+.cfi_startproc ;\
+FNAME(reg, op, s, i): ;\
+
+#define ENDF .cfi_endproc ;\
+
+// Access check functions for 1,2 and 4 byte types, which require extra checks.
+#define ASAN_MEMORY_ACCESS_INITIAL_CHECK_ADD(reg, op, s) \
+        mov    %##reg,%r10 ;\
+        shr    $0x3,%r10 ;\
+        movsbl ASAN_SHADOW_OFFSET_CONST(%r10),%r10d ;\
+        test   %r10d,%r10d ;\
+        jne    CLABEL(reg, op, s, add) ;\
+RLABEL(reg, op, s, add): ;\
+        retq  ;\
+
+#define ASAN_MEMORY_ACCESS_EXTRA_CHECK_1(reg, op, i) \
+CLABEL(reg, op, 1, i): ;\
+        push   %rcx ;\
+        mov    %##reg,%rcx ;\
+        and    $0x7,%ecx ;\
+        cmp    %r10d,%ecx ;\
+        pop    %rcx ;\
+        jl     RLABEL(reg, op, 1, i);\
+        mov    %##reg,%rdi ;\
+        jmp    __asan_report_##op##1@PLT ;\
+
+#define ASAN_MEMORY_ACCESS_EXTRA_CHECK_2(reg, op, i) \
+CLABEL(reg, op, 2, i): ;\
+        push   %rcx ;\
+        mov    %##reg,%rcx ;\
+        and    $0x7,%ecx ;\
+        add    $0x1,%ecx ;\
+        cmp    %r10d,%ecx ;\
+        pop    %rcx ;\
+        jl     RLABEL(reg, op, 2, i);\
+        mov    %##reg,%rdi ;\
+        jmp    __asan_report_##op##2@PLT ;\
+
+#define ASAN_MEMORY_ACCESS_EXTRA_CHECK_4(reg, op, i) \
+CLABEL(reg, op, 4, i): ;\
+        push   %rcx ;\
+        mov    %##reg,%rcx ;\
+        and    $0x7,%ecx ;\
+        add    $0x3,%ecx ;\
+        cmp    %r10d,%ecx ;\
+        pop    %rcx ;\
+        jl     RLABEL(reg, op, 4, i);\
+        mov    %##reg,%rdi ;\
+        jmp    __asan_report_##op##4@PLT ;\
+
+#define ASAN_MEMORY_ACCESS_CALLBACK_ADD_1(reg, op) \
+BEGINF(reg, op, 1, add) ;\
+        ASAN_MEMORY_ACCESS_INITIAL_CHECK_ADD(reg, op, 1) ;\
+        ASAN_MEMORY_ACCESS_EXTRA_CHECK_1(reg, op, add) ;\
+ENDF
+
+#define ASAN_MEMORY_ACCESS_CALLBACK_ADD_2(reg, op) \
+BEGINF(reg, op, 2, add) ;\
+        ASAN_MEMORY_ACCESS_INITIAL_CHECK_ADD(reg, op, 2) ;\
+        ASAN_MEMORY_ACCESS_EXTRA_CHECK_2(reg, op, add) ;\
+ENDF
+
+#define ASAN_MEMORY_ACCESS_CALLBACK_ADD_4(reg, op) \
+BEGINF(reg, op, 4, add) ;\
+        ASAN_MEMORY_ACCESS_INITIAL_CHECK_ADD(reg, op, 4) ;\
+        ASAN_MEMORY_ACCESS_EXTRA_CHECK_4(reg, op, add) ;\
+ENDF
+
+// Access check functions for 8 and 16 byte types: no extra checks required.
+#define ASAN_MEMORY_ACCESS_CHECK_ADD(reg, op, s, c) \
+        mov    %##reg,%r10 ;\
+        shr    $0x3,%r10 ;\
+        ##c    $0x0,ASAN_SHADOW_OFFSET_CONST(%r10) ;\
+        jne    FLABEL(reg, op, s, add) ;\
+        retq  ;\
+
+#define ASAN_MEMORY_ACCESS_FAIL(reg, op, s, i) \
+FLABEL(reg, op, s, i): ;\
+        mov    %##reg,%rdi ;\
+        jmp    __asan_report_##op##s@PLT;\
+
+#define ASAN_MEMORY_ACCESS_CALLBACK_ADD_8(reg, op) \
+BEGINF(reg, op, 8, add) ;\
+        ASAN_MEMORY_ACCESS_CHECK_ADD(reg, op, 8, cmpb) ;\
+        ASAN_MEMORY_ACCESS_FAIL(reg, op, 8, add) ;\
+ENDF
+
+#define ASAN_MEMORY_ACCESS_CALLBACK_ADD_16(reg, op) \
+BEGINF(reg, op, 16, add) ;\
+        ASAN_MEMORY_ACCESS_CHECK_ADD(reg, op, 16, cmpw) ;\
+        ASAN_MEMORY_ACCESS_FAIL(reg, op, 16, add) ;\
+ENDF
+
+#define ASAN_MEMORY_ACCESS_CALLBACKS_ADD(reg) \
+ASAN_MEMORY_ACCESS_CALLBACK_ADD_1(reg, load) \
+ASAN_MEMORY_ACCESS_CALLBACK_ADD_1(reg, store) \
+ASAN_MEMORY_ACCESS_CALLBACK_ADD_2(reg, load) \
+ASAN_MEMORY_ACCESS_CALLBACK_ADD_2(reg, store) \
+ASAN_MEMORY_ACCESS_CALLBACK_ADD_4(reg, load) \
+ASAN_MEMORY_ACCESS_CALLBACK_ADD_4(reg, store) \
+ASAN_MEMORY_ACCESS_CALLBACK_ADD_8(reg, load) \
+ASAN_MEMORY_ACCESS_CALLBACK_ADD_8(reg, store) \
+ASAN_MEMORY_ACCESS_CALLBACK_ADD_16(reg, load) \
+ASAN_MEMORY_ACCESS_CALLBACK_ADD_16(reg, store) \
+
+
+// Instantiate all but R10 and R11 callbacks. We are using PLTSafe class with
+// the intrinsic, which guarantees that the code generation will never emit
+// R10 or R11 callback.
+ASAN_MEMORY_ACCESS_CALLBACKS_ADD(RAX)
+ASAN_MEMORY_ACCESS_CALLBACKS_ADD(RBX)
+ASAN_MEMORY_ACCESS_CALLBACKS_ADD(RCX)
+ASAN_MEMORY_ACCESS_CALLBACKS_ADD(RDX)
+ASAN_MEMORY_ACCESS_CALLBACKS_ADD(RSI)
+ASAN_MEMORY_ACCESS_CALLBACKS_ADD(RDI)
+ASAN_MEMORY_ACCESS_CALLBACKS_ADD(RBP)
+ASAN_MEMORY_ACCESS_CALLBACKS_ADD(R8)
+ASAN_MEMORY_ACCESS_CALLBACKS_ADD(R9)
+ASAN_MEMORY_ACCESS_CALLBACKS_ADD(R12)
+ASAN_MEMORY_ACCESS_CALLBACKS_ADD(R13)
+ASAN_MEMORY_ACCESS_CALLBACKS_ADD(R14)
+ASAN_MEMORY_ACCESS_CALLBACKS_ADD(R15)
+
+#endif
+
+NO_EXEC_STACK_DIRECTIVE
diff --git a/compiler-rt/lib/asan/asan_thread.cpp b/compiler-rt/lib/asan/asan_thread.cpp
index 930139968ec3..2b06c3c4e7c0 100644
--- a/compiler-rt/lib/asan/asan_thread.cpp
+++ b/compiler-rt/lib/asan/asan_thread.cpp
@@ -305,7 +305,7 @@ void AsanThread::SetThreadStackAndTls(const InitOptions *options) {
   uptr stack_size = 0;
   GetThreadStackAndTls(tid() == kMainTid, &stack_bottom_, &stack_size,
                        &tls_begin_, &tls_size);
-  stack_top_ = RoundDownTo(stack_bottom_ + stack_size, SHADOW_GRANULARITY);
+  stack_top_ = RoundDownTo(stack_bottom_ + stack_size, ASAN_SHADOW_GRANULARITY);
   tls_end_ = tls_begin_ + tls_size;
   dtls_ = DTLS_Get();
 
@@ -321,8 +321,8 @@ void AsanThread::ClearShadowForThreadStackAndTLS() {
   if (stack_top_ != stack_bottom_)
     PoisonShadow(stack_bottom_, stack_top_ - stack_bottom_, 0);
   if (tls_begin_ != tls_end_) {
-    uptr tls_begin_aligned = RoundDownTo(tls_begin_, SHADOW_GRANULARITY);
-    uptr tls_end_aligned = RoundUpTo(tls_end_, SHADOW_GRANULARITY);
+    uptr tls_begin_aligned = RoundDownTo(tls_begin_, ASAN_SHADOW_GRANULARITY);
+    uptr tls_end_aligned = RoundUpTo(tls_end_, ASAN_SHADOW_GRANULARITY);
     FastPoisonShadowPartialRightRedzone(tls_begin_aligned,
                                         tls_end_ - tls_begin_aligned,
                                         tls_end_aligned - tls_end_, 0);
@@ -346,27 +346,27 @@ bool AsanThread::GetStackFrameAccessByAddr(uptr addr,
     return true;
   }
   uptr aligned_addr = RoundDownTo(addr, SANITIZER_WORDSIZE / 8);  // align addr.
-  uptr mem_ptr = RoundDownTo(aligned_addr, SHADOW_GRANULARITY);
+  uptr mem_ptr = RoundDownTo(aligned_addr, ASAN_SHADOW_GRANULARITY);
   u8 *shadow_ptr = (u8*)MemToShadow(aligned_addr);
   u8 *shadow_bottom = (u8*)MemToShadow(bottom);
 
   while (shadow_ptr >= shadow_bottom &&
          *shadow_ptr != kAsanStackLeftRedzoneMagic) {
     shadow_ptr--;
-    mem_ptr -= SHADOW_GRANULARITY;
+    mem_ptr -= ASAN_SHADOW_GRANULARITY;
   }
 
   while (shadow_ptr >= shadow_bottom &&
          *shadow_ptr == kAsanStackLeftRedzoneMagic) {
     shadow_ptr--;
-    mem_ptr -= SHADOW_GRANULARITY;
+    mem_ptr -= ASAN_SHADOW_GRANULARITY;
   }
 
   if (shadow_ptr < shadow_bottom) {
     return false;
   }
 
-  uptr* ptr = (uptr*)(mem_ptr + SHADOW_GRANULARITY);
+  uptr *ptr = (uptr *)(mem_ptr + ASAN_SHADOW_GRANULARITY);
   CHECK(ptr[0] == kCurrentStackFrameMagic);
   access->offset = addr - (uptr)ptr;
   access->frame_pc = ptr[2];
diff --git a/compiler-rt/lib/asan/asan_win.cpp b/compiler-rt/lib/asan/asan_win.cpp
index 1577c83cf994..53a0e3bfd385 100644
--- a/compiler-rt/lib/asan/asan_win.cpp
+++ b/compiler-rt/lib/asan/asan_win.cpp
@@ -253,7 +253,7 @@ void *AsanDoesNotSupportStaticLinkage() {
 }
 
 uptr FindDynamicShadowStart() {
-  return MapDynamicShadow(MemToShadowSize(kHighMemEnd), SHADOW_SCALE,
+  return MapDynamicShadow(MemToShadowSize(kHighMemEnd), ASAN_SHADOW_SCALE,
                           /*min_shadow_base_alignment*/ 0, kHighMemEnd);
 }
 
diff --git a/compiler-rt/lib/builtins/cpu_model.c b/compiler-rt/lib/builtins/cpu_model.c
index 53e2d89708dc..cf12aa021d3d 100644
--- a/compiler-rt/lib/builtins/cpu_model.c
+++ b/compiler-rt/lib/builtins/cpu_model.c
@@ -798,6 +798,10 @@ _Bool __aarch64_have_lse_atomics
 #ifndef HWCAP_ATOMICS
 #define HWCAP_ATOMICS (1 << 8)
 #endif
+#if defined(__ANDROID__)
+#include <string.h>
+#include <sys/system_properties.h>
+#endif
 static void CONSTRUCTOR_ATTRIBUTE init_have_lse_atomics(void) {
 #if defined(__FreeBSD__)
   unsigned long hwcap;
@@ -805,8 +809,20 @@ static void CONSTRUCTOR_ATTRIBUTE init_have_lse_atomics(void) {
   __aarch64_have_lse_atomics = result == 0 && (hwcap & HWCAP_ATOMICS) != 0;
 #else
   unsigned long hwcap = getauxval(AT_HWCAP);
-  __aarch64_have_lse_atomics = (hwcap & HWCAP_ATOMICS) != 0;
-#endif
+  _Bool result = (hwcap & HWCAP_ATOMICS) != 0;
+#if defined(__ANDROID__)
+  if (result) {
+    char arch[PROP_VALUE_MAX];
+    if (__system_property_get("ro.arch", arch) > 0 &&
+        strncmp(arch, "exynos9810", sizeof("exynos9810") - 1) == 0) {
+      // Some cores of Exynos 9810 are ARMv8.2 and others are ARMv8.0,
+      // so disable the lse atomics completely.
+      result = false;
+    }
+  }
+#endif // defined(__ANDROID__)
+  __aarch64_have_lse_atomics = result;
+#endif // defined(__FreeBSD__)
 }
 #endif // defined(__has_include)
 #endif // __has_include(<sys/auxv.h>)
diff --git a/compiler-rt/lib/dfsan/dfsan.cpp b/compiler-rt/lib/dfsan/dfsan.cpp
index ce2c04df83a8..ee7221c7b9a8 100644
--- a/compiler-rt/lib/dfsan/dfsan.cpp
+++ b/compiler-rt/lib/dfsan/dfsan.cpp
@@ -630,22 +630,16 @@ void PrintInvalidOriginWarning(dfsan_label label, const void *address) {
       d.Warning(), label, address, d.Default());
 }
 
-bool PrintOriginTraceToStr(const void *addr, const char *description,
-                           InternalScopedString *out) {
-  CHECK(out);
-  CHECK(dfsan_get_track_origins());
+void PrintInvalidOriginIdWarning(dfsan_origin origin) {
   Decorator d;
+  Printf(
+      "  %sOrigin Id %d has invalid origin tracking. This can "
+      "be a DFSan bug.%s\n",
+      d.Warning(), origin, d.Default());
+}
 
-  const dfsan_label label = *__dfsan::shadow_for(addr);
-  CHECK(label);
-
-  const dfsan_origin origin = *__dfsan::origin_for(addr);
-
-  out->append("  %sTaint value 0x%x (at %p) origin tracking (%s)%s\n",
-              d.Origin(), label, addr, description ? description : "",
-              d.Default());
-
-  Origin o = Origin::FromRawId(origin);
+bool PrintOriginTraceFramesToStr(Origin o, InternalScopedString *out) {
+  Decorator d;
   bool found = false;
 
   while (o.isChainedOrigin()) {
@@ -668,6 +662,25 @@ bool PrintOriginTraceToStr(const void *addr, const char *description,
   return found;
 }
 
+bool PrintOriginTraceToStr(const void *addr, const char *description,
+                           InternalScopedString *out) {
+  CHECK(out);
+  CHECK(dfsan_get_track_origins());
+  Decorator d;
+
+  const dfsan_label label = *__dfsan::shadow_for(addr);
+  CHECK(label);
+
+  const dfsan_origin origin = *__dfsan::origin_for(addr);
+
+  out->append("  %sTaint value 0x%x (at %p) origin tracking (%s)%s\n",
+              d.Origin(), label, addr, description ? description : "",
+              d.Default());
+
+  Origin o = Origin::FromRawId(origin);
+  return PrintOriginTraceFramesToStr(o, out);
+}
+
 }  // namespace
 
 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void dfsan_print_origin_trace(
@@ -725,6 +738,50 @@ dfsan_sprint_origin_trace(const void *addr, const char *description,
   return trace.length();
 }
 
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void dfsan_print_origin_id_trace(
+    dfsan_origin origin) {
+  if (!dfsan_get_track_origins()) {
+    PrintNoOriginTrackingWarning();
+    return;
+  }
+  Origin o = Origin::FromRawId(origin);
+
+  InternalScopedString trace;
+  bool success = PrintOriginTraceFramesToStr(o, &trace);
+
+  if (trace.length())
+    Printf("%s", trace.data());
+
+  if (!success)
+    PrintInvalidOriginIdWarning(origin);
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE uptr dfsan_sprint_origin_id_trace(
+    dfsan_origin origin, char *out_buf, uptr out_buf_size) {
+  CHECK(out_buf);
+
+  if (!dfsan_get_track_origins()) {
+    PrintNoOriginTrackingWarning();
+    return 0;
+  }
+  Origin o = Origin::FromRawId(origin);
+
+  InternalScopedString trace;
+  bool success = PrintOriginTraceFramesToStr(o, &trace);
+
+  if (!success) {
+    PrintInvalidOriginIdWarning(origin);
+    return 0;
+  }
+
+  if (out_buf_size) {
+    internal_strncpy(out_buf, trace.data(), out_buf_size - 1);
+    out_buf[out_buf_size - 1] = '\0';
+  }
+
+  return trace.length();
+}
+
 extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_origin
 dfsan_get_init_origin(const void *addr) {
   if (!dfsan_get_track_origins())
diff --git a/compiler-rt/lib/dfsan/dfsan_allocator.cpp b/compiler-rt/lib/dfsan/dfsan_allocator.cpp
index b2e94564446e..c50aee7a55a0 100644
--- a/compiler-rt/lib/dfsan/dfsan_allocator.cpp
+++ b/compiler-rt/lib/dfsan/dfsan_allocator.cpp
@@ -87,6 +87,12 @@ static void *DFsanAllocate(uptr size, uptr alignment, bool zeroise) {
     BufferedStackTrace stack;
     ReportAllocationSizeTooBig(size, max_malloc_size, &stack);
   }
+  if (UNLIKELY(IsRssLimitExceeded())) {
+    if (AllocatorMayReturnNull())
+      return nullptr;
+    BufferedStackTrace stack;
+    ReportRssLimitExceeded(&stack);
+  }
   DFsanThread *t = GetCurrentThread();
   void *allocated;
   if (t) {
diff --git a/compiler-rt/lib/dfsan/done_abilist.txt b/compiler-rt/lib/dfsan/done_abilist.txt
index eef7c48948cc..fc2dd02ccf5f 100644
--- a/compiler-rt/lib/dfsan/done_abilist.txt
+++ b/compiler-rt/lib/dfsan/done_abilist.txt
@@ -30,12 +30,18 @@ fun:dfsan_flush=uninstrumented
 fun:dfsan_flush=discard
 fun:dfsan_print_origin_trace=uninstrumented
 fun:dfsan_print_origin_trace=discard
+fun:dfsan_print_origin_id_trace=uninstrumented
+fun:dfsan_print_origin_id_trace=discard
 fun:dfsan_sprint_origin_trace=uninstrumented
 fun:dfsan_sprint_origin_trace=discard
+fun:dfsan_sprint_origin_id_trace=uninstrumented
+fun:dfsan_sprint_origin_id_trace=discard
 fun:dfsan_sprint_stack_trace=uninstrumented
 fun:dfsan_sprint_stack_trace=discard
 fun:dfsan_get_origin=uninstrumented
 fun:dfsan_get_origin=custom
+fun:dfsan_read_origin_of_first_taint=uninstrumented
+fun:dfsan_read_origin_of_first_taint=discard
 fun:dfsan_get_init_origin=uninstrumented
 fun:dfsan_get_init_origin=discard
 fun:dfsan_get_track_origins=uninstrumented
diff --git a/compiler-rt/lib/hwasan/hwasan_allocator.cpp b/compiler-rt/lib/hwasan/hwasan_allocator.cpp
index 9e1729964e27..84e183f2384f 100644
--- a/compiler-rt/lib/hwasan/hwasan_allocator.cpp
+++ b/compiler-rt/lib/hwasan/hwasan_allocator.cpp
@@ -132,6 +132,11 @@ static void *HwasanAllocate(StackTrace *stack, uptr orig_size, uptr alignment,
     }
     ReportAllocationSizeTooBig(orig_size, kMaxAllowedMallocSize, stack);
   }
+  if (UNLIKELY(IsRssLimitExceeded())) {
+    if (AllocatorMayReturnNull())
+      return nullptr;
+    ReportRssLimitExceeded(stack);
+  }
 
   alignment = Max(alignment, kShadowAlignment);
   uptr size = TaggedSize(orig_size);
diff --git a/compiler-rt/lib/hwasan/hwasan_interceptors.cpp b/compiler-rt/lib/hwasan/hwasan_interceptors.cpp
index f96ed8804102..8dc886e587e7 100644
--- a/compiler-rt/lib/hwasan/hwasan_interceptors.cpp
+++ b/compiler-rt/lib/hwasan/hwasan_interceptors.cpp
@@ -47,6 +47,12 @@ INTERCEPTOR(int, pthread_create, void *th, void *attr, void *(*callback)(void*),
   return res;
 }
 
+INTERCEPTOR(int, pthread_join, void *t, void **arg) {
+  return REAL(pthread_join)(t, arg);
+}
+
+DEFINE_REAL_PTHREAD_FUNCTIONS
+
 DEFINE_REAL(int, vfork)
 DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(int, vfork)
 
@@ -189,7 +195,8 @@ void InitializeInterceptors() {
   INTERCEPT_FUNCTION(vfork);
 #endif  // __linux__
   INTERCEPT_FUNCTION(pthread_create);
-#endif
+  INTERCEPT_FUNCTION(pthread_join);
+#  endif
 
   inited = 1;
 }
diff --git a/compiler-rt/lib/lsan/lsan.h b/compiler-rt/lib/lsan/lsan.h
index 1e82ad72f005..af8efa6153a5 100644
--- a/compiler-rt/lib/lsan/lsan.h
+++ b/compiler-rt/lib/lsan/lsan.h
@@ -13,17 +13,17 @@
 
 #include "lsan_thread.h"
 #if SANITIZER_POSIX
-#include "lsan_posix.h"
+#  include "lsan_posix.h"
 #elif SANITIZER_FUCHSIA
-#include "lsan_fuchsia.h"
+#  include "lsan_fuchsia.h"
 #endif
 #include "sanitizer_common/sanitizer_flags.h"
 #include "sanitizer_common/sanitizer_stacktrace.h"
 
-#define GET_STACK_TRACE(max_size, fast)                       \
-  __sanitizer::BufferedStackTrace stack;                      \
-  stack.Unwind(StackTrace::GetCurrentPc(),                    \
-               GET_CURRENT_FRAME(), nullptr, fast, max_size);
+#define GET_STACK_TRACE(max_size, fast)                                        \
+  __sanitizer::BufferedStackTrace stack;                                       \
+  stack.Unwind(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), nullptr, fast, \
+               max_size);
 
 #define GET_STACK_TRACE_FATAL \
   GET_STACK_TRACE(kStackTraceMax, common_flags()->fast_unwind_on_fatal)
@@ -40,11 +40,12 @@ void InitializeInterceptors();
 void ReplaceSystemMalloc();
 void LsanOnDeadlySignal(int signo, void *siginfo, void *context);
 
-#define ENSURE_LSAN_INITED do {   \
-  CHECK(!lsan_init_is_running);   \
-  if (!lsan_inited)               \
-    __lsan_init();                \
-} while (0)
+#define ENSURE_LSAN_INITED        \
+  do {                            \
+    CHECK(!lsan_init_is_running); \
+    if (!lsan_inited)             \
+      __lsan_init();              \
+  } while (0)
 
 }  // namespace __lsan
 
diff --git a/compiler-rt/lib/lsan/lsan_allocator.cpp b/compiler-rt/lib/lsan/lsan_allocator.cpp
index 91e34ebb3214..ea4c6c9cf647 100644
--- a/compiler-rt/lib/lsan/lsan_allocator.cpp
+++ b/compiler-rt/lib/lsan/lsan_allocator.cpp
@@ -27,11 +27,11 @@ extern "C" void *memset(void *ptr, int value, uptr num);
 
 namespace __lsan {
 #if defined(__i386__) || defined(__arm__)
-static const uptr kMaxAllowedMallocSize = 1UL << 30;
+static const uptr kMaxAllowedMallocSize = 1ULL << 30;
 #elif defined(__mips64) || defined(__aarch64__)
-static const uptr kMaxAllowedMallocSize = 4UL << 30;
+static const uptr kMaxAllowedMallocSize = 4ULL << 30;
 #else
-static const uptr kMaxAllowedMallocSize = 8UL << 30;
+static const uptr kMaxAllowedMallocSize = 8ULL << 30;
 #endif
 
 static Allocator allocator;
@@ -88,6 +88,11 @@ void *Allocate(const StackTrace &stack, uptr size, uptr alignment,
     size = 1;
   if (size > max_malloc_size)
     return ReportAllocationSizeTooBig(size, stack);
+  if (UNLIKELY(IsRssLimitExceeded())) {
+    if (AllocatorMayReturnNull())
+      return nullptr;
+    ReportRssLimitExceeded(&stack);
+  }
   void *p = allocator.Allocate(GetAllocatorCache(), size, alignment);
   if (UNLIKELY(!p)) {
     SetAllocatorOutOfMemory();
diff --git a/compiler-rt/lib/lsan/lsan_common.cpp b/compiler-rt/lib/lsan/lsan_common.cpp
index 308dbb3e41da..fd7aa38d99db 100644
--- a/compiler-rt/lib/lsan/lsan_common.cpp
+++ b/compiler-rt/lib/lsan/lsan_common.cpp
@@ -34,7 +34,6 @@ Mutex global_mutex;
 
 Flags lsan_flags;
 
-
 void DisableCounterUnderflow() {
   if (common_flags()->detect_leaks) {
     Report("Unmatched call to __lsan_enable().\n");
@@ -43,44 +42,48 @@ void DisableCounterUnderflow() {
 }
 
 void Flags::SetDefaults() {
-#define LSAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue;
-#include "lsan_flags.inc"
-#undef LSAN_FLAG
+#  define LSAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue;
+#  include "lsan_flags.inc"
+#  undef LSAN_FLAG
 }
 
 void RegisterLsanFlags(FlagParser *parser, Flags *f) {
-#define LSAN_FLAG(Type, Name, DefaultValue, Description) \
-  RegisterFlag(parser, #Name, Description, &f->Name);
-#include "lsan_flags.inc"
-#undef LSAN_FLAG
+#  define LSAN_FLAG(Type, Name, DefaultValue, Description) \
+    RegisterFlag(parser, #Name, Description, &f->Name);
+#  include "lsan_flags.inc"
+#  undef LSAN_FLAG
 }
 
-#define LOG_POINTERS(...)                           \
-  do {                                              \
-    if (flags()->log_pointers) Report(__VA_ARGS__); \
-  } while (0)
+#  define LOG_POINTERS(...)      \
+    do {                         \
+      if (flags()->log_pointers) \
+        Report(__VA_ARGS__);     \
+    } while (0)
 
-#define LOG_THREADS(...)                           \
-  do {                                             \
-    if (flags()->log_threads) Report(__VA_ARGS__); \
-  } while (0)
+#  define LOG_THREADS(...)      \
+    do {                        \
+      if (flags()->log_threads) \
+        Report(__VA_ARGS__);    \
+    } while (0)
 
 class LeakSuppressionContext {
   bool parsed = false;
   SuppressionContext context;
   bool suppressed_stacks_sorted = true;
   InternalMmapVector<u32> suppressed_stacks;
+  const LoadedModule *suppress_module = nullptr;
 
-  Suppression *GetSuppressionForAddr(uptr addr);
   void LazyInit();
+  Suppression *GetSuppressionForAddr(uptr addr);
+  bool SuppressInvalid(const StackTrace &stack);
+  bool SuppressByRule(const StackTrace &stack, uptr hit_count, uptr total_size);
 
  public:
   LeakSuppressionContext(const char *supprression_types[],
                          int suppression_types_num)
       : context(supprression_types, suppression_types_num) {}
 
-  Suppression *GetSuppressionForStack(u32 stack_trace_id,
-                                      const StackTrace &stack);
+  bool Suppress(u32 stack_trace_id, uptr hit_count, uptr total_size);
 
   const InternalMmapVector<u32> &GetSortedSuppressedStacks() {
     if (!suppressed_stacks_sorted) {
@@ -95,17 +98,17 @@ class LeakSuppressionContext {
 ALIGNED(64) static char suppression_placeholder[sizeof(LeakSuppressionContext)];
 static LeakSuppressionContext *suppression_ctx = nullptr;
 static const char kSuppressionLeak[] = "leak";
-static const char *kSuppressionTypes[] = { kSuppressionLeak };
+static const char *kSuppressionTypes[] = {kSuppressionLeak};
 static const char kStdSuppressions[] =
-#if SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT
+#  if SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT
     // For more details refer to the SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT
     // definition.
     "leak:*pthread_exit*\n"
-#endif  // SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT
-#if SANITIZER_MAC
+#  endif  // SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT
+#  if SANITIZER_MAC
     // For Darwin and os_log/os_trace: https://reviews.llvm.org/D35173
     "leak:*_os_trace*\n"
-#endif
+#  endif
     // TLS leak in some glibc versions, described in
     // https://sourceware.org/bugzilla/show_bug.cgi?id=12650.
     "leak:*tls_get_addr*\n";
@@ -123,9 +126,92 @@ void LeakSuppressionContext::LazyInit() {
     if (&__lsan_default_suppressions)
       context.Parse(__lsan_default_suppressions());
     context.Parse(kStdSuppressions);
+    if (flags()->use_tls && flags()->use_ld_allocations)
+      suppress_module = GetLinker();
   }
 }
 
+Suppression *LeakSuppressionContext::GetSuppressionForAddr(uptr addr) {
+  Suppression *s = nullptr;
+
+  // Suppress by module name.
+  if (const char *module_name =
+          Symbolizer::GetOrInit()->GetModuleNameForPc(addr))
+    if (context.Match(module_name, kSuppressionLeak, &s))
+      return s;
+
+  // Suppress by file or function name.
+  SymbolizedStack *frames = Symbolizer::GetOrInit()->SymbolizePC(addr);
+  for (SymbolizedStack *cur = frames; cur; cur = cur->next) {
+    if (context.Match(cur->info.function, kSuppressionLeak, &s) ||
+        context.Match(cur->info.file, kSuppressionLeak, &s)) {
+      break;
+    }
+  }
+  frames->ClearAll();
+  return s;
+}
+
+static uptr GetCallerPC(const StackTrace &stack) {
+  // The top frame is our malloc/calloc/etc. The next frame is the caller.
+  if (stack.size >= 2)
+    return stack.trace[1];
+  return 0;
+}
+
+// On Linux, treats all chunks allocated from ld-linux.so as reachable, which
+// covers dynamically allocated TLS blocks, internal dynamic loader's loaded
+// modules accounting etc.
+// Dynamic TLS blocks contain the TLS variables of dynamically loaded modules.
+// They are allocated with a __libc_memalign() call in allocate_and_init()
+// (elf/dl-tls.c). Glibc won't tell us the address ranges occupied by those
+// blocks, but we can make sure they come from our own allocator by intercepting
+// __libc_memalign(). On top of that, there is no easy way to reach them. Their
+// addresses are stored in a dynamically allocated array (the DTV) which is
+// referenced from the static TLS. Unfortunately, we can't just rely on the DTV
+// being reachable from the static TLS, and the dynamic TLS being reachable from
+// the DTV. This is because the initial DTV is allocated before our interception
+// mechanism kicks in, and thus we don't recognize it as allocated memory. We
+// can't special-case it either, since we don't know its size.
+// Our solution is to include in the root set all allocations made from
+// ld-linux.so (which is where allocate_and_init() is implemented). This is
+// guaranteed to include all dynamic TLS blocks (and possibly other allocations
+// which we don't care about).
+// On all other platforms, this simply checks to ensure that the caller pc is
+// valid before reporting chunks as leaked.
+bool LeakSuppressionContext::SuppressInvalid(const StackTrace &stack) {
+  uptr caller_pc = GetCallerPC(stack);
+  // If caller_pc is unknown, this chunk may be allocated in a coroutine. Mark
+  // it as reachable, as we can't properly report its allocation stack anyway.
+  return !caller_pc ||
+         (suppress_module && suppress_module->containsAddress(caller_pc));
+}
+
+bool LeakSuppressionContext::SuppressByRule(const StackTrace &stack,
+                                            uptr hit_count, uptr total_size) {
+  for (uptr i = 0; i < stack.size; i++) {
+    Suppression *s = GetSuppressionForAddr(
+        StackTrace::GetPreviousInstructionPc(stack.trace[i]));
+    if (s) {
+      s->weight += total_size;
+      atomic_fetch_add(&s->hit_count, hit_count, memory_order_relaxed);
+      return true;
+    }
+  }
+  return false;
+}
+
+bool LeakSuppressionContext::Suppress(u32 stack_trace_id, uptr hit_count,
+                                      uptr total_size) {
+  LazyInit();
+  StackTrace stack = StackDepotGet(stack_trace_id);
+  if (!SuppressInvalid(stack) && !SuppressByRule(stack, hit_count, total_size))
+    return false;
+  suppressed_stacks_sorted = false;
+  suppressed_stacks.push_back(stack_trace_id);
+  return true;
+}
+
 static LeakSuppressionContext *GetSuppressionContext() {
   CHECK(suppression_ctx);
   return suppression_ctx;
@@ -146,9 +232,9 @@ void InitCommonLsan() {
   }
 }
 
-class Decorator: public __sanitizer::SanitizerCommonDecorator {
+class Decorator : public __sanitizer::SanitizerCommonDecorator {
  public:
-  Decorator() : SanitizerCommonDecorator() { }
+  Decorator() : SanitizerCommonDecorator() {}
   const char *Error() { return Red(); }
   const char *Leak() { return Blue(); }
 };
@@ -157,19 +243,19 @@ static inline bool CanBeAHeapPointer(uptr p) {
   // Since our heap is located in mmap-ed memory, we can assume a sensible lower
   // bound on heap addresses.
   const uptr kMinAddress = 4 * 4096;
-  if (p < kMinAddress) return false;
-#if defined(__x86_64__)
+  if (p < kMinAddress)
+    return false;
+#  if defined(__x86_64__)
   // Accept only canonical form user-space addresses.
   return ((p >> 47) == 0);
-#elif defined(__mips64)
+#  elif defined(__mips64)
   return ((p >> 40) == 0);
-#elif defined(__aarch64__)
-  unsigned runtimeVMA =
-    (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1);
+#  elif defined(__aarch64__)
+  unsigned runtimeVMA = (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1);
   return ((p >> runtimeVMA) == 0);
-#else
+#  else
   return true;
-#endif
+#  endif
 }
 
 // Scans the memory range, looking for byte patterns that point into allocator
@@ -178,8 +264,7 @@ static inline bool CanBeAHeapPointer(uptr p) {
 // (|tag| = kReachable) and finding indirectly leaked chunks
 // (|tag| = kIndirectlyLeaked). In the second case, there's no flood fill,
 // so |frontier| = 0.
-void ScanRangeForPointers(uptr begin, uptr end,
-                          Frontier *frontier,
+void ScanRangeForPointers(uptr begin, uptr end, Frontier *frontier,
                           const char *region_type, ChunkTag tag) {
   CHECK(tag == kReachable || tag == kIndirectlyLeaked);
   const uptr alignment = flags()->pointer_alignment();
@@ -190,13 +275,17 @@ void ScanRangeForPointers(uptr begin, uptr end,
     pp = pp + alignment - pp % alignment;
   for (; pp + sizeof(void *) <= end; pp += alignment) {
     void *p = *reinterpret_cast<void **>(pp);
-    if (!CanBeAHeapPointer(reinterpret_cast<uptr>(p))) continue;
+    if (!CanBeAHeapPointer(reinterpret_cast<uptr>(p)))
+      continue;
     uptr chunk = PointsIntoChunk(p);
-    if (!chunk) continue;
+    if (!chunk)
+      continue;
     // Pointers to self don't count. This matters when tag == kIndirectlyLeaked.
-    if (chunk == begin) continue;
+    if (chunk == begin)
+      continue;
     LsanMetadata m(chunk);
-    if (m.tag() == kReachable || m.tag() == kIgnored) continue;
+    if (m.tag() == kReachable || m.tag() == kIgnored)
+      continue;
 
     // Do this check relatively late so we can log only the interesting cases.
     if (!flags()->use_poisoned && WordIsPoisoned(pp)) {
@@ -234,23 +323,23 @@ void ScanGlobalRange(uptr begin, uptr end, Frontier *frontier) {
   }
 }
 
-void ForEachExtraStackRangeCb(uptr begin, uptr end, void* arg) {
+void ForEachExtraStackRangeCb(uptr begin, uptr end, void *arg) {
   Frontier *frontier = reinterpret_cast<Frontier *>(arg);
   ScanRangeForPointers(begin, end, frontier, "FAKE STACK", kReachable);
 }
 
-#if SANITIZER_FUCHSIA
+#  if SANITIZER_FUCHSIA
 
 // Fuchsia handles all threads together with its own callback.
 static void ProcessThreads(SuspendedThreadsList const &, Frontier *) {}
 
-#else
+#  else
 
-#if SANITIZER_ANDROID
+#    if SANITIZER_ANDROID
 // FIXME: Move this out into *libcdep.cpp
 extern "C" SANITIZER_WEAK_ATTRIBUTE void __libc_iterate_dynamic_tls(
     pid_t, void (*cb)(void *, void *, uptr, void *), void *);
-#endif
+#    endif
 
 static void ProcessThreadRegistry(Frontier *frontier) {
   InternalMmapVector<uptr> ptrs;
@@ -282,9 +371,9 @@ static void ProcessThreads(SuspendedThreadsList const &suspended_threads,
     LOG_THREADS("Processing thread %llu.\n", os_id);
     uptr stack_begin, stack_end, tls_begin, tls_end, cache_begin, cache_end;
     DTLS *dtls;
-    bool thread_found = GetThreadRangesLocked(os_id, &stack_begin, &stack_end,
-                                              &tls_begin, &tls_end,
-                                              &cache_begin, &cache_end, &dtls);
+    bool thread_found =
+        GetThreadRangesLocked(os_id, &stack_begin, &stack_end, &tls_begin,
+                              &tls_end, &cache_begin, &cache_end, &dtls);
     if (!thread_found) {
       // If a thread can't be found in the thread registry, it's probably in the
       // process of destruction. Log this event and move on.
@@ -298,7 +387,8 @@ static void ProcessThreads(SuspendedThreadsList const &suspended_threads,
       Report("Unable to get registers from thread %llu.\n", os_id);
       // If unable to get SP, consider the entire stack to be reachable unless
       // GetRegistersAndSP failed with ESRCH.
-      if (have_registers == REGISTERS_UNAVAILABLE_FATAL) continue;
+      if (have_registers == REGISTERS_UNAVAILABLE_FATAL)
+        continue;
       sp = stack_begin;
     }
 
@@ -353,7 +443,7 @@ static void ProcessThreads(SuspendedThreadsList const &suspended_threads,
                                  kReachable);
         }
       }
-#if SANITIZER_ANDROID
+#    if SANITIZER_ANDROID
       auto *cb = +[](void *dtls_begin, void *dtls_end, uptr /*dso_idd*/,
                      void *arg) -> void {
         ScanRangeForPointers(reinterpret_cast<uptr>(dtls_begin),
@@ -366,7 +456,7 @@ static void ProcessThreads(SuspendedThreadsList const &suspended_threads,
       // thread is suspended in the middle of updating its DTLS. IOWs, we
       // could scan already freed memory. (probably fine for now)
       __libc_iterate_dynamic_tls(os_id, cb, frontier);
-#else
+#    else
       if (dtls && !DTLSInDestruction(dtls)) {
         ForEachDVT(dtls, [&](const DTLS::DTV &dtv, int id) {
           uptr dtls_beg = dtv.beg;
@@ -383,7 +473,7 @@ static void ProcessThreads(SuspendedThreadsList const &suspended_threads,
         // this and continue.
         LOG_THREADS("Thread %llu has DTLS under destruction.\n", os_id);
       }
-#endif
+#    endif
     }
   }
 
@@ -391,13 +481,14 @@ static void ProcessThreads(SuspendedThreadsList const &suspended_threads,
   ProcessThreadRegistry(frontier);
 }
 
-#endif  // SANITIZER_FUCHSIA
+#  endif  // SANITIZER_FUCHSIA
 
 void ScanRootRegion(Frontier *frontier, const RootRegion &root_region,
                     uptr region_begin, uptr region_end, bool is_readable) {
   uptr intersection_begin = Max(root_region.begin, region_begin);
   uptr intersection_end = Min(region_end, root_region.begin + root_region.size);
-  if (intersection_begin >= intersection_end) return;
+  if (intersection_begin >= intersection_end)
+    return;
   LOG_POINTERS("Root region %p-%p intersects with mapped region %p-%p (%s)\n",
                (void *)root_region.begin,
                (void *)(root_region.begin + root_region.size),
@@ -420,7 +511,8 @@ static void ProcessRootRegion(Frontier *frontier,
 
 // Scans root regions for heap pointers.
 static void ProcessRootRegions(Frontier *frontier) {
-  if (!flags()->use_root_regions) return;
+  if (!flags()->use_root_regions)
+    return;
   for (uptr i = 0; i < root_regions.size(); i++)
     ProcessRootRegion(frontier, root_regions[i]);
 }
@@ -477,68 +569,6 @@ static void CollectIgnoredCb(uptr chunk, void *arg) {
   }
 }
 
-static uptr GetCallerPC(const StackTrace &stack) {
-  // The top frame is our malloc/calloc/etc. The next frame is the caller.
-  if (stack.size >= 2)
-    return stack.trace[1];
-  return 0;
-}
-
-struct InvalidPCParam {
-  Frontier *frontier;
-  bool skip_linker_allocations;
-};
-
-// ForEachChunk callback. If the caller pc is invalid or is within the linker,
-// mark as reachable. Called by ProcessPlatformSpecificAllocations.
-static void MarkInvalidPCCb(uptr chunk, void *arg) {
-  CHECK(arg);
-  InvalidPCParam *param = reinterpret_cast<InvalidPCParam *>(arg);
-  chunk = GetUserBegin(chunk);
-  LsanMetadata m(chunk);
-  if (m.allocated() && m.tag() != kReachable && m.tag() != kIgnored) {
-    u32 stack_id = m.stack_trace_id();
-    uptr caller_pc = 0;
-    if (stack_id > 0)
-      caller_pc = GetCallerPC(StackDepotGet(stack_id));
-    // If caller_pc is unknown, this chunk may be allocated in a coroutine. Mark
-    // it as reachable, as we can't properly report its allocation stack anyway.
-    if (caller_pc == 0 || (param->skip_linker_allocations &&
-                           GetLinker()->containsAddress(caller_pc))) {
-      m.set_tag(kReachable);
-      param->frontier->push_back(chunk);
-    }
-  }
-}
-
-// On Linux, treats all chunks allocated from ld-linux.so as reachable, which
-// covers dynamically allocated TLS blocks, internal dynamic loader's loaded
-// modules accounting etc.
-// Dynamic TLS blocks contain the TLS variables of dynamically loaded modules.
-// They are allocated with a __libc_memalign() call in allocate_and_init()
-// (elf/dl-tls.c). Glibc won't tell us the address ranges occupied by those
-// blocks, but we can make sure they come from our own allocator by intercepting
-// __libc_memalign(). On top of that, there is no easy way to reach them. Their
-// addresses are stored in a dynamically allocated array (the DTV) which is
-// referenced from the static TLS. Unfortunately, we can't just rely on the DTV
-// being reachable from the static TLS, and the dynamic TLS being reachable from
-// the DTV. This is because the initial DTV is allocated before our interception
-// mechanism kicks in, and thus we don't recognize it as allocated memory. We
-// can't special-case it either, since we don't know its size.
-// Our solution is to include in the root set all allocations made from
-// ld-linux.so (which is where allocate_and_init() is implemented). This is
-// guaranteed to include all dynamic TLS blocks (and possibly other allocations
-// which we don't care about).
-// On all other platforms, this simply checks to ensure that the caller pc is
-// valid before reporting chunks as leaked.
-static void ProcessPC(Frontier *frontier) {
-  InvalidPCParam arg;
-  arg.frontier = frontier;
-  arg.skip_linker_allocations =
-      flags()->use_tls && flags()->use_ld_allocations && GetLinker() != nullptr;
-  ForEachChunk(MarkInvalidPCCb, &arg);
-}
-
 // Sets the appropriate tag on each chunk.
 static void ClassifyAllChunks(SuspendedThreadsList const &suspended_threads,
                               Frontier *frontier) {
@@ -554,9 +584,6 @@ static void ClassifyAllChunks(SuspendedThreadsList const &suspended_threads,
   ProcessRootRegions(frontier);
   FloodFillTag(frontier, kReachable);
 
-  CHECK_EQ(0, frontier->size());
-  ProcessPC(frontier);
-
   // The check here is relatively expensive, so we do this in a separate flood
   // fill. That way we can skip the check for chunks that are reachable
   // otherwise.
@@ -583,14 +610,13 @@ static void ResetTagsCb(uptr chunk, void *arg) {
 // a LeakReport.
 static void CollectLeaksCb(uptr chunk, void *arg) {
   CHECK(arg);
-  LeakReport *leak_report = reinterpret_cast<LeakReport *>(arg);
+  LeakedChunks *leaks = reinterpret_cast<LeakedChunks *>(arg);
   chunk = GetUserBegin(chunk);
   LsanMetadata m(chunk);
-  if (!m.allocated()) return;
-  if (m.tag() == kDirectlyLeaked || m.tag() == kIndirectlyLeaked) {
-    leak_report->AddLeakedChunk(chunk, m.stack_trace_id(), m.requested_size(),
-                                m.tag());
-  }
+  if (!m.allocated())
+    return;
+  if (m.tag() == kDirectlyLeaked || m.tag() == kIndirectlyLeaked)
+    leaks->push_back({chunk, m.stack_trace_id(), m.requested_size(), m.tag()});
 }
 
 void LeakSuppressionContext::PrintMatchedSuppressions() {
@@ -622,13 +648,13 @@ static void ReportIfNotSuspended(ThreadContextBase *tctx, void *arg) {
   }
 }
 
-#if SANITIZER_FUCHSIA
+#  if SANITIZER_FUCHSIA
 
 // Fuchsia provides a libc interface that guarantees all threads are
 // covered, and SuspendedThreadList is never really used.
 static void ReportUnsuspendedThreads(const SuspendedThreadsList &) {}
 
-#else  // !SANITIZER_FUCHSIA
+#  else  // !SANITIZER_FUCHSIA
 
 static void ReportUnsuspendedThreads(
     const SuspendedThreadsList &suspended_threads) {
@@ -642,7 +668,7 @@ static void ReportUnsuspendedThreads(
       &ReportIfNotSuspended, &threads);
 }
 
-#endif  // !SANITIZER_FUCHSIA
+#  endif  // !SANITIZER_FUCHSIA
 
 static void CheckForLeaksCallback(const SuspendedThreadsList &suspended_threads,
                                   void *arg) {
@@ -651,7 +677,7 @@ static void CheckForLeaksCallback(const SuspendedThreadsList &suspended_threads,
   CHECK(!param->success);
   ReportUnsuspendedThreads(suspended_threads);
   ClassifyAllChunks(suspended_threads, &param->frontier);
-  ForEachChunk(CollectLeaksCb, &param->leak_report);
+  ForEachChunk(CollectLeaksCb, &param->leaks);
   // Clean up for subsequent leak checks. This assumes we did not overwrite any
   // kIgnored tags.
   ForEachChunk(ResetTagsCb, nullptr);
@@ -700,17 +726,20 @@ static bool CheckForLeaks() {
           "etc)\n");
       Die();
     }
+    LeakReport leak_report;
+    leak_report.AddLeakedChunks(param.leaks);
+
     // No new suppressions stacks, so rerun will not help and we can report.
-    if (!param.leak_report.ApplySuppressions())
-      return PrintResults(param.leak_report);
+    if (!leak_report.ApplySuppressions())
+      return PrintResults(leak_report);
 
     // No indirect leaks to report, so we are done here.
-    if (!param.leak_report.IndirectUnsuppressedLeakCount())
-      return PrintResults(param.leak_report);
+    if (!leak_report.IndirectUnsuppressedLeakCount())
+      return PrintResults(leak_report);
 
     if (i >= 8) {
       Report("WARNING: LeakSanitizer gave up on indirect leaks suppression.\n");
-      return PrintResults(param.leak_report);
+      return PrintResults(leak_report);
     }
 
     // We found a new previously unseen suppressed call stack. Rerun to make
@@ -726,10 +755,12 @@ bool HasReportedLeaks() { return has_reported_leaks; }
 void DoLeakCheck() {
   Lock l(&global_mutex);
   static bool already_done;
-  if (already_done) return;
+  if (already_done)
+    return;
   already_done = true;
   has_reported_leaks = CheckForLeaks();
-  if (has_reported_leaks) HandleLeaks();
+  if (has_reported_leaks)
+    HandleLeaks();
 }
 
 static int DoRecoverableLeakCheck() {
@@ -740,80 +771,50 @@ static int DoRecoverableLeakCheck() {
 
 void DoRecoverableLeakCheckVoid() { DoRecoverableLeakCheck(); }
 
-Suppression *LeakSuppressionContext::GetSuppressionForAddr(uptr addr) {
-  Suppression *s = nullptr;
-
-  // Suppress by module name.
-  if (const char *module_name =
-          Symbolizer::GetOrInit()->GetModuleNameForPc(addr))
-    if (context.Match(module_name, kSuppressionLeak, &s))
-      return s;
-
-  // Suppress by file or function name.
-  SymbolizedStack *frames = Symbolizer::GetOrInit()->SymbolizePC(addr);
-  for (SymbolizedStack *cur = frames; cur; cur = cur->next) {
-    if (context.Match(cur->info.function, kSuppressionLeak, &s) ||
-        context.Match(cur->info.file, kSuppressionLeak, &s)) {
-      break;
-    }
-  }
-  frames->ClearAll();
-  return s;
-}
-
-Suppression *LeakSuppressionContext::GetSuppressionForStack(
-    u32 stack_trace_id, const StackTrace &stack) {
-  LazyInit();
-  for (uptr i = 0; i < stack.size; i++) {
-    Suppression *s = GetSuppressionForAddr(
-        StackTrace::GetPreviousInstructionPc(stack.trace[i]));
-    if (s) {
-      suppressed_stacks_sorted = false;
-      suppressed_stacks.push_back(stack_trace_id);
-      return s;
-    }
-  }
-  return nullptr;
-}
-
 ///// LeakReport implementation. /////
 
 // A hard limit on the number of distinct leaks, to avoid quadratic complexity
 // in LeakReport::AddLeakedChunk(). We don't expect to ever see this many leaks
 // in real-world applications.
-// FIXME: Get rid of this limit by changing the implementation of LeakReport to
-// use a hash table.
+// FIXME: Get rid of this limit by moving logic into DedupLeaks.
 const uptr kMaxLeaksConsidered = 5000;
 
-void LeakReport::AddLeakedChunk(uptr chunk, u32 stack_trace_id,
-                                uptr leaked_size, ChunkTag tag) {
-  CHECK(tag == kDirectlyLeaked || tag == kIndirectlyLeaked);
-
-  if (u32 resolution = flags()->resolution) {
-    StackTrace stack = StackDepotGet(stack_trace_id);
-    stack.size = Min(stack.size, resolution);
-    stack_trace_id = StackDepotPut(stack);
-  }
+void LeakReport::AddLeakedChunks(const LeakedChunks &chunks) {
+  for (const LeakedChunk &leak : chunks) {
+    uptr chunk = leak.chunk;
+    u32 stack_trace_id = leak.stack_trace_id;
+    uptr leaked_size = leak.leaked_size;
+    ChunkTag tag = leak.tag;
+    CHECK(tag == kDirectlyLeaked || tag == kIndirectlyLeaked);
+
+    if (u32 resolution = flags()->resolution) {
+      StackTrace stack = StackDepotGet(stack_trace_id);
+      stack.size = Min(stack.size, resolution);
+      stack_trace_id = StackDepotPut(stack);
+    }
 
-  bool is_directly_leaked = (tag == kDirectlyLeaked);
-  uptr i;
-  for (i = 0; i < leaks_.size(); i++) {
-    if (leaks_[i].stack_trace_id == stack_trace_id &&
-        leaks_[i].is_directly_leaked == is_directly_leaked) {
-      leaks_[i].hit_count++;
-      leaks_[i].total_size += leaked_size;
-      break;
+    bool is_directly_leaked = (tag == kDirectlyLeaked);
+    uptr i;
+    for (i = 0; i < leaks_.size(); i++) {
+      if (leaks_[i].stack_trace_id == stack_trace_id &&
+          leaks_[i].is_directly_leaked == is_directly_leaked) {
+        leaks_[i].hit_count++;
+        leaks_[i].total_size += leaked_size;
+        break;
+      }
+    }
+    if (i == leaks_.size()) {
+      if (leaks_.size() == kMaxLeaksConsidered)
+        return;
+      Leak leak = {next_id_++,         /* hit_count */ 1,
+                   leaked_size,        stack_trace_id,
+                   is_directly_leaked, /* is_suppressed */ false};
+      leaks_.push_back(leak);
+    }
+    if (flags()->report_objects) {
+      LeakedObject obj = {leaks_[i].id, chunk, leaked_size};
+      leaked_objects_.push_back(obj);
     }
-  }
-  if (i == leaks_.size()) {
-    if (leaks_.size() == kMaxLeaksConsidered) return;
-    Leak leak = { next_id_++, /* hit_count */ 1, leaked_size, stack_trace_id,
-                  is_directly_leaked, /* is_suppressed */ false };
-    leaks_.push_back(leak);
-  }
-  if (flags()->report_objects) {
-    LeakedObject obj = {leaks_[i].id, chunk, leaked_size};
-    leaked_objects_.push_back(obj);
   }
 }
 
@@ -828,9 +829,10 @@ void LeakReport::ReportTopLeaks(uptr num_leaks_to_report) {
   CHECK(leaks_.size() <= kMaxLeaksConsidered);
   Printf("\n");
   if (leaks_.size() == kMaxLeaksConsidered)
-    Printf("Too many leaks! Only the first %zu leaks encountered will be "
-           "reported.\n",
-           kMaxLeaksConsidered);
+    Printf(
+        "Too many leaks! Only the first %zu leaks encountered will be "
+        "reported.\n",
+        kMaxLeaksConsidered);
 
   uptr unsuppressed_count = UnsuppressedLeakCount();
   if (num_leaks_to_report > 0 && num_leaks_to_report < unsuppressed_count)
@@ -838,10 +840,12 @@ void LeakReport::ReportTopLeaks(uptr num_leaks_to_report) {
   Sort(leaks_.data(), leaks_.size(), &LeakComparator);
   uptr leaks_reported = 0;
   for (uptr i = 0; i < leaks_.size(); i++) {
-    if (leaks_[i].is_suppressed) continue;
+    if (leaks_[i].is_suppressed)
+      continue;
     PrintReportForLeak(i);
     leaks_reported++;
-    if (leaks_reported == num_leaks_to_report) break;
+    if (leaks_reported == num_leaks_to_report)
+      break;
   }
   if (leaks_reported < unsuppressed_count) {
     uptr remaining = unsuppressed_count - leaks_reported;
@@ -880,9 +884,10 @@ void LeakReport::PrintSummary() {
   CHECK(leaks_.size() <= kMaxLeaksConsidered);
   uptr bytes = 0, allocations = 0;
   for (uptr i = 0; i < leaks_.size(); i++) {
-      if (leaks_[i].is_suppressed) continue;
-      bytes += leaks_[i].total_size;
-      allocations += leaks_[i].hit_count;
+    if (leaks_[i].is_suppressed)
+      continue;
+    bytes += leaks_[i].total_size;
+    allocations += leaks_[i].hit_count;
   }
   InternalScopedString summary;
   summary.append("%zu byte(s) leaked in %zu allocation(s).", bytes,
@@ -894,12 +899,8 @@ uptr LeakReport::ApplySuppressions() {
   LeakSuppressionContext *suppressions = GetSuppressionContext();
   uptr new_suppressions = false;
   for (uptr i = 0; i < leaks_.size(); i++) {
-    Suppression *s = suppressions->GetSuppressionForStack(
-        leaks_[i].stack_trace_id, StackDepotGet(leaks_[i].stack_trace_id));
-    if (s) {
-      s->weight += leaks_[i].total_size;
-      atomic_store_relaxed(&s->hit_count, atomic_load_relaxed(&s->hit_count) +
-          leaks_[i].hit_count);
+    if (suppressions->Suppress(leaks_[i].stack_trace_id, leaks_[i].hit_count,
+                               leaks_[i].total_size)) {
       leaks_[i].is_suppressed = true;
       ++new_suppressions;
     }
@@ -910,7 +911,8 @@ uptr LeakReport::ApplySuppressions() {
 uptr LeakReport::UnsuppressedLeakCount() {
   uptr result = 0;
   for (uptr i = 0; i < leaks_.size(); i++)
-    if (!leaks_[i].is_suppressed) result++;
+    if (!leaks_[i].is_suppressed)
+      result++;
   return result;
 }
 
@@ -922,16 +924,16 @@ uptr LeakReport::IndirectUnsuppressedLeakCount() {
   return result;
 }
 
-} // namespace __lsan
-#else // CAN_SANITIZE_LEAKS
+}  // namespace __lsan
+#else   // CAN_SANITIZE_LEAKS
 namespace __lsan {
-void InitCommonLsan() { }
-void DoLeakCheck() { }
-void DoRecoverableLeakCheckVoid() { }
-void DisableInThisThread() { }
-void EnableInThisThread() { }
-}
-#endif // CAN_SANITIZE_LEAKS
+void InitCommonLsan() {}
+void DoLeakCheck() {}
+void DoRecoverableLeakCheckVoid() {}
+void DisableInThisThread() {}
+void EnableInThisThread() {}
+}  // namespace __lsan
+#endif  // CAN_SANITIZE_LEAKS
 
 using namespace __lsan;
 
@@ -948,11 +950,13 @@ void __lsan_ignore_object(const void *p) {
   if (res == kIgnoreObjectInvalid)
     VReport(1, "__lsan_ignore_object(): no heap object found at %p", p);
   if (res == kIgnoreObjectAlreadyIgnored)
-    VReport(1, "__lsan_ignore_object(): "
-           "heap object at %p is already being ignored\n", p);
+    VReport(1,
+            "__lsan_ignore_object(): "
+            "heap object at %p is already being ignored\n",
+            p);
   if (res == kIgnoreObjectSuccess)
     VReport(1, "__lsan_ignore_object(): ignoring heap object at %p\n", p);
-#endif // CAN_SANITIZE_LEAKS
+#endif  // CAN_SANITIZE_LEAKS
 }
 
 SANITIZER_INTERFACE_ATTRIBUTE
@@ -962,7 +966,7 @@ void __lsan_register_root_region(const void *begin, uptr size) {
   RootRegion region = {reinterpret_cast<uptr>(begin), size};
   root_regions.push_back(region);
   VReport(1, "Registered root region at %p of size %zu\n", begin, size);
-#endif // CAN_SANITIZE_LEAKS
+#endif  // CAN_SANITIZE_LEAKS
 }
 
 SANITIZER_INTERFACE_ATTRIBUTE
@@ -988,7 +992,7 @@ void __lsan_unregister_root_region(const void *begin, uptr size) {
         begin, size);
     Die();
   }
-#endif // CAN_SANITIZE_LEAKS
+#endif  // CAN_SANITIZE_LEAKS
 }
 
 SANITIZER_INTERFACE_ATTRIBUTE
@@ -1010,7 +1014,7 @@ void __lsan_do_leak_check() {
 #if CAN_SANITIZE_LEAKS
   if (common_flags()->detect_leaks)
     __lsan::DoLeakCheck();
-#endif // CAN_SANITIZE_LEAKS
+#endif  // CAN_SANITIZE_LEAKS
 }
 
 SANITIZER_INTERFACE_ATTRIBUTE
@@ -1018,7 +1022,7 @@ int __lsan_do_recoverable_leak_check() {
 #if CAN_SANITIZE_LEAKS
   if (common_flags()->detect_leaks)
     return __lsan::DoRecoverableLeakCheck();
-#endif // CAN_SANITIZE_LEAKS
+#endif  // CAN_SANITIZE_LEAKS
   return 0;
 }
 
@@ -1027,14 +1031,14 @@ SANITIZER_INTERFACE_WEAK_DEF(const char *, __lsan_default_options, void) {
 }
 
 #if !SANITIZER_SUPPORTS_WEAK_HOOKS
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-int __lsan_is_turned_off() {
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE int
+__lsan_is_turned_off() {
   return 0;
 }
 
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-const char *__lsan_default_suppressions() {
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE const char *
+__lsan_default_suppressions() {
   return "";
 }
 #endif
-} // extern "C"
+}  // extern "C"
diff --git a/compiler-rt/lib/lsan/lsan_common.h b/compiler-rt/lib/lsan/lsan_common.h
index f9b55e4e8006..61b64d4dc30f 100644
--- a/compiler-rt/lib/lsan/lsan_common.h
+++ b/compiler-rt/lib/lsan/lsan_common.h
@@ -33,21 +33,21 @@
 // Exclude leak-detection on arm32 for Android because `__aeabi_read_tp`
 // is missing. This caused a link error.
 #if SANITIZER_ANDROID && (__ANDROID_API__ < 28 || defined(__arm__))
-#define CAN_SANITIZE_LEAKS 0
+#  define CAN_SANITIZE_LEAKS 0
 #elif (SANITIZER_LINUX || SANITIZER_MAC) && (SANITIZER_WORDSIZE == 64) && \
     (defined(__x86_64__) || defined(__mips64) || defined(__aarch64__) ||  \
      defined(__powerpc64__) || defined(__s390x__))
-#define CAN_SANITIZE_LEAKS 1
+#  define CAN_SANITIZE_LEAKS 1
 #elif defined(__i386__) && (SANITIZER_LINUX || SANITIZER_MAC)
-#define CAN_SANITIZE_LEAKS 1
+#  define CAN_SANITIZE_LEAKS 1
 #elif defined(__arm__) && SANITIZER_LINUX
-#define CAN_SANITIZE_LEAKS 1
+#  define CAN_SANITIZE_LEAKS 1
 #elif SANITIZER_RISCV64 && SANITIZER_LINUX
-#define CAN_SANITIZE_LEAKS 1
+#  define CAN_SANITIZE_LEAKS 1
 #elif SANITIZER_NETBSD || SANITIZER_FUCHSIA
-#define CAN_SANITIZE_LEAKS 1
+#  define CAN_SANITIZE_LEAKS 1
 #else
-#define CAN_SANITIZE_LEAKS 0
+#  define CAN_SANITIZE_LEAKS 0
 #endif
 
 namespace __sanitizer {
@@ -82,6 +82,15 @@ extern Flags lsan_flags;
 inline Flags *flags() { return &lsan_flags; }
 void RegisterLsanFlags(FlagParser *parser, Flags *f);
 
+struct LeakedChunk {
+  uptr chunk;
+  u32 stack_trace_id;
+  uptr leaked_size;
+  ChunkTag tag;
+};
+
+using LeakedChunks = InternalMmapVector<LeakedChunk>;
+
 struct Leak {
   u32 id;
   uptr hit_count;
@@ -101,8 +110,7 @@ struct LeakedObject {
 class LeakReport {
  public:
   LeakReport() {}
-  void AddLeakedChunk(uptr chunk, u32 stack_trace_id, uptr leaked_size,
-                      ChunkTag tag);
+  void AddLeakedChunks(const LeakedChunks &chunks);
   void ReportTopLeaks(uptr max_leaks);
   void PrintSummary();
   uptr ApplySuppressions();
@@ -136,7 +144,7 @@ struct RootRegion {
 // threads and enumerating roots.
 struct CheckForLeaksParam {
   Frontier frontier;
-  LeakReport leak_report;
+  LeakedChunks leaks;
   bool success = false;
 };
 
@@ -224,6 +232,22 @@ bool WordIsPoisoned(uptr addr);
 // Wrappers for ThreadRegistry access.
 void LockThreadRegistry() NO_THREAD_SAFETY_ANALYSIS;
 void UnlockThreadRegistry() NO_THREAD_SAFETY_ANALYSIS;
+
+struct ScopedStopTheWorldLock {
+  ScopedStopTheWorldLock() {
+    LockThreadRegistry();
+    LockAllocator();
+  }
+
+  ~ScopedStopTheWorldLock() {
+    UnlockAllocator();
+    UnlockThreadRegistry();
+  }
+
+  ScopedStopTheWorldLock &operator=(const ScopedStopTheWorldLock &) = delete;
+  ScopedStopTheWorldLock(const ScopedStopTheWorldLock &) = delete;
+};
+
 ThreadRegistry *GetThreadRegistryLocked();
 bool GetThreadRangesLocked(tid_t os_id, uptr *stack_begin, uptr *stack_end,
                            uptr *tls_begin, uptr *tls_end, uptr *cache_begin,
diff --git a/compiler-rt/lib/lsan/lsan_common_fuchsia.cpp b/compiler-rt/lib/lsan/lsan_common_fuchsia.cpp
index 2d35fa5b1cff..e5e8cd2b4c23 100644
--- a/compiler-rt/lib/lsan/lsan_common_fuchsia.cpp
+++ b/compiler-rt/lib/lsan/lsan_common_fuchsia.cpp
@@ -58,8 +58,7 @@ int ExitHook(int status) {
 
 void LockStuffAndStopTheWorld(StopTheWorldCallback callback,
                               CheckForLeaksParam *argument) {
-  LockThreadRegistry();
-  LockAllocator();
+  ScopedStopTheWorldLock lock;
 
   struct Params {
     InternalMmapVector<uptr> allocator_caches;
@@ -149,9 +148,6 @@ void LockStuffAndStopTheWorld(StopTheWorldCallback callback,
         params->callback(SuspendedThreadsListFuchsia(), params->argument);
       },
       &params);
-
-  UnlockAllocator();
-  UnlockThreadRegistry();
 }
 
 }  // namespace __lsan
diff --git a/compiler-rt/lib/lsan/lsan_common_linux.cpp b/compiler-rt/lib/lsan/lsan_common_linux.cpp
index 3af586e220f6..692ad35169e1 100644
--- a/compiler-rt/lib/lsan/lsan_common_linux.cpp
+++ b/compiler-rt/lib/lsan/lsan_common_linux.cpp
@@ -122,12 +122,9 @@ void HandleLeaks() {
 
 static int LockStuffAndStopTheWorldCallback(struct dl_phdr_info *info,
                                             size_t size, void *data) {
-  LockThreadRegistry();
-  LockAllocator();
+  ScopedStopTheWorldLock lock;
   DoStopTheWorldParam *param = reinterpret_cast<DoStopTheWorldParam *>(data);
   StopTheWorld(param->callback, param->argument);
-  UnlockAllocator();
-  UnlockThreadRegistry();
   return 1;
 }
 
diff --git a/compiler-rt/lib/lsan/lsan_common_mac.cpp b/compiler-rt/lib/lsan/lsan_common_mac.cpp
index 4301dcc615d7..6e8a6dfe16b4 100644
--- a/compiler-rt/lib/lsan/lsan_common_mac.cpp
+++ b/compiler-rt/lib/lsan/lsan_common_mac.cpp
@@ -195,11 +195,8 @@ void HandleLeaks() {}
 
 void LockStuffAndStopTheWorld(StopTheWorldCallback callback,
                               CheckForLeaksParam *argument) {
-  LockThreadRegistry();
-  LockAllocator();
+  ScopedStopTheWorldLock lock;
   StopTheWorld(callback, argument);
-  UnlockAllocator();
-  UnlockThreadRegistry();
 }
 
 } // namespace __lsan
diff --git a/compiler-rt/lib/lsan/lsan_interceptors.cpp b/compiler-rt/lib/lsan/lsan_interceptors.cpp
index ee723f210c9d..205e85685a7f 100644
--- a/compiler-rt/lib/lsan/lsan_interceptors.cpp
+++ b/compiler-rt/lib/lsan/lsan_interceptors.cpp
@@ -479,6 +479,12 @@ INTERCEPTOR(int, pthread_create, void *th, void *attr,
   return res;
 }
 
+INTERCEPTOR(int, pthread_join, void *t, void **arg) {
+  return REAL(pthread_join)(t, arg);
+}
+
+DEFINE_REAL_PTHREAD_FUNCTIONS
+
 INTERCEPTOR(void, _exit, int status) {
   if (status == 0 && HasReportedLeaks()) status = common_flags()->exitcode;
   REAL(_exit)(status);
@@ -511,6 +517,7 @@ void InitializeInterceptors() {
   LSAN_MAYBE_INTERCEPT_MALLINFO;
   LSAN_MAYBE_INTERCEPT_MALLOPT;
   INTERCEPT_FUNCTION(pthread_create);
+  INTERCEPT_FUNCTION(pthread_join);
   INTERCEPT_FUNCTION(_exit);
 
   LSAN_MAYBE_INTERCEPT__LWP_EXIT;
diff --git a/compiler-rt/lib/memprof/memprof_allocator.cpp b/compiler-rt/lib/memprof/memprof_allocator.cpp
index 059ce283b8c9..adbdf365bc4c 100644
--- a/compiler-rt/lib/memprof/memprof_allocator.cpp
+++ b/compiler-rt/lib/memprof/memprof_allocator.cpp
@@ -218,7 +218,6 @@ struct Allocator {
   AllocatorCache fallback_allocator_cache;
 
   uptr max_user_defined_malloc_size;
-  atomic_uint8_t rss_limit_exceeded;
 
   // Holds the mapping of stack ids to MemInfoBlocks.
   MIBMapTy MIBMap;
@@ -301,20 +300,12 @@ struct Allocator {
                                        : kMaxAllowedMallocSize;
   }
 
-  bool RssLimitExceeded() {
-    return atomic_load(&rss_limit_exceeded, memory_order_relaxed);
-  }
-
-  void SetRssLimitExceeded(bool limit_exceeded) {
-    atomic_store(&rss_limit_exceeded, limit_exceeded, memory_order_relaxed);
-  }
-
   // -------------------- Allocation/Deallocation routines ---------------
   void *Allocate(uptr size, uptr alignment, BufferedStackTrace *stack,
                  AllocType alloc_type) {
     if (UNLIKELY(!memprof_inited))
       MemprofInitFromRtl();
-    if (RssLimitExceeded()) {
+    if (UNLIKELY(IsRssLimitExceeded())) {
       if (AllocatorMayReturnNull())
         return nullptr;
       ReportRssLimitExceeded(stack);
@@ -662,10 +653,6 @@ uptr memprof_malloc_usable_size(const void *ptr, uptr pc, uptr bp) {
   return usable_size;
 }
 
-void MemprofSoftRssLimitExceededCallback(bool limit_exceeded) {
-  instance.SetRssLimitExceeded(limit_exceeded);
-}
-
 } // namespace __memprof
 
 // ---------------------- Interface ---------------- {{{1
diff --git a/compiler-rt/lib/memprof/memprof_allocator.h b/compiler-rt/lib/memprof/memprof_allocator.h
index f1438baaa20e..001502cde08a 100644
--- a/compiler-rt/lib/memprof/memprof_allocator.h
+++ b/compiler-rt/lib/memprof/memprof_allocator.h
@@ -98,7 +98,6 @@ int memprof_posix_memalign(void **memptr, uptr alignment, uptr size,
 uptr memprof_malloc_usable_size(const void *ptr, uptr pc, uptr bp);
 
 void PrintInternalAllocatorStats();
-void MemprofSoftRssLimitExceededCallback(bool exceeded);
 
 } // namespace __memprof
 #endif // MEMPROF_ALLOCATOR_H
diff --git a/compiler-rt/lib/memprof/memprof_rtl.cpp b/compiler-rt/lib/memprof/memprof_rtl.cpp
index fb2ef37e51a2..c3d1c5f096fb 100644
--- a/compiler-rt/lib/memprof/memprof_rtl.cpp
+++ b/compiler-rt/lib/memprof/memprof_rtl.cpp
@@ -133,13 +133,6 @@ void PrintAddressSpaceLayout() {
   CHECK(SHADOW_SCALE >= 3 && SHADOW_SCALE <= 7);
 }
 
-static bool UNUSED __local_memprof_dyninit = [] {
-  MaybeStartBackgroudThread();
-  SetSoftRssLimitExceededCallback(MemprofSoftRssLimitExceededCallback);
-
-  return false;
-}();
-
 static void MemprofInitInternal() {
   if (LIKELY(memprof_inited))
     return;
diff --git a/compiler-rt/lib/msan/msan_allocator.cpp b/compiler-rt/lib/msan/msan_allocator.cpp
index a97bd8371e08..dc006457a59f 100644
--- a/compiler-rt/lib/msan/msan_allocator.cpp
+++ b/compiler-rt/lib/msan/msan_allocator.cpp
@@ -160,6 +160,11 @@ static void *MsanAllocate(StackTrace *stack, uptr size, uptr alignment,
     }
     ReportAllocationSizeTooBig(size, max_malloc_size, stack);
   }
+  if (UNLIKELY(IsRssLimitExceeded())) {
+    if (AllocatorMayReturnNull())
+      return nullptr;
+    ReportRssLimitExceeded(stack);
+  }
   MsanThread *t = GetCurrentThread();
   void *allocated;
   if (t) {
diff --git a/compiler-rt/lib/msan/msan_interceptors.cpp b/compiler-rt/lib/msan/msan_interceptors.cpp
index eaa3b3ae9404..df0cdecf79c7 100644
--- a/compiler-rt/lib/msan/msan_interceptors.cpp
+++ b/compiler-rt/lib/msan/msan_interceptors.cpp
@@ -1065,6 +1065,8 @@ INTERCEPTOR(int, pthread_join, void *th, void **retval) {
   return res;
 }
 
+DEFINE_REAL_PTHREAD_FUNCTIONS
+
 extern char *tzname[2];
 
 INTERCEPTOR(void, tzset, int fake) {
@@ -1705,6 +1707,7 @@ void InitializeInterceptors() {
 #else
   INTERCEPT_FUNCTION(pthread_create);
 #endif
+  INTERCEPT_FUNCTION(pthread_join);
   INTERCEPT_FUNCTION(pthread_key_create);
 
 #if SANITIZER_NETBSD
diff --git a/compiler-rt/lib/profile/InstrProfiling.c b/compiler-rt/lib/profile/InstrProfiling.c
index 6df65f66df73..7c1d357d96fe 100644
--- a/compiler-rt/lib/profile/InstrProfiling.c
+++ b/compiler-rt/lib/profile/InstrProfiling.c
@@ -38,7 +38,7 @@ __llvm_profile_get_num_padding_bytes(uint64_t SizeInBytes) {
 }
 
 COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_version(void) {
-  return __llvm_profile_raw_version;
+  return INSTR_PROF_RAW_VERSION_VAR;
 }
 
 COMPILER_RT_VISIBILITY void __llvm_profile_reset_counters(void) {
diff --git a/compiler-rt/lib/profile/InstrProfilingMerge.c b/compiler-rt/lib/profile/InstrProfilingMerge.c
index 80db2527461e..bf99521d4da7 100644
--- a/compiler-rt/lib/profile/InstrProfilingMerge.c
+++ b/compiler-rt/lib/profile/InstrProfilingMerge.c
@@ -95,6 +95,14 @@ static uintptr_t signextIfWin64(void *V) {
 COMPILER_RT_VISIBILITY
 int __llvm_profile_merge_from_buffer(const char *ProfileData,
                                      uint64_t ProfileSize) {
+  if (__llvm_profile_get_version() & VARIANT_MASK_DBG_CORRELATE) {
+    PROF_ERR(
+        "%s\n",
+        "Debug info correlation does not support profile merging at runtime. "
+        "Instead, merge raw profiles using the llvm-profdata tool.");
+    return 1;
+  }
+
   __llvm_profile_data *SrcDataStart, *SrcDataEnd, *SrcData, *DstData;
   __llvm_profile_header *Header = (__llvm_profile_header *)ProfileData;
   uint64_t *SrcCountersStart;
diff --git a/compiler-rt/lib/profile/InstrProfilingWriter.c b/compiler-rt/lib/profile/InstrProfilingWriter.c
index 9cb05570989d..e5c0dc1479d9 100644
--- a/compiler-rt/lib/profile/InstrProfilingWriter.c
+++ b/compiler-rt/lib/profile/InstrProfilingWriter.c
@@ -259,16 +259,19 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
                    const uint64_t *CountersBegin, const uint64_t *CountersEnd,
                    VPDataReaderType *VPDataReader, const char *NamesBegin,
                    const char *NamesEnd, int SkipNameDataWrite) {
+  int DebugInfoCorrelate =
+      (__llvm_profile_get_version() & VARIANT_MASK_DBG_CORRELATE) != 0ULL;
 
   /* Calculate size of sections. */
-  const uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
+  const uint64_t DataSize =
+      DebugInfoCorrelate ? 0 : __llvm_profile_get_data_size(DataBegin, DataEnd);
   const uint64_t CountersSize = CountersEnd - CountersBegin;
-  const uint64_t NamesSize = NamesEnd - NamesBegin;
+  const uint64_t NamesSize = DebugInfoCorrelate ? 0 : NamesEnd - NamesBegin;
 
   /* Create the header. */
   __llvm_profile_header Header;
 
-  if (!DataSize)
+  if (!DataSize && (!DebugInfoCorrelate || !CountersSize))
     return 0;
 
   /* Determine how much padding is needed before/after the counters and after
@@ -289,6 +292,12 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
   Header.CountersDelta = (uint32_t)Header.CountersDelta;
 #endif
 
+  /* The data and names sections are omitted in lightweight mode. */
+  if (DebugInfoCorrelate) {
+    Header.CountersDelta = 0;
+    Header.NamesDelta = 0;
+  }
+
   /* Write the profile header. */
   ProfDataIOVec IOVec[] = {{&Header, sizeof(__llvm_profile_header), 1, 0}};
   if (Writer->Write(Writer, IOVec, sizeof(IOVec) / sizeof(*IOVec)))
@@ -300,11 +309,13 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
 
   /* Write the profile data. */
   ProfDataIOVec IOVecData[] = {
-      {DataBegin, sizeof(__llvm_profile_data), DataSize, 0},
+      {DebugInfoCorrelate ? NULL : DataBegin, sizeof(__llvm_profile_data),
+       DataSize, 0},
       {NULL, sizeof(uint8_t), PaddingBytesBeforeCounters, 1},
       {CountersBegin, sizeof(uint64_t), CountersSize, 0},
       {NULL, sizeof(uint8_t), PaddingBytesAfterCounters, 1},
-      {SkipNameDataWrite ? NULL : NamesBegin, sizeof(uint8_t), NamesSize, 0},
+      {(SkipNameDataWrite || DebugInfoCorrelate) ? NULL : NamesBegin,
+       sizeof(uint8_t), NamesSize, 0},
       {NULL, sizeof(uint8_t), PaddingBytesAfterNames, 1}};
   if (Writer->Write(Writer, IOVecData, sizeof(IOVecData) / sizeof(*IOVecData)))
     return -1;
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cpp
index af0b0949a88e..c5a5fb7371dd 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cpp
@@ -17,6 +17,7 @@
 #include "sanitizer_allocator_internal.h"
 #include "sanitizer_atomic.h"
 #include "sanitizer_common.h"
+#include "sanitizer_platform.h"
 
 namespace __sanitizer {
 
@@ -195,4 +196,14 @@ void PrintHintAllocatorCannotReturnNull() {
          "allocator_may_return_null=1\n");
 }
 
+static atomic_uint8_t rss_limit_exceeded;
+
+bool IsRssLimitExceeded() {
+  return atomic_load(&rss_limit_exceeded, memory_order_relaxed);
+}
+
+void SetRssLimitExceeded(bool limit_exceeded) {
+  atomic_store(&rss_limit_exceeded, limit_exceeded, memory_order_relaxed);
+}
+
 } // namespace __sanitizer
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h b/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h
index ec23465d9584..76b936ff5eaa 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h
@@ -70,6 +70,9 @@ inline void RandomShuffle(T *a, u32 n, u32 *rand_state) {
 #include "sanitizer_allocator_secondary.h"
 #include "sanitizer_allocator_combined.h"
 
+bool IsRssLimitExceeded();
+void SetRssLimitExceeded(bool limit_exceeded);
+
 } // namespace __sanitizer
 
 #endif // SANITIZER_ALLOCATOR_H
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_common.cpp
index 5fae8e33b905..e9379b7bdc96 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_common.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_common.cpp
@@ -138,9 +138,17 @@ void LoadedModule::set(const char *module_name, uptr base_address,
   set(module_name, base_address);
   arch_ = arch;
   internal_memcpy(uuid_, uuid, sizeof(uuid_));
+  uuid_size_ = kModuleUUIDSize;
   instrumented_ = instrumented;
 }
 
+void LoadedModule::setUuid(const char *uuid, uptr size) {
+  if (size > kModuleUUIDSize)
+    size = kModuleUUIDSize;
+  internal_memcpy(uuid_, uuid, size);
+  uuid_size_ = size;
+}
+
 void LoadedModule::clear() {
   InternalFree(full_name_);
   base_address_ = 0;
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common.h b/compiler-rt/lib/sanitizer_common/sanitizer_common.h
index 6ec6bb4bd856..9ddb099a8dbc 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_common.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_common.h
@@ -326,12 +326,6 @@ void SetUserDieCallback(DieCallbackType callback);
 
 void SetCheckUnwindCallback(void (*callback)());
 
-// Callback will be called if soft_rss_limit_mb is given and the limit is
-// exceeded (exceeded==true) or if rss went down below the limit
-// (exceeded==false).
-// The callback should be registered once at the tool init time.
-void SetSoftRssLimitExceededCallback(void (*Callback)(bool exceeded));
-
 // Functions related to signal handling.
 typedef void (*SignalHandlerType)(int, void *, void *);
 HandleSignalMode GetHandleSignalMode(int signum);
@@ -673,11 +667,9 @@ void Sort(T *v, uptr size, Compare comp = {}) {
 
 // Works like std::lower_bound: finds the first element that is not less
 // than the val.
-template <class Container,
+template <class Container, class T,
           class Compare = CompareLess<typename Container::value_type>>
-uptr InternalLowerBound(const Container &v,
-                        const typename Container::value_type &val,
-                        Compare comp = {}) {
+uptr InternalLowerBound(const Container &v, const T &val, Compare comp = {}) {
   uptr first = 0;
   uptr last = v.size();
   while (last > first) {
@@ -778,7 +770,7 @@ inline const char *ModuleArchToString(ModuleArch arch) {
   return "";
 }
 
-const uptr kModuleUUIDSize = 16;
+const uptr kModuleUUIDSize = 32;
 const uptr kMaxSegName = 16;
 
 // Represents a binary loaded into virtual memory (e.g. this can be an
@@ -790,6 +782,7 @@ class LoadedModule {
         base_address_(0),
         max_executable_address_(0),
         arch_(kModuleArchUnknown),
+        uuid_size_(0),
         instrumented_(false) {
     internal_memset(uuid_, 0, kModuleUUIDSize);
     ranges_.clear();
@@ -797,6 +790,7 @@ class LoadedModule {
   void set(const char *module_name, uptr base_address);
   void set(const char *module_name, uptr base_address, ModuleArch arch,
            u8 uuid[kModuleUUIDSize], bool instrumented);
+  void setUuid(const char *uuid, uptr size);
   void clear();
   void addAddressRange(uptr beg, uptr end, bool executable, bool writable,
                        const char *name = nullptr);
@@ -807,6 +801,7 @@ class LoadedModule {
   uptr max_executable_address() const { return max_executable_address_; }
   ModuleArch arch() const { return arch_; }
   const u8 *uuid() const { return uuid_; }
+  uptr uuid_size() const { return uuid_size_; }
   bool instrumented() const { return instrumented_; }
 
   struct AddressRange {
@@ -835,6 +830,7 @@ class LoadedModule {
   uptr base_address_;
   uptr max_executable_address_;
   ModuleArch arch_;
+  uptr uuid_size_;
   u8 uuid_[kModuleUUIDSize];
   bool instrumented_;
   IntrusiveList<AddressRange> ranges_;
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
index d219734fa0a3..b0ab08dff1db 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
@@ -7857,12 +7857,12 @@ INTERCEPTOR(void, setbuf, __sanitizer_FILE *stream, char *buf) {
       unpoison_file(stream);
 }
 
-INTERCEPTOR(void, setbuffer, __sanitizer_FILE *stream, char *buf, int mode) {
+INTERCEPTOR(void, setbuffer, __sanitizer_FILE *stream, char *buf, SIZE_T size) {
   void *ctx;
-  COMMON_INTERCEPTOR_ENTER(ctx, setbuffer, stream, buf, mode);
-  REAL(setbuffer)(stream, buf, mode);
+  COMMON_INTERCEPTOR_ENTER(ctx, setbuffer, stream, buf, size);
+  REAL(setbuffer)(stream, buf, size);
   if (buf) {
-    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, __sanitizer_bufsiz);
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, size);
   }
   if (stream)
     unpoison_file(stream);
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common_interface_posix.inc b/compiler-rt/lib/sanitizer_common/sanitizer_common_interface_posix.inc
index 38f9531148d4..a5259be9335a 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_common_interface_posix.inc
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_common_interface_posix.inc
@@ -11,3 +11,5 @@ INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_code)
 INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_data)
 INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_demangle)
 INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_flush)
+INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_set_demangle)
+INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_set_inline_frames)
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cpp
index bc4b477e350f..c4cc0e45193e 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cpp
@@ -10,25 +10,21 @@
 // run-time libraries.
 //===----------------------------------------------------------------------===//
 
+#include "sanitizer_allocator.h"
 #include "sanitizer_allocator_interface.h"
 #include "sanitizer_common.h"
 #include "sanitizer_flags.h"
 #include "sanitizer_procmaps.h"
-
+#include "sanitizer_stackdepot.h"
 
 namespace __sanitizer {
 
-static void (*SoftRssLimitExceededCallback)(bool exceeded);
-void SetSoftRssLimitExceededCallback(void (*Callback)(bool exceeded)) {
-  CHECK_EQ(SoftRssLimitExceededCallback, nullptr);
-  SoftRssLimitExceededCallback = Callback;
-}
-
 #if (SANITIZER_LINUX || SANITIZER_NETBSD) && !SANITIZER_GO
 // Weak default implementation for when sanitizer_stackdepot is not linked in.
 SANITIZER_WEAK_ATTRIBUTE StackDepotStats StackDepotGetStats() { return {}; }
 
 void *BackgroundThread(void *arg) {
+  VPrintf(1, "%s: Started BackgroundThread\n", SanitizerToolName);
   const uptr hard_rss_limit_mb = common_flags()->hard_rss_limit_mb;
   const uptr soft_rss_limit_mb = common_flags()->soft_rss_limit_mb;
   const bool heap_profile = common_flags()->heap_profile;
@@ -66,13 +62,11 @@ void *BackgroundThread(void *arg) {
         reached_soft_rss_limit = true;
         Report("%s: soft rss limit exhausted (%zdMb vs %zdMb)\n",
                SanitizerToolName, soft_rss_limit_mb, current_rss_mb);
-        if (SoftRssLimitExceededCallback)
-          SoftRssLimitExceededCallback(true);
+        SetRssLimitExceeded(true);
       } else if (soft_rss_limit_mb >= current_rss_mb &&
                  reached_soft_rss_limit) {
         reached_soft_rss_limit = false;
-        if (SoftRssLimitExceededCallback)
-          SoftRssLimitExceededCallback(false);
+        SetRssLimitExceeded(false);
       }
     }
     if (heap_profile &&
@@ -83,6 +77,38 @@ void *BackgroundThread(void *arg) {
     }
   }
 }
+
+void MaybeStartBackgroudThread() {
+  // Need to implement/test on other platforms.
+  // Start the background thread if one of the rss limits is given.
+  if (!common_flags()->hard_rss_limit_mb &&
+      !common_flags()->soft_rss_limit_mb &&
+      !common_flags()->heap_profile) return;
+  if (!&real_pthread_create) {
+    VPrintf(1, "%s: real_pthread_create undefined\n", SanitizerToolName);
+    return;  // Can't spawn the thread anyway.
+  }
+
+  static bool started = false;
+  if (!started) {
+    started = true;
+    internal_start_thread(BackgroundThread, nullptr);
+  }
+}
+
+#  if !SANITIZER_START_BACKGROUND_THREAD_IN_ASAN_INTERNAL
+#    pragma clang diagnostic push
+// We avoid global-constructors to be sure that globals are ready when
+// sanitizers need them. This can happend before global constructors executed.
+// Here we don't mind if thread is started on later stages.
+#    pragma clang diagnostic ignored "-Wglobal-constructors"
+static struct BackgroudThreadStarted {
+  BackgroudThreadStarted() { MaybeStartBackgroudThread(); }
+} background_thread_strarter UNUSED;
+#    pragma clang diagnostic pop
+#  endif
+#else
+void MaybeStartBackgroudThread() {}
 #endif
 
 void WriteToSyslog(const char *msg) {
@@ -105,18 +131,6 @@ void WriteToSyslog(const char *msg) {
     WriteOneLineToSyslog(p);
 }
 
-void MaybeStartBackgroudThread() {
-#if (SANITIZER_LINUX || SANITIZER_NETBSD) && \
-    !SANITIZER_GO  // Need to implement/test on other platforms.
-  // Start the background thread if one of the rss limits is given.
-  if (!common_flags()->hard_rss_limit_mb &&
-      !common_flags()->soft_rss_limit_mb &&
-      !common_flags()->heap_profile) return;
-  if (!&real_pthread_create) return;  // Can't spawn the thread anyway.
-  internal_start_thread(BackgroundThread, nullptr);
-#endif
-}
-
 static void (*sandboxing_callback)();
 void SetSandboxingCallback(void (*f)()) {
   sandboxing_callback = f;
@@ -185,10 +199,22 @@ void ProtectGap(uptr addr, uptr size, uptr zero_base_shadow_start,
 
 #endif  // !SANITIZER_FUCHSIA
 
+#if !SANITIZER_WINDOWS && !SANITIZER_GO
+// Weak default implementation for when sanitizer_stackdepot is not linked in.
+SANITIZER_WEAK_ATTRIBUTE void StackDepotStopBackgroundThread() {}
+static void StopStackDepotBackgroundThread() {
+  StackDepotStopBackgroundThread();
+}
+#else
+// SANITIZER_WEAK_ATTRIBUTE is unsupported.
+static void StopStackDepotBackgroundThread() {}
+#endif
+
 }  // namespace __sanitizer
 
 SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_sandbox_on_notify,
                              __sanitizer_sandbox_arguments *args) {
+  __sanitizer::StopStackDepotBackgroundThread();
   __sanitizer::PlatformPrepareForSandboxing(args);
   if (__sanitizer::sandboxing_callback)
     __sanitizer::sandboxing_callback();
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc b/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc
index 95da82b1a1da..0ca91aff8dd4 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc
@@ -179,6 +179,7 @@ COMMON_FLAG(bool, use_madv_dontdump, true,
           "in core file.")
 COMMON_FLAG(bool, symbolize_inline_frames, true,
             "Print inlined frames in stacktraces. Defaults to true.")
+COMMON_FLAG(bool, demangle, true, "Print demangled symbols.")
 COMMON_FLAG(bool, symbolize_vs_style, false,
             "Print file locations in Visual Studio style (e.g: "
             " file(10,42): ...")
@@ -191,6 +192,8 @@ COMMON_FLAG(const char *, stack_trace_format, "DEFAULT",
             "Format string used to render stack frames. "
             "See sanitizer_stacktrace_printer.h for the format description. "
             "Use DEFAULT to get default format.")
+COMMON_FLAG(int, compress_stack_depot, 0,
+            "Compress stack depot to save memory.")
 COMMON_FLAG(bool, no_huge_pages_for_shadow, true,
             "If true, the shadow is not allowed to use huge pages. ")
 COMMON_FLAG(bool, strict_string_checks, false,
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cpp
index 9b5f6f1da1a1..66a0fd64a05a 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cpp
@@ -275,11 +275,11 @@ void *MmapFixedNoAccess(uptr fixed_addr, uptr size, const char *name) {
 }
 
 bool MprotectNoAccess(uptr addr, uptr size) {
-  return _zx_vmar_protect(_zx_vmar_root_self(), 0, Addr, Size) == ZX_OK;
+  return _zx_vmar_protect(_zx_vmar_root_self(), 0, addr, size) == ZX_OK;
 }
 
 bool MprotectReadOnly(uptr addr, uptr size) {
-  return _zx_vmar_protect(_zx_vmar_root_self(), ZX_VM_PERM_READ, Addr, Size) ==
+  return _zx_vmar_protect(_zx_vmar_root_self(), ZX_VM_PERM_READ, addr, size) ==
          ZX_OK;
 }
 
@@ -484,6 +484,9 @@ u32 GetNumberOfCPUs() { return zx_system_get_num_cpus(); }
 
 uptr GetRSS() { UNIMPLEMENTED(); }
 
+void *internal_start_thread(void *(*func)(void *arg), void *arg) { return 0; }
+void internal_join_thread(void *th) {}
+
 void InitializePlatformCommonFlags(CommonFlags *cf) {}
 
 }  // namespace __sanitizer
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp
index 2d787332a445..a92ea01ccccc 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp
@@ -163,6 +163,12 @@ ScopedBlockSignals::ScopedBlockSignals(__sanitizer_sigset_t *copy) {
   // See test/sanitizer_common/TestCases/Linux/setuid.c.
   internal_sigdelset(&set, 33);
 #  endif
+#  if SANITIZER_LINUX
+  // Seccomp-BPF-sandboxed processes rely on SIGSYS to handle trapped syscalls.
+  // If this signal is blocked, such calls cannot be handled and the process may
+  // hang.
+  internal_sigdelset(&set, 31);
+#  endif
   SetSigProcMask(&set, &saved_);
   if (copy)
     internal_memcpy(copy, &saved_, sizeof(saved_));
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_linux.h b/compiler-rt/lib/sanitizer_common/sanitizer_linux.h
index 6a235db0ee2e..ebd60e0b10f2 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_linux.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_linux.h
@@ -55,6 +55,9 @@ struct ScopedBlockSignals {
   explicit ScopedBlockSignals(__sanitizer_sigset_t *copy);
   ~ScopedBlockSignals();
 
+  ScopedBlockSignals &operator=(const ScopedBlockSignals &) = delete;
+  ScopedBlockSignals(const ScopedBlockSignals &) = delete;
+
  private:
   __sanitizer_sigset_t saved_;
 };
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp
index 7ce9e25da342..3c15c35cf488 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp
@@ -603,6 +603,32 @@ static int AddModuleSegments(const char *module_name, dl_phdr_info *info,
       bool writable = phdr->p_flags & PF_W;
       cur_module.addAddressRange(cur_beg, cur_end, executable,
                                  writable);
+    } else if (phdr->p_type == PT_NOTE) {
+      uptr off = 0;
+      while (off < phdr->p_memsz - sizeof(ElfW(Nhdr))) {
+        auto *nhdr = reinterpret_cast<const ElfW(Nhdr) *>(info->dlpi_addr +
+                                                          phdr->p_vaddr + off);
+        constexpr auto kGnuNamesz = 4;  // "GNU" with NUL-byte.
+        static_assert(kGnuNamesz % 4 == 0, "kGnuNameSize is aligned to 4.");
+        if (nhdr->n_type == NT_GNU_BUILD_ID && nhdr->n_namesz == kGnuNamesz) {
+          if (off + sizeof(ElfW(Nhdr)) + nhdr->n_namesz + nhdr->n_descsz >
+              phdr->p_memsz) {
+            // Something is very wrong, bail out instead of reading potentially
+            // arbitrary memory.
+            break;
+          }
+          const char *name =
+              reinterpret_cast<const char *>(nhdr) + sizeof(*nhdr);
+          if (internal_memcmp(name, "GNU", 3) == 0) {
+            const char *value = reinterpret_cast<const char *>(nhdr) +
+                                sizeof(*nhdr) + kGnuNamesz;
+            cur_module.setUuid(value, nhdr->n_descsz);
+            break;
+          }
+        }
+        off += sizeof(*nhdr) + RoundUpTo(nhdr->n_namesz, 4) +
+               RoundUpTo(nhdr->n_descsz, 4);
+      }
     }
   }
   modules->push_back(cur_module);
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_lzw.h b/compiler-rt/lib/sanitizer_common/sanitizer_lzw.h
new file mode 100644
index 000000000000..42acfbdcea09
--- /dev/null
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_lzw.h
@@ -0,0 +1,159 @@
+//===-- sanitizer_lzw.h -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Lempel–Ziv–Welch encoding/decoding
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_LZW_H
+#define SANITIZER_LZW_H
+
+#include "sanitizer_dense_map.h"
+
+namespace __sanitizer {
+
+using LzwCodeType = u32;
+
+template <class T, class ItIn, class ItOut>
+ItOut LzwEncode(ItIn begin, ItIn end, ItOut out) {
+  using Substring =
+      detail::DenseMapPair<LzwCodeType /* Prefix */, T /* Next input */>;
+
+  // Sentinel value for substrings of len 1.
+  static constexpr LzwCodeType kNoPrefix =
+      Min(DenseMapInfo<Substring>::getEmptyKey().first,
+          DenseMapInfo<Substring>::getTombstoneKey().first) -
+      1;
+  DenseMap<Substring, LzwCodeType> prefix_to_code;
+  {
+    // Add all substring of len 1 as initial dictionary.
+    InternalMmapVector<T> dict_len1;
+    for (auto it = begin; it != end; ++it)
+      if (prefix_to_code.try_emplace({kNoPrefix, *it}, 0).second)
+        dict_len1.push_back(*it);
+
+    // Slightly helps with later delta encoding.
+    Sort(dict_len1.data(), dict_len1.size());
+
+    // For large sizeof(T) we have to store dict_len1. Smaller types like u8 can
+    // just generate them.
+    *out = dict_len1.size();
+    ++out;
+
+    for (uptr i = 0; i != dict_len1.size(); ++i) {
+      // Remap after the Sort.
+      prefix_to_code[{kNoPrefix, dict_len1[i]}] = i;
+      *out = dict_len1[i];
+      ++out;
+    }
+    CHECK_EQ(prefix_to_code.size(), dict_len1.size());
+  }
+
+  if (begin == end)
+    return out;
+
+  // Main LZW encoding loop.
+  LzwCodeType match = prefix_to_code.find({kNoPrefix, *begin})->second;
+  ++begin;
+  for (auto it = begin; it != end; ++it) {
+    // Extend match with the new item.
+    auto ins = prefix_to_code.try_emplace({match, *it}, prefix_to_code.size());
+    if (ins.second) {
+      // This is a new substring, but emit the code for the current match
+      // (before extend). This allows LZW decoder to recover the dictionary.
+      *out = match;
+      ++out;
+      // Reset the match to a single item, which must be already in the map.
+      match = prefix_to_code.find({kNoPrefix, *it})->second;
+    } else {
+      // Already known, use as the current match.
+      match = ins.first->second;
+    }
+  }
+
+  *out = match;
+  ++out;
+
+  return out;
+}
+
+template <class T, class ItIn, class ItOut>
+ItOut LzwDecode(ItIn begin, ItIn end, ItOut out) {
+  if (begin == end)
+    return out;
+
+  // Load dictionary of len 1 substrings. Theses correspont to lowest codes.
+  InternalMmapVector<T> dict_len1(*begin);
+  ++begin;
+
+  if (begin == end)
+    return out;
+
+  for (auto& v : dict_len1) {
+    v = *begin;
+    ++begin;
+  }
+
+  // Substrings of len 2 and up. Indexes are shifted because [0,
+  // dict_len1.size()) stored in dict_len1. Substings get here after being
+  // emitted to the output, so we can use output position.
+  InternalMmapVector<detail::DenseMapPair<ItOut /* begin. */, ItOut /* end */>>
+      code_to_substr;
+
+  // Copies already emitted substrings into the output again.
+  auto copy = [&code_to_substr, &dict_len1](LzwCodeType code, ItOut out) {
+    if (code < dict_len1.size()) {
+      *out = dict_len1[code];
+      ++out;
+      return out;
+    }
+    const auto& s = code_to_substr[code - dict_len1.size()];
+
+    for (ItOut it = s.first; it != s.second; ++it, ++out) *out = *it;
+    return out;
+  };
+
+  // Returns lens of the substring with the given code.
+  auto code_to_len = [&code_to_substr, &dict_len1](LzwCodeType code) -> uptr {
+    if (code < dict_len1.size())
+      return 1;
+    const auto& s = code_to_substr[code - dict_len1.size()];
+    return s.second - s.first;
+  };
+
+  // Main LZW decoding loop.
+  LzwCodeType prev_code = *begin;
+  ++begin;
+  out = copy(prev_code, out);
+  for (auto it = begin; it != end; ++it) {
+    LzwCodeType code = *it;
+    auto start = out;
+    if (code == dict_len1.size() + code_to_substr.size()) {
+      // Special LZW case. The code is not in the dictionary yet. This is
+      // possible only when the new substring is the same as previous one plus
+      // the first item of the previous substring. We can emit that in two
+      // steps.
+      out = copy(prev_code, out);
+      *out = *start;
+      ++out;
+    } else {
+      out = copy(code, out);
+    }
+
+    // Every time encoded emits the code, it also creates substing of len + 1
+    // including the first item of the just emmited substring. Do the same here.
+    uptr len = code_to_len(prev_code);
+    code_to_substr.push_back({start - len, start + 1});
+
+    prev_code = code;
+  }
+  return out;
+}
+
+}  // namespace __sanitizer
+#endif
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
index f9b5c531aeee..a2fc310ad1a2 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
@@ -888,7 +888,7 @@ bool SignalContext::IsTrueFaultingAddress() const {
     (uptr)ptrauth_strip(     \
         (void *)arm_thread_state64_get_##r(ucontext->uc_mcontext->__ss), 0)
 #else
-  #define AARCH64_GET_REG(r) ucontext->uc_mcontext->__ss.__##r
+  #define AARCH64_GET_REG(r) (uptr)ucontext->uc_mcontext->__ss.__##r
 #endif
 
 static void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) {
@@ -1223,7 +1223,7 @@ uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,
 
   uptr largest_gap_found = 0;
   uptr max_occupied_addr = 0;
-  VReport(2, "FindDynamicShadowStart, space_size = %p\n", space_size);
+  VReport(2, "FindDynamicShadowStart, space_size = %p\n", (void *)space_size);
   uptr shadow_start =
       FindAvailableMemoryRange(space_size, alignment, granularity,
                                &largest_gap_found, &max_occupied_addr);
@@ -1232,20 +1232,21 @@ uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,
     VReport(
         2,
         "Shadow doesn't fit, largest_gap_found = %p, max_occupied_addr = %p\n",
-        largest_gap_found, max_occupied_addr);
+        (void *)largest_gap_found, (void *)max_occupied_addr);
     uptr new_max_vm = RoundDownTo(largest_gap_found << shadow_scale, alignment);
     if (new_max_vm < max_occupied_addr) {
       Report("Unable to find a memory range for dynamic shadow.\n");
       Report(
           "space_size = %p, largest_gap_found = %p, max_occupied_addr = %p, "
           "new_max_vm = %p\n",
-          space_size, largest_gap_found, max_occupied_addr, new_max_vm);
+          (void *)space_size, (void *)largest_gap_found,
+          (void *)max_occupied_addr, (void *)new_max_vm);
       CHECK(0 && "cannot place shadow");
     }
     RestrictMemoryToMaxAddress(new_max_vm);
     high_mem_end = new_max_vm - 1;
     space_size = (high_mem_end >> shadow_scale) + left_padding;
-    VReport(2, "FindDynamicShadowStart, space_size = %p\n", space_size);
+    VReport(2, "FindDynamicShadowStart, space_size = %p\n", (void *)space_size);
     shadow_start = FindAvailableMemoryRange(space_size, alignment, granularity,
                                             nullptr, nullptr);
     if (shadow_start == 0) {
@@ -1325,7 +1326,7 @@ void SignalContext::DumpAllRegisters(void *context) {
 # define DUMPREG64(r) \
     Printf("%s = 0x%016llx  ", #r, ucontext->uc_mcontext->__ss.__ ## r);
 # define DUMPREGA64(r) \
-    Printf("   %s = 0x%016llx  ", #r, AARCH64_GET_REG(r));
+    Printf("   %s = 0x%016lx  ", #r, AARCH64_GET_REG(r));
 # define DUMPREG32(r) \
     Printf("%s = 0x%08x  ", #r, ucontext->uc_mcontext->__ss.__ ## r);
 # define DUMPREG_(r)   Printf(" "); DUMPREG(r);
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform.h b/compiler-rt/lib/sanitizer_common/sanitizer_platform.h
index 3153de34e5a3..8de765cf6669 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_platform.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform.h
@@ -22,103 +22,103 @@
 // function declarations into a .S file which doesn't compile.
 // https://crbug.com/1162741
 #if __has_include(<features.h>) && !defined(__ANDROID__)
-#include <features.h>
+#  include <features.h>
 #endif
 
 #if defined(__linux__)
-# define SANITIZER_LINUX   1
+#  define SANITIZER_LINUX 1
 #else
-# define SANITIZER_LINUX   0
+#  define SANITIZER_LINUX 0
 #endif
 
 #if defined(__GLIBC__)
-# define SANITIZER_GLIBC   1
+#  define SANITIZER_GLIBC 1
 #else
-# define SANITIZER_GLIBC   0
+#  define SANITIZER_GLIBC 0
 #endif
 
 #if defined(__FreeBSD__)
-# define SANITIZER_FREEBSD 1
+#  define SANITIZER_FREEBSD 1
 #else
-# define SANITIZER_FREEBSD 0
+#  define SANITIZER_FREEBSD 0
 #endif
 
 #if defined(__NetBSD__)
-# define SANITIZER_NETBSD 1
+#  define SANITIZER_NETBSD 1
 #else
-# define SANITIZER_NETBSD 0
+#  define SANITIZER_NETBSD 0
 #endif
 
 #if defined(__sun__) && defined(__svr4__)
-# define SANITIZER_SOLARIS 1
+#  define SANITIZER_SOLARIS 1
 #else
-# define SANITIZER_SOLARIS 0
+#  define SANITIZER_SOLARIS 0
 #endif
 
 #if defined(__APPLE__)
-# define SANITIZER_MAC     1
-# include <TargetConditionals.h>
-# if TARGET_OS_OSX
-#  define SANITIZER_OSX    1
-# else
-#  define SANITIZER_OSX    0
-# endif
-# if TARGET_OS_IPHONE
-#  define SANITIZER_IOS    1
-# else
-#  define SANITIZER_IOS    0
-# endif
-# if TARGET_OS_SIMULATOR
-#  define SANITIZER_IOSSIM 1
-# else
-#  define SANITIZER_IOSSIM 0
-# endif
+#  define SANITIZER_MAC 1
+#  include <TargetConditionals.h>
+#  if TARGET_OS_OSX
+#    define SANITIZER_OSX 1
+#  else
+#    define SANITIZER_OSX 0
+#  endif
+#  if TARGET_OS_IPHONE
+#    define SANITIZER_IOS 1
+#  else
+#    define SANITIZER_IOS 0
+#  endif
+#  if TARGET_OS_SIMULATOR
+#    define SANITIZER_IOSSIM 1
+#  else
+#    define SANITIZER_IOSSIM 0
+#  endif
 #else
-# define SANITIZER_MAC     0
-# define SANITIZER_IOS     0
-# define SANITIZER_IOSSIM  0
-# define SANITIZER_OSX     0
+#  define SANITIZER_MAC 0
+#  define SANITIZER_IOS 0
+#  define SANITIZER_IOSSIM 0
+#  define SANITIZER_OSX 0
 #endif
 
 #if defined(__APPLE__) && TARGET_OS_IPHONE && TARGET_OS_WATCH
-# define SANITIZER_WATCHOS 1
+#  define SANITIZER_WATCHOS 1
 #else
-# define SANITIZER_WATCHOS 0
+#  define SANITIZER_WATCHOS 0
 #endif
 
 #if defined(__APPLE__) && TARGET_OS_IPHONE && TARGET_OS_TV
-# define SANITIZER_TVOS 1
+#  define SANITIZER_TVOS 1
 #else
-# define SANITIZER_TVOS 0
+#  define SANITIZER_TVOS 0
 #endif
 
 #if defined(_WIN32)
-# define SANITIZER_WINDOWS 1
+#  define SANITIZER_WINDOWS 1
 #else
-# define SANITIZER_WINDOWS 0
+#  define SANITIZER_WINDOWS 0
 #endif
 
 #if defined(_WIN64)
-# define SANITIZER_WINDOWS64 1
+#  define SANITIZER_WINDOWS64 1
 #else
-# define SANITIZER_WINDOWS64 0
+#  define SANITIZER_WINDOWS64 0
 #endif
 
 #if defined(__ANDROID__)
-# define SANITIZER_ANDROID 1
+#  define SANITIZER_ANDROID 1
 #else
-# define SANITIZER_ANDROID 0
+#  define SANITIZER_ANDROID 0
 #endif
 
 #if defined(__Fuchsia__)
-# define SANITIZER_FUCHSIA 1
+#  define SANITIZER_FUCHSIA 1
 #else
-# define SANITIZER_FUCHSIA 0
+#  define SANITIZER_FUCHSIA 0
 #endif
 
-#define SANITIZER_POSIX \
+#define SANITIZER_POSIX                                     \
   (SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC || \
-    SANITIZER_NETBSD || SANITIZER_SOLARIS)
+   SANITIZER_NETBSD || SANITIZER_SOLARIS)
 
 #if __LP64__ || defined(_WIN64)
 #  define SANITIZER_WORDSIZE 64
@@ -127,58 +127,64 @@
 #endif
 
 #if SANITIZER_WORDSIZE == 64
-# define FIRST_32_SECOND_64(a, b) (b)
+#  define FIRST_32_SECOND_64(a, b) (b)
 #else
-# define FIRST_32_SECOND_64(a, b) (a)
+#  define FIRST_32_SECOND_64(a, b) (a)
 #endif
 
 #if defined(__x86_64__) && !defined(_LP64)
-# define SANITIZER_X32 1
+#  define SANITIZER_X32 1
 #else
-# define SANITIZER_X32 0
+#  define SANITIZER_X32 0
+#endif
+
+#if defined(__x86_64__) || defined(_M_X64)
+#  define SANITIZER_X64 1
+#else
+#  define SANITIZER_X64 0
 #endif
 
 #if defined(__i386__) || defined(_M_IX86)
-# define SANITIZER_I386 1
+#  define SANITIZER_I386 1
 #else
-# define SANITIZER_I386 0
+#  define SANITIZER_I386 0
 #endif
 
 #if defined(__mips__)
-# define SANITIZER_MIPS 1
-# if defined(__mips64)
+#  define SANITIZER_MIPS 1
+#  if defined(__mips64)
+#    define SANITIZER_MIPS32 0
+#    define SANITIZER_MIPS64 1
+#  else
+#    define SANITIZER_MIPS32 1
+#    define SANITIZER_MIPS64 0
+#  endif
+#else
+#  define SANITIZER_MIPS 0
 #  define SANITIZER_MIPS32 0
-#  define SANITIZER_MIPS64 1
-# else
-#  define SANITIZER_MIPS32 1
 #  define SANITIZER_MIPS64 0
-# endif
-#else
-# define SANITIZER_MIPS 0
-# define SANITIZER_MIPS32 0
-# define SANITIZER_MIPS64 0
 #endif
 
 #if defined(__s390__)
-# define SANITIZER_S390 1
-# if defined(__s390x__)
+#  define SANITIZER_S390 1
+#  if defined(__s390x__)
+#    define SANITIZER_S390_31 0
+#    define SANITIZER_S390_64 1
+#  else
+#    define SANITIZER_S390_31 1
+#    define SANITIZER_S390_64 0
+#  endif
+#else
+#  define SANITIZER_S390 0
 #  define SANITIZER_S390_31 0
-#  define SANITIZER_S390_64 1
-# else
-#  define SANITIZER_S390_31 1
 #  define SANITIZER_S390_64 0
-# endif
-#else
-# define SANITIZER_S390 0
-# define SANITIZER_S390_31 0
-# define SANITIZER_S390_64 0
 #endif
 
 #if defined(__powerpc__)
-# define SANITIZER_PPC 1
-# if defined(__powerpc64__)
-#  define SANITIZER_PPC32 0
-#  define SANITIZER_PPC64 1
+#  define SANITIZER_PPC 1
+#  if defined(__powerpc64__)
+#    define SANITIZER_PPC32 0
+#    define SANITIZER_PPC64 1
 // 64-bit PPC has two ABIs (v1 and v2).  The old powerpc64 target is
 // big-endian, and uses v1 ABI (known for its function descriptors),
 // while the new powerpc64le target is little-endian and uses v2.
@@ -186,43 +192,49 @@
 // (eg. big-endian v2), but you won't find such combinations in the wild
 // (it'd require bootstrapping a whole system, which would be quite painful
 // - there's no target triple for that).  LLVM doesn't support them either.
-#  if _CALL_ELF == 2
-#   define SANITIZER_PPC64V1 0
-#   define SANITIZER_PPC64V2 1
+#    if _CALL_ELF == 2
+#      define SANITIZER_PPC64V1 0
+#      define SANITIZER_PPC64V2 1
+#    else
+#      define SANITIZER_PPC64V1 1
+#      define SANITIZER_PPC64V2 0
+#    endif
 #  else
-#   define SANITIZER_PPC64V1 1
-#   define SANITIZER_PPC64V2 0
+#    define SANITIZER_PPC32 1
+#    define SANITIZER_PPC64 0
+#    define SANITIZER_PPC64V1 0
+#    define SANITIZER_PPC64V2 0
 #  endif
-# else
-#  define SANITIZER_PPC32 1
+#else
+#  define SANITIZER_PPC 0
+#  define SANITIZER_PPC32 0
 #  define SANITIZER_PPC64 0
 #  define SANITIZER_PPC64V1 0
 #  define SANITIZER_PPC64V2 0
-# endif
+#endif
+
+#if defined(__arm__) || defined(_M_ARM)
+#  define SANITIZER_ARM 1
 #else
-# define SANITIZER_PPC 0
-# define SANITIZER_PPC32 0
-# define SANITIZER_PPC64 0
-# define SANITIZER_PPC64V1 0
-# define SANITIZER_PPC64V2 0
+#  define SANITIZER_ARM 0
 #endif
 
-#if defined(__arm__)
-# define SANITIZER_ARM 1
+#if defined(__aarch64__) || defined(_M_ARM64)
+#  define SANITIZER_ARM64 1
 #else
-# define SANITIZER_ARM 0
+#  define SANITIZER_ARM64 0
 #endif
 
 #if SANITIZER_SOLARIS && SANITIZER_WORDSIZE == 32
-# define SANITIZER_SOLARIS32 1
+#  define SANITIZER_SOLARIS32 1
 #else
-# define SANITIZER_SOLARIS32 0
+#  define SANITIZER_SOLARIS32 0
 #endif
 
 #if defined(__riscv) && (__riscv_xlen == 64)
-#define SANITIZER_RISCV64 1
+#  define SANITIZER_RISCV64 1
 #else
-#define SANITIZER_RISCV64 0
+#  define SANITIZER_RISCV64 0
 #endif
 
 // By default we allow to use SizeClassAllocator64 on 64-bit platform.
@@ -231,50 +243,52 @@
 // For such platforms build this code with -DSANITIZER_CAN_USE_ALLOCATOR64=0 or
 // change the definition of SANITIZER_CAN_USE_ALLOCATOR64 here.
 #ifndef SANITIZER_CAN_USE_ALLOCATOR64
-# if (SANITIZER_ANDROID && defined(__aarch64__)) || SANITIZER_FUCHSIA
-#  define SANITIZER_CAN_USE_ALLOCATOR64 1
-# elif defined(__mips64) || defined(__aarch64__)
-#  define SANITIZER_CAN_USE_ALLOCATOR64 0
-# else
-#  define SANITIZER_CAN_USE_ALLOCATOR64 (SANITIZER_WORDSIZE == 64)
-# endif
+#  if (SANITIZER_ANDROID && defined(__aarch64__)) || SANITIZER_FUCHSIA
+#    define SANITIZER_CAN_USE_ALLOCATOR64 1
+#  elif defined(__mips64) || defined(__aarch64__)
+#    define SANITIZER_CAN_USE_ALLOCATOR64 0
+#  else
+#    define SANITIZER_CAN_USE_ALLOCATOR64 (SANITIZER_WORDSIZE == 64)
+#  endif
 #endif
 
 // The range of addresses which can be returned my mmap.
 // FIXME: this value should be different on different platforms.  Larger values
 // will still work but will consume more memory for TwoLevelByteMap.
 #if defined(__mips__)
-#if SANITIZER_GO && defined(__mips64)
-#define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47)
-#else
-# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 40)
-#endif
+#  if SANITIZER_GO && defined(__mips64)
+#    define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47)
+#  else
+#    define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 40)
+#  endif
 #elif SANITIZER_RISCV64
-#define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 38)
+#  define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 38)
 #elif defined(__aarch64__)
-# if SANITIZER_MAC
-#  if SANITIZER_OSX || SANITIZER_IOSSIM
-#   define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47)
+#  if SANITIZER_MAC
+#    if SANITIZER_OSX || SANITIZER_IOSSIM
+#      define SANITIZER_MMAP_RANGE_SIZE \
+        FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47)
+#    else
+// Darwin iOS/ARM64 has a 36-bit VMA, 64GiB VM
+#      define SANITIZER_MMAP_RANGE_SIZE \
+        FIRST_32_SECOND_64(1ULL << 32, 1ULL << 36)
+#    endif
 #  else
-    // Darwin iOS/ARM64 has a 36-bit VMA, 64GiB VM
-#   define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 36)
+#    define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 48)
 #  endif
-# else
-#  define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 48)
-# endif
 #elif defined(__sparc__)
-#define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 52)
+#  define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 52)
 #else
-# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47)
+#  define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47)
 #endif
 
 // Whether the addresses are sign-extended from the VMA range to the word.
 // The SPARC64 Linux port implements this to split the VMA space into two
 // non-contiguous halves with a huge hole in the middle.
 #if defined(__sparc__) && SANITIZER_WORDSIZE == 64
-#define SANITIZER_SIGN_EXTENDED_ADDRESSES 1
+#  define SANITIZER_SIGN_EXTENDED_ADDRESSES 1
 #else
-#define SANITIZER_SIGN_EXTENDED_ADDRESSES 0
+#  define SANITIZER_SIGN_EXTENDED_ADDRESSES 0
 #endif
 
 // The AArch64 and RISC-V linux ports use the canonical syscall set as
@@ -297,15 +311,15 @@
 // Since we don't want to include libc headers here, we check the
 // target only.
 #if defined(__arm__) || SANITIZER_X32 || defined(__sparc__)
-#define SANITIZER_USES_UID16_SYSCALLS 1
+#  define SANITIZER_USES_UID16_SYSCALLS 1
 #else
-#define SANITIZER_USES_UID16_SYSCALLS 0
+#  define SANITIZER_USES_UID16_SYSCALLS 0
 #endif
 
 #if defined(__mips__)
-# define SANITIZER_POINTER_FORMAT_LENGTH FIRST_32_SECOND_64(8, 10)
+#  define SANITIZER_POINTER_FORMAT_LENGTH FIRST_32_SECOND_64(8, 10)
 #else
-# define SANITIZER_POINTER_FORMAT_LENGTH FIRST_32_SECOND_64(8, 12)
+#  define SANITIZER_POINTER_FORMAT_LENGTH FIRST_32_SECOND_64(8, 12)
 #endif
 
 /// \macro MSC_PREREQ
@@ -314,15 +328,15 @@
 ///  * 1800: Microsoft Visual Studio 2013 / 12.0
 ///  * 1900: Microsoft Visual Studio 2015 / 14.0
 #ifdef _MSC_VER
-# define MSC_PREREQ(version) (_MSC_VER >= (version))
+#  define MSC_PREREQ(version) (_MSC_VER >= (version))
 #else
-# define MSC_PREREQ(version) 0
+#  define MSC_PREREQ(version) 0
 #endif
 
 #if SANITIZER_MAC && !(defined(__arm64__) && SANITIZER_IOS)
-# define SANITIZER_NON_UNIQUE_TYPEINFO 0
+#  define SANITIZER_NON_UNIQUE_TYPEINFO 0
 #else
-# define SANITIZER_NON_UNIQUE_TYPEINFO 1
+#  define SANITIZER_NON_UNIQUE_TYPEINFO 1
 #endif
 
 // On linux, some architectures had an ABI transition from 64-bit long double
@@ -330,11 +344,11 @@
 // involving long doubles come in two versions, and we need to pass the
 // correct one to dlvsym when intercepting them.
 #if SANITIZER_LINUX && (SANITIZER_S390 || SANITIZER_PPC32 || SANITIZER_PPC64V1)
-#define SANITIZER_NLDBL_VERSION "GLIBC_2.4"
+#  define SANITIZER_NLDBL_VERSION "GLIBC_2.4"
 #endif
 
 #if SANITIZER_GO == 0
-# define SANITIZER_GO 0
+#  define SANITIZER_GO 0
 #endif
 
 // On PowerPC and ARM Thumb, calling pthread_exit() causes LSan to detect leaks.
@@ -342,40 +356,39 @@
 // dlopen mallocs "libgcc_s.so" string which confuses LSan, it fails to realize
 // that this allocation happens in dynamic linker and should be ignored.
 #if SANITIZER_PPC || defined(__thumb__)
-# define SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT 1
+#  define SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT 1
 #else
-# define SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT 0
+#  define SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT 0
 #endif
 
-#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD || \
-  SANITIZER_SOLARIS
-# define SANITIZER_MADVISE_DONTNEED MADV_FREE
+#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD || SANITIZER_SOLARIS
+#  define SANITIZER_MADVISE_DONTNEED MADV_FREE
 #else
-# define SANITIZER_MADVISE_DONTNEED MADV_DONTNEED
+#  define SANITIZER_MADVISE_DONTNEED MADV_DONTNEED
 #endif
 
 // Older gcc have issues aligning to a constexpr, and require an integer.
 // See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=56859 among others.
 #if defined(__powerpc__) || defined(__powerpc64__)
-# define SANITIZER_CACHE_LINE_SIZE 128
+#  define SANITIZER_CACHE_LINE_SIZE 128
 #else
-# define SANITIZER_CACHE_LINE_SIZE 64
+#  define SANITIZER_CACHE_LINE_SIZE 64
 #endif
 
 // Enable offline markup symbolizer for Fuchsia.
 #if SANITIZER_FUCHSIA
 #  define SANITIZER_SYMBOLIZER_MARKUP 1
 #else
-#define SANITIZER_SYMBOLIZER_MARKUP 0
+#  define SANITIZER_SYMBOLIZER_MARKUP 0
 #endif
 
 // Enable ability to support sanitizer initialization that is
 // compatible with the sanitizer library being loaded via
 // `dlopen()`.
 #if SANITIZER_MAC
-#define SANITIZER_SUPPORTS_INIT_FOR_DLOPEN 1
+#  define SANITIZER_SUPPORTS_INIT_FOR_DLOPEN 1
 #else
-#define SANITIZER_SUPPORTS_INIT_FOR_DLOPEN 0
+#  define SANITIZER_SUPPORTS_INIT_FOR_DLOPEN 0
 #endif
 
 // SANITIZER_SUPPORTS_THREADLOCAL
@@ -392,4 +405,15 @@
 #  endif
 #endif
 
-#endif // SANITIZER_PLATFORM_H
+#if defined(__thumb__) && defined(__linux__)
+// Workaround for
+// https://lab.llvm.org/buildbot/#/builders/clang-thumbv7-full-2stage
+// or
+// https://lab.llvm.org/staging/#/builders/clang-thumbv7-full-2stage
+// It fails *rss_limit_mb_test* without meaningful errors.
+#  define SANITIZER_START_BACKGROUND_THREAD_IN_ASAN_INTERNAL 1
+#else
+#  define SANITIZER_START_BACKGROUND_THREAD_IN_ASAN_INTERNAL 0
+#endif
+
+#endif  // SANITIZER_PLATFORM_H
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.cpp
index b1c15d8c2834..4791a3a35bdb 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.cpp
@@ -10,6 +10,10 @@
 
 #include "sanitizer_atomic.h"
 #include "sanitizer_common.h"
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_leb128.h"
+#include "sanitizer_lzw.h"
+#include "sanitizer_placement_new.h"
 #include "sanitizer_stacktrace.h"
 
 namespace __sanitizer {
@@ -52,7 +56,7 @@ StackTrace StackStore::Load(Id id) {
   uptr idx = IdToOffset(id);
   uptr block_idx = GetBlockIdx(idx);
   CHECK_LT(block_idx, ARRAY_SIZE(blocks_));
-  const uptr *stack_trace = blocks_[block_idx].GetOrUnpack();
+  const uptr *stack_trace = blocks_[block_idx].GetOrUnpack(this);
   if (!stack_trace)
     return {};
   stack_trace += GetInBlockIdx(idx);
@@ -61,11 +65,7 @@ StackTrace StackStore::Load(Id id) {
 }
 
 uptr StackStore::Allocated() const {
-  uptr next_block = GetBlockIdx(
-      RoundUpTo(atomic_load_relaxed(&total_frames_), kBlockSizeFrames));
-  uptr res = 0;
-  for (uptr i = 0; i < next_block; ++i) res += blocks_[i].Allocated();
-  return res + sizeof(*this);
+  return atomic_load_relaxed(&allocated_) + sizeof(*this);
 }
 
 uptr *StackStore::Alloc(uptr count, uptr *idx, uptr *pack) {
@@ -79,7 +79,7 @@ uptr *StackStore::Alloc(uptr count, uptr *idx, uptr *pack) {
       // Fits into the a single block.
       CHECK_LT(block_idx, ARRAY_SIZE(blocks_));
       *idx = start;
-      return blocks_[block_idx].GetOrCreate() + GetInBlockIdx(start);
+      return blocks_[block_idx].GetOrCreate(this) + GetInBlockIdx(start);
     }
 
     // Retry. We can't use range allocated in two different blocks.
@@ -92,43 +92,157 @@ uptr *StackStore::Alloc(uptr count, uptr *idx, uptr *pack) {
   }
 }
 
+void *StackStore::Map(uptr size, const char *mem_type) {
+  atomic_fetch_add(&allocated_, size, memory_order_relaxed);
+  return MmapNoReserveOrDie(size, mem_type);
+}
+
+void StackStore::Unmap(void *addr, uptr size) {
+  atomic_fetch_sub(&allocated_, size, memory_order_relaxed);
+  UnmapOrDie(addr, size);
+}
+
 uptr StackStore::Pack(Compression type) {
   uptr res = 0;
-  for (BlockInfo &b : blocks_) res += b.Pack(type);
+  for (BlockInfo &b : blocks_) res += b.Pack(type, this);
   return res;
 }
 
+void StackStore::LockAll() {
+  for (BlockInfo &b : blocks_) b.Lock();
+}
+
+void StackStore::UnlockAll() {
+  for (BlockInfo &b : blocks_) b.Unlock();
+}
+
 void StackStore::TestOnlyUnmap() {
-  for (BlockInfo &b : blocks_) b.TestOnlyUnmap();
+  for (BlockInfo &b : blocks_) b.TestOnlyUnmap(this);
   internal_memset(this, 0, sizeof(*this));
 }
 
 uptr *StackStore::BlockInfo::Get() const {
   // Idiomatic double-checked locking uses memory_order_acquire here. But
-  // relaxed is find for us, justification is similar to
+  // relaxed is fine for us, justification is similar to
   // TwoLevelMap::GetOrCreate.
   return reinterpret_cast<uptr *>(atomic_load_relaxed(&data_));
 }
 
-uptr *StackStore::BlockInfo::Create() {
+uptr *StackStore::BlockInfo::Create(StackStore *store) {
   SpinMutexLock l(&mtx_);
   uptr *ptr = Get();
   if (!ptr) {
-    ptr = reinterpret_cast<uptr *>(
-        MmapNoReserveOrDie(kBlockSizeBytes, "StackStore"));
+    ptr = reinterpret_cast<uptr *>(store->Map(kBlockSizeBytes, "StackStore"));
     atomic_store(&data_, reinterpret_cast<uptr>(ptr), memory_order_release);
   }
   return ptr;
 }
 
-uptr *StackStore::BlockInfo::GetOrCreate() {
+uptr *StackStore::BlockInfo::GetOrCreate(StackStore *store) {
   uptr *ptr = Get();
   if (LIKELY(ptr))
     return ptr;
-  return Create();
+  return Create(store);
+}
+
+class SLeb128Encoder {
+ public:
+  SLeb128Encoder(u8 *begin, u8 *end) : begin(begin), end(end) {}
+
+  bool operator==(const SLeb128Encoder &other) const {
+    return begin == other.begin;
+  }
+
+  bool operator!=(const SLeb128Encoder &other) const {
+    return begin != other.begin;
+  }
+
+  SLeb128Encoder &operator=(uptr v) {
+    sptr diff = v - previous;
+    begin = EncodeSLEB128(diff, begin, end);
+    previous = v;
+    return *this;
+  }
+  SLeb128Encoder &operator*() { return *this; }
+  SLeb128Encoder &operator++() { return *this; }
+
+  u8 *base() const { return begin; }
+
+ private:
+  u8 *begin;
+  u8 *end;
+  uptr previous = 0;
+};
+
+class SLeb128Decoder {
+ public:
+  SLeb128Decoder(const u8 *begin, const u8 *end) : begin(begin), end(end) {}
+
+  bool operator==(const SLeb128Decoder &other) const {
+    return begin == other.begin;
+  }
+
+  bool operator!=(const SLeb128Decoder &other) const {
+    return begin != other.begin;
+  }
+
+  uptr operator*() {
+    sptr diff;
+    begin = DecodeSLEB128(begin, end, &diff);
+    previous += diff;
+    return previous;
+  }
+  SLeb128Decoder &operator++() { return *this; }
+
+  SLeb128Decoder operator++(int) { return *this; }
+
+ private:
+  const u8 *begin;
+  const u8 *end;
+  uptr previous = 0;
+};
+
+static u8 *CompressDelta(const uptr *from, const uptr *from_end, u8 *to,
+                         u8 *to_end) {
+  SLeb128Encoder encoder(to, to_end);
+  for (; from != from_end; ++from, ++encoder) *encoder = *from;
+  return encoder.base();
 }
 
-uptr *StackStore::BlockInfo::GetOrUnpack() {
+static uptr *UncompressDelta(const u8 *from, const u8 *from_end, uptr *to,
+                             uptr *to_end) {
+  SLeb128Decoder decoder(from, from_end);
+  SLeb128Decoder end(from_end, from_end);
+  for (; decoder != end; ++to, ++decoder) *to = *decoder;
+  CHECK_EQ(to, to_end);
+  return to;
+}
+
+static u8 *CompressLzw(const uptr *from, const uptr *from_end, u8 *to,
+                       u8 *to_end) {
+  SLeb128Encoder encoder(to, to_end);
+  encoder = LzwEncode<uptr>(from, from_end, encoder);
+  return encoder.base();
+}
+
+static uptr *UncompressLzw(const u8 *from, const u8 *from_end, uptr *to,
+                           uptr *to_end) {
+  SLeb128Decoder decoder(from, from_end);
+  SLeb128Decoder end(from_end, from_end);
+  to = LzwDecode<uptr>(decoder, end, to);
+  CHECK_EQ(to, to_end);
+  return to;
+}
+
+namespace {
+struct PackedHeader {
+  uptr size;
+  StackStore::Compression type;
+  u8 data[];
+};
+}  // namespace
+
+uptr *StackStore::BlockInfo::GetOrUnpack(StackStore *store) {
   SpinMutexLock l(&mtx_);
   switch (state) {
     case State::Storing:
@@ -140,15 +254,43 @@ uptr *StackStore::BlockInfo::GetOrUnpack() {
       break;
   }
 
-  uptr *ptr = Get();
+  u8 *ptr = reinterpret_cast<u8 *>(Get());
   CHECK_NE(nullptr, ptr);
-  // Fake unpacking.
-  for (uptr i = 0; i < kBlockSizeFrames; ++i) ptr[i] = ~ptr[i];
+  const PackedHeader *header = reinterpret_cast<const PackedHeader *>(ptr);
+  CHECK_LE(header->size, kBlockSizeBytes);
+  CHECK_GE(header->size, sizeof(PackedHeader));
+
+  uptr packed_size_aligned = RoundUpTo(header->size, GetPageSizeCached());
+
+  uptr *unpacked =
+      reinterpret_cast<uptr *>(store->Map(kBlockSizeBytes, "StackStoreUnpack"));
+
+  uptr *unpacked_end;
+  switch (header->type) {
+    case Compression::Delta:
+      unpacked_end = UncompressDelta(header->data, ptr + header->size, unpacked,
+                                     unpacked + kBlockSizeFrames);
+      break;
+    case Compression::LZW:
+      unpacked_end = UncompressLzw(header->data, ptr + header->size, unpacked,
+                                   unpacked + kBlockSizeFrames);
+      break;
+    default:
+      UNREACHABLE("Unexpected type");
+      break;
+  }
+
+  CHECK_EQ(kBlockSizeFrames, unpacked_end - unpacked);
+
+  MprotectReadOnly(reinterpret_cast<uptr>(unpacked), kBlockSizeBytes);
+  atomic_store(&data_, reinterpret_cast<uptr>(unpacked), memory_order_release);
+  store->Unmap(ptr, packed_size_aligned);
+
   state = State::Unpacked;
   return Get();
 }
 
-uptr StackStore::BlockInfo::Pack(Compression type) {
+uptr StackStore::BlockInfo::Pack(Compression type, StackStore *store) {
   if (type == Compression::None)
     return 0;
 
@@ -165,26 +307,55 @@ uptr StackStore::BlockInfo::Pack(Compression type) {
   if (!ptr || !Stored(0))
     return 0;
 
-  // Fake packing.
-  for (uptr i = 0; i < kBlockSizeFrames; ++i) ptr[i] = ~ptr[i];
-  state = State::Packed;
-  return kBlockSizeBytes - kBlockSizeBytes / 10;
-}
+  u8 *packed =
+      reinterpret_cast<u8 *>(store->Map(kBlockSizeBytes, "StackStorePack"));
+  PackedHeader *header = reinterpret_cast<PackedHeader *>(packed);
+  u8 *alloc_end = packed + kBlockSizeBytes;
 
-uptr StackStore::BlockInfo::Allocated() const {
-  SpinMutexLock l(&mtx_);
-  switch (state) {
-    case State::Packed:
-      return kBlockSizeBytes / 10;
-    case State::Unpacked:
-    case State::Storing:
-      return kBlockSizeBytes;
+  u8 *packed_end = nullptr;
+  switch (type) {
+    case Compression::Delta:
+      packed_end =
+          CompressDelta(ptr, ptr + kBlockSizeFrames, header->data, alloc_end);
+      break;
+    case Compression::LZW:
+      packed_end =
+          CompressLzw(ptr, ptr + kBlockSizeFrames, header->data, alloc_end);
+      break;
+    default:
+      UNREACHABLE("Unexpected type");
+      break;
   }
+
+  header->type = type;
+  header->size = packed_end - packed;
+
+  VPrintf(1, "Packed block of %zu KiB to %zu KiB\n", kBlockSizeBytes >> 10,
+          header->size >> 10);
+
+  if (kBlockSizeBytes - header->size < kBlockSizeBytes / 8) {
+    VPrintf(1, "Undo and keep block unpacked\n");
+    MprotectReadOnly(reinterpret_cast<uptr>(ptr), kBlockSizeBytes);
+    store->Unmap(packed, kBlockSizeBytes);
+    state = State::Unpacked;
+    return 0;
+  }
+
+  uptr packed_size_aligned = RoundUpTo(header->size, GetPageSizeCached());
+  store->Unmap(packed + packed_size_aligned,
+               kBlockSizeBytes - packed_size_aligned);
+  MprotectReadOnly(reinterpret_cast<uptr>(packed), packed_size_aligned);
+
+  atomic_store(&data_, reinterpret_cast<uptr>(packed), memory_order_release);
+  store->Unmap(ptr, kBlockSizeBytes);
+
+  state = State::Packed;
+  return kBlockSizeBytes - packed_size_aligned;
 }
 
-void StackStore::BlockInfo::TestOnlyUnmap() {
+void StackStore::BlockInfo::TestOnlyUnmap(StackStore *store) {
   if (uptr *ptr = Get())
-    UnmapOrDie(ptr, StackStore::kBlockSizeBytes);
+    store->Unmap(ptr, kBlockSizeBytes);
 }
 
 bool StackStore::BlockInfo::Stored(uptr n) {
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.h b/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.h
index e0bc4e9c4a45..1bfad811f712 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_stack_store.h
@@ -25,7 +25,8 @@ class StackStore {
  public:
   enum class Compression : u8 {
     None = 0,
-    Test,
+    Delta,
+    LZW,
   };
 
   constexpr StackStore() = default;
@@ -45,6 +46,9 @@ class StackStore {
   // Returns the number of released bytes.
   uptr Pack(Compression type);
 
+  void LockAll();
+  void UnlockAll();
+
   void TestOnlyUnmap();
 
  private:
@@ -71,9 +75,15 @@ class StackStore {
 
   uptr *Alloc(uptr count, uptr *idx, uptr *pack);
 
+  void *Map(uptr size, const char *mem_type);
+  void Unmap(void *addr, uptr size);
+
   // Total number of allocated frames.
   atomic_uintptr_t total_frames_ = {};
 
+  // Tracks total allocated memory in bytes.
+  atomic_uintptr_t allocated_ = {};
+
   // Each block will hold pointer to exactly kBlockSizeFrames.
   class BlockInfo {
     atomic_uintptr_t data_;
@@ -89,17 +99,18 @@ class StackStore {
     };
     State state GUARDED_BY(mtx_);
 
-    uptr *Create();
+    uptr *Create(StackStore *store);
 
    public:
     uptr *Get() const;
-    uptr *GetOrCreate();
-    uptr *GetOrUnpack();
-    uptr Pack(Compression type);
-    uptr Allocated() const;
-    void TestOnlyUnmap();
+    uptr *GetOrCreate(StackStore *store);
+    uptr *GetOrUnpack(StackStore *store);
+    uptr Pack(Compression type, StackStore *store);
+    void TestOnlyUnmap(StackStore *store);
     bool Stored(uptr n);
     bool IsPacked() const;
+    void Lock() NO_THREAD_SAFETY_ANALYSIS { mtx_.Lock(); }
+    void Unlock() NO_THREAD_SAFETY_ANALYSIS { mtx_.Unlock(); }
   };
 
   BlockInfo blocks_[kBlockCount] = {};
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.cpp
index 527221b0c85c..c755b1829d2a 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.cpp
@@ -12,8 +12,10 @@
 
 #include "sanitizer_stackdepot.h"
 
+#include "sanitizer_atomic.h"
 #include "sanitizer_common.h"
 #include "sanitizer_hash.h"
+#include "sanitizer_mutex.h"
 #include "sanitizer_stack_store.h"
 #include "sanitizer_stackdepotbase.h"
 
@@ -72,12 +74,125 @@ uptr StackDepotNode::allocated() {
   return stackStore.Allocated() + useCounts.MemoryUsage();
 }
 
+static void CompressStackStore() {
+  u64 start = MonotonicNanoTime();
+  uptr diff = stackStore.Pack(static_cast<StackStore::Compression>(
+      Abs(common_flags()->compress_stack_depot)));
+  if (!diff)
+    return;
+  u64 finish = MonotonicNanoTime();
+  uptr total_before = theDepot.GetStats().allocated + diff;
+  VPrintf(1, "%s: StackDepot released %zu KiB out of %zu KiB in %llu ms\n",
+          SanitizerToolName, diff >> 10, total_before >> 10,
+          (finish - start) / 1000000);
+}
+
+namespace {
+
+class CompressThread {
+ public:
+  constexpr CompressThread() = default;
+  void NewWorkNotify();
+  void Stop();
+  void LockAndStop() NO_THREAD_SAFETY_ANALYSIS;
+  void Unlock() NO_THREAD_SAFETY_ANALYSIS;
+
+ private:
+  enum class State {
+    NotStarted = 0,
+    Started,
+    Failed,
+    Stopped,
+  };
+
+  void Run();
+
+  bool WaitForWork() {
+    semaphore_.Wait();
+    return atomic_load(&run_, memory_order_acquire);
+  }
+
+  Semaphore semaphore_ = {};
+  StaticSpinMutex mutex_ = {};
+  State state_ GUARDED_BY(mutex_) = State::NotStarted;
+  void *thread_ GUARDED_BY(mutex_) = nullptr;
+  atomic_uint8_t run_ = {};
+};
+
+static CompressThread compress_thread;
+
+void CompressThread::NewWorkNotify() {
+  int compress = common_flags()->compress_stack_depot;
+  if (!compress)
+    return;
+  if (compress > 0 /* for testing or debugging */) {
+    SpinMutexLock l(&mutex_);
+    if (state_ == State::NotStarted) {
+      atomic_store(&run_, 1, memory_order_release);
+      CHECK_EQ(nullptr, thread_);
+      thread_ = internal_start_thread(
+          [](void *arg) -> void * {
+            reinterpret_cast<CompressThread *>(arg)->Run();
+            return nullptr;
+          },
+          this);
+      state_ = thread_ ? State::Started : State::Failed;
+    }
+    if (state_ == State::Started) {
+      semaphore_.Post();
+      return;
+    }
+  }
+  CompressStackStore();
+}
+
+void CompressThread::Run() {
+  VPrintf(1, "%s: StackDepot compression thread started\n", SanitizerToolName);
+  while (WaitForWork()) CompressStackStore();
+  VPrintf(1, "%s: StackDepot compression thread stopped\n", SanitizerToolName);
+}
+
+void CompressThread::Stop() {
+  void *t = nullptr;
+  {
+    SpinMutexLock l(&mutex_);
+    if (state_ != State::Started)
+      return;
+    state_ = State::Stopped;
+    CHECK_NE(nullptr, thread_);
+    t = thread_;
+    thread_ = nullptr;
+  }
+  atomic_store(&run_, 0, memory_order_release);
+  semaphore_.Post();
+  internal_join_thread(t);
+}
+
+void CompressThread::LockAndStop() {
+  mutex_.Lock();
+  if (state_ != State::Started)
+    return;
+  CHECK_NE(nullptr, thread_);
+
+  atomic_store(&run_, 0, memory_order_release);
+  semaphore_.Post();
+  internal_join_thread(thread_);
+  // Allow to restart after Unlock() if needed.
+  state_ = State::NotStarted;
+  thread_ = nullptr;
+}
+
+void CompressThread::Unlock() { mutex_.Unlock(); }
+
+}  // namespace
+
 void StackDepotNode::store(u32 id, const args_type &args, hash_type hash) {
   stack_hash = hash;
   uptr pack = 0;
   store_id = stackStore.Store(args, &pack);
-  if (pack)
-    stackStore.Pack(StackStore::Compression::None);
+  if (LIKELY(!pack))
+    return;
+  compress_thread.NewWorkNotify();
 }
 
 StackDepotNode::args_type StackDepotNode::load(u32 id) const {
@@ -100,9 +215,13 @@ StackTrace StackDepotGet(u32 id) {
 
 void StackDepotLockAll() {
   theDepot.LockAll();
+  compress_thread.LockAndStop();
+  stackStore.LockAll();
 }
 
 void StackDepotUnlockAll() {
+  stackStore.UnlockAll();
+  compress_thread.Unlock();
   theDepot.UnlockAll();
 }
 
@@ -112,6 +231,8 @@ void StackDepotPrintAll() {
 #endif
 }
 
+void StackDepotStopBackgroundThread() { compress_thread.Stop(); }
+
 StackDepotHandle StackDepotNode::get_handle(u32 id) {
   return StackDepotHandle(&theDepot.nodes[id], id);
 }
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.h b/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.h
index 56d655d9404c..cca6fd534688 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_stackdepot.h
@@ -42,6 +42,7 @@ StackTrace StackDepotGet(u32 id);
 void StackDepotLockAll();
 void StackDepotUnlockAll();
 void StackDepotPrintAll();
+void StackDepotStopBackgroundThread();
 
 void StackDepotTestOnlyUnmap();
 
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cpp
index c6356dae23c1..2d0eccc1602a 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cpp
@@ -104,6 +104,19 @@ static const char *DemangleFunctionName(const char *function) {
   return function;
 }
 
+static void MaybeBuildIdToBuffer(const AddressInfo &info, bool PrefixSpace,
+                                 InternalScopedString *buffer) {
+  if (info.uuid_size) {
+    if (PrefixSpace)
+      buffer->append(" ");
+    buffer->append("(BuildId: ");
+    for (uptr i = 0; i < info.uuid_size; ++i) {
+      buffer->append("%02x", info.uuid[i]);
+    }
+    buffer->append(")");
+  }
+}
+
 static const char kDefaultFormat[] = "    #%n %p %F %L";
 
 void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no,
@@ -140,6 +153,9 @@ void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no,
     case 'o':
       buffer->append("0x%zx", info->module_offset);
       break;
+    case 'b':
+      MaybeBuildIdToBuffer(*info, /*PrefixSpace=*/false, buffer);
+      break;
     case 'f':
       buffer->append("%s", DemangleFunctionName(StripFunctionName(
                                info->function, strip_func_prefix)));
@@ -181,6 +197,8 @@ void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no,
       } else if (info->module) {
         RenderModuleLocation(buffer, info->module, info->module_offset,
                              info->module_arch, strip_path_prefix);
+
+        MaybeBuildIdToBuffer(*info, /*PrefixSpace=*/true, buffer);
       } else {
         buffer->append("(<unknown module>)");
       }
@@ -193,6 +211,7 @@ void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no,
         // Always strip the module name for %M.
         RenderModuleLocation(buffer, StripModuleName(info->module),
                              info->module_offset, info->module_arch, "");
+        MaybeBuildIdToBuffer(*info, /*PrefixSpace=*/true, buffer);
       } else {
         buffer->append("(%p)", (void *)address);
       }
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_win.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_win.cpp
new file mode 100644
index 000000000000..e12b9e5bee06
--- /dev/null
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_win.cpp
@@ -0,0 +1,175 @@
+//===-- sanitizer_stoptheworld_win.cpp ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// See sanitizer_stoptheworld.h for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+
+#if SANITIZER_WINDOWS
+
+#  define WIN32_LEAN_AND_MEAN
+#  include <windows.h>
+// windows.h needs to be included before tlhelp32.h
+#  include <tlhelp32.h>
+
+#  include "sanitizer_stoptheworld.h"
+
+namespace __sanitizer {
+
+namespace {
+
+struct SuspendedThreadsListWindows final : public SuspendedThreadsList {
+  InternalMmapVector<HANDLE> threadHandles;
+  InternalMmapVector<DWORD> threadIds;
+
+  SuspendedThreadsListWindows() {
+    threadIds.reserve(1024);
+    threadHandles.reserve(1024);
+  }
+
+  PtraceRegistersStatus GetRegistersAndSP(uptr index,
+                                          InternalMmapVector<uptr> *buffer,
+                                          uptr *sp) const override;
+
+  tid_t GetThreadID(uptr index) const override;
+  uptr ThreadCount() const override;
+};
+
+// Stack Pointer register names on different architectures
+#  if SANITIZER_X64
+#    define SP_REG Rsp
+#  elif SANITIZER_I386
+#    define SP_REG Esp
+#  elif SANITIZER_ARM | SANITIZER_ARM64
+#    define SP_REG Sp
+#  else
+#    error Architecture not supported!
+#  endif
+
+PtraceRegistersStatus SuspendedThreadsListWindows::GetRegistersAndSP(
+    uptr index, InternalMmapVector<uptr> *buffer, uptr *sp) const {
+  CHECK_LT(index, threadHandles.size());
+
+  buffer->resize(RoundUpTo(sizeof(CONTEXT), sizeof(uptr)) / sizeof(uptr));
+  CONTEXT *thread_context = reinterpret_cast<CONTEXT *>(buffer->data());
+  thread_context->ContextFlags = CONTEXT_ALL;
+  CHECK(GetThreadContext(threadHandles[index], thread_context));
+  *sp = thread_context->SP_REG;
+
+  return REGISTERS_AVAILABLE;
+}
+
+tid_t SuspendedThreadsListWindows::GetThreadID(uptr index) const {
+  CHECK_LT(index, threadIds.size());
+  return threadIds[index];
+}
+
+uptr SuspendedThreadsListWindows::ThreadCount() const {
+  return threadIds.size();
+}
+
+struct RunThreadArgs {
+  StopTheWorldCallback callback;
+  void *argument;
+};
+
+DWORD WINAPI RunThread(void *argument) {
+  RunThreadArgs *run_args = (RunThreadArgs *)argument;
+
+  const DWORD this_thread = GetCurrentThreadId();
+  const DWORD this_process = GetCurrentProcessId();
+
+  SuspendedThreadsListWindows suspended_threads_list;
+  bool new_thread_found;
+
+  do {
+    // Take a snapshot of all Threads
+    const HANDLE threads = CreateToolhelp32Snapshot(TH32CS_SNAPTHREAD, 0);
+    CHECK(threads != INVALID_HANDLE_VALUE);
+
+    THREADENTRY32 thread_entry;
+    thread_entry.dwSize = sizeof(thread_entry);
+    new_thread_found = false;
+
+    if (!Thread32First(threads, &thread_entry))
+      break;
+
+    do {
+      if (thread_entry.th32ThreadID == this_thread ||
+          thread_entry.th32OwnerProcessID != this_process)
+        continue;
+
+      bool suspended_thread = false;
+      for (const auto thread_id : suspended_threads_list.threadIds) {
+        if (thread_id == thread_entry.th32ThreadID) {
+          suspended_thread = true;
+          break;
+        }
+      }
+
+      // Skip the Thread if it was already suspended
+      if (suspended_thread)
+        continue;
+
+      const HANDLE thread =
+          OpenThread(THREAD_ALL_ACCESS, FALSE, thread_entry.th32ThreadID);
+      CHECK(thread);
+
+      if (SuspendThread(thread) == -1) {
+        DWORD last_error = GetLastError();
+
+        VPrintf(1, "Could not suspend thread %lu (error %lu)",
+                thread_entry.th32ThreadID, last_error);
+        continue;
+      }
+
+      suspended_threads_list.threadIds.push_back(thread_entry.th32ThreadID);
+      suspended_threads_list.threadHandles.push_back(thread);
+      new_thread_found = true;
+    } while (Thread32Next(threads, &thread_entry));
+
+    CloseHandle(threads);
+
+    // Between the call to `CreateToolhelp32Snapshot` and suspending the
+    // relevant Threads, new Threads could have potentially been created. So
+    // continue to find and suspend new Threads until we don't find any.
+  } while (new_thread_found);
+
+  // Now all Threads of this Process except of this Thread should be suspended.
+  // Execute the callback function.
+  run_args->callback(suspended_threads_list, run_args->argument);
+
+  // Resume all Threads
+  for (const auto suspended_thread_handle :
+       suspended_threads_list.threadHandles) {
+    CHECK_NE(ResumeThread(suspended_thread_handle), -1);
+    CloseHandle(suspended_thread_handle);
+  }
+
+  return 0;
+}
+
+}  // namespace
+
+void StopTheWorld(StopTheWorldCallback callback, void *argument) {
+  struct RunThreadArgs arg = {callback, argument};
+  DWORD trace_thread_id;
+
+  auto trace_thread =
+      CreateThread(nullptr, 0, RunThread, &arg, 0, &trace_thread_id);
+  CHECK(trace_thread);
+
+  WaitForSingleObject(trace_thread, INFINITE);
+  CloseHandle(trace_thread);
+}
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_WINDOWS
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.cpp
index 0c4b84c767aa..d3cffaa6eeff 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.cpp
@@ -11,10 +11,11 @@
 //===----------------------------------------------------------------------===//
 
 #include "sanitizer_allocator_internal.h"
-#include "sanitizer_platform.h"
+#include "sanitizer_common.h"
 #include "sanitizer_internal_defs.h"
 #include "sanitizer_libc.h"
 #include "sanitizer_placement_new.h"
+#include "sanitizer_platform.h"
 #include "sanitizer_symbolizer_internal.h"
 
 namespace __sanitizer {
@@ -30,6 +31,7 @@ void AddressInfo::Clear() {
   InternalFree(file);
   internal_memset(this, 0, sizeof(AddressInfo));
   function_offset = kUnknown;
+  uuid_size = 0;
 }
 
 void AddressInfo::FillModuleInfo(const char *mod_name, uptr mod_offset,
@@ -37,6 +39,16 @@ void AddressInfo::FillModuleInfo(const char *mod_name, uptr mod_offset,
   module = internal_strdup(mod_name);
   module_offset = mod_offset;
   module_arch = mod_arch;
+  uuid_size = 0;
+}
+
+void AddressInfo::FillModuleInfo(const LoadedModule &mod) {
+  module = internal_strdup(mod.full_name());
+  module_offset = address - mod.base_address();
+  module_arch = mod.arch();
+  if (mod.uuid_size())
+    internal_memcpy(uuid, mod.uuid(), mod.uuid_size());
+  uuid_size = mod.uuid_size();
 }
 
 SymbolizedStack::SymbolizedStack() : next(nullptr), info() {}
@@ -126,10 +138,4 @@ Symbolizer::SymbolizerScope::~SymbolizerScope() {
     sym_->end_hook_();
 }
 
-void Symbolizer::LateInitializeTools() {
-  for (auto &tool : tools_) {
-    tool.LateInitialize();
-  }
-}
-
 }  // namespace __sanitizer
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.h b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.h
index 42bd157fa627..bad4761e345f 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.h
@@ -32,6 +32,8 @@ struct AddressInfo {
   char *module;
   uptr module_offset;
   ModuleArch module_arch;
+  u8 uuid[kModuleUUIDSize];
+  uptr uuid_size;
 
   static const uptr kUnknown = ~(uptr)0;
   char *function;
@@ -45,6 +47,8 @@ struct AddressInfo {
   // Deletes all strings and resets all fields.
   void Clear();
   void FillModuleInfo(const char *mod_name, uptr mod_offset, ModuleArch arch);
+  void FillModuleInfo(const LoadedModule &mod);
+  uptr module_base() const { return address - module_offset; }
 };
 
 // Linked list of symbolized frames (each frame is described by AddressInfo).
@@ -209,9 +213,6 @@ class Symbolizer final {
    private:
     const Symbolizer *sym_;
   };
-
-  // Calls `LateInitialize()` on all items in `tools_`.
-  void LateInitializeTools();
 };
 
 #ifdef SANITIZER_WINDOWS
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_internal.h b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_internal.h
index b8670941a05e..df122ed3425c 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_internal.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_internal.h
@@ -70,11 +70,6 @@ class SymbolizerTool {
     return nullptr;
   }
 
-  // Called during the LateInitialize phase of Sanitizer initialization.
-  // Usually this is a safe place to call code that might need to use user
-  // memory allocators.
-  virtual void LateInitialize() {}
-
  protected:
   ~SymbolizerTool() {}
 };
@@ -91,7 +86,7 @@ class SymbolizerProcess {
   ~SymbolizerProcess() {}
 
   /// The maximum number of arguments required to invoke a tool process.
-  static const unsigned kArgVMax = 6;
+  static const unsigned kArgVMax = 16;
 
   // Customizable by subclasses.
   virtual bool StartSymbolizerSubprocess();
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cpp
index 3fc994fd3deb..8bbd4af0c7c2 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cpp
@@ -84,15 +84,12 @@ const char *ExtractTokenUpToDelimiter(const char *str, const char *delimiter,
 
 SymbolizedStack *Symbolizer::SymbolizePC(uptr addr) {
   Lock l(&mu_);
-  const char *module_name = nullptr;
-  uptr module_offset;
-  ModuleArch arch;
   SymbolizedStack *res = SymbolizedStack::New(addr);
-  if (!FindModuleNameAndOffsetForAddress(addr, &module_name, &module_offset,
-                                         &arch))
+  auto *mod = FindModuleForAddress(addr);
+  if (!mod)
     return res;
   // Always fill data about module name and offset.
-  res->info.FillModuleInfo(module_name, module_offset, arch);
+  res->info.FillModuleInfo(*mod);
   for (auto &tool : tools_) {
     SymbolizerScope sym_scope(this);
     if (tool.SymbolizePC(addr, res)) {
@@ -277,14 +274,17 @@ class LLVMSymbolizerProcess final : public SymbolizerProcess {
     const char* const kSymbolizerArch = "--default-arch=unknown";
 #endif
 
-    const char *const inline_flag = common_flags()->symbolize_inline_frames
-                                        ? "--inlines"
-                                        : "--no-inlines";
+    const char *const demangle_flag =
+        common_flags()->demangle ? "--demangle" : "--no-demangle";
+    const char *const inline_flag =
+        common_flags()->symbolize_inline_frames ? "--inlines" : "--no-inlines";
     int i = 0;
     argv[i++] = path_to_binary;
+    argv[i++] = demangle_flag;
     argv[i++] = inline_flag;
     argv[i++] = kSymbolizerArch;
     argv[i++] = nullptr;
+    CHECK_LE(i, kArgVMax);
   }
 };
 
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_mac.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_mac.cpp
index 5c25b28b5dc9..ac811c8a9136 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_mac.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_mac.cpp
@@ -20,7 +20,6 @@
 
 #include <dlfcn.h>
 #include <errno.h>
-#include <mach/mach.h>
 #include <stdlib.h>
 #include <sys/wait.h>
 #include <unistd.h>
@@ -58,13 +57,6 @@ bool DlAddrSymbolizer::SymbolizeData(uptr addr, DataInfo *datainfo) {
   return true;
 }
 
-#define K_ATOS_ENV_VAR "__check_mach_ports_lookup"
-
-// This cannot live in `AtosSymbolizerProcess` because instances of that object
-// are allocated by the internal allocator which under ASan is poisoned with
-// kAsanInternalHeapMagic.
-static char kAtosMachPortEnvEntry[] = K_ATOS_ENV_VAR "=000000000000000";
-
 class AtosSymbolizerProcess final : public SymbolizerProcess {
  public:
   explicit AtosSymbolizerProcess(const char *path)
@@ -72,51 +64,13 @@ class AtosSymbolizerProcess final : public SymbolizerProcess {
     pid_str_[0] = '\0';
   }
 
-  void LateInitialize() {
-    if (SANITIZER_IOSSIM) {
-      // `putenv()` may call malloc/realloc so it is only safe to do this
-      // during LateInitialize() or later (i.e. we can't do this in the
-      // constructor).  We also can't do this in `StartSymbolizerSubprocess()`
-      // because in TSan we switch allocators when we're symbolizing.
-      // We use `putenv()` rather than `setenv()` so that we can later directly
-      // write into the storage without LibC getting involved to change what the
-      // variable is set to
-      int result = putenv(kAtosMachPortEnvEntry);
-      CHECK_EQ(result, 0);
-    }
-  }
-
  private:
   bool StartSymbolizerSubprocess() override {
-    // Configure sandbox before starting atos process.
-
     // Put the string command line argument in the object so that it outlives
     // the call to GetArgV.
-    internal_snprintf(pid_str_, sizeof(pid_str_), "%d", internal_getpid());
-
-    if (SANITIZER_IOSSIM) {
-      // `atos` in the simulator is restricted in its ability to retrieve the
-      // task port for the target process (us) so we need to do extra work
-      // to pass our task port to it.
-      mach_port_t ports[]{mach_task_self()};
-      kern_return_t ret =
-          mach_ports_register(mach_task_self(), ports, /*count=*/1);
-      CHECK_EQ(ret, KERN_SUCCESS);
-
-      // Set environment variable that signals to `atos` that it should look
-      // for our task port. We can't call `setenv()` here because it might call
-      // malloc/realloc. To avoid that we instead update the
-      // `mach_port_env_var_entry_` variable with our current PID.
-      uptr count = internal_snprintf(kAtosMachPortEnvEntry,
-                                     sizeof(kAtosMachPortEnvEntry),
-                                     K_ATOS_ENV_VAR "=%s", pid_str_);
-      CHECK_GE(count, sizeof(K_ATOS_ENV_VAR) + internal_strlen(pid_str_));
-      // Document our assumption but without calling `getenv()` in normal
-      // builds.
-      DCHECK(getenv(K_ATOS_ENV_VAR));
-      DCHECK_EQ(internal_strcmp(getenv(K_ATOS_ENV_VAR), pid_str_), 0);
-    }
+    internal_snprintf(pid_str_, sizeof(pid_str_), "%d", (int)internal_getpid());
 
+    // Configure sandbox before starting atos process.
     return SymbolizerProcess::StartSymbolizerSubprocess();
   }
 
@@ -137,13 +91,10 @@ class AtosSymbolizerProcess final : public SymbolizerProcess {
       argv[i++] = "-d";
     }
     argv[i++] = nullptr;
+    CHECK_LE(i, kArgVMax);
   }
 
   char pid_str_[16];
-  // Space for `\0` in `K_ATOS_ENV_VAR` is reused for `=`.
-  static_assert(sizeof(kAtosMachPortEnvEntry) ==
-                    (sizeof(K_ATOS_ENV_VAR) + sizeof(pid_str_)),
-                "sizes should match");
 };
 
 #undef K_ATOS_ENV_VAR
@@ -249,8 +200,6 @@ bool AtosSymbolizer::SymbolizeData(uptr addr, DataInfo *info) {
   return true;
 }
 
-void AtosSymbolizer::LateInitialize() { process_->LateInitialize(); }
-
 }  // namespace __sanitizer
 
 #endif  // SANITIZER_MAC
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_mac.h b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_mac.h
index 401d30fa5033..d5abe9d98c1f 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_mac.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_mac.h
@@ -35,7 +35,6 @@ class AtosSymbolizer final : public SymbolizerTool {
 
   bool SymbolizePC(uptr addr, SymbolizedStack *stack) override;
   bool SymbolizeData(uptr addr, DataInfo *info) override;
-  void LateInitialize() override;
 
  private:
   AtosSymbolizerProcess *process_;
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_markup.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_markup.cpp
index 9a5b4a8c54c7..1ec0c5cad7a2 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_markup.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_markup.cpp
@@ -100,9 +100,7 @@ Symbolizer *Symbolizer::PlatformInit() {
   return new (symbolizer_allocator_) Symbolizer({});
 }
 
-void Symbolizer::LateInitialize() {
-  Symbolizer::GetOrInit()->LateInitializeTools();
-}
+void Symbolizer::LateInitialize() { Symbolizer::GetOrInit(); }
 
 void StartReportDeadlySignal() {}
 void ReportDeadlySignal(const SignalContext &sig, u32 tid,
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp
index 4cd4b4636f0a..5f6e4cc3180e 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp
@@ -213,9 +213,14 @@ class Addr2LineProcess final : public SymbolizerProcess {
                const char *(&argv)[kArgVMax]) const override {
     int i = 0;
     argv[i++] = path_to_binary;
-    argv[i++] = "-iCfe";
+    if (common_flags()->demangle)
+      argv[i++] = "-C";
+    if (common_flags()->symbolize_inline_frames)
+      argv[i++] = "-i";
+    argv[i++] = "-fe";
     argv[i++] = module_name_;
     argv[i++] = nullptr;
+    CHECK_LE(i, kArgVMax);
   }
 
   bool ReachedEndOfOutput(const char *buffer, uptr length) const override;
@@ -312,37 +317,42 @@ class Addr2LinePool final : public SymbolizerTool {
       FIRST_32_SECOND_64(UINT32_MAX, UINT64_MAX);
 };
 
-#if SANITIZER_SUPPORTS_WEAK_HOOKS
+#  if SANITIZER_SUPPORTS_WEAK_HOOKS
 extern "C" {
 SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE bool
 __sanitizer_symbolize_code(const char *ModuleName, u64 ModuleOffset,
-                           char *Buffer, int MaxLength,
-                           bool SymbolizeInlineFrames);
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-bool __sanitizer_symbolize_data(const char *ModuleName, u64 ModuleOffset,
-                                char *Buffer, int MaxLength);
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-void __sanitizer_symbolize_flush();
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-int __sanitizer_symbolize_demangle(const char *Name, char *Buffer,
-                                   int MaxLength);
+                           char *Buffer, int MaxLength);
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE bool
+__sanitizer_symbolize_data(const char *ModuleName, u64 ModuleOffset,
+                           char *Buffer, int MaxLength);
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+__sanitizer_symbolize_flush();
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE int
+__sanitizer_symbolize_demangle(const char *Name, char *Buffer, int MaxLength);
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE bool
+__sanitizer_symbolize_set_demangle(bool Demangle);
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE bool
+__sanitizer_symbolize_set_inline_frames(bool InlineFrames);
 }  // extern "C"
 
 class InternalSymbolizer final : public SymbolizerTool {
  public:
   static InternalSymbolizer *get(LowLevelAllocator *alloc) {
-    if (__sanitizer_symbolize_code != 0 &&
-        __sanitizer_symbolize_data != 0) {
-      return new(*alloc) InternalSymbolizer();
-    }
+    if (__sanitizer_symbolize_set_demangle)
+      CHECK(__sanitizer_symbolize_set_demangle(common_flags()->demangle));
+    if (__sanitizer_symbolize_set_inline_frames)
+      CHECK(__sanitizer_symbolize_set_inline_frames(
+          common_flags()->symbolize_inline_frames));
+    if (__sanitizer_symbolize_code && __sanitizer_symbolize_data)
+      return new (*alloc) InternalSymbolizer();
     return 0;
   }
 
   bool SymbolizePC(uptr addr, SymbolizedStack *stack) override {
     bool result = __sanitizer_symbolize_code(
-        stack->info.module, stack->info.module_offset, buffer_, kBufferSize,
-        common_flags()->symbolize_inline_frames);
-    if (result) ParseSymbolizePCOutput(buffer_, stack);
+        stack->info.module, stack->info.module_offset, buffer_, kBufferSize);
+    if (result)
+      ParseSymbolizePCOutput(buffer_, stack);
     return result;
   }
 
@@ -365,7 +375,7 @@ class InternalSymbolizer final : public SymbolizerTool {
     if (__sanitizer_symbolize_demangle) {
       for (uptr res_length = 1024;
            res_length <= InternalSizeClassMap::kMaxSize;) {
-        char *res_buff = static_cast<char*>(InternalAlloc(res_length));
+        char *res_buff = static_cast<char *>(InternalAlloc(res_length));
         uptr req_length =
             __sanitizer_symbolize_demangle(name, res_buff, res_length);
         if (req_length > res_length) {
@@ -380,19 +390,19 @@ class InternalSymbolizer final : public SymbolizerTool {
   }
 
  private:
-  InternalSymbolizer() { }
+  InternalSymbolizer() {}
 
   static const int kBufferSize = 16 * 1024;
   char buffer_[kBufferSize];
 };
-#else  // SANITIZER_SUPPORTS_WEAK_HOOKS
+#  else  // SANITIZER_SUPPORTS_WEAK_HOOKS
 
 class InternalSymbolizer final : public SymbolizerTool {
  public:
   static InternalSymbolizer *get(LowLevelAllocator *alloc) { return 0; }
 };
 
-#endif  // SANITIZER_SUPPORTS_WEAK_HOOKS
+#  endif  // SANITIZER_SUPPORTS_WEAK_HOOKS
 
 const char *Symbolizer::PlatformDemangle(const char *name) {
   return DemangleSwiftAndCXX(name);
@@ -492,7 +502,7 @@ Symbolizer *Symbolizer::PlatformInit() {
 }
 
 void Symbolizer::LateInitialize() {
-  Symbolizer::GetOrInit()->LateInitializeTools();
+  Symbolizer::GetOrInit();
   InitializeSwiftDemangler();
 }
 
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_win.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_win.cpp
index 702d901353db..c647ab107ec5 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_win.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_win.cpp
@@ -318,7 +318,7 @@ Symbolizer *Symbolizer::PlatformInit() {
 }
 
 void Symbolizer::LateInitialize() {
-  Symbolizer::GetOrInit()->LateInitializeTools();
+  Symbolizer::GetOrInit();
 }
 
 }  // namespace __sanitizer
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cpp
index 2e1cd0238812..278f6defca95 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cpp
@@ -110,7 +110,7 @@ ThreadRegistry::ThreadRegistry(ThreadContextFactory factory, u32 max_threads,
       max_threads_(max_threads),
       thread_quarantine_size_(thread_quarantine_size),
       max_reuse_(max_reuse),
-      mtx_(),
+      mtx_(MutexThreadRegistry),
       total_threads_(0),
       alive_threads_(0),
       max_alive_threads_(0),
@@ -365,4 +365,20 @@ void ThreadRegistry::SetThreadUserId(u32 tid, uptr user_id) {
   CHECK(live_.try_emplace(user_id, tctx->tid).second);
 }
 
+u32 ThreadRegistry::OnFork(u32 tid) {
+  ThreadRegistryLock l(this);
+  // We only purge user_id (pthread_t) of live threads because
+  // they cause CHECK failures if new threads with matching pthread_t
+  // created after fork.
+  // Potentially we could purge more info (ThreadContextBase themselves),
+  // but it's hard to test and easy to introduce new issues by doing this.
+  for (auto *tctx : threads_) {
+    if (tctx->tid == tid || !tctx->user_id)
+      continue;
+    CHECK(live_.erase(tctx->user_id));
+    tctx->user_id = 0;
+  }
+  return alive_threads_;
+}
+
 }  // namespace __sanitizer
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.h b/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.h
index a259b324220f..9975d78ec0bb 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.h
@@ -104,6 +104,8 @@ class MUTEX ThreadRegistry {
     return threads_.empty() ? nullptr : threads_[tid];
   }
 
+  u32 NumThreadsLocked() const { return threads_.size(); }
+
   u32 CreateThread(uptr user_id, bool detached, u32 parent_tid, void *arg);
 
   typedef void (*ThreadCallback)(ThreadContextBase *tctx, void *arg);
@@ -131,6 +133,11 @@ class MUTEX ThreadRegistry {
   u32 ConsumeThreadUserId(uptr user_id);
   void SetThreadUserId(u32 tid, uptr user_id);
 
+  // OnFork must be called in the child process after fork to purge old
+  // threads that don't exist anymore (except for the current thread tid).
+  // Returns number of alive threads before fork.
+  u32 OnFork(u32 tid);
+
  private:
   const ThreadContextFactory context_factory_;
   const u32 max_threads_;
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp
index 1a31ce02af4c..cfe6cc2b394b 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp
@@ -16,7 +16,6 @@
 
 #define WIN32_LEAN_AND_MEAN
 #define NOGDI
-#include <direct.h>
 #include <windows.h>
 #include <io.h>
 #include <psapi.h>
@@ -571,7 +570,9 @@ void Abort() {
   internal__exit(3);
 }
 
-bool CreateDir(const char *pathname) { return _mkdir(pathname) == 0; }
+bool CreateDir(const char *pathname) {
+  return CreateDirectoryA(pathname, nullptr) != 0;
+}
 
 #if !SANITIZER_GO
 // Read the file to extract the ImageBase field from the PE header. If ASLR is
diff --git a/compiler-rt/lib/sanitizer_common/symbolizer/sanitizer_symbolize.cpp b/compiler-rt/lib/sanitizer_common/symbolizer/sanitizer_symbolize.cpp
index 3809880d50b4..80cab36426c5 100644
--- a/compiler-rt/lib/sanitizer_common/symbolizer/sanitizer_symbolize.cpp
+++ b/compiler-rt/lib/sanitizer_common/symbolizer/sanitizer_symbolize.cpp
@@ -17,10 +17,17 @@
 #include "llvm/DebugInfo/Symbolize/DIPrinter.h"
 #include "llvm/DebugInfo/Symbolize/Symbolize.h"
 
+static llvm::symbolize::LLVMSymbolizer *Symbolizer = nullptr;
+static bool Demangle = true;
+static bool InlineFrames = true;
+
 static llvm::symbolize::LLVMSymbolizer *getDefaultSymbolizer() {
-  static llvm::symbolize::LLVMSymbolizer *DefaultSymbolizer =
-      new llvm::symbolize::LLVMSymbolizer();
-  return DefaultSymbolizer;
+  if (Symbolizer)
+    return Symbolizer;
+  llvm::symbolize::LLVMSymbolizer::Options Opts;
+  Opts.Demangle = Demangle;
+  Symbolizer = new llvm::symbolize::LLVMSymbolizer(Opts);
+  return Symbolizer;
 }
 
 static llvm::symbolize::PrinterConfig getDefaultPrinterConfig() {
@@ -43,8 +50,7 @@ extern "C" {
 typedef uint64_t u64;
 
 bool __sanitizer_symbolize_code(const char *ModuleName, uint64_t ModuleOffset,
-                                char *Buffer, int MaxLength,
-                                bool SymbolizeInlineFrames) {
+                                char *Buffer, int MaxLength) {
   std::string Result;
   {
     llvm::raw_string_ostream OS(Result);
@@ -55,7 +61,7 @@ bool __sanitizer_symbolize_code(const char *ModuleName, uint64_t ModuleOffset,
 
     // TODO: it is neccessary to set proper SectionIndex here.
     // object::SectionedAddress::UndefSection works for only absolute addresses.
-    if (SymbolizeInlineFrames) {
+    if (InlineFrames) {
       auto ResOrErr = getDefaultSymbolizer()->symbolizeInlinedCode(
           ModuleName,
           {ModuleOffset, llvm::object::SectionedAddress::UndefSection});
@@ -93,7 +99,10 @@ bool __sanitizer_symbolize_data(const char *ModuleName, uint64_t ModuleOffset,
                                         Result.c_str()) < MaxLength;
 }
 
-void __sanitizer_symbolize_flush() { getDefaultSymbolizer()->flush(); }
+void __sanitizer_symbolize_flush() {
+  if (Symbolizer)
+    Symbolizer->flush();
+}
 
 int __sanitizer_symbolize_demangle(const char *Name, char *Buffer,
                                    int MaxLength) {
@@ -105,6 +114,19 @@ int __sanitizer_symbolize_demangle(const char *Name, char *Buffer,
              : 0;
 }
 
+bool __sanitizer_symbolize_set_demangle(bool Value) {
+  // Must be called before LLVMSymbolizer created.
+  if (Symbolizer)
+    return false;
+  Demangle = Value;
+  return true;
+}
+
+bool __sanitizer_symbolize_set_inline_frames(bool Value) {
+  InlineFrames = Value;
+  return true;
+}
+
 // Override __cxa_atexit and ignore callbacks.
 // This prevents crashes in a configuration when the symbolizer
 // is built into sanitizer runtime and consequently into the test process.
diff --git a/compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh b/compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
index d1d61fb7ab2a..599f063b45c9 100755
--- a/compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
+++ b/compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
@@ -143,7 +143,7 @@ if [[ ! -d ${LLVM_BUILD} ]]; then
   $LLVM_SRC
 fi
 cd ${LLVM_BUILD}
-ninja LLVMSymbolize LLVMObject LLVMBinaryFormat LLVMDebugInfoDWARF LLVMSupport LLVMDebugInfoPDB LLVMMC LLVMDemangle LLVMTextAPI
+ninja LLVMSymbolize LLVMObject LLVMBinaryFormat LLVMDebugInfoDWARF LLVMSupport LLVMDebugInfoPDB LLVMDebuginfod LLVMMC LLVMDemangle LLVMTextAPI
 
 cd ${BUILD_DIR}
 rm -rf ${SYMBOLIZER_BUILD}
@@ -155,7 +155,12 @@ SYMBOLIZER_FLAGS="$LLVM_FLAGS -I${LLVM_SRC}/include -I${LLVM_BUILD}/include -std
 $CXX $SYMBOLIZER_FLAGS ${SRC_DIR}/sanitizer_symbolize.cpp ${SRC_DIR}/sanitizer_wrappers.cpp -c
 $AR rc symbolizer.a sanitizer_symbolize.o sanitizer_wrappers.o
 
-SYMBOLIZER_API_LIST=__sanitizer_symbolize_code,__sanitizer_symbolize_data,__sanitizer_symbolize_flush,__sanitizer_symbolize_demangle
+SYMBOLIZER_API_LIST=__sanitizer_symbolize_code
+SYMBOLIZER_API_LIST+=,__sanitizer_symbolize_data
+SYMBOLIZER_API_LIST+=,__sanitizer_symbolize_flush
+SYMBOLIZER_API_LIST+=,__sanitizer_symbolize_demangle
+SYMBOLIZER_API_LIST+=,__sanitizer_symbolize_set_demangle
+SYMBOLIZER_API_LIST+=,__sanitizer_symbolize_set_inline_frames
 
 LIBCXX_ARCHIVE_DIR=$(dirname $(find $LIBCXX_BUILD -name libc++.a | head -n1))
 
@@ -170,6 +175,7 @@ $SCRIPT_DIR/ar_to_bc.sh $LIBCXX_ARCHIVE_DIR/libc++.a \
                         $LLVM_BUILD/lib/libLLVMDebugInfoPDB.a \
                         $LLVM_BUILD/lib/libLLVMDebugInfoMSF.a \
                         $LLVM_BUILD/lib/libLLVMDebugInfoCodeView.a \
+                        $LLVM_BUILD/lib/libLLVMDebuginfod.a \
                         $LLVM_BUILD/lib/libLLVMDemangle.a \
                         $LLVM_BUILD/lib/libLLVMMC.a \
                         $LLVM_BUILD/lib/libLLVMTextAPI.a \
diff --git a/compiler-rt/lib/sanitizer_common/symbolizer/scripts/global_symbols.txt b/compiler-rt/lib/sanitizer_common/symbolizer/scripts/global_symbols.txt
index 29b2960e11fe..0bb38ba951a8 100644
--- a/compiler-rt/lib/sanitizer_common/symbolizer/scripts/global_symbols.txt
+++ b/compiler-rt/lib/sanitizer_common/symbolizer/scripts/global_symbols.txt
@@ -38,6 +38,8 @@ __sanitizer_symbolize_code T
 __sanitizer_symbolize_data T
 __sanitizer_symbolize_demangle T
 __sanitizer_symbolize_flush T
+__sanitizer_symbolize_set_demangle T
+__sanitizer_symbolize_set_inline_frames T
 __strdup U
 __udivdi3 U
 __umoddi3 U
@@ -51,8 +53,8 @@ catgets U
 catopen U
 ceil U
 ceilf U
-clock_gettime U
 cfgetospeed U
+clock_gettime U
 dl_iterate_phdr U
 dlsym U
 dup U
@@ -76,15 +78,17 @@ getcwd U
 getenv U
 getpagesize U
 getpid U
+getpwuid U
 getrlimit U
 gettimeofday U
+getuid U
 ioctl U
 isalnum U
 isalpha U
 isatty U
 islower U
-isspace U
 isprint U
+isspace U
 isupper U
 isxdigit U
 log10 U
@@ -111,12 +115,17 @@ posix_spawn_file_actions_addopen U
 posix_spawn_file_actions_destroy U
 posix_spawn_file_actions_init U
 qsort U
+raise U
 rand U
 readlink U
 realloc U
 remove U
+rename U
 setrlimit U
 setvbuf U
+sigaction U
+sigaltstack U
+sigemptyset U
 sigfillset U
 sigprocmask U
 snprintf U
@@ -146,6 +155,7 @@ strtold_l U
 strtoll_l U
 strtoull_l U
 syscall U
+sysconf U
 tcgetattr U
 uname U
 ungetc U
diff --git a/compiler-rt/lib/sanitizer_common/weak_symbols.txt b/compiler-rt/lib/sanitizer_common/weak_symbols.txt
index 5a2b275184f4..d07f81bc8c12 100644
--- a/compiler-rt/lib/sanitizer_common/weak_symbols.txt
+++ b/compiler-rt/lib/sanitizer_common/weak_symbols.txt
@@ -6,3 +6,5 @@ ___sanitizer_symbolize_code
 ___sanitizer_symbolize_data
 ___sanitizer_symbolize_demangle
 ___sanitizer_symbolize_flush
+___sanitizer_symbolize_set_demangle
+___sanitizer_symbolize_set_inline_frames
diff --git a/compiler-rt/lib/tsan/go/tsan_go.cpp b/compiler-rt/lib/tsan/go/tsan_go.cpp
index 104c5b325aee..c689a51fb5e1 100644
--- a/compiler-rt/lib/tsan/go/tsan_go.cpp
+++ b/compiler-rt/lib/tsan/go/tsan_go.cpp
@@ -214,7 +214,7 @@ void __tsan_malloc(ThreadState *thr, uptr pc, uptr p, uptr sz) {
 }
 
 void __tsan_free(uptr p, uptr sz) {
-  ctx->metamap.FreeRange(get_cur_proc(), p, sz);
+  ctx->metamap.FreeRange(get_cur_proc(), p, sz, false);
 }
 
 void __tsan_go_start(ThreadState *parent, ThreadState **pthr, void *pc) {
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan.syms.extra b/compiler-rt/lib/tsan/rtl-old/tsan.syms.extra
new file mode 100644
index 000000000000..4838bb0a7279
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan.syms.extra
@@ -0,0 +1,31 @@
+__tsan_init
+__tsan_flush_memory
+__tsan_read*
+__tsan_write*
+__tsan_vptr*
+__tsan_func*
+__tsan_atomic*
+__tsan_java*
+__tsan_unaligned*
+__tsan_release
+__tsan_acquire
+__tsan_mutex_create
+__tsan_mutex_destroy
+__tsan_mutex_pre_lock
+__tsan_mutex_post_lock
+__tsan_mutex_pre_unlock
+__tsan_mutex_post_unlock
+__tsan_mutex_pre_signal
+__tsan_mutex_post_signal
+__tsan_mutex_pre_divert
+__tsan_mutex_post_divert
+__tsan_get_current_fiber
+__tsan_create_fiber
+__tsan_destroy_fiber
+__tsan_switch_to_fiber
+__tsan_set_fiber_name
+__ubsan_*
+Annotate*
+WTFAnnotate*
+RunningOnValgrind
+ValgrindSlowdown
diff --git a/compiler-rt/lib/tsan/rtl/tsan_clock.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_clock.cpp
index d122b67c0aaa..d122b67c0aaa 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_clock.cpp
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_clock.cpp
diff --git a/compiler-rt/lib/tsan/rtl/tsan_clock.h b/compiler-rt/lib/tsan/rtl-old/tsan_clock.h
index 11cbc0c0b86b..11cbc0c0b86b 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_clock.h
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_clock.h
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_debugging.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_debugging.cpp
new file mode 100644
index 000000000000..1d3c3849a446
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_debugging.cpp
@@ -0,0 +1,262 @@
+//===-- tsan_debugging.cpp ------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// TSan debugging API implementation.
+//===----------------------------------------------------------------------===//
+#include "tsan_interface.h"
+#include "tsan_report.h"
+#include "tsan_rtl.h"
+
+#include "sanitizer_common/sanitizer_stackdepot.h"
+
+using namespace __tsan;
+
+static const char *ReportTypeDescription(ReportType typ) {
+  switch (typ) {
+    case ReportTypeRace: return "data-race";
+    case ReportTypeVptrRace: return "data-race-vptr";
+    case ReportTypeUseAfterFree: return "heap-use-after-free";
+    case ReportTypeVptrUseAfterFree: return "heap-use-after-free-vptr";
+    case ReportTypeExternalRace: return "external-race";
+    case ReportTypeThreadLeak: return "thread-leak";
+    case ReportTypeMutexDestroyLocked: return "locked-mutex-destroy";
+    case ReportTypeMutexDoubleLock: return "mutex-double-lock";
+    case ReportTypeMutexInvalidAccess: return "mutex-invalid-access";
+    case ReportTypeMutexBadUnlock: return "mutex-bad-unlock";
+    case ReportTypeMutexBadReadLock: return "mutex-bad-read-lock";
+    case ReportTypeMutexBadReadUnlock: return "mutex-bad-read-unlock";
+    case ReportTypeSignalUnsafe: return "signal-unsafe-call";
+    case ReportTypeErrnoInSignal: return "errno-in-signal-handler";
+    case ReportTypeDeadlock: return "lock-order-inversion";
+    // No default case so compiler warns us if we miss one
+  }
+  UNREACHABLE("missing case");
+}
+
+static const char *ReportLocationTypeDescription(ReportLocationType typ) {
+  switch (typ) {
+    case ReportLocationGlobal: return "global";
+    case ReportLocationHeap: return "heap";
+    case ReportLocationStack: return "stack";
+    case ReportLocationTLS: return "tls";
+    case ReportLocationFD: return "fd";
+    // No default case so compiler warns us if we miss one
+  }
+  UNREACHABLE("missing case");
+}
+
+static void CopyTrace(SymbolizedStack *first_frame, void **trace,
+                      uptr trace_size) {
+  uptr i = 0;
+  for (SymbolizedStack *frame = first_frame; frame != nullptr;
+       frame = frame->next) {
+    trace[i++] = (void *)frame->info.address;
+    if (i >= trace_size) break;
+  }
+}
+
+// Meant to be called by the debugger.
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__tsan_get_current_report() {
+  return const_cast<ReportDesc*>(cur_thread()->current_report);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_report_data(void *report, const char **description, int *count,
+                           int *stack_count, int *mop_count, int *loc_count,
+                           int *mutex_count, int *thread_count,
+                           int *unique_tid_count, void **sleep_trace,
+                           uptr trace_size) {
+  const ReportDesc *rep = (ReportDesc *)report;
+  *description = ReportTypeDescription(rep->typ);
+  *count = rep->count;
+  *stack_count = rep->stacks.Size();
+  *mop_count = rep->mops.Size();
+  *loc_count = rep->locs.Size();
+  *mutex_count = rep->mutexes.Size();
+  *thread_count = rep->threads.Size();
+  *unique_tid_count = rep->unique_tids.Size();
+  if (rep->sleep) CopyTrace(rep->sleep->frames, sleep_trace, trace_size);
+  return 1;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_report_tag(void *report, uptr *tag) {
+  const ReportDesc *rep = (ReportDesc *)report;
+  *tag = rep->tag;
+  return 1;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_report_stack(void *report, uptr idx, void **trace,
+                            uptr trace_size) {
+  const ReportDesc *rep = (ReportDesc *)report;
+  CHECK_LT(idx, rep->stacks.Size());
+  ReportStack *stack = rep->stacks[idx];
+  if (stack) CopyTrace(stack->frames, trace, trace_size);
+  return stack ? 1 : 0;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_report_mop(void *report, uptr idx, int *tid, void **addr,
+                          int *size, int *write, int *atomic, void **trace,
+                          uptr trace_size) {
+  const ReportDesc *rep = (ReportDesc *)report;
+  CHECK_LT(idx, rep->mops.Size());
+  ReportMop *mop = rep->mops[idx];
+  *tid = mop->tid;
+  *addr = (void *)mop->addr;
+  *size = mop->size;
+  *write = mop->write ? 1 : 0;
+  *atomic = mop->atomic ? 1 : 0;
+  if (mop->stack) CopyTrace(mop->stack->frames, trace, trace_size);
+  return 1;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_report_loc(void *report, uptr idx, const char **type,
+                          void **addr, uptr *start, uptr *size, int *tid,
+                          int *fd, int *suppressable, void **trace,
+                          uptr trace_size) {
+  const ReportDesc *rep = (ReportDesc *)report;
+  CHECK_LT(idx, rep->locs.Size());
+  ReportLocation *loc = rep->locs[idx];
+  *type = ReportLocationTypeDescription(loc->type);
+  *addr = (void *)loc->global.start;
+  *start = loc->heap_chunk_start;
+  *size = loc->heap_chunk_size;
+  *tid = loc->tid;
+  *fd = loc->fd;
+  *suppressable = loc->suppressable;
+  if (loc->stack) CopyTrace(loc->stack->frames, trace, trace_size);
+  return 1;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_report_loc_object_type(void *report, uptr idx,
+                                      const char **object_type) {
+  const ReportDesc *rep = (ReportDesc *)report;
+  CHECK_LT(idx, rep->locs.Size());
+  ReportLocation *loc = rep->locs[idx];
+  *object_type = GetObjectTypeFromTag(loc->external_tag);
+  return 1;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_report_mutex(void *report, uptr idx, uptr *mutex_id, void **addr,
+                            int *destroyed, void **trace, uptr trace_size) {
+  const ReportDesc *rep = (ReportDesc *)report;
+  CHECK_LT(idx, rep->mutexes.Size());
+  ReportMutex *mutex = rep->mutexes[idx];
+  *mutex_id = mutex->id;
+  *addr = (void *)mutex->addr;
+  *destroyed = mutex->destroyed;
+  if (mutex->stack) CopyTrace(mutex->stack->frames, trace, trace_size);
+  return 1;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_report_thread(void *report, uptr idx, int *tid, tid_t *os_id,
+                             int *running, const char **name, int *parent_tid,
+                             void **trace, uptr trace_size) {
+  const ReportDesc *rep = (ReportDesc *)report;
+  CHECK_LT(idx, rep->threads.Size());
+  ReportThread *thread = rep->threads[idx];
+  *tid = thread->id;
+  *os_id = thread->os_id;
+  *running = thread->running;
+  *name = thread->name;
+  *parent_tid = thread->parent_tid;
+  if (thread->stack) CopyTrace(thread->stack->frames, trace, trace_size);
+  return 1;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_report_unique_tid(void *report, uptr idx, int *tid) {
+  const ReportDesc *rep = (ReportDesc *)report;
+  CHECK_LT(idx, rep->unique_tids.Size());
+  *tid = rep->unique_tids[idx];
+  return 1;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+const char *__tsan_locate_address(uptr addr, char *name, uptr name_size,
+                                  uptr *region_address_ptr,
+                                  uptr *region_size_ptr) {
+  uptr region_address = 0;
+  uptr region_size = 0;
+  const char *region_kind = nullptr;
+  if (name && name_size > 0) name[0] = 0;
+
+  if (IsMetaMem(reinterpret_cast<u32 *>(addr))) {
+    region_kind = "meta shadow";
+  } else if (IsShadowMem(reinterpret_cast<RawShadow *>(addr))) {
+    region_kind = "shadow";
+  } else {
+    bool is_stack = false;
+    MBlock *b = 0;
+    Allocator *a = allocator();
+    if (a->PointerIsMine((void *)addr)) {
+      void *block_begin = a->GetBlockBegin((void *)addr);
+      if (block_begin) b = ctx->metamap.GetBlock((uptr)block_begin);
+    }
+
+    if (b != 0) {
+      region_address = (uptr)allocator()->GetBlockBegin((void *)addr);
+      region_size = b->siz;
+      region_kind = "heap";
+    } else {
+      // TODO(kuba.brecka): We should not lock. This is supposed to be called
+      // from within the debugger when other threads are stopped.
+      ctx->thread_registry.Lock();
+      ThreadContext *tctx = IsThreadStackOrTls(addr, &is_stack);
+      ctx->thread_registry.Unlock();
+      if (tctx) {
+        region_kind = is_stack ? "stack" : "tls";
+      } else {
+        region_kind = "global";
+        DataInfo info;
+        if (Symbolizer::GetOrInit()->SymbolizeData(addr, &info)) {
+          internal_strncpy(name, info.name, name_size);
+          region_address = info.start;
+          region_size = info.size;
+        }
+      }
+    }
+  }
+
+  CHECK(region_kind);
+  if (region_address_ptr) *region_address_ptr = region_address;
+  if (region_size_ptr) *region_size_ptr = region_size;
+  return region_kind;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_alloc_stack(uptr addr, uptr *trace, uptr size, int *thread_id,
+                           tid_t *os_id) {
+  MBlock *b = 0;
+  Allocator *a = allocator();
+  if (a->PointerIsMine((void *)addr)) {
+    void *block_begin = a->GetBlockBegin((void *)addr);
+    if (block_begin) b = ctx->metamap.GetBlock((uptr)block_begin);
+  }
+  if (b == 0) return 0;
+
+  *thread_id = b->tid;
+  // No locking.  This is supposed to be called from within the debugger when
+  // other threads are stopped.
+  ThreadContextBase *tctx = ctx->thread_registry.GetThreadLocked(b->tid);
+  *os_id = tctx->os_id;
+
+  StackTrace stack = StackDepotGet(b->stk);
+  size = Min(size, (uptr)stack.size);
+  for (uptr i = 0; i < size; i++) trace[i] = stack.trace[stack.size - i - 1];
+  return size;
+}
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_defs.h b/compiler-rt/lib/tsan/rtl-old/tsan_defs.h
new file mode 100644
index 000000000000..4712c2be1813
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_defs.h
@@ -0,0 +1,236 @@
+//===-- tsan_defs.h ---------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef TSAN_DEFS_H
+#define TSAN_DEFS_H
+
+#include "sanitizer_common/sanitizer_internal_defs.h"
+#include "sanitizer_common/sanitizer_libc.h"
+#include "sanitizer_common/sanitizer_mutex.h"
+#include "ubsan/ubsan_platform.h"
+
+#ifndef TSAN_VECTORIZE
+#  define TSAN_VECTORIZE __SSE4_2__
+#endif
+
+#if TSAN_VECTORIZE
+// <emmintrin.h> transitively includes <stdlib.h>,
+// and it's prohibited to include std headers into tsan runtime.
+// So we do this dirty trick.
+#  define _MM_MALLOC_H_INCLUDED
+#  define __MM_MALLOC_H
+#  include <emmintrin.h>
+#  include <smmintrin.h>
+#  define VECTOR_ALIGNED ALIGNED(16)
+typedef __m128i m128;
+#else
+#  define VECTOR_ALIGNED
+#endif
+
+// Setup defaults for compile definitions.
+#ifndef TSAN_NO_HISTORY
+# define TSAN_NO_HISTORY 0
+#endif
+
+#ifndef TSAN_CONTAINS_UBSAN
+# if CAN_SANITIZE_UB && !SANITIZER_GO
+#  define TSAN_CONTAINS_UBSAN 1
+# else
+#  define TSAN_CONTAINS_UBSAN 0
+# endif
+#endif
+
+namespace __tsan {
+
+constexpr uptr kByteBits = 8;
+
+// Thread slot ID.
+enum class Sid : u8 {};
+constexpr uptr kThreadSlotCount = 256;
+constexpr Sid kFreeSid = static_cast<Sid>(255);
+
+// Abstract time unit, vector clock element.
+enum class Epoch : u16 {};
+constexpr uptr kEpochBits = 14;
+constexpr Epoch kEpochZero = static_cast<Epoch>(0);
+constexpr Epoch kEpochOver = static_cast<Epoch>(1 << kEpochBits);
+
+const int kClkBits = 42;
+const unsigned kMaxTidReuse = (1 << (64 - kClkBits)) - 1;
+
+struct ClockElem {
+  u64 epoch  : kClkBits;
+  u64 reused : 64 - kClkBits;  // tid reuse count
+};
+
+struct ClockBlock {
+  static const uptr kSize = 512;
+  static const uptr kTableSize = kSize / sizeof(u32);
+  static const uptr kClockCount = kSize / sizeof(ClockElem);
+  static const uptr kRefIdx = kTableSize - 1;
+  static const uptr kBlockIdx = kTableSize - 2;
+
+  union {
+    u32       table[kTableSize];
+    ClockElem clock[kClockCount];
+  };
+
+  ClockBlock() {
+  }
+};
+
+const int kTidBits = 13;
+// Reduce kMaxTid by kClockCount because one slot in ClockBlock table is
+// occupied by reference counter, so total number of elements we can store
+// in SyncClock is kClockCount * (kTableSize - 1).
+const unsigned kMaxTid = (1 << kTidBits) - ClockBlock::kClockCount;
+#if !SANITIZER_GO
+const unsigned kMaxTidInClock = kMaxTid * 2;  // This includes msb 'freed' bit.
+#else
+const unsigned kMaxTidInClock = kMaxTid;  // Go does not track freed memory.
+#endif
+const uptr kShadowStackSize = 64 * 1024;
+
+// Count of shadow values in a shadow cell.
+const uptr kShadowCnt = 4;
+
+// That many user bytes are mapped onto a single shadow cell.
+const uptr kShadowCell = 8;
+
+// Single shadow value.
+typedef u64 RawShadow;
+const uptr kShadowSize = sizeof(RawShadow);
+
+// Shadow memory is kShadowMultiplier times larger than user memory.
+const uptr kShadowMultiplier = kShadowSize * kShadowCnt / kShadowCell;
+
+// That many user bytes are mapped onto a single meta shadow cell.
+// Must be less or equal to minimal memory allocator alignment.
+const uptr kMetaShadowCell = 8;
+
+// Size of a single meta shadow value (u32).
+const uptr kMetaShadowSize = 4;
+
+// All addresses and PCs are assumed to be compressable to that many bits.
+const uptr kCompressedAddrBits = 44;
+
+#if TSAN_NO_HISTORY
+const bool kCollectHistory = false;
+#else
+const bool kCollectHistory = true;
+#endif
+
+// The following "build consistency" machinery ensures that all source files
+// are built in the same configuration. Inconsistent builds lead to
+// hard to debug crashes.
+#if SANITIZER_DEBUG
+void build_consistency_debug();
+#else
+void build_consistency_release();
+#endif
+
+static inline void USED build_consistency() {
+#if SANITIZER_DEBUG
+  build_consistency_debug();
+#else
+  build_consistency_release();
+#endif
+}
+
+template<typename T>
+T min(T a, T b) {
+  return a < b ? a : b;
+}
+
+template<typename T>
+T max(T a, T b) {
+  return a > b ? a : b;
+}
+
+template<typename T>
+T RoundUp(T p, u64 align) {
+  DCHECK_EQ(align & (align - 1), 0);
+  return (T)(((u64)p + align - 1) & ~(align - 1));
+}
+
+template<typename T>
+T RoundDown(T p, u64 align) {
+  DCHECK_EQ(align & (align - 1), 0);
+  return (T)((u64)p & ~(align - 1));
+}
+
+// Zeroizes high part, returns 'bits' lsb bits.
+template<typename T>
+T GetLsb(T v, int bits) {
+  return (T)((u64)v & ((1ull << bits) - 1));
+}
+
+struct MD5Hash {
+  u64 hash[2];
+  bool operator==(const MD5Hash &other) const;
+};
+
+MD5Hash md5_hash(const void *data, uptr size);
+
+struct Processor;
+struct ThreadState;
+class ThreadContext;
+struct Context;
+struct ReportStack;
+class ReportDesc;
+class RegionAlloc;
+
+typedef uptr AccessType;
+
+enum : AccessType {
+  kAccessWrite = 0,
+  kAccessRead = 1 << 0,
+  kAccessAtomic = 1 << 1,
+  kAccessVptr = 1 << 2,  // read or write of an object virtual table pointer
+  kAccessFree = 1 << 3,  // synthetic memory access during memory freeing
+  kAccessExternalPC = 1 << 4,  // access PC can have kExternalPCBit set
+};
+
+// Descriptor of user's memory block.
+struct MBlock {
+  u64  siz : 48;
+  u64  tag : 16;
+  StackID stk;
+  Tid tid;
+};
+
+COMPILER_CHECK(sizeof(MBlock) == 16);
+
+enum ExternalTag : uptr {
+  kExternalTagNone = 0,
+  kExternalTagSwiftModifyingAccess = 1,
+  kExternalTagFirstUserAvailable = 2,
+  kExternalTagMax = 1024,
+  // Don't set kExternalTagMax over 65,536, since MBlock only stores tags
+  // as 16-bit values, see tsan_defs.h.
+};
+
+enum MutexType {
+  MutexTypeTrace = MutexLastCommon,
+  MutexTypeReport,
+  MutexTypeSyncVar,
+  MutexTypeAnnotations,
+  MutexTypeAtExit,
+  MutexTypeFired,
+  MutexTypeRacy,
+  MutexTypeGlobalProc,
+  MutexTypeInternalAlloc,
+};
+
+}  // namespace __tsan
+
+#endif  // TSAN_DEFS_H
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_dense_alloc.h b/compiler-rt/lib/tsan/rtl-old/tsan_dense_alloc.h
new file mode 100644
index 000000000000..9e15f74a0615
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_dense_alloc.h
@@ -0,0 +1,156 @@
+//===-- tsan_dense_alloc.h --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// A DenseSlabAlloc is a freelist-based allocator of fixed-size objects.
+// DenseSlabAllocCache is a thread-local cache for DenseSlabAlloc.
+// The only difference with traditional slab allocators is that DenseSlabAlloc
+// allocates/free indices of objects and provide a functionality to map
+// the index onto the real pointer. The index is u32, that is, 2 times smaller
+// than uptr (hense the Dense prefix).
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_DENSE_ALLOC_H
+#define TSAN_DENSE_ALLOC_H
+
+#include "sanitizer_common/sanitizer_common.h"
+#include "tsan_defs.h"
+
+namespace __tsan {
+
+class DenseSlabAllocCache {
+  static const uptr kSize = 128;
+  typedef u32 IndexT;
+  uptr pos;
+  IndexT cache[kSize];
+  template <typename, uptr, uptr, u64>
+  friend class DenseSlabAlloc;
+};
+
+template <typename T, uptr kL1Size, uptr kL2Size, u64 kReserved = 0>
+class DenseSlabAlloc {
+ public:
+  typedef DenseSlabAllocCache Cache;
+  typedef typename Cache::IndexT IndexT;
+
+  static_assert((kL1Size & (kL1Size - 1)) == 0,
+                "kL1Size must be a power-of-two");
+  static_assert((kL2Size & (kL2Size - 1)) == 0,
+                "kL2Size must be a power-of-two");
+  static_assert((kL1Size * kL2Size) <= (1ull << (sizeof(IndexT) * 8)),
+                "kL1Size/kL2Size are too large");
+  static_assert(((kL1Size * kL2Size - 1) & kReserved) == 0,
+                "reserved bits don't fit");
+  static_assert(sizeof(T) > sizeof(IndexT),
+                "it doesn't make sense to use dense alloc");
+
+  DenseSlabAlloc(LinkerInitialized, const char *name) : name_(name) {}
+
+  explicit DenseSlabAlloc(const char *name)
+      : DenseSlabAlloc(LINKER_INITIALIZED, name) {
+    // It can be very large.
+    // Don't page it in for linker initialized objects.
+    internal_memset(map_, 0, sizeof(map_));
+  }
+
+  ~DenseSlabAlloc() {
+    for (uptr i = 0; i < kL1Size; i++) {
+      if (map_[i] != 0)
+        UnmapOrDie(map_[i], kL2Size * sizeof(T));
+    }
+  }
+
+  IndexT Alloc(Cache *c) {
+    if (c->pos == 0)
+      Refill(c);
+    return c->cache[--c->pos];
+  }
+
+  void Free(Cache *c, IndexT idx) {
+    DCHECK_NE(idx, 0);
+    if (c->pos == Cache::kSize)
+      Drain(c);
+    c->cache[c->pos++] = idx;
+  }
+
+  T *Map(IndexT idx) {
+    DCHECK_NE(idx, 0);
+    DCHECK_LE(idx, kL1Size * kL2Size);
+    return &map_[idx / kL2Size][idx % kL2Size];
+  }
+
+  void FlushCache(Cache *c) {
+    if (!c->pos)
+      return;
+    SpinMutexLock lock(&mtx_);
+    while (c->pos) {
+      IndexT idx = c->cache[--c->pos];
+      *(IndexT*)Map(idx) = freelist_;
+      freelist_ = idx;
+    }
+  }
+
+  void InitCache(Cache *c) {
+    c->pos = 0;
+    internal_memset(c->cache, 0, sizeof(c->cache));
+  }
+
+  uptr AllocatedMemory() const {
+    return atomic_load_relaxed(&fillpos_) * kL2Size * sizeof(T);
+  }
+
+ private:
+  T *map_[kL1Size];
+  SpinMutex mtx_;
+  IndexT freelist_ = {0};
+  atomic_uintptr_t fillpos_ = {0};
+  const char *const name_;
+
+  void Refill(Cache *c) {
+    SpinMutexLock lock(&mtx_);
+    if (freelist_ == 0) {
+      uptr fillpos = atomic_load_relaxed(&fillpos_);
+      if (fillpos == kL1Size) {
+        Printf("ThreadSanitizer: %s overflow (%zu*%zu). Dying.\n",
+            name_, kL1Size, kL2Size);
+        Die();
+      }
+      VPrintf(2, "ThreadSanitizer: growing %s: %zu out of %zu*%zu\n", name_,
+              fillpos, kL1Size, kL2Size);
+      T *batch = (T*)MmapOrDie(kL2Size * sizeof(T), name_);
+      // Reserve 0 as invalid index.
+      IndexT start = fillpos == 0 ? 1 : 0;
+      for (IndexT i = start; i < kL2Size; i++) {
+        new(batch + i) T;
+        *(IndexT *)(batch + i) = i + 1 + fillpos * kL2Size;
+      }
+      *(IndexT*)(batch + kL2Size - 1) = 0;
+      freelist_ = fillpos * kL2Size + start;
+      map_[fillpos] = batch;
+      atomic_store_relaxed(&fillpos_, fillpos + 1);
+    }
+    for (uptr i = 0; i < Cache::kSize / 2 && freelist_ != 0; i++) {
+      IndexT idx = freelist_;
+      c->cache[c->pos++] = idx;
+      freelist_ = *(IndexT*)Map(idx);
+    }
+  }
+
+  void Drain(Cache *c) {
+    SpinMutexLock lock(&mtx_);
+    for (uptr i = 0; i < Cache::kSize / 2; i++) {
+      IndexT idx = c->cache[--c->pos];
+      *(IndexT*)Map(idx) = freelist_;
+      freelist_ = idx;
+    }
+  }
+};
+
+}  // namespace __tsan
+
+#endif  // TSAN_DENSE_ALLOC_H
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_dispatch_defs.h b/compiler-rt/lib/tsan/rtl-old/tsan_dispatch_defs.h
new file mode 100644
index 000000000000..94e0b50fed36
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_dispatch_defs.h
@@ -0,0 +1,73 @@
+//===-- tsan_dispatch_defs.h ------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_DISPATCH_DEFS_H
+#define TSAN_DISPATCH_DEFS_H
+
+#include "sanitizer_common/sanitizer_internal_defs.h"
+
+typedef struct dispatch_object_s {} *dispatch_object_t;
+
+#define DISPATCH_DECL(name) \
+  typedef struct name##_s : public dispatch_object_s {} *name##_t
+
+DISPATCH_DECL(dispatch_queue);
+DISPATCH_DECL(dispatch_source);
+DISPATCH_DECL(dispatch_group);
+DISPATCH_DECL(dispatch_data);
+DISPATCH_DECL(dispatch_semaphore);
+DISPATCH_DECL(dispatch_io);
+
+typedef void (*dispatch_function_t)(void *arg);
+typedef void (^dispatch_block_t)(void);
+typedef void (^dispatch_io_handler_t)(bool done, dispatch_data_t data,
+                                      int error);
+
+typedef long dispatch_once_t;
+typedef __sanitizer::u64 dispatch_time_t;
+typedef int dispatch_fd_t;
+typedef unsigned long dispatch_io_type_t;
+typedef unsigned long dispatch_io_close_flags_t;
+
+extern "C" {
+void *dispatch_get_context(dispatch_object_t object);
+void dispatch_retain(dispatch_object_t object);
+void dispatch_release(dispatch_object_t object);
+
+extern const dispatch_block_t _dispatch_data_destructor_free;
+extern const dispatch_block_t _dispatch_data_destructor_munmap;
+} // extern "C"
+
+#define DISPATCH_DATA_DESTRUCTOR_DEFAULT nullptr
+#define DISPATCH_DATA_DESTRUCTOR_FREE    _dispatch_data_destructor_free
+#define DISPATCH_DATA_DESTRUCTOR_MUNMAP  _dispatch_data_destructor_munmap
+
+#if __has_attribute(noescape)
+# define DISPATCH_NOESCAPE __attribute__((__noescape__))
+#else
+# define DISPATCH_NOESCAPE
+#endif
+
+#if SANITIZER_MAC
+# define SANITIZER_WEAK_IMPORT extern "C" __attribute((weak_import))
+#else
+# define SANITIZER_WEAK_IMPORT extern "C" __attribute((weak))
+#endif
+
+
+// Data types used in dispatch APIs
+typedef unsigned long size_t;
+typedef unsigned long uintptr_t;
+typedef __sanitizer::s64 off_t;
+typedef __sanitizer::u16 mode_t;
+typedef long long_t;
+
+#endif  // TSAN_DISPATCH_DEFS_H
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_external.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_external.cpp
new file mode 100644
index 000000000000..19ae174f20a5
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_external.cpp
@@ -0,0 +1,126 @@
+//===-- tsan_external.cpp -------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#include "tsan_rtl.h"
+#include "sanitizer_common/sanitizer_ptrauth.h"
+
+#if !SANITIZER_GO
+#  include "tsan_interceptors.h"
+#endif
+
+namespace __tsan {
+
+#define CALLERPC ((uptr)__builtin_return_address(0))
+
+struct TagData {
+  const char *object_type;
+  const char *header;
+};
+
+static TagData registered_tags[kExternalTagMax] = {
+  {},
+  {"Swift variable", "Swift access race"},
+};
+static atomic_uint32_t used_tags{kExternalTagFirstUserAvailable};
+static TagData *GetTagData(uptr tag) {
+  // Invalid/corrupted tag?  Better return NULL and let the caller deal with it.
+  if (tag >= atomic_load(&used_tags, memory_order_relaxed)) return nullptr;
+  return &registered_tags[tag];
+}
+
+const char *GetObjectTypeFromTag(uptr tag) {
+  TagData *tag_data = GetTagData(tag);
+  return tag_data ? tag_data->object_type : nullptr;
+}
+
+const char *GetReportHeaderFromTag(uptr tag) {
+  TagData *tag_data = GetTagData(tag);
+  return tag_data ? tag_data->header : nullptr;
+}
+
+void InsertShadowStackFrameForTag(ThreadState *thr, uptr tag) {
+  FuncEntry(thr, (uptr)&registered_tags[tag]);
+}
+
+uptr TagFromShadowStackFrame(uptr pc) {
+  uptr tag_count = atomic_load(&used_tags, memory_order_relaxed);
+  void *pc_ptr = (void *)pc;
+  if (pc_ptr < GetTagData(0) || pc_ptr > GetTagData(tag_count - 1))
+    return 0;
+  return (TagData *)pc_ptr - GetTagData(0);
+}
+
+#if !SANITIZER_GO
+
+void ExternalAccess(void *addr, uptr caller_pc, void *tag, AccessType typ) {
+  CHECK_LT(tag, atomic_load(&used_tags, memory_order_relaxed));
+  ThreadState *thr = cur_thread();
+  if (caller_pc) FuncEntry(thr, caller_pc);
+  InsertShadowStackFrameForTag(thr, (uptr)tag);
+  bool in_ignored_lib;
+  if (!caller_pc || !libignore()->IsIgnored(caller_pc, &in_ignored_lib))
+    MemoryAccess(thr, CALLERPC, (uptr)addr, 1, typ);
+  FuncExit(thr);
+  if (caller_pc) FuncExit(thr);
+}
+
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__tsan_external_register_tag(const char *object_type) {
+  uptr new_tag = atomic_fetch_add(&used_tags, 1, memory_order_relaxed);
+  CHECK_LT(new_tag, kExternalTagMax);
+  GetTagData(new_tag)->object_type = internal_strdup(object_type);
+  char header[127] = {0};
+  internal_snprintf(header, sizeof(header), "race on %s", object_type);
+  GetTagData(new_tag)->header = internal_strdup(header);
+  return (void *)new_tag;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_external_register_header(void *tag, const char *header) {
+  CHECK_GE((uptr)tag, kExternalTagFirstUserAvailable);
+  CHECK_LT((uptr)tag, kExternalTagMax);
+  atomic_uintptr_t *header_ptr =
+      (atomic_uintptr_t *)&GetTagData((uptr)tag)->header;
+  header = internal_strdup(header);
+  char *old_header =
+      (char *)atomic_exchange(header_ptr, (uptr)header, memory_order_seq_cst);
+  Free(old_header);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_external_assign_tag(void *addr, void *tag) {
+  CHECK_LT(tag, atomic_load(&used_tags, memory_order_relaxed));
+  Allocator *a = allocator();
+  MBlock *b = nullptr;
+  if (a->PointerIsMine((void *)addr)) {
+    void *block_begin = a->GetBlockBegin((void *)addr);
+    if (block_begin) b = ctx->metamap.GetBlock((uptr)block_begin);
+  }
+  if (b) {
+    b->tag = (uptr)tag;
+  }
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_external_read(void *addr, void *caller_pc, void *tag) {
+  ExternalAccess(addr, STRIP_PAC_PC(caller_pc), tag, kAccessRead);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_external_write(void *addr, void *caller_pc, void *tag) {
+  ExternalAccess(addr, STRIP_PAC_PC(caller_pc), tag, kAccessWrite);
+}
+}  // extern "C"
+
+#endif  // !SANITIZER_GO
+
+}  // namespace __tsan
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_fd.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_fd.cpp
new file mode 100644
index 000000000000..255ffa8daf76
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_fd.cpp
@@ -0,0 +1,316 @@
+//===-- tsan_fd.cpp -------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+
+#include "tsan_fd.h"
+#include "tsan_rtl.h"
+#include <sanitizer_common/sanitizer_atomic.h>
+
+namespace __tsan {
+
+const int kTableSizeL1 = 1024;
+const int kTableSizeL2 = 1024;
+const int kTableSize = kTableSizeL1 * kTableSizeL2;
+
+struct FdSync {
+  atomic_uint64_t rc;
+};
+
+struct FdDesc {
+  FdSync *sync;
+  Tid creation_tid;
+  StackID creation_stack;
+};
+
+struct FdContext {
+  atomic_uintptr_t tab[kTableSizeL1];
+  // Addresses used for synchronization.
+  FdSync globsync;
+  FdSync filesync;
+  FdSync socksync;
+  u64 connectsync;
+};
+
+static FdContext fdctx;
+
+static bool bogusfd(int fd) {
+  // Apparently a bogus fd value.
+  return fd < 0 || fd >= kTableSize;
+}
+
+static FdSync *allocsync(ThreadState *thr, uptr pc) {
+  FdSync *s = (FdSync*)user_alloc_internal(thr, pc, sizeof(FdSync),
+      kDefaultAlignment, false);
+  atomic_store(&s->rc, 1, memory_order_relaxed);
+  return s;
+}
+
+static FdSync *ref(FdSync *s) {
+  if (s && atomic_load(&s->rc, memory_order_relaxed) != (u64)-1)
+    atomic_fetch_add(&s->rc, 1, memory_order_relaxed);
+  return s;
+}
+
+static void unref(ThreadState *thr, uptr pc, FdSync *s) {
+  if (s && atomic_load(&s->rc, memory_order_relaxed) != (u64)-1) {
+    if (atomic_fetch_sub(&s->rc, 1, memory_order_acq_rel) == 1) {
+      CHECK_NE(s, &fdctx.globsync);
+      CHECK_NE(s, &fdctx.filesync);
+      CHECK_NE(s, &fdctx.socksync);
+      user_free(thr, pc, s, false);
+    }
+  }
+}
+
+static FdDesc *fddesc(ThreadState *thr, uptr pc, int fd) {
+  CHECK_GE(fd, 0);
+  CHECK_LT(fd, kTableSize);
+  atomic_uintptr_t *pl1 = &fdctx.tab[fd / kTableSizeL2];
+  uptr l1 = atomic_load(pl1, memory_order_consume);
+  if (l1 == 0) {
+    uptr size = kTableSizeL2 * sizeof(FdDesc);
+    // We need this to reside in user memory to properly catch races on it.
+    void *p = user_alloc_internal(thr, pc, size, kDefaultAlignment, false);
+    internal_memset(p, 0, size);
+    MemoryResetRange(thr, (uptr)&fddesc, (uptr)p, size);
+    if (atomic_compare_exchange_strong(pl1, &l1, (uptr)p, memory_order_acq_rel))
+      l1 = (uptr)p;
+    else
+      user_free(thr, pc, p, false);
+  }
+  FdDesc *fds = reinterpret_cast<FdDesc *>(l1);
+  return &fds[fd % kTableSizeL2];
+}
+
+// pd must be already ref'ed.
+static void init(ThreadState *thr, uptr pc, int fd, FdSync *s,
+    bool write = true) {
+  FdDesc *d = fddesc(thr, pc, fd);
+  // As a matter of fact, we don't intercept all close calls.
+  // See e.g. libc __res_iclose().
+  if (d->sync) {
+    unref(thr, pc, d->sync);
+    d->sync = 0;
+  }
+  if (flags()->io_sync == 0) {
+    unref(thr, pc, s);
+  } else if (flags()->io_sync == 1) {
+    d->sync = s;
+  } else if (flags()->io_sync == 2) {
+    unref(thr, pc, s);
+    d->sync = &fdctx.globsync;
+  }
+  d->creation_tid = thr->tid;
+  d->creation_stack = CurrentStackId(thr, pc);
+  if (write) {
+    // To catch races between fd usage and open.
+    MemoryRangeImitateWrite(thr, pc, (uptr)d, 8);
+  } else {
+    // See the dup-related comment in FdClose.
+    MemoryAccess(thr, pc, (uptr)d, 8, kAccessRead);
+  }
+}
+
+void FdInit() {
+  atomic_store(&fdctx.globsync.rc, (u64)-1, memory_order_relaxed);
+  atomic_store(&fdctx.filesync.rc, (u64)-1, memory_order_relaxed);
+  atomic_store(&fdctx.socksync.rc, (u64)-1, memory_order_relaxed);
+}
+
+void FdOnFork(ThreadState *thr, uptr pc) {
+  // On fork() we need to reset all fd's, because the child is going
+  // close all them, and that will cause races between previous read/write
+  // and the close.
+  for (int l1 = 0; l1 < kTableSizeL1; l1++) {
+    FdDesc *tab = (FdDesc*)atomic_load(&fdctx.tab[l1], memory_order_relaxed);
+    if (tab == 0)
+      break;
+    for (int l2 = 0; l2 < kTableSizeL2; l2++) {
+      FdDesc *d = &tab[l2];
+      MemoryResetRange(thr, pc, (uptr)d, 8);
+    }
+  }
+}
+
+bool FdLocation(uptr addr, int *fd, Tid *tid, StackID *stack) {
+  for (int l1 = 0; l1 < kTableSizeL1; l1++) {
+    FdDesc *tab = (FdDesc*)atomic_load(&fdctx.tab[l1], memory_order_relaxed);
+    if (tab == 0)
+      break;
+    if (addr >= (uptr)tab && addr < (uptr)(tab + kTableSizeL2)) {
+      int l2 = (addr - (uptr)tab) / sizeof(FdDesc);
+      FdDesc *d = &tab[l2];
+      *fd = l1 * kTableSizeL1 + l2;
+      *tid = d->creation_tid;
+      *stack = d->creation_stack;
+      return true;
+    }
+  }
+  return false;
+}
+
+void FdAcquire(ThreadState *thr, uptr pc, int fd) {
+  if (bogusfd(fd))
+    return;
+  FdDesc *d = fddesc(thr, pc, fd);
+  FdSync *s = d->sync;
+  DPrintf("#%d: FdAcquire(%d) -> %p\n", thr->tid, fd, s);
+  MemoryAccess(thr, pc, (uptr)d, 8, kAccessRead);
+  if (s)
+    Acquire(thr, pc, (uptr)s);
+}
+
+void FdRelease(ThreadState *thr, uptr pc, int fd) {
+  if (bogusfd(fd))
+    return;
+  FdDesc *d = fddesc(thr, pc, fd);
+  FdSync *s = d->sync;
+  DPrintf("#%d: FdRelease(%d) -> %p\n", thr->tid, fd, s);
+  MemoryAccess(thr, pc, (uptr)d, 8, kAccessRead);
+  if (s)
+    Release(thr, pc, (uptr)s);
+}
+
+void FdAccess(ThreadState *thr, uptr pc, int fd) {
+  DPrintf("#%d: FdAccess(%d)\n", thr->tid, fd);
+  if (bogusfd(fd))
+    return;
+  FdDesc *d = fddesc(thr, pc, fd);
+  MemoryAccess(thr, pc, (uptr)d, 8, kAccessRead);
+}
+
+void FdClose(ThreadState *thr, uptr pc, int fd, bool write) {
+  DPrintf("#%d: FdClose(%d)\n", thr->tid, fd);
+  if (bogusfd(fd))
+    return;
+  FdDesc *d = fddesc(thr, pc, fd);
+  if (write) {
+    // To catch races between fd usage and close.
+    MemoryAccess(thr, pc, (uptr)d, 8, kAccessWrite);
+  } else {
+    // This path is used only by dup2/dup3 calls.
+    // We do read instead of write because there is a number of legitimate
+    // cases where write would lead to false positives:
+    // 1. Some software dups a closed pipe in place of a socket before closing
+    //    the socket (to prevent races actually).
+    // 2. Some daemons dup /dev/null in place of stdin/stdout.
+    // On the other hand we have not seen cases when write here catches real
+    // bugs.
+    MemoryAccess(thr, pc, (uptr)d, 8, kAccessRead);
+  }
+  // We need to clear it, because if we do not intercept any call out there
+  // that creates fd, we will hit false postives.
+  MemoryResetRange(thr, pc, (uptr)d, 8);
+  unref(thr, pc, d->sync);
+  d->sync = 0;
+  d->creation_tid = kInvalidTid;
+  d->creation_stack = kInvalidStackID;
+}
+
+void FdFileCreate(ThreadState *thr, uptr pc, int fd) {
+  DPrintf("#%d: FdFileCreate(%d)\n", thr->tid, fd);
+  if (bogusfd(fd))
+    return;
+  init(thr, pc, fd, &fdctx.filesync);
+}
+
+void FdDup(ThreadState *thr, uptr pc, int oldfd, int newfd, bool write) {
+  DPrintf("#%d: FdDup(%d, %d)\n", thr->tid, oldfd, newfd);
+  if (bogusfd(oldfd) || bogusfd(newfd))
+    return;
+  // Ignore the case when user dups not yet connected socket.
+  FdDesc *od = fddesc(thr, pc, oldfd);
+  MemoryAccess(thr, pc, (uptr)od, 8, kAccessRead);
+  FdClose(thr, pc, newfd, write);
+  init(thr, pc, newfd, ref(od->sync), write);
+}
+
+void FdPipeCreate(ThreadState *thr, uptr pc, int rfd, int wfd) {
+  DPrintf("#%d: FdCreatePipe(%d, %d)\n", thr->tid, rfd, wfd);
+  FdSync *s = allocsync(thr, pc);
+  init(thr, pc, rfd, ref(s));
+  init(thr, pc, wfd, ref(s));
+  unref(thr, pc, s);
+}
+
+void FdEventCreate(ThreadState *thr, uptr pc, int fd) {
+  DPrintf("#%d: FdEventCreate(%d)\n", thr->tid, fd);
+  if (bogusfd(fd))
+    return;
+  init(thr, pc, fd, allocsync(thr, pc));
+}
+
+void FdSignalCreate(ThreadState *thr, uptr pc, int fd) {
+  DPrintf("#%d: FdSignalCreate(%d)\n", thr->tid, fd);
+  if (bogusfd(fd))
+    return;
+  init(thr, pc, fd, 0);
+}
+
+void FdInotifyCreate(ThreadState *thr, uptr pc, int fd) {
+  DPrintf("#%d: FdInotifyCreate(%d)\n", thr->tid, fd);
+  if (bogusfd(fd))
+    return;
+  init(thr, pc, fd, 0);
+}
+
+void FdPollCreate(ThreadState *thr, uptr pc, int fd) {
+  DPrintf("#%d: FdPollCreate(%d)\n", thr->tid, fd);
+  if (bogusfd(fd))
+    return;
+  init(thr, pc, fd, allocsync(thr, pc));
+}
+
+void FdSocketCreate(ThreadState *thr, uptr pc, int fd) {
+  DPrintf("#%d: FdSocketCreate(%d)\n", thr->tid, fd);
+  if (bogusfd(fd))
+    return;
+  // It can be a UDP socket.
+  init(thr, pc, fd, &fdctx.socksync);
+}
+
+void FdSocketAccept(ThreadState *thr, uptr pc, int fd, int newfd) {
+  DPrintf("#%d: FdSocketAccept(%d, %d)\n", thr->tid, fd, newfd);
+  if (bogusfd(fd))
+    return;
+  // Synchronize connect->accept.
+  Acquire(thr, pc, (uptr)&fdctx.connectsync);
+  init(thr, pc, newfd, &fdctx.socksync);
+}
+
+void FdSocketConnecting(ThreadState *thr, uptr pc, int fd) {
+  DPrintf("#%d: FdSocketConnecting(%d)\n", thr->tid, fd);
+  if (bogusfd(fd))
+    return;
+  // Synchronize connect->accept.
+  Release(thr, pc, (uptr)&fdctx.connectsync);
+}
+
+void FdSocketConnect(ThreadState *thr, uptr pc, int fd) {
+  DPrintf("#%d: FdSocketConnect(%d)\n", thr->tid, fd);
+  if (bogusfd(fd))
+    return;
+  init(thr, pc, fd, &fdctx.socksync);
+}
+
+uptr File2addr(const char *path) {
+  (void)path;
+  static u64 addr;
+  return (uptr)&addr;
+}
+
+uptr Dir2addr(const char *path) {
+  (void)path;
+  static u64 addr;
+  return (uptr)&addr;
+}
+
+}  //  namespace __tsan
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_fd.h b/compiler-rt/lib/tsan/rtl-old/tsan_fd.h
new file mode 100644
index 000000000000..d9648178481c
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_fd.h
@@ -0,0 +1,64 @@
+//===-- tsan_fd.h -----------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// This file handles synchronization via IO.
+// People use IO for synchronization along the lines of:
+//
+// int X;
+// int client_socket;  // initialized elsewhere
+// int server_socket;  // initialized elsewhere
+//
+// Thread 1:
+// X = 42;
+// send(client_socket, ...);
+//
+// Thread 2:
+// if (recv(server_socket, ...) > 0)
+//   assert(X == 42);
+//
+// This file determines the scope of the file descriptor (pipe, socket,
+// all local files, etc) and executes acquire and release operations on
+// the scope as necessary.  Some scopes are very fine grained (e.g. pipe
+// operations synchronize only with operations on the same pipe), while
+// others are corse-grained (e.g. all operations on local files synchronize
+// with each other).
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_FD_H
+#define TSAN_FD_H
+
+#include "tsan_rtl.h"
+
+namespace __tsan {
+
+void FdInit();
+void FdAcquire(ThreadState *thr, uptr pc, int fd);
+void FdRelease(ThreadState *thr, uptr pc, int fd);
+void FdAccess(ThreadState *thr, uptr pc, int fd);
+void FdClose(ThreadState *thr, uptr pc, int fd, bool write = true);
+void FdFileCreate(ThreadState *thr, uptr pc, int fd);
+void FdDup(ThreadState *thr, uptr pc, int oldfd, int newfd, bool write);
+void FdPipeCreate(ThreadState *thr, uptr pc, int rfd, int wfd);
+void FdEventCreate(ThreadState *thr, uptr pc, int fd);
+void FdSignalCreate(ThreadState *thr, uptr pc, int fd);
+void FdInotifyCreate(ThreadState *thr, uptr pc, int fd);
+void FdPollCreate(ThreadState *thr, uptr pc, int fd);
+void FdSocketCreate(ThreadState *thr, uptr pc, int fd);
+void FdSocketAccept(ThreadState *thr, uptr pc, int fd, int newfd);
+void FdSocketConnecting(ThreadState *thr, uptr pc, int fd);
+void FdSocketConnect(ThreadState *thr, uptr pc, int fd);
+bool FdLocation(uptr addr, int *fd, Tid *tid, StackID *stack);
+void FdOnFork(ThreadState *thr, uptr pc);
+
+uptr File2addr(const char *path);
+uptr Dir2addr(const char *path);
+
+}  // namespace __tsan
+
+#endif  // TSAN_INTERFACE_H
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_flags.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_flags.cpp
new file mode 100644
index 000000000000..ee89862d17bd
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_flags.cpp
@@ -0,0 +1,126 @@
+//===-- tsan_flags.cpp ----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_flags.h"
+#include "sanitizer_common/sanitizer_flag_parser.h"
+#include "sanitizer_common/sanitizer_libc.h"
+#include "tsan_flags.h"
+#include "tsan_rtl.h"
+#include "tsan_mman.h"
+#include "ubsan/ubsan_flags.h"
+
+namespace __tsan {
+
+// Can be overriden in frontend.
+#ifdef TSAN_EXTERNAL_HOOKS
+extern "C" const char* __tsan_default_options();
+#else
+SANITIZER_WEAK_DEFAULT_IMPL
+const char *__tsan_default_options() {
+  return "";
+}
+#endif
+
+void Flags::SetDefaults() {
+#define TSAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue;
+#include "tsan_flags.inc"
+#undef TSAN_FLAG
+  // DDFlags
+  second_deadlock_stack = false;
+}
+
+void RegisterTsanFlags(FlagParser *parser, Flags *f) {
+#define TSAN_FLAG(Type, Name, DefaultValue, Description) \
+  RegisterFlag(parser, #Name, Description, &f->Name);
+#include "tsan_flags.inc"
+#undef TSAN_FLAG
+  // DDFlags
+  RegisterFlag(parser, "second_deadlock_stack",
+      "Report where each mutex is locked in deadlock reports",
+      &f->second_deadlock_stack);
+}
+
+void InitializeFlags(Flags *f, const char *env, const char *env_option_name) {
+  SetCommonFlagsDefaults();
+  {
+    // Override some common flags defaults.
+    CommonFlags cf;
+    cf.CopyFrom(*common_flags());
+    cf.external_symbolizer_path = GetEnv("TSAN_SYMBOLIZER_PATH");
+    cf.allow_addr2line = true;
+    if (SANITIZER_GO) {
+      // Does not work as expected for Go: runtime handles SIGABRT and crashes.
+      cf.abort_on_error = false;
+      // Go does not have mutexes.
+      cf.detect_deadlocks = false;
+    }
+    cf.print_suppressions = false;
+    cf.stack_trace_format = "    #%n %f %S %M";
+    cf.exitcode = 66;
+    cf.intercept_tls_get_addr = true;
+    OverrideCommonFlags(cf);
+  }
+
+  f->SetDefaults();
+
+  FlagParser parser;
+  RegisterTsanFlags(&parser, f);
+  RegisterCommonFlags(&parser);
+
+#if TSAN_CONTAINS_UBSAN
+  __ubsan::Flags *uf = __ubsan::flags();
+  uf->SetDefaults();
+
+  FlagParser ubsan_parser;
+  __ubsan::RegisterUbsanFlags(&ubsan_parser, uf);
+  RegisterCommonFlags(&ubsan_parser);
+#endif
+
+  // Let a frontend override.
+  parser.ParseString(__tsan_default_options());
+#if TSAN_CONTAINS_UBSAN
+  const char *ubsan_default_options = __ubsan_default_options();
+  ubsan_parser.ParseString(ubsan_default_options);
+#endif
+  // Override from command line.
+  parser.ParseString(env, env_option_name);
+#if TSAN_CONTAINS_UBSAN
+  ubsan_parser.ParseStringFromEnv("UBSAN_OPTIONS");
+#endif
+
+  // Sanity check.
+  if (!f->report_bugs) {
+    f->report_thread_leaks = false;
+    f->report_destroy_locked = false;
+    f->report_signal_unsafe = false;
+  }
+
+  InitializeCommonFlags();
+
+  if (Verbosity()) ReportUnrecognizedFlags();
+
+  if (common_flags()->help) parser.PrintFlagDescriptions();
+
+  if (f->history_size < 0 || f->history_size > 7) {
+    Printf("ThreadSanitizer: incorrect value for history_size"
+           " (must be [0..7])\n");
+    Die();
+  }
+
+  if (f->io_sync < 0 || f->io_sync > 2) {
+    Printf("ThreadSanitizer: incorrect value for io_sync"
+           " (must be [0..2])\n");
+    Die();
+  }
+}
+
+}  // namespace __tsan
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_flags.h b/compiler-rt/lib/tsan/rtl-old/tsan_flags.h
new file mode 100644
index 000000000000..da27d5b992bc
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_flags.h
@@ -0,0 +1,34 @@
+//===-- tsan_flags.h --------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+// NOTE: This file may be included into user code.
+//===----------------------------------------------------------------------===//
+
+#ifndef TSAN_FLAGS_H
+#define TSAN_FLAGS_H
+
+#include "sanitizer_common/sanitizer_flags.h"
+#include "sanitizer_common/sanitizer_deadlock_detector_interface.h"
+
+namespace __tsan {
+
+struct Flags : DDFlags {
+#define TSAN_FLAG(Type, Name, DefaultValue, Description) Type Name;
+#include "tsan_flags.inc"
+#undef TSAN_FLAG
+
+  void SetDefaults();
+  void ParseFromString(const char *str);
+};
+
+void InitializeFlags(Flags *flags, const char *env,
+                     const char *env_option_name = nullptr);
+}  // namespace __tsan
+
+#endif  // TSAN_FLAGS_H
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_flags.inc b/compiler-rt/lib/tsan/rtl-old/tsan_flags.inc
new file mode 100644
index 000000000000..7954a4307fa1
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_flags.inc
@@ -0,0 +1,84 @@
+//===-- tsan_flags.inc ------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// TSan runtime flags.
+//
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_FLAG
+# error "Define TSAN_FLAG prior to including this file!"
+#endif
+
+// TSAN_FLAG(Type, Name, DefaultValue, Description)
+// See COMMON_FLAG in sanitizer_flags.inc for more details.
+
+TSAN_FLAG(bool, enable_annotations, true,
+          "Enable dynamic annotations, otherwise they are no-ops.")
+// Suppress a race report if we've already output another race report
+// with the same stack.
+TSAN_FLAG(bool, suppress_equal_stacks, true,
+          "Suppress a race report if we've already output another race report "
+          "with the same stack.")
+TSAN_FLAG(bool, suppress_equal_addresses, true,
+          "Suppress a race report if we've already output another race report "
+          "on the same address.")
+
+TSAN_FLAG(bool, report_bugs, true,
+          "Turns off bug reporting entirely (useful for benchmarking).")
+TSAN_FLAG(bool, report_thread_leaks, true, "Report thread leaks at exit?")
+TSAN_FLAG(bool, report_destroy_locked, true,
+          "Report destruction of a locked mutex?")
+TSAN_FLAG(bool, report_mutex_bugs, true,
+          "Report incorrect usages of mutexes and mutex annotations?")
+TSAN_FLAG(bool, report_signal_unsafe, true,
+          "Report violations of async signal-safety "
+          "(e.g. malloc() call from a signal handler).")
+TSAN_FLAG(bool, report_atomic_races, true,
+          "Report races between atomic and plain memory accesses.")
+TSAN_FLAG(
+    bool, force_seq_cst_atomics, false,
+    "If set, all atomics are effectively sequentially consistent (seq_cst), "
+    "regardless of what user actually specified.")
+TSAN_FLAG(bool, halt_on_error, false, "Exit after first reported error.")
+TSAN_FLAG(int, atexit_sleep_ms, 1000,
+          "Sleep in main thread before exiting for that many ms "
+          "(useful to catch \"at exit\" races).")
+TSAN_FLAG(const char *, profile_memory, "",
+          "If set, periodically write memory profile to that file.")
+TSAN_FLAG(int, flush_memory_ms, 0, "Flush shadow memory every X ms.")
+TSAN_FLAG(int, flush_symbolizer_ms, 5000, "Flush symbolizer caches every X ms.")
+TSAN_FLAG(
+    int, memory_limit_mb, 0,
+    "Resident memory limit in MB to aim at."
+    "If the process consumes more memory, then TSan will flush shadow memory.")
+TSAN_FLAG(bool, stop_on_start, false,
+          "Stops on start until __tsan_resume() is called (for debugging).")
+TSAN_FLAG(bool, running_on_valgrind, false,
+          "Controls whether RunningOnValgrind() returns true or false.")
+// There are a lot of goroutines in Go, so we use smaller history.
+TSAN_FLAG(
+    int, history_size, SANITIZER_GO ? 1 : 3,
+    "Per-thread history size, controls how many previous memory accesses "
+    "are remembered per thread.  Possible values are [0..7]. "
+    "history_size=0 amounts to 32K memory accesses.  Each next value doubles "
+    "the amount of memory accesses, up to history_size=7 that amounts to "
+    "4M memory accesses.  The default value is 2 (128K memory accesses).")
+TSAN_FLAG(int, io_sync, 1,
+          "Controls level of synchronization implied by IO operations. "
+          "0 - no synchronization "
+          "1 - reasonable level of synchronization (write->read)"
+          "2 - global synchronization of all IO operations.")
+TSAN_FLAG(bool, die_after_fork, true,
+          "Die after multi-threaded fork if the child creates new threads.")
+TSAN_FLAG(const char *, suppressions, "", "Suppressions file name.")
+TSAN_FLAG(bool, ignore_interceptors_accesses, SANITIZER_MAC ? true : false,
+          "Ignore reads and writes from all interceptors.")
+TSAN_FLAG(bool, ignore_noninstrumented_modules, SANITIZER_MAC ? true : false,
+          "Interceptors should only detect races when called from instrumented "
+          "modules.")
+TSAN_FLAG(bool, shared_ptr_interceptor, true,
+          "Track atomic reference counting in libc++ shared_ptr and weak_ptr.")
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_ignoreset.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_ignoreset.cpp
new file mode 100644
index 000000000000..1fca1cf4f9fc
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_ignoreset.cpp
@@ -0,0 +1,38 @@
+//===-- tsan_ignoreset.cpp ------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#include "tsan_ignoreset.h"
+
+namespace __tsan {
+
+const uptr IgnoreSet::kMaxSize;
+
+IgnoreSet::IgnoreSet()
+    : size_() {
+}
+
+void IgnoreSet::Add(StackID stack_id) {
+  if (size_ == kMaxSize)
+    return;
+  for (uptr i = 0; i < size_; i++) {
+    if (stacks_[i] == stack_id)
+      return;
+  }
+  stacks_[size_++] = stack_id;
+}
+
+StackID IgnoreSet::At(uptr i) const {
+  CHECK_LT(i, size_);
+  CHECK_LE(size_, kMaxSize);
+  return stacks_[i];
+}
+
+}  // namespace __tsan
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_ignoreset.h b/compiler-rt/lib/tsan/rtl-old/tsan_ignoreset.h
new file mode 100644
index 000000000000..4e2511291ce4
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_ignoreset.h
@@ -0,0 +1,36 @@
+//===-- tsan_ignoreset.h ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// IgnoreSet holds a set of stack traces where ignores were enabled.
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_IGNORESET_H
+#define TSAN_IGNORESET_H
+
+#include "tsan_defs.h"
+
+namespace __tsan {
+
+class IgnoreSet {
+ public:
+  IgnoreSet();
+  void Add(StackID stack_id);
+  void Reset() { size_ = 0; }
+  uptr Size() const { return size_; }
+  StackID At(uptr i) const;
+
+ private:
+  static constexpr uptr kMaxSize = 16;
+  uptr size_;
+  StackID stacks_[kMaxSize];
+};
+
+}  // namespace __tsan
+
+#endif  // TSAN_IGNORESET_H
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_ilist.h b/compiler-rt/lib/tsan/rtl-old/tsan_ilist.h
new file mode 100644
index 000000000000..d7d8be219dbe
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_ilist.h
@@ -0,0 +1,189 @@
+//===-- tsan_ilist.h --------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_ILIST_H
+#define TSAN_ILIST_H
+
+#include "sanitizer_common/sanitizer_internal_defs.h"
+
+namespace __tsan {
+
+class INode {
+ public:
+  INode() = default;
+
+ private:
+  INode* next_ = nullptr;
+  INode* prev_ = nullptr;
+
+  template <typename Base, INode Base::*Node, typename Elem>
+  friend class IList;
+  INode(const INode&) = delete;
+  void operator=(const INode&) = delete;
+};
+
+// Intrusive doubly-linked list.
+//
+// The node class (MyNode) needs to include "INode foo" field,
+// then the list can be declared as IList<MyNode, &MyNode::foo>.
+// This design allows to link MyNode into multiple lists using
+// different INode fields.
+// The optional Elem template argument allows to specify node MDT
+// (most derived type) if it's different from MyNode.
+template <typename Base, INode Base::*Node, typename Elem = Base>
+class IList {
+ public:
+  IList();
+
+  void PushFront(Elem* e);
+  void PushBack(Elem* e);
+  void Remove(Elem* e);
+
+  Elem* PopFront();
+  Elem* PopBack();
+  Elem* Front();
+  Elem* Back();
+
+  // Prev links point towards front of the queue.
+  Elem* Prev(Elem* e);
+  // Next links point towards back of the queue.
+  Elem* Next(Elem* e);
+
+  uptr Size() const;
+  bool Empty() const;
+  bool Queued(Elem* e) const;
+
+ private:
+  INode node_;
+  uptr size_ = 0;
+
+  void Push(Elem* e, INode* after);
+  static INode* ToNode(Elem* e);
+  static Elem* ToElem(INode* n);
+
+  IList(const IList&) = delete;
+  void operator=(const IList&) = delete;
+};
+
+template <typename Base, INode Base::*Node, typename Elem>
+IList<Base, Node, Elem>::IList() {
+  node_.next_ = node_.prev_ = &node_;
+}
+
+template <typename Base, INode Base::*Node, typename Elem>
+void IList<Base, Node, Elem>::PushFront(Elem* e) {
+  Push(e, &node_);
+}
+
+template <typename Base, INode Base::*Node, typename Elem>
+void IList<Base, Node, Elem>::PushBack(Elem* e) {
+  Push(e, node_.prev_);
+}
+
+template <typename Base, INode Base::*Node, typename Elem>
+void IList<Base, Node, Elem>::Push(Elem* e, INode* after) {
+  INode* n = ToNode(e);
+  DCHECK_EQ(n->next_, nullptr);
+  DCHECK_EQ(n->prev_, nullptr);
+  INode* next = after->next_;
+  n->next_ = next;
+  n->prev_ = after;
+  next->prev_ = n;
+  after->next_ = n;
+  size_++;
+}
+
+template <typename Base, INode Base::*Node, typename Elem>
+void IList<Base, Node, Elem>::Remove(Elem* e) {
+  INode* n = ToNode(e);
+  INode* next = n->next_;
+  INode* prev = n->prev_;
+  DCHECK(next);
+  DCHECK(prev);
+  DCHECK(size_);
+  next->prev_ = prev;
+  prev->next_ = next;
+  n->prev_ = n->next_ = nullptr;
+  size_--;
+}
+
+template <typename Base, INode Base::*Node, typename Elem>
+Elem* IList<Base, Node, Elem>::PopFront() {
+  Elem* e = Front();
+  if (e)
+    Remove(e);
+  return e;
+}
+
+template <typename Base, INode Base::*Node, typename Elem>
+Elem* IList<Base, Node, Elem>::PopBack() {
+  Elem* e = Back();
+  if (e)
+    Remove(e);
+  return e;
+}
+
+template <typename Base, INode Base::*Node, typename Elem>
+Elem* IList<Base, Node, Elem>::Front() {
+  return size_ ? ToElem(node_.next_) : nullptr;
+}
+
+template <typename Base, INode Base::*Node, typename Elem>
+Elem* IList<Base, Node, Elem>::Back() {
+  return size_ ? ToElem(node_.prev_) : nullptr;
+}
+
+template <typename Base, INode Base::*Node, typename Elem>
+Elem* IList<Base, Node, Elem>::Prev(Elem* e) {
+  INode* n = ToNode(e);
+  DCHECK(n->prev_);
+  return n->prev_ != &node_ ? ToElem(n->prev_) : nullptr;
+}
+
+template <typename Base, INode Base::*Node, typename Elem>
+Elem* IList<Base, Node, Elem>::Next(Elem* e) {
+  INode* n = ToNode(e);
+  DCHECK(n->next_);
+  return n->next_ != &node_ ? ToElem(n->next_) : nullptr;
+}
+
+template <typename Base, INode Base::*Node, typename Elem>
+uptr IList<Base, Node, Elem>::Size() const {
+  return size_;
+}
+
+template <typename Base, INode Base::*Node, typename Elem>
+bool IList<Base, Node, Elem>::Empty() const {
+  return size_ == 0;
+}
+
+template <typename Base, INode Base::*Node, typename Elem>
+bool IList<Base, Node, Elem>::Queued(Elem* e) const {
+  INode* n = ToNode(e);
+  DCHECK_EQ(!n->next_, !n->prev_);
+  return n->next_;
+}
+
+template <typename Base, INode Base::*Node, typename Elem>
+INode* IList<Base, Node, Elem>::ToNode(Elem* e) {
+  return &(e->*Node);
+}
+
+template <typename Base, INode Base::*Node, typename Elem>
+Elem* IList<Base, Node, Elem>::ToElem(INode* n) {
+  return static_cast<Elem*>(reinterpret_cast<Base*>(
+      reinterpret_cast<uptr>(n) -
+      reinterpret_cast<uptr>(&(reinterpret_cast<Elem*>(0)->*Node))));
+}
+
+}  // namespace __tsan
+
+#endif
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_interceptors.h b/compiler-rt/lib/tsan/rtl-old/tsan_interceptors.h
new file mode 100644
index 000000000000..61dbb81ffec4
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_interceptors.h
@@ -0,0 +1,93 @@
+#ifndef TSAN_INTERCEPTORS_H
+#define TSAN_INTERCEPTORS_H
+
+#include "sanitizer_common/sanitizer_stacktrace.h"
+#include "tsan_rtl.h"
+
+namespace __tsan {
+
+class ScopedInterceptor {
+ public:
+  ScopedInterceptor(ThreadState *thr, const char *fname, uptr pc);
+  ~ScopedInterceptor();
+  void DisableIgnores() {
+    if (UNLIKELY(ignoring_))
+      DisableIgnoresImpl();
+  }
+  void EnableIgnores() {
+    if (UNLIKELY(ignoring_))
+      EnableIgnoresImpl();
+  }
+
+ private:
+  ThreadState *const thr_;
+  bool in_ignored_lib_;
+  bool ignoring_;
+
+  void DisableIgnoresImpl();
+  void EnableIgnoresImpl();
+};
+
+LibIgnore *libignore();
+
+#if !SANITIZER_GO
+inline bool in_symbolizer() {
+  return UNLIKELY(cur_thread_init()->in_symbolizer);
+}
+#endif
+
+}  // namespace __tsan
+
+#define SCOPED_INTERCEPTOR_RAW(func, ...)            \
+  ThreadState *thr = cur_thread_init();              \
+  ScopedInterceptor si(thr, #func, GET_CALLER_PC()); \
+  UNUSED const uptr pc = GET_CURRENT_PC();
+
+#ifdef __powerpc64__
+// Debugging of crashes on powerpc after commit:
+// c80604f7a3 ("tsan: remove real func check from interceptors")
+// Somehow replacing if with DCHECK leads to strange failures in:
+// SanitizerCommon-tsan-powerpc64le-Linux :: Linux/ptrace.cpp
+// https://lab.llvm.org/buildbot/#/builders/105
+// https://lab.llvm.org/buildbot/#/builders/121
+// https://lab.llvm.org/buildbot/#/builders/57
+#  define CHECK_REAL_FUNC(func)                                          \
+    if (REAL(func) == 0) {                                               \
+      Report("FATAL: ThreadSanitizer: failed to intercept %s\n", #func); \
+      Die();                                                             \
+    }
+#else
+#  define CHECK_REAL_FUNC(func) DCHECK(REAL(func))
+#endif
+
+#define SCOPED_TSAN_INTERCEPTOR(func, ...)                                \
+  SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__);                              \
+  CHECK_REAL_FUNC(func);                                                  \
+  if (!thr->is_inited || thr->ignore_interceptors || thr->in_ignored_lib) \
+    return REAL(func)(__VA_ARGS__);
+
+#define SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START() \
+    si.DisableIgnores();
+
+#define SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END() \
+    si.EnableIgnores();
+
+#define TSAN_INTERCEPTOR(ret, func, ...) INTERCEPTOR(ret, func, __VA_ARGS__)
+
+#if SANITIZER_NETBSD
+# define TSAN_INTERCEPTOR_NETBSD_ALIAS(ret, func, ...) \
+  TSAN_INTERCEPTOR(ret, __libc_##func, __VA_ARGS__) \
+  ALIAS(WRAPPER_NAME(pthread_##func));
+# define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(ret, func, ...) \
+  TSAN_INTERCEPTOR(ret, __libc_thr_##func, __VA_ARGS__) \
+  ALIAS(WRAPPER_NAME(pthread_##func));
+# define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR2(ret, func, func2, ...) \
+  TSAN_INTERCEPTOR(ret, __libc_thr_##func, __VA_ARGS__) \
+  ALIAS(WRAPPER_NAME(pthread_##func2));
+#else
+# define TSAN_INTERCEPTOR_NETBSD_ALIAS(ret, func, ...)
+# define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(ret, func, ...)
+# define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR2(ret, func, func2, ...)
+#endif
+
+#endif  // TSAN_INTERCEPTORS_H
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_interceptors_libdispatch.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_interceptors_libdispatch.cpp
new file mode 100644
index 000000000000..cbbb7ecb2397
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_interceptors_libdispatch.cpp
@@ -0,0 +1,814 @@
+//===-- tsan_interceptors_libdispatch.cpp ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// Support for intercepting libdispatch (GCD).
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_common.h"
+#include "interception/interception.h"
+#include "tsan_interceptors.h"
+#include "tsan_rtl.h"
+
+#include "BlocksRuntime/Block.h"
+#include "tsan_dispatch_defs.h"
+
+#if SANITIZER_MAC
+# include <Availability.h>
+#endif
+
+namespace __tsan {
+  typedef u16 uint16_t;
+
+typedef struct {
+  dispatch_queue_t queue;
+  void *orig_context;
+  dispatch_function_t orig_work;
+  bool free_context_in_callback;
+  bool submitted_synchronously;
+  bool is_barrier_block;
+  uptr non_queue_sync_object;
+} block_context_t;
+
+// The offsets of different fields of the dispatch_queue_t structure, exported
+// by libdispatch.dylib.
+extern "C" struct dispatch_queue_offsets_s {
+  const uint16_t dqo_version;
+  const uint16_t dqo_label;
+  const uint16_t dqo_label_size;
+  const uint16_t dqo_flags;
+  const uint16_t dqo_flags_size;
+  const uint16_t dqo_serialnum;
+  const uint16_t dqo_serialnum_size;
+  const uint16_t dqo_width;
+  const uint16_t dqo_width_size;
+  const uint16_t dqo_running;
+  const uint16_t dqo_running_size;
+  const uint16_t dqo_suspend_cnt;
+  const uint16_t dqo_suspend_cnt_size;
+  const uint16_t dqo_target_queue;
+  const uint16_t dqo_target_queue_size;
+  const uint16_t dqo_priority;
+  const uint16_t dqo_priority_size;
+} dispatch_queue_offsets;
+
+static bool IsQueueSerial(dispatch_queue_t q) {
+  CHECK_EQ(dispatch_queue_offsets.dqo_width_size, 2);
+  uptr width = *(uint16_t *)(((uptr)q) + dispatch_queue_offsets.dqo_width);
+  CHECK_NE(width, 0);
+  return width == 1;
+}
+
+static dispatch_queue_t GetTargetQueueFromQueue(dispatch_queue_t q) {
+  CHECK_EQ(dispatch_queue_offsets.dqo_target_queue_size, 8);
+  dispatch_queue_t tq = *(
+      dispatch_queue_t *)(((uptr)q) + dispatch_queue_offsets.dqo_target_queue);
+  return tq;
+}
+
+static dispatch_queue_t GetTargetQueueFromSource(dispatch_source_t source) {
+  dispatch_queue_t tq = GetTargetQueueFromQueue((dispatch_queue_t)source);
+  CHECK_NE(tq, 0);
+  return tq;
+}
+
+static block_context_t *AllocContext(ThreadState *thr, uptr pc,
+                                     dispatch_queue_t queue, void *orig_context,
+                                     dispatch_function_t orig_work) {
+  block_context_t *new_context =
+      (block_context_t *)user_alloc_internal(thr, pc, sizeof(block_context_t));
+  new_context->queue = queue;
+  new_context->orig_context = orig_context;
+  new_context->orig_work = orig_work;
+  new_context->free_context_in_callback = true;
+  new_context->submitted_synchronously = false;
+  new_context->is_barrier_block = false;
+  new_context->non_queue_sync_object = 0;
+  return new_context;
+}
+
+#define GET_QUEUE_SYNC_VARS(context, q)                                  \
+  bool is_queue_serial = q && IsQueueSerial(q);                          \
+  uptr sync_ptr = (uptr)q ?: context->non_queue_sync_object;             \
+  uptr serial_sync = (uptr)sync_ptr;                                     \
+  uptr concurrent_sync = sync_ptr ? ((uptr)sync_ptr) + sizeof(uptr) : 0; \
+  bool serial_task = context->is_barrier_block || is_queue_serial
+
+static void dispatch_sync_pre_execute(ThreadState *thr, uptr pc,
+                                      block_context_t *context) {
+  uptr submit_sync = (uptr)context;
+  Acquire(thr, pc, submit_sync);
+
+  dispatch_queue_t q = context->queue;
+  do {
+    GET_QUEUE_SYNC_VARS(context, q);
+    if (serial_sync) Acquire(thr, pc, serial_sync);
+    if (serial_task && concurrent_sync) Acquire(thr, pc, concurrent_sync);
+
+    if (q) q = GetTargetQueueFromQueue(q);
+  } while (q);
+}
+
+static void dispatch_sync_post_execute(ThreadState *thr, uptr pc,
+                                       block_context_t *context) {
+  uptr submit_sync = (uptr)context;
+  if (context->submitted_synchronously) Release(thr, pc, submit_sync);
+
+  dispatch_queue_t q = context->queue;
+  do {
+    GET_QUEUE_SYNC_VARS(context, q);
+    if (serial_task && serial_sync) Release(thr, pc, serial_sync);
+    if (!serial_task && concurrent_sync) Release(thr, pc, concurrent_sync);
+
+    if (q) q = GetTargetQueueFromQueue(q);
+  } while (q);
+}
+
+static void dispatch_callback_wrap(void *param) {
+  SCOPED_INTERCEPTOR_RAW(dispatch_callback_wrap);
+  block_context_t *context = (block_context_t *)param;
+
+  dispatch_sync_pre_execute(thr, pc, context);
+
+  SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
+  context->orig_work(context->orig_context);
+  SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
+
+  dispatch_sync_post_execute(thr, pc, context);
+
+  if (context->free_context_in_callback) user_free(thr, pc, context);
+}
+
+static void invoke_block(void *param) {
+  dispatch_block_t block = (dispatch_block_t)param;
+  block();
+}
+
+static void invoke_and_release_block(void *param) {
+  dispatch_block_t block = (dispatch_block_t)param;
+  block();
+  Block_release(block);
+}
+
+#define DISPATCH_INTERCEPT_ASYNC_B(name, barrier)                            \
+  TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \
+    SCOPED_TSAN_INTERCEPTOR(name, q, block);                                 \
+    SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                           \
+    dispatch_block_t heap_block = Block_copy(block);                         \
+    SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                             \
+    block_context_t *new_context =                                           \
+        AllocContext(thr, pc, q, heap_block, &invoke_and_release_block);     \
+    new_context->is_barrier_block = barrier;                                 \
+    Release(thr, pc, (uptr)new_context);                                     \
+    SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                           \
+    REAL(name##_f)(q, new_context, dispatch_callback_wrap);                  \
+    SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                             \
+  }
+
+#define DISPATCH_INTERCEPT_SYNC_B(name, barrier)                             \
+  TSAN_INTERCEPTOR(void, name, dispatch_queue_t q,                           \
+                   DISPATCH_NOESCAPE dispatch_block_t block) {               \
+    SCOPED_TSAN_INTERCEPTOR(name, q, block);                                 \
+    block_context_t new_context = {                                          \
+        q, block, &invoke_block, false, true, barrier, 0};                   \
+    Release(thr, pc, (uptr)&new_context);                                    \
+    SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                           \
+    REAL(name##_f)(q, &new_context, dispatch_callback_wrap);                 \
+    SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                             \
+    Acquire(thr, pc, (uptr)&new_context);                                    \
+  }
+
+#define DISPATCH_INTERCEPT_ASYNC_F(name, barrier)                 \
+  TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \
+                   dispatch_function_t work) {                    \
+    SCOPED_TSAN_INTERCEPTOR(name, q, context, work);              \
+    block_context_t *new_context =                                \
+        AllocContext(thr, pc, q, context, work);                  \
+    new_context->is_barrier_block = barrier;                      \
+    Release(thr, pc, (uptr)new_context);                          \
+    SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                \
+    REAL(name)(q, new_context, dispatch_callback_wrap);           \
+    SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                  \
+  }
+
+#define DISPATCH_INTERCEPT_SYNC_F(name, barrier)                              \
+  TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context,             \
+                   dispatch_function_t work) {                                \
+    SCOPED_TSAN_INTERCEPTOR(name, q, context, work);                          \
+    block_context_t new_context = {                                           \
+        q, context, work, false, true, barrier, 0};                           \
+    Release(thr, pc, (uptr)&new_context);                                     \
+    SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                            \
+    REAL(name)(q, &new_context, dispatch_callback_wrap);                      \
+    SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                              \
+    Acquire(thr, pc, (uptr)&new_context);                                     \
+  }
+
+#define DISPATCH_INTERCEPT(name, barrier)             \
+  DISPATCH_INTERCEPT_ASYNC_F(name##_async_f, barrier) \
+  DISPATCH_INTERCEPT_ASYNC_B(name##_async, barrier)   \
+  DISPATCH_INTERCEPT_SYNC_F(name##_sync_f, barrier)   \
+  DISPATCH_INTERCEPT_SYNC_B(name##_sync, barrier)
+
+// We wrap dispatch_async, dispatch_sync and friends where we allocate a new
+// context, which is used to synchronize (we release the context before
+// submitting, and the callback acquires it before executing the original
+// callback).
+DISPATCH_INTERCEPT(dispatch, false)
+DISPATCH_INTERCEPT(dispatch_barrier, true)
+
+// dispatch_async_and_wait() and friends were introduced in macOS 10.14.
+// Linking of these interceptors fails when using an older SDK.
+#if !SANITIZER_MAC || defined(__MAC_10_14)
+// macOS 10.14 is greater than our minimal deployment target.  To ensure we
+// generate a weak reference so the TSan dylib continues to work on older
+// systems, we need to forward declare the intercepted functions as "weak
+// imports".   Note that this file is multi-platform, so we cannot include the
+// actual header file (#include <dispatch/dispatch.h>).
+SANITIZER_WEAK_IMPORT void dispatch_async_and_wait(
+    dispatch_queue_t queue, DISPATCH_NOESCAPE dispatch_block_t block);
+SANITIZER_WEAK_IMPORT void dispatch_async_and_wait_f(
+    dispatch_queue_t queue, void *context, dispatch_function_t work);
+SANITIZER_WEAK_IMPORT void dispatch_barrier_async_and_wait(
+    dispatch_queue_t queue, DISPATCH_NOESCAPE dispatch_block_t block);
+SANITIZER_WEAK_IMPORT void dispatch_barrier_async_and_wait_f(
+    dispatch_queue_t queue, void *context, dispatch_function_t work);
+
+DISPATCH_INTERCEPT_SYNC_F(dispatch_async_and_wait_f, false)
+DISPATCH_INTERCEPT_SYNC_B(dispatch_async_and_wait, false)
+DISPATCH_INTERCEPT_SYNC_F(dispatch_barrier_async_and_wait_f, true)
+DISPATCH_INTERCEPT_SYNC_B(dispatch_barrier_async_and_wait, true)
+#endif
+
+
+DECLARE_REAL(void, dispatch_after_f, dispatch_time_t when,
+             dispatch_queue_t queue, void *context, dispatch_function_t work)
+
+TSAN_INTERCEPTOR(void, dispatch_after, dispatch_time_t when,
+                 dispatch_queue_t queue, dispatch_block_t block) {
+  SCOPED_TSAN_INTERCEPTOR(dispatch_after, when, queue, block);
+  SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
+  dispatch_block_t heap_block = Block_copy(block);
+  SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
+  block_context_t *new_context =
+      AllocContext(thr, pc, queue, heap_block, &invoke_and_release_block);
+  Release(thr, pc, (uptr)new_context);
+  SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
+  REAL(dispatch_after_f)(when, queue, new_context, dispatch_callback_wrap);
+  SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
+}
+
+TSAN_INTERCEPTOR(void, dispatch_after_f, dispatch_time_t when,
+                 dispatch_queue_t queue, void *context,
+                 dispatch_function_t work) {
+  SCOPED_TSAN_INTERCEPTOR(dispatch_after_f, when, queue, context, work);
+  WRAP(dispatch_after)(when, queue, ^(void) {
+    work(context);
+  });
+}
+
+// GCD's dispatch_once implementation has a fast path that contains a racy read
+// and it's inlined into user's code. Furthermore, this fast path doesn't
+// establish a proper happens-before relations between the initialization and
+// code following the call to dispatch_once. We could deal with this in
+// instrumented code, but there's not much we can do about it in system
+// libraries. Let's disable the fast path (by never storing the value ~0 to
+// predicate), so the interceptor is always called, and let's add proper release
+// and acquire semantics. Since TSan does not see its own atomic stores, the
+// race on predicate won't be reported - the only accesses to it that TSan sees
+// are the loads on the fast path. Loads don't race. Secondly, dispatch_once is
+// both a macro and a real function, we want to intercept the function, so we
+// need to undefine the macro.
+#undef dispatch_once
+TSAN_INTERCEPTOR(void, dispatch_once, dispatch_once_t *predicate,
+                 DISPATCH_NOESCAPE dispatch_block_t block) {
+  SCOPED_INTERCEPTOR_RAW(dispatch_once, predicate, block);
+  atomic_uint32_t *a = reinterpret_cast<atomic_uint32_t *>(predicate);
+  u32 v = atomic_load(a, memory_order_acquire);
+  if (v == 0 &&
+      atomic_compare_exchange_strong(a, &v, 1, memory_order_relaxed)) {
+    SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
+    block();
+    SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
+    Release(thr, pc, (uptr)a);
+    atomic_store(a, 2, memory_order_release);
+  } else {
+    while (v != 2) {
+      internal_sched_yield();
+      v = atomic_load(a, memory_order_acquire);
+    }
+    Acquire(thr, pc, (uptr)a);
+  }
+}
+
+#undef dispatch_once_f
+TSAN_INTERCEPTOR(void, dispatch_once_f, dispatch_once_t *predicate,
+                 void *context, dispatch_function_t function) {
+  SCOPED_INTERCEPTOR_RAW(dispatch_once_f, predicate, context, function);
+  SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
+  WRAP(dispatch_once)(predicate, ^(void) {
+    function(context);
+  });
+  SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
+}
+
+TSAN_INTERCEPTOR(long_t, dispatch_semaphore_signal,
+                 dispatch_semaphore_t dsema) {
+  SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_signal, dsema);
+  Release(thr, pc, (uptr)dsema);
+  return REAL(dispatch_semaphore_signal)(dsema);
+}
+
+TSAN_INTERCEPTOR(long_t, dispatch_semaphore_wait, dispatch_semaphore_t dsema,
+                 dispatch_time_t timeout) {
+  SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_wait, dsema, timeout);
+  long_t result = REAL(dispatch_semaphore_wait)(dsema, timeout);
+  if (result == 0) Acquire(thr, pc, (uptr)dsema);
+  return result;
+}
+
+TSAN_INTERCEPTOR(long_t, dispatch_group_wait, dispatch_group_t group,
+                 dispatch_time_t timeout) {
+  SCOPED_TSAN_INTERCEPTOR(dispatch_group_wait, group, timeout);
+  long_t result = REAL(dispatch_group_wait)(group, timeout);
+  if (result == 0) Acquire(thr, pc, (uptr)group);
+  return result;
+}
+
+// Used, but not intercepted.
+extern "C" void dispatch_group_enter(dispatch_group_t group);
+
+TSAN_INTERCEPTOR(void, dispatch_group_leave, dispatch_group_t group) {
+  SCOPED_TSAN_INTERCEPTOR(dispatch_group_leave, group);
+  // Acquired in the group notification callback in dispatch_group_notify[_f].
+  Release(thr, pc, (uptr)group);
+  REAL(dispatch_group_leave)(group);
+}
+
+TSAN_INTERCEPTOR(void, dispatch_group_async, dispatch_group_t group,
+                 dispatch_queue_t queue, dispatch_block_t block) {
+  SCOPED_TSAN_INTERCEPTOR(dispatch_group_async, group, queue, block);
+  dispatch_retain(group);
+  dispatch_group_enter(group);
+  __block dispatch_block_t block_copy = (dispatch_block_t)Block_copy(block);
+  WRAP(dispatch_async)(queue, ^(void) {
+    block_copy();
+    Block_release(block_copy);
+    WRAP(dispatch_group_leave)(group);
+    dispatch_release(group);
+  });
+}
+
+TSAN_INTERCEPTOR(void, dispatch_group_async_f, dispatch_group_t group,
+                 dispatch_queue_t queue, void *context,
+                 dispatch_function_t work) {
+  SCOPED_TSAN_INTERCEPTOR(dispatch_group_async_f, group, queue, context, work);
+  dispatch_retain(group);
+  dispatch_group_enter(group);
+  WRAP(dispatch_async)(queue, ^(void) {
+    work(context);
+    WRAP(dispatch_group_leave)(group);
+    dispatch_release(group);
+  });
+}
+
+DECLARE_REAL(void, dispatch_group_notify_f, dispatch_group_t group,
+             dispatch_queue_t q, void *context, dispatch_function_t work)
+
+TSAN_INTERCEPTOR(void, dispatch_group_notify, dispatch_group_t group,
+                 dispatch_queue_t q, dispatch_block_t block) {
+  SCOPED_TSAN_INTERCEPTOR(dispatch_group_notify, group, q, block);
+
+  // To make sure the group is still available in the callback (otherwise
+  // it can be already destroyed).  Will be released in the callback.
+  dispatch_retain(group);
+
+  SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
+  dispatch_block_t heap_block = Block_copy(^(void) {
+    {
+      SCOPED_INTERCEPTOR_RAW(dispatch_read_callback);
+      // Released when leaving the group (dispatch_group_leave).
+      Acquire(thr, pc, (uptr)group);
+    }
+    dispatch_release(group);
+    block();
+  });
+  SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
+  block_context_t *new_context =
+      AllocContext(thr, pc, q, heap_block, &invoke_and_release_block);
+  new_context->is_barrier_block = true;
+  Release(thr, pc, (uptr)new_context);
+  REAL(dispatch_group_notify_f)(group, q, new_context, dispatch_callback_wrap);
+}
+
+TSAN_INTERCEPTOR(void, dispatch_group_notify_f, dispatch_group_t group,
+                 dispatch_queue_t q, void *context, dispatch_function_t work) {
+  WRAP(dispatch_group_notify)(group, q, ^(void) { work(context); });
+}
+
+TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler,
+                 dispatch_source_t source, dispatch_block_t handler) {
+  SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler, source, handler);
+  if (handler == nullptr)
+    return REAL(dispatch_source_set_event_handler)(source, nullptr);
+  dispatch_queue_t q = GetTargetQueueFromSource(source);
+  __block block_context_t new_context = {
+      q, handler, &invoke_block, false, false, false, 0 };
+  dispatch_block_t new_handler = Block_copy(^(void) {
+    new_context.orig_context = handler;  // To explicitly capture "handler".
+    dispatch_callback_wrap(&new_context);
+  });
+  uptr submit_sync = (uptr)&new_context;
+  Release(thr, pc, submit_sync);
+  REAL(dispatch_source_set_event_handler)(source, new_handler);
+  Block_release(new_handler);
+}
+
+TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler_f,
+                 dispatch_source_t source, dispatch_function_t handler) {
+  SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler_f, source, handler);
+  if (handler == nullptr)
+    return REAL(dispatch_source_set_event_handler)(source, nullptr);
+  dispatch_block_t block = ^(void) {
+    handler(dispatch_get_context(source));
+  };
+  WRAP(dispatch_source_set_event_handler)(source, block);
+}
+
+TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler,
+                 dispatch_source_t source, dispatch_block_t handler) {
+  SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler, source, handler);
+  if (handler == nullptr)
+    return REAL(dispatch_source_set_cancel_handler)(source, nullptr);
+  dispatch_queue_t q = GetTargetQueueFromSource(source);
+  __block block_context_t new_context = {
+      q, handler, &invoke_block, false, false, false, 0};
+  dispatch_block_t new_handler = Block_copy(^(void) {
+    new_context.orig_context = handler;  // To explicitly capture "handler".
+    dispatch_callback_wrap(&new_context);
+  });
+  uptr submit_sync = (uptr)&new_context;
+  Release(thr, pc, submit_sync);
+  REAL(dispatch_source_set_cancel_handler)(source, new_handler);
+  Block_release(new_handler);
+}
+
+TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler_f,
+                 dispatch_source_t source, dispatch_function_t handler) {
+  SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler_f, source,
+                          handler);
+  if (handler == nullptr)
+    return REAL(dispatch_source_set_cancel_handler)(source, nullptr);
+  dispatch_block_t block = ^(void) {
+    handler(dispatch_get_context(source));
+  };
+  WRAP(dispatch_source_set_cancel_handler)(source, block);
+}
+
+TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler,
+                 dispatch_source_t source, dispatch_block_t handler) {
+  SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler, source,
+                          handler);
+  if (handler == nullptr)
+    return REAL(dispatch_source_set_registration_handler)(source, nullptr);
+  dispatch_queue_t q = GetTargetQueueFromSource(source);
+  __block block_context_t new_context = {
+      q, handler, &invoke_block, false, false, false, 0};
+  dispatch_block_t new_handler = Block_copy(^(void) {
+    new_context.orig_context = handler;  // To explicitly capture "handler".
+    dispatch_callback_wrap(&new_context);
+  });
+  uptr submit_sync = (uptr)&new_context;
+  Release(thr, pc, submit_sync);
+  REAL(dispatch_source_set_registration_handler)(source, new_handler);
+  Block_release(new_handler);
+}
+
+TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler_f,
+                 dispatch_source_t source, dispatch_function_t handler) {
+  SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler_f, source,
+                          handler);
+  if (handler == nullptr)
+    return REAL(dispatch_source_set_registration_handler)(source, nullptr);
+  dispatch_block_t block = ^(void) {
+    handler(dispatch_get_context(source));
+  };
+  WRAP(dispatch_source_set_registration_handler)(source, block);
+}
+
+TSAN_INTERCEPTOR(void, dispatch_apply, size_t iterations,
+                 dispatch_queue_t queue,
+                 DISPATCH_NOESCAPE void (^block)(size_t)) {
+  SCOPED_TSAN_INTERCEPTOR(dispatch_apply, iterations, queue, block);
+
+  u8 sync1, sync2;
+  uptr parent_to_child_sync = (uptr)&sync1;
+  uptr child_to_parent_sync = (uptr)&sync2;
+
+  Release(thr, pc, parent_to_child_sync);
+  void (^new_block)(size_t) = ^(size_t iteration) {
+    SCOPED_INTERCEPTOR_RAW(dispatch_apply);
+    Acquire(thr, pc, parent_to_child_sync);
+    SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
+    block(iteration);
+    SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
+    Release(thr, pc, child_to_parent_sync);
+  };
+  SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
+  REAL(dispatch_apply)(iterations, queue, new_block);
+  SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
+  Acquire(thr, pc, child_to_parent_sync);
+}
+
+static void invoke_block_iteration(void *param, size_t iteration) {
+  auto block = (void (^)(size_t)) param;
+  block(iteration);
+}
+
+TSAN_INTERCEPTOR(void, dispatch_apply_f, size_t iterations,
+                 dispatch_queue_t queue, void *context,
+                 void (*work)(void *, size_t)) {
+  SCOPED_TSAN_INTERCEPTOR(dispatch_apply_f, iterations, queue, context, work);
+
+  // Unfortunately, we cannot delegate to dispatch_apply, since libdispatch
+  // implements dispatch_apply in terms of dispatch_apply_f.
+  u8 sync1, sync2;
+  uptr parent_to_child_sync = (uptr)&sync1;
+  uptr child_to_parent_sync = (uptr)&sync2;
+
+  Release(thr, pc, parent_to_child_sync);
+  void (^new_block)(size_t) = ^(size_t iteration) {
+    SCOPED_INTERCEPTOR_RAW(dispatch_apply_f);
+    Acquire(thr, pc, parent_to_child_sync);
+    SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
+    work(context, iteration);
+    SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
+    Release(thr, pc, child_to_parent_sync);
+  };
+  SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
+  REAL(dispatch_apply_f)(iterations, queue, new_block, invoke_block_iteration);
+  SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
+  Acquire(thr, pc, child_to_parent_sync);
+}
+
+DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr)
+DECLARE_REAL_AND_INTERCEPTOR(int, munmap, void *addr, long_t sz)
+
+TSAN_INTERCEPTOR(dispatch_data_t, dispatch_data_create, const void *buffer,
+                 size_t size, dispatch_queue_t q, dispatch_block_t destructor) {
+  SCOPED_TSAN_INTERCEPTOR(dispatch_data_create, buffer, size, q, destructor);
+  if ((q == nullptr) || (destructor == DISPATCH_DATA_DESTRUCTOR_DEFAULT))
+    return REAL(dispatch_data_create)(buffer, size, q, destructor);
+
+  if (destructor == DISPATCH_DATA_DESTRUCTOR_FREE)
+    destructor = ^(void) { WRAP(free)((void *)(uintptr_t)buffer); };
+  else if (destructor == DISPATCH_DATA_DESTRUCTOR_MUNMAP)
+    destructor = ^(void) { WRAP(munmap)((void *)(uintptr_t)buffer, size); };
+
+  SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
+  dispatch_block_t heap_block = Block_copy(destructor);
+  SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
+  block_context_t *new_context =
+      AllocContext(thr, pc, q, heap_block, &invoke_and_release_block);
+  uptr submit_sync = (uptr)new_context;
+  Release(thr, pc, submit_sync);
+  return REAL(dispatch_data_create)(buffer, size, q, ^(void) {
+    dispatch_callback_wrap(new_context);
+  });
+}
+
+typedef void (^fd_handler_t)(dispatch_data_t data, int error);
+typedef void (^cleanup_handler_t)(int error);
+
+TSAN_INTERCEPTOR(void, dispatch_read, dispatch_fd_t fd, size_t length,
+                 dispatch_queue_t q, fd_handler_t h) {
+  SCOPED_TSAN_INTERCEPTOR(dispatch_read, fd, length, q, h);
+  __block block_context_t new_context = {
+      q, nullptr, &invoke_block, false, false, false, 0};
+  fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) {
+    new_context.orig_context = ^(void) {
+      h(data, error);
+    };
+    dispatch_callback_wrap(&new_context);
+  });
+  uptr submit_sync = (uptr)&new_context;
+  Release(thr, pc, submit_sync);
+  REAL(dispatch_read)(fd, length, q, new_h);
+  Block_release(new_h);
+}
+
+TSAN_INTERCEPTOR(void, dispatch_write, dispatch_fd_t fd, dispatch_data_t data,
+                 dispatch_queue_t q, fd_handler_t h) {
+  SCOPED_TSAN_INTERCEPTOR(dispatch_write, fd, data, q, h);
+  __block block_context_t new_context = {
+      q, nullptr, &invoke_block, false, false, false, 0};
+  fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) {
+    new_context.orig_context = ^(void) {
+      h(data, error);
+    };
+    dispatch_callback_wrap(&new_context);
+  });
+  uptr submit_sync = (uptr)&new_context;
+  Release(thr, pc, submit_sync);
+  REAL(dispatch_write)(fd, data, q, new_h);
+  Block_release(new_h);
+}
+
+TSAN_INTERCEPTOR(void, dispatch_io_read, dispatch_io_t channel, off_t offset,
+                 size_t length, dispatch_queue_t q, dispatch_io_handler_t h) {
+  SCOPED_TSAN_INTERCEPTOR(dispatch_io_read, channel, offset, length, q, h);
+  __block block_context_t new_context = {
+      q, nullptr, &invoke_block, false, false, false, 0};
+  dispatch_io_handler_t new_h =
+      Block_copy(^(bool done, dispatch_data_t data, int error) {
+        new_context.orig_context = ^(void) {
+          h(done, data, error);
+        };
+        dispatch_callback_wrap(&new_context);
+      });
+  uptr submit_sync = (uptr)&new_context;
+  Release(thr, pc, submit_sync);
+  REAL(dispatch_io_read)(channel, offset, length, q, new_h);
+  Block_release(new_h);
+}
+
+TSAN_INTERCEPTOR(void, dispatch_io_write, dispatch_io_t channel, off_t offset,
+                 dispatch_data_t data, dispatch_queue_t q,
+                 dispatch_io_handler_t h) {
+  SCOPED_TSAN_INTERCEPTOR(dispatch_io_write, channel, offset, data, q, h);
+  __block block_context_t new_context = {
+      q, nullptr, &invoke_block, false, false, false, 0};
+  dispatch_io_handler_t new_h =
+      Block_copy(^(bool done, dispatch_data_t data, int error) {
+        new_context.orig_context = ^(void) {
+          h(done, data, error);
+        };
+        dispatch_callback_wrap(&new_context);
+      });
+  uptr submit_sync = (uptr)&new_context;
+  Release(thr, pc, submit_sync);
+  REAL(dispatch_io_write)(channel, offset, data, q, new_h);
+  Block_release(new_h);
+}
+
+TSAN_INTERCEPTOR(void, dispatch_io_barrier, dispatch_io_t channel,
+                 dispatch_block_t barrier) {
+  SCOPED_TSAN_INTERCEPTOR(dispatch_io_barrier, channel, barrier);
+  __block block_context_t new_context = {
+      nullptr, nullptr, &invoke_block, false, false, false, 0};
+  new_context.non_queue_sync_object = (uptr)channel;
+  new_context.is_barrier_block = true;
+  dispatch_block_t new_block = Block_copy(^(void) {
+    new_context.orig_context = ^(void) {
+      barrier();
+    };
+    dispatch_callback_wrap(&new_context);
+  });
+  uptr submit_sync = (uptr)&new_context;
+  Release(thr, pc, submit_sync);
+  REAL(dispatch_io_barrier)(channel, new_block);
+  Block_release(new_block);
+}
+
+TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create, dispatch_io_type_t type,
+                 dispatch_fd_t fd, dispatch_queue_t q, cleanup_handler_t h) {
+  SCOPED_TSAN_INTERCEPTOR(dispatch_io_create, type, fd, q, h);
+  __block dispatch_io_t new_channel = nullptr;
+  __block block_context_t new_context = {
+      q, nullptr, &invoke_block, false, false, false, 0};
+  cleanup_handler_t new_h = Block_copy(^(int error) {
+    {
+      SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
+      Acquire(thr, pc, (uptr)new_channel);  // Release() in dispatch_io_close.
+    }
+    new_context.orig_context = ^(void) {
+      h(error);
+    };
+    dispatch_callback_wrap(&new_context);
+  });
+  uptr submit_sync = (uptr)&new_context;
+  Release(thr, pc, submit_sync);
+  new_channel = REAL(dispatch_io_create)(type, fd, q, new_h);
+  Block_release(new_h);
+  return new_channel;
+}
+
+TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_path,
+                 dispatch_io_type_t type, const char *path, int oflag,
+                 mode_t mode, dispatch_queue_t q, cleanup_handler_t h) {
+  SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_path, type, path, oflag, mode,
+                          q, h);
+  __block dispatch_io_t new_channel = nullptr;
+  __block block_context_t new_context = {
+      q, nullptr, &invoke_block, false, false, false, 0};
+  cleanup_handler_t new_h = Block_copy(^(int error) {
+    {
+      SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
+      Acquire(thr, pc, (uptr)new_channel);  // Release() in dispatch_io_close.
+    }
+    new_context.orig_context = ^(void) {
+      h(error);
+    };
+    dispatch_callback_wrap(&new_context);
+  });
+  uptr submit_sync = (uptr)&new_context;
+  Release(thr, pc, submit_sync);
+  new_channel =
+      REAL(dispatch_io_create_with_path)(type, path, oflag, mode, q, new_h);
+  Block_release(new_h);
+  return new_channel;
+}
+
+TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_io,
+                 dispatch_io_type_t type, dispatch_io_t io, dispatch_queue_t q,
+                 cleanup_handler_t h) {
+  SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_io, type, io, q, h);
+  __block dispatch_io_t new_channel = nullptr;
+  __block block_context_t new_context = {
+      q, nullptr, &invoke_block, false, false, false, 0};
+  cleanup_handler_t new_h = Block_copy(^(int error) {
+    {
+      SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
+      Acquire(thr, pc, (uptr)new_channel);  // Release() in dispatch_io_close.
+    }
+    new_context.orig_context = ^(void) {
+      h(error);
+    };
+    dispatch_callback_wrap(&new_context);
+  });
+  uptr submit_sync = (uptr)&new_context;
+  Release(thr, pc, submit_sync);
+  new_channel = REAL(dispatch_io_create_with_io)(type, io, q, new_h);
+  Block_release(new_h);
+  return new_channel;
+}
+
+TSAN_INTERCEPTOR(void, dispatch_io_close, dispatch_io_t channel,
+                 dispatch_io_close_flags_t flags) {
+  SCOPED_TSAN_INTERCEPTOR(dispatch_io_close, channel, flags);
+  Release(thr, pc, (uptr)channel);  // Acquire() in dispatch_io_create[_*].
+  return REAL(dispatch_io_close)(channel, flags);
+}
+
+// Resuming a suspended queue needs to synchronize with all subsequent
+// executions of blocks in that queue.
+TSAN_INTERCEPTOR(void, dispatch_resume, dispatch_object_t o) {
+  SCOPED_TSAN_INTERCEPTOR(dispatch_resume, o);
+  Release(thr, pc, (uptr)o);  // Synchronizes with the Acquire() on serial_sync
+                              // in dispatch_sync_pre_execute
+  return REAL(dispatch_resume)(o);
+}
+
+void InitializeLibdispatchInterceptors() {
+  INTERCEPT_FUNCTION(dispatch_async);
+  INTERCEPT_FUNCTION(dispatch_async_f);
+  INTERCEPT_FUNCTION(dispatch_sync);
+  INTERCEPT_FUNCTION(dispatch_sync_f);
+  INTERCEPT_FUNCTION(dispatch_barrier_async);
+  INTERCEPT_FUNCTION(dispatch_barrier_async_f);
+  INTERCEPT_FUNCTION(dispatch_barrier_sync);
+  INTERCEPT_FUNCTION(dispatch_barrier_sync_f);
+  INTERCEPT_FUNCTION(dispatch_async_and_wait);
+  INTERCEPT_FUNCTION(dispatch_async_and_wait_f);
+  INTERCEPT_FUNCTION(dispatch_barrier_async_and_wait);
+  INTERCEPT_FUNCTION(dispatch_barrier_async_and_wait_f);
+  INTERCEPT_FUNCTION(dispatch_after);
+  INTERCEPT_FUNCTION(dispatch_after_f);
+  INTERCEPT_FUNCTION(dispatch_once);
+  INTERCEPT_FUNCTION(dispatch_once_f);
+  INTERCEPT_FUNCTION(dispatch_semaphore_signal);
+  INTERCEPT_FUNCTION(dispatch_semaphore_wait);
+  INTERCEPT_FUNCTION(dispatch_group_wait);
+  INTERCEPT_FUNCTION(dispatch_group_leave);
+  INTERCEPT_FUNCTION(dispatch_group_async);
+  INTERCEPT_FUNCTION(dispatch_group_async_f);
+  INTERCEPT_FUNCTION(dispatch_group_notify);
+  INTERCEPT_FUNCTION(dispatch_group_notify_f);
+  INTERCEPT_FUNCTION(dispatch_source_set_event_handler);
+  INTERCEPT_FUNCTION(dispatch_source_set_event_handler_f);
+  INTERCEPT_FUNCTION(dispatch_source_set_cancel_handler);
+  INTERCEPT_FUNCTION(dispatch_source_set_cancel_handler_f);
+  INTERCEPT_FUNCTION(dispatch_source_set_registration_handler);
+  INTERCEPT_FUNCTION(dispatch_source_set_registration_handler_f);
+  INTERCEPT_FUNCTION(dispatch_apply);
+  INTERCEPT_FUNCTION(dispatch_apply_f);
+  INTERCEPT_FUNCTION(dispatch_data_create);
+  INTERCEPT_FUNCTION(dispatch_read);
+  INTERCEPT_FUNCTION(dispatch_write);
+  INTERCEPT_FUNCTION(dispatch_io_read);
+  INTERCEPT_FUNCTION(dispatch_io_write);
+  INTERCEPT_FUNCTION(dispatch_io_barrier);
+  INTERCEPT_FUNCTION(dispatch_io_create);
+  INTERCEPT_FUNCTION(dispatch_io_create_with_path);
+  INTERCEPT_FUNCTION(dispatch_io_create_with_io);
+  INTERCEPT_FUNCTION(dispatch_io_close);
+  INTERCEPT_FUNCTION(dispatch_resume);
+}
+
+}  // namespace __tsan
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_interceptors_mac.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_interceptors_mac.cpp
new file mode 100644
index 000000000000..ed064150d005
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_interceptors_mac.cpp
@@ -0,0 +1,521 @@
+//===-- tsan_interceptors_mac.cpp -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// Mac-specific interceptors.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_platform.h"
+#if SANITIZER_MAC
+
+#include "interception/interception.h"
+#include "tsan_interceptors.h"
+#include "tsan_interface.h"
+#include "tsan_interface_ann.h"
+#include "sanitizer_common/sanitizer_addrhashmap.h"
+
+#include <errno.h>
+#include <libkern/OSAtomic.h>
+#include <objc/objc-sync.h>
+#include <os/lock.h>
+#include <sys/ucontext.h>
+
+#if defined(__has_include) && __has_include(<xpc/xpc.h>)
+#include <xpc/xpc.h>
+#endif  // #if defined(__has_include) && __has_include(<xpc/xpc.h>)
+
+typedef long long_t;
+
+extern "C" {
+int getcontext(ucontext_t *ucp) __attribute__((returns_twice));
+int setcontext(const ucontext_t *ucp);
+}
+
+namespace __tsan {
+
+// The non-barrier versions of OSAtomic* functions are semantically mo_relaxed,
+// but the two variants (e.g. OSAtomicAdd32 and OSAtomicAdd32Barrier) are
+// actually aliases of each other, and we cannot have different interceptors for
+// them, because they're actually the same function.  Thus, we have to stay
+// conservative and treat the non-barrier versions as mo_acq_rel.
+static constexpr morder kMacOrderBarrier = mo_acq_rel;
+static constexpr morder kMacOrderNonBarrier = mo_acq_rel;
+static constexpr morder kMacFailureOrder = mo_relaxed;
+
+#define OSATOMIC_INTERCEPTOR(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
+  TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) {                 \
+    SCOPED_TSAN_INTERCEPTOR(f, x, ptr);                                 \
+    return tsan_atomic_f((volatile tsan_t *)ptr, x, mo);                \
+  }
+
+#define OSATOMIC_INTERCEPTOR_PLUS_X(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
+  TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) {                        \
+    SCOPED_TSAN_INTERCEPTOR(f, x, ptr);                                        \
+    return tsan_atomic_f((volatile tsan_t *)ptr, x, mo) + x;                   \
+  }
+
+#define OSATOMIC_INTERCEPTOR_PLUS_1(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
+  TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) {                             \
+    SCOPED_TSAN_INTERCEPTOR(f, ptr);                                           \
+    return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) + 1;                   \
+  }
+
+#define OSATOMIC_INTERCEPTOR_MINUS_1(return_t, t, tsan_t, f, tsan_atomic_f, \
+                                     mo)                                    \
+  TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) {                          \
+    SCOPED_TSAN_INTERCEPTOR(f, ptr);                                        \
+    return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) - 1;                \
+  }
+
+#define OSATOMIC_INTERCEPTORS_ARITHMETIC(f, tsan_atomic_f, m)                  \
+  m(int32_t, int32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f,             \
+    kMacOrderNonBarrier)                                                       \
+  m(int32_t, int32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f,    \
+    kMacOrderBarrier)                                                          \
+  m(int64_t, int64_t, a64, f##64, __tsan_atomic64_##tsan_atomic_f,             \
+    kMacOrderNonBarrier)                                                       \
+  m(int64_t, int64_t, a64, f##64##Barrier, __tsan_atomic64_##tsan_atomic_f,    \
+    kMacOrderBarrier)
+
+#define OSATOMIC_INTERCEPTORS_BITWISE(f, tsan_atomic_f, m, m_orig)             \
+  m(int32_t, uint32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f,            \
+    kMacOrderNonBarrier)                                                       \
+  m(int32_t, uint32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f,   \
+    kMacOrderBarrier)                                                          \
+  m_orig(int32_t, uint32_t, a32, f##32##Orig, __tsan_atomic32_##tsan_atomic_f, \
+    kMacOrderNonBarrier)                                                       \
+  m_orig(int32_t, uint32_t, a32, f##32##OrigBarrier,                           \
+    __tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier)
+
+OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicAdd, fetch_add,
+                                 OSATOMIC_INTERCEPTOR_PLUS_X)
+OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicIncrement, fetch_add,
+                                 OSATOMIC_INTERCEPTOR_PLUS_1)
+OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicDecrement, fetch_sub,
+                                 OSATOMIC_INTERCEPTOR_MINUS_1)
+OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicOr, fetch_or, OSATOMIC_INTERCEPTOR_PLUS_X,
+                              OSATOMIC_INTERCEPTOR)
+OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicAnd, fetch_and,
+                              OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR)
+OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicXor, fetch_xor,
+                              OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR)
+
+#define OSATOMIC_INTERCEPTORS_CAS(f, tsan_atomic_f, tsan_t, t)              \
+  TSAN_INTERCEPTOR(bool, f, t old_value, t new_value, t volatile *ptr) {    \
+    SCOPED_TSAN_INTERCEPTOR(f, old_value, new_value, ptr);                  \
+    return tsan_atomic_f##_compare_exchange_strong(                         \
+        (volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value,    \
+        kMacOrderNonBarrier, kMacFailureOrder);                             \
+  }                                                                         \
+                                                                            \
+  TSAN_INTERCEPTOR(bool, f##Barrier, t old_value, t new_value,              \
+                   t volatile *ptr) {                                       \
+    SCOPED_TSAN_INTERCEPTOR(f##Barrier, old_value, new_value, ptr);         \
+    return tsan_atomic_f##_compare_exchange_strong(                         \
+        (volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value,    \
+        kMacOrderBarrier, kMacFailureOrder);                                \
+  }
+
+OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapInt, __tsan_atomic32, a32, int)
+OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapLong, __tsan_atomic64, a64,
+                          long_t)
+OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapPtr, __tsan_atomic64, a64,
+                          void *)
+OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap32, __tsan_atomic32, a32,
+                          int32_t)
+OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap64, __tsan_atomic64, a64,
+                          int64_t)
+
+#define OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, mo)             \
+  TSAN_INTERCEPTOR(bool, f, uint32_t n, volatile void *ptr) {    \
+    SCOPED_TSAN_INTERCEPTOR(f, n, ptr);                          \
+    volatile char *byte_ptr = ((volatile char *)ptr) + (n >> 3); \
+    char bit = 0x80u >> (n & 7);                                 \
+    char mask = clear ? ~bit : bit;                              \
+    char orig_byte = op((volatile a8 *)byte_ptr, mask, mo);      \
+    return orig_byte & bit;                                      \
+  }
+
+#define OSATOMIC_INTERCEPTORS_BITOP(f, op, clear)               \
+  OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, kMacOrderNonBarrier) \
+  OSATOMIC_INTERCEPTOR_BITOP(f##Barrier, op, clear, kMacOrderBarrier)
+
+OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndSet, __tsan_atomic8_fetch_or, false)
+OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndClear, __tsan_atomic8_fetch_and,
+                            true)
+
+TSAN_INTERCEPTOR(void, OSAtomicEnqueue, OSQueueHead *list, void *item,
+                 size_t offset) {
+  SCOPED_TSAN_INTERCEPTOR(OSAtomicEnqueue, list, item, offset);
+  __tsan_release(item);
+  REAL(OSAtomicEnqueue)(list, item, offset);
+}
+
+TSAN_INTERCEPTOR(void *, OSAtomicDequeue, OSQueueHead *list, size_t offset) {
+  SCOPED_TSAN_INTERCEPTOR(OSAtomicDequeue, list, offset);
+  void *item = REAL(OSAtomicDequeue)(list, offset);
+  if (item) __tsan_acquire(item);
+  return item;
+}
+
+// OSAtomicFifoEnqueue and OSAtomicFifoDequeue are only on OS X.
+#if !SANITIZER_IOS
+
+TSAN_INTERCEPTOR(void, OSAtomicFifoEnqueue, OSFifoQueueHead *list, void *item,
+                 size_t offset) {
+  SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoEnqueue, list, item, offset);
+  __tsan_release(item);
+  REAL(OSAtomicFifoEnqueue)(list, item, offset);
+}
+
+TSAN_INTERCEPTOR(void *, OSAtomicFifoDequeue, OSFifoQueueHead *list,
+                 size_t offset) {
+  SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoDequeue, list, offset);
+  void *item = REAL(OSAtomicFifoDequeue)(list, offset);
+  if (item) __tsan_acquire(item);
+  return item;
+}
+
+#endif
+
+TSAN_INTERCEPTOR(void, OSSpinLockLock, volatile OSSpinLock *lock) {
+  CHECK(!cur_thread()->is_dead);
+  if (!cur_thread()->is_inited) {
+    return REAL(OSSpinLockLock)(lock);
+  }
+  SCOPED_TSAN_INTERCEPTOR(OSSpinLockLock, lock);
+  REAL(OSSpinLockLock)(lock);
+  Acquire(thr, pc, (uptr)lock);
+}
+
+TSAN_INTERCEPTOR(bool, OSSpinLockTry, volatile OSSpinLock *lock) {
+  CHECK(!cur_thread()->is_dead);
+  if (!cur_thread()->is_inited) {
+    return REAL(OSSpinLockTry)(lock);
+  }
+  SCOPED_TSAN_INTERCEPTOR(OSSpinLockTry, lock);
+  bool result = REAL(OSSpinLockTry)(lock);
+  if (result)
+    Acquire(thr, pc, (uptr)lock);
+  return result;
+}
+
+TSAN_INTERCEPTOR(void, OSSpinLockUnlock, volatile OSSpinLock *lock) {
+  CHECK(!cur_thread()->is_dead);
+  if (!cur_thread()->is_inited) {
+    return REAL(OSSpinLockUnlock)(lock);
+  }
+  SCOPED_TSAN_INTERCEPTOR(OSSpinLockUnlock, lock);
+  Release(thr, pc, (uptr)lock);
+  REAL(OSSpinLockUnlock)(lock);
+}
+
+TSAN_INTERCEPTOR(void, os_lock_lock, void *lock) {
+  CHECK(!cur_thread()->is_dead);
+  if (!cur_thread()->is_inited) {
+    return REAL(os_lock_lock)(lock);
+  }
+  SCOPED_TSAN_INTERCEPTOR(os_lock_lock, lock);
+  REAL(os_lock_lock)(lock);
+  Acquire(thr, pc, (uptr)lock);
+}
+
+TSAN_INTERCEPTOR(bool, os_lock_trylock, void *lock) {
+  CHECK(!cur_thread()->is_dead);
+  if (!cur_thread()->is_inited) {
+    return REAL(os_lock_trylock)(lock);
+  }
+  SCOPED_TSAN_INTERCEPTOR(os_lock_trylock, lock);
+  bool result = REAL(os_lock_trylock)(lock);
+  if (result)
+    Acquire(thr, pc, (uptr)lock);
+  return result;
+}
+
+TSAN_INTERCEPTOR(void, os_lock_unlock, void *lock) {
+  CHECK(!cur_thread()->is_dead);
+  if (!cur_thread()->is_inited) {
+    return REAL(os_lock_unlock)(lock);
+  }
+  SCOPED_TSAN_INTERCEPTOR(os_lock_unlock, lock);
+  Release(thr, pc, (uptr)lock);
+  REAL(os_lock_unlock)(lock);
+}
+
+TSAN_INTERCEPTOR(void, os_unfair_lock_lock, os_unfair_lock_t lock) {
+  if (!cur_thread()->is_inited || cur_thread()->is_dead) {
+    return REAL(os_unfair_lock_lock)(lock);
+  }
+  SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_lock, lock);
+  REAL(os_unfair_lock_lock)(lock);
+  Acquire(thr, pc, (uptr)lock);
+}
+
+TSAN_INTERCEPTOR(void, os_unfair_lock_lock_with_options, os_unfair_lock_t lock,
+                 u32 options) {
+  if (!cur_thread()->is_inited || cur_thread()->is_dead) {
+    return REAL(os_unfair_lock_lock_with_options)(lock, options);
+  }
+  SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_lock_with_options, lock, options);
+  REAL(os_unfair_lock_lock_with_options)(lock, options);
+  Acquire(thr, pc, (uptr)lock);
+}
+
+TSAN_INTERCEPTOR(bool, os_unfair_lock_trylock, os_unfair_lock_t lock) {
+  if (!cur_thread()->is_inited || cur_thread()->is_dead) {
+    return REAL(os_unfair_lock_trylock)(lock);
+  }
+  SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_trylock, lock);
+  bool result = REAL(os_unfair_lock_trylock)(lock);
+  if (result)
+    Acquire(thr, pc, (uptr)lock);
+  return result;
+}
+
+TSAN_INTERCEPTOR(void, os_unfair_lock_unlock, os_unfair_lock_t lock) {
+  if (!cur_thread()->is_inited || cur_thread()->is_dead) {
+    return REAL(os_unfair_lock_unlock)(lock);
+  }
+  SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_unlock, lock);
+  Release(thr, pc, (uptr)lock);
+  REAL(os_unfair_lock_unlock)(lock);
+}
+
+#if defined(__has_include) && __has_include(<xpc/xpc.h>)
+
+TSAN_INTERCEPTOR(void, xpc_connection_set_event_handler,
+                 xpc_connection_t connection, xpc_handler_t handler) {
+  SCOPED_TSAN_INTERCEPTOR(xpc_connection_set_event_handler, connection,
+                          handler);
+  Release(thr, pc, (uptr)connection);
+  xpc_handler_t new_handler = ^(xpc_object_t object) {
+    {
+      SCOPED_INTERCEPTOR_RAW(xpc_connection_set_event_handler);
+      Acquire(thr, pc, (uptr)connection);
+    }
+    handler(object);
+  };
+  REAL(xpc_connection_set_event_handler)(connection, new_handler);
+}
+
+TSAN_INTERCEPTOR(void, xpc_connection_send_barrier, xpc_connection_t connection,
+                 dispatch_block_t barrier) {
+  SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_barrier, connection, barrier);
+  Release(thr, pc, (uptr)connection);
+  dispatch_block_t new_barrier = ^() {
+    {
+      SCOPED_INTERCEPTOR_RAW(xpc_connection_send_barrier);
+      Acquire(thr, pc, (uptr)connection);
+    }
+    barrier();
+  };
+  REAL(xpc_connection_send_barrier)(connection, new_barrier);
+}
+
+TSAN_INTERCEPTOR(void, xpc_connection_send_message_with_reply,
+                 xpc_connection_t connection, xpc_object_t message,
+                 dispatch_queue_t replyq, xpc_handler_t handler) {
+  SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_message_with_reply, connection,
+                          message, replyq, handler);
+  Release(thr, pc, (uptr)connection);
+  xpc_handler_t new_handler = ^(xpc_object_t object) {
+    {
+      SCOPED_INTERCEPTOR_RAW(xpc_connection_send_message_with_reply);
+      Acquire(thr, pc, (uptr)connection);
+    }
+    handler(object);
+  };
+  REAL(xpc_connection_send_message_with_reply)
+  (connection, message, replyq, new_handler);
+}
+
+TSAN_INTERCEPTOR(void, xpc_connection_cancel, xpc_connection_t connection) {
+  SCOPED_TSAN_INTERCEPTOR(xpc_connection_cancel, connection);
+  Release(thr, pc, (uptr)connection);
+  REAL(xpc_connection_cancel)(connection);
+}
+
+#endif  // #if defined(__has_include) && __has_include(<xpc/xpc.h>)
+
+// Determines whether the Obj-C object pointer is a tagged pointer. Tagged
+// pointers encode the object data directly in their pointer bits and do not
+// have an associated memory allocation. The Obj-C runtime uses tagged pointers
+// to transparently optimize small objects.
+static bool IsTaggedObjCPointer(id obj) {
+  const uptr kPossibleTaggedBits = 0x8000000000000001ull;
+  return ((uptr)obj & kPossibleTaggedBits) != 0;
+}
+
+// Returns an address which can be used to inform TSan about synchronization
+// points (MutexLock/Unlock). The TSan infrastructure expects this to be a valid
+// address in the process space. We do a small allocation here to obtain a
+// stable address (the array backing the hash map can change). The memory is
+// never free'd (leaked) and allocation and locking are slow, but this code only
+// runs for @synchronized with tagged pointers, which is very rare.
+static uptr GetOrCreateSyncAddress(uptr addr, ThreadState *thr, uptr pc) {
+  typedef AddrHashMap<uptr, 5> Map;
+  static Map Addresses;
+  Map::Handle h(&Addresses, addr);
+  if (h.created()) {
+    ThreadIgnoreBegin(thr, pc);
+    *h = (uptr) user_alloc(thr, pc, /*size=*/1);
+    ThreadIgnoreEnd(thr);
+  }
+  return *h;
+}
+
+// Returns an address on which we can synchronize given an Obj-C object pointer.
+// For normal object pointers, this is just the address of the object in memory.
+// Tagged pointers are not backed by an actual memory allocation, so we need to
+// synthesize a valid address.
+static uptr SyncAddressForObjCObject(id obj, ThreadState *thr, uptr pc) {
+  if (IsTaggedObjCPointer(obj))
+    return GetOrCreateSyncAddress((uptr)obj, thr, pc);
+  return (uptr)obj;
+}
+
+TSAN_INTERCEPTOR(int, objc_sync_enter, id obj) {
+  SCOPED_TSAN_INTERCEPTOR(objc_sync_enter, obj);
+  if (!obj) return REAL(objc_sync_enter)(obj);
+  uptr addr = SyncAddressForObjCObject(obj, thr, pc);
+  MutexPreLock(thr, pc, addr, MutexFlagWriteReentrant);
+  int result = REAL(objc_sync_enter)(obj);
+  CHECK_EQ(result, OBJC_SYNC_SUCCESS);
+  MutexPostLock(thr, pc, addr, MutexFlagWriteReentrant);
+  return result;
+}
+
+TSAN_INTERCEPTOR(int, objc_sync_exit, id obj) {
+  SCOPED_TSAN_INTERCEPTOR(objc_sync_exit, obj);
+  if (!obj) return REAL(objc_sync_exit)(obj);
+  uptr addr = SyncAddressForObjCObject(obj, thr, pc);
+  MutexUnlock(thr, pc, addr);
+  int result = REAL(objc_sync_exit)(obj);
+  if (result != OBJC_SYNC_SUCCESS) MutexInvalidAccess(thr, pc, addr);
+  return result;
+}
+
+TSAN_INTERCEPTOR(int, swapcontext, ucontext_t *oucp, const ucontext_t *ucp) {
+  {
+    SCOPED_INTERCEPTOR_RAW(swapcontext, oucp, ucp);
+  }
+  // Because of swapcontext() semantics we have no option but to copy its
+  // implementation here
+  if (!oucp || !ucp) {
+    errno = EINVAL;
+    return -1;
+  }
+  ThreadState *thr = cur_thread();
+  const int UCF_SWAPPED = 0x80000000;
+  oucp->uc_onstack &= ~UCF_SWAPPED;
+  thr->ignore_interceptors++;
+  int ret = getcontext(oucp);
+  if (!(oucp->uc_onstack & UCF_SWAPPED)) {
+    thr->ignore_interceptors--;
+    if (!ret) {
+      oucp->uc_onstack |= UCF_SWAPPED;
+      ret = setcontext(ucp);
+    }
+  }
+  return ret;
+}
+
+// On macOS, libc++ is always linked dynamically, so intercepting works the
+// usual way.
+#define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR
+
+namespace {
+struct fake_shared_weak_count {
+  volatile a64 shared_owners;
+  volatile a64 shared_weak_owners;
+  virtual void _unused_0x0() = 0;
+  virtual void _unused_0x8() = 0;
+  virtual void on_zero_shared() = 0;
+  virtual void _unused_0x18() = 0;
+  virtual void on_zero_shared_weak() = 0;
+  virtual ~fake_shared_weak_count() = 0;  // suppress -Wnon-virtual-dtor
+};
+}  // namespace
+
+// The following code adds libc++ interceptors for:
+//     void __shared_weak_count::__release_shared() _NOEXCEPT;
+//     bool __shared_count::__release_shared() _NOEXCEPT;
+// Shared and weak pointers in C++ maintain reference counts via atomics in
+// libc++.dylib, which are TSan-invisible, and this leads to false positives in
+// destructor code. These interceptors re-implements the whole functions so that
+// the mo_acq_rel semantics of the atomic decrement are visible.
+//
+// Unfortunately, the interceptors cannot simply Acquire/Release some sync
+// object and call the original function, because it would have a race between
+// the sync and the destruction of the object.  Calling both under a lock will
+// not work because the destructor can invoke this interceptor again (and even
+// in a different thread, so recursive locks don't help).
+
+STDCXX_INTERCEPTOR(void, _ZNSt3__119__shared_weak_count16__release_sharedEv,
+                   fake_shared_weak_count *o) {
+  if (!flags()->shared_ptr_interceptor)
+    return REAL(_ZNSt3__119__shared_weak_count16__release_sharedEv)(o);
+
+  SCOPED_TSAN_INTERCEPTOR(_ZNSt3__119__shared_weak_count16__release_sharedEv,
+                          o);
+  if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) {
+    Acquire(thr, pc, (uptr)&o->shared_owners);
+    o->on_zero_shared();
+    if (__tsan_atomic64_fetch_add(&o->shared_weak_owners, -1, mo_release) ==
+        0) {
+      Acquire(thr, pc, (uptr)&o->shared_weak_owners);
+      o->on_zero_shared_weak();
+    }
+  }
+}
+
+STDCXX_INTERCEPTOR(bool, _ZNSt3__114__shared_count16__release_sharedEv,
+                   fake_shared_weak_count *o) {
+  if (!flags()->shared_ptr_interceptor)
+    return REAL(_ZNSt3__114__shared_count16__release_sharedEv)(o);
+
+  SCOPED_TSAN_INTERCEPTOR(_ZNSt3__114__shared_count16__release_sharedEv, o);
+  if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) {
+    Acquire(thr, pc, (uptr)&o->shared_owners);
+    o->on_zero_shared();
+    return true;
+  }
+  return false;
+}
+
+namespace {
+struct call_once_callback_args {
+  void (*orig_func)(void *arg);
+  void *orig_arg;
+  void *flag;
+};
+
+void call_once_callback_wrapper(void *arg) {
+  call_once_callback_args *new_args = (call_once_callback_args *)arg;
+  new_args->orig_func(new_args->orig_arg);
+  __tsan_release(new_args->flag);
+}
+}  // namespace
+
+// This adds a libc++ interceptor for:
+//     void __call_once(volatile unsigned long&, void*, void(*)(void*));
+// C++11 call_once is implemented via an internal function __call_once which is
+// inside libc++.dylib, and the atomic release store inside it is thus
+// TSan-invisible. To avoid false positives, this interceptor wraps the callback
+// function and performs an explicit Release after the user code has run.
+STDCXX_INTERCEPTOR(void, _ZNSt3__111__call_onceERVmPvPFvS2_E, void *flag,
+                   void *arg, void (*func)(void *arg)) {
+  call_once_callback_args new_args = {func, arg, flag};
+  REAL(_ZNSt3__111__call_onceERVmPvPFvS2_E)(flag, &new_args,
+                                            call_once_callback_wrapper);
+}
+
+}  // namespace __tsan
+
+#endif  // SANITIZER_MAC
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_interceptors_mach_vm.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_interceptors_mach_vm.cpp
new file mode 100644
index 000000000000..6d62ff6a8382
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_interceptors_mach_vm.cpp
@@ -0,0 +1,53 @@
+//===-- tsan_interceptors_mach_vm.cpp -------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// Interceptors for mach_vm_* user space memory routines on Darwin.
+//===----------------------------------------------------------------------===//
+
+#include "interception/interception.h"
+#include "tsan_interceptors.h"
+#include "tsan_platform.h"
+
+#include <mach/mach.h>
+
+namespace __tsan {
+
+static bool intersects_with_shadow(mach_vm_address_t address,
+                                   mach_vm_size_t size, int flags) {
+  // VM_FLAGS_FIXED is 0x0, so we have to test for VM_FLAGS_ANYWHERE.
+  if (flags & VM_FLAGS_ANYWHERE) return false;
+  return !IsAppMem(address) || !IsAppMem(address + size - 1);
+}
+
+TSAN_INTERCEPTOR(kern_return_t, mach_vm_allocate, vm_map_t target,
+                 mach_vm_address_t *address, mach_vm_size_t size, int flags) {
+  SCOPED_TSAN_INTERCEPTOR(mach_vm_allocate, target, address, size, flags);
+  if (target != mach_task_self())
+    return REAL(mach_vm_allocate)(target, address, size, flags);
+  if (address && intersects_with_shadow(*address, size, flags))
+    return KERN_NO_SPACE;
+  kern_return_t kr = REAL(mach_vm_allocate)(target, address, size, flags);
+  if (kr == KERN_SUCCESS)
+    MemoryRangeImitateWriteOrResetRange(thr, pc, *address, size);
+  return kr;
+}
+
+TSAN_INTERCEPTOR(kern_return_t, mach_vm_deallocate, vm_map_t target,
+                 mach_vm_address_t address, mach_vm_size_t size) {
+  SCOPED_TSAN_INTERCEPTOR(mach_vm_deallocate, target, address, size);
+  if (target != mach_task_self())
+    return REAL(mach_vm_deallocate)(target, address, size);
+  kern_return_t kr = REAL(mach_vm_deallocate)(target, address, size);
+  if (kr == KERN_SUCCESS && address)
+    UnmapShadow(thr, address, size);
+  return kr;
+}
+
+}  // namespace __tsan
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_interceptors_posix.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_interceptors_posix.cpp
new file mode 100644
index 000000000000..cf3dc90d96a1
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_interceptors_posix.cpp
@@ -0,0 +1,3015 @@
+//===-- tsan_interceptors_posix.cpp ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// FIXME: move as many interceptors as possible into
+// sanitizer_common/sanitizer_common_interceptors.inc
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_atomic.h"
+#include "sanitizer_common/sanitizer_errno.h"
+#include "sanitizer_common/sanitizer_libc.h"
+#include "sanitizer_common/sanitizer_linux.h"
+#include "sanitizer_common/sanitizer_platform_limits_netbsd.h"
+#include "sanitizer_common/sanitizer_platform_limits_posix.h"
+#include "sanitizer_common/sanitizer_placement_new.h"
+#include "sanitizer_common/sanitizer_posix.h"
+#include "sanitizer_common/sanitizer_stacktrace.h"
+#include "sanitizer_common/sanitizer_tls_get_addr.h"
+#include "interception/interception.h"
+#include "tsan_interceptors.h"
+#include "tsan_interface.h"
+#include "tsan_platform.h"
+#include "tsan_suppressions.h"
+#include "tsan_rtl.h"
+#include "tsan_mman.h"
+#include "tsan_fd.h"
+
+#include <stdarg.h>
+
+using namespace __tsan;
+
+#if SANITIZER_FREEBSD || SANITIZER_MAC
+#define stdout __stdoutp
+#define stderr __stderrp
+#endif
+
+#if SANITIZER_NETBSD
+#define dirfd(dirp) (*(int *)(dirp))
+#define fileno_unlocked(fp)              \
+  (((__sanitizer_FILE *)fp)->_file == -1 \
+       ? -1                              \
+       : (int)(unsigned short)(((__sanitizer_FILE *)fp)->_file))
+
+#define stdout ((__sanitizer_FILE*)&__sF[1])
+#define stderr ((__sanitizer_FILE*)&__sF[2])
+
+#define nanosleep __nanosleep50
+#define vfork __vfork14
+#endif
+
+#ifdef __mips__
+const int kSigCount = 129;
+#else
+const int kSigCount = 65;
+#endif
+
+#ifdef __mips__
+struct ucontext_t {
+  u64 opaque[768 / sizeof(u64) + 1];
+};
+#else
+struct ucontext_t {
+  // The size is determined by looking at sizeof of real ucontext_t on linux.
+  u64 opaque[936 / sizeof(u64) + 1];
+};
+#endif
+
+#if defined(__x86_64__) || defined(__mips__) || SANITIZER_PPC64V1 || \
+    defined(__s390x__)
+#define PTHREAD_ABI_BASE  "GLIBC_2.3.2"
+#elif defined(__aarch64__) || SANITIZER_PPC64V2
+#define PTHREAD_ABI_BASE  "GLIBC_2.17"
+#endif
+
+extern "C" int pthread_attr_init(void *attr);
+extern "C" int pthread_attr_destroy(void *attr);
+DECLARE_REAL(int, pthread_attr_getdetachstate, void *, void *)
+extern "C" int pthread_attr_setstacksize(void *attr, uptr stacksize);
+extern "C" int pthread_atfork(void (*prepare)(void), void (*parent)(void),
+                              void (*child)(void));
+extern "C" int pthread_key_create(unsigned *key, void (*destructor)(void* v));
+extern "C" int pthread_setspecific(unsigned key, const void *v);
+DECLARE_REAL(int, pthread_mutexattr_gettype, void *, void *)
+DECLARE_REAL(int, fflush, __sanitizer_FILE *fp)
+DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr size)
+DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr)
+extern "C" int pthread_equal(void *t1, void *t2);
+extern "C" void *pthread_self();
+extern "C" void _exit(int status);
+#if !SANITIZER_NETBSD
+extern "C" int fileno_unlocked(void *stream);
+extern "C" int dirfd(void *dirp);
+#endif
+#if SANITIZER_NETBSD
+extern __sanitizer_FILE __sF[];
+#else
+extern __sanitizer_FILE *stdout, *stderr;
+#endif
+#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD
+const int PTHREAD_MUTEX_RECURSIVE = 1;
+const int PTHREAD_MUTEX_RECURSIVE_NP = 1;
+#else
+const int PTHREAD_MUTEX_RECURSIVE = 2;
+const int PTHREAD_MUTEX_RECURSIVE_NP = 2;
+#endif
+#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD
+const int EPOLL_CTL_ADD = 1;
+#endif
+const int SIGILL = 4;
+const int SIGTRAP = 5;
+const int SIGABRT = 6;
+const int SIGFPE = 8;
+const int SIGSEGV = 11;
+const int SIGPIPE = 13;
+const int SIGTERM = 15;
+#if defined(__mips__) || SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD
+const int SIGBUS = 10;
+const int SIGSYS = 12;
+#else
+const int SIGBUS = 7;
+const int SIGSYS = 31;
+#endif
+void *const MAP_FAILED = (void*)-1;
+#if SANITIZER_NETBSD
+const int PTHREAD_BARRIER_SERIAL_THREAD = 1234567;
+#elif !SANITIZER_MAC
+const int PTHREAD_BARRIER_SERIAL_THREAD = -1;
+#endif
+const int MAP_FIXED = 0x10;
+typedef long long_t;
+typedef __sanitizer::u16 mode_t;
+
+// From /usr/include/unistd.h
+# define F_ULOCK 0      /* Unlock a previously locked region.  */
+# define F_LOCK  1      /* Lock a region for exclusive use.  */
+# define F_TLOCK 2      /* Test and lock a region for exclusive use.  */
+# define F_TEST  3      /* Test a region for other processes locks.  */
+
+#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD
+const int SA_SIGINFO = 0x40;
+const int SIG_SETMASK = 3;
+#elif defined(__mips__)
+const int SA_SIGINFO = 8;
+const int SIG_SETMASK = 3;
+#else
+const int SA_SIGINFO = 4;
+const int SIG_SETMASK = 2;
+#endif
+
+#define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED \
+  (!cur_thread_init()->is_inited)
+
+namespace __tsan {
+struct SignalDesc {
+  bool armed;
+  __sanitizer_siginfo siginfo;
+  ucontext_t ctx;
+};
+
+struct ThreadSignalContext {
+  int int_signal_send;
+  atomic_uintptr_t in_blocking_func;
+  SignalDesc pending_signals[kSigCount];
+  // emptyset and oldset are too big for stack.
+  __sanitizer_sigset_t emptyset;
+  __sanitizer_sigset_t oldset;
+};
+
+// The sole reason tsan wraps atexit callbacks is to establish synchronization
+// between callback setup and callback execution.
+struct AtExitCtx {
+  void (*f)();
+  void *arg;
+  uptr pc;
+};
+
+// InterceptorContext holds all global data required for interceptors.
+// It's explicitly constructed in InitializeInterceptors with placement new
+// and is never destroyed. This allows usage of members with non-trivial
+// constructors and destructors.
+struct InterceptorContext {
+  // The object is 64-byte aligned, because we want hot data to be located
+  // in a single cache line if possible (it's accessed in every interceptor).
+  ALIGNED(64) LibIgnore libignore;
+  __sanitizer_sigaction sigactions[kSigCount];
+#if !SANITIZER_MAC && !SANITIZER_NETBSD
+  unsigned finalize_key;
+#endif
+
+  Mutex atexit_mu;
+  Vector<struct AtExitCtx *> AtExitStack;
+
+  InterceptorContext() : libignore(LINKER_INITIALIZED), atexit_mu(MutexTypeAtExit), AtExitStack() {}
+};
+
+static ALIGNED(64) char interceptor_placeholder[sizeof(InterceptorContext)];
+InterceptorContext *interceptor_ctx() {
+  return reinterpret_cast<InterceptorContext*>(&interceptor_placeholder[0]);
+}
+
+LibIgnore *libignore() {
+  return &interceptor_ctx()->libignore;
+}
+
+void InitializeLibIgnore() {
+  const SuppressionContext &supp = *Suppressions();
+  const uptr n = supp.SuppressionCount();
+  for (uptr i = 0; i < n; i++) {
+    const Suppression *s = supp.SuppressionAt(i);
+    if (0 == internal_strcmp(s->type, kSuppressionLib))
+      libignore()->AddIgnoredLibrary(s->templ);
+  }
+  if (flags()->ignore_noninstrumented_modules)
+    libignore()->IgnoreNoninstrumentedModules(true);
+  libignore()->OnLibraryLoaded(0);
+}
+
+// The following two hooks can be used by for cooperative scheduling when
+// locking.
+#ifdef TSAN_EXTERNAL_HOOKS
+void OnPotentiallyBlockingRegionBegin();
+void OnPotentiallyBlockingRegionEnd();
+#else
+SANITIZER_WEAK_CXX_DEFAULT_IMPL void OnPotentiallyBlockingRegionBegin() {}
+SANITIZER_WEAK_CXX_DEFAULT_IMPL void OnPotentiallyBlockingRegionEnd() {}
+#endif
+
+}  // namespace __tsan
+
+static ThreadSignalContext *SigCtx(ThreadState *thr) {
+  ThreadSignalContext *ctx = (ThreadSignalContext*)thr->signal_ctx;
+  if (ctx == 0 && !thr->is_dead) {
+    ctx = (ThreadSignalContext*)MmapOrDie(sizeof(*ctx), "ThreadSignalContext");
+    MemoryResetRange(thr, (uptr)&SigCtx, (uptr)ctx, sizeof(*ctx));
+    thr->signal_ctx = ctx;
+  }
+  return ctx;
+}
+
+ScopedInterceptor::ScopedInterceptor(ThreadState *thr, const char *fname,
+                                     uptr pc)
+    : thr_(thr), in_ignored_lib_(false), ignoring_(false) {
+  LazyInitialize(thr);
+  if (!thr_->is_inited) return;
+  if (!thr_->ignore_interceptors) FuncEntry(thr, pc);
+  DPrintf("#%d: intercept %s()\n", thr_->tid, fname);
+  ignoring_ =
+      !thr_->in_ignored_lib && (flags()->ignore_interceptors_accesses ||
+                                libignore()->IsIgnored(pc, &in_ignored_lib_));
+  EnableIgnores();
+}
+
+ScopedInterceptor::~ScopedInterceptor() {
+  if (!thr_->is_inited) return;
+  DisableIgnores();
+  if (!thr_->ignore_interceptors) {
+    ProcessPendingSignals(thr_);
+    FuncExit(thr_);
+    CheckedMutex::CheckNoLocks();
+  }
+}
+
+NOINLINE
+void ScopedInterceptor::EnableIgnoresImpl() {
+  ThreadIgnoreBegin(thr_, 0);
+  if (flags()->ignore_noninstrumented_modules)
+    thr_->suppress_reports++;
+  if (in_ignored_lib_) {
+    DCHECK(!thr_->in_ignored_lib);
+    thr_->in_ignored_lib = true;
+  }
+}
+
+NOINLINE
+void ScopedInterceptor::DisableIgnoresImpl() {
+  ThreadIgnoreEnd(thr_);
+  if (flags()->ignore_noninstrumented_modules)
+    thr_->suppress_reports--;
+  if (in_ignored_lib_) {
+    DCHECK(thr_->in_ignored_lib);
+    thr_->in_ignored_lib = false;
+  }
+}
+
+#define TSAN_INTERCEPT(func) INTERCEPT_FUNCTION(func)
+#if SANITIZER_FREEBSD
+# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func)
+# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func)
+# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func)
+#elif SANITIZER_NETBSD
+# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func)
+# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) \
+         INTERCEPT_FUNCTION(__libc_##func)
+# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) \
+         INTERCEPT_FUNCTION(__libc_thr_##func)
+#else
+# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION_VER(func, ver)
+# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func)
+# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func)
+#endif
+
+#define READ_STRING_OF_LEN(thr, pc, s, len, n)                 \
+  MemoryAccessRange((thr), (pc), (uptr)(s),                         \
+    common_flags()->strict_string_checks ? (len) + 1 : (n), false)
+
+#define READ_STRING(thr, pc, s, n)                             \
+    READ_STRING_OF_LEN((thr), (pc), (s), internal_strlen(s), (n))
+
+#define BLOCK_REAL(name) (BlockingCall(thr), REAL(name))
+
+struct BlockingCall {
+  explicit BlockingCall(ThreadState *thr)
+      : thr(thr)
+      , ctx(SigCtx(thr)) {
+    for (;;) {
+      atomic_store(&ctx->in_blocking_func, 1, memory_order_relaxed);
+      if (atomic_load(&thr->pending_signals, memory_order_relaxed) == 0)
+        break;
+      atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed);
+      ProcessPendingSignals(thr);
+    }
+    // When we are in a "blocking call", we process signals asynchronously
+    // (right when they arrive). In this context we do not expect to be
+    // executing any user/runtime code. The known interceptor sequence when
+    // this is not true is: pthread_join -> munmap(stack). It's fine
+    // to ignore munmap in this case -- we handle stack shadow separately.
+    thr->ignore_interceptors++;
+  }
+
+  ~BlockingCall() {
+    thr->ignore_interceptors--;
+    atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed);
+  }
+
+  ThreadState *thr;
+  ThreadSignalContext *ctx;
+};
+
+TSAN_INTERCEPTOR(unsigned, sleep, unsigned sec) {
+  SCOPED_TSAN_INTERCEPTOR(sleep, sec);
+  unsigned res = BLOCK_REAL(sleep)(sec);
+  AfterSleep(thr, pc);
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, usleep, long_t usec) {
+  SCOPED_TSAN_INTERCEPTOR(usleep, usec);
+  int res = BLOCK_REAL(usleep)(usec);
+  AfterSleep(thr, pc);
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, nanosleep, void *req, void *rem) {
+  SCOPED_TSAN_INTERCEPTOR(nanosleep, req, rem);
+  int res = BLOCK_REAL(nanosleep)(req, rem);
+  AfterSleep(thr, pc);
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, pause, int fake) {
+  SCOPED_TSAN_INTERCEPTOR(pause, fake);
+  return BLOCK_REAL(pause)(fake);
+}
+
+// Note: we specifically call the function in such strange way
+// with "installed_at" because in reports it will appear between
+// callback frames and the frame that installed the callback.
+static void at_exit_callback_installed_at() {
+  AtExitCtx *ctx;
+  {
+    // Ensure thread-safety.
+    Lock l(&interceptor_ctx()->atexit_mu);
+
+    // Pop AtExitCtx from the top of the stack of callback functions
+    uptr element = interceptor_ctx()->AtExitStack.Size() - 1;
+    ctx = interceptor_ctx()->AtExitStack[element];
+    interceptor_ctx()->AtExitStack.PopBack();
+  }
+
+  ThreadState *thr = cur_thread();
+  Acquire(thr, ctx->pc, (uptr)ctx);
+  FuncEntry(thr, ctx->pc);
+  ((void(*)())ctx->f)();
+  FuncExit(thr);
+  Free(ctx);
+}
+
+static void cxa_at_exit_callback_installed_at(void *arg) {
+  ThreadState *thr = cur_thread();
+  AtExitCtx *ctx = (AtExitCtx*)arg;
+  Acquire(thr, ctx->pc, (uptr)arg);
+  FuncEntry(thr, ctx->pc);
+  ((void(*)(void *arg))ctx->f)(ctx->arg);
+  FuncExit(thr);
+  Free(ctx);
+}
+
+static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(),
+      void *arg, void *dso);
+
+#if !SANITIZER_ANDROID
+TSAN_INTERCEPTOR(int, atexit, void (*f)()) {
+  if (in_symbolizer())
+    return 0;
+  // We want to setup the atexit callback even if we are in ignored lib
+  // or after fork.
+  SCOPED_INTERCEPTOR_RAW(atexit, f);
+  return setup_at_exit_wrapper(thr, GET_CALLER_PC(), (void (*)())f, 0, 0);
+}
+#endif
+
+TSAN_INTERCEPTOR(int, __cxa_atexit, void (*f)(void *a), void *arg, void *dso) {
+  if (in_symbolizer())
+    return 0;
+  SCOPED_TSAN_INTERCEPTOR(__cxa_atexit, f, arg, dso);
+  return setup_at_exit_wrapper(thr, GET_CALLER_PC(), (void (*)())f, arg, dso);
+}
+
+static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(),
+      void *arg, void *dso) {
+  auto *ctx = New<AtExitCtx>();
+  ctx->f = f;
+  ctx->arg = arg;
+  ctx->pc = pc;
+  Release(thr, pc, (uptr)ctx);
+  // Memory allocation in __cxa_atexit will race with free during exit,
+  // because we do not see synchronization around atexit callback list.
+  ThreadIgnoreBegin(thr, pc);
+  int res;
+  if (!dso) {
+    // NetBSD does not preserve the 2nd argument if dso is equal to 0
+    // Store ctx in a local stack-like structure
+
+    // Ensure thread-safety.
+    Lock l(&interceptor_ctx()->atexit_mu);
+    // __cxa_atexit calls calloc. If we don't ignore interceptors, we will fail
+    // due to atexit_mu held on exit from the calloc interceptor.
+    ScopedIgnoreInterceptors ignore;
+
+    res = REAL(__cxa_atexit)((void (*)(void *a))at_exit_callback_installed_at,
+                             0, 0);
+    // Push AtExitCtx on the top of the stack of callback functions
+    if (!res) {
+      interceptor_ctx()->AtExitStack.PushBack(ctx);
+    }
+  } else {
+    res = REAL(__cxa_atexit)(cxa_at_exit_callback_installed_at, ctx, dso);
+  }
+  ThreadIgnoreEnd(thr);
+  return res;
+}
+
+#if !SANITIZER_MAC && !SANITIZER_NETBSD
+static void on_exit_callback_installed_at(int status, void *arg) {
+  ThreadState *thr = cur_thread();
+  AtExitCtx *ctx = (AtExitCtx*)arg;
+  Acquire(thr, ctx->pc, (uptr)arg);
+  FuncEntry(thr, ctx->pc);
+  ((void(*)(int status, void *arg))ctx->f)(status, ctx->arg);
+  FuncExit(thr);
+  Free(ctx);
+}
+
+TSAN_INTERCEPTOR(int, on_exit, void(*f)(int, void*), void *arg) {
+  if (in_symbolizer())
+    return 0;
+  SCOPED_TSAN_INTERCEPTOR(on_exit, f, arg);
+  auto *ctx = New<AtExitCtx>();
+  ctx->f = (void(*)())f;
+  ctx->arg = arg;
+  ctx->pc = GET_CALLER_PC();
+  Release(thr, pc, (uptr)ctx);
+  // Memory allocation in __cxa_atexit will race with free during exit,
+  // because we do not see synchronization around atexit callback list.
+  ThreadIgnoreBegin(thr, pc);
+  int res = REAL(on_exit)(on_exit_callback_installed_at, ctx);
+  ThreadIgnoreEnd(thr);
+  return res;
+}
+#define TSAN_MAYBE_INTERCEPT_ON_EXIT TSAN_INTERCEPT(on_exit)
+#else
+#define TSAN_MAYBE_INTERCEPT_ON_EXIT
+#endif
+
+// Cleanup old bufs.
+static void JmpBufGarbageCollect(ThreadState *thr, uptr sp) {
+  for (uptr i = 0; i < thr->jmp_bufs.Size(); i++) {
+    JmpBuf *buf = &thr->jmp_bufs[i];
+    if (buf->sp <= sp) {
+      uptr sz = thr->jmp_bufs.Size();
+      internal_memcpy(buf, &thr->jmp_bufs[sz - 1], sizeof(*buf));
+      thr->jmp_bufs.PopBack();
+      i--;
+    }
+  }
+}
+
+static void SetJmp(ThreadState *thr, uptr sp) {
+  if (!thr->is_inited)  // called from libc guts during bootstrap
+    return;
+  // Cleanup old bufs.
+  JmpBufGarbageCollect(thr, sp);
+  // Remember the buf.
+  JmpBuf *buf = thr->jmp_bufs.PushBack();
+  buf->sp = sp;
+  buf->shadow_stack_pos = thr->shadow_stack_pos;
+  ThreadSignalContext *sctx = SigCtx(thr);
+  buf->int_signal_send = sctx ? sctx->int_signal_send : 0;
+  buf->in_blocking_func = sctx ?
+      atomic_load(&sctx->in_blocking_func, memory_order_relaxed) :
+      false;
+  buf->in_signal_handler = atomic_load(&thr->in_signal_handler,
+      memory_order_relaxed);
+}
+
+static void LongJmp(ThreadState *thr, uptr *env) {
+  uptr sp = ExtractLongJmpSp(env);
+  // Find the saved buf with matching sp.
+  for (uptr i = 0; i < thr->jmp_bufs.Size(); i++) {
+    JmpBuf *buf = &thr->jmp_bufs[i];
+    if (buf->sp == sp) {
+      CHECK_GE(thr->shadow_stack_pos, buf->shadow_stack_pos);
+      // Unwind the stack.
+      while (thr->shadow_stack_pos > buf->shadow_stack_pos)
+        FuncExit(thr);
+      ThreadSignalContext *sctx = SigCtx(thr);
+      if (sctx) {
+        sctx->int_signal_send = buf->int_signal_send;
+        atomic_store(&sctx->in_blocking_func, buf->in_blocking_func,
+            memory_order_relaxed);
+      }
+      atomic_store(&thr->in_signal_handler, buf->in_signal_handler,
+          memory_order_relaxed);
+      JmpBufGarbageCollect(thr, buf->sp - 1);  // do not collect buf->sp
+      return;
+    }
+  }
+  Printf("ThreadSanitizer: can't find longjmp buf\n");
+  CHECK(0);
+}
+
+// FIXME: put everything below into a common extern "C" block?
+extern "C" void __tsan_setjmp(uptr sp) { SetJmp(cur_thread_init(), sp); }
+
+#if SANITIZER_MAC
+TSAN_INTERCEPTOR(int, setjmp, void *env);
+TSAN_INTERCEPTOR(int, _setjmp, void *env);
+TSAN_INTERCEPTOR(int, sigsetjmp, void *env);
+#else  // SANITIZER_MAC
+
+#if SANITIZER_NETBSD
+#define setjmp_symname __setjmp14
+#define sigsetjmp_symname __sigsetjmp14
+#else
+#define setjmp_symname setjmp
+#define sigsetjmp_symname sigsetjmp
+#endif
+
+#define TSAN_INTERCEPTOR_SETJMP_(x) __interceptor_ ## x
+#define TSAN_INTERCEPTOR_SETJMP__(x) TSAN_INTERCEPTOR_SETJMP_(x)
+#define TSAN_INTERCEPTOR_SETJMP TSAN_INTERCEPTOR_SETJMP__(setjmp_symname)
+#define TSAN_INTERCEPTOR_SIGSETJMP TSAN_INTERCEPTOR_SETJMP__(sigsetjmp_symname)
+
+#define TSAN_STRING_SETJMP SANITIZER_STRINGIFY(setjmp_symname)
+#define TSAN_STRING_SIGSETJMP SANITIZER_STRINGIFY(sigsetjmp_symname)
+
+// Not called.  Merely to satisfy TSAN_INTERCEPT().
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE
+int TSAN_INTERCEPTOR_SETJMP(void *env);
+extern "C" int TSAN_INTERCEPTOR_SETJMP(void *env) {
+  CHECK(0);
+  return 0;
+}
+
+// FIXME: any reason to have a separate declaration?
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE
+int __interceptor__setjmp(void *env);
+extern "C" int __interceptor__setjmp(void *env) {
+  CHECK(0);
+  return 0;
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE
+int TSAN_INTERCEPTOR_SIGSETJMP(void *env);
+extern "C" int TSAN_INTERCEPTOR_SIGSETJMP(void *env) {
+  CHECK(0);
+  return 0;
+}
+
+#if !SANITIZER_NETBSD
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE
+int __interceptor___sigsetjmp(void *env);
+extern "C" int __interceptor___sigsetjmp(void *env) {
+  CHECK(0);
+  return 0;
+}
+#endif
+
+extern "C" int setjmp_symname(void *env);
+extern "C" int _setjmp(void *env);
+extern "C" int sigsetjmp_symname(void *env);
+#if !SANITIZER_NETBSD
+extern "C" int __sigsetjmp(void *env);
+#endif
+DEFINE_REAL(int, setjmp_symname, void *env)
+DEFINE_REAL(int, _setjmp, void *env)
+DEFINE_REAL(int, sigsetjmp_symname, void *env)
+#if !SANITIZER_NETBSD
+DEFINE_REAL(int, __sigsetjmp, void *env)
+#endif
+#endif  // SANITIZER_MAC
+
+#if SANITIZER_NETBSD
+#define longjmp_symname __longjmp14
+#define siglongjmp_symname __siglongjmp14
+#else
+#define longjmp_symname longjmp
+#define siglongjmp_symname siglongjmp
+#endif
+
+TSAN_INTERCEPTOR(void, longjmp_symname, uptr *env, int val) {
+  // Note: if we call REAL(longjmp) in the context of ScopedInterceptor,
+  // bad things will happen. We will jump over ScopedInterceptor dtor and can
+  // leave thr->in_ignored_lib set.
+  {
+    SCOPED_INTERCEPTOR_RAW(longjmp_symname, env, val);
+  }
+  LongJmp(cur_thread(), env);
+  REAL(longjmp_symname)(env, val);
+}
+
+TSAN_INTERCEPTOR(void, siglongjmp_symname, uptr *env, int val) {
+  {
+    SCOPED_INTERCEPTOR_RAW(siglongjmp_symname, env, val);
+  }
+  LongJmp(cur_thread(), env);
+  REAL(siglongjmp_symname)(env, val);
+}
+
+#if SANITIZER_NETBSD
+TSAN_INTERCEPTOR(void, _longjmp, uptr *env, int val) {
+  {
+    SCOPED_INTERCEPTOR_RAW(_longjmp, env, val);
+  }
+  LongJmp(cur_thread(), env);
+  REAL(_longjmp)(env, val);
+}
+#endif
+
+#if !SANITIZER_MAC
+TSAN_INTERCEPTOR(void*, malloc, uptr size) {
+  if (in_symbolizer())
+    return InternalAlloc(size);
+  void *p = 0;
+  {
+    SCOPED_INTERCEPTOR_RAW(malloc, size);
+    p = user_alloc(thr, pc, size);
+  }
+  invoke_malloc_hook(p, size);
+  return p;
+}
+
+// In glibc<2.25, dynamic TLS blocks are allocated by __libc_memalign. Intercept
+// __libc_memalign so that (1) we can detect races (2) free will not be called
+// on libc internally allocated blocks.
+TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) {
+  SCOPED_INTERCEPTOR_RAW(__libc_memalign, align, sz);
+  return user_memalign(thr, pc, align, sz);
+}
+
+TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) {
+  if (in_symbolizer())
+    return InternalCalloc(size, n);
+  void *p = 0;
+  {
+    SCOPED_INTERCEPTOR_RAW(calloc, size, n);
+    p = user_calloc(thr, pc, size, n);
+  }
+  invoke_malloc_hook(p, n * size);
+  return p;
+}
+
+TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) {
+  if (in_symbolizer())
+    return InternalRealloc(p, size);
+  if (p)
+    invoke_free_hook(p);
+  {
+    SCOPED_INTERCEPTOR_RAW(realloc, p, size);
+    p = user_realloc(thr, pc, p, size);
+  }
+  invoke_malloc_hook(p, size);
+  return p;
+}
+
+TSAN_INTERCEPTOR(void*, reallocarray, void *p, uptr size, uptr n) {
+  if (in_symbolizer())
+    return InternalReallocArray(p, size, n);
+  if (p)
+    invoke_free_hook(p);
+  {
+    SCOPED_INTERCEPTOR_RAW(reallocarray, p, size, n);
+    p = user_reallocarray(thr, pc, p, size, n);
+  }
+  invoke_malloc_hook(p, size);
+  return p;
+}
+
+TSAN_INTERCEPTOR(void, free, void *p) {
+  if (p == 0)
+    return;
+  if (in_symbolizer())
+    return InternalFree(p);
+  invoke_free_hook(p);
+  SCOPED_INTERCEPTOR_RAW(free, p);
+  user_free(thr, pc, p);
+}
+
+TSAN_INTERCEPTOR(void, cfree, void *p) {
+  if (p == 0)
+    return;
+  if (in_symbolizer())
+    return InternalFree(p);
+  invoke_free_hook(p);
+  SCOPED_INTERCEPTOR_RAW(cfree, p);
+  user_free(thr, pc, p);
+}
+
+TSAN_INTERCEPTOR(uptr, malloc_usable_size, void *p) {
+  SCOPED_INTERCEPTOR_RAW(malloc_usable_size, p);
+  return user_alloc_usable_size(p);
+}
+#endif
+
+TSAN_INTERCEPTOR(char *, strcpy, char *dst, const char *src) {
+  SCOPED_TSAN_INTERCEPTOR(strcpy, dst, src);
+  uptr srclen = internal_strlen(src);
+  MemoryAccessRange(thr, pc, (uptr)dst, srclen + 1, true);
+  MemoryAccessRange(thr, pc, (uptr)src, srclen + 1, false);
+  return REAL(strcpy)(dst, src);
+}
+
+TSAN_INTERCEPTOR(char*, strncpy, char *dst, char *src, uptr n) {
+  SCOPED_TSAN_INTERCEPTOR(strncpy, dst, src, n);
+  uptr srclen = internal_strnlen(src, n);
+  MemoryAccessRange(thr, pc, (uptr)dst, n, true);
+  MemoryAccessRange(thr, pc, (uptr)src, min(srclen + 1, n), false);
+  return REAL(strncpy)(dst, src, n);
+}
+
+TSAN_INTERCEPTOR(char*, strdup, const char *str) {
+  SCOPED_TSAN_INTERCEPTOR(strdup, str);
+  // strdup will call malloc, so no instrumentation is required here.
+  return REAL(strdup)(str);
+}
+
+// Zero out addr if it points into shadow memory and was provided as a hint
+// only, i.e., MAP_FIXED is not set.
+static bool fix_mmap_addr(void **addr, long_t sz, int flags) {
+  if (*addr) {
+    if (!IsAppMem((uptr)*addr) || !IsAppMem((uptr)*addr + sz - 1)) {
+      if (flags & MAP_FIXED) {
+        errno = errno_EINVAL;
+        return false;
+      } else {
+        *addr = 0;
+      }
+    }
+  }
+  return true;
+}
+
+template <class Mmap>
+static void *mmap_interceptor(ThreadState *thr, uptr pc, Mmap real_mmap,
+                              void *addr, SIZE_T sz, int prot, int flags,
+                              int fd, OFF64_T off) {
+  if (!fix_mmap_addr(&addr, sz, flags)) return MAP_FAILED;
+  void *res = real_mmap(addr, sz, prot, flags, fd, off);
+  if (res != MAP_FAILED) {
+    if (!IsAppMem((uptr)res) || !IsAppMem((uptr)res + sz - 1)) {
+      Report("ThreadSanitizer: mmap at bad address: addr=%p size=%p res=%p\n",
+             addr, (void*)sz, res);
+      Die();
+    }
+    if (fd > 0) FdAccess(thr, pc, fd);
+    MemoryRangeImitateWriteOrResetRange(thr, pc, (uptr)res, sz);
+  }
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, munmap, void *addr, long_t sz) {
+  SCOPED_TSAN_INTERCEPTOR(munmap, addr, sz);
+  UnmapShadow(thr, (uptr)addr, sz);
+  int res = REAL(munmap)(addr, sz);
+  return res;
+}
+
+#if SANITIZER_LINUX
+TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) {
+  SCOPED_INTERCEPTOR_RAW(memalign, align, sz);
+  return user_memalign(thr, pc, align, sz);
+}
+#define TSAN_MAYBE_INTERCEPT_MEMALIGN TSAN_INTERCEPT(memalign)
+#else
+#define TSAN_MAYBE_INTERCEPT_MEMALIGN
+#endif
+
+#if !SANITIZER_MAC
+TSAN_INTERCEPTOR(void*, aligned_alloc, uptr align, uptr sz) {
+  if (in_symbolizer())
+    return InternalAlloc(sz, nullptr, align);
+  SCOPED_INTERCEPTOR_RAW(aligned_alloc, align, sz);
+  return user_aligned_alloc(thr, pc, align, sz);
+}
+
+TSAN_INTERCEPTOR(void*, valloc, uptr sz) {
+  if (in_symbolizer())
+    return InternalAlloc(sz, nullptr, GetPageSizeCached());
+  SCOPED_INTERCEPTOR_RAW(valloc, sz);
+  return user_valloc(thr, pc, sz);
+}
+#endif
+
+#if SANITIZER_LINUX
+TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) {
+  if (in_symbolizer()) {
+    uptr PageSize = GetPageSizeCached();
+    sz = sz ? RoundUpTo(sz, PageSize) : PageSize;
+    return InternalAlloc(sz, nullptr, PageSize);
+  }
+  SCOPED_INTERCEPTOR_RAW(pvalloc, sz);
+  return user_pvalloc(thr, pc, sz);
+}
+#define TSAN_MAYBE_INTERCEPT_PVALLOC TSAN_INTERCEPT(pvalloc)
+#else
+#define TSAN_MAYBE_INTERCEPT_PVALLOC
+#endif
+
+#if !SANITIZER_MAC
+TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) {
+  if (in_symbolizer()) {
+    void *p = InternalAlloc(sz, nullptr, align);
+    if (!p)
+      return errno_ENOMEM;
+    *memptr = p;
+    return 0;
+  }
+  SCOPED_INTERCEPTOR_RAW(posix_memalign, memptr, align, sz);
+  return user_posix_memalign(thr, pc, memptr, align, sz);
+}
+#endif
+
+// Both __cxa_guard_acquire and pthread_once 0-initialize
+// the object initially. pthread_once does not have any
+// other ABI requirements. __cxa_guard_acquire assumes
+// that any non-0 value in the first byte means that
+// initialization is completed. Contents of the remaining
+// bytes are up to us.
+constexpr u32 kGuardInit = 0;
+constexpr u32 kGuardDone = 1;
+constexpr u32 kGuardRunning = 1 << 16;
+constexpr u32 kGuardWaiter = 1 << 17;
+
+static int guard_acquire(ThreadState *thr, uptr pc, atomic_uint32_t *g,
+                         bool blocking_hooks = true) {
+  if (blocking_hooks)
+    OnPotentiallyBlockingRegionBegin();
+  auto on_exit = at_scope_exit([blocking_hooks] {
+    if (blocking_hooks)
+      OnPotentiallyBlockingRegionEnd();
+  });
+
+  for (;;) {
+    u32 cmp = atomic_load(g, memory_order_acquire);
+    if (cmp == kGuardInit) {
+      if (atomic_compare_exchange_strong(g, &cmp, kGuardRunning,
+                                         memory_order_relaxed))
+        return 1;
+    } else if (cmp == kGuardDone) {
+      if (!thr->in_ignored_lib)
+        Acquire(thr, pc, (uptr)g);
+      return 0;
+    } else {
+      if ((cmp & kGuardWaiter) ||
+          atomic_compare_exchange_strong(g, &cmp, cmp | kGuardWaiter,
+                                         memory_order_relaxed))
+        FutexWait(g, cmp | kGuardWaiter);
+    }
+  }
+}
+
+static void guard_release(ThreadState *thr, uptr pc, atomic_uint32_t *g,
+                          u32 v) {
+  if (!thr->in_ignored_lib)
+    Release(thr, pc, (uptr)g);
+  u32 old = atomic_exchange(g, v, memory_order_release);
+  if (old & kGuardWaiter)
+    FutexWake(g, 1 << 30);
+}
+
+// __cxa_guard_acquire and friends need to be intercepted in a special way -
+// regular interceptors will break statically-linked libstdc++. Linux
+// interceptors are especially defined as weak functions (so that they don't
+// cause link errors when user defines them as well). So they silently
+// auto-disable themselves when such symbol is already present in the binary. If
+// we link libstdc++ statically, it will bring own __cxa_guard_acquire which
+// will silently replace our interceptor.  That's why on Linux we simply export
+// these interceptors with INTERFACE_ATTRIBUTE.
+// On OS X, we don't support statically linking, so we just use a regular
+// interceptor.
+#if SANITIZER_MAC
+#define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR
+#else
+#define STDCXX_INTERCEPTOR(rettype, name, ...) \
+  extern "C" rettype INTERFACE_ATTRIBUTE name(__VA_ARGS__)
+#endif
+
+// Used in thread-safe function static initialization.
+STDCXX_INTERCEPTOR(int, __cxa_guard_acquire, atomic_uint32_t *g) {
+  SCOPED_INTERCEPTOR_RAW(__cxa_guard_acquire, g);
+  return guard_acquire(thr, pc, g);
+}
+
+STDCXX_INTERCEPTOR(void, __cxa_guard_release, atomic_uint32_t *g) {
+  SCOPED_INTERCEPTOR_RAW(__cxa_guard_release, g);
+  guard_release(thr, pc, g, kGuardDone);
+}
+
+STDCXX_INTERCEPTOR(void, __cxa_guard_abort, atomic_uint32_t *g) {
+  SCOPED_INTERCEPTOR_RAW(__cxa_guard_abort, g);
+  guard_release(thr, pc, g, kGuardInit);
+}
+
+namespace __tsan {
+void DestroyThreadState() {
+  ThreadState *thr = cur_thread();
+  Processor *proc = thr->proc();
+  ThreadFinish(thr);
+  ProcUnwire(proc, thr);
+  ProcDestroy(proc);
+  DTLS_Destroy();
+  cur_thread_finalize();
+}
+
+void PlatformCleanUpThreadState(ThreadState *thr) {
+  ThreadSignalContext *sctx = thr->signal_ctx;
+  if (sctx) {
+    thr->signal_ctx = 0;
+    UnmapOrDie(sctx, sizeof(*sctx));
+  }
+}
+}  // namespace __tsan
+
+#if !SANITIZER_MAC && !SANITIZER_NETBSD && !SANITIZER_FREEBSD
+static void thread_finalize(void *v) {
+  uptr iter = (uptr)v;
+  if (iter > 1) {
+    if (pthread_setspecific(interceptor_ctx()->finalize_key,
+        (void*)(iter - 1))) {
+      Printf("ThreadSanitizer: failed to set thread key\n");
+      Die();
+    }
+    return;
+  }
+  DestroyThreadState();
+}
+#endif
+
+
+struct ThreadParam {
+  void* (*callback)(void *arg);
+  void *param;
+  Tid tid;
+  Semaphore created;
+  Semaphore started;
+};
+
+extern "C" void *__tsan_thread_start_func(void *arg) {
+  ThreadParam *p = (ThreadParam*)arg;
+  void* (*callback)(void *arg) = p->callback;
+  void *param = p->param;
+  {
+    ThreadState *thr = cur_thread_init();
+    // Thread-local state is not initialized yet.
+    ScopedIgnoreInterceptors ignore;
+#if !SANITIZER_MAC && !SANITIZER_NETBSD && !SANITIZER_FREEBSD
+    ThreadIgnoreBegin(thr, 0);
+    if (pthread_setspecific(interceptor_ctx()->finalize_key,
+                            (void *)GetPthreadDestructorIterations())) {
+      Printf("ThreadSanitizer: failed to set thread key\n");
+      Die();
+    }
+    ThreadIgnoreEnd(thr);
+#endif
+    p->created.Wait();
+    Processor *proc = ProcCreate();
+    ProcWire(proc, thr);
+    ThreadStart(thr, p->tid, GetTid(), ThreadType::Regular);
+    p->started.Post();
+  }
+  void *res = callback(param);
+  // Prevent the callback from being tail called,
+  // it mixes up stack traces.
+  volatile int foo = 42;
+  foo++;
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, pthread_create,
+    void *th, void *attr, void *(*callback)(void*), void * param) {
+  SCOPED_INTERCEPTOR_RAW(pthread_create, th, attr, callback, param);
+
+  MaybeSpawnBackgroundThread();
+
+  if (ctx->after_multithreaded_fork) {
+    if (flags()->die_after_fork) {
+      Report("ThreadSanitizer: starting new threads after multi-threaded "
+          "fork is not supported. Dying (set die_after_fork=0 to override)\n");
+      Die();
+    } else {
+      VPrintf(1,
+              "ThreadSanitizer: starting new threads after multi-threaded "
+              "fork is not supported (pid %lu). Continuing because of "
+              "die_after_fork=0, but you are on your own\n",
+              internal_getpid());
+    }
+  }
+  __sanitizer_pthread_attr_t myattr;
+  if (attr == 0) {
+    pthread_attr_init(&myattr);
+    attr = &myattr;
+  }
+  int detached = 0;
+  REAL(pthread_attr_getdetachstate)(attr, &detached);
+  AdjustStackSize(attr);
+
+  ThreadParam p;
+  p.callback = callback;
+  p.param = param;
+  p.tid = kMainTid;
+  int res = -1;
+  {
+    // Otherwise we see false positives in pthread stack manipulation.
+    ScopedIgnoreInterceptors ignore;
+    ThreadIgnoreBegin(thr, pc);
+    res = REAL(pthread_create)(th, attr, __tsan_thread_start_func, &p);
+    ThreadIgnoreEnd(thr);
+  }
+  if (res == 0) {
+    p.tid = ThreadCreate(thr, pc, *(uptr *)th, IsStateDetached(detached));
+    CHECK_NE(p.tid, kMainTid);
+    // Synchronization on p.tid serves two purposes:
+    // 1. ThreadCreate must finish before the new thread starts.
+    //    Otherwise the new thread can call pthread_detach, but the pthread_t
+    //    identifier is not yet registered in ThreadRegistry by ThreadCreate.
+    // 2. ThreadStart must finish before this thread continues.
+    //    Otherwise, this thread can call pthread_detach and reset thr->sync
+    //    before the new thread got a chance to acquire from it in ThreadStart.
+    p.created.Post();
+    p.started.Wait();
+  }
+  if (attr == &myattr)
+    pthread_attr_destroy(&myattr);
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, pthread_join, void *th, void **ret) {
+  SCOPED_INTERCEPTOR_RAW(pthread_join, th, ret);
+  Tid tid = ThreadConsumeTid(thr, pc, (uptr)th);
+  ThreadIgnoreBegin(thr, pc);
+  int res = BLOCK_REAL(pthread_join)(th, ret);
+  ThreadIgnoreEnd(thr);
+  if (res == 0) {
+    ThreadJoin(thr, pc, tid);
+  }
+  return res;
+}
+
+DEFINE_REAL_PTHREAD_FUNCTIONS
+
+TSAN_INTERCEPTOR(int, pthread_detach, void *th) {
+  SCOPED_INTERCEPTOR_RAW(pthread_detach, th);
+  Tid tid = ThreadConsumeTid(thr, pc, (uptr)th);
+  int res = REAL(pthread_detach)(th);
+  if (res == 0) {
+    ThreadDetach(thr, pc, tid);
+  }
+  return res;
+}
+
+TSAN_INTERCEPTOR(void, pthread_exit, void *retval) {
+  {
+    SCOPED_INTERCEPTOR_RAW(pthread_exit, retval);
+#if !SANITIZER_MAC && !SANITIZER_ANDROID
+    CHECK_EQ(thr, &cur_thread_placeholder);
+#endif
+  }
+  REAL(pthread_exit)(retval);
+}
+
+#if SANITIZER_LINUX
+TSAN_INTERCEPTOR(int, pthread_tryjoin_np, void *th, void **ret) {
+  SCOPED_INTERCEPTOR_RAW(pthread_tryjoin_np, th, ret);
+  Tid tid = ThreadConsumeTid(thr, pc, (uptr)th);
+  ThreadIgnoreBegin(thr, pc);
+  int res = REAL(pthread_tryjoin_np)(th, ret);
+  ThreadIgnoreEnd(thr);
+  if (res == 0)
+    ThreadJoin(thr, pc, tid);
+  else
+    ThreadNotJoined(thr, pc, tid, (uptr)th);
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, pthread_timedjoin_np, void *th, void **ret,
+                 const struct timespec *abstime) {
+  SCOPED_INTERCEPTOR_RAW(pthread_timedjoin_np, th, ret, abstime);
+  Tid tid = ThreadConsumeTid(thr, pc, (uptr)th);
+  ThreadIgnoreBegin(thr, pc);
+  int res = BLOCK_REAL(pthread_timedjoin_np)(th, ret, abstime);
+  ThreadIgnoreEnd(thr);
+  if (res == 0)
+    ThreadJoin(thr, pc, tid);
+  else
+    ThreadNotJoined(thr, pc, tid, (uptr)th);
+  return res;
+}
+#endif
+
+// Problem:
+// NPTL implementation of pthread_cond has 2 versions (2.2.5 and 2.3.2).
+// pthread_cond_t has different size in the different versions.
+// If call new REAL functions for old pthread_cond_t, they will corrupt memory
+// after pthread_cond_t (old cond is smaller).
+// If we call old REAL functions for new pthread_cond_t, we will lose  some
+// functionality (e.g. old functions do not support waiting against
+// CLOCK_REALTIME).
+// Proper handling would require to have 2 versions of interceptors as well.
+// But this is messy, in particular requires linker scripts when sanitizer
+// runtime is linked into a shared library.
+// Instead we assume we don't have dynamic libraries built against old
+// pthread (2.2.5 is dated by 2002). And provide legacy_pthread_cond flag
+// that allows to work with old libraries (but this mode does not support
+// some features, e.g. pthread_condattr_getpshared).
+static void *init_cond(void *c, bool force = false) {
+  // sizeof(pthread_cond_t) >= sizeof(uptr) in both versions.
+  // So we allocate additional memory on the side large enough to hold
+  // any pthread_cond_t object. Always call new REAL functions, but pass
+  // the aux object to them.
+  // Note: the code assumes that PTHREAD_COND_INITIALIZER initializes
+  // first word of pthread_cond_t to zero.
+  // It's all relevant only for linux.
+  if (!common_flags()->legacy_pthread_cond)
+    return c;
+  atomic_uintptr_t *p = (atomic_uintptr_t*)c;
+  uptr cond = atomic_load(p, memory_order_acquire);
+  if (!force && cond != 0)
+    return (void*)cond;
+  void *newcond = WRAP(malloc)(pthread_cond_t_sz);
+  internal_memset(newcond, 0, pthread_cond_t_sz);
+  if (atomic_compare_exchange_strong(p, &cond, (uptr)newcond,
+      memory_order_acq_rel))
+    return newcond;
+  WRAP(free)(newcond);
+  return (void*)cond;
+}
+
+namespace {
+
+template <class Fn>
+struct CondMutexUnlockCtx {
+  ScopedInterceptor *si;
+  ThreadState *thr;
+  uptr pc;
+  void *m;
+  void *c;
+  const Fn &fn;
+
+  int Cancel() const { return fn(); }
+  void Unlock() const;
+};
+
+template <class Fn>
+void CondMutexUnlockCtx<Fn>::Unlock() const {
+  // pthread_cond_wait interceptor has enabled async signal delivery
+  // (see BlockingCall below). Disable async signals since we are running
+  // tsan code. Also ScopedInterceptor and BlockingCall destructors won't run
+  // since the thread is cancelled, so we have to manually execute them
+  // (the thread still can run some user code due to pthread_cleanup_push).
+  ThreadSignalContext *ctx = SigCtx(thr);
+  CHECK_EQ(atomic_load(&ctx->in_blocking_func, memory_order_relaxed), 1);
+  atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed);
+  MutexPostLock(thr, pc, (uptr)m, MutexFlagDoPreLockOnPostLock);
+  // Undo BlockingCall ctor effects.
+  thr->ignore_interceptors--;
+  si->~ScopedInterceptor();
+}
+}  // namespace
+
+INTERCEPTOR(int, pthread_cond_init, void *c, void *a) {
+  void *cond = init_cond(c, true);
+  SCOPED_TSAN_INTERCEPTOR(pthread_cond_init, cond, a);
+  MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), true);
+  return REAL(pthread_cond_init)(cond, a);
+}
+
+template <class Fn>
+int cond_wait(ThreadState *thr, uptr pc, ScopedInterceptor *si, const Fn &fn,
+              void *c, void *m) {
+  MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false);
+  MutexUnlock(thr, pc, (uptr)m);
+  int res = 0;
+  // This ensures that we handle mutex lock even in case of pthread_cancel.
+  // See test/tsan/cond_cancel.cpp.
+  {
+    // Enable signal delivery while the thread is blocked.
+    BlockingCall bc(thr);
+    CondMutexUnlockCtx<Fn> arg = {si, thr, pc, m, c, fn};
+    res = call_pthread_cancel_with_cleanup(
+        [](void *arg) -> int {
+          return ((const CondMutexUnlockCtx<Fn> *)arg)->Cancel();
+        },
+        [](void *arg) { ((const CondMutexUnlockCtx<Fn> *)arg)->Unlock(); },
+        &arg);
+  }
+  if (res == errno_EOWNERDEAD) MutexRepair(thr, pc, (uptr)m);
+  MutexPostLock(thr, pc, (uptr)m, MutexFlagDoPreLockOnPostLock);
+  return res;
+}
+
+INTERCEPTOR(int, pthread_cond_wait, void *c, void *m) {
+  void *cond = init_cond(c);
+  SCOPED_TSAN_INTERCEPTOR(pthread_cond_wait, cond, m);
+  return cond_wait(
+      thr, pc, &si, [=]() { return REAL(pthread_cond_wait)(cond, m); }, cond,
+      m);
+}
+
+INTERCEPTOR(int, pthread_cond_timedwait, void *c, void *m, void *abstime) {
+  void *cond = init_cond(c);
+  SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait, cond, m, abstime);
+  return cond_wait(
+      thr, pc, &si,
+      [=]() { return REAL(pthread_cond_timedwait)(cond, m, abstime); }, cond,
+      m);
+}
+
+#if SANITIZER_LINUX
+INTERCEPTOR(int, pthread_cond_clockwait, void *c, void *m,
+            __sanitizer_clockid_t clock, void *abstime) {
+  void *cond = init_cond(c);
+  SCOPED_TSAN_INTERCEPTOR(pthread_cond_clockwait, cond, m, clock, abstime);
+  return cond_wait(
+      thr, pc, &si,
+      [=]() { return REAL(pthread_cond_clockwait)(cond, m, clock, abstime); },
+      cond, m);
+}
+#define TSAN_MAYBE_PTHREAD_COND_CLOCKWAIT TSAN_INTERCEPT(pthread_cond_clockwait)
+#else
+#define TSAN_MAYBE_PTHREAD_COND_CLOCKWAIT
+#endif
+
+#if SANITIZER_MAC
+INTERCEPTOR(int, pthread_cond_timedwait_relative_np, void *c, void *m,
+            void *reltime) {
+  void *cond = init_cond(c);
+  SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait_relative_np, cond, m, reltime);
+  return cond_wait(
+      thr, pc, &si,
+      [=]() {
+        return REAL(pthread_cond_timedwait_relative_np)(cond, m, reltime);
+      },
+      cond, m);
+}
+#endif
+
+INTERCEPTOR(int, pthread_cond_signal, void *c) {
+  void *cond = init_cond(c);
+  SCOPED_TSAN_INTERCEPTOR(pthread_cond_signal, cond);
+  MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false);
+  return REAL(pthread_cond_signal)(cond);
+}
+
+INTERCEPTOR(int, pthread_cond_broadcast, void *c) {
+  void *cond = init_cond(c);
+  SCOPED_TSAN_INTERCEPTOR(pthread_cond_broadcast, cond);
+  MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false);
+  return REAL(pthread_cond_broadcast)(cond);
+}
+
+INTERCEPTOR(int, pthread_cond_destroy, void *c) {
+  void *cond = init_cond(c);
+  SCOPED_TSAN_INTERCEPTOR(pthread_cond_destroy, cond);
+  MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), true);
+  int res = REAL(pthread_cond_destroy)(cond);
+  if (common_flags()->legacy_pthread_cond) {
+    // Free our aux cond and zero the pointer to not leave dangling pointers.
+    WRAP(free)(cond);
+    atomic_store((atomic_uintptr_t*)c, 0, memory_order_relaxed);
+  }
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, pthread_mutex_init, void *m, void *a) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_mutex_init, m, a);
+  int res = REAL(pthread_mutex_init)(m, a);
+  if (res == 0) {
+    u32 flagz = 0;
+    if (a) {
+      int type = 0;
+      if (REAL(pthread_mutexattr_gettype)(a, &type) == 0)
+        if (type == PTHREAD_MUTEX_RECURSIVE ||
+            type == PTHREAD_MUTEX_RECURSIVE_NP)
+          flagz |= MutexFlagWriteReentrant;
+    }
+    MutexCreate(thr, pc, (uptr)m, flagz);
+  }
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_mutex_destroy, m);
+  int res = REAL(pthread_mutex_destroy)(m);
+  if (res == 0 || res == errno_EBUSY) {
+    MutexDestroy(thr, pc, (uptr)m);
+  }
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_mutex_trylock, m);
+  int res = REAL(pthread_mutex_trylock)(m);
+  if (res == errno_EOWNERDEAD)
+    MutexRepair(thr, pc, (uptr)m);
+  if (res == 0 || res == errno_EOWNERDEAD)
+    MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock);
+  return res;
+}
+
+#if !SANITIZER_MAC
+TSAN_INTERCEPTOR(int, pthread_mutex_timedlock, void *m, void *abstime) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_mutex_timedlock, m, abstime);
+  int res = REAL(pthread_mutex_timedlock)(m, abstime);
+  if (res == 0) {
+    MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock);
+  }
+  return res;
+}
+#endif
+
+#if !SANITIZER_MAC
+TSAN_INTERCEPTOR(int, pthread_spin_init, void *m, int pshared) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_spin_init, m, pshared);
+  int res = REAL(pthread_spin_init)(m, pshared);
+  if (res == 0) {
+    MutexCreate(thr, pc, (uptr)m);
+  }
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, pthread_spin_destroy, void *m) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_spin_destroy, m);
+  int res = REAL(pthread_spin_destroy)(m);
+  if (res == 0) {
+    MutexDestroy(thr, pc, (uptr)m);
+  }
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, pthread_spin_lock, void *m) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_spin_lock, m);
+  MutexPreLock(thr, pc, (uptr)m);
+  int res = REAL(pthread_spin_lock)(m);
+  if (res == 0) {
+    MutexPostLock(thr, pc, (uptr)m);
+  }
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, pthread_spin_trylock, void *m) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_spin_trylock, m);
+  int res = REAL(pthread_spin_trylock)(m);
+  if (res == 0) {
+    MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock);
+  }
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, pthread_spin_unlock, void *m) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_spin_unlock, m);
+  MutexUnlock(thr, pc, (uptr)m);
+  int res = REAL(pthread_spin_unlock)(m);
+  return res;
+}
+#endif
+
+TSAN_INTERCEPTOR(int, pthread_rwlock_init, void *m, void *a) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_init, m, a);
+  int res = REAL(pthread_rwlock_init)(m, a);
+  if (res == 0) {
+    MutexCreate(thr, pc, (uptr)m);
+  }
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, pthread_rwlock_destroy, void *m) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_destroy, m);
+  int res = REAL(pthread_rwlock_destroy)(m);
+  if (res == 0) {
+    MutexDestroy(thr, pc, (uptr)m);
+  }
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, pthread_rwlock_rdlock, void *m) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_rdlock, m);
+  MutexPreReadLock(thr, pc, (uptr)m);
+  int res = REAL(pthread_rwlock_rdlock)(m);
+  if (res == 0) {
+    MutexPostReadLock(thr, pc, (uptr)m);
+  }
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, pthread_rwlock_tryrdlock, void *m) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_tryrdlock, m);
+  int res = REAL(pthread_rwlock_tryrdlock)(m);
+  if (res == 0) {
+    MutexPostReadLock(thr, pc, (uptr)m, MutexFlagTryLock);
+  }
+  return res;
+}
+
+#if !SANITIZER_MAC
+TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock, void *m, void *abstime) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedrdlock, m, abstime);
+  int res = REAL(pthread_rwlock_timedrdlock)(m, abstime);
+  if (res == 0) {
+    MutexPostReadLock(thr, pc, (uptr)m);
+  }
+  return res;
+}
+#endif
+
+TSAN_INTERCEPTOR(int, pthread_rwlock_wrlock, void *m) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_wrlock, m);
+  MutexPreLock(thr, pc, (uptr)m);
+  int res = REAL(pthread_rwlock_wrlock)(m);
+  if (res == 0) {
+    MutexPostLock(thr, pc, (uptr)m);
+  }
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, pthread_rwlock_trywrlock, void *m) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_trywrlock, m);
+  int res = REAL(pthread_rwlock_trywrlock)(m);
+  if (res == 0) {
+    MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock);
+  }
+  return res;
+}
+
+#if !SANITIZER_MAC
+TSAN_INTERCEPTOR(int, pthread_rwlock_timedwrlock, void *m, void *abstime) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedwrlock, m, abstime);
+  int res = REAL(pthread_rwlock_timedwrlock)(m, abstime);
+  if (res == 0) {
+    MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock);
+  }
+  return res;
+}
+#endif
+
+TSAN_INTERCEPTOR(int, pthread_rwlock_unlock, void *m) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_unlock, m);
+  MutexReadOrWriteUnlock(thr, pc, (uptr)m);
+  int res = REAL(pthread_rwlock_unlock)(m);
+  return res;
+}
+
+#if !SANITIZER_MAC
+TSAN_INTERCEPTOR(int, pthread_barrier_init, void *b, void *a, unsigned count) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_barrier_init, b, a, count);
+  MemoryAccess(thr, pc, (uptr)b, 1, kAccessWrite);
+  int res = REAL(pthread_barrier_init)(b, a, count);
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, pthread_barrier_destroy, void *b) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_barrier_destroy, b);
+  MemoryAccess(thr, pc, (uptr)b, 1, kAccessWrite);
+  int res = REAL(pthread_barrier_destroy)(b);
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, pthread_barrier_wait, void *b) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_barrier_wait, b);
+  Release(thr, pc, (uptr)b);
+  MemoryAccess(thr, pc, (uptr)b, 1, kAccessRead);
+  int res = REAL(pthread_barrier_wait)(b);
+  MemoryAccess(thr, pc, (uptr)b, 1, kAccessRead);
+  if (res == 0 || res == PTHREAD_BARRIER_SERIAL_THREAD) {
+    Acquire(thr, pc, (uptr)b);
+  }
+  return res;
+}
+#endif
+
+TSAN_INTERCEPTOR(int, pthread_once, void *o, void (*f)()) {
+  SCOPED_INTERCEPTOR_RAW(pthread_once, o, f);
+  if (o == 0 || f == 0)
+    return errno_EINVAL;
+  atomic_uint32_t *a;
+
+  if (SANITIZER_MAC)
+    a = static_cast<atomic_uint32_t*>((void *)((char *)o + sizeof(long_t)));
+  else if (SANITIZER_NETBSD)
+    a = static_cast<atomic_uint32_t*>
+          ((void *)((char *)o + __sanitizer::pthread_mutex_t_sz));
+  else
+    a = static_cast<atomic_uint32_t*>(o);
+
+  // Mac OS X appears to use pthread_once() where calling BlockingRegion hooks
+  // result in crashes due to too little stack space.
+  if (guard_acquire(thr, pc, a, !SANITIZER_MAC)) {
+    (*f)();
+    guard_release(thr, pc, a, kGuardDone);
+  }
+  return 0;
+}
+
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+TSAN_INTERCEPTOR(int, __fxstat, int version, int fd, void *buf) {
+  SCOPED_TSAN_INTERCEPTOR(__fxstat, version, fd, buf);
+  if (fd > 0)
+    FdAccess(thr, pc, fd);
+  return REAL(__fxstat)(version, fd, buf);
+}
+#define TSAN_MAYBE_INTERCEPT___FXSTAT TSAN_INTERCEPT(__fxstat)
+#else
+#define TSAN_MAYBE_INTERCEPT___FXSTAT
+#endif
+
+TSAN_INTERCEPTOR(int, fstat, int fd, void *buf) {
+#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_ANDROID || SANITIZER_NETBSD
+  SCOPED_TSAN_INTERCEPTOR(fstat, fd, buf);
+  if (fd > 0)
+    FdAccess(thr, pc, fd);
+  return REAL(fstat)(fd, buf);
+#else
+  SCOPED_TSAN_INTERCEPTOR(__fxstat, 0, fd, buf);
+  if (fd > 0)
+    FdAccess(thr, pc, fd);
+  return REAL(__fxstat)(0, fd, buf);
+#endif
+}
+
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+TSAN_INTERCEPTOR(int, __fxstat64, int version, int fd, void *buf) {
+  SCOPED_TSAN_INTERCEPTOR(__fxstat64, version, fd, buf);
+  if (fd > 0)
+    FdAccess(thr, pc, fd);
+  return REAL(__fxstat64)(version, fd, buf);
+}
+#define TSAN_MAYBE_INTERCEPT___FXSTAT64 TSAN_INTERCEPT(__fxstat64)
+#else
+#define TSAN_MAYBE_INTERCEPT___FXSTAT64
+#endif
+
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+TSAN_INTERCEPTOR(int, fstat64, int fd, void *buf) {
+  SCOPED_TSAN_INTERCEPTOR(__fxstat64, 0, fd, buf);
+  if (fd > 0)
+    FdAccess(thr, pc, fd);
+  return REAL(__fxstat64)(0, fd, buf);
+}
+#define TSAN_MAYBE_INTERCEPT_FSTAT64 TSAN_INTERCEPT(fstat64)
+#else
+#define TSAN_MAYBE_INTERCEPT_FSTAT64
+#endif
+
+TSAN_INTERCEPTOR(int, open, const char *name, int oflag, ...) {
+  va_list ap;
+  va_start(ap, oflag);
+  mode_t mode = va_arg(ap, int);
+  va_end(ap);
+  SCOPED_TSAN_INTERCEPTOR(open, name, oflag, mode);
+  READ_STRING(thr, pc, name, 0);
+  int fd = REAL(open)(name, oflag, mode);
+  if (fd >= 0)
+    FdFileCreate(thr, pc, fd);
+  return fd;
+}
+
+#if SANITIZER_LINUX
+TSAN_INTERCEPTOR(int, open64, const char *name, int oflag, ...) {
+  va_list ap;
+  va_start(ap, oflag);
+  mode_t mode = va_arg(ap, int);
+  va_end(ap);
+  SCOPED_TSAN_INTERCEPTOR(open64, name, oflag, mode);
+  READ_STRING(thr, pc, name, 0);
+  int fd = REAL(open64)(name, oflag, mode);
+  if (fd >= 0)
+    FdFileCreate(thr, pc, fd);
+  return fd;
+}
+#define TSAN_MAYBE_INTERCEPT_OPEN64 TSAN_INTERCEPT(open64)
+#else
+#define TSAN_MAYBE_INTERCEPT_OPEN64
+#endif
+
+TSAN_INTERCEPTOR(int, creat, const char *name, int mode) {
+  SCOPED_TSAN_INTERCEPTOR(creat, name, mode);
+  READ_STRING(thr, pc, name, 0);
+  int fd = REAL(creat)(name, mode);
+  if (fd >= 0)
+    FdFileCreate(thr, pc, fd);
+  return fd;
+}
+
+#if SANITIZER_LINUX
+TSAN_INTERCEPTOR(int, creat64, const char *name, int mode) {
+  SCOPED_TSAN_INTERCEPTOR(creat64, name, mode);
+  READ_STRING(thr, pc, name, 0);
+  int fd = REAL(creat64)(name, mode);
+  if (fd >= 0)
+    FdFileCreate(thr, pc, fd);
+  return fd;
+}
+#define TSAN_MAYBE_INTERCEPT_CREAT64 TSAN_INTERCEPT(creat64)
+#else
+#define TSAN_MAYBE_INTERCEPT_CREAT64
+#endif
+
+TSAN_INTERCEPTOR(int, dup, int oldfd) {
+  SCOPED_TSAN_INTERCEPTOR(dup, oldfd);
+  int newfd = REAL(dup)(oldfd);
+  if (oldfd >= 0 && newfd >= 0 && newfd != oldfd)
+    FdDup(thr, pc, oldfd, newfd, true);
+  return newfd;
+}
+
+TSAN_INTERCEPTOR(int, dup2, int oldfd, int newfd) {
+  SCOPED_TSAN_INTERCEPTOR(dup2, oldfd, newfd);
+  int newfd2 = REAL(dup2)(oldfd, newfd);
+  if (oldfd >= 0 && newfd2 >= 0 && newfd2 != oldfd)
+    FdDup(thr, pc, oldfd, newfd2, false);
+  return newfd2;
+}
+
+#if !SANITIZER_MAC
+TSAN_INTERCEPTOR(int, dup3, int oldfd, int newfd, int flags) {
+  SCOPED_TSAN_INTERCEPTOR(dup3, oldfd, newfd, flags);
+  int newfd2 = REAL(dup3)(oldfd, newfd, flags);
+  if (oldfd >= 0 && newfd2 >= 0 && newfd2 != oldfd)
+    FdDup(thr, pc, oldfd, newfd2, false);
+  return newfd2;
+}
+#endif
+
+#if SANITIZER_LINUX
+TSAN_INTERCEPTOR(int, eventfd, unsigned initval, int flags) {
+  SCOPED_TSAN_INTERCEPTOR(eventfd, initval, flags);
+  int fd = REAL(eventfd)(initval, flags);
+  if (fd >= 0)
+    FdEventCreate(thr, pc, fd);
+  return fd;
+}
+#define TSAN_MAYBE_INTERCEPT_EVENTFD TSAN_INTERCEPT(eventfd)
+#else
+#define TSAN_MAYBE_INTERCEPT_EVENTFD
+#endif
+
+#if SANITIZER_LINUX
+TSAN_INTERCEPTOR(int, signalfd, int fd, void *mask, int flags) {
+  SCOPED_TSAN_INTERCEPTOR(signalfd, fd, mask, flags);
+  if (fd >= 0)
+    FdClose(thr, pc, fd);
+  fd = REAL(signalfd)(fd, mask, flags);
+  if (fd >= 0)
+    FdSignalCreate(thr, pc, fd);
+  return fd;
+}
+#define TSAN_MAYBE_INTERCEPT_SIGNALFD TSAN_INTERCEPT(signalfd)
+#else
+#define TSAN_MAYBE_INTERCEPT_SIGNALFD
+#endif
+
+#if SANITIZER_LINUX
+TSAN_INTERCEPTOR(int, inotify_init, int fake) {
+  SCOPED_TSAN_INTERCEPTOR(inotify_init, fake);
+  int fd = REAL(inotify_init)(fake);
+  if (fd >= 0)
+    FdInotifyCreate(thr, pc, fd);
+  return fd;
+}
+#define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT TSAN_INTERCEPT(inotify_init)
+#else
+#define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT
+#endif
+
+#if SANITIZER_LINUX
+TSAN_INTERCEPTOR(int, inotify_init1, int flags) {
+  SCOPED_TSAN_INTERCEPTOR(inotify_init1, flags);
+  int fd = REAL(inotify_init1)(flags);
+  if (fd >= 0)
+    FdInotifyCreate(thr, pc, fd);
+  return fd;
+}
+#define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1 TSAN_INTERCEPT(inotify_init1)
+#else
+#define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1
+#endif
+
+TSAN_INTERCEPTOR(int, socket, int domain, int type, int protocol) {
+  SCOPED_TSAN_INTERCEPTOR(socket, domain, type, protocol);
+  int fd = REAL(socket)(domain, type, protocol);
+  if (fd >= 0)
+    FdSocketCreate(thr, pc, fd);
+  return fd;
+}
+
+TSAN_INTERCEPTOR(int, socketpair, int domain, int type, int protocol, int *fd) {
+  SCOPED_TSAN_INTERCEPTOR(socketpair, domain, type, protocol, fd);
+  int res = REAL(socketpair)(domain, type, protocol, fd);
+  if (res == 0 && fd[0] >= 0 && fd[1] >= 0)
+    FdPipeCreate(thr, pc, fd[0], fd[1]);
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, connect, int fd, void *addr, unsigned addrlen) {
+  SCOPED_TSAN_INTERCEPTOR(connect, fd, addr, addrlen);
+  FdSocketConnecting(thr, pc, fd);
+  int res = REAL(connect)(fd, addr, addrlen);
+  if (res == 0 && fd >= 0)
+    FdSocketConnect(thr, pc, fd);
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, bind, int fd, void *addr, unsigned addrlen) {
+  SCOPED_TSAN_INTERCEPTOR(bind, fd, addr, addrlen);
+  int res = REAL(bind)(fd, addr, addrlen);
+  if (fd > 0 && res == 0)
+    FdAccess(thr, pc, fd);
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, listen, int fd, int backlog) {
+  SCOPED_TSAN_INTERCEPTOR(listen, fd, backlog);
+  int res = REAL(listen)(fd, backlog);
+  if (fd > 0 && res == 0)
+    FdAccess(thr, pc, fd);
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, close, int fd) {
+  SCOPED_TSAN_INTERCEPTOR(close, fd);
+  if (fd >= 0)
+    FdClose(thr, pc, fd);
+  return REAL(close)(fd);
+}
+
+#if SANITIZER_LINUX
+TSAN_INTERCEPTOR(int, __close, int fd) {
+  SCOPED_TSAN_INTERCEPTOR(__close, fd);
+  if (fd >= 0)
+    FdClose(thr, pc, fd);
+  return REAL(__close)(fd);
+}
+#define TSAN_MAYBE_INTERCEPT___CLOSE TSAN_INTERCEPT(__close)
+#else
+#define TSAN_MAYBE_INTERCEPT___CLOSE
+#endif
+
+// glibc guts
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+TSAN_INTERCEPTOR(void, __res_iclose, void *state, bool free_addr) {
+  SCOPED_TSAN_INTERCEPTOR(__res_iclose, state, free_addr);
+  int fds[64];
+  int cnt = ExtractResolvFDs(state, fds, ARRAY_SIZE(fds));
+  for (int i = 0; i < cnt; i++) {
+    if (fds[i] > 0)
+      FdClose(thr, pc, fds[i]);
+  }
+  REAL(__res_iclose)(state, free_addr);
+}
+#define TSAN_MAYBE_INTERCEPT___RES_ICLOSE TSAN_INTERCEPT(__res_iclose)
+#else
+#define TSAN_MAYBE_INTERCEPT___RES_ICLOSE
+#endif
+
+TSAN_INTERCEPTOR(int, pipe, int *pipefd) {
+  SCOPED_TSAN_INTERCEPTOR(pipe, pipefd);
+  int res = REAL(pipe)(pipefd);
+  if (res == 0 && pipefd[0] >= 0 && pipefd[1] >= 0)
+    FdPipeCreate(thr, pc, pipefd[0], pipefd[1]);
+  return res;
+}
+
+#if !SANITIZER_MAC
+TSAN_INTERCEPTOR(int, pipe2, int *pipefd, int flags) {
+  SCOPED_TSAN_INTERCEPTOR(pipe2, pipefd, flags);
+  int res = REAL(pipe2)(pipefd, flags);
+  if (res == 0 && pipefd[0] >= 0 && pipefd[1] >= 0)
+    FdPipeCreate(thr, pc, pipefd[0], pipefd[1]);
+  return res;
+}
+#endif
+
+TSAN_INTERCEPTOR(int, unlink, char *path) {
+  SCOPED_TSAN_INTERCEPTOR(unlink, path);
+  Release(thr, pc, File2addr(path));
+  int res = REAL(unlink)(path);
+  return res;
+}
+
+TSAN_INTERCEPTOR(void*, tmpfile, int fake) {
+  SCOPED_TSAN_INTERCEPTOR(tmpfile, fake);
+  void *res = REAL(tmpfile)(fake);
+  if (res) {
+    int fd = fileno_unlocked(res);
+    if (fd >= 0)
+      FdFileCreate(thr, pc, fd);
+  }
+  return res;
+}
+
+#if SANITIZER_LINUX
+TSAN_INTERCEPTOR(void*, tmpfile64, int fake) {
+  SCOPED_TSAN_INTERCEPTOR(tmpfile64, fake);
+  void *res = REAL(tmpfile64)(fake);
+  if (res) {
+    int fd = fileno_unlocked(res);
+    if (fd >= 0)
+      FdFileCreate(thr, pc, fd);
+  }
+  return res;
+}
+#define TSAN_MAYBE_INTERCEPT_TMPFILE64 TSAN_INTERCEPT(tmpfile64)
+#else
+#define TSAN_MAYBE_INTERCEPT_TMPFILE64
+#endif
+
+static void FlushStreams() {
+  // Flushing all the streams here may freeze the process if a child thread is
+  // performing file stream operations at the same time.
+  REAL(fflush)(stdout);
+  REAL(fflush)(stderr);
+}
+
+TSAN_INTERCEPTOR(void, abort, int fake) {
+  SCOPED_TSAN_INTERCEPTOR(abort, fake);
+  FlushStreams();
+  REAL(abort)(fake);
+}
+
+TSAN_INTERCEPTOR(int, rmdir, char *path) {
+  SCOPED_TSAN_INTERCEPTOR(rmdir, path);
+  Release(thr, pc, Dir2addr(path));
+  int res = REAL(rmdir)(path);
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, closedir, void *dirp) {
+  SCOPED_TSAN_INTERCEPTOR(closedir, dirp);
+  if (dirp) {
+    int fd = dirfd(dirp);
+    FdClose(thr, pc, fd);
+  }
+  return REAL(closedir)(dirp);
+}
+
+#if SANITIZER_LINUX
+TSAN_INTERCEPTOR(int, epoll_create, int size) {
+  SCOPED_TSAN_INTERCEPTOR(epoll_create, size);
+  int fd = REAL(epoll_create)(size);
+  if (fd >= 0)
+    FdPollCreate(thr, pc, fd);
+  return fd;
+}
+
+TSAN_INTERCEPTOR(int, epoll_create1, int flags) {
+  SCOPED_TSAN_INTERCEPTOR(epoll_create1, flags);
+  int fd = REAL(epoll_create1)(flags);
+  if (fd >= 0)
+    FdPollCreate(thr, pc, fd);
+  return fd;
+}
+
+TSAN_INTERCEPTOR(int, epoll_ctl, int epfd, int op, int fd, void *ev) {
+  SCOPED_TSAN_INTERCEPTOR(epoll_ctl, epfd, op, fd, ev);
+  if (epfd >= 0)
+    FdAccess(thr, pc, epfd);
+  if (epfd >= 0 && fd >= 0)
+    FdAccess(thr, pc, fd);
+  if (op == EPOLL_CTL_ADD && epfd >= 0)
+    FdRelease(thr, pc, epfd);
+  int res = REAL(epoll_ctl)(epfd, op, fd, ev);
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, epoll_wait, int epfd, void *ev, int cnt, int timeout) {
+  SCOPED_TSAN_INTERCEPTOR(epoll_wait, epfd, ev, cnt, timeout);
+  if (epfd >= 0)
+    FdAccess(thr, pc, epfd);
+  int res = BLOCK_REAL(epoll_wait)(epfd, ev, cnt, timeout);
+  if (res > 0 && epfd >= 0)
+    FdAcquire(thr, pc, epfd);
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, epoll_pwait, int epfd, void *ev, int cnt, int timeout,
+                 void *sigmask) {
+  SCOPED_TSAN_INTERCEPTOR(epoll_pwait, epfd, ev, cnt, timeout, sigmask);
+  if (epfd >= 0)
+    FdAccess(thr, pc, epfd);
+  int res = BLOCK_REAL(epoll_pwait)(epfd, ev, cnt, timeout, sigmask);
+  if (res > 0 && epfd >= 0)
+    FdAcquire(thr, pc, epfd);
+  return res;
+}
+
+#define TSAN_MAYBE_INTERCEPT_EPOLL \
+    TSAN_INTERCEPT(epoll_create); \
+    TSAN_INTERCEPT(epoll_create1); \
+    TSAN_INTERCEPT(epoll_ctl); \
+    TSAN_INTERCEPT(epoll_wait); \
+    TSAN_INTERCEPT(epoll_pwait)
+#else
+#define TSAN_MAYBE_INTERCEPT_EPOLL
+#endif
+
+// The following functions are intercepted merely to process pending signals.
+// If program blocks signal X, we must deliver the signal before the function
+// returns. Similarly, if program unblocks a signal (or returns from sigsuspend)
+// it's better to deliver the signal straight away.
+TSAN_INTERCEPTOR(int, sigsuspend, const __sanitizer_sigset_t *mask) {
+  SCOPED_TSAN_INTERCEPTOR(sigsuspend, mask);
+  return REAL(sigsuspend)(mask);
+}
+
+TSAN_INTERCEPTOR(int, sigblock, int mask) {
+  SCOPED_TSAN_INTERCEPTOR(sigblock, mask);
+  return REAL(sigblock)(mask);
+}
+
+TSAN_INTERCEPTOR(int, sigsetmask, int mask) {
+  SCOPED_TSAN_INTERCEPTOR(sigsetmask, mask);
+  return REAL(sigsetmask)(mask);
+}
+
+TSAN_INTERCEPTOR(int, pthread_sigmask, int how, const __sanitizer_sigset_t *set,
+    __sanitizer_sigset_t *oldset) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_sigmask, how, set, oldset);
+  return REAL(pthread_sigmask)(how, set, oldset);
+}
+
+namespace __tsan {
+
+static void ReportErrnoSpoiling(ThreadState *thr, uptr pc) {
+  VarSizeStackTrace stack;
+  // StackTrace::GetNestInstructionPc(pc) is used because return address is
+  // expected, OutputReport() will undo this.
+  ObtainCurrentStack(thr, StackTrace::GetNextInstructionPc(pc), &stack);
+  ThreadRegistryLock l(&ctx->thread_registry);
+  ScopedReport rep(ReportTypeErrnoInSignal);
+  if (!IsFiredSuppression(ctx, ReportTypeErrnoInSignal, stack)) {
+    rep.AddStack(stack, true);
+    OutputReport(thr, rep);
+  }
+}
+
+static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire,
+                                  int sig, __sanitizer_siginfo *info,
+                                  void *uctx) {
+  __sanitizer_sigaction *sigactions = interceptor_ctx()->sigactions;
+  if (acquire)
+    Acquire(thr, 0, (uptr)&sigactions[sig]);
+  // Signals are generally asynchronous, so if we receive a signals when
+  // ignores are enabled we should disable ignores. This is critical for sync
+  // and interceptors, because otherwise we can miss synchronization and report
+  // false races.
+  int ignore_reads_and_writes = thr->ignore_reads_and_writes;
+  int ignore_interceptors = thr->ignore_interceptors;
+  int ignore_sync = thr->ignore_sync;
+  // For symbolizer we only process SIGSEGVs synchronously
+  // (bug in symbolizer or in tsan). But we want to reset
+  // in_symbolizer to fail gracefully. Symbolizer and user code
+  // use different memory allocators, so if we don't reset
+  // in_symbolizer we can get memory allocated with one being
+  // feed with another, which can cause more crashes.
+  int in_symbolizer = thr->in_symbolizer;
+  if (!ctx->after_multithreaded_fork) {
+    thr->ignore_reads_and_writes = 0;
+    thr->fast_state.ClearIgnoreBit();
+    thr->ignore_interceptors = 0;
+    thr->ignore_sync = 0;
+    thr->in_symbolizer = 0;
+  }
+  // Ensure that the handler does not spoil errno.
+  const int saved_errno = errno;
+  errno = 99;
+  // This code races with sigaction. Be careful to not read sa_sigaction twice.
+  // Also need to remember pc for reporting before the call,
+  // because the handler can reset it.
+  volatile uptr pc = (sigactions[sig].sa_flags & SA_SIGINFO)
+                         ? (uptr)sigactions[sig].sigaction
+                         : (uptr)sigactions[sig].handler;
+  if (pc != sig_dfl && pc != sig_ign) {
+    // The callback can be either sa_handler or sa_sigaction.
+    // They have different signatures, but we assume that passing
+    // additional arguments to sa_handler works and is harmless.
+    ((__sanitizer_sigactionhandler_ptr)pc)(sig, info, uctx);
+  }
+  if (!ctx->after_multithreaded_fork) {
+    thr->ignore_reads_and_writes = ignore_reads_and_writes;
+    if (ignore_reads_and_writes)
+      thr->fast_state.SetIgnoreBit();
+    thr->ignore_interceptors = ignore_interceptors;
+    thr->ignore_sync = ignore_sync;
+    thr->in_symbolizer = in_symbolizer;
+  }
+  // We do not detect errno spoiling for SIGTERM,
+  // because some SIGTERM handlers do spoil errno but reraise SIGTERM,
+  // tsan reports false positive in such case.
+  // It's difficult to properly detect this situation (reraise),
+  // because in async signal processing case (when handler is called directly
+  // from rtl_generic_sighandler) we have not yet received the reraised
+  // signal; and it looks too fragile to intercept all ways to reraise a signal.
+  if (ShouldReport(thr, ReportTypeErrnoInSignal) && !sync && sig != SIGTERM &&
+      errno != 99)
+    ReportErrnoSpoiling(thr, pc);
+  errno = saved_errno;
+}
+
+void ProcessPendingSignalsImpl(ThreadState *thr) {
+  atomic_store(&thr->pending_signals, 0, memory_order_relaxed);
+  ThreadSignalContext *sctx = SigCtx(thr);
+  if (sctx == 0)
+    return;
+  atomic_fetch_add(&thr->in_signal_handler, 1, memory_order_relaxed);
+  internal_sigfillset(&sctx->emptyset);
+  int res = REAL(pthread_sigmask)(SIG_SETMASK, &sctx->emptyset, &sctx->oldset);
+  CHECK_EQ(res, 0);
+  for (int sig = 0; sig < kSigCount; sig++) {
+    SignalDesc *signal = &sctx->pending_signals[sig];
+    if (signal->armed) {
+      signal->armed = false;
+      CallUserSignalHandler(thr, false, true, sig, &signal->siginfo,
+                            &signal->ctx);
+    }
+  }
+  res = REAL(pthread_sigmask)(SIG_SETMASK, &sctx->oldset, 0);
+  CHECK_EQ(res, 0);
+  atomic_fetch_add(&thr->in_signal_handler, -1, memory_order_relaxed);
+}
+
+}  // namespace __tsan
+
+static bool is_sync_signal(ThreadSignalContext *sctx, int sig) {
+  return sig == SIGSEGV || sig == SIGBUS || sig == SIGILL || sig == SIGTRAP ||
+         sig == SIGABRT || sig == SIGFPE || sig == SIGPIPE || sig == SIGSYS ||
+         // If we are sending signal to ourselves, we must process it now.
+         (sctx && sig == sctx->int_signal_send);
+}
+
+void sighandler(int sig, __sanitizer_siginfo *info, void *ctx) {
+  ThreadState *thr = cur_thread_init();
+  ThreadSignalContext *sctx = SigCtx(thr);
+  if (sig < 0 || sig >= kSigCount) {
+    VPrintf(1, "ThreadSanitizer: ignoring signal %d\n", sig);
+    return;
+  }
+  // Don't mess with synchronous signals.
+  const bool sync = is_sync_signal(sctx, sig);
+  if (sync ||
+      // If we are in blocking function, we can safely process it now
+      // (but check if we are in a recursive interceptor,
+      // i.e. pthread_join()->munmap()).
+      (sctx && atomic_load(&sctx->in_blocking_func, memory_order_relaxed))) {
+    atomic_fetch_add(&thr->in_signal_handler, 1, memory_order_relaxed);
+    if (sctx && atomic_load(&sctx->in_blocking_func, memory_order_relaxed)) {
+      atomic_store(&sctx->in_blocking_func, 0, memory_order_relaxed);
+      CallUserSignalHandler(thr, sync, true, sig, info, ctx);
+      atomic_store(&sctx->in_blocking_func, 1, memory_order_relaxed);
+    } else {
+      // Be very conservative with when we do acquire in this case.
+      // It's unsafe to do acquire in async handlers, because ThreadState
+      // can be in inconsistent state.
+      // SIGSYS looks relatively safe -- it's synchronous and can actually
+      // need some global state.
+      bool acq = (sig == SIGSYS);
+      CallUserSignalHandler(thr, sync, acq, sig, info, ctx);
+    }
+    atomic_fetch_add(&thr->in_signal_handler, -1, memory_order_relaxed);
+    return;
+  }
+
+  if (sctx == 0)
+    return;
+  SignalDesc *signal = &sctx->pending_signals[sig];
+  if (signal->armed == false) {
+    signal->armed = true;
+    internal_memcpy(&signal->siginfo, info, sizeof(*info));
+    internal_memcpy(&signal->ctx, ctx, sizeof(signal->ctx));
+    atomic_store(&thr->pending_signals, 1, memory_order_relaxed);
+  }
+}
+
+TSAN_INTERCEPTOR(int, raise, int sig) {
+  SCOPED_TSAN_INTERCEPTOR(raise, sig);
+  ThreadSignalContext *sctx = SigCtx(thr);
+  CHECK_NE(sctx, 0);
+  int prev = sctx->int_signal_send;
+  sctx->int_signal_send = sig;
+  int res = REAL(raise)(sig);
+  CHECK_EQ(sctx->int_signal_send, sig);
+  sctx->int_signal_send = prev;
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, kill, int pid, int sig) {
+  SCOPED_TSAN_INTERCEPTOR(kill, pid, sig);
+  ThreadSignalContext *sctx = SigCtx(thr);
+  CHECK_NE(sctx, 0);
+  int prev = sctx->int_signal_send;
+  if (pid == (int)internal_getpid()) {
+    sctx->int_signal_send = sig;
+  }
+  int res = REAL(kill)(pid, sig);
+  if (pid == (int)internal_getpid()) {
+    CHECK_EQ(sctx->int_signal_send, sig);
+    sctx->int_signal_send = prev;
+  }
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, pthread_kill, void *tid, int sig) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_kill, tid, sig);
+  ThreadSignalContext *sctx = SigCtx(thr);
+  CHECK_NE(sctx, 0);
+  int prev = sctx->int_signal_send;
+  bool self = pthread_equal(tid, pthread_self());
+  if (self)
+    sctx->int_signal_send = sig;
+  int res = REAL(pthread_kill)(tid, sig);
+  if (self) {
+    CHECK_EQ(sctx->int_signal_send, sig);
+    sctx->int_signal_send = prev;
+  }
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, gettimeofday, void *tv, void *tz) {
+  SCOPED_TSAN_INTERCEPTOR(gettimeofday, tv, tz);
+  // It's intercepted merely to process pending signals.
+  return REAL(gettimeofday)(tv, tz);
+}
+
+TSAN_INTERCEPTOR(int, getaddrinfo, void *node, void *service,
+    void *hints, void *rv) {
+  SCOPED_TSAN_INTERCEPTOR(getaddrinfo, node, service, hints, rv);
+  // We miss atomic synchronization in getaddrinfo,
+  // and can report false race between malloc and free
+  // inside of getaddrinfo. So ignore memory accesses.
+  ThreadIgnoreBegin(thr, pc);
+  int res = REAL(getaddrinfo)(node, service, hints, rv);
+  ThreadIgnoreEnd(thr);
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, fork, int fake) {
+  if (in_symbolizer())
+    return REAL(fork)(fake);
+  SCOPED_INTERCEPTOR_RAW(fork, fake);
+  return REAL(fork)(fake);
+}
+
+void atfork_prepare() {
+  if (in_symbolizer())
+    return;
+  ThreadState *thr = cur_thread();
+  const uptr pc = StackTrace::GetCurrentPc();
+  ForkBefore(thr, pc);
+}
+
+void atfork_parent() {
+  if (in_symbolizer())
+    return;
+  ThreadState *thr = cur_thread();
+  const uptr pc = StackTrace::GetCurrentPc();
+  ForkParentAfter(thr, pc);
+}
+
+void atfork_child() {
+  if (in_symbolizer())
+    return;
+  ThreadState *thr = cur_thread();
+  const uptr pc = StackTrace::GetCurrentPc();
+  ForkChildAfter(thr, pc, true);
+  FdOnFork(thr, pc);
+}
+
+TSAN_INTERCEPTOR(int, vfork, int fake) {
+  // Some programs (e.g. openjdk) call close for all file descriptors
+  // in the child process. Under tsan it leads to false positives, because
+  // address space is shared, so the parent process also thinks that
+  // the descriptors are closed (while they are actually not).
+  // This leads to false positives due to missed synchronization.
+  // Strictly saying this is undefined behavior, because vfork child is not
+  // allowed to call any functions other than exec/exit. But this is what
+  // openjdk does, so we want to handle it.
+  // We could disable interceptors in the child process. But it's not possible
+  // to simply intercept and wrap vfork, because vfork child is not allowed
+  // to return from the function that calls vfork, and that's exactly what
+  // we would do. So this would require some assembly trickery as well.
+  // Instead we simply turn vfork into fork.
+  return WRAP(fork)(fake);
+}
+
+#if SANITIZER_LINUX
+TSAN_INTERCEPTOR(int, clone, int (*fn)(void *), void *stack, int flags,
+                 void *arg, int *parent_tid, void *tls, pid_t *child_tid) {
+  SCOPED_INTERCEPTOR_RAW(clone, fn, stack, flags, arg, parent_tid, tls,
+                         child_tid);
+  struct Arg {
+    int (*fn)(void *);
+    void *arg;
+  };
+  auto wrapper = +[](void *p) -> int {
+    auto *thr = cur_thread();
+    uptr pc = GET_CURRENT_PC();
+    // Start the background thread for fork, but not for clone.
+    // For fork we did this always and it's known to work (or user code has
+    // adopted). But if we do this for the new clone interceptor some code
+    // (sandbox2) fails. So model we used to do for years and don't start the
+    // background thread after clone.
+    ForkChildAfter(thr, pc, false);
+    FdOnFork(thr, pc);
+    auto *arg = static_cast<Arg *>(p);
+    return arg->fn(arg->arg);
+  };
+  ForkBefore(thr, pc);
+  Arg arg_wrapper = {fn, arg};
+  int pid = REAL(clone)(wrapper, stack, flags, &arg_wrapper, parent_tid, tls,
+                        child_tid);
+  ForkParentAfter(thr, pc);
+  return pid;
+}
+#endif
+
+#if !SANITIZER_MAC && !SANITIZER_ANDROID
+typedef int (*dl_iterate_phdr_cb_t)(__sanitizer_dl_phdr_info *info, SIZE_T size,
+                                    void *data);
+struct dl_iterate_phdr_data {
+  ThreadState *thr;
+  uptr pc;
+  dl_iterate_phdr_cb_t cb;
+  void *data;
+};
+
+static bool IsAppNotRodata(uptr addr) {
+  return IsAppMem(addr) && *MemToShadow(addr) != kShadowRodata;
+}
+
+static int dl_iterate_phdr_cb(__sanitizer_dl_phdr_info *info, SIZE_T size,
+                              void *data) {
+  dl_iterate_phdr_data *cbdata = (dl_iterate_phdr_data *)data;
+  // dlopen/dlclose allocate/free dynamic-linker-internal memory, which is later
+  // accessible in dl_iterate_phdr callback. But we don't see synchronization
+  // inside of dynamic linker, so we "unpoison" it here in order to not
+  // produce false reports. Ignoring malloc/free in dlopen/dlclose is not enough
+  // because some libc functions call __libc_dlopen.
+  if (info && IsAppNotRodata((uptr)info->dlpi_name))
+    MemoryResetRange(cbdata->thr, cbdata->pc, (uptr)info->dlpi_name,
+                     internal_strlen(info->dlpi_name));
+  int res = cbdata->cb(info, size, cbdata->data);
+  // Perform the check one more time in case info->dlpi_name was overwritten
+  // by user callback.
+  if (info && IsAppNotRodata((uptr)info->dlpi_name))
+    MemoryResetRange(cbdata->thr, cbdata->pc, (uptr)info->dlpi_name,
+                     internal_strlen(info->dlpi_name));
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, dl_iterate_phdr, dl_iterate_phdr_cb_t cb, void *data) {
+  SCOPED_TSAN_INTERCEPTOR(dl_iterate_phdr, cb, data);
+  dl_iterate_phdr_data cbdata;
+  cbdata.thr = thr;
+  cbdata.pc = pc;
+  cbdata.cb = cb;
+  cbdata.data = data;
+  int res = REAL(dl_iterate_phdr)(dl_iterate_phdr_cb, &cbdata);
+  return res;
+}
+#endif
+
+static int OnExit(ThreadState *thr) {
+  int status = Finalize(thr);
+  FlushStreams();
+  return status;
+}
+
+struct TsanInterceptorContext {
+  ThreadState *thr;
+  const uptr pc;
+};
+
+#if !SANITIZER_MAC
+static void HandleRecvmsg(ThreadState *thr, uptr pc,
+    __sanitizer_msghdr *msg) {
+  int fds[64];
+  int cnt = ExtractRecvmsgFDs(msg, fds, ARRAY_SIZE(fds));
+  for (int i = 0; i < cnt; i++)
+    FdEventCreate(thr, pc, fds[i]);
+}
+#endif
+
+#include "sanitizer_common/sanitizer_platform_interceptors.h"
+// Causes interceptor recursion (getaddrinfo() and fopen())
+#undef SANITIZER_INTERCEPT_GETADDRINFO
+// We define our own.
+#if SANITIZER_INTERCEPT_TLS_GET_ADDR
+#define NEED_TLS_GET_ADDR
+#endif
+#undef SANITIZER_INTERCEPT_TLS_GET_ADDR
+#define SANITIZER_INTERCEPT_TLS_GET_OFFSET 1
+#undef SANITIZER_INTERCEPT_PTHREAD_SIGMASK
+
+#define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name)
+#define COMMON_INTERCEPT_FUNCTION_VER(name, ver)                          \
+  INTERCEPT_FUNCTION_VER(name, ver)
+#define COMMON_INTERCEPT_FUNCTION_VER_UNVERSIONED_FALLBACK(name, ver) \
+  (INTERCEPT_FUNCTION_VER(name, ver) || INTERCEPT_FUNCTION(name))
+
+#define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size)                    \
+  MemoryAccessRange(((TsanInterceptorContext *)ctx)->thr,                 \
+                    ((TsanInterceptorContext *)ctx)->pc, (uptr)ptr, size, \
+                    true)
+
+#define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size)                       \
+  MemoryAccessRange(((TsanInterceptorContext *) ctx)->thr,                  \
+                    ((TsanInterceptorContext *) ctx)->pc, (uptr) ptr, size, \
+                    false)
+
+#define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \
+  SCOPED_TSAN_INTERCEPTOR(func, __VA_ARGS__);    \
+  TsanInterceptorContext _ctx = {thr, pc};       \
+  ctx = (void *)&_ctx;                           \
+  (void)ctx;
+
+#define COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, func, ...) \
+  SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__);              \
+  TsanInterceptorContext _ctx = {thr, pc};                \
+  ctx = (void *)&_ctx;                                    \
+  (void)ctx;
+
+#define COMMON_INTERCEPTOR_FILE_OPEN(ctx, file, path) \
+  if (path)                                           \
+    Acquire(thr, pc, File2addr(path));                \
+  if (file) {                                         \
+    int fd = fileno_unlocked(file);                   \
+    if (fd >= 0) FdFileCreate(thr, pc, fd);           \
+  }
+
+#define COMMON_INTERCEPTOR_FILE_CLOSE(ctx, file) \
+  if (file) {                                    \
+    int fd = fileno_unlocked(file);              \
+    if (fd >= 0) FdClose(thr, pc, fd);           \
+  }
+
+#define COMMON_INTERCEPTOR_DLOPEN(filename, flag) \
+  ({                                              \
+    CheckNoDeepBind(filename, flag);              \
+    ThreadIgnoreBegin(thr, 0);                    \
+    void *res = REAL(dlopen)(filename, flag);     \
+    ThreadIgnoreEnd(thr);                         \
+    res;                                          \
+  })
+
+#define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, handle) \
+  libignore()->OnLibraryLoaded(filename)
+
+#define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() \
+  libignore()->OnLibraryUnloaded()
+
+#define COMMON_INTERCEPTOR_ACQUIRE(ctx, u) \
+  Acquire(((TsanInterceptorContext *) ctx)->thr, pc, u)
+
+#define COMMON_INTERCEPTOR_RELEASE(ctx, u) \
+  Release(((TsanInterceptorContext *) ctx)->thr, pc, u)
+
+#define COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path) \
+  Acquire(((TsanInterceptorContext *) ctx)->thr, pc, Dir2addr(path))
+
+#define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \
+  FdAcquire(((TsanInterceptorContext *) ctx)->thr, pc, fd)
+
+#define COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd) \
+  FdRelease(((TsanInterceptorContext *) ctx)->thr, pc, fd)
+
+#define COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd) \
+  FdAccess(((TsanInterceptorContext *) ctx)->thr, pc, fd)
+
+#define COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, newfd) \
+  FdSocketAccept(((TsanInterceptorContext *) ctx)->thr, pc, fd, newfd)
+
+#define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) \
+  ThreadSetName(((TsanInterceptorContext *) ctx)->thr, name)
+
+#define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name)         \
+  if (pthread_equal(pthread_self(), reinterpret_cast<void *>(thread))) \
+    COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name);                     \
+  else                                                                 \
+    __tsan::ctx->thread_registry.SetThreadNameByUserId(thread, name)
+
+#define COMMON_INTERCEPTOR_BLOCK_REAL(name) BLOCK_REAL(name)
+
+#define COMMON_INTERCEPTOR_ON_EXIT(ctx) \
+  OnExit(((TsanInterceptorContext *) ctx)->thr)
+
+#define COMMON_INTERCEPTOR_MUTEX_PRE_LOCK(ctx, m) \
+  MutexPreLock(((TsanInterceptorContext *)ctx)->thr, \
+            ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
+
+#define COMMON_INTERCEPTOR_MUTEX_POST_LOCK(ctx, m) \
+  MutexPostLock(((TsanInterceptorContext *)ctx)->thr, \
+            ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
+
+#define COMMON_INTERCEPTOR_MUTEX_UNLOCK(ctx, m) \
+  MutexUnlock(((TsanInterceptorContext *)ctx)->thr, \
+            ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
+
+#define COMMON_INTERCEPTOR_MUTEX_REPAIR(ctx, m) \
+  MutexRepair(((TsanInterceptorContext *)ctx)->thr, \
+            ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
+
+#define COMMON_INTERCEPTOR_MUTEX_INVALID(ctx, m) \
+  MutexInvalidAccess(((TsanInterceptorContext *)ctx)->thr, \
+                     ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
+
+#define COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, sz, prot, flags, fd,  \
+                                     off)                                   \
+  do {                                                                      \
+    return mmap_interceptor(thr, pc, REAL(mmap), addr, sz, prot, flags, fd, \
+                            off);                                           \
+  } while (false)
+
+#if !SANITIZER_MAC
+#define COMMON_INTERCEPTOR_HANDLE_RECVMSG(ctx, msg) \
+  HandleRecvmsg(((TsanInterceptorContext *)ctx)->thr, \
+      ((TsanInterceptorContext *)ctx)->pc, msg)
+#endif
+
+#define COMMON_INTERCEPTOR_GET_TLS_RANGE(begin, end)                           \
+  if (TsanThread *t = GetCurrentThread()) {                                    \
+    *begin = t->tls_begin();                                                   \
+    *end = t->tls_end();                                                       \
+  } else {                                                                     \
+    *begin = *end = 0;                                                         \
+  }
+
+#define COMMON_INTERCEPTOR_USER_CALLBACK_START() \
+  SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START()
+
+#define COMMON_INTERCEPTOR_USER_CALLBACK_END() \
+  SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END()
+
+#include "sanitizer_common/sanitizer_common_interceptors.inc"
+
+static int sigaction_impl(int sig, const __sanitizer_sigaction *act,
+                          __sanitizer_sigaction *old);
+static __sanitizer_sighandler_ptr signal_impl(int sig,
+                                              __sanitizer_sighandler_ptr h);
+
+#define SIGNAL_INTERCEPTOR_SIGACTION_IMPL(signo, act, oldact) \
+  { return sigaction_impl(signo, act, oldact); }
+
+#define SIGNAL_INTERCEPTOR_SIGNAL_IMPL(func, signo, handler) \
+  { return (uptr)signal_impl(signo, (__sanitizer_sighandler_ptr)handler); }
+
+#include "sanitizer_common/sanitizer_signal_interceptors.inc"
+
+int sigaction_impl(int sig, const __sanitizer_sigaction *act,
+                   __sanitizer_sigaction *old) {
+  // Note: if we call REAL(sigaction) directly for any reason without proxying
+  // the signal handler through sighandler, very bad things will happen.
+  // The handler will run synchronously and corrupt tsan per-thread state.
+  SCOPED_INTERCEPTOR_RAW(sigaction, sig, act, old);
+  if (sig <= 0 || sig >= kSigCount) {
+    errno = errno_EINVAL;
+    return -1;
+  }
+  __sanitizer_sigaction *sigactions = interceptor_ctx()->sigactions;
+  __sanitizer_sigaction old_stored;
+  if (old) internal_memcpy(&old_stored, &sigactions[sig], sizeof(old_stored));
+  __sanitizer_sigaction newact;
+  if (act) {
+    // Copy act into sigactions[sig].
+    // Can't use struct copy, because compiler can emit call to memcpy.
+    // Can't use internal_memcpy, because it copies byte-by-byte,
+    // and signal handler reads the handler concurrently. It it can read
+    // some bytes from old value and some bytes from new value.
+    // Use volatile to prevent insertion of memcpy.
+    sigactions[sig].handler =
+        *(volatile __sanitizer_sighandler_ptr const *)&act->handler;
+    sigactions[sig].sa_flags = *(volatile int const *)&act->sa_flags;
+    internal_memcpy(&sigactions[sig].sa_mask, &act->sa_mask,
+                    sizeof(sigactions[sig].sa_mask));
+#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD
+    sigactions[sig].sa_restorer = act->sa_restorer;
+#endif
+    internal_memcpy(&newact, act, sizeof(newact));
+    internal_sigfillset(&newact.sa_mask);
+    if ((act->sa_flags & SA_SIGINFO) ||
+        ((uptr)act->handler != sig_ign && (uptr)act->handler != sig_dfl)) {
+      newact.sa_flags |= SA_SIGINFO;
+      newact.sigaction = sighandler;
+    }
+    ReleaseStore(thr, pc, (uptr)&sigactions[sig]);
+    act = &newact;
+  }
+  int res = REAL(sigaction)(sig, act, old);
+  if (res == 0 && old && old->sigaction == sighandler)
+    internal_memcpy(old, &old_stored, sizeof(*old));
+  return res;
+}
+
+static __sanitizer_sighandler_ptr signal_impl(int sig,
+                                              __sanitizer_sighandler_ptr h) {
+  __sanitizer_sigaction act;
+  act.handler = h;
+  internal_memset(&act.sa_mask, -1, sizeof(act.sa_mask));
+  act.sa_flags = 0;
+  __sanitizer_sigaction old;
+  int res = sigaction_symname(sig, &act, &old);
+  if (res) return (__sanitizer_sighandler_ptr)sig_err;
+  return old.handler;
+}
+
+#define TSAN_SYSCALL()             \
+  ThreadState *thr = cur_thread(); \
+  if (thr->ignore_interceptors)    \
+    return;                        \
+  ScopedSyscall scoped_syscall(thr)
+
+struct ScopedSyscall {
+  ThreadState *thr;
+
+  explicit ScopedSyscall(ThreadState *thr) : thr(thr) { LazyInitialize(thr); }
+
+  ~ScopedSyscall() {
+    ProcessPendingSignals(thr);
+  }
+};
+
+#if !SANITIZER_FREEBSD && !SANITIZER_MAC
+static void syscall_access_range(uptr pc, uptr p, uptr s, bool write) {
+  TSAN_SYSCALL();
+  MemoryAccessRange(thr, pc, p, s, write);
+}
+
+static USED void syscall_acquire(uptr pc, uptr addr) {
+  TSAN_SYSCALL();
+  Acquire(thr, pc, addr);
+  DPrintf("syscall_acquire(0x%zx))\n", addr);
+}
+
+static USED void syscall_release(uptr pc, uptr addr) {
+  TSAN_SYSCALL();
+  DPrintf("syscall_release(0x%zx)\n", addr);
+  Release(thr, pc, addr);
+}
+
+static void syscall_fd_close(uptr pc, int fd) {
+  TSAN_SYSCALL();
+  FdClose(thr, pc, fd);
+}
+
+static USED void syscall_fd_acquire(uptr pc, int fd) {
+  TSAN_SYSCALL();
+  FdAcquire(thr, pc, fd);
+  DPrintf("syscall_fd_acquire(%d)\n", fd);
+}
+
+static USED void syscall_fd_release(uptr pc, int fd) {
+  TSAN_SYSCALL();
+  DPrintf("syscall_fd_release(%d)\n", fd);
+  FdRelease(thr, pc, fd);
+}
+
+static void syscall_pre_fork(uptr pc) { ForkBefore(cur_thread(), pc); }
+
+static void syscall_post_fork(uptr pc, int pid) {
+  ThreadState *thr = cur_thread();
+  if (pid == 0) {
+    // child
+    ForkChildAfter(thr, pc, true);
+    FdOnFork(thr, pc);
+  } else if (pid > 0) {
+    // parent
+    ForkParentAfter(thr, pc);
+  } else {
+    // error
+    ForkParentAfter(thr, pc);
+  }
+}
+#endif
+
+#define COMMON_SYSCALL_PRE_READ_RANGE(p, s) \
+  syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), false)
+
+#define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) \
+  syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), true)
+
+#define COMMON_SYSCALL_POST_READ_RANGE(p, s) \
+  do {                                       \
+    (void)(p);                               \
+    (void)(s);                               \
+  } while (false)
+
+#define COMMON_SYSCALL_POST_WRITE_RANGE(p, s) \
+  do {                                        \
+    (void)(p);                                \
+    (void)(s);                                \
+  } while (false)
+
+#define COMMON_SYSCALL_ACQUIRE(addr) \
+    syscall_acquire(GET_CALLER_PC(), (uptr)(addr))
+
+#define COMMON_SYSCALL_RELEASE(addr) \
+    syscall_release(GET_CALLER_PC(), (uptr)(addr))
+
+#define COMMON_SYSCALL_FD_CLOSE(fd) syscall_fd_close(GET_CALLER_PC(), fd)
+
+#define COMMON_SYSCALL_FD_ACQUIRE(fd) syscall_fd_acquire(GET_CALLER_PC(), fd)
+
+#define COMMON_SYSCALL_FD_RELEASE(fd) syscall_fd_release(GET_CALLER_PC(), fd)
+
+#define COMMON_SYSCALL_PRE_FORK() \
+  syscall_pre_fork(GET_CALLER_PC())
+
+#define COMMON_SYSCALL_POST_FORK(res) \
+  syscall_post_fork(GET_CALLER_PC(), res)
+
+#include "sanitizer_common/sanitizer_common_syscalls.inc"
+#include "sanitizer_common/sanitizer_syscalls_netbsd.inc"
+
+#ifdef NEED_TLS_GET_ADDR
+
+static void handle_tls_addr(void *arg, void *res) {
+  ThreadState *thr = cur_thread();
+  if (!thr)
+    return;
+  DTLS::DTV *dtv = DTLS_on_tls_get_addr(arg, res, thr->tls_addr,
+                                        thr->tls_addr + thr->tls_size);
+  if (!dtv)
+    return;
+  // New DTLS block has been allocated.
+  MemoryResetRange(thr, 0, dtv->beg, dtv->size);
+}
+
+#if !SANITIZER_S390
+// Define own interceptor instead of sanitizer_common's for three reasons:
+// 1. It must not process pending signals.
+//    Signal handlers may contain MOVDQA instruction (see below).
+// 2. It must be as simple as possible to not contain MOVDQA.
+// 3. Sanitizer_common version uses COMMON_INTERCEPTOR_INITIALIZE_RANGE which
+//    is empty for tsan (meant only for msan).
+// Note: __tls_get_addr can be called with mis-aligned stack due to:
+// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58066
+// So the interceptor must work with mis-aligned stack, in particular, does not
+// execute MOVDQA with stack addresses.
+TSAN_INTERCEPTOR(void *, __tls_get_addr, void *arg) {
+  void *res = REAL(__tls_get_addr)(arg);
+  handle_tls_addr(arg, res);
+  return res;
+}
+#else // SANITIZER_S390
+TSAN_INTERCEPTOR(uptr, __tls_get_addr_internal, void *arg) {
+  uptr res = __tls_get_offset_wrapper(arg, REAL(__tls_get_offset));
+  char *tp = static_cast<char *>(__builtin_thread_pointer());
+  handle_tls_addr(arg, res + tp);
+  return res;
+}
+#endif
+#endif
+
+#if SANITIZER_NETBSD
+TSAN_INTERCEPTOR(void, _lwp_exit) {
+  SCOPED_TSAN_INTERCEPTOR(_lwp_exit);
+  DestroyThreadState();
+  REAL(_lwp_exit)();
+}
+#define TSAN_MAYBE_INTERCEPT__LWP_EXIT TSAN_INTERCEPT(_lwp_exit)
+#else
+#define TSAN_MAYBE_INTERCEPT__LWP_EXIT
+#endif
+
+#if SANITIZER_FREEBSD
+TSAN_INTERCEPTOR(void, thr_exit, tid_t *state) {
+  SCOPED_TSAN_INTERCEPTOR(thr_exit, state);
+  DestroyThreadState();
+  REAL(thr_exit(state));
+}
+#define TSAN_MAYBE_INTERCEPT_THR_EXIT TSAN_INTERCEPT(thr_exit)
+#else
+#define TSAN_MAYBE_INTERCEPT_THR_EXIT
+#endif
+
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_init, void *c, void *a)
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_signal, void *c)
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_broadcast, void *c)
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_wait, void *c, void *m)
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_destroy, void *c)
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_init, void *m, void *a)
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_destroy, void *m)
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_trylock, void *m)
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_init, void *m, void *a)
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_destroy, void *m)
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_rdlock, void *m)
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_tryrdlock, void *m)
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_wrlock, void *m)
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_trywrlock, void *m)
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_unlock, void *m)
+TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(int, once, void *o, void (*f)())
+TSAN_INTERCEPTOR_NETBSD_ALIAS_THR2(int, sigsetmask, sigmask, int a, void *b,
+  void *c)
+
+namespace __tsan {
+
+static void finalize(void *arg) {
+  ThreadState *thr = cur_thread();
+  int status = Finalize(thr);
+  // Make sure the output is not lost.
+  FlushStreams();
+  if (status)
+    Die();
+}
+
+#if !SANITIZER_MAC && !SANITIZER_ANDROID
+static void unreachable() {
+  Report("FATAL: ThreadSanitizer: unreachable called\n");
+  Die();
+}
+#endif
+
+// Define default implementation since interception of libdispatch  is optional.
+SANITIZER_WEAK_ATTRIBUTE void InitializeLibdispatchInterceptors() {}
+
+void InitializeInterceptors() {
+#if !SANITIZER_MAC
+  // We need to setup it early, because functions like dlsym() can call it.
+  REAL(memset) = internal_memset;
+  REAL(memcpy) = internal_memcpy;
+#endif
+
+  new(interceptor_ctx()) InterceptorContext();
+
+  InitializeCommonInterceptors();
+  InitializeSignalInterceptors();
+  InitializeLibdispatchInterceptors();
+
+#if !SANITIZER_MAC
+  // We can not use TSAN_INTERCEPT to get setjmp addr,
+  // because it does &setjmp and setjmp is not present in some versions of libc.
+  using __interception::InterceptFunction;
+  InterceptFunction(TSAN_STRING_SETJMP, (uptr*)&REAL(setjmp_symname), 0, 0);
+  InterceptFunction("_setjmp", (uptr*)&REAL(_setjmp), 0, 0);
+  InterceptFunction(TSAN_STRING_SIGSETJMP, (uptr*)&REAL(sigsetjmp_symname), 0,
+                    0);
+#if !SANITIZER_NETBSD
+  InterceptFunction("__sigsetjmp", (uptr*)&REAL(__sigsetjmp), 0, 0);
+#endif
+#endif
+
+  TSAN_INTERCEPT(longjmp_symname);
+  TSAN_INTERCEPT(siglongjmp_symname);
+#if SANITIZER_NETBSD
+  TSAN_INTERCEPT(_longjmp);
+#endif
+
+  TSAN_INTERCEPT(malloc);
+  TSAN_INTERCEPT(__libc_memalign);
+  TSAN_INTERCEPT(calloc);
+  TSAN_INTERCEPT(realloc);
+  TSAN_INTERCEPT(reallocarray);
+  TSAN_INTERCEPT(free);
+  TSAN_INTERCEPT(cfree);
+  TSAN_INTERCEPT(munmap);
+  TSAN_MAYBE_INTERCEPT_MEMALIGN;
+  TSAN_INTERCEPT(valloc);
+  TSAN_MAYBE_INTERCEPT_PVALLOC;
+  TSAN_INTERCEPT(posix_memalign);
+
+  TSAN_INTERCEPT(strcpy);
+  TSAN_INTERCEPT(strncpy);
+  TSAN_INTERCEPT(strdup);
+
+  TSAN_INTERCEPT(pthread_create);
+  TSAN_INTERCEPT(pthread_join);
+  TSAN_INTERCEPT(pthread_detach);
+  TSAN_INTERCEPT(pthread_exit);
+  #if SANITIZER_LINUX
+  TSAN_INTERCEPT(pthread_tryjoin_np);
+  TSAN_INTERCEPT(pthread_timedjoin_np);
+  #endif
+
+  TSAN_INTERCEPT_VER(pthread_cond_init, PTHREAD_ABI_BASE);
+  TSAN_INTERCEPT_VER(pthread_cond_signal, PTHREAD_ABI_BASE);
+  TSAN_INTERCEPT_VER(pthread_cond_broadcast, PTHREAD_ABI_BASE);
+  TSAN_INTERCEPT_VER(pthread_cond_wait, PTHREAD_ABI_BASE);
+  TSAN_INTERCEPT_VER(pthread_cond_timedwait, PTHREAD_ABI_BASE);
+  TSAN_INTERCEPT_VER(pthread_cond_destroy, PTHREAD_ABI_BASE);
+
+  TSAN_MAYBE_PTHREAD_COND_CLOCKWAIT;
+
+  TSAN_INTERCEPT(pthread_mutex_init);
+  TSAN_INTERCEPT(pthread_mutex_destroy);
+  TSAN_INTERCEPT(pthread_mutex_trylock);
+  TSAN_INTERCEPT(pthread_mutex_timedlock);
+
+  TSAN_INTERCEPT(pthread_spin_init);
+  TSAN_INTERCEPT(pthread_spin_destroy);
+  TSAN_INTERCEPT(pthread_spin_lock);
+  TSAN_INTERCEPT(pthread_spin_trylock);
+  TSAN_INTERCEPT(pthread_spin_unlock);
+
+  TSAN_INTERCEPT(pthread_rwlock_init);
+  TSAN_INTERCEPT(pthread_rwlock_destroy);
+  TSAN_INTERCEPT(pthread_rwlock_rdlock);
+  TSAN_INTERCEPT(pthread_rwlock_tryrdlock);
+  TSAN_INTERCEPT(pthread_rwlock_timedrdlock);
+  TSAN_INTERCEPT(pthread_rwlock_wrlock);
+  TSAN_INTERCEPT(pthread_rwlock_trywrlock);
+  TSAN_INTERCEPT(pthread_rwlock_timedwrlock);
+  TSAN_INTERCEPT(pthread_rwlock_unlock);
+
+  TSAN_INTERCEPT(pthread_barrier_init);
+  TSAN_INTERCEPT(pthread_barrier_destroy);
+  TSAN_INTERCEPT(pthread_barrier_wait);
+
+  TSAN_INTERCEPT(pthread_once);
+
+  TSAN_INTERCEPT(fstat);
+  TSAN_MAYBE_INTERCEPT___FXSTAT;
+  TSAN_MAYBE_INTERCEPT_FSTAT64;
+  TSAN_MAYBE_INTERCEPT___FXSTAT64;
+  TSAN_INTERCEPT(open);
+  TSAN_MAYBE_INTERCEPT_OPEN64;
+  TSAN_INTERCEPT(creat);
+  TSAN_MAYBE_INTERCEPT_CREAT64;
+  TSAN_INTERCEPT(dup);
+  TSAN_INTERCEPT(dup2);
+  TSAN_INTERCEPT(dup3);
+  TSAN_MAYBE_INTERCEPT_EVENTFD;
+  TSAN_MAYBE_INTERCEPT_SIGNALFD;
+  TSAN_MAYBE_INTERCEPT_INOTIFY_INIT;
+  TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1;
+  TSAN_INTERCEPT(socket);
+  TSAN_INTERCEPT(socketpair);
+  TSAN_INTERCEPT(connect);
+  TSAN_INTERCEPT(bind);
+  TSAN_INTERCEPT(listen);
+  TSAN_MAYBE_INTERCEPT_EPOLL;
+  TSAN_INTERCEPT(close);
+  TSAN_MAYBE_INTERCEPT___CLOSE;
+  TSAN_MAYBE_INTERCEPT___RES_ICLOSE;
+  TSAN_INTERCEPT(pipe);
+  TSAN_INTERCEPT(pipe2);
+
+  TSAN_INTERCEPT(unlink);
+  TSAN_INTERCEPT(tmpfile);
+  TSAN_MAYBE_INTERCEPT_TMPFILE64;
+  TSAN_INTERCEPT(abort);
+  TSAN_INTERCEPT(rmdir);
+  TSAN_INTERCEPT(closedir);
+
+  TSAN_INTERCEPT(sigsuspend);
+  TSAN_INTERCEPT(sigblock);
+  TSAN_INTERCEPT(sigsetmask);
+  TSAN_INTERCEPT(pthread_sigmask);
+  TSAN_INTERCEPT(raise);
+  TSAN_INTERCEPT(kill);
+  TSAN_INTERCEPT(pthread_kill);
+  TSAN_INTERCEPT(sleep);
+  TSAN_INTERCEPT(usleep);
+  TSAN_INTERCEPT(nanosleep);
+  TSAN_INTERCEPT(pause);
+  TSAN_INTERCEPT(gettimeofday);
+  TSAN_INTERCEPT(getaddrinfo);
+
+  TSAN_INTERCEPT(fork);
+  TSAN_INTERCEPT(vfork);
+#if SANITIZER_LINUX
+  TSAN_INTERCEPT(clone);
+#endif
+#if !SANITIZER_ANDROID
+  TSAN_INTERCEPT(dl_iterate_phdr);
+#endif
+  TSAN_MAYBE_INTERCEPT_ON_EXIT;
+  TSAN_INTERCEPT(__cxa_atexit);
+  TSAN_INTERCEPT(_exit);
+
+#ifdef NEED_TLS_GET_ADDR
+#if !SANITIZER_S390
+  TSAN_INTERCEPT(__tls_get_addr);
+#else
+  TSAN_INTERCEPT(__tls_get_addr_internal);
+  TSAN_INTERCEPT(__tls_get_offset);
+#endif
+#endif
+
+  TSAN_MAYBE_INTERCEPT__LWP_EXIT;
+  TSAN_MAYBE_INTERCEPT_THR_EXIT;
+
+#if !SANITIZER_MAC && !SANITIZER_ANDROID
+  // Need to setup it, because interceptors check that the function is resolved.
+  // But atexit is emitted directly into the module, so can't be resolved.
+  REAL(atexit) = (int(*)(void(*)()))unreachable;
+#endif
+
+  if (REAL(__cxa_atexit)(&finalize, 0, 0)) {
+    Printf("ThreadSanitizer: failed to setup atexit callback\n");
+    Die();
+  }
+  if (pthread_atfork(atfork_prepare, atfork_parent, atfork_child)) {
+    Printf("ThreadSanitizer: failed to setup atfork callbacks\n");
+    Die();
+  }
+
+#if !SANITIZER_MAC && !SANITIZER_NETBSD && !SANITIZER_FREEBSD
+  if (pthread_key_create(&interceptor_ctx()->finalize_key, &thread_finalize)) {
+    Printf("ThreadSanitizer: failed to create thread key\n");
+    Die();
+  }
+#endif
+
+  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_init);
+  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_signal);
+  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_broadcast);
+  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_wait);
+  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_destroy);
+  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_init);
+  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_destroy);
+  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_trylock);
+  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_init);
+  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_destroy);
+  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_rdlock);
+  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_tryrdlock);
+  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_wrlock);
+  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_trywrlock);
+  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_unlock);
+  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(once);
+  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(sigsetmask);
+
+  FdInit();
+}
+
+}  // namespace __tsan
+
+// Invisible barrier for tests.
+// There were several unsuccessful iterations for this functionality:
+// 1. Initially it was implemented in user code using
+//    REAL(pthread_barrier_wait). But pthread_barrier_wait is not supported on
+//    MacOS. Futexes are linux-specific for this matter.
+// 2. Then we switched to atomics+usleep(10). But usleep produced parasitic
+//    "as-if synchronized via sleep" messages in reports which failed some
+//    output tests.
+// 3. Then we switched to atomics+sched_yield. But this produced tons of tsan-
+//    visible events, which lead to "failed to restore stack trace" failures.
+// Note that no_sanitize_thread attribute does not turn off atomic interception
+// so attaching it to the function defined in user code does not help.
+// That's why we now have what we have.
+constexpr u32 kBarrierThreadBits = 10;
+constexpr u32 kBarrierThreads = 1 << kBarrierThreadBits;
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __tsan_testonly_barrier_init(
+    atomic_uint32_t *barrier, u32 num_threads) {
+  if (num_threads >= kBarrierThreads) {
+    Printf("barrier_init: count is too large (%d)\n", num_threads);
+    Die();
+  }
+  // kBarrierThreadBits lsb is thread count,
+  // the remaining are count of entered threads.
+  atomic_store(barrier, num_threads, memory_order_relaxed);
+}
+
+static u32 barrier_epoch(u32 value) {
+  return (value >> kBarrierThreadBits) / (value & (kBarrierThreads - 1));
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __tsan_testonly_barrier_wait(
+    atomic_uint32_t *barrier) {
+  u32 old = atomic_fetch_add(barrier, kBarrierThreads, memory_order_relaxed);
+  u32 old_epoch = barrier_epoch(old);
+  if (barrier_epoch(old + kBarrierThreads) != old_epoch) {
+    FutexWake(barrier, (1 << 30));
+    return;
+  }
+  for (;;) {
+    u32 cur = atomic_load(barrier, memory_order_relaxed);
+    if (barrier_epoch(cur) != old_epoch)
+      return;
+    FutexWait(barrier, cur);
+  }
+}
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_interface.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_interface.cpp
new file mode 100644
index 000000000000..048715185151
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_interface.cpp
@@ -0,0 +1,106 @@
+//===-- tsan_interface.cpp ------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+
+#include "tsan_interface.h"
+#include "tsan_interface_ann.h"
+#include "tsan_rtl.h"
+#include "sanitizer_common/sanitizer_internal_defs.h"
+#include "sanitizer_common/sanitizer_ptrauth.h"
+
+#define CALLERPC ((uptr)__builtin_return_address(0))
+
+using namespace __tsan;
+
+void __tsan_init() { Initialize(cur_thread_init()); }
+
+void __tsan_flush_memory() {
+  FlushShadowMemory();
+}
+
+void __tsan_read16(void *addr) {
+  uptr pc = CALLERPC;
+  ThreadState *thr = cur_thread();
+  MemoryAccess(thr, pc, (uptr)addr, 8, kAccessRead);
+  MemoryAccess(thr, pc, (uptr)addr + 8, 8, kAccessRead);
+}
+
+void __tsan_write16(void *addr) {
+  uptr pc = CALLERPC;
+  ThreadState *thr = cur_thread();
+  MemoryAccess(thr, pc, (uptr)addr, 8, kAccessWrite);
+  MemoryAccess(thr, pc, (uptr)addr + 8, 8, kAccessWrite);
+}
+
+void __tsan_read16_pc(void *addr, void *pc) {
+  uptr pc_no_pac = STRIP_PAC_PC(pc);
+  ThreadState *thr = cur_thread();
+  MemoryAccess(thr, pc_no_pac, (uptr)addr, 8, kAccessRead);
+  MemoryAccess(thr, pc_no_pac, (uptr)addr + 8, 8, kAccessRead);
+}
+
+void __tsan_write16_pc(void *addr, void *pc) {
+  uptr pc_no_pac = STRIP_PAC_PC(pc);
+  ThreadState *thr = cur_thread();
+  MemoryAccess(thr, pc_no_pac, (uptr)addr, 8, kAccessWrite);
+  MemoryAccess(thr, pc_no_pac, (uptr)addr + 8, 8, kAccessWrite);
+}
+
+// __tsan_unaligned_read/write calls are emitted by compiler.
+
+void __tsan_unaligned_read16(const void *addr) {
+  uptr pc = CALLERPC;
+  ThreadState *thr = cur_thread();
+  UnalignedMemoryAccess(thr, pc, (uptr)addr, 8, kAccessRead);
+  UnalignedMemoryAccess(thr, pc, (uptr)addr + 8, 8, kAccessRead);
+}
+
+void __tsan_unaligned_write16(void *addr) {
+  uptr pc = CALLERPC;
+  ThreadState *thr = cur_thread();
+  UnalignedMemoryAccess(thr, pc, (uptr)addr, 8, kAccessWrite);
+  UnalignedMemoryAccess(thr, pc, (uptr)addr + 8, 8, kAccessWrite);
+}
+
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__tsan_get_current_fiber() {
+  return cur_thread();
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__tsan_create_fiber(unsigned flags) {
+  return FiberCreate(cur_thread(), CALLERPC, flags);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_destroy_fiber(void *fiber) {
+  FiberDestroy(cur_thread(), CALLERPC, static_cast<ThreadState *>(fiber));
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_switch_to_fiber(void *fiber, unsigned flags) {
+  FiberSwitch(cur_thread(), CALLERPC, static_cast<ThreadState *>(fiber), flags);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_set_fiber_name(void *fiber, const char *name) {
+  ThreadSetName(static_cast<ThreadState *>(fiber), name);
+}
+}  // extern "C"
+
+void __tsan_acquire(void *addr) {
+  Acquire(cur_thread(), CALLERPC, (uptr)addr);
+}
+
+void __tsan_release(void *addr) {
+  Release(cur_thread(), CALLERPC, (uptr)addr);
+}
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_interface.h b/compiler-rt/lib/tsan/rtl-old/tsan_interface.h
new file mode 100644
index 000000000000..711f064174c2
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_interface.h
@@ -0,0 +1,424 @@
+//===-- tsan_interface.h ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// The functions declared in this header will be inserted by the instrumentation
+// module.
+// This header can be included by the instrumented program or by TSan tests.
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_INTERFACE_H
+#define TSAN_INTERFACE_H
+
+#include <sanitizer_common/sanitizer_internal_defs.h>
+using __sanitizer::uptr;
+using __sanitizer::tid_t;
+
+// This header should NOT include any other headers.
+// All functions in this header are extern "C" and start with __tsan_.
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#if !SANITIZER_GO
+
+// This function should be called at the very beginning of the process,
+// before any instrumented code is executed and before any call to malloc.
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_init();
+
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_flush_memory();
+
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read1(void *addr);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read2(void *addr);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read4(void *addr);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read8(void *addr);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read16(void *addr);
+
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write1(void *addr);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write2(void *addr);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write4(void *addr);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write8(void *addr);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write16(void *addr);
+
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_read2(const void *addr);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_read4(const void *addr);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_read8(const void *addr);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_read16(const void *addr);
+
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write2(void *addr);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write4(void *addr);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write8(void *addr);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write16(void *addr);
+
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read1_pc(void *addr, void *pc);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read2_pc(void *addr, void *pc);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read4_pc(void *addr, void *pc);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read8_pc(void *addr, void *pc);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read16_pc(void *addr, void *pc);
+
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write1_pc(void *addr, void *pc);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write2_pc(void *addr, void *pc);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write4_pc(void *addr, void *pc);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write8_pc(void *addr, void *pc);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write16_pc(void *addr, void *pc);
+
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_vptr_read(void **vptr_p);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_vptr_update(void **vptr_p, void *new_val);
+
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_func_entry(void *call_pc);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_func_exit();
+
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_ignore_thread_begin();
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_ignore_thread_end();
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__tsan_external_register_tag(const char *object_type);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_external_register_header(void *tag, const char *header);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_external_assign_tag(void *addr, void *tag);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_external_read(void *addr, void *caller_pc, void *tag);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_external_write(void *addr, void *caller_pc, void *tag);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_read_range(void *addr, unsigned long size);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_write_range(void *addr, unsigned long size);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_read_range_pc(void *addr, unsigned long size, void *pc);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_write_range_pc(void *addr, unsigned long size, void *pc);
+
+// User may provide function that would be called right when TSan detects
+// an error. The argument 'report' is an opaque pointer that can be used to
+// gather additional information using other TSan report API functions.
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_on_report(void *report);
+
+// If TSan is currently reporting a detected issue on the current thread,
+// returns an opaque pointer to the current report. Otherwise returns NULL.
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__tsan_get_current_report();
+
+// Returns a report's description (issue type), number of duplicate issues
+// found, counts of array data (stack traces, memory operations, locations,
+// mutexes, threads, unique thread IDs) and a stack trace of a sleep() call (if
+// one was involved in the issue).
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_report_data(void *report, const char **description, int *count,
+                           int *stack_count, int *mop_count, int *loc_count,
+                           int *mutex_count, int *thread_count,
+                           int *unique_tid_count, void **sleep_trace,
+                           uptr trace_size);
+
+/// Retrieves the "tag" from a report (for external-race report types). External
+/// races can be associated with a tag which give them more meaning. For example
+/// tag value '1' means "Swift access race". Tag value '0' indicated a plain
+/// external race.
+///
+/// \param report opaque pointer to the current report (obtained as argument in
+///               __tsan_on_report, or from __tsan_get_current_report)
+/// \param [out] tag points to storage that will be filled with the tag value
+///
+/// \returns non-zero value on success, zero on failure
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_report_tag(void *report, uptr *tag);
+
+// Returns information about stack traces included in the report.
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_report_stack(void *report, uptr idx, void **trace,
+                            uptr trace_size);
+
+// Returns information about memory operations included in the report.
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_report_mop(void *report, uptr idx, int *tid, void **addr,
+                          int *size, int *write, int *atomic, void **trace,
+                          uptr trace_size);
+
+// Returns information about locations included in the report.
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_report_loc(void *report, uptr idx, const char **type,
+                          void **addr, uptr *start, uptr *size, int *tid,
+                          int *fd, int *suppressable, void **trace,
+                          uptr trace_size);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_report_loc_object_type(void *report, uptr idx,
+                                      const char **object_type);
+
+// Returns information about mutexes included in the report.
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_report_mutex(void *report, uptr idx, uptr *mutex_id, void **addr,
+                            int *destroyed, void **trace, uptr trace_size);
+
+// Returns information about threads included in the report.
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_report_thread(void *report, uptr idx, int *tid, tid_t *os_id,
+                             int *running, const char **name, int *parent_tid,
+                             void **trace, uptr trace_size);
+
+// Returns information about unique thread IDs included in the report.
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_report_unique_tid(void *report, uptr idx, int *tid);
+
+// Returns the type of the pointer (heap, stack, global, ...) and if possible
+// also the starting address (e.g. of a heap allocation) and size.
+SANITIZER_INTERFACE_ATTRIBUTE
+const char *__tsan_locate_address(uptr addr, char *name, uptr name_size,
+                                  uptr *region_address, uptr *region_size);
+
+// Returns the allocation stack for a heap pointer.
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_alloc_stack(uptr addr, uptr *trace, uptr size, int *thread_id,
+                           tid_t *os_id);
+
+#endif  // SANITIZER_GO
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+namespace __tsan {
+
+// These should match declarations from public tsan_interface_atomic.h header.
+typedef unsigned char      a8;
+typedef unsigned short a16;
+typedef unsigned int       a32;
+typedef unsigned long long a64;
+#if !SANITIZER_GO && (defined(__SIZEOF_INT128__) \
+    || (__clang_major__ * 100 + __clang_minor__ >= 302)) && \
+    !defined(__mips64) && !defined(__s390x__)
+__extension__ typedef __int128 a128;
+# define __TSAN_HAS_INT128 1
+#else
+# define __TSAN_HAS_INT128 0
+#endif
+
+// Part of ABI, do not change.
+// https://github.com/llvm/llvm-project/blob/main/libcxx/include/atomic
+typedef enum {
+  mo_relaxed,
+  mo_consume,
+  mo_acquire,
+  mo_release,
+  mo_acq_rel,
+  mo_seq_cst
+} morder;
+
+struct ThreadState;
+
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE
+a8 __tsan_atomic8_load(const volatile a8 *a, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a16 __tsan_atomic16_load(const volatile a16 *a, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a32 __tsan_atomic32_load(const volatile a32 *a, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a64 __tsan_atomic64_load(const volatile a64 *a, morder mo);
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+a128 __tsan_atomic128_load(const volatile a128 *a, morder mo);
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_atomic8_store(volatile a8 *a, a8 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_atomic16_store(volatile a16 *a, a16 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_atomic32_store(volatile a32 *a, a32 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_atomic64_store(volatile a64 *a, a64 v, morder mo);
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_atomic128_store(volatile a128 *a, a128 v, morder mo);
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, morder mo);
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, morder mo);
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, morder mo);
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, morder mo);
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, morder mo);
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, morder mo);
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, morder mo);
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, morder mo);
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, morder mo);
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, morder mo);
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, morder mo);
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, morder mo);
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, morder mo);
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, morder mo);
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic8_compare_exchange_strong(volatile a8 *a, a8 *c, a8 v,
+                                           morder mo, morder fmo);
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic16_compare_exchange_strong(volatile a16 *a, a16 *c, a16 v,
+                                            morder mo, morder fmo);
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic32_compare_exchange_strong(volatile a32 *a, a32 *c, a32 v,
+                                            morder mo, morder fmo);
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic64_compare_exchange_strong(volatile a64 *a, a64 *c, a64 v,
+                                            morder mo, morder fmo);
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic128_compare_exchange_strong(volatile a128 *a, a128 *c, a128 v,
+                                             morder mo, morder fmo);
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic8_compare_exchange_weak(volatile a8 *a, a8 *c, a8 v, morder mo,
+                                         morder fmo);
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic16_compare_exchange_weak(volatile a16 *a, a16 *c, a16 v,
+                                          morder mo, morder fmo);
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic32_compare_exchange_weak(volatile a32 *a, a32 *c, a32 v,
+                                          morder mo, morder fmo);
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic64_compare_exchange_weak(volatile a64 *a, a64 *c, a64 v,
+                                          morder mo, morder fmo);
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic128_compare_exchange_weak(volatile a128 *a, a128 *c, a128 v,
+                                           morder mo, morder fmo);
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v, morder mo,
+                                       morder fmo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v,
+                                         morder mo, morder fmo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v,
+                                         morder mo, morder fmo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v,
+                                         morder mo, morder fmo);
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+a128 __tsan_atomic128_compare_exchange_val(volatile a128 *a, a128 c, a128 v,
+                                           morder mo, morder fmo);
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_atomic_thread_fence(morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_atomic_signal_fence(morder mo);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic32_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic64_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic32_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic64_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic32_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic64_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic32_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic64_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic32_compare_exchange(ThreadState *thr, uptr cpc, uptr pc,
+                                         u8 *a);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic64_compare_exchange(ThreadState *thr, uptr cpc, uptr pc,
+                                         u8 *a);
+
+}  // extern "C"
+
+}  // namespace __tsan
+
+#endif  // TSAN_INTERFACE_H
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_interface.inc b/compiler-rt/lib/tsan/rtl-old/tsan_interface.inc
new file mode 100644
index 000000000000..0031800e851f
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_interface.inc
@@ -0,0 +1,182 @@
+//===-- tsan_interface.inc --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_ptrauth.h"
+#include "tsan_interface.h"
+#include "tsan_rtl.h"
+
+#define CALLERPC ((uptr)__builtin_return_address(0))
+
+using namespace __tsan;
+
+void __tsan_read1(void *addr) {
+  MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 1, kAccessRead);
+}
+
+void __tsan_read2(void *addr) {
+  MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, kAccessRead);
+}
+
+void __tsan_read4(void *addr) {
+  MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, kAccessRead);
+}
+
+void __tsan_read8(void *addr) {
+  MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, kAccessRead);
+}
+
+void __tsan_write1(void *addr) {
+  MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 1, kAccessWrite);
+}
+
+void __tsan_write2(void *addr) {
+  MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, kAccessWrite);
+}
+
+void __tsan_write4(void *addr) {
+  MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, kAccessWrite);
+}
+
+void __tsan_write8(void *addr) {
+  MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, kAccessWrite);
+}
+
+void __tsan_read1_pc(void *addr, void *pc) {
+  MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 1, kAccessRead | kAccessExternalPC);
+}
+
+void __tsan_read2_pc(void *addr, void *pc) {
+  MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 2, kAccessRead | kAccessExternalPC);
+}
+
+void __tsan_read4_pc(void *addr, void *pc) {
+  MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 4, kAccessRead | kAccessExternalPC);
+}
+
+void __tsan_read8_pc(void *addr, void *pc) {
+  MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 8, kAccessRead | kAccessExternalPC);
+}
+
+void __tsan_write1_pc(void *addr, void *pc) {
+  MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 1, kAccessWrite | kAccessExternalPC);
+}
+
+void __tsan_write2_pc(void *addr, void *pc) {
+  MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 2, kAccessWrite | kAccessExternalPC);
+}
+
+void __tsan_write4_pc(void *addr, void *pc) {
+  MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 4, kAccessWrite | kAccessExternalPC);
+}
+
+void __tsan_write8_pc(void *addr, void *pc) {
+  MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 8, kAccessWrite | kAccessExternalPC);
+}
+
+ALWAYS_INLINE USED void __tsan_unaligned_read2(const void *addr) {
+  UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, kAccessRead);
+}
+
+ALWAYS_INLINE USED void __tsan_unaligned_read4(const void *addr) {
+  UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, kAccessRead);
+}
+
+ALWAYS_INLINE USED void __tsan_unaligned_read8(const void *addr) {
+  UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, kAccessRead);
+}
+
+ALWAYS_INLINE USED void __tsan_unaligned_write2(void *addr) {
+  UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, kAccessWrite);
+}
+
+ALWAYS_INLINE USED void __tsan_unaligned_write4(void *addr) {
+  UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, kAccessWrite);
+}
+
+ALWAYS_INLINE USED void __tsan_unaligned_write8(void *addr) {
+  UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, kAccessWrite);
+}
+
+extern "C" {
+// __sanitizer_unaligned_load/store are for user instrumentation.
+SANITIZER_INTERFACE_ATTRIBUTE
+u16 __sanitizer_unaligned_load16(const uu16 *addr) {
+  __tsan_unaligned_read2(addr);
+  return *addr;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+u32 __sanitizer_unaligned_load32(const uu32 *addr) {
+  __tsan_unaligned_read4(addr);
+  return *addr;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+u64 __sanitizer_unaligned_load64(const uu64 *addr) {
+  __tsan_unaligned_read8(addr);
+  return *addr;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_unaligned_store16(uu16 *addr, u16 v) {
+  *addr = v;
+  __tsan_unaligned_write2(addr);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_unaligned_store32(uu32 *addr, u32 v) {
+  *addr = v;
+  __tsan_unaligned_write4(addr);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_unaligned_store64(uu64 *addr, u64 v) {
+  *addr = v;
+  __tsan_unaligned_write8(addr);
+}
+}
+
+void __tsan_vptr_update(void **vptr_p, void *new_val) {
+  if (*vptr_p == new_val)
+    return;
+  MemoryAccess(cur_thread(), CALLERPC, (uptr)vptr_p, sizeof(*vptr_p),
+               kAccessWrite | kAccessVptr);
+}
+
+void __tsan_vptr_read(void **vptr_p) {
+  MemoryAccess(cur_thread(), CALLERPC, (uptr)vptr_p, sizeof(*vptr_p),
+               kAccessRead | kAccessVptr);
+}
+
+void __tsan_func_entry(void *pc) { FuncEntry(cur_thread(), STRIP_PAC_PC(pc)); }
+
+void __tsan_func_exit() { FuncExit(cur_thread()); }
+
+void __tsan_ignore_thread_begin() { ThreadIgnoreBegin(cur_thread(), CALLERPC); }
+
+void __tsan_ignore_thread_end() { ThreadIgnoreEnd(cur_thread()); }
+
+void __tsan_read_range(void *addr, uptr size) {
+  MemoryAccessRange(cur_thread(), CALLERPC, (uptr)addr, size, false);
+}
+
+void __tsan_write_range(void *addr, uptr size) {
+  MemoryAccessRange(cur_thread(), CALLERPC, (uptr)addr, size, true);
+}
+
+void __tsan_read_range_pc(void *addr, uptr size, void *pc) {
+  MemoryAccessRange(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, size, false);
+}
+
+void __tsan_write_range_pc(void *addr, uptr size, void *pc) {
+  MemoryAccessRange(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, size, true);
+}
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_interface_ann.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_interface_ann.cpp
new file mode 100644
index 000000000000..6bd72e18d942
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_interface_ann.cpp
@@ -0,0 +1,438 @@
+//===-- tsan_interface_ann.cpp --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#include "sanitizer_common/sanitizer_libc.h"
+#include "sanitizer_common/sanitizer_internal_defs.h"
+#include "sanitizer_common/sanitizer_placement_new.h"
+#include "sanitizer_common/sanitizer_stacktrace.h"
+#include "sanitizer_common/sanitizer_vector.h"
+#include "tsan_interface_ann.h"
+#include "tsan_report.h"
+#include "tsan_rtl.h"
+#include "tsan_mman.h"
+#include "tsan_flags.h"
+#include "tsan_platform.h"
+
+#define CALLERPC ((uptr)__builtin_return_address(0))
+
+using namespace __tsan;
+
+namespace __tsan {
+
+class ScopedAnnotation {
+ public:
+  ScopedAnnotation(ThreadState *thr, const char *aname, uptr pc)
+      : thr_(thr) {
+    FuncEntry(thr_, pc);
+    DPrintf("#%d: annotation %s()\n", thr_->tid, aname);
+  }
+
+  ~ScopedAnnotation() {
+    FuncExit(thr_);
+    CheckedMutex::CheckNoLocks();
+  }
+ private:
+  ThreadState *const thr_;
+};
+
+#define SCOPED_ANNOTATION_RET(typ, ret)                     \
+  if (!flags()->enable_annotations)                         \
+    return ret;                                             \
+  ThreadState *thr = cur_thread();                          \
+  const uptr caller_pc = (uptr)__builtin_return_address(0); \
+  ScopedAnnotation sa(thr, __func__, caller_pc);            \
+  const uptr pc = StackTrace::GetCurrentPc();               \
+  (void)pc;
+
+#define SCOPED_ANNOTATION(typ) SCOPED_ANNOTATION_RET(typ, )
+
+static const int kMaxDescLen = 128;
+
+struct ExpectRace {
+  ExpectRace *next;
+  ExpectRace *prev;
+  atomic_uintptr_t hitcount;
+  atomic_uintptr_t addcount;
+  uptr addr;
+  uptr size;
+  char *file;
+  int line;
+  char desc[kMaxDescLen];
+};
+
+struct DynamicAnnContext {
+  Mutex mtx;
+  ExpectRace benign;
+
+  DynamicAnnContext() : mtx(MutexTypeAnnotations) {}
+};
+
+static DynamicAnnContext *dyn_ann_ctx;
+static char dyn_ann_ctx_placeholder[sizeof(DynamicAnnContext)] ALIGNED(64);
+
+static void AddExpectRace(ExpectRace *list,
+    char *f, int l, uptr addr, uptr size, char *desc) {
+  ExpectRace *race = list->next;
+  for (; race != list; race = race->next) {
+    if (race->addr == addr && race->size == size) {
+      atomic_store_relaxed(&race->addcount,
+          atomic_load_relaxed(&race->addcount) + 1);
+      return;
+    }
+  }
+  race = static_cast<ExpectRace *>(Alloc(sizeof(ExpectRace)));
+  race->addr = addr;
+  race->size = size;
+  race->file = f;
+  race->line = l;
+  race->desc[0] = 0;
+  atomic_store_relaxed(&race->hitcount, 0);
+  atomic_store_relaxed(&race->addcount, 1);
+  if (desc) {
+    int i = 0;
+    for (; i < kMaxDescLen - 1 && desc[i]; i++)
+      race->desc[i] = desc[i];
+    race->desc[i] = 0;
+  }
+  race->prev = list;
+  race->next = list->next;
+  race->next->prev = race;
+  list->next = race;
+}
+
+static ExpectRace *FindRace(ExpectRace *list, uptr addr, uptr size) {
+  for (ExpectRace *race = list->next; race != list; race = race->next) {
+    uptr maxbegin = max(race->addr, addr);
+    uptr minend = min(race->addr + race->size, addr + size);
+    if (maxbegin < minend)
+      return race;
+  }
+  return 0;
+}
+
+static bool CheckContains(ExpectRace *list, uptr addr, uptr size) {
+  ExpectRace *race = FindRace(list, addr, size);
+  if (race == 0)
+    return false;
+  DPrintf("Hit expected/benign race: %s addr=%zx:%d %s:%d\n",
+      race->desc, race->addr, (int)race->size, race->file, race->line);
+  atomic_fetch_add(&race->hitcount, 1, memory_order_relaxed);
+  return true;
+}
+
+static void InitList(ExpectRace *list) {
+  list->next = list;
+  list->prev = list;
+}
+
+void InitializeDynamicAnnotations() {
+  dyn_ann_ctx = new(dyn_ann_ctx_placeholder) DynamicAnnContext;
+  InitList(&dyn_ann_ctx->benign);
+}
+
+bool IsExpectedReport(uptr addr, uptr size) {
+  ReadLock lock(&dyn_ann_ctx->mtx);
+  return CheckContains(&dyn_ann_ctx->benign, addr, size);
+}
+}  // namespace __tsan
+
+using namespace __tsan;
+
+extern "C" {
+void INTERFACE_ATTRIBUTE AnnotateHappensBefore(char *f, int l, uptr addr) {
+  SCOPED_ANNOTATION(AnnotateHappensBefore);
+  Release(thr, pc, addr);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateHappensAfter(char *f, int l, uptr addr) {
+  SCOPED_ANNOTATION(AnnotateHappensAfter);
+  Acquire(thr, pc, addr);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateCondVarSignal(char *f, int l, uptr cv) {
+}
+
+void INTERFACE_ATTRIBUTE AnnotateCondVarSignalAll(char *f, int l, uptr cv) {
+}
+
+void INTERFACE_ATTRIBUTE AnnotateMutexIsNotPHB(char *f, int l, uptr mu) {
+}
+
+void INTERFACE_ATTRIBUTE AnnotateCondVarWait(char *f, int l, uptr cv,
+                                             uptr lock) {
+}
+
+void INTERFACE_ATTRIBUTE AnnotateRWLockCreate(char *f, int l, uptr m) {
+  SCOPED_ANNOTATION(AnnotateRWLockCreate);
+  MutexCreate(thr, pc, m, MutexFlagWriteReentrant);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateRWLockCreateStatic(char *f, int l, uptr m) {
+  SCOPED_ANNOTATION(AnnotateRWLockCreateStatic);
+  MutexCreate(thr, pc, m, MutexFlagWriteReentrant | MutexFlagLinkerInit);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateRWLockDestroy(char *f, int l, uptr m) {
+  SCOPED_ANNOTATION(AnnotateRWLockDestroy);
+  MutexDestroy(thr, pc, m);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateRWLockAcquired(char *f, int l, uptr m,
+                                                uptr is_w) {
+  SCOPED_ANNOTATION(AnnotateRWLockAcquired);
+  if (is_w)
+    MutexPostLock(thr, pc, m, MutexFlagDoPreLockOnPostLock);
+  else
+    MutexPostReadLock(thr, pc, m, MutexFlagDoPreLockOnPostLock);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateRWLockReleased(char *f, int l, uptr m,
+                                                uptr is_w) {
+  SCOPED_ANNOTATION(AnnotateRWLockReleased);
+  if (is_w)
+    MutexUnlock(thr, pc, m);
+  else
+    MutexReadUnlock(thr, pc, m);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateTraceMemory(char *f, int l, uptr mem) {
+}
+
+void INTERFACE_ATTRIBUTE AnnotateFlushState(char *f, int l) {
+}
+
+void INTERFACE_ATTRIBUTE AnnotateNewMemory(char *f, int l, uptr mem,
+                                           uptr size) {
+}
+
+void INTERFACE_ATTRIBUTE AnnotateNoOp(char *f, int l, uptr mem) {
+}
+
+void INTERFACE_ATTRIBUTE AnnotateFlushExpectedRaces(char *f, int l) {
+}
+
+void INTERFACE_ATTRIBUTE AnnotateEnableRaceDetection(
+    char *f, int l, int enable) {
+}
+
+void INTERFACE_ATTRIBUTE AnnotateMutexIsUsedAsCondVar(
+    char *f, int l, uptr mu) {
+}
+
+void INTERFACE_ATTRIBUTE AnnotatePCQGet(
+    char *f, int l, uptr pcq) {
+}
+
+void INTERFACE_ATTRIBUTE AnnotatePCQPut(
+    char *f, int l, uptr pcq) {
+}
+
+void INTERFACE_ATTRIBUTE AnnotatePCQDestroy(
+    char *f, int l, uptr pcq) {
+}
+
+void INTERFACE_ATTRIBUTE AnnotatePCQCreate(
+    char *f, int l, uptr pcq) {
+}
+
+void INTERFACE_ATTRIBUTE AnnotateExpectRace(
+    char *f, int l, uptr mem, char *desc) {
+}
+
+static void BenignRaceImpl(char *f, int l, uptr mem, uptr size, char *desc) {
+  Lock lock(&dyn_ann_ctx->mtx);
+  AddExpectRace(&dyn_ann_ctx->benign,
+                f, l, mem, size, desc);
+  DPrintf("Add benign race: %s addr=%zx %s:%d\n", desc, mem, f, l);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateBenignRaceSized(
+    char *f, int l, uptr mem, uptr size, char *desc) {
+  SCOPED_ANNOTATION(AnnotateBenignRaceSized);
+  BenignRaceImpl(f, l, mem, size, desc);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateBenignRace(
+    char *f, int l, uptr mem, char *desc) {
+  SCOPED_ANNOTATION(AnnotateBenignRace);
+  BenignRaceImpl(f, l, mem, 1, desc);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateIgnoreReadsBegin(char *f, int l) {
+  SCOPED_ANNOTATION(AnnotateIgnoreReadsBegin);
+  ThreadIgnoreBegin(thr, pc);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateIgnoreReadsEnd(char *f, int l) {
+  SCOPED_ANNOTATION(AnnotateIgnoreReadsEnd);
+  ThreadIgnoreEnd(thr);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateIgnoreWritesBegin(char *f, int l) {
+  SCOPED_ANNOTATION(AnnotateIgnoreWritesBegin);
+  ThreadIgnoreBegin(thr, pc);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateIgnoreWritesEnd(char *f, int l) {
+  SCOPED_ANNOTATION(AnnotateIgnoreWritesEnd);
+  ThreadIgnoreEnd(thr);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateIgnoreSyncBegin(char *f, int l) {
+  SCOPED_ANNOTATION(AnnotateIgnoreSyncBegin);
+  ThreadIgnoreSyncBegin(thr, pc);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateIgnoreSyncEnd(char *f, int l) {
+  SCOPED_ANNOTATION(AnnotateIgnoreSyncEnd);
+  ThreadIgnoreSyncEnd(thr);
+}
+
+void INTERFACE_ATTRIBUTE AnnotatePublishMemoryRange(
+    char *f, int l, uptr addr, uptr size) {
+}
+
+void INTERFACE_ATTRIBUTE AnnotateUnpublishMemoryRange(
+    char *f, int l, uptr addr, uptr size) {
+}
+
+void INTERFACE_ATTRIBUTE AnnotateThreadName(
+    char *f, int l, char *name) {
+  SCOPED_ANNOTATION(AnnotateThreadName);
+  ThreadSetName(thr, name);
+}
+
+// We deliberately omit the implementation of WTFAnnotateHappensBefore() and
+// WTFAnnotateHappensAfter(). Those are being used by Webkit to annotate
+// atomic operations, which should be handled by ThreadSanitizer correctly.
+void INTERFACE_ATTRIBUTE WTFAnnotateHappensBefore(char *f, int l, uptr addr) {
+}
+
+void INTERFACE_ATTRIBUTE WTFAnnotateHappensAfter(char *f, int l, uptr addr) {
+}
+
+void INTERFACE_ATTRIBUTE WTFAnnotateBenignRaceSized(
+    char *f, int l, uptr mem, uptr sz, char *desc) {
+  SCOPED_ANNOTATION(AnnotateBenignRaceSized);
+  BenignRaceImpl(f, l, mem, sz, desc);
+}
+
+int INTERFACE_ATTRIBUTE RunningOnValgrind() {
+  return flags()->running_on_valgrind;
+}
+
+double __attribute__((weak)) INTERFACE_ATTRIBUTE ValgrindSlowdown(void) {
+  return 10.0;
+}
+
+const char INTERFACE_ATTRIBUTE* ThreadSanitizerQuery(const char *query) {
+  if (internal_strcmp(query, "pure_happens_before") == 0)
+    return "1";
+  else
+    return "0";
+}
+
+void INTERFACE_ATTRIBUTE
+AnnotateMemoryIsInitialized(char *f, int l, uptr mem, uptr sz) {}
+void INTERFACE_ATTRIBUTE
+AnnotateMemoryIsUninitialized(char *f, int l, uptr mem, uptr sz) {}
+
+// Note: the parameter is called flagz, because flags is already taken
+// by the global function that returns flags.
+INTERFACE_ATTRIBUTE
+void __tsan_mutex_create(void *m, unsigned flagz) {
+  SCOPED_ANNOTATION(__tsan_mutex_create);
+  MutexCreate(thr, pc, (uptr)m, flagz & MutexCreationFlagMask);
+}
+
+INTERFACE_ATTRIBUTE
+void __tsan_mutex_destroy(void *m, unsigned flagz) {
+  SCOPED_ANNOTATION(__tsan_mutex_destroy);
+  MutexDestroy(thr, pc, (uptr)m, flagz);
+}
+
+INTERFACE_ATTRIBUTE
+void __tsan_mutex_pre_lock(void *m, unsigned flagz) {
+  SCOPED_ANNOTATION(__tsan_mutex_pre_lock);
+  if (!(flagz & MutexFlagTryLock)) {
+    if (flagz & MutexFlagReadLock)
+      MutexPreReadLock(thr, pc, (uptr)m);
+    else
+      MutexPreLock(thr, pc, (uptr)m);
+  }
+  ThreadIgnoreBegin(thr, 0);
+  ThreadIgnoreSyncBegin(thr, 0);
+}
+
+INTERFACE_ATTRIBUTE
+void __tsan_mutex_post_lock(void *m, unsigned flagz, int rec) {
+  SCOPED_ANNOTATION(__tsan_mutex_post_lock);
+  ThreadIgnoreSyncEnd(thr);
+  ThreadIgnoreEnd(thr);
+  if (!(flagz & MutexFlagTryLockFailed)) {
+    if (flagz & MutexFlagReadLock)
+      MutexPostReadLock(thr, pc, (uptr)m, flagz);
+    else
+      MutexPostLock(thr, pc, (uptr)m, flagz, rec);
+  }
+}
+
+INTERFACE_ATTRIBUTE
+int __tsan_mutex_pre_unlock(void *m, unsigned flagz) {
+  SCOPED_ANNOTATION_RET(__tsan_mutex_pre_unlock, 0);
+  int ret = 0;
+  if (flagz & MutexFlagReadLock) {
+    CHECK(!(flagz & MutexFlagRecursiveUnlock));
+    MutexReadUnlock(thr, pc, (uptr)m);
+  } else {
+    ret = MutexUnlock(thr, pc, (uptr)m, flagz);
+  }
+  ThreadIgnoreBegin(thr, 0);
+  ThreadIgnoreSyncBegin(thr, 0);
+  return ret;
+}
+
+INTERFACE_ATTRIBUTE
+void __tsan_mutex_post_unlock(void *m, unsigned flagz) {
+  SCOPED_ANNOTATION(__tsan_mutex_post_unlock);
+  ThreadIgnoreSyncEnd(thr);
+  ThreadIgnoreEnd(thr);
+}
+
+INTERFACE_ATTRIBUTE
+void __tsan_mutex_pre_signal(void *addr, unsigned flagz) {
+  SCOPED_ANNOTATION(__tsan_mutex_pre_signal);
+  ThreadIgnoreBegin(thr, 0);
+  ThreadIgnoreSyncBegin(thr, 0);
+}
+
+INTERFACE_ATTRIBUTE
+void __tsan_mutex_post_signal(void *addr, unsigned flagz) {
+  SCOPED_ANNOTATION(__tsan_mutex_post_signal);
+  ThreadIgnoreSyncEnd(thr);
+  ThreadIgnoreEnd(thr);
+}
+
+INTERFACE_ATTRIBUTE
+void __tsan_mutex_pre_divert(void *addr, unsigned flagz) {
+  SCOPED_ANNOTATION(__tsan_mutex_pre_divert);
+  // Exit from ignore region started in __tsan_mutex_pre_lock/unlock/signal.
+  ThreadIgnoreSyncEnd(thr);
+  ThreadIgnoreEnd(thr);
+}
+
+INTERFACE_ATTRIBUTE
+void __tsan_mutex_post_divert(void *addr, unsigned flagz) {
+  SCOPED_ANNOTATION(__tsan_mutex_post_divert);
+  ThreadIgnoreBegin(thr, 0);
+  ThreadIgnoreSyncBegin(thr, 0);
+}
+}  // extern "C"
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_interface_ann.h b/compiler-rt/lib/tsan/rtl-old/tsan_interface_ann.h
new file mode 100644
index 000000000000..458d61f53356
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_interface_ann.h
@@ -0,0 +1,32 @@
+//===-- tsan_interface_ann.h ------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// Interface for dynamic annotations.
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_INTERFACE_ANN_H
+#define TSAN_INTERFACE_ANN_H
+
+#include <sanitizer_common/sanitizer_internal_defs.h>
+
+// This header should NOT include any other headers.
+// All functions in this header are extern "C" and start with __tsan_.
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_acquire(void *addr);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_release(void *addr);
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+#endif  // TSAN_INTERFACE_ANN_H
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_interface_atomic.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_interface_atomic.cpp
new file mode 100644
index 000000000000..24ba3bb1f65d
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_interface_atomic.cpp
@@ -0,0 +1,920 @@
+//===-- tsan_interface_atomic.cpp -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+
+// ThreadSanitizer atomic operations are based on C++11/C1x standards.
+// For background see C++11 standard.  A slightly older, publicly
+// available draft of the standard (not entirely up-to-date, but close enough
+// for casual browsing) is available here:
+// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2011/n3242.pdf
+// The following page contains more background information:
+// http://www.hpl.hp.com/personal/Hans_Boehm/c++mm/
+
+#include "sanitizer_common/sanitizer_placement_new.h"
+#include "sanitizer_common/sanitizer_stacktrace.h"
+#include "sanitizer_common/sanitizer_mutex.h"
+#include "tsan_flags.h"
+#include "tsan_interface.h"
+#include "tsan_rtl.h"
+
+using namespace __tsan;
+
+#if !SANITIZER_GO && __TSAN_HAS_INT128
+// Protects emulation of 128-bit atomic operations.
+static StaticSpinMutex mutex128;
+#endif
+
+#if SANITIZER_DEBUG
+static bool IsLoadOrder(morder mo) {
+  return mo == mo_relaxed || mo == mo_consume
+      || mo == mo_acquire || mo == mo_seq_cst;
+}
+
+static bool IsStoreOrder(morder mo) {
+  return mo == mo_relaxed || mo == mo_release || mo == mo_seq_cst;
+}
+#endif
+
+static bool IsReleaseOrder(morder mo) {
+  return mo == mo_release || mo == mo_acq_rel || mo == mo_seq_cst;
+}
+
+static bool IsAcquireOrder(morder mo) {
+  return mo == mo_consume || mo == mo_acquire
+      || mo == mo_acq_rel || mo == mo_seq_cst;
+}
+
+static bool IsAcqRelOrder(morder mo) {
+  return mo == mo_acq_rel || mo == mo_seq_cst;
+}
+
+template<typename T> T func_xchg(volatile T *v, T op) {
+  T res = __sync_lock_test_and_set(v, op);
+  // __sync_lock_test_and_set does not contain full barrier.
+  __sync_synchronize();
+  return res;
+}
+
+template<typename T> T func_add(volatile T *v, T op) {
+  return __sync_fetch_and_add(v, op);
+}
+
+template<typename T> T func_sub(volatile T *v, T op) {
+  return __sync_fetch_and_sub(v, op);
+}
+
+template<typename T> T func_and(volatile T *v, T op) {
+  return __sync_fetch_and_and(v, op);
+}
+
+template<typename T> T func_or(volatile T *v, T op) {
+  return __sync_fetch_and_or(v, op);
+}
+
+template<typename T> T func_xor(volatile T *v, T op) {
+  return __sync_fetch_and_xor(v, op);
+}
+
+template<typename T> T func_nand(volatile T *v, T op) {
+  // clang does not support __sync_fetch_and_nand.
+  T cmp = *v;
+  for (;;) {
+    T newv = ~(cmp & op);
+    T cur = __sync_val_compare_and_swap(v, cmp, newv);
+    if (cmp == cur)
+      return cmp;
+    cmp = cur;
+  }
+}
+
+template<typename T> T func_cas(volatile T *v, T cmp, T xch) {
+  return __sync_val_compare_and_swap(v, cmp, xch);
+}
+
+// clang does not support 128-bit atomic ops.
+// Atomic ops are executed under tsan internal mutex,
+// here we assume that the atomic variables are not accessed
+// from non-instrumented code.
+#if !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16) && !SANITIZER_GO \
+    && __TSAN_HAS_INT128
+a128 func_xchg(volatile a128 *v, a128 op) {
+  SpinMutexLock lock(&mutex128);
+  a128 cmp = *v;
+  *v = op;
+  return cmp;
+}
+
+a128 func_add(volatile a128 *v, a128 op) {
+  SpinMutexLock lock(&mutex128);
+  a128 cmp = *v;
+  *v = cmp + op;
+  return cmp;
+}
+
+a128 func_sub(volatile a128 *v, a128 op) {
+  SpinMutexLock lock(&mutex128);
+  a128 cmp = *v;
+  *v = cmp - op;
+  return cmp;
+}
+
+a128 func_and(volatile a128 *v, a128 op) {
+  SpinMutexLock lock(&mutex128);
+  a128 cmp = *v;
+  *v = cmp & op;
+  return cmp;
+}
+
+a128 func_or(volatile a128 *v, a128 op) {
+  SpinMutexLock lock(&mutex128);
+  a128 cmp = *v;
+  *v = cmp | op;
+  return cmp;
+}
+
+a128 func_xor(volatile a128 *v, a128 op) {
+  SpinMutexLock lock(&mutex128);
+  a128 cmp = *v;
+  *v = cmp ^ op;
+  return cmp;
+}
+
+a128 func_nand(volatile a128 *v, a128 op) {
+  SpinMutexLock lock(&mutex128);
+  a128 cmp = *v;
+  *v = ~(cmp & op);
+  return cmp;
+}
+
+a128 func_cas(volatile a128 *v, a128 cmp, a128 xch) {
+  SpinMutexLock lock(&mutex128);
+  a128 cur = *v;
+  if (cur == cmp)
+    *v = xch;
+  return cur;
+}
+#endif
+
+template <typename T>
+static int AccessSize() {
+  if (sizeof(T) <= 1)
+    return 1;
+  else if (sizeof(T) <= 2)
+    return 2;
+  else if (sizeof(T) <= 4)
+    return 4;
+  else
+    return 8;
+  // For 16-byte atomics we also use 8-byte memory access,
+  // this leads to false negatives only in very obscure cases.
+}
+
+#if !SANITIZER_GO
+static atomic_uint8_t *to_atomic(const volatile a8 *a) {
+  return reinterpret_cast<atomic_uint8_t *>(const_cast<a8 *>(a));
+}
+
+static atomic_uint16_t *to_atomic(const volatile a16 *a) {
+  return reinterpret_cast<atomic_uint16_t *>(const_cast<a16 *>(a));
+}
+#endif
+
+static atomic_uint32_t *to_atomic(const volatile a32 *a) {
+  return reinterpret_cast<atomic_uint32_t *>(const_cast<a32 *>(a));
+}
+
+static atomic_uint64_t *to_atomic(const volatile a64 *a) {
+  return reinterpret_cast<atomic_uint64_t *>(const_cast<a64 *>(a));
+}
+
+static memory_order to_mo(morder mo) {
+  switch (mo) {
+  case mo_relaxed: return memory_order_relaxed;
+  case mo_consume: return memory_order_consume;
+  case mo_acquire: return memory_order_acquire;
+  case mo_release: return memory_order_release;
+  case mo_acq_rel: return memory_order_acq_rel;
+  case mo_seq_cst: return memory_order_seq_cst;
+  }
+  DCHECK(0);
+  return memory_order_seq_cst;
+}
+
+template<typename T>
+static T NoTsanAtomicLoad(const volatile T *a, morder mo) {
+  return atomic_load(to_atomic(a), to_mo(mo));
+}
+
+#if __TSAN_HAS_INT128 && !SANITIZER_GO
+static a128 NoTsanAtomicLoad(const volatile a128 *a, morder mo) {
+  SpinMutexLock lock(&mutex128);
+  return *a;
+}
+#endif
+
+template <typename T>
+static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a, morder mo) {
+  DCHECK(IsLoadOrder(mo));
+  // This fast-path is critical for performance.
+  // Assume the access is atomic.
+  if (!IsAcquireOrder(mo)) {
+    MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(),
+                 kAccessRead | kAccessAtomic);
+    return NoTsanAtomicLoad(a, mo);
+  }
+  // Don't create sync object if it does not exist yet. For example, an atomic
+  // pointer is initialized to nullptr and then periodically acquire-loaded.
+  T v = NoTsanAtomicLoad(a, mo);
+  SyncVar *s = ctx->metamap.GetSyncIfExists((uptr)a);
+  if (s) {
+    ReadLock l(&s->mtx);
+    AcquireImpl(thr, pc, &s->clock);
+    // Re-read under sync mutex because we need a consistent snapshot
+    // of the value and the clock we acquire.
+    v = NoTsanAtomicLoad(a, mo);
+  }
+  MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), kAccessRead | kAccessAtomic);
+  return v;
+}
+
+template<typename T>
+static void NoTsanAtomicStore(volatile T *a, T v, morder mo) {
+  atomic_store(to_atomic(a), v, to_mo(mo));
+}
+
+#if __TSAN_HAS_INT128 && !SANITIZER_GO
+static void NoTsanAtomicStore(volatile a128 *a, a128 v, morder mo) {
+  SpinMutexLock lock(&mutex128);
+  *a = v;
+}
+#endif
+
+template <typename T>
+static void AtomicStore(ThreadState *thr, uptr pc, volatile T *a, T v,
+                        morder mo) {
+  DCHECK(IsStoreOrder(mo));
+  MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), kAccessWrite | kAccessAtomic);
+  // This fast-path is critical for performance.
+  // Assume the access is atomic.
+  // Strictly saying even relaxed store cuts off release sequence,
+  // so must reset the clock.
+  if (!IsReleaseOrder(mo)) {
+    NoTsanAtomicStore(a, v, mo);
+    return;
+  }
+  __sync_synchronize();
+  SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false);
+  Lock l(&s->mtx);
+  thr->fast_state.IncrementEpoch();
+  // Can't increment epoch w/o writing to the trace as well.
+  TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
+  ReleaseStoreImpl(thr, pc, &s->clock);
+  NoTsanAtomicStore(a, v, mo);
+}
+
+template <typename T, T (*F)(volatile T *v, T op)>
+static T AtomicRMW(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) {
+  MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), kAccessWrite | kAccessAtomic);
+  if (LIKELY(mo == mo_relaxed))
+    return F(a, v);
+  SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false);
+  Lock l(&s->mtx);
+  thr->fast_state.IncrementEpoch();
+  // Can't increment epoch w/o writing to the trace as well.
+  TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
+  if (IsAcqRelOrder(mo))
+    AcquireReleaseImpl(thr, pc, &s->clock);
+  else if (IsReleaseOrder(mo))
+    ReleaseImpl(thr, pc, &s->clock);
+  else if (IsAcquireOrder(mo))
+    AcquireImpl(thr, pc, &s->clock);
+  return F(a, v);
+}
+
+template<typename T>
+static T NoTsanAtomicExchange(volatile T *a, T v, morder mo) {
+  return func_xchg(a, v);
+}
+
+template<typename T>
+static T NoTsanAtomicFetchAdd(volatile T *a, T v, morder mo) {
+  return func_add(a, v);
+}
+
+template<typename T>
+static T NoTsanAtomicFetchSub(volatile T *a, T v, morder mo) {
+  return func_sub(a, v);
+}
+
+template<typename T>
+static T NoTsanAtomicFetchAnd(volatile T *a, T v, morder mo) {
+  return func_and(a, v);
+}
+
+template<typename T>
+static T NoTsanAtomicFetchOr(volatile T *a, T v, morder mo) {
+  return func_or(a, v);
+}
+
+template<typename T>
+static T NoTsanAtomicFetchXor(volatile T *a, T v, morder mo) {
+  return func_xor(a, v);
+}
+
+template<typename T>
+static T NoTsanAtomicFetchNand(volatile T *a, T v, morder mo) {
+  return func_nand(a, v);
+}
+
+template<typename T>
+static T AtomicExchange(ThreadState *thr, uptr pc, volatile T *a, T v,
+    morder mo) {
+  return AtomicRMW<T, func_xchg>(thr, pc, a, v, mo);
+}
+
+template<typename T>
+static T AtomicFetchAdd(ThreadState *thr, uptr pc, volatile T *a, T v,
+    morder mo) {
+  return AtomicRMW<T, func_add>(thr, pc, a, v, mo);
+}
+
+template<typename T>
+static T AtomicFetchSub(ThreadState *thr, uptr pc, volatile T *a, T v,
+    morder mo) {
+  return AtomicRMW<T, func_sub>(thr, pc, a, v, mo);
+}
+
+template<typename T>
+static T AtomicFetchAnd(ThreadState *thr, uptr pc, volatile T *a, T v,
+    morder mo) {
+  return AtomicRMW<T, func_and>(thr, pc, a, v, mo);
+}
+
+template<typename T>
+static T AtomicFetchOr(ThreadState *thr, uptr pc, volatile T *a, T v,
+    morder mo) {
+  return AtomicRMW<T, func_or>(thr, pc, a, v, mo);
+}
+
+template<typename T>
+static T AtomicFetchXor(ThreadState *thr, uptr pc, volatile T *a, T v,
+    morder mo) {
+  return AtomicRMW<T, func_xor>(thr, pc, a, v, mo);
+}
+
+template<typename T>
+static T AtomicFetchNand(ThreadState *thr, uptr pc, volatile T *a, T v,
+    morder mo) {
+  return AtomicRMW<T, func_nand>(thr, pc, a, v, mo);
+}
+
+template<typename T>
+static bool NoTsanAtomicCAS(volatile T *a, T *c, T v, morder mo, morder fmo) {
+  return atomic_compare_exchange_strong(to_atomic(a), c, v, to_mo(mo));
+}
+
+#if __TSAN_HAS_INT128
+static bool NoTsanAtomicCAS(volatile a128 *a, a128 *c, a128 v,
+    morder mo, morder fmo) {
+  a128 old = *c;
+  a128 cur = func_cas(a, old, v);
+  if (cur == old)
+    return true;
+  *c = cur;
+  return false;
+}
+#endif
+
+template<typename T>
+static T NoTsanAtomicCAS(volatile T *a, T c, T v, morder mo, morder fmo) {
+  NoTsanAtomicCAS(a, &c, v, mo, fmo);
+  return c;
+}
+
+template <typename T>
+static bool AtomicCAS(ThreadState *thr, uptr pc, volatile T *a, T *c, T v,
+                      morder mo, morder fmo) {
+  // 31.7.2.18: "The failure argument shall not be memory_order_release
+  // nor memory_order_acq_rel". LLVM (2021-05) fallbacks to Monotonic
+  // (mo_relaxed) when those are used.
+  DCHECK(IsLoadOrder(fmo));
+
+  MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), kAccessWrite | kAccessAtomic);
+  if (LIKELY(mo == mo_relaxed && fmo == mo_relaxed)) {
+    T cc = *c;
+    T pr = func_cas(a, cc, v);
+    if (pr == cc)
+      return true;
+    *c = pr;
+    return false;
+  }
+
+  bool release = IsReleaseOrder(mo);
+  SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false);
+  RWLock l(&s->mtx, release);
+  T cc = *c;
+  T pr = func_cas(a, cc, v);
+  bool success = pr == cc;
+  if (!success) {
+    *c = pr;
+    mo = fmo;
+  }
+  thr->fast_state.IncrementEpoch();
+  // Can't increment epoch w/o writing to the trace as well.
+  TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
+
+  if (success && IsAcqRelOrder(mo))
+    AcquireReleaseImpl(thr, pc, &s->clock);
+  else if (success && IsReleaseOrder(mo))
+    ReleaseImpl(thr, pc, &s->clock);
+  else if (IsAcquireOrder(mo))
+    AcquireImpl(thr, pc, &s->clock);
+  return success;
+}
+
+template<typename T>
+static T AtomicCAS(ThreadState *thr, uptr pc,
+    volatile T *a, T c, T v, morder mo, morder fmo) {
+  AtomicCAS(thr, pc, a, &c, v, mo, fmo);
+  return c;
+}
+
+#if !SANITIZER_GO
+static void NoTsanAtomicFence(morder mo) {
+  __sync_synchronize();
+}
+
+static void AtomicFence(ThreadState *thr, uptr pc, morder mo) {
+  // FIXME(dvyukov): not implemented.
+  __sync_synchronize();
+}
+#endif
+
+// Interface functions follow.
+#if !SANITIZER_GO
+
+// C/C++
+
+static morder convert_morder(morder mo) {
+  if (flags()->force_seq_cst_atomics)
+    return (morder)mo_seq_cst;
+
+  // Filter out additional memory order flags:
+  // MEMMODEL_SYNC        = 1 << 15
+  // __ATOMIC_HLE_ACQUIRE = 1 << 16
+  // __ATOMIC_HLE_RELEASE = 1 << 17
+  //
+  // HLE is an optimization, and we pretend that elision always fails.
+  // MEMMODEL_SYNC is used when lowering __sync_ atomics,
+  // since we use __sync_ atomics for actual atomic operations,
+  // we can safely ignore it as well. It also subtly affects semantics,
+  // but we don't model the difference.
+  return (morder)(mo & 0x7fff);
+}
+
+#  define ATOMIC_IMPL(func, ...)                                \
+    ThreadState *const thr = cur_thread();                      \
+    ProcessPendingSignals(thr);                                 \
+    if (UNLIKELY(thr->ignore_sync || thr->ignore_interceptors)) \
+      return NoTsanAtomic##func(__VA_ARGS__);                   \
+    mo = convert_morder(mo);                                    \
+    return Atomic##func(thr, GET_CALLER_PC(), __VA_ARGS__);
+
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE
+a8 __tsan_atomic8_load(const volatile a8 *a, morder mo) {
+  ATOMIC_IMPL(Load, a, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a16 __tsan_atomic16_load(const volatile a16 *a, morder mo) {
+  ATOMIC_IMPL(Load, a, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a32 __tsan_atomic32_load(const volatile a32 *a, morder mo) {
+  ATOMIC_IMPL(Load, a, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a64 __tsan_atomic64_load(const volatile a64 *a, morder mo) {
+  ATOMIC_IMPL(Load, a, mo);
+}
+
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+a128 __tsan_atomic128_load(const volatile a128 *a, morder mo) {
+  ATOMIC_IMPL(Load, a, mo);
+}
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_atomic8_store(volatile a8 *a, a8 v, morder mo) {
+  ATOMIC_IMPL(Store, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_atomic16_store(volatile a16 *a, a16 v, morder mo) {
+  ATOMIC_IMPL(Store, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_atomic32_store(volatile a32 *a, a32 v, morder mo) {
+  ATOMIC_IMPL(Store, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_atomic64_store(volatile a64 *a, a64 v, morder mo) {
+  ATOMIC_IMPL(Store, a, v, mo);
+}
+
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_atomic128_store(volatile a128 *a, a128 v, morder mo) {
+  ATOMIC_IMPL(Store, a, v, mo);
+}
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, morder mo) {
+  ATOMIC_IMPL(Exchange, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, morder mo) {
+  ATOMIC_IMPL(Exchange, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, morder mo) {
+  ATOMIC_IMPL(Exchange, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, morder mo) {
+  ATOMIC_IMPL(Exchange, a, v, mo);
+}
+
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, morder mo) {
+  ATOMIC_IMPL(Exchange, a, v, mo);
+}
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, morder mo) {
+  ATOMIC_IMPL(FetchAdd, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, morder mo) {
+  ATOMIC_IMPL(FetchAdd, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, morder mo) {
+  ATOMIC_IMPL(FetchAdd, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, morder mo) {
+  ATOMIC_IMPL(FetchAdd, a, v, mo);
+}
+
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, morder mo) {
+  ATOMIC_IMPL(FetchAdd, a, v, mo);
+}
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, morder mo) {
+  ATOMIC_IMPL(FetchSub, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, morder mo) {
+  ATOMIC_IMPL(FetchSub, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, morder mo) {
+  ATOMIC_IMPL(FetchSub, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, morder mo) {
+  ATOMIC_IMPL(FetchSub, a, v, mo);
+}
+
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, morder mo) {
+  ATOMIC_IMPL(FetchSub, a, v, mo);
+}
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, morder mo) {
+  ATOMIC_IMPL(FetchAnd, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, morder mo) {
+  ATOMIC_IMPL(FetchAnd, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, morder mo) {
+  ATOMIC_IMPL(FetchAnd, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, morder mo) {
+  ATOMIC_IMPL(FetchAnd, a, v, mo);
+}
+
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, morder mo) {
+  ATOMIC_IMPL(FetchAnd, a, v, mo);
+}
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, morder mo) {
+  ATOMIC_IMPL(FetchOr, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, morder mo) {
+  ATOMIC_IMPL(FetchOr, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, morder mo) {
+  ATOMIC_IMPL(FetchOr, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, morder mo) {
+  ATOMIC_IMPL(FetchOr, a, v, mo);
+}
+
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, morder mo) {
+  ATOMIC_IMPL(FetchOr, a, v, mo);
+}
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, morder mo) {
+  ATOMIC_IMPL(FetchXor, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, morder mo) {
+  ATOMIC_IMPL(FetchXor, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, morder mo) {
+  ATOMIC_IMPL(FetchXor, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, morder mo) {
+  ATOMIC_IMPL(FetchXor, a, v, mo);
+}
+
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, morder mo) {
+  ATOMIC_IMPL(FetchXor, a, v, mo);
+}
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, morder mo) {
+  ATOMIC_IMPL(FetchNand, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, morder mo) {
+  ATOMIC_IMPL(FetchNand, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, morder mo) {
+  ATOMIC_IMPL(FetchNand, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, morder mo) {
+  ATOMIC_IMPL(FetchNand, a, v, mo);
+}
+
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, morder mo) {
+  ATOMIC_IMPL(FetchNand, a, v, mo);
+}
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic8_compare_exchange_strong(volatile a8 *a, a8 *c, a8 v,
+    morder mo, morder fmo) {
+  ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic16_compare_exchange_strong(volatile a16 *a, a16 *c, a16 v,
+    morder mo, morder fmo) {
+  ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic32_compare_exchange_strong(volatile a32 *a, a32 *c, a32 v,
+    morder mo, morder fmo) {
+  ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic64_compare_exchange_strong(volatile a64 *a, a64 *c, a64 v,
+    morder mo, morder fmo) {
+  ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
+}
+
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic128_compare_exchange_strong(volatile a128 *a, a128 *c, a128 v,
+    morder mo, morder fmo) {
+  ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
+}
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic8_compare_exchange_weak(volatile a8 *a, a8 *c, a8 v,
+    morder mo, morder fmo) {
+  ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic16_compare_exchange_weak(volatile a16 *a, a16 *c, a16 v,
+    morder mo, morder fmo) {
+  ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic32_compare_exchange_weak(volatile a32 *a, a32 *c, a32 v,
+    morder mo, morder fmo) {
+  ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic64_compare_exchange_weak(volatile a64 *a, a64 *c, a64 v,
+    morder mo, morder fmo) {
+  ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
+}
+
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic128_compare_exchange_weak(volatile a128 *a, a128 *c, a128 v,
+    morder mo, morder fmo) {
+  ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
+}
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v,
+    morder mo, morder fmo) {
+  ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v,
+    morder mo, morder fmo) {
+  ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v,
+    morder mo, morder fmo) {
+  ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v,
+    morder mo, morder fmo) {
+  ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
+}
+
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+a128 __tsan_atomic128_compare_exchange_val(volatile a128 *a, a128 c, a128 v,
+    morder mo, morder fmo) {
+  ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
+}
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_atomic_thread_fence(morder mo) { ATOMIC_IMPL(Fence, mo); }
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_atomic_signal_fence(morder mo) {
+}
+}  // extern "C"
+
+#else  // #if !SANITIZER_GO
+
+// Go
+
+#  define ATOMIC(func, ...)               \
+    if (thr->ignore_sync) {               \
+      NoTsanAtomic##func(__VA_ARGS__);    \
+    } else {                              \
+      FuncEntry(thr, cpc);                \
+      Atomic##func(thr, pc, __VA_ARGS__); \
+      FuncExit(thr);                      \
+    }
+
+#  define ATOMIC_RET(func, ret, ...)              \
+    if (thr->ignore_sync) {                       \
+      (ret) = NoTsanAtomic##func(__VA_ARGS__);    \
+    } else {                                      \
+      FuncEntry(thr, cpc);                        \
+      (ret) = Atomic##func(thr, pc, __VA_ARGS__); \
+      FuncExit(thr);                              \
+    }
+
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic32_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
+  ATOMIC_RET(Load, *(a32*)(a+8), *(a32**)a, mo_acquire);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic64_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
+  ATOMIC_RET(Load, *(a64*)(a+8), *(a64**)a, mo_acquire);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic32_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
+  ATOMIC(Store, *(a32**)a, *(a32*)(a+8), mo_release);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic64_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
+  ATOMIC(Store, *(a64**)a, *(a64*)(a+8), mo_release);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic32_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
+  ATOMIC_RET(FetchAdd, *(a32*)(a+16), *(a32**)a, *(a32*)(a+8), mo_acq_rel);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic64_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
+  ATOMIC_RET(FetchAdd, *(a64*)(a+16), *(a64**)a, *(a64*)(a+8), mo_acq_rel);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic32_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
+  ATOMIC_RET(Exchange, *(a32*)(a+16), *(a32**)a, *(a32*)(a+8), mo_acq_rel);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic64_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
+  ATOMIC_RET(Exchange, *(a64*)(a+16), *(a64**)a, *(a64*)(a+8), mo_acq_rel);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic32_compare_exchange(
+    ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
+  a32 cur = 0;
+  a32 cmp = *(a32*)(a+8);
+  ATOMIC_RET(CAS, cur, *(a32**)a, cmp, *(a32*)(a+12), mo_acq_rel, mo_acquire);
+  *(bool*)(a+16) = (cur == cmp);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic64_compare_exchange(
+    ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
+  a64 cur = 0;
+  a64 cmp = *(a64*)(a+8);
+  ATOMIC_RET(CAS, cur, *(a64**)a, cmp, *(a64*)(a+16), mo_acq_rel, mo_acquire);
+  *(bool*)(a+24) = (cur == cmp);
+}
+}  // extern "C"
+#endif  // #if !SANITIZER_GO
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_interface_java.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_interface_java.cpp
new file mode 100644
index 000000000000..c090c1f08cbe
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_interface_java.cpp
@@ -0,0 +1,258 @@
+//===-- tsan_interface_java.cpp -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+
+#include "tsan_interface_java.h"
+#include "tsan_rtl.h"
+#include "sanitizer_common/sanitizer_internal_defs.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_placement_new.h"
+#include "sanitizer_common/sanitizer_stacktrace.h"
+#include "sanitizer_common/sanitizer_procmaps.h"
+
+using namespace __tsan;
+
+const jptr kHeapAlignment = 8;
+
+namespace __tsan {
+
+struct JavaContext {
+  const uptr heap_begin;
+  const uptr heap_size;
+
+  JavaContext(jptr heap_begin, jptr heap_size)
+      : heap_begin(heap_begin)
+      , heap_size(heap_size) {
+  }
+};
+
+static u64 jctx_buf[sizeof(JavaContext) / sizeof(u64) + 1];
+static JavaContext *jctx;
+
+MBlock *JavaHeapBlock(uptr addr, uptr *start) {
+  if (!jctx || addr < jctx->heap_begin ||
+      addr >= jctx->heap_begin + jctx->heap_size)
+    return nullptr;
+  for (uptr p = RoundDown(addr, kMetaShadowCell); p >= jctx->heap_begin;
+       p -= kMetaShadowCell) {
+    MBlock *b = ctx->metamap.GetBlock(p);
+    if (!b)
+      continue;
+    if (p + b->siz <= addr)
+      return nullptr;
+    *start = p;
+    return b;
+  }
+  return nullptr;
+}
+
+}  // namespace __tsan
+
+#define JAVA_FUNC_ENTER(func)      \
+  ThreadState *thr = cur_thread(); \
+  (void)thr;
+
+void __tsan_java_init(jptr heap_begin, jptr heap_size) {
+  JAVA_FUNC_ENTER(__tsan_java_init);
+  Initialize(thr);
+  DPrintf("#%d: java_init(0x%zx, 0x%zx)\n", thr->tid, heap_begin, heap_size);
+  DCHECK_EQ(jctx, 0);
+  DCHECK_GT(heap_begin, 0);
+  DCHECK_GT(heap_size, 0);
+  DCHECK_EQ(heap_begin % kHeapAlignment, 0);
+  DCHECK_EQ(heap_size % kHeapAlignment, 0);
+  DCHECK_LT(heap_begin, heap_begin + heap_size);
+  jctx = new(jctx_buf) JavaContext(heap_begin, heap_size);
+}
+
+int  __tsan_java_fini() {
+  JAVA_FUNC_ENTER(__tsan_java_fini);
+  DPrintf("#%d: java_fini()\n", thr->tid);
+  DCHECK_NE(jctx, 0);
+  // FIXME(dvyukov): this does not call atexit() callbacks.
+  int status = Finalize(thr);
+  DPrintf("#%d: java_fini() = %d\n", thr->tid, status);
+  return status;
+}
+
+void __tsan_java_alloc(jptr ptr, jptr size) {
+  JAVA_FUNC_ENTER(__tsan_java_alloc);
+  DPrintf("#%d: java_alloc(0x%zx, 0x%zx)\n", thr->tid, ptr, size);
+  DCHECK_NE(jctx, 0);
+  DCHECK_NE(size, 0);
+  DCHECK_EQ(ptr % kHeapAlignment, 0);
+  DCHECK_EQ(size % kHeapAlignment, 0);
+  DCHECK_GE(ptr, jctx->heap_begin);
+  DCHECK_LE(ptr + size, jctx->heap_begin + jctx->heap_size);
+
+  OnUserAlloc(thr, 0, ptr, size, false);
+}
+
+void __tsan_java_free(jptr ptr, jptr size) {
+  JAVA_FUNC_ENTER(__tsan_java_free);
+  DPrintf("#%d: java_free(0x%zx, 0x%zx)\n", thr->tid, ptr, size);
+  DCHECK_NE(jctx, 0);
+  DCHECK_NE(size, 0);
+  DCHECK_EQ(ptr % kHeapAlignment, 0);
+  DCHECK_EQ(size % kHeapAlignment, 0);
+  DCHECK_GE(ptr, jctx->heap_begin);
+  DCHECK_LE(ptr + size, jctx->heap_begin + jctx->heap_size);
+
+  ctx->metamap.FreeRange(thr->proc(), ptr, size);
+}
+
+void __tsan_java_move(jptr src, jptr dst, jptr size) {
+  JAVA_FUNC_ENTER(__tsan_java_move);
+  DPrintf("#%d: java_move(0x%zx, 0x%zx, 0x%zx)\n", thr->tid, src, dst, size);
+  DCHECK_NE(jctx, 0);
+  DCHECK_NE(size, 0);
+  DCHECK_EQ(src % kHeapAlignment, 0);
+  DCHECK_EQ(dst % kHeapAlignment, 0);
+  DCHECK_EQ(size % kHeapAlignment, 0);
+  DCHECK_GE(src, jctx->heap_begin);
+  DCHECK_LE(src + size, jctx->heap_begin + jctx->heap_size);
+  DCHECK_GE(dst, jctx->heap_begin);
+  DCHECK_LE(dst + size, jctx->heap_begin + jctx->heap_size);
+  DCHECK_NE(dst, src);
+  DCHECK_NE(size, 0);
+
+  // Assuming it's not running concurrently with threads that do
+  // memory accesses and mutex operations (stop-the-world phase).
+  ctx->metamap.MoveMemory(src, dst, size);
+
+  // Clear the destination shadow range.
+  // We used to move shadow from src to dst, but the trace format does not
+  // support that anymore as it contains addresses of accesses.
+  RawShadow *d = MemToShadow(dst);
+  RawShadow *dend = MemToShadow(dst + size);
+  internal_memset(d, 0, (dend - d) * sizeof(*d));
+}
+
+jptr __tsan_java_find(jptr *from_ptr, jptr to) {
+  JAVA_FUNC_ENTER(__tsan_java_find);
+  DPrintf("#%d: java_find(&0x%zx, 0x%zx)\n", thr->tid, *from_ptr, to);
+  DCHECK_EQ((*from_ptr) % kHeapAlignment, 0);
+  DCHECK_EQ(to % kHeapAlignment, 0);
+  DCHECK_GE(*from_ptr, jctx->heap_begin);
+  DCHECK_LE(to, jctx->heap_begin + jctx->heap_size);
+  for (uptr from = *from_ptr; from < to; from += kHeapAlignment) {
+    MBlock *b = ctx->metamap.GetBlock(from);
+    if (b) {
+      *from_ptr = from;
+      return b->siz;
+    }
+  }
+  return 0;
+}
+
+void __tsan_java_finalize() {
+  JAVA_FUNC_ENTER(__tsan_java_finalize);
+  DPrintf("#%d: java_finalize()\n", thr->tid);
+  AcquireGlobal(thr);
+}
+
+void __tsan_java_mutex_lock(jptr addr) {
+  JAVA_FUNC_ENTER(__tsan_java_mutex_lock);
+  DPrintf("#%d: java_mutex_lock(0x%zx)\n", thr->tid, addr);
+  DCHECK_NE(jctx, 0);
+  DCHECK_GE(addr, jctx->heap_begin);
+  DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
+
+  MutexPostLock(thr, 0, addr,
+                MutexFlagLinkerInit | MutexFlagWriteReentrant |
+                    MutexFlagDoPreLockOnPostLock);
+}
+
+void __tsan_java_mutex_unlock(jptr addr) {
+  JAVA_FUNC_ENTER(__tsan_java_mutex_unlock);
+  DPrintf("#%d: java_mutex_unlock(0x%zx)\n", thr->tid, addr);
+  DCHECK_NE(jctx, 0);
+  DCHECK_GE(addr, jctx->heap_begin);
+  DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
+
+  MutexUnlock(thr, 0, addr);
+}
+
+void __tsan_java_mutex_read_lock(jptr addr) {
+  JAVA_FUNC_ENTER(__tsan_java_mutex_read_lock);
+  DPrintf("#%d: java_mutex_read_lock(0x%zx)\n", thr->tid, addr);
+  DCHECK_NE(jctx, 0);
+  DCHECK_GE(addr, jctx->heap_begin);
+  DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
+
+  MutexPostReadLock(thr, 0, addr,
+                    MutexFlagLinkerInit | MutexFlagWriteReentrant |
+                        MutexFlagDoPreLockOnPostLock);
+}
+
+void __tsan_java_mutex_read_unlock(jptr addr) {
+  JAVA_FUNC_ENTER(__tsan_java_mutex_read_unlock);
+  DPrintf("#%d: java_mutex_read_unlock(0x%zx)\n", thr->tid, addr);
+  DCHECK_NE(jctx, 0);
+  DCHECK_GE(addr, jctx->heap_begin);
+  DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
+
+  MutexReadUnlock(thr, 0, addr);
+}
+
+void __tsan_java_mutex_lock_rec(jptr addr, int rec) {
+  JAVA_FUNC_ENTER(__tsan_java_mutex_lock_rec);
+  DPrintf("#%d: java_mutex_lock_rec(0x%zx, %d)\n", thr->tid, addr, rec);
+  DCHECK_NE(jctx, 0);
+  DCHECK_GE(addr, jctx->heap_begin);
+  DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
+  DCHECK_GT(rec, 0);
+
+  MutexPostLock(thr, 0, addr,
+                MutexFlagLinkerInit | MutexFlagWriteReentrant |
+                    MutexFlagDoPreLockOnPostLock | MutexFlagRecursiveLock,
+                rec);
+}
+
+int __tsan_java_mutex_unlock_rec(jptr addr) {
+  JAVA_FUNC_ENTER(__tsan_java_mutex_unlock_rec);
+  DPrintf("#%d: java_mutex_unlock_rec(0x%zx)\n", thr->tid, addr);
+  DCHECK_NE(jctx, 0);
+  DCHECK_GE(addr, jctx->heap_begin);
+  DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
+
+  return MutexUnlock(thr, 0, addr, MutexFlagRecursiveUnlock);
+}
+
+void __tsan_java_acquire(jptr addr) {
+  JAVA_FUNC_ENTER(__tsan_java_acquire);
+  DPrintf("#%d: java_acquire(0x%zx)\n", thr->tid, addr);
+  DCHECK_NE(jctx, 0);
+  DCHECK_GE(addr, jctx->heap_begin);
+  DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
+
+  Acquire(thr, 0, addr);
+}
+
+void __tsan_java_release(jptr addr) {
+  JAVA_FUNC_ENTER(__tsan_java_release);
+  DPrintf("#%d: java_release(0x%zx)\n", thr->tid, addr);
+  DCHECK_NE(jctx, 0);
+  DCHECK_GE(addr, jctx->heap_begin);
+  DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
+
+  Release(thr, 0, addr);
+}
+
+void __tsan_java_release_store(jptr addr) {
+  JAVA_FUNC_ENTER(__tsan_java_release);
+  DPrintf("#%d: java_release_store(0x%zx)\n", thr->tid, addr);
+  DCHECK_NE(jctx, 0);
+  DCHECK_GE(addr, jctx->heap_begin);
+  DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
+
+  ReleaseStore(thr, 0, addr);
+}
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_interface_java.h b/compiler-rt/lib/tsan/rtl-old/tsan_interface_java.h
new file mode 100644
index 000000000000..51b445251e09
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_interface_java.h
@@ -0,0 +1,99 @@
+//===-- tsan_interface_java.h -----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// Interface for verification of Java or mixed Java/C++ programs.
+// The interface is intended to be used from within a JVM and notify TSan
+// about such events like Java locks and GC memory compaction.
+//
+// For plain memory accesses and function entry/exit a JVM is intended to use
+// C++ interfaces: __tsan_readN/writeN and __tsan_func_enter/exit.
+//
+// For volatile memory accesses and atomic operations JVM is intended to use
+// standard atomics API: __tsan_atomicN_load/store/etc.
+//
+// For usage examples see lit_tests/java_*.cpp
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_INTERFACE_JAVA_H
+#define TSAN_INTERFACE_JAVA_H
+
+#ifndef INTERFACE_ATTRIBUTE
+# define INTERFACE_ATTRIBUTE __attribute__((visibility("default")))
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef unsigned long jptr;
+
+// Must be called before any other callback from Java.
+void __tsan_java_init(jptr heap_begin, jptr heap_size) INTERFACE_ATTRIBUTE;
+// Must be called when the application exits.
+// Not necessary the last callback (concurrently running threads are OK).
+// Returns exit status or 0 if tsan does not want to override it.
+int  __tsan_java_fini() INTERFACE_ATTRIBUTE;
+
+// Callback for memory allocations.
+// May be omitted for allocations that are not subject to data races
+// nor contain synchronization objects (e.g. String).
+void __tsan_java_alloc(jptr ptr, jptr size) INTERFACE_ATTRIBUTE;
+// Callback for memory free.
+// Can be aggregated for several objects (preferably).
+void __tsan_java_free(jptr ptr, jptr size) INTERFACE_ATTRIBUTE;
+// Callback for memory move by GC.
+// Can be aggregated for several objects (preferably).
+// The ranges can overlap.
+void __tsan_java_move(jptr src, jptr dst, jptr size) INTERFACE_ATTRIBUTE;
+// This function must be called on the finalizer thread
+// before executing a batch of finalizers.
+// It ensures necessary synchronization between
+// java object creation and finalization.
+void __tsan_java_finalize() INTERFACE_ATTRIBUTE;
+// Finds the first allocated memory block in the [*from_ptr, to) range, saves
+// its address in *from_ptr and returns its size. Returns 0 if there are no
+// allocated memory blocks in the range.
+jptr __tsan_java_find(jptr *from_ptr, jptr to) INTERFACE_ATTRIBUTE;
+
+// Mutex lock.
+// Addr is any unique address associated with the mutex.
+// Can be called on recursive reentry.
+void __tsan_java_mutex_lock(jptr addr) INTERFACE_ATTRIBUTE;
+// Mutex unlock.
+void __tsan_java_mutex_unlock(jptr addr) INTERFACE_ATTRIBUTE;
+// Mutex read lock.
+void __tsan_java_mutex_read_lock(jptr addr) INTERFACE_ATTRIBUTE;
+// Mutex read unlock.
+void __tsan_java_mutex_read_unlock(jptr addr) INTERFACE_ATTRIBUTE;
+// Recursive mutex lock, intended for handling of Object.wait().
+// The 'rec' value must be obtained from the previous
+// __tsan_java_mutex_unlock_rec().
+void __tsan_java_mutex_lock_rec(jptr addr, int rec) INTERFACE_ATTRIBUTE;
+// Recursive mutex unlock, intended for handling of Object.wait().
+// The return value says how many times this thread called lock()
+// w/o a pairing unlock() (i.e. how many recursive levels it unlocked).
+// It must be passed back to __tsan_java_mutex_lock_rec() to restore
+// the same recursion level.
+int __tsan_java_mutex_unlock_rec(jptr addr) INTERFACE_ATTRIBUTE;
+
+// Raw acquire/release primitives.
+// Can be used to establish happens-before edges on volatile/final fields,
+// in atomic operations, etc. release_store is the same as release, but it
+// breaks release sequence on addr (see C++ standard 1.10/7 for details).
+void __tsan_java_acquire(jptr addr) INTERFACE_ATTRIBUTE;
+void __tsan_java_release(jptr addr) INTERFACE_ATTRIBUTE;
+void __tsan_java_release_store(jptr addr) INTERFACE_ATTRIBUTE;
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+#undef INTERFACE_ATTRIBUTE
+
+#endif  // #ifndef TSAN_INTERFACE_JAVA_H
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_malloc_mac.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_malloc_mac.cpp
new file mode 100644
index 000000000000..0e861bf1f962
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_malloc_mac.cpp
@@ -0,0 +1,71 @@
+//===-- tsan_malloc_mac.cpp -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// Mac-specific malloc interception.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_platform.h"
+#if SANITIZER_MAC
+
+#include "sanitizer_common/sanitizer_errno.h"
+#include "tsan_interceptors.h"
+#include "tsan_stack_trace.h"
+
+using namespace __tsan;
+#define COMMON_MALLOC_ZONE_NAME "tsan"
+#define COMMON_MALLOC_ENTER()
+#define COMMON_MALLOC_SANITIZER_INITIALIZED (cur_thread()->is_inited)
+#define COMMON_MALLOC_FORCE_LOCK()
+#define COMMON_MALLOC_FORCE_UNLOCK()
+#define COMMON_MALLOC_MEMALIGN(alignment, size) \
+  void *p =                                     \
+      user_memalign(cur_thread(), StackTrace::GetCurrentPc(), alignment, size)
+#define COMMON_MALLOC_MALLOC(size)                             \
+  if (in_symbolizer()) return InternalAlloc(size);             \
+  SCOPED_INTERCEPTOR_RAW(malloc, size);                        \
+  void *p = user_alloc(thr, pc, size)
+#define COMMON_MALLOC_REALLOC(ptr, size)                              \
+  if (in_symbolizer()) return InternalRealloc(ptr, size);             \
+  SCOPED_INTERCEPTOR_RAW(realloc, ptr, size);                         \
+  void *p = user_realloc(thr, pc, ptr, size)
+#define COMMON_MALLOC_CALLOC(count, size)                              \
+  if (in_symbolizer()) return InternalCalloc(count, size);             \
+  SCOPED_INTERCEPTOR_RAW(calloc, size, count);                         \
+  void *p = user_calloc(thr, pc, size, count)
+#define COMMON_MALLOC_POSIX_MEMALIGN(memptr, alignment, size)      \
+  if (in_symbolizer()) {                                           \
+    void *p = InternalAlloc(size, nullptr, alignment);             \
+    if (!p) return errno_ENOMEM;                                   \
+    *memptr = p;                                                   \
+    return 0;                                                      \
+  }                                                                \
+  SCOPED_INTERCEPTOR_RAW(posix_memalign, memptr, alignment, size); \
+  int res = user_posix_memalign(thr, pc, memptr, alignment, size);
+#define COMMON_MALLOC_VALLOC(size)                            \
+  if (in_symbolizer())                                        \
+    return InternalAlloc(size, nullptr, GetPageSizeCached()); \
+  SCOPED_INTERCEPTOR_RAW(valloc, size);                       \
+  void *p = user_valloc(thr, pc, size)
+#define COMMON_MALLOC_FREE(ptr)                              \
+  if (in_symbolizer()) return InternalFree(ptr);             \
+  SCOPED_INTERCEPTOR_RAW(free, ptr);                         \
+  user_free(thr, pc, ptr)
+#define COMMON_MALLOC_SIZE(ptr) uptr size = user_alloc_usable_size(ptr);
+#define COMMON_MALLOC_FILL_STATS(zone, stats)
+#define COMMON_MALLOC_REPORT_UNKNOWN_REALLOC(ptr, zone_ptr, zone_name) \
+  (void)zone_name; \
+  Report("mz_realloc(%p) -- attempting to realloc unallocated memory.\n", ptr);
+#define COMMON_MALLOC_NAMESPACE __tsan
+#define COMMON_MALLOC_HAS_ZONE_ENUMERATOR 0
+#define COMMON_MALLOC_HAS_EXTRA_INTROSPECTION_INIT 0
+
+#include "sanitizer_common/sanitizer_malloc_mac.inc"
+
+#endif
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_md5.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_md5.cpp
new file mode 100644
index 000000000000..72857b773fed
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_md5.cpp
@@ -0,0 +1,250 @@
+//===-- tsan_md5.cpp ------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#include "tsan_defs.h"
+
+namespace __tsan {
+
+#define F(x, y, z)      ((z) ^ ((x) & ((y) ^ (z))))
+#define G(x, y, z)      ((y) ^ ((z) & ((x) ^ (y))))
+#define H(x, y, z)      ((x) ^ (y) ^ (z))
+#define I(x, y, z)      ((y) ^ ((x) | ~(z)))
+
+#define STEP(f, a, b, c, d, x, t, s) \
+  (a) += f((b), (c), (d)) + (x) + (t); \
+  (a) = (((a) << (s)) | (((a) & 0xffffffff) >> (32 - (s)))); \
+  (a) += (b);
+
+#define SET(n) \
+  (*(const MD5_u32plus *)&ptr[(n) * 4])
+#define GET(n) \
+  SET(n)
+
+typedef unsigned int MD5_u32plus;
+typedef unsigned long ulong_t;
+
+typedef struct {
+  MD5_u32plus lo, hi;
+  MD5_u32plus a, b, c, d;
+  unsigned char buffer[64];
+  MD5_u32plus block[16];
+} MD5_CTX;
+
+static const void *body(MD5_CTX *ctx, const void *data, ulong_t size) {
+  const unsigned char *ptr = (const unsigned char *)data;
+  MD5_u32plus a, b, c, d;
+  MD5_u32plus saved_a, saved_b, saved_c, saved_d;
+
+  a = ctx->a;
+  b = ctx->b;
+  c = ctx->c;
+  d = ctx->d;
+
+  do {
+    saved_a = a;
+    saved_b = b;
+    saved_c = c;
+    saved_d = d;
+
+    STEP(F, a, b, c, d, SET(0), 0xd76aa478, 7)
+    STEP(F, d, a, b, c, SET(1), 0xe8c7b756, 12)
+    STEP(F, c, d, a, b, SET(2), 0x242070db, 17)
+    STEP(F, b, c, d, a, SET(3), 0xc1bdceee, 22)
+    STEP(F, a, b, c, d, SET(4), 0xf57c0faf, 7)
+    STEP(F, d, a, b, c, SET(5), 0x4787c62a, 12)
+    STEP(F, c, d, a, b, SET(6), 0xa8304613, 17)
+    STEP(F, b, c, d, a, SET(7), 0xfd469501, 22)
+    STEP(F, a, b, c, d, SET(8), 0x698098d8, 7)
+    STEP(F, d, a, b, c, SET(9), 0x8b44f7af, 12)
+    STEP(F, c, d, a, b, SET(10), 0xffff5bb1, 17)
+    STEP(F, b, c, d, a, SET(11), 0x895cd7be, 22)
+    STEP(F, a, b, c, d, SET(12), 0x6b901122, 7)
+    STEP(F, d, a, b, c, SET(13), 0xfd987193, 12)
+    STEP(F, c, d, a, b, SET(14), 0xa679438e, 17)
+    STEP(F, b, c, d, a, SET(15), 0x49b40821, 22)
+
+    STEP(G, a, b, c, d, GET(1), 0xf61e2562, 5)
+    STEP(G, d, a, b, c, GET(6), 0xc040b340, 9)
+    STEP(G, c, d, a, b, GET(11), 0x265e5a51, 14)
+    STEP(G, b, c, d, a, GET(0), 0xe9b6c7aa, 20)
+    STEP(G, a, b, c, d, GET(5), 0xd62f105d, 5)
+    STEP(G, d, a, b, c, GET(10), 0x02441453, 9)
+    STEP(G, c, d, a, b, GET(15), 0xd8a1e681, 14)
+    STEP(G, b, c, d, a, GET(4), 0xe7d3fbc8, 20)
+    STEP(G, a, b, c, d, GET(9), 0x21e1cde6, 5)
+    STEP(G, d, a, b, c, GET(14), 0xc33707d6, 9)
+    STEP(G, c, d, a, b, GET(3), 0xf4d50d87, 14)
+    STEP(G, b, c, d, a, GET(8), 0x455a14ed, 20)
+    STEP(G, a, b, c, d, GET(13), 0xa9e3e905, 5)
+    STEP(G, d, a, b, c, GET(2), 0xfcefa3f8, 9)
+    STEP(G, c, d, a, b, GET(7), 0x676f02d9, 14)
+    STEP(G, b, c, d, a, GET(12), 0x8d2a4c8a, 20)
+
+    STEP(H, a, b, c, d, GET(5), 0xfffa3942, 4)
+    STEP(H, d, a, b, c, GET(8), 0x8771f681, 11)
+    STEP(H, c, d, a, b, GET(11), 0x6d9d6122, 16)
+    STEP(H, b, c, d, a, GET(14), 0xfde5380c, 23)
+    STEP(H, a, b, c, d, GET(1), 0xa4beea44, 4)
+    STEP(H, d, a, b, c, GET(4), 0x4bdecfa9, 11)
+    STEP(H, c, d, a, b, GET(7), 0xf6bb4b60, 16)
+    STEP(H, b, c, d, a, GET(10), 0xbebfbc70, 23)
+    STEP(H, a, b, c, d, GET(13), 0x289b7ec6, 4)
+    STEP(H, d, a, b, c, GET(0), 0xeaa127fa, 11)
+    STEP(H, c, d, a, b, GET(3), 0xd4ef3085, 16)
+    STEP(H, b, c, d, a, GET(6), 0x04881d05, 23)
+    STEP(H, a, b, c, d, GET(9), 0xd9d4d039, 4)
+    STEP(H, d, a, b, c, GET(12), 0xe6db99e5, 11)
+    STEP(H, c, d, a, b, GET(15), 0x1fa27cf8, 16)
+    STEP(H, b, c, d, a, GET(2), 0xc4ac5665, 23)
+
+    STEP(I, a, b, c, d, GET(0), 0xf4292244, 6)
+    STEP(I, d, a, b, c, GET(7), 0x432aff97, 10)
+    STEP(I, c, d, a, b, GET(14), 0xab9423a7, 15)
+    STEP(I, b, c, d, a, GET(5), 0xfc93a039, 21)
+    STEP(I, a, b, c, d, GET(12), 0x655b59c3, 6)
+    STEP(I, d, a, b, c, GET(3), 0x8f0ccc92, 10)
+    STEP(I, c, d, a, b, GET(10), 0xffeff47d, 15)
+    STEP(I, b, c, d, a, GET(1), 0x85845dd1, 21)
+    STEP(I, a, b, c, d, GET(8), 0x6fa87e4f, 6)
+    STEP(I, d, a, b, c, GET(15), 0xfe2ce6e0, 10)
+    STEP(I, c, d, a, b, GET(6), 0xa3014314, 15)
+    STEP(I, b, c, d, a, GET(13), 0x4e0811a1, 21)
+    STEP(I, a, b, c, d, GET(4), 0xf7537e82, 6)
+    STEP(I, d, a, b, c, GET(11), 0xbd3af235, 10)
+    STEP(I, c, d, a, b, GET(2), 0x2ad7d2bb, 15)
+    STEP(I, b, c, d, a, GET(9), 0xeb86d391, 21)
+
+    a += saved_a;
+    b += saved_b;
+    c += saved_c;
+    d += saved_d;
+
+    ptr += 64;
+  } while (size -= 64);
+
+  ctx->a = a;
+  ctx->b = b;
+  ctx->c = c;
+  ctx->d = d;
+
+  return ptr;
+}
+
+#undef F
+#undef G
+#undef H
+#undef I
+#undef STEP
+#undef SET
+#undef GET
+
+void MD5_Init(MD5_CTX *ctx) {
+  ctx->a = 0x67452301;
+  ctx->b = 0xefcdab89;
+  ctx->c = 0x98badcfe;
+  ctx->d = 0x10325476;
+
+  ctx->lo = 0;
+  ctx->hi = 0;
+}
+
+void MD5_Update(MD5_CTX *ctx, const void *data, ulong_t size) {
+  MD5_u32plus saved_lo;
+  ulong_t used, free;
+
+  saved_lo = ctx->lo;
+  if ((ctx->lo = (saved_lo + size) & 0x1fffffff) < saved_lo)
+    ctx->hi++;
+  ctx->hi += size >> 29;
+
+  used = saved_lo & 0x3f;
+
+  if (used) {
+    free = 64 - used;
+
+    if (size < free) {
+      internal_memcpy(&ctx->buffer[used], data, size);
+      return;
+    }
+
+    internal_memcpy(&ctx->buffer[used], data, free);
+    data = (const unsigned char *)data + free;
+    size -= free;
+    body(ctx, ctx->buffer, 64);
+  }
+
+  if (size >= 64) {
+    data = body(ctx, data, size & ~(ulong_t)0x3f);
+    size &= 0x3f;
+  }
+
+  internal_memcpy(ctx->buffer, data, size);
+}
+
+void MD5_Final(unsigned char *result, MD5_CTX *ctx) {
+  ulong_t used, free;
+
+  used = ctx->lo & 0x3f;
+
+  ctx->buffer[used++] = 0x80;
+
+  free = 64 - used;
+
+  if (free < 8) {
+    internal_memset(&ctx->buffer[used], 0, free);
+    body(ctx, ctx->buffer, 64);
+    used = 0;
+    free = 64;
+  }
+
+  internal_memset(&ctx->buffer[used], 0, free - 8);
+
+  ctx->lo <<= 3;
+  ctx->buffer[56] = ctx->lo;
+  ctx->buffer[57] = ctx->lo >> 8;
+  ctx->buffer[58] = ctx->lo >> 16;
+  ctx->buffer[59] = ctx->lo >> 24;
+  ctx->buffer[60] = ctx->hi;
+  ctx->buffer[61] = ctx->hi >> 8;
+  ctx->buffer[62] = ctx->hi >> 16;
+  ctx->buffer[63] = ctx->hi >> 24;
+
+  body(ctx, ctx->buffer, 64);
+
+  result[0] = ctx->a;
+  result[1] = ctx->a >> 8;
+  result[2] = ctx->a >> 16;
+  result[3] = ctx->a >> 24;
+  result[4] = ctx->b;
+  result[5] = ctx->b >> 8;
+  result[6] = ctx->b >> 16;
+  result[7] = ctx->b >> 24;
+  result[8] = ctx->c;
+  result[9] = ctx->c >> 8;
+  result[10] = ctx->c >> 16;
+  result[11] = ctx->c >> 24;
+  result[12] = ctx->d;
+  result[13] = ctx->d >> 8;
+  result[14] = ctx->d >> 16;
+  result[15] = ctx->d >> 24;
+
+  internal_memset(ctx, 0, sizeof(*ctx));
+}
+
+MD5Hash md5_hash(const void *data, uptr size) {
+  MD5Hash res;
+  MD5_CTX ctx;
+  MD5_Init(&ctx);
+  MD5_Update(&ctx, data, size);
+  MD5_Final((unsigned char*)&res.hash[0], &ctx);
+  return res;
+}
+}  // namespace __tsan
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_mman.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_mman.cpp
new file mode 100644
index 000000000000..75044c38d5d2
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_mman.cpp
@@ -0,0 +1,436 @@
+//===-- tsan_mman.cpp -----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#include "sanitizer_common/sanitizer_allocator_checks.h"
+#include "sanitizer_common/sanitizer_allocator_interface.h"
+#include "sanitizer_common/sanitizer_allocator_report.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_errno.h"
+#include "sanitizer_common/sanitizer_placement_new.h"
+#include "tsan_mman.h"
+#include "tsan_rtl.h"
+#include "tsan_report.h"
+#include "tsan_flags.h"
+
+// May be overriden by front-end.
+SANITIZER_WEAK_DEFAULT_IMPL
+void __sanitizer_malloc_hook(void *ptr, uptr size) {
+  (void)ptr;
+  (void)size;
+}
+
+SANITIZER_WEAK_DEFAULT_IMPL
+void __sanitizer_free_hook(void *ptr) {
+  (void)ptr;
+}
+
+namespace __tsan {
+
+struct MapUnmapCallback {
+  void OnMap(uptr p, uptr size) const { }
+  void OnUnmap(uptr p, uptr size) const {
+    // We are about to unmap a chunk of user memory.
+    // Mark the corresponding shadow memory as not needed.
+    DontNeedShadowFor(p, size);
+    // Mark the corresponding meta shadow memory as not needed.
+    // Note the block does not contain any meta info at this point
+    // (this happens after free).
+    const uptr kMetaRatio = kMetaShadowCell / kMetaShadowSize;
+    const uptr kPageSize = GetPageSizeCached() * kMetaRatio;
+    // Block came from LargeMmapAllocator, so must be large.
+    // We rely on this in the calculations below.
+    CHECK_GE(size, 2 * kPageSize);
+    uptr diff = RoundUp(p, kPageSize) - p;
+    if (diff != 0) {
+      p += diff;
+      size -= diff;
+    }
+    diff = p + size - RoundDown(p + size, kPageSize);
+    if (diff != 0)
+      size -= diff;
+    uptr p_meta = (uptr)MemToMeta(p);
+    ReleaseMemoryPagesToOS(p_meta, p_meta + size / kMetaRatio);
+  }
+};
+
+static char allocator_placeholder[sizeof(Allocator)] ALIGNED(64);
+Allocator *allocator() {
+  return reinterpret_cast<Allocator*>(&allocator_placeholder);
+}
+
+struct GlobalProc {
+  Mutex mtx;
+  Processor *proc;
+  // This mutex represents the internal allocator combined for
+  // the purposes of deadlock detection. The internal allocator
+  // uses multiple mutexes, moreover they are locked only occasionally
+  // and they are spin mutexes which don't support deadlock detection.
+  // So we use this fake mutex to serve as a substitute for these mutexes.
+  CheckedMutex internal_alloc_mtx;
+
+  GlobalProc()
+      : mtx(MutexTypeGlobalProc),
+        proc(ProcCreate()),
+        internal_alloc_mtx(MutexTypeInternalAlloc) {}
+};
+
+static char global_proc_placeholder[sizeof(GlobalProc)] ALIGNED(64);
+GlobalProc *global_proc() {
+  return reinterpret_cast<GlobalProc*>(&global_proc_placeholder);
+}
+
+static void InternalAllocAccess() {
+  global_proc()->internal_alloc_mtx.Lock();
+  global_proc()->internal_alloc_mtx.Unlock();
+}
+
+ScopedGlobalProcessor::ScopedGlobalProcessor() {
+  GlobalProc *gp = global_proc();
+  ThreadState *thr = cur_thread();
+  if (thr->proc())
+    return;
+  // If we don't have a proc, use the global one.
+  // There are currently only two known case where this path is triggered:
+  //   __interceptor_free
+  //   __nptl_deallocate_tsd
+  //   start_thread
+  //   clone
+  // and:
+  //   ResetRange
+  //   __interceptor_munmap
+  //   __deallocate_stack
+  //   start_thread
+  //   clone
+  // Ideally, we destroy thread state (and unwire proc) when a thread actually
+  // exits (i.e. when we join/wait it). Then we would not need the global proc
+  gp->mtx.Lock();
+  ProcWire(gp->proc, thr);
+}
+
+ScopedGlobalProcessor::~ScopedGlobalProcessor() {
+  GlobalProc *gp = global_proc();
+  ThreadState *thr = cur_thread();
+  if (thr->proc() != gp->proc)
+    return;
+  ProcUnwire(gp->proc, thr);
+  gp->mtx.Unlock();
+}
+
+void AllocatorLock() NO_THREAD_SAFETY_ANALYSIS {
+  global_proc()->mtx.Lock();
+  global_proc()->internal_alloc_mtx.Lock();
+  InternalAllocatorLock();
+}
+
+void AllocatorUnlock() NO_THREAD_SAFETY_ANALYSIS {
+  InternalAllocatorUnlock();
+  global_proc()->internal_alloc_mtx.Unlock();
+  global_proc()->mtx.Unlock();
+}
+
+static constexpr uptr kMaxAllowedMallocSize = 1ull << 40;
+static uptr max_user_defined_malloc_size;
+
+void InitializeAllocator() {
+  SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
+  allocator()->Init(common_flags()->allocator_release_to_os_interval_ms);
+  max_user_defined_malloc_size = common_flags()->max_allocation_size_mb
+                                     ? common_flags()->max_allocation_size_mb
+                                           << 20
+                                     : kMaxAllowedMallocSize;
+}
+
+void InitializeAllocatorLate() {
+  new(global_proc()) GlobalProc();
+}
+
+void AllocatorProcStart(Processor *proc) {
+  allocator()->InitCache(&proc->alloc_cache);
+  internal_allocator()->InitCache(&proc->internal_alloc_cache);
+}
+
+void AllocatorProcFinish(Processor *proc) {
+  allocator()->DestroyCache(&proc->alloc_cache);
+  internal_allocator()->DestroyCache(&proc->internal_alloc_cache);
+}
+
+void AllocatorPrintStats() {
+  allocator()->PrintStats();
+}
+
+static void SignalUnsafeCall(ThreadState *thr, uptr pc) {
+  if (atomic_load_relaxed(&thr->in_signal_handler) == 0 ||
+      !ShouldReport(thr, ReportTypeSignalUnsafe))
+    return;
+  VarSizeStackTrace stack;
+  ObtainCurrentStack(thr, pc, &stack);
+  if (IsFiredSuppression(ctx, ReportTypeSignalUnsafe, stack))
+    return;
+  ThreadRegistryLock l(&ctx->thread_registry);
+  ScopedReport rep(ReportTypeSignalUnsafe);
+  rep.AddStack(stack, true);
+  OutputReport(thr, rep);
+}
+
+
+void *user_alloc_internal(ThreadState *thr, uptr pc, uptr sz, uptr align,
+                          bool signal) {
+  if (sz >= kMaxAllowedMallocSize || align >= kMaxAllowedMallocSize ||
+      sz > max_user_defined_malloc_size) {
+    if (AllocatorMayReturnNull())
+      return nullptr;
+    uptr malloc_limit =
+        Min(kMaxAllowedMallocSize, max_user_defined_malloc_size);
+    GET_STACK_TRACE_FATAL(thr, pc);
+    ReportAllocationSizeTooBig(sz, malloc_limit, &stack);
+  }
+  if (UNLIKELY(IsRssLimitExceeded())) {
+    if (AllocatorMayReturnNull())
+      return nullptr;
+    GET_STACK_TRACE_FATAL(thr, pc);
+    ReportRssLimitExceeded(&stack);
+  }
+  void *p = allocator()->Allocate(&thr->proc()->alloc_cache, sz, align);
+  if (UNLIKELY(!p)) {
+    SetAllocatorOutOfMemory();
+    if (AllocatorMayReturnNull())
+      return nullptr;
+    GET_STACK_TRACE_FATAL(thr, pc);
+    ReportOutOfMemory(sz, &stack);
+  }
+  if (ctx && ctx->initialized)
+    OnUserAlloc(thr, pc, (uptr)p, sz, true);
+  if (signal)
+    SignalUnsafeCall(thr, pc);
+  return p;
+}
+
+void user_free(ThreadState *thr, uptr pc, void *p, bool signal) {
+  ScopedGlobalProcessor sgp;
+  if (ctx && ctx->initialized)
+    OnUserFree(thr, pc, (uptr)p, true);
+  allocator()->Deallocate(&thr->proc()->alloc_cache, p);
+  if (signal)
+    SignalUnsafeCall(thr, pc);
+}
+
+void *user_alloc(ThreadState *thr, uptr pc, uptr sz) {
+  return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, kDefaultAlignment));
+}
+
+void *user_calloc(ThreadState *thr, uptr pc, uptr size, uptr n) {
+  if (UNLIKELY(CheckForCallocOverflow(size, n))) {
+    if (AllocatorMayReturnNull())
+      return SetErrnoOnNull(nullptr);
+    GET_STACK_TRACE_FATAL(thr, pc);
+    ReportCallocOverflow(n, size, &stack);
+  }
+  void *p = user_alloc_internal(thr, pc, n * size);
+  if (p)
+    internal_memset(p, 0, n * size);
+  return SetErrnoOnNull(p);
+}
+
+void *user_reallocarray(ThreadState *thr, uptr pc, void *p, uptr size, uptr n) {
+  if (UNLIKELY(CheckForCallocOverflow(size, n))) {
+    if (AllocatorMayReturnNull())
+      return SetErrnoOnNull(nullptr);
+    GET_STACK_TRACE_FATAL(thr, pc);
+    ReportReallocArrayOverflow(size, n, &stack);
+  }
+  return user_realloc(thr, pc, p, size * n);
+}
+
+void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write) {
+  DPrintf("#%d: alloc(%zu) = 0x%zx\n", thr->tid, sz, p);
+  ctx->metamap.AllocBlock(thr, pc, p, sz);
+  if (write && thr->ignore_reads_and_writes == 0 && thr->is_inited)
+    MemoryRangeImitateWrite(thr, pc, (uptr)p, sz);
+  else
+    MemoryResetRange(thr, pc, (uptr)p, sz);
+}
+
+void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write) {
+  CHECK_NE(p, (void*)0);
+  uptr sz = ctx->metamap.FreeBlock(thr->proc(), p);
+  DPrintf("#%d: free(0x%zx, %zu)\n", thr->tid, p, sz);
+  if (write && thr->ignore_reads_and_writes == 0 && thr->is_inited)
+    MemoryRangeFreed(thr, pc, (uptr)p, sz);
+}
+
+void *user_realloc(ThreadState *thr, uptr pc, void *p, uptr sz) {
+  // FIXME: Handle "shrinking" more efficiently,
+  // it seems that some software actually does this.
+  if (!p)
+    return SetErrnoOnNull(user_alloc_internal(thr, pc, sz));
+  if (!sz) {
+    user_free(thr, pc, p);
+    return nullptr;
+  }
+  void *new_p = user_alloc_internal(thr, pc, sz);
+  if (new_p) {
+    uptr old_sz = user_alloc_usable_size(p);
+    internal_memcpy(new_p, p, min(old_sz, sz));
+    user_free(thr, pc, p);
+  }
+  return SetErrnoOnNull(new_p);
+}
+
+void *user_memalign(ThreadState *thr, uptr pc, uptr align, uptr sz) {
+  if (UNLIKELY(!IsPowerOfTwo(align))) {
+    errno = errno_EINVAL;
+    if (AllocatorMayReturnNull())
+      return nullptr;
+    GET_STACK_TRACE_FATAL(thr, pc);
+    ReportInvalidAllocationAlignment(align, &stack);
+  }
+  return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, align));
+}
+
+int user_posix_memalign(ThreadState *thr, uptr pc, void **memptr, uptr align,
+                        uptr sz) {
+  if (UNLIKELY(!CheckPosixMemalignAlignment(align))) {
+    if (AllocatorMayReturnNull())
+      return errno_EINVAL;
+    GET_STACK_TRACE_FATAL(thr, pc);
+    ReportInvalidPosixMemalignAlignment(align, &stack);
+  }
+  void *ptr = user_alloc_internal(thr, pc, sz, align);
+  if (UNLIKELY(!ptr))
+    // OOM error is already taken care of by user_alloc_internal.
+    return errno_ENOMEM;
+  CHECK(IsAligned((uptr)ptr, align));
+  *memptr = ptr;
+  return 0;
+}
+
+void *user_aligned_alloc(ThreadState *thr, uptr pc, uptr align, uptr sz) {
+  if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(align, sz))) {
+    errno = errno_EINVAL;
+    if (AllocatorMayReturnNull())
+      return nullptr;
+    GET_STACK_TRACE_FATAL(thr, pc);
+    ReportInvalidAlignedAllocAlignment(sz, align, &stack);
+  }
+  return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, align));
+}
+
+void *user_valloc(ThreadState *thr, uptr pc, uptr sz) {
+  return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, GetPageSizeCached()));
+}
+
+void *user_pvalloc(ThreadState *thr, uptr pc, uptr sz) {
+  uptr PageSize = GetPageSizeCached();
+  if (UNLIKELY(CheckForPvallocOverflow(sz, PageSize))) {
+    errno = errno_ENOMEM;
+    if (AllocatorMayReturnNull())
+      return nullptr;
+    GET_STACK_TRACE_FATAL(thr, pc);
+    ReportPvallocOverflow(sz, &stack);
+  }
+  // pvalloc(0) should allocate one page.
+  sz = sz ? RoundUpTo(sz, PageSize) : PageSize;
+  return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, PageSize));
+}
+
+uptr user_alloc_usable_size(const void *p) {
+  if (p == 0)
+    return 0;
+  MBlock *b = ctx->metamap.GetBlock((uptr)p);
+  if (!b)
+    return 0;  // Not a valid pointer.
+  if (b->siz == 0)
+    return 1;  // Zero-sized allocations are actually 1 byte.
+  return b->siz;
+}
+
+void invoke_malloc_hook(void *ptr, uptr size) {
+  ThreadState *thr = cur_thread();
+  if (ctx == 0 || !ctx->initialized || thr->ignore_interceptors)
+    return;
+  __sanitizer_malloc_hook(ptr, size);
+  RunMallocHooks(ptr, size);
+}
+
+void invoke_free_hook(void *ptr) {
+  ThreadState *thr = cur_thread();
+  if (ctx == 0 || !ctx->initialized || thr->ignore_interceptors)
+    return;
+  __sanitizer_free_hook(ptr);
+  RunFreeHooks(ptr);
+}
+
+void *Alloc(uptr sz) {
+  ThreadState *thr = cur_thread();
+  if (thr->nomalloc) {
+    thr->nomalloc = 0;  // CHECK calls internal_malloc().
+    CHECK(0);
+  }
+  InternalAllocAccess();
+  return InternalAlloc(sz, &thr->proc()->internal_alloc_cache);
+}
+
+void FreeImpl(void *p) {
+  ThreadState *thr = cur_thread();
+  if (thr->nomalloc) {
+    thr->nomalloc = 0;  // CHECK calls internal_malloc().
+    CHECK(0);
+  }
+  InternalAllocAccess();
+  InternalFree(p, &thr->proc()->internal_alloc_cache);
+}
+
+}  // namespace __tsan
+
+using namespace __tsan;
+
+extern "C" {
+uptr __sanitizer_get_current_allocated_bytes() {
+  uptr stats[AllocatorStatCount];
+  allocator()->GetStats(stats);
+  return stats[AllocatorStatAllocated];
+}
+
+uptr __sanitizer_get_heap_size() {
+  uptr stats[AllocatorStatCount];
+  allocator()->GetStats(stats);
+  return stats[AllocatorStatMapped];
+}
+
+uptr __sanitizer_get_free_bytes() {
+  return 1;
+}
+
+uptr __sanitizer_get_unmapped_bytes() {
+  return 1;
+}
+
+uptr __sanitizer_get_estimated_allocated_size(uptr size) {
+  return size;
+}
+
+int __sanitizer_get_ownership(const void *p) {
+  return allocator()->GetBlockBegin(p) != 0;
+}
+
+uptr __sanitizer_get_allocated_size(const void *p) {
+  return user_alloc_usable_size(p);
+}
+
+void __tsan_on_thread_idle() {
+  ThreadState *thr = cur_thread();
+  thr->clock.ResetCached(&thr->proc()->clock_cache);
+  thr->last_sleep_clock.ResetCached(&thr->proc()->clock_cache);
+  allocator()->SwallowCache(&thr->proc()->alloc_cache);
+  internal_allocator()->SwallowCache(&thr->proc()->internal_alloc_cache);
+  ctx->metamap.OnProcIdle(thr->proc());
+}
+}  // extern "C"
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_mman.h b/compiler-rt/lib/tsan/rtl-old/tsan_mman.h
new file mode 100644
index 000000000000..db8488eabbe2
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_mman.h
@@ -0,0 +1,78 @@
+//===-- tsan_mman.h ---------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_MMAN_H
+#define TSAN_MMAN_H
+
+#include "tsan_defs.h"
+
+namespace __tsan {
+
+const uptr kDefaultAlignment = 16;
+
+void InitializeAllocator();
+void InitializeAllocatorLate();
+void ReplaceSystemMalloc();
+void AllocatorProcStart(Processor *proc);
+void AllocatorProcFinish(Processor *proc);
+void AllocatorPrintStats();
+void AllocatorLock();
+void AllocatorUnlock();
+
+// For user allocations.
+void *user_alloc_internal(ThreadState *thr, uptr pc, uptr sz,
+                          uptr align = kDefaultAlignment, bool signal = true);
+// Does not accept NULL.
+void user_free(ThreadState *thr, uptr pc, void *p, bool signal = true);
+// Interceptor implementations.
+void *user_alloc(ThreadState *thr, uptr pc, uptr sz);
+void *user_calloc(ThreadState *thr, uptr pc, uptr sz, uptr n);
+void *user_realloc(ThreadState *thr, uptr pc, void *p, uptr sz);
+void *user_reallocarray(ThreadState *thr, uptr pc, void *p, uptr sz, uptr n);
+void *user_memalign(ThreadState *thr, uptr pc, uptr align, uptr sz);
+int user_posix_memalign(ThreadState *thr, uptr pc, void **memptr, uptr align,
+                        uptr sz);
+void *user_aligned_alloc(ThreadState *thr, uptr pc, uptr align, uptr sz);
+void *user_valloc(ThreadState *thr, uptr pc, uptr sz);
+void *user_pvalloc(ThreadState *thr, uptr pc, uptr sz);
+uptr user_alloc_usable_size(const void *p);
+
+// Invoking malloc/free hooks that may be installed by the user.
+void invoke_malloc_hook(void *ptr, uptr size);
+void invoke_free_hook(void *ptr);
+
+// For internal data structures.
+void *Alloc(uptr sz);
+void FreeImpl(void *p);
+
+template <typename T, typename... Args>
+T *New(Args &&...args) {
+  return new (Alloc(sizeof(T))) T(static_cast<Args &&>(args)...);
+}
+
+template <typename T>
+void Free(T *&p) {
+  if (p == nullptr)
+    return;
+  FreeImpl(p);
+  p = nullptr;
+}
+
+template <typename T>
+void DestroyAndFree(T *&p) {
+  if (p == nullptr)
+    return;
+  p->~T();
+  Free(p);
+}
+
+}  // namespace __tsan
+#endif  // TSAN_MMAN_H
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_mutexset.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_mutexset.cpp
new file mode 100644
index 000000000000..735179686ba9
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_mutexset.cpp
@@ -0,0 +1,132 @@
+//===-- tsan_mutexset.cpp -------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#include "tsan_mutexset.h"
+
+#include "sanitizer_common/sanitizer_placement_new.h"
+#include "tsan_rtl.h"
+
+namespace __tsan {
+
+MutexSet::MutexSet() {
+}
+
+void MutexSet::Add(u64 id, bool write, u64 epoch) {
+  // Look up existing mutex with the same id.
+  for (uptr i = 0; i < size_; i++) {
+    if (descs_[i].id == id) {
+      descs_[i].count++;
+      descs_[i].epoch = epoch;
+      return;
+    }
+  }
+  // On overflow, find the oldest mutex and drop it.
+  if (size_ == kMaxSize) {
+    u64 minepoch = (u64)-1;
+    u64 mini = (u64)-1;
+    for (uptr i = 0; i < size_; i++) {
+      if (descs_[i].epoch < minepoch) {
+        minepoch = descs_[i].epoch;
+        mini = i;
+      }
+    }
+    RemovePos(mini);
+    CHECK_EQ(size_, kMaxSize - 1);
+  }
+  // Add new mutex descriptor.
+  descs_[size_].addr = 0;
+  descs_[size_].stack_id = kInvalidStackID;
+  descs_[size_].id = id;
+  descs_[size_].write = write;
+  descs_[size_].epoch = epoch;
+  descs_[size_].seq = seq_++;
+  descs_[size_].count = 1;
+  size_++;
+}
+
+void MutexSet::Del(u64 id, bool write) {
+  for (uptr i = 0; i < size_; i++) {
+    if (descs_[i].id == id) {
+      if (--descs_[i].count == 0)
+        RemovePos(i);
+      return;
+    }
+  }
+}
+
+void MutexSet::Remove(u64 id) {
+  for (uptr i = 0; i < size_; i++) {
+    if (descs_[i].id == id) {
+      RemovePos(i);
+      return;
+    }
+  }
+}
+
+void MutexSet::AddAddr(uptr addr, StackID stack_id, bool write) {
+  // Look up existing mutex with the same id.
+  for (uptr i = 0; i < size_; i++) {
+    if (descs_[i].addr == addr) {
+      descs_[i].count++;
+      descs_[i].seq = seq_++;
+      return;
+    }
+  }
+  // On overflow, find the oldest mutex and drop it.
+  if (size_ == kMaxSize) {
+    uptr min = 0;
+    for (uptr i = 0; i < size_; i++) {
+      if (descs_[i].seq < descs_[min].seq)
+        min = i;
+    }
+    RemovePos(min);
+    CHECK_EQ(size_, kMaxSize - 1);
+  }
+  // Add new mutex descriptor.
+  descs_[size_].addr = addr;
+  descs_[size_].stack_id = stack_id;
+  descs_[size_].id = 0;
+  descs_[size_].write = write;
+  descs_[size_].epoch = 0;
+  descs_[size_].seq = seq_++;
+  descs_[size_].count = 1;
+  size_++;
+}
+
+void MutexSet::DelAddr(uptr addr, bool destroy) {
+  for (uptr i = 0; i < size_; i++) {
+    if (descs_[i].addr == addr) {
+      if (destroy || --descs_[i].count == 0)
+        RemovePos(i);
+      return;
+    }
+  }
+}
+
+void MutexSet::RemovePos(uptr i) {
+  CHECK_LT(i, size_);
+  descs_[i] = descs_[size_ - 1];
+  size_--;
+}
+
+uptr MutexSet::Size() const {
+  return size_;
+}
+
+MutexSet::Desc MutexSet::Get(uptr i) const {
+  CHECK_LT(i, size_);
+  return descs_[i];
+}
+
+DynamicMutexSet::DynamicMutexSet() : ptr_(New<MutexSet>()) {}
+DynamicMutexSet::~DynamicMutexSet() { DestroyAndFree(ptr_); }
+
+}  // namespace __tsan
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_mutexset.h b/compiler-rt/lib/tsan/rtl-old/tsan_mutexset.h
new file mode 100644
index 000000000000..93776a664135
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_mutexset.h
@@ -0,0 +1,98 @@
+//===-- tsan_mutexset.h -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// MutexSet holds the set of mutexes currently held by a thread.
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_MUTEXSET_H
+#define TSAN_MUTEXSET_H
+
+#include "tsan_defs.h"
+
+namespace __tsan {
+
+class MutexSet {
+ public:
+  // Holds limited number of mutexes.
+  // The oldest mutexes are discarded on overflow.
+  static constexpr uptr kMaxSize = 16;
+  struct Desc {
+    uptr addr;
+    StackID stack_id;
+    u64 id;
+    u64 epoch;
+    u32 seq;
+    u32 count;
+    bool write;
+
+    Desc() { internal_memset(this, 0, sizeof(*this)); }
+    Desc(const Desc& other) { *this = other; }
+    Desc& operator=(const MutexSet::Desc& other) {
+      internal_memcpy(this, &other, sizeof(*this));
+      return *this;
+    }
+  };
+
+  MutexSet();
+  // The 'id' is obtained from SyncVar::GetId().
+  void Add(u64 id, bool write, u64 epoch);
+  void Del(u64 id, bool write);
+  void Remove(u64 id);  // Removes the mutex completely (if it's destroyed).
+  void AddAddr(uptr addr, StackID stack_id, bool write);
+  void DelAddr(uptr addr, bool destroy = false);
+  uptr Size() const;
+  Desc Get(uptr i) const;
+
+ private:
+#if !SANITIZER_GO
+  u32 seq_ = 0;
+  uptr size_ = 0;
+  Desc descs_[kMaxSize];
+
+  void RemovePos(uptr i);
+#endif
+};
+
+// MutexSet is too large to live on stack.
+// DynamicMutexSet can be use used to create local MutexSet's.
+class DynamicMutexSet {
+ public:
+  DynamicMutexSet();
+  ~DynamicMutexSet();
+  MutexSet* operator->() { return ptr_; }
+  operator MutexSet*() { return ptr_; }
+  DynamicMutexSet(const DynamicMutexSet&) = delete;
+  DynamicMutexSet& operator=(const DynamicMutexSet&) = delete;
+
+ private:
+  MutexSet* ptr_;
+#if SANITIZER_GO
+  MutexSet set_;
+#endif
+};
+
+// Go does not have mutexes, so do not spend memory and time.
+// (Go sync.Mutex is actually a semaphore -- can be unlocked
+// in different goroutine).
+#if SANITIZER_GO
+MutexSet::MutexSet() {}
+void MutexSet::Add(u64 id, bool write, u64 epoch) {}
+void MutexSet::Del(u64 id, bool write) {}
+void MutexSet::Remove(u64 id) {}
+void MutexSet::AddAddr(uptr addr, StackID stack_id, bool write) {}
+void MutexSet::DelAddr(uptr addr, bool destroy) {}
+uptr MutexSet::Size() const { return 0; }
+MutexSet::Desc MutexSet::Get(uptr i) const { return Desc(); }
+DynamicMutexSet::DynamicMutexSet() : ptr_(&set_) {}
+DynamicMutexSet::~DynamicMutexSet() {}
+#endif
+
+}  // namespace __tsan
+
+#endif  // TSAN_MUTEXSET_H
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_new_delete.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_new_delete.cpp
new file mode 100644
index 000000000000..fc44a5221b5b
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_new_delete.cpp
@@ -0,0 +1,199 @@
+//===-- tsan_new_delete.cpp ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// Interceptors for operators new and delete.
+//===----------------------------------------------------------------------===//
+#include "interception/interception.h"
+#include "sanitizer_common/sanitizer_allocator.h"
+#include "sanitizer_common/sanitizer_allocator_report.h"
+#include "sanitizer_common/sanitizer_internal_defs.h"
+#include "tsan_interceptors.h"
+#include "tsan_rtl.h"
+
+using namespace __tsan;
+
+namespace std {
+struct nothrow_t {};
+enum class align_val_t: __sanitizer::uptr {};
+}  // namespace std
+
+DECLARE_REAL(void *, malloc, uptr size)
+DECLARE_REAL(void, free, void *ptr)
+
+// TODO(alekseys): throw std::bad_alloc instead of dying on OOM.
+#define OPERATOR_NEW_BODY(mangled_name, nothrow) \
+  if (in_symbolizer()) \
+    return InternalAlloc(size); \
+  void *p = 0; \
+  {  \
+    SCOPED_INTERCEPTOR_RAW(mangled_name, size); \
+    p = user_alloc(thr, pc, size); \
+    if (!nothrow && UNLIKELY(!p)) { \
+      GET_STACK_TRACE_FATAL(thr, pc); \
+      ReportOutOfMemory(size, &stack); \
+    } \
+  }  \
+  invoke_malloc_hook(p, size);  \
+  return p;
+
+#define OPERATOR_NEW_BODY_ALIGN(mangled_name, nothrow) \
+  if (in_symbolizer()) \
+    return InternalAlloc(size, nullptr, (uptr)align); \
+  void *p = 0; \
+  {  \
+    SCOPED_INTERCEPTOR_RAW(mangled_name, size); \
+    p = user_memalign(thr, pc, (uptr)align, size); \
+    if (!nothrow && UNLIKELY(!p)) { \
+      GET_STACK_TRACE_FATAL(thr, pc); \
+      ReportOutOfMemory(size, &stack); \
+    } \
+  }  \
+  invoke_malloc_hook(p, size);  \
+  return p;
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void *operator new(__sanitizer::uptr size);
+void *operator new(__sanitizer::uptr size) {
+  OPERATOR_NEW_BODY(_Znwm, false /*nothrow*/);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void *operator new[](__sanitizer::uptr size);
+void *operator new[](__sanitizer::uptr size) {
+  OPERATOR_NEW_BODY(_Znam, false /*nothrow*/);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void *operator new(__sanitizer::uptr size, std::nothrow_t const&);
+void *operator new(__sanitizer::uptr size, std::nothrow_t const&) {
+  OPERATOR_NEW_BODY(_ZnwmRKSt9nothrow_t, true /*nothrow*/);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void *operator new[](__sanitizer::uptr size, std::nothrow_t const&);
+void *operator new[](__sanitizer::uptr size, std::nothrow_t const&) {
+  OPERATOR_NEW_BODY(_ZnamRKSt9nothrow_t, true /*nothrow*/);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void *operator new(__sanitizer::uptr size, std::align_val_t align);
+void *operator new(__sanitizer::uptr size, std::align_val_t align) {
+  OPERATOR_NEW_BODY_ALIGN(_ZnwmSt11align_val_t, false /*nothrow*/);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void *operator new[](__sanitizer::uptr size, std::align_val_t align);
+void *operator new[](__sanitizer::uptr size, std::align_val_t align) {
+  OPERATOR_NEW_BODY_ALIGN(_ZnamSt11align_val_t, false /*nothrow*/);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void *operator new(__sanitizer::uptr size, std::align_val_t align,
+                   std::nothrow_t const&);
+void *operator new(__sanitizer::uptr size, std::align_val_t align,
+                   std::nothrow_t const&) {
+  OPERATOR_NEW_BODY_ALIGN(_ZnwmSt11align_val_tRKSt9nothrow_t,
+                          true /*nothrow*/);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void *operator new[](__sanitizer::uptr size, std::align_val_t align,
+                     std::nothrow_t const&);
+void *operator new[](__sanitizer::uptr size, std::align_val_t align,
+                     std::nothrow_t const&) {
+  OPERATOR_NEW_BODY_ALIGN(_ZnamSt11align_val_tRKSt9nothrow_t,
+                          true /*nothrow*/);
+}
+
+#define OPERATOR_DELETE_BODY(mangled_name) \
+  if (ptr == 0) return;  \
+  if (in_symbolizer()) \
+    return InternalFree(ptr); \
+  invoke_free_hook(ptr);  \
+  SCOPED_INTERCEPTOR_RAW(mangled_name, ptr);  \
+  user_free(thr, pc, ptr);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void operator delete(void *ptr) NOEXCEPT;
+void operator delete(void *ptr) NOEXCEPT {
+  OPERATOR_DELETE_BODY(_ZdlPv);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void operator delete[](void *ptr) NOEXCEPT;
+void operator delete[](void *ptr) NOEXCEPT {
+  OPERATOR_DELETE_BODY(_ZdaPv);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void operator delete(void *ptr, std::nothrow_t const&);
+void operator delete(void *ptr, std::nothrow_t const&) {
+  OPERATOR_DELETE_BODY(_ZdlPvRKSt9nothrow_t);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void operator delete[](void *ptr, std::nothrow_t const&);
+void operator delete[](void *ptr, std::nothrow_t const&) {
+  OPERATOR_DELETE_BODY(_ZdaPvRKSt9nothrow_t);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void operator delete(void *ptr, __sanitizer::uptr size) NOEXCEPT;
+void operator delete(void *ptr, __sanitizer::uptr size) NOEXCEPT {
+  OPERATOR_DELETE_BODY(_ZdlPvm);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void operator delete[](void *ptr, __sanitizer::uptr size) NOEXCEPT;
+void operator delete[](void *ptr, __sanitizer::uptr size) NOEXCEPT {
+  OPERATOR_DELETE_BODY(_ZdaPvm);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void operator delete(void *ptr, std::align_val_t align) NOEXCEPT;
+void operator delete(void *ptr, std::align_val_t align) NOEXCEPT {
+  OPERATOR_DELETE_BODY(_ZdlPvSt11align_val_t);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void operator delete[](void *ptr, std::align_val_t align) NOEXCEPT;
+void operator delete[](void *ptr, std::align_val_t align) NOEXCEPT {
+  OPERATOR_DELETE_BODY(_ZdaPvSt11align_val_t);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void operator delete(void *ptr, std::align_val_t align, std::nothrow_t const&);
+void operator delete(void *ptr, std::align_val_t align, std::nothrow_t const&) {
+  OPERATOR_DELETE_BODY(_ZdlPvSt11align_val_tRKSt9nothrow_t);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void operator delete[](void *ptr, std::align_val_t align,
+                       std::nothrow_t const&);
+void operator delete[](void *ptr, std::align_val_t align,
+                       std::nothrow_t const&) {
+  OPERATOR_DELETE_BODY(_ZdaPvSt11align_val_tRKSt9nothrow_t);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void operator delete(void *ptr, __sanitizer::uptr size,
+                     std::align_val_t align) NOEXCEPT;
+void operator delete(void *ptr, __sanitizer::uptr size,
+                     std::align_val_t align) NOEXCEPT {
+  OPERATOR_DELETE_BODY(_ZdlPvmSt11align_val_t);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void operator delete[](void *ptr, __sanitizer::uptr size,
+                       std::align_val_t align) NOEXCEPT;
+void operator delete[](void *ptr, __sanitizer::uptr size,
+                       std::align_val_t align) NOEXCEPT {
+  OPERATOR_DELETE_BODY(_ZdaPvmSt11align_val_t);
+}
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_platform.h b/compiler-rt/lib/tsan/rtl-old/tsan_platform.h
new file mode 100644
index 000000000000..7ff0acace8f6
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_platform.h
@@ -0,0 +1,988 @@
+//===-- tsan_platform.h -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// Platform-specific code.
+//===----------------------------------------------------------------------===//
+
+#ifndef TSAN_PLATFORM_H
+#define TSAN_PLATFORM_H
+
+#if !defined(__LP64__) && !defined(_WIN64)
+# error "Only 64-bit is supported"
+#endif
+
+#include "tsan_defs.h"
+#include "tsan_trace.h"
+
+namespace __tsan {
+
+enum {
+  // App memory is not mapped onto shadow memory range.
+  kBrokenMapping = 1 << 0,
+  // Mapping app memory and back does not produce the same address,
+  // this can lead to wrong addresses in reports and potentially
+  // other bad consequences.
+  kBrokenReverseMapping = 1 << 1,
+  // Mapping is non-linear for linear user range.
+  // This is bad and can lead to unpredictable memory corruptions, etc
+  // because range access functions assume linearity.
+  kBrokenLinearity = 1 << 2,
+};
+
+/*
+C/C++ on linux/x86_64 and freebsd/x86_64
+0000 0000 1000 - 0080 0000 0000: main binary and/or MAP_32BIT mappings (512GB)
+0040 0000 0000 - 0100 0000 0000: -
+0100 0000 0000 - 2000 0000 0000: shadow
+2000 0000 0000 - 3000 0000 0000: -
+3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
+4000 0000 0000 - 5500 0000 0000: -
+5500 0000 0000 - 5680 0000 0000: pie binaries without ASLR or on 4.1+ kernels
+5680 0000 0000 - 6000 0000 0000: -
+6000 0000 0000 - 6200 0000 0000: traces
+6200 0000 0000 - 7d00 0000 0000: -
+7b00 0000 0000 - 7c00 0000 0000: heap
+7c00 0000 0000 - 7e80 0000 0000: -
+7e80 0000 0000 - 8000 0000 0000: modules and main thread stack
+
+C/C++ on netbsd/amd64 can reuse the same mapping:
+ * The address space starts from 0x1000 (option with 0x0) and ends with
+   0x7f7ffffff000.
+ * LoAppMem-kHeapMemEnd can be reused as it is.
+ * No VDSO support.
+ * No MidAppMem region.
+ * No additional HeapMem region.
+ * HiAppMem contains the stack, loader, shared libraries and heap.
+ * Stack on NetBSD/amd64 has prereserved 128MB.
+ * Heap grows downwards (top-down).
+ * ASLR must be disabled per-process or globally.
+*/
+struct Mapping48AddressSpace {
+  static const uptr kMetaShadowBeg = 0x300000000000ull;
+  static const uptr kMetaShadowEnd = 0x340000000000ull;
+  static const uptr kTraceMemBeg   = 0x600000000000ull;
+  static const uptr kTraceMemEnd   = 0x620000000000ull;
+  static const uptr kShadowBeg     = 0x010000000000ull;
+  static const uptr kShadowEnd     = 0x200000000000ull;
+  static const uptr kHeapMemBeg    = 0x7b0000000000ull;
+  static const uptr kHeapMemEnd    = 0x7c0000000000ull;
+  static const uptr kLoAppMemBeg   = 0x000000001000ull;
+  static const uptr kLoAppMemEnd   = 0x008000000000ull;
+  static const uptr kMidAppMemBeg  = 0x550000000000ull;
+  static const uptr kMidAppMemEnd  = 0x568000000000ull;
+  static const uptr kHiAppMemBeg   = 0x7e8000000000ull;
+  static const uptr kHiAppMemEnd   = 0x800000000000ull;
+  static const uptr kShadowMsk = 0x780000000000ull;
+  static const uptr kShadowXor = 0x040000000000ull;
+  static const uptr kShadowAdd = 0x000000000000ull;
+  static const uptr kVdsoBeg       = 0xf000000000000000ull;
+};
+
+/*
+C/C++ on linux/mips64 (40-bit VMA)
+0000 0000 00 - 0100 0000 00: -                                           (4 GB)
+0100 0000 00 - 0200 0000 00: main binary                                 (4 GB)
+0200 0000 00 - 2000 0000 00: -                                         (120 GB)
+2000 0000 00 - 4000 0000 00: shadow                                    (128 GB)
+4000 0000 00 - 5000 0000 00: metainfo (memory blocks and sync objects)  (64 GB)
+5000 0000 00 - aa00 0000 00: -                                         (360 GB)
+aa00 0000 00 - ab00 0000 00: main binary (PIE)                           (4 GB)
+ab00 0000 00 - b000 0000 00: -                                          (20 GB)
+b000 0000 00 - b200 0000 00: traces                                      (8 GB)
+b200 0000 00 - fe00 0000 00: -                                         (304 GB)
+fe00 0000 00 - ff00 0000 00: heap                                        (4 GB)
+ff00 0000 00 - ff80 0000 00: -                                           (2 GB)
+ff80 0000 00 - ffff ffff ff: modules and main thread stack              (<2 GB)
+*/
+struct MappingMips64_40 {
+  static const uptr kMetaShadowBeg = 0x4000000000ull;
+  static const uptr kMetaShadowEnd = 0x5000000000ull;
+  static const uptr kTraceMemBeg   = 0xb000000000ull;
+  static const uptr kTraceMemEnd   = 0xb200000000ull;
+  static const uptr kShadowBeg     = 0x2000000000ull;
+  static const uptr kShadowEnd     = 0x4000000000ull;
+  static const uptr kHeapMemBeg    = 0xfe00000000ull;
+  static const uptr kHeapMemEnd    = 0xff00000000ull;
+  static const uptr kLoAppMemBeg   = 0x0100000000ull;
+  static const uptr kLoAppMemEnd   = 0x0200000000ull;
+  static const uptr kMidAppMemBeg  = 0xaa00000000ull;
+  static const uptr kMidAppMemEnd  = 0xab00000000ull;
+  static const uptr kHiAppMemBeg   = 0xff80000000ull;
+  static const uptr kHiAppMemEnd   = 0xffffffffffull;
+  static const uptr kShadowMsk = 0xf800000000ull;
+  static const uptr kShadowXor = 0x0800000000ull;
+  static const uptr kShadowAdd = 0x0000000000ull;
+  static const uptr kVdsoBeg       = 0xfffff00000ull;
+};
+
+/*
+C/C++ on Darwin/iOS/ARM64 (36-bit VMA, 64 GB VM)
+0000 0000 00 - 0100 0000 00: -                                    (4 GB)
+0100 0000 00 - 0200 0000 00: main binary, modules, thread stacks  (4 GB)
+0200 0000 00 - 0300 0000 00: heap                                 (4 GB)
+0300 0000 00 - 0400 0000 00: -                                    (4 GB)
+0400 0000 00 - 0c00 0000 00: shadow memory                       (32 GB)
+0c00 0000 00 - 0d00 0000 00: -                                    (4 GB)
+0d00 0000 00 - 0e00 0000 00: metainfo                             (4 GB)
+0e00 0000 00 - 0f00 0000 00: -                                    (4 GB)
+0f00 0000 00 - 0fc0 0000 00: traces                               (3 GB)
+0fc0 0000 00 - 1000 0000 00: -
+*/
+struct MappingAppleAarch64 {
+  static const uptr kLoAppMemBeg   = 0x0100000000ull;
+  static const uptr kLoAppMemEnd   = 0x0200000000ull;
+  static const uptr kHeapMemBeg    = 0x0200000000ull;
+  static const uptr kHeapMemEnd    = 0x0300000000ull;
+  static const uptr kShadowBeg     = 0x0400000000ull;
+  static const uptr kShadowEnd     = 0x0c00000000ull;
+  static const uptr kMetaShadowBeg = 0x0d00000000ull;
+  static const uptr kMetaShadowEnd = 0x0e00000000ull;
+  static const uptr kTraceMemBeg   = 0x0f00000000ull;
+  static const uptr kTraceMemEnd   = 0x0fc0000000ull;
+  static const uptr kHiAppMemBeg   = 0x0fc0000000ull;
+  static const uptr kHiAppMemEnd   = 0x0fc0000000ull;
+  static const uptr kShadowMsk = 0x0ull;
+  static const uptr kShadowXor = 0x0ull;
+  static const uptr kShadowAdd = 0x0ull;
+  static const uptr kVdsoBeg       = 0x7000000000000000ull;
+  static const uptr kMidAppMemBeg = 0;
+  static const uptr kMidAppMemEnd = 0;
+};
+
+/*
+C/C++ on linux/aarch64 (39-bit VMA)
+0000 0010 00 - 0100 0000 00: main binary
+0100 0000 00 - 0800 0000 00: -
+0800 0000 00 - 2000 0000 00: shadow memory
+2000 0000 00 - 3100 0000 00: -
+3100 0000 00 - 3400 0000 00: metainfo
+3400 0000 00 - 5500 0000 00: -
+5500 0000 00 - 5600 0000 00: main binary (PIE)
+5600 0000 00 - 6000 0000 00: -
+6000 0000 00 - 6200 0000 00: traces
+6200 0000 00 - 7d00 0000 00: -
+7c00 0000 00 - 7d00 0000 00: heap
+7d00 0000 00 - 7fff ffff ff: modules and main thread stack
+*/
+struct MappingAarch64_39 {
+  static const uptr kLoAppMemBeg   = 0x0000001000ull;
+  static const uptr kLoAppMemEnd   = 0x0100000000ull;
+  static const uptr kShadowBeg     = 0x0800000000ull;
+  static const uptr kShadowEnd     = 0x2000000000ull;
+  static const uptr kMetaShadowBeg = 0x3100000000ull;
+  static const uptr kMetaShadowEnd = 0x3400000000ull;
+  static const uptr kMidAppMemBeg  = 0x5500000000ull;
+  static const uptr kMidAppMemEnd  = 0x5600000000ull;
+  static const uptr kTraceMemBeg   = 0x6000000000ull;
+  static const uptr kTraceMemEnd   = 0x6200000000ull;
+  static const uptr kHeapMemBeg    = 0x7c00000000ull;
+  static const uptr kHeapMemEnd    = 0x7d00000000ull;
+  static const uptr kHiAppMemBeg   = 0x7e00000000ull;
+  static const uptr kHiAppMemEnd   = 0x7fffffffffull;
+  static const uptr kShadowMsk = 0x7800000000ull;
+  static const uptr kShadowXor = 0x0200000000ull;
+  static const uptr kShadowAdd = 0x0000000000ull;
+  static const uptr kVdsoBeg       = 0x7f00000000ull;
+};
+
+/*
+C/C++ on linux/aarch64 (42-bit VMA)
+00000 0010 00 - 01000 0000 00: main binary
+01000 0000 00 - 10000 0000 00: -
+10000 0000 00 - 20000 0000 00: shadow memory
+20000 0000 00 - 26000 0000 00: -
+26000 0000 00 - 28000 0000 00: metainfo
+28000 0000 00 - 2aa00 0000 00: -
+2aa00 0000 00 - 2ab00 0000 00: main binary (PIE)
+2ab00 0000 00 - 36200 0000 00: -
+36200 0000 00 - 36240 0000 00: traces
+36240 0000 00 - 3e000 0000 00: -
+3e000 0000 00 - 3f000 0000 00: heap
+3f000 0000 00 - 3ffff ffff ff: modules and main thread stack
+*/
+struct MappingAarch64_42 {
+  static const uptr kBroken = kBrokenReverseMapping;
+  static const uptr kLoAppMemBeg   = 0x00000001000ull;
+  static const uptr kLoAppMemEnd   = 0x01000000000ull;
+  static const uptr kShadowBeg     = 0x10000000000ull;
+  static const uptr kShadowEnd     = 0x20000000000ull;
+  static const uptr kMetaShadowBeg = 0x26000000000ull;
+  static const uptr kMetaShadowEnd = 0x28000000000ull;
+  static const uptr kMidAppMemBeg  = 0x2aa00000000ull;
+  static const uptr kMidAppMemEnd  = 0x2ab00000000ull;
+  static const uptr kTraceMemBeg   = 0x36200000000ull;
+  static const uptr kTraceMemEnd   = 0x36400000000ull;
+  static const uptr kHeapMemBeg    = 0x3e000000000ull;
+  static const uptr kHeapMemEnd    = 0x3f000000000ull;
+  static const uptr kHiAppMemBeg   = 0x3f000000000ull;
+  static const uptr kHiAppMemEnd   = 0x3ffffffffffull;
+  static const uptr kShadowMsk = 0x3c000000000ull;
+  static const uptr kShadowXor = 0x04000000000ull;
+  static const uptr kShadowAdd = 0x00000000000ull;
+  static const uptr kVdsoBeg       = 0x37f00000000ull;
+};
+
+struct MappingAarch64_48 {
+  static const uptr kLoAppMemBeg   = 0x0000000001000ull;
+  static const uptr kLoAppMemEnd   = 0x0000200000000ull;
+  static const uptr kShadowBeg     = 0x0002000000000ull;
+  static const uptr kShadowEnd     = 0x0004000000000ull;
+  static const uptr kMetaShadowBeg = 0x0005000000000ull;
+  static const uptr kMetaShadowEnd = 0x0006000000000ull;
+  static const uptr kMidAppMemBeg  = 0x0aaaa00000000ull;
+  static const uptr kMidAppMemEnd  = 0x0aaaf00000000ull;
+  static const uptr kTraceMemBeg   = 0x0f06000000000ull;
+  static const uptr kTraceMemEnd   = 0x0f06200000000ull;
+  static const uptr kHeapMemBeg    = 0x0ffff00000000ull;
+  static const uptr kHeapMemEnd    = 0x0ffff00000000ull;
+  static const uptr kHiAppMemBeg   = 0x0ffff00000000ull;
+  static const uptr kHiAppMemEnd   = 0x1000000000000ull;
+  static const uptr kShadowMsk = 0x0fff800000000ull;
+  static const uptr kShadowXor = 0x0000800000000ull;
+  static const uptr kShadowAdd = 0x0000000000000ull;
+  static const uptr kVdsoBeg       = 0xffff000000000ull;
+};
+
+/*
+C/C++ on linux/powerpc64 (44-bit VMA)
+0000 0000 0100 - 0001 0000 0000: main binary
+0001 0000 0000 - 0001 0000 0000: -
+0001 0000 0000 - 0b00 0000 0000: shadow
+0b00 0000 0000 - 0b00 0000 0000: -
+0b00 0000 0000 - 0d00 0000 0000: metainfo (memory blocks and sync objects)
+0d00 0000 0000 - 0d00 0000 0000: -
+0d00 0000 0000 - 0f00 0000 0000: traces
+0f00 0000 0000 - 0f00 0000 0000: -
+0f00 0000 0000 - 0f50 0000 0000: heap
+0f50 0000 0000 - 0f60 0000 0000: -
+0f60 0000 0000 - 1000 0000 0000: modules and main thread stack
+*/
+struct MappingPPC64_44 {
+  static const uptr kBroken =
+      kBrokenMapping | kBrokenReverseMapping | kBrokenLinearity;
+  static const uptr kMetaShadowBeg = 0x0b0000000000ull;
+  static const uptr kMetaShadowEnd = 0x0d0000000000ull;
+  static const uptr kTraceMemBeg   = 0x0d0000000000ull;
+  static const uptr kTraceMemEnd   = 0x0f0000000000ull;
+  static const uptr kShadowBeg     = 0x000100000000ull;
+  static const uptr kShadowEnd     = 0x0b0000000000ull;
+  static const uptr kLoAppMemBeg   = 0x000000000100ull;
+  static const uptr kLoAppMemEnd   = 0x000100000000ull;
+  static const uptr kHeapMemBeg    = 0x0f0000000000ull;
+  static const uptr kHeapMemEnd    = 0x0f5000000000ull;
+  static const uptr kHiAppMemBeg   = 0x0f6000000000ull;
+  static const uptr kHiAppMemEnd   = 0x100000000000ull; // 44 bits
+  static const uptr kShadowMsk = 0x0f0000000000ull;
+  static const uptr kShadowXor = 0x002100000000ull;
+  static const uptr kShadowAdd = 0x000000000000ull;
+  static const uptr kVdsoBeg       = 0x3c0000000000000ull;
+  static const uptr kMidAppMemBeg = 0;
+  static const uptr kMidAppMemEnd = 0;
+};
+
+/*
+C/C++ on linux/powerpc64 (46-bit VMA)
+0000 0000 1000 - 0100 0000 0000: main binary
+0100 0000 0000 - 0200 0000 0000: -
+0100 0000 0000 - 1000 0000 0000: shadow
+1000 0000 0000 - 1000 0000 0000: -
+1000 0000 0000 - 2000 0000 0000: metainfo (memory blocks and sync objects)
+2000 0000 0000 - 2000 0000 0000: -
+2000 0000 0000 - 2200 0000 0000: traces
+2200 0000 0000 - 3d00 0000 0000: -
+3d00 0000 0000 - 3e00 0000 0000: heap
+3e00 0000 0000 - 3e80 0000 0000: -
+3e80 0000 0000 - 4000 0000 0000: modules and main thread stack
+*/
+struct MappingPPC64_46 {
+  static const uptr kMetaShadowBeg = 0x100000000000ull;
+  static const uptr kMetaShadowEnd = 0x200000000000ull;
+  static const uptr kTraceMemBeg   = 0x200000000000ull;
+  static const uptr kTraceMemEnd   = 0x220000000000ull;
+  static const uptr kShadowBeg     = 0x010000000000ull;
+  static const uptr kShadowEnd     = 0x100000000000ull;
+  static const uptr kHeapMemBeg    = 0x3d0000000000ull;
+  static const uptr kHeapMemEnd    = 0x3e0000000000ull;
+  static const uptr kLoAppMemBeg   = 0x000000001000ull;
+  static const uptr kLoAppMemEnd   = 0x010000000000ull;
+  static const uptr kHiAppMemBeg   = 0x3e8000000000ull;
+  static const uptr kHiAppMemEnd   = 0x400000000000ull; // 46 bits
+  static const uptr kShadowMsk = 0x3c0000000000ull;
+  static const uptr kShadowXor = 0x020000000000ull;
+  static const uptr kShadowAdd = 0x000000000000ull;
+  static const uptr kVdsoBeg       = 0x7800000000000000ull;
+  static const uptr kMidAppMemBeg = 0;
+  static const uptr kMidAppMemEnd = 0;
+};
+
+/*
+C/C++ on linux/powerpc64 (47-bit VMA)
+0000 0000 1000 - 0100 0000 0000: main binary
+0100 0000 0000 - 0200 0000 0000: -
+0100 0000 0000 - 1000 0000 0000: shadow
+1000 0000 0000 - 1000 0000 0000: -
+1000 0000 0000 - 2000 0000 0000: metainfo (memory blocks and sync objects)
+2000 0000 0000 - 2000 0000 0000: -
+2000 0000 0000 - 2200 0000 0000: traces
+2200 0000 0000 - 7d00 0000 0000: -
+7d00 0000 0000 - 7e00 0000 0000: heap
+7e00 0000 0000 - 7e80 0000 0000: -
+7e80 0000 0000 - 8000 0000 0000: modules and main thread stack
+*/
+struct MappingPPC64_47 {
+  static const uptr kMetaShadowBeg = 0x100000000000ull;
+  static const uptr kMetaShadowEnd = 0x200000000000ull;
+  static const uptr kTraceMemBeg   = 0x200000000000ull;
+  static const uptr kTraceMemEnd   = 0x220000000000ull;
+  static const uptr kShadowBeg     = 0x010000000000ull;
+  static const uptr kShadowEnd     = 0x100000000000ull;
+  static const uptr kHeapMemBeg    = 0x7d0000000000ull;
+  static const uptr kHeapMemEnd    = 0x7e0000000000ull;
+  static const uptr kLoAppMemBeg   = 0x000000001000ull;
+  static const uptr kLoAppMemEnd   = 0x010000000000ull;
+  static const uptr kHiAppMemBeg   = 0x7e8000000000ull;
+  static const uptr kHiAppMemEnd   = 0x800000000000ull; // 47 bits
+  static const uptr kShadowMsk = 0x7c0000000000ull;
+  static const uptr kShadowXor = 0x020000000000ull;
+  static const uptr kShadowAdd = 0x000000000000ull;
+  static const uptr kVdsoBeg       = 0x7800000000000000ull;
+  static const uptr kMidAppMemBeg = 0;
+  static const uptr kMidAppMemEnd = 0;
+};
+
+/*
+C/C++ on linux/s390x
+While the kernel provides a 64-bit address space, we have to restrict ourselves
+to 48 bits due to how e.g. SyncVar::GetId() works.
+0000 0000 1000 - 0e00 0000 0000: binary, modules, stacks - 14 TiB
+0e00 0000 0000 - 4000 0000 0000: -
+4000 0000 0000 - 8000 0000 0000: shadow - 64TiB (4 * app)
+8000 0000 0000 - 9000 0000 0000: -
+9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app)
+9800 0000 0000 - a000 0000 0000: -
+a000 0000 0000 - b000 0000 0000: traces - 16TiB (max history * 128k threads)
+b000 0000 0000 - be00 0000 0000: -
+be00 0000 0000 - c000 0000 0000: heap - 2TiB (max supported by the allocator)
+*/
+struct MappingS390x {
+  static const uptr kMetaShadowBeg = 0x900000000000ull;
+  static const uptr kMetaShadowEnd = 0x980000000000ull;
+  static const uptr kTraceMemBeg   = 0xa00000000000ull;
+  static const uptr kTraceMemEnd   = 0xb00000000000ull;
+  static const uptr kShadowBeg     = 0x400000000000ull;
+  static const uptr kShadowEnd     = 0x800000000000ull;
+  static const uptr kHeapMemBeg    = 0xbe0000000000ull;
+  static const uptr kHeapMemEnd    = 0xc00000000000ull;
+  static const uptr kLoAppMemBeg   = 0x000000001000ull;
+  static const uptr kLoAppMemEnd   = 0x0e0000000000ull;
+  static const uptr kHiAppMemBeg   = 0xc00000004000ull;
+  static const uptr kHiAppMemEnd   = 0xc00000004000ull;
+  static const uptr kShadowMsk = 0xb00000000000ull;
+  static const uptr kShadowXor = 0x100000000000ull;
+  static const uptr kShadowAdd = 0x000000000000ull;
+  static const uptr kVdsoBeg       = 0xfffffffff000ull;
+  static const uptr kMidAppMemBeg = 0;
+  static const uptr kMidAppMemEnd = 0;
+};
+
+/* Go on linux, darwin and freebsd on x86_64
+0000 0000 1000 - 0000 1000 0000: executable
+0000 1000 0000 - 00c0 0000 0000: -
+00c0 0000 0000 - 00e0 0000 0000: heap
+00e0 0000 0000 - 2000 0000 0000: -
+2000 0000 0000 - 2380 0000 0000: shadow
+2380 0000 0000 - 3000 0000 0000: -
+3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
+4000 0000 0000 - 6000 0000 0000: -
+6000 0000 0000 - 6200 0000 0000: traces
+6200 0000 0000 - 8000 0000 0000: -
+*/
+
+struct MappingGo48 {
+  static const uptr kMetaShadowBeg = 0x300000000000ull;
+  static const uptr kMetaShadowEnd = 0x400000000000ull;
+  static const uptr kTraceMemBeg   = 0x600000000000ull;
+  static const uptr kTraceMemEnd   = 0x620000000000ull;
+  static const uptr kShadowBeg     = 0x200000000000ull;
+  static const uptr kShadowEnd     = 0x238000000000ull;
+  static const uptr kLoAppMemBeg = 0x000000001000ull;
+  static const uptr kLoAppMemEnd = 0x00e000000000ull;
+  static const uptr kMidAppMemBeg = 0;
+  static const uptr kMidAppMemEnd = 0;
+  static const uptr kHiAppMemBeg = 0;
+  static const uptr kHiAppMemEnd = 0;
+  static const uptr kHeapMemBeg = 0;
+  static const uptr kHeapMemEnd = 0;
+  static const uptr kVdsoBeg = 0;
+  static const uptr kShadowMsk = 0;
+  static const uptr kShadowXor = 0;
+  static const uptr kShadowAdd = 0x200000000000ull;
+};
+
+/* Go on windows
+0000 0000 1000 - 0000 1000 0000: executable
+0000 1000 0000 - 00f8 0000 0000: -
+00c0 0000 0000 - 00e0 0000 0000: heap
+00e0 0000 0000 - 0100 0000 0000: -
+0100 0000 0000 - 0500 0000 0000: shadow
+0500 0000 0000 - 0700 0000 0000: traces
+0700 0000 0000 - 0770 0000 0000: metainfo (memory blocks and sync objects)
+07d0 0000 0000 - 8000 0000 0000: -
+*/
+
+struct MappingGoWindows {
+  static const uptr kMetaShadowBeg = 0x070000000000ull;
+  static const uptr kMetaShadowEnd = 0x077000000000ull;
+  static const uptr kTraceMemBeg = 0x050000000000ull;
+  static const uptr kTraceMemEnd = 0x070000000000ull;
+  static const uptr kShadowBeg     = 0x010000000000ull;
+  static const uptr kShadowEnd     = 0x050000000000ull;
+  static const uptr kLoAppMemBeg = 0x000000001000ull;
+  static const uptr kLoAppMemEnd = 0x00e000000000ull;
+  static const uptr kMidAppMemBeg = 0;
+  static const uptr kMidAppMemEnd = 0;
+  static const uptr kHiAppMemBeg = 0;
+  static const uptr kHiAppMemEnd = 0;
+  static const uptr kHeapMemBeg = 0;
+  static const uptr kHeapMemEnd = 0;
+  static const uptr kVdsoBeg = 0;
+  static const uptr kShadowMsk = 0;
+  static const uptr kShadowXor = 0;
+  static const uptr kShadowAdd = 0x010000000000ull;
+};
+
+/* Go on linux/powerpc64 (46-bit VMA)
+0000 0000 1000 - 0000 1000 0000: executable
+0000 1000 0000 - 00c0 0000 0000: -
+00c0 0000 0000 - 00e0 0000 0000: heap
+00e0 0000 0000 - 2000 0000 0000: -
+2000 0000 0000 - 2380 0000 0000: shadow
+2380 0000 0000 - 2400 0000 0000: -
+2400 0000 0000 - 3400 0000 0000: metainfo (memory blocks and sync objects)
+3400 0000 0000 - 3600 0000 0000: -
+3600 0000 0000 - 3800 0000 0000: traces
+3800 0000 0000 - 4000 0000 0000: -
+*/
+
+struct MappingGoPPC64_46 {
+  static const uptr kMetaShadowBeg = 0x240000000000ull;
+  static const uptr kMetaShadowEnd = 0x340000000000ull;
+  static const uptr kTraceMemBeg   = 0x360000000000ull;
+  static const uptr kTraceMemEnd   = 0x380000000000ull;
+  static const uptr kShadowBeg     = 0x200000000000ull;
+  static const uptr kShadowEnd     = 0x238000000000ull;
+  static const uptr kLoAppMemBeg = 0x000000001000ull;
+  static const uptr kLoAppMemEnd = 0x00e000000000ull;
+  static const uptr kMidAppMemBeg = 0;
+  static const uptr kMidAppMemEnd = 0;
+  static const uptr kHiAppMemBeg = 0;
+  static const uptr kHiAppMemEnd = 0;
+  static const uptr kHeapMemBeg = 0;
+  static const uptr kHeapMemEnd = 0;
+  static const uptr kVdsoBeg = 0;
+  static const uptr kShadowMsk = 0;
+  static const uptr kShadowXor = 0;
+  static const uptr kShadowAdd = 0x200000000000ull;
+};
+
+/* Go on linux/powerpc64 (47-bit VMA)
+0000 0000 1000 - 0000 1000 0000: executable
+0000 1000 0000 - 00c0 0000 0000: -
+00c0 0000 0000 - 00e0 0000 0000: heap
+00e0 0000 0000 - 2000 0000 0000: -
+2000 0000 0000 - 3000 0000 0000: shadow
+3000 0000 0000 - 3000 0000 0000: -
+3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
+4000 0000 0000 - 6000 0000 0000: -
+6000 0000 0000 - 6200 0000 0000: traces
+6200 0000 0000 - 8000 0000 0000: -
+*/
+
+struct MappingGoPPC64_47 {
+  static const uptr kMetaShadowBeg = 0x300000000000ull;
+  static const uptr kMetaShadowEnd = 0x400000000000ull;
+  static const uptr kTraceMemBeg   = 0x600000000000ull;
+  static const uptr kTraceMemEnd   = 0x620000000000ull;
+  static const uptr kShadowBeg     = 0x200000000000ull;
+  static const uptr kShadowEnd     = 0x300000000000ull;
+  static const uptr kLoAppMemBeg = 0x000000001000ull;
+  static const uptr kLoAppMemEnd = 0x00e000000000ull;
+  static const uptr kMidAppMemBeg = 0;
+  static const uptr kMidAppMemEnd = 0;
+  static const uptr kHiAppMemBeg = 0;
+  static const uptr kHiAppMemEnd = 0;
+  static const uptr kHeapMemBeg = 0;
+  static const uptr kHeapMemEnd = 0;
+  static const uptr kVdsoBeg = 0;
+  static const uptr kShadowMsk = 0;
+  static const uptr kShadowXor = 0;
+  static const uptr kShadowAdd = 0x200000000000ull;
+};
+
+/* Go on linux/aarch64 (48-bit VMA) and darwin/aarch64 (47-bit VMA)
+0000 0000 1000 - 0000 1000 0000: executable
+0000 1000 0000 - 00c0 0000 0000: -
+00c0 0000 0000 - 00e0 0000 0000: heap
+00e0 0000 0000 - 2000 0000 0000: -
+2000 0000 0000 - 3000 0000 0000: shadow
+3000 0000 0000 - 3000 0000 0000: -
+3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
+4000 0000 0000 - 6000 0000 0000: -
+6000 0000 0000 - 6200 0000 0000: traces
+6200 0000 0000 - 8000 0000 0000: -
+*/
+struct MappingGoAarch64 {
+  static const uptr kMetaShadowBeg = 0x300000000000ull;
+  static const uptr kMetaShadowEnd = 0x400000000000ull;
+  static const uptr kTraceMemBeg   = 0x600000000000ull;
+  static const uptr kTraceMemEnd   = 0x620000000000ull;
+  static const uptr kShadowBeg     = 0x200000000000ull;
+  static const uptr kShadowEnd     = 0x300000000000ull;
+  static const uptr kLoAppMemBeg = 0x000000001000ull;
+  static const uptr kLoAppMemEnd = 0x00e000000000ull;
+  static const uptr kMidAppMemBeg = 0;
+  static const uptr kMidAppMemEnd = 0;
+  static const uptr kHiAppMemBeg = 0;
+  static const uptr kHiAppMemEnd = 0;
+  static const uptr kHeapMemBeg = 0;
+  static const uptr kHeapMemEnd = 0;
+  static const uptr kVdsoBeg = 0;
+  static const uptr kShadowMsk = 0;
+  static const uptr kShadowXor = 0;
+  static const uptr kShadowAdd = 0x200000000000ull;
+};
+
+/*
+Go on linux/mips64 (47-bit VMA)
+0000 0000 1000 - 0000 1000 0000: executable
+0000 1000 0000 - 00c0 0000 0000: -
+00c0 0000 0000 - 00e0 0000 0000: heap
+00e0 0000 0000 - 2000 0000 0000: -
+2000 0000 0000 - 3000 0000 0000: shadow
+3000 0000 0000 - 3000 0000 0000: -
+3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
+4000 0000 0000 - 6000 0000 0000: -
+6000 0000 0000 - 6200 0000 0000: traces
+6200 0000 0000 - 8000 0000 0000: -
+*/
+struct MappingGoMips64_47 {
+  static const uptr kMetaShadowBeg = 0x300000000000ull;
+  static const uptr kMetaShadowEnd = 0x400000000000ull;
+  static const uptr kTraceMemBeg = 0x600000000000ull;
+  static const uptr kTraceMemEnd = 0x620000000000ull;
+  static const uptr kShadowBeg = 0x200000000000ull;
+  static const uptr kShadowEnd = 0x300000000000ull;
+  static const uptr kLoAppMemBeg = 0x000000001000ull;
+  static const uptr kLoAppMemEnd = 0x00e000000000ull;
+  static const uptr kMidAppMemBeg = 0;
+  static const uptr kMidAppMemEnd = 0;
+  static const uptr kHiAppMemBeg = 0;
+  static const uptr kHiAppMemEnd = 0;
+  static const uptr kHeapMemBeg = 0;
+  static const uptr kHeapMemEnd = 0;
+  static const uptr kVdsoBeg = 0;
+  static const uptr kShadowMsk = 0;
+  static const uptr kShadowXor = 0;
+  static const uptr kShadowAdd = 0x200000000000ull;
+};
+
+/*
+Go on linux/s390x
+0000 0000 1000 - 1000 0000 0000: executable and heap - 16 TiB
+1000 0000 0000 - 4000 0000 0000: -
+4000 0000 0000 - 8000 0000 0000: shadow - 64TiB (4 * app)
+8000 0000 0000 - 9000 0000 0000: -
+9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app)
+9800 0000 0000 - a000 0000 0000: -
+a000 0000 0000 - b000 0000 0000: traces - 16TiB (max history * 128k threads)
+*/
+struct MappingGoS390x {
+  static const uptr kMetaShadowBeg = 0x900000000000ull;
+  static const uptr kMetaShadowEnd = 0x980000000000ull;
+  static const uptr kTraceMemBeg   = 0xa00000000000ull;
+  static const uptr kTraceMemEnd   = 0xb00000000000ull;
+  static const uptr kShadowBeg     = 0x400000000000ull;
+  static const uptr kShadowEnd     = 0x800000000000ull;
+  static const uptr kLoAppMemBeg = 0x000000001000ull;
+  static const uptr kLoAppMemEnd = 0x100000000000ull;
+  static const uptr kMidAppMemBeg = 0;
+  static const uptr kMidAppMemEnd = 0;
+  static const uptr kHiAppMemBeg = 0;
+  static const uptr kHiAppMemEnd = 0;
+  static const uptr kHeapMemBeg = 0;
+  static const uptr kHeapMemEnd = 0;
+  static const uptr kVdsoBeg = 0;
+  static const uptr kShadowMsk = 0;
+  static const uptr kShadowXor = 0;
+  static const uptr kShadowAdd = 0x400000000000ull;
+};
+
+extern uptr vmaSize;
+
+template <typename Func, typename Arg>
+ALWAYS_INLINE auto SelectMapping(Arg arg) {
+#if SANITIZER_GO
+#  if defined(__powerpc64__)
+  switch (vmaSize) {
+    case 46:
+      return Func::template Apply<MappingGoPPC64_46>(arg);
+    case 47:
+      return Func::template Apply<MappingGoPPC64_47>(arg);
+  }
+#  elif defined(__mips64)
+  return Func::template Apply<MappingGoMips64_47>(arg);
+#  elif defined(__s390x__)
+  return Func::template Apply<MappingGoS390x>(arg);
+#  elif defined(__aarch64__)
+  return Func::template Apply<MappingGoAarch64>(arg);
+#  elif SANITIZER_WINDOWS
+  return Func::template Apply<MappingGoWindows>(arg);
+#  else
+  return Func::template Apply<MappingGo48>(arg);
+#  endif
+#else  // SANITIZER_GO
+#  if defined(__x86_64__) || SANITIZER_IOSSIM || SANITIZER_MAC && !SANITIZER_IOS
+  return Func::template Apply<Mapping48AddressSpace>(arg);
+#  elif defined(__aarch64__) && defined(__APPLE__)
+  return Func::template Apply<MappingAppleAarch64>(arg);
+#  elif defined(__aarch64__) && !defined(__APPLE__)
+  switch (vmaSize) {
+    case 39:
+      return Func::template Apply<MappingAarch64_39>(arg);
+    case 42:
+      return Func::template Apply<MappingAarch64_42>(arg);
+    case 48:
+      return Func::template Apply<MappingAarch64_48>(arg);
+  }
+#  elif defined(__powerpc64__)
+  switch (vmaSize) {
+    case 44:
+      return Func::template Apply<MappingPPC64_44>(arg);
+    case 46:
+      return Func::template Apply<MappingPPC64_46>(arg);
+    case 47:
+      return Func::template Apply<MappingPPC64_47>(arg);
+  }
+#  elif defined(__mips64)
+  return Func::template Apply<MappingMips64_40>(arg);
+#  elif defined(__s390x__)
+  return Func::template Apply<MappingS390x>(arg);
+#  else
+#    error "unsupported platform"
+#  endif
+#endif
+  Die();
+}
+
+template <typename Func>
+void ForEachMapping() {
+  Func::template Apply<Mapping48AddressSpace>();
+  Func::template Apply<MappingMips64_40>();
+  Func::template Apply<MappingAppleAarch64>();
+  Func::template Apply<MappingAarch64_39>();
+  Func::template Apply<MappingAarch64_42>();
+  Func::template Apply<MappingAarch64_48>();
+  Func::template Apply<MappingPPC64_44>();
+  Func::template Apply<MappingPPC64_46>();
+  Func::template Apply<MappingPPC64_47>();
+  Func::template Apply<MappingS390x>();
+  Func::template Apply<MappingGo48>();
+  Func::template Apply<MappingGoWindows>();
+  Func::template Apply<MappingGoPPC64_46>();
+  Func::template Apply<MappingGoPPC64_47>();
+  Func::template Apply<MappingGoAarch64>();
+  Func::template Apply<MappingGoMips64_47>();
+  Func::template Apply<MappingGoS390x>();
+}
+
+enum MappingType {
+  kLoAppMemBeg,
+  kLoAppMemEnd,
+  kHiAppMemBeg,
+  kHiAppMemEnd,
+  kMidAppMemBeg,
+  kMidAppMemEnd,
+  kHeapMemBeg,
+  kHeapMemEnd,
+  kShadowBeg,
+  kShadowEnd,
+  kMetaShadowBeg,
+  kMetaShadowEnd,
+  kTraceMemBeg,
+  kTraceMemEnd,
+  kVdsoBeg,
+};
+
+struct MappingField {
+  template <typename Mapping>
+  static uptr Apply(MappingType type) {
+    switch (type) {
+      case kLoAppMemBeg:
+        return Mapping::kLoAppMemBeg;
+      case kLoAppMemEnd:
+        return Mapping::kLoAppMemEnd;
+      case kMidAppMemBeg:
+        return Mapping::kMidAppMemBeg;
+      case kMidAppMemEnd:
+        return Mapping::kMidAppMemEnd;
+      case kHiAppMemBeg:
+        return Mapping::kHiAppMemBeg;
+      case kHiAppMemEnd:
+        return Mapping::kHiAppMemEnd;
+      case kHeapMemBeg:
+        return Mapping::kHeapMemBeg;
+      case kHeapMemEnd:
+        return Mapping::kHeapMemEnd;
+      case kVdsoBeg:
+        return Mapping::kVdsoBeg;
+      case kShadowBeg:
+        return Mapping::kShadowBeg;
+      case kShadowEnd:
+        return Mapping::kShadowEnd;
+      case kMetaShadowBeg:
+        return Mapping::kMetaShadowBeg;
+      case kMetaShadowEnd:
+        return Mapping::kMetaShadowEnd;
+      case kTraceMemBeg:
+        return Mapping::kTraceMemBeg;
+      case kTraceMemEnd:
+        return Mapping::kTraceMemEnd;
+    }
+    Die();
+  }
+};
+
+ALWAYS_INLINE
+uptr LoAppMemBeg(void) { return SelectMapping<MappingField>(kLoAppMemBeg); }
+ALWAYS_INLINE
+uptr LoAppMemEnd(void) { return SelectMapping<MappingField>(kLoAppMemEnd); }
+
+ALWAYS_INLINE
+uptr MidAppMemBeg(void) { return SelectMapping<MappingField>(kMidAppMemBeg); }
+ALWAYS_INLINE
+uptr MidAppMemEnd(void) { return SelectMapping<MappingField>(kMidAppMemEnd); }
+
+ALWAYS_INLINE
+uptr HeapMemBeg(void) { return SelectMapping<MappingField>(kHeapMemBeg); }
+ALWAYS_INLINE
+uptr HeapMemEnd(void) { return SelectMapping<MappingField>(kHeapMemEnd); }
+
+ALWAYS_INLINE
+uptr HiAppMemBeg(void) { return SelectMapping<MappingField>(kHiAppMemBeg); }
+ALWAYS_INLINE
+uptr HiAppMemEnd(void) { return SelectMapping<MappingField>(kHiAppMemEnd); }
+
+ALWAYS_INLINE
+uptr VdsoBeg(void) { return SelectMapping<MappingField>(kVdsoBeg); }
+
+ALWAYS_INLINE
+uptr ShadowBeg(void) { return SelectMapping<MappingField>(kShadowBeg); }
+ALWAYS_INLINE
+uptr ShadowEnd(void) { return SelectMapping<MappingField>(kShadowEnd); }
+
+ALWAYS_INLINE
+uptr MetaShadowBeg(void) { return SelectMapping<MappingField>(kMetaShadowBeg); }
+ALWAYS_INLINE
+uptr MetaShadowEnd(void) { return SelectMapping<MappingField>(kMetaShadowEnd); }
+
+ALWAYS_INLINE
+uptr TraceMemBeg(void) { return SelectMapping<MappingField>(kTraceMemBeg); }
+ALWAYS_INLINE
+uptr TraceMemEnd(void) { return SelectMapping<MappingField>(kTraceMemEnd); }
+
+struct IsAppMemImpl {
+  template <typename Mapping>
+  static bool Apply(uptr mem) {
+  return (mem >= Mapping::kHeapMemBeg && mem < Mapping::kHeapMemEnd) ||
+         (mem >= Mapping::kMidAppMemBeg && mem < Mapping::kMidAppMemEnd) ||
+         (mem >= Mapping::kLoAppMemBeg && mem < Mapping::kLoAppMemEnd) ||
+         (mem >= Mapping::kHiAppMemBeg && mem < Mapping::kHiAppMemEnd);
+  }
+};
+
+ALWAYS_INLINE
+bool IsAppMem(uptr mem) { return SelectMapping<IsAppMemImpl>(mem); }
+
+struct IsShadowMemImpl {
+  template <typename Mapping>
+  static bool Apply(uptr mem) {
+    return mem >= Mapping::kShadowBeg && mem <= Mapping::kShadowEnd;
+  }
+};
+
+ALWAYS_INLINE
+bool IsShadowMem(RawShadow *p) {
+  return SelectMapping<IsShadowMemImpl>(reinterpret_cast<uptr>(p));
+}
+
+struct IsMetaMemImpl {
+  template <typename Mapping>
+  static bool Apply(uptr mem) {
+    return mem >= Mapping::kMetaShadowBeg && mem <= Mapping::kMetaShadowEnd;
+  }
+};
+
+ALWAYS_INLINE
+bool IsMetaMem(const u32 *p) {
+  return SelectMapping<IsMetaMemImpl>(reinterpret_cast<uptr>(p));
+}
+
+struct MemToShadowImpl {
+  template <typename Mapping>
+  static uptr Apply(uptr x) {
+    DCHECK(IsAppMemImpl::Apply<Mapping>(x));
+    return (((x) & ~(Mapping::kShadowMsk | (kShadowCell - 1))) ^
+            Mapping::kShadowXor) *
+               kShadowMultiplier +
+           Mapping::kShadowAdd;
+  }
+};
+
+ALWAYS_INLINE
+RawShadow *MemToShadow(uptr x) {
+  return reinterpret_cast<RawShadow *>(SelectMapping<MemToShadowImpl>(x));
+}
+
+struct MemToMetaImpl {
+  template <typename Mapping>
+  static u32 *Apply(uptr x) {
+    DCHECK(IsAppMemImpl::Apply<Mapping>(x));
+    return (u32 *)(((((x) & ~(Mapping::kShadowMsk | (kMetaShadowCell - 1)))) /
+                    kMetaShadowCell * kMetaShadowSize) |
+                   Mapping::kMetaShadowBeg);
+  }
+};
+
+ALWAYS_INLINE
+u32 *MemToMeta(uptr x) { return SelectMapping<MemToMetaImpl>(x); }
+
+struct ShadowToMemImpl {
+  template <typename Mapping>
+  static uptr Apply(uptr sp) {
+    if (!IsShadowMemImpl::Apply<Mapping>(sp))
+      return 0;
+    // The shadow mapping is non-linear and we've lost some bits, so we don't
+    // have an easy way to restore the original app address. But the mapping is
+    // a bijection, so we try to restore the address as belonging to
+    // low/mid/high range consecutively and see if shadow->app->shadow mapping
+    // gives us the same address.
+    uptr p =
+        ((sp - Mapping::kShadowAdd) / kShadowMultiplier) ^ Mapping::kShadowXor;
+    if (p >= Mapping::kLoAppMemBeg && p < Mapping::kLoAppMemEnd &&
+        MemToShadowImpl::Apply<Mapping>(p) == sp)
+      return p;
+    if (Mapping::kMidAppMemBeg) {
+      uptr p_mid = p + (Mapping::kMidAppMemBeg & Mapping::kShadowMsk);
+      if (p_mid >= Mapping::kMidAppMemBeg && p_mid < Mapping::kMidAppMemEnd &&
+          MemToShadowImpl::Apply<Mapping>(p_mid) == sp)
+        return p_mid;
+    }
+    return p | Mapping::kShadowMsk;
+  }
+};
+
+ALWAYS_INLINE
+uptr ShadowToMem(RawShadow *s) {
+  return SelectMapping<ShadowToMemImpl>(reinterpret_cast<uptr>(s));
+}
+
+// Compresses addr to kCompressedAddrBits stored in least significant bits.
+ALWAYS_INLINE uptr CompressAddr(uptr addr) {
+  return addr & ((1ull << kCompressedAddrBits) - 1);
+}
+
+struct RestoreAddrImpl {
+  typedef uptr Result;
+  template <typename Mapping>
+  static Result Apply(uptr addr) {
+    // To restore the address we go over all app memory ranges and check if top
+    // 3 bits of the compressed addr match that of the app range. If yes, we
+    // assume that the compressed address come from that range and restore the
+    // missing top bits to match the app range address.
+    const uptr ranges[] = {
+        Mapping::kLoAppMemBeg,  Mapping::kLoAppMemEnd, Mapping::kMidAppMemBeg,
+        Mapping::kMidAppMemEnd, Mapping::kHiAppMemBeg, Mapping::kHiAppMemEnd,
+        Mapping::kHeapMemBeg,   Mapping::kHeapMemEnd,
+    };
+    const uptr indicator = 0x0e0000000000ull;
+    const uptr ind_lsb = 1ull << LeastSignificantSetBitIndex(indicator);
+    for (uptr i = 0; i < ARRAY_SIZE(ranges); i += 2) {
+      uptr beg = ranges[i];
+      uptr end = ranges[i + 1];
+      if (beg == end)
+        continue;
+      for (uptr p = beg; p < end; p = RoundDown(p + ind_lsb, ind_lsb)) {
+        if ((addr & indicator) == (p & indicator))
+          return addr | (p & ~(ind_lsb - 1));
+      }
+    }
+    Printf("ThreadSanitizer: failed to restore address 0x%zx\n", addr);
+    Die();
+  }
+};
+
+// Restores compressed addr from kCompressedAddrBits to full representation.
+// This is called only during reporting and is not performance-critical.
+inline uptr RestoreAddr(uptr addr) {
+  return SelectMapping<RestoreAddrImpl>(addr);
+}
+
+// The additional page is to catch shadow stack overflow as paging fault.
+// Windows wants 64K alignment for mmaps.
+const uptr kTotalTraceSize = (kTraceSize * sizeof(Event) + sizeof(Trace)
+    + (64 << 10) + (64 << 10) - 1) & ~((64 << 10) - 1);
+
+struct GetThreadTraceImpl {
+  template <typename Mapping>
+  static uptr Apply(uptr tid) {
+    uptr p = Mapping::kTraceMemBeg + tid * kTotalTraceSize;
+    DCHECK_LT(p, Mapping::kTraceMemEnd);
+    return p;
+  }
+};
+
+ALWAYS_INLINE
+uptr GetThreadTrace(int tid) { return SelectMapping<GetThreadTraceImpl>(tid); }
+
+struct GetThreadTraceHeaderImpl {
+  template <typename Mapping>
+  static uptr Apply(uptr tid) {
+    uptr p = Mapping::kTraceMemBeg + tid * kTotalTraceSize +
+             kTraceSize * sizeof(Event);
+    DCHECK_LT(p, Mapping::kTraceMemEnd);
+    return p;
+  }
+};
+
+ALWAYS_INLINE
+uptr GetThreadTraceHeader(int tid) {
+  return SelectMapping<GetThreadTraceHeaderImpl>(tid);
+}
+
+void InitializePlatform();
+void InitializePlatformEarly();
+void CheckAndProtect();
+void InitializeShadowMemoryPlatform();
+void FlushShadowMemory();
+void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns);
+int ExtractResolvFDs(void *state, int *fds, int nfd);
+int ExtractRecvmsgFDs(void *msg, int *fds, int nfd);
+uptr ExtractLongJmpSp(uptr *env);
+void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size);
+
+int call_pthread_cancel_with_cleanup(int (*fn)(void *arg),
+                                     void (*cleanup)(void *arg), void *arg);
+
+void DestroyThreadState();
+void PlatformCleanUpThreadState(ThreadState *thr);
+
+}  // namespace __tsan
+
+#endif  // TSAN_PLATFORM_H
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_platform_linux.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_platform_linux.cpp
new file mode 100644
index 000000000000..73ec14892d28
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_platform_linux.cpp
@@ -0,0 +1,545 @@
+//===-- tsan_platform_linux.cpp -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// Linux- and BSD-specific code.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_platform.h"
+#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD
+
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_libc.h"
+#include "sanitizer_common/sanitizer_linux.h"
+#include "sanitizer_common/sanitizer_platform_limits_netbsd.h"
+#include "sanitizer_common/sanitizer_platform_limits_posix.h"
+#include "sanitizer_common/sanitizer_posix.h"
+#include "sanitizer_common/sanitizer_procmaps.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
+#include "sanitizer_common/sanitizer_stoptheworld.h"
+#include "tsan_flags.h"
+#include "tsan_platform.h"
+#include "tsan_rtl.h"
+
+#include <fcntl.h>
+#include <pthread.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdarg.h>
+#include <sys/mman.h>
+#if SANITIZER_LINUX
+#include <sys/personality.h>
+#include <setjmp.h>
+#endif
+#include <sys/syscall.h>
+#include <sys/socket.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/resource.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <sched.h>
+#include <dlfcn.h>
+#if SANITIZER_LINUX
+#define __need_res_state
+#include <resolv.h>
+#endif
+
+#ifdef sa_handler
+# undef sa_handler
+#endif
+
+#ifdef sa_sigaction
+# undef sa_sigaction
+#endif
+
+#if SANITIZER_FREEBSD
+extern "C" void *__libc_stack_end;
+void *__libc_stack_end = 0;
+#endif
+
+#if SANITIZER_LINUX && defined(__aarch64__) && !SANITIZER_GO
+# define INIT_LONGJMP_XOR_KEY 1
+#else
+# define INIT_LONGJMP_XOR_KEY 0
+#endif
+
+#if INIT_LONGJMP_XOR_KEY
+#include "interception/interception.h"
+// Must be declared outside of other namespaces.
+DECLARE_REAL(int, _setjmp, void *env)
+#endif
+
+namespace __tsan {
+
+#if INIT_LONGJMP_XOR_KEY
+static void InitializeLongjmpXorKey();
+static uptr longjmp_xor_key;
+#endif
+
+// Runtime detected VMA size.
+uptr vmaSize;
+
+enum {
+  MemTotal,
+  MemShadow,
+  MemMeta,
+  MemFile,
+  MemMmap,
+  MemTrace,
+  MemHeap,
+  MemOther,
+  MemCount,
+};
+
+void FillProfileCallback(uptr p, uptr rss, bool file, uptr *mem) {
+  mem[MemTotal] += rss;
+  if (p >= ShadowBeg() && p < ShadowEnd())
+    mem[MemShadow] += rss;
+  else if (p >= MetaShadowBeg() && p < MetaShadowEnd())
+    mem[MemMeta] += rss;
+  else if ((p >= LoAppMemBeg() && p < LoAppMemEnd()) ||
+           (p >= MidAppMemBeg() && p < MidAppMemEnd()) ||
+           (p >= HiAppMemBeg() && p < HiAppMemEnd()))
+    mem[file ? MemFile : MemMmap] += rss;
+  else if (p >= HeapMemBeg() && p < HeapMemEnd())
+    mem[MemHeap] += rss;
+  else if (p >= TraceMemBeg() && p < TraceMemEnd())
+    mem[MemTrace] += rss;
+  else
+    mem[MemOther] += rss;
+}
+
+void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns) {
+  uptr mem[MemCount];
+  internal_memset(mem, 0, sizeof(mem));
+  GetMemoryProfile(FillProfileCallback, mem);
+  auto meta = ctx->metamap.GetMemoryStats();
+  StackDepotStats stacks = StackDepotGetStats();
+  uptr nthread, nlive;
+  ctx->thread_registry.GetNumberOfThreads(&nthread, &nlive);
+  uptr internal_stats[AllocatorStatCount];
+  internal_allocator()->GetStats(internal_stats);
+  // All these are allocated from the common mmap region.
+  mem[MemMmap] -= meta.mem_block + meta.sync_obj + stacks.allocated +
+                  internal_stats[AllocatorStatMapped];
+  if (s64(mem[MemMmap]) < 0)
+    mem[MemMmap] = 0;
+  internal_snprintf(
+      buf, buf_size,
+      "%llus: RSS %zd MB: shadow:%zd meta:%zd file:%zd mmap:%zd"
+      " trace:%zd heap:%zd other:%zd intalloc:%zd memblocks:%zd syncobj:%zu"
+      " stacks=%zd[%zd] nthr=%zd/%zd\n",
+      uptime_ns / (1000 * 1000 * 1000), mem[MemTotal] >> 20,
+      mem[MemShadow] >> 20, mem[MemMeta] >> 20, mem[MemFile] >> 20,
+      mem[MemMmap] >> 20, mem[MemTrace] >> 20, mem[MemHeap] >> 20,
+      mem[MemOther] >> 20, internal_stats[AllocatorStatMapped] >> 20,
+      meta.mem_block >> 20, meta.sync_obj >> 20, stacks.allocated >> 20,
+      stacks.n_uniq_ids, nlive, nthread);
+}
+
+#  if SANITIZER_LINUX
+void FlushShadowMemoryCallback(
+    const SuspendedThreadsList &suspended_threads_list,
+    void *argument) {
+  ReleaseMemoryPagesToOS(ShadowBeg(), ShadowEnd());
+}
+#endif
+
+void FlushShadowMemory() {
+#if SANITIZER_LINUX
+  StopTheWorld(FlushShadowMemoryCallback, 0);
+#endif
+}
+
+#if !SANITIZER_GO
+// Mark shadow for .rodata sections with the special kShadowRodata marker.
+// Accesses to .rodata can't race, so this saves time, memory and trace space.
+static void MapRodata() {
+  // First create temp file.
+  const char *tmpdir = GetEnv("TMPDIR");
+  if (tmpdir == 0)
+    tmpdir = GetEnv("TEST_TMPDIR");
+#ifdef P_tmpdir
+  if (tmpdir == 0)
+    tmpdir = P_tmpdir;
+#endif
+  if (tmpdir == 0)
+    return;
+  char name[256];
+  internal_snprintf(name, sizeof(name), "%s/tsan.rodata.%d",
+                    tmpdir, (int)internal_getpid());
+  uptr openrv = internal_open(name, O_RDWR | O_CREAT | O_EXCL, 0600);
+  if (internal_iserror(openrv))
+    return;
+  internal_unlink(name);  // Unlink it now, so that we can reuse the buffer.
+  fd_t fd = openrv;
+  // Fill the file with kShadowRodata.
+  const uptr kMarkerSize = 512 * 1024 / sizeof(RawShadow);
+  InternalMmapVector<RawShadow> marker(kMarkerSize);
+  // volatile to prevent insertion of memset
+  for (volatile RawShadow *p = marker.data(); p < marker.data() + kMarkerSize;
+       p++)
+    *p = kShadowRodata;
+  internal_write(fd, marker.data(), marker.size() * sizeof(RawShadow));
+  // Map the file into memory.
+  uptr page = internal_mmap(0, GetPageSizeCached(), PROT_READ | PROT_WRITE,
+                            MAP_PRIVATE | MAP_ANONYMOUS, fd, 0);
+  if (internal_iserror(page)) {
+    internal_close(fd);
+    return;
+  }
+  // Map the file into shadow of .rodata sections.
+  MemoryMappingLayout proc_maps(/*cache_enabled*/true);
+  // Reusing the buffer 'name'.
+  MemoryMappedSegment segment(name, ARRAY_SIZE(name));
+  while (proc_maps.Next(&segment)) {
+    if (segment.filename[0] != 0 && segment.filename[0] != '[' &&
+        segment.IsReadable() && segment.IsExecutable() &&
+        !segment.IsWritable() && IsAppMem(segment.start)) {
+      // Assume it's .rodata
+      char *shadow_start = (char *)MemToShadow(segment.start);
+      char *shadow_end = (char *)MemToShadow(segment.end);
+      for (char *p = shadow_start; p < shadow_end;
+           p += marker.size() * sizeof(RawShadow)) {
+        internal_mmap(
+            p, Min<uptr>(marker.size() * sizeof(RawShadow), shadow_end - p),
+            PROT_READ, MAP_PRIVATE | MAP_FIXED, fd, 0);
+      }
+    }
+  }
+  internal_close(fd);
+}
+
+void InitializeShadowMemoryPlatform() {
+  MapRodata();
+}
+
+#endif  // #if !SANITIZER_GO
+
+void InitializePlatformEarly() {
+  vmaSize =
+    (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1);
+#if defined(__aarch64__)
+# if !SANITIZER_GO
+  if (vmaSize != 39 && vmaSize != 42 && vmaSize != 48) {
+    Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
+    Printf("FATAL: Found %zd - Supported 39, 42 and 48\n", vmaSize);
+    Die();
+  }
+#else
+  if (vmaSize != 48) {
+    Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
+    Printf("FATAL: Found %zd - Supported 48\n", vmaSize);
+    Die();
+  }
+#endif
+#elif defined(__powerpc64__)
+# if !SANITIZER_GO
+  if (vmaSize != 44 && vmaSize != 46 && vmaSize != 47) {
+    Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
+    Printf("FATAL: Found %zd - Supported 44, 46, and 47\n", vmaSize);
+    Die();
+  }
+# else
+  if (vmaSize != 46 && vmaSize != 47) {
+    Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
+    Printf("FATAL: Found %zd - Supported 46, and 47\n", vmaSize);
+    Die();
+  }
+# endif
+#elif defined(__mips64)
+# if !SANITIZER_GO
+  if (vmaSize != 40) {
+    Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
+    Printf("FATAL: Found %zd - Supported 40\n", vmaSize);
+    Die();
+  }
+# else
+  if (vmaSize != 47) {
+    Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
+    Printf("FATAL: Found %zd - Supported 47\n", vmaSize);
+    Die();
+  }
+# endif
+#endif
+}
+
+void InitializePlatform() {
+  DisableCoreDumperIfNecessary();
+
+  // Go maps shadow memory lazily and works fine with limited address space.
+  // Unlimited stack is not a problem as well, because the executable
+  // is not compiled with -pie.
+#if !SANITIZER_GO
+  {
+    bool reexec = false;
+    // TSan doesn't play well with unlimited stack size (as stack
+    // overlaps with shadow memory). If we detect unlimited stack size,
+    // we re-exec the program with limited stack size as a best effort.
+    if (StackSizeIsUnlimited()) {
+      const uptr kMaxStackSize = 32 * 1024 * 1024;
+      VReport(1, "Program is run with unlimited stack size, which wouldn't "
+                 "work with ThreadSanitizer.\n"
+                 "Re-execing with stack size limited to %zd bytes.\n",
+              kMaxStackSize);
+      SetStackSizeLimitInBytes(kMaxStackSize);
+      reexec = true;
+    }
+
+    if (!AddressSpaceIsUnlimited()) {
+      Report("WARNING: Program is run with limited virtual address space,"
+             " which wouldn't work with ThreadSanitizer.\n");
+      Report("Re-execing with unlimited virtual address space.\n");
+      SetAddressSpaceUnlimited();
+      reexec = true;
+    }
+#if SANITIZER_LINUX && defined(__aarch64__)
+    // After patch "arm64: mm: support ARCH_MMAP_RND_BITS." is introduced in
+    // linux kernel, the random gap between stack and mapped area is increased
+    // from 128M to 36G on 39-bit aarch64. As it is almost impossible to cover
+    // this big range, we should disable randomized virtual space on aarch64.
+    int old_personality = personality(0xffffffff);
+    if (old_personality != -1 && (old_personality & ADDR_NO_RANDOMIZE) == 0) {
+      VReport(1, "WARNING: Program is run with randomized virtual address "
+              "space, which wouldn't work with ThreadSanitizer.\n"
+              "Re-execing with fixed virtual address space.\n");
+      CHECK_NE(personality(old_personality | ADDR_NO_RANDOMIZE), -1);
+      reexec = true;
+    }
+    // Initialize the xor key used in {sig}{set,long}jump.
+    InitializeLongjmpXorKey();
+#endif
+    if (reexec)
+      ReExec();
+  }
+
+  CheckAndProtect();
+  InitTlsSize();
+#endif  // !SANITIZER_GO
+}
+
+#if !SANITIZER_GO
+// Extract file descriptors passed to glibc internal __res_iclose function.
+// This is required to properly "close" the fds, because we do not see internal
+// closes within glibc. The code is a pure hack.
+int ExtractResolvFDs(void *state, int *fds, int nfd) {
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+  int cnt = 0;
+  struct __res_state *statp = (struct __res_state*)state;
+  for (int i = 0; i < MAXNS && cnt < nfd; i++) {
+    if (statp->_u._ext.nsaddrs[i] && statp->_u._ext.nssocks[i] != -1)
+      fds[cnt++] = statp->_u._ext.nssocks[i];
+  }
+  return cnt;
+#else
+  return 0;
+#endif
+}
+
+// Extract file descriptors passed via UNIX domain sockets.
+// This is required to properly handle "open" of these fds.
+// see 'man recvmsg' and 'man 3 cmsg'.
+int ExtractRecvmsgFDs(void *msgp, int *fds, int nfd) {
+  int res = 0;
+  msghdr *msg = (msghdr*)msgp;
+  struct cmsghdr *cmsg = CMSG_FIRSTHDR(msg);
+  for (; cmsg; cmsg = CMSG_NXTHDR(msg, cmsg)) {
+    if (cmsg->cmsg_level != SOL_SOCKET || cmsg->cmsg_type != SCM_RIGHTS)
+      continue;
+    int n = (cmsg->cmsg_len - CMSG_LEN(0)) / sizeof(fds[0]);
+    for (int i = 0; i < n; i++) {
+      fds[res++] = ((int*)CMSG_DATA(cmsg))[i];
+      if (res == nfd)
+        return res;
+    }
+  }
+  return res;
+}
+
+// Reverse operation of libc stack pointer mangling
+static uptr UnmangleLongJmpSp(uptr mangled_sp) {
+#if defined(__x86_64__)
+# if SANITIZER_LINUX
+  // Reverse of:
+  //   xor  %fs:0x30, %rsi
+  //   rol  $0x11, %rsi
+  uptr sp;
+  asm("ror  $0x11,     %0 \n"
+      "xor  %%fs:0x30, %0 \n"
+      : "=r" (sp)
+      : "0" (mangled_sp));
+  return sp;
+# else
+  return mangled_sp;
+# endif
+#elif defined(__aarch64__)
+# if SANITIZER_LINUX
+  return mangled_sp ^ longjmp_xor_key;
+# else
+  return mangled_sp;
+# endif
+#elif defined(__powerpc64__)
+  // Reverse of:
+  //   ld   r4, -28696(r13)
+  //   xor  r4, r3, r4
+  uptr xor_key;
+  asm("ld  %0, -28696(%%r13)" : "=r" (xor_key));
+  return mangled_sp ^ xor_key;
+#elif defined(__mips__)
+  return mangled_sp;
+#elif defined(__s390x__)
+  // tcbhead_t.stack_guard
+  uptr xor_key = ((uptr *)__builtin_thread_pointer())[5];
+  return mangled_sp ^ xor_key;
+#else
+  #error "Unknown platform"
+#endif
+}
+
+#if SANITIZER_NETBSD
+# ifdef __x86_64__
+#  define LONG_JMP_SP_ENV_SLOT 6
+# else
+#  error unsupported
+# endif
+#elif defined(__powerpc__)
+# define LONG_JMP_SP_ENV_SLOT 0
+#elif SANITIZER_FREEBSD
+# define LONG_JMP_SP_ENV_SLOT 2
+#elif SANITIZER_LINUX
+# ifdef __aarch64__
+#  define LONG_JMP_SP_ENV_SLOT 13
+# elif defined(__mips64)
+#  define LONG_JMP_SP_ENV_SLOT 1
+# elif defined(__s390x__)
+#  define LONG_JMP_SP_ENV_SLOT 9
+# else
+#  define LONG_JMP_SP_ENV_SLOT 6
+# endif
+#endif
+
+uptr ExtractLongJmpSp(uptr *env) {
+  uptr mangled_sp = env[LONG_JMP_SP_ENV_SLOT];
+  return UnmangleLongJmpSp(mangled_sp);
+}
+
+#if INIT_LONGJMP_XOR_KEY
+// GLIBC mangles the function pointers in jmp_buf (used in {set,long}*jmp
+// functions) by XORing them with a random key.  For AArch64 it is a global
+// variable rather than a TCB one (as for x86_64/powerpc).  We obtain the key by
+// issuing a setjmp and XORing the SP pointer values to derive the key.
+static void InitializeLongjmpXorKey() {
+  // 1. Call REAL(setjmp), which stores the mangled SP in env.
+  jmp_buf env;
+  REAL(_setjmp)(env);
+
+  // 2. Retrieve vanilla/mangled SP.
+  uptr sp;
+  asm("mov  %0, sp" : "=r" (sp));
+  uptr mangled_sp = ((uptr *)&env)[LONG_JMP_SP_ENV_SLOT];
+
+  // 3. xor SPs to obtain key.
+  longjmp_xor_key = mangled_sp ^ sp;
+}
+#endif
+
+extern "C" void __tsan_tls_initialization() {}
+
+void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size) {
+  // Check that the thr object is in tls;
+  const uptr thr_beg = (uptr)thr;
+  const uptr thr_end = (uptr)thr + sizeof(*thr);
+  CHECK_GE(thr_beg, tls_addr);
+  CHECK_LE(thr_beg, tls_addr + tls_size);
+  CHECK_GE(thr_end, tls_addr);
+  CHECK_LE(thr_end, tls_addr + tls_size);
+  // Since the thr object is huge, skip it.
+  const uptr pc = StackTrace::GetNextInstructionPc(
+      reinterpret_cast<uptr>(__tsan_tls_initialization));
+  MemoryRangeImitateWrite(thr, pc, tls_addr, thr_beg - tls_addr);
+  MemoryRangeImitateWrite(thr, pc, thr_end, tls_addr + tls_size - thr_end);
+}
+
+// Note: this function runs with async signals enabled,
+// so it must not touch any tsan state.
+int call_pthread_cancel_with_cleanup(int (*fn)(void *arg),
+                                     void (*cleanup)(void *arg), void *arg) {
+  // pthread_cleanup_push/pop are hardcore macros mess.
+  // We can't intercept nor call them w/o including pthread.h.
+  int res;
+  pthread_cleanup_push(cleanup, arg);
+  res = fn(arg);
+  pthread_cleanup_pop(0);
+  return res;
+}
+#endif  // !SANITIZER_GO
+
+#if !SANITIZER_GO
+void ReplaceSystemMalloc() { }
+#endif
+
+#if !SANITIZER_GO
+#if SANITIZER_ANDROID
+// On Android, one thread can call intercepted functions after
+// DestroyThreadState(), so add a fake thread state for "dead" threads.
+static ThreadState *dead_thread_state = nullptr;
+
+ThreadState *cur_thread() {
+  ThreadState* thr = reinterpret_cast<ThreadState*>(*get_android_tls_ptr());
+  if (thr == nullptr) {
+    __sanitizer_sigset_t emptyset;
+    internal_sigfillset(&emptyset);
+    __sanitizer_sigset_t oldset;
+    CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &emptyset, &oldset));
+    thr = reinterpret_cast<ThreadState*>(*get_android_tls_ptr());
+    if (thr == nullptr) {
+      thr = reinterpret_cast<ThreadState*>(MmapOrDie(sizeof(ThreadState),
+                                                     "ThreadState"));
+      *get_android_tls_ptr() = reinterpret_cast<uptr>(thr);
+      if (dead_thread_state == nullptr) {
+        dead_thread_state = reinterpret_cast<ThreadState*>(
+            MmapOrDie(sizeof(ThreadState), "ThreadState"));
+        dead_thread_state->fast_state.SetIgnoreBit();
+        dead_thread_state->ignore_interceptors = 1;
+        dead_thread_state->is_dead = true;
+        *const_cast<u32*>(&dead_thread_state->tid) = -1;
+        CHECK_EQ(0, internal_mprotect(dead_thread_state, sizeof(ThreadState),
+                                      PROT_READ));
+      }
+    }
+    CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &oldset, nullptr));
+  }
+  return thr;
+}
+
+void set_cur_thread(ThreadState *thr) {
+  *get_android_tls_ptr() = reinterpret_cast<uptr>(thr);
+}
+
+void cur_thread_finalize() {
+  __sanitizer_sigset_t emptyset;
+  internal_sigfillset(&emptyset);
+  __sanitizer_sigset_t oldset;
+  CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &emptyset, &oldset));
+  ThreadState* thr = reinterpret_cast<ThreadState*>(*get_android_tls_ptr());
+  if (thr != dead_thread_state) {
+    *get_android_tls_ptr() = reinterpret_cast<uptr>(dead_thread_state);
+    UnmapOrDie(thr, sizeof(ThreadState));
+  }
+  CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &oldset, nullptr));
+}
+#endif  // SANITIZER_ANDROID
+#endif  // if !SANITIZER_GO
+
+}  // namespace __tsan
+
+#endif  // SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_platform_mac.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_platform_mac.cpp
new file mode 100644
index 000000000000..1465f9953c19
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_platform_mac.cpp
@@ -0,0 +1,326 @@
+//===-- tsan_platform_mac.cpp ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// Mac-specific code.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_platform.h"
+#if SANITIZER_MAC
+
+#include "sanitizer_common/sanitizer_atomic.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_libc.h"
+#include "sanitizer_common/sanitizer_posix.h"
+#include "sanitizer_common/sanitizer_procmaps.h"
+#include "sanitizer_common/sanitizer_ptrauth.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
+#include "tsan_platform.h"
+#include "tsan_rtl.h"
+#include "tsan_flags.h"
+
+#include <limits.h>
+#include <mach/mach.h>
+#include <pthread.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdarg.h>
+#include <sys/mman.h>
+#include <sys/syscall.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/resource.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <errno.h>
+#include <sched.h>
+
+namespace __tsan {
+
+#if !SANITIZER_GO
+static char main_thread_state[sizeof(ThreadState)] ALIGNED(
+    SANITIZER_CACHE_LINE_SIZE);
+static ThreadState *dead_thread_state;
+static pthread_key_t thread_state_key;
+
+// We rely on the following documented, but Darwin-specific behavior to keep the
+// reference to the ThreadState object alive in TLS:
+// pthread_key_create man page:
+//   If, after all the destructors have been called for all non-NULL values with
+//   associated destructors, there are still some non-NULL values with
+//   associated destructors, then the process is repeated.  If, after at least
+//   [PTHREAD_DESTRUCTOR_ITERATIONS] iterations of destructor calls for
+//   outstanding non-NULL values, there are still some non-NULL values with
+//   associated destructors, the implementation stops calling destructors.
+static_assert(PTHREAD_DESTRUCTOR_ITERATIONS == 4, "Small number of iterations");
+static void ThreadStateDestructor(void *thr) {
+  int res = pthread_setspecific(thread_state_key, thr);
+  CHECK_EQ(res, 0);
+}
+
+static void InitializeThreadStateStorage() {
+  int res;
+  CHECK_EQ(thread_state_key, 0);
+  res = pthread_key_create(&thread_state_key, ThreadStateDestructor);
+  CHECK_EQ(res, 0);
+  res = pthread_setspecific(thread_state_key, main_thread_state);
+  CHECK_EQ(res, 0);
+
+  auto dts = (ThreadState *)MmapOrDie(sizeof(ThreadState), "ThreadState");
+  dts->fast_state.SetIgnoreBit();
+  dts->ignore_interceptors = 1;
+  dts->is_dead = true;
+  const_cast<Tid &>(dts->tid) = kInvalidTid;
+  res = internal_mprotect(dts, sizeof(ThreadState), PROT_READ);  // immutable
+  CHECK_EQ(res, 0);
+  dead_thread_state = dts;
+}
+
+ThreadState *cur_thread() {
+  // Some interceptors get called before libpthread has been initialized and in
+  // these cases we must avoid calling any pthread APIs.
+  if (UNLIKELY(!thread_state_key)) {
+    return (ThreadState *)main_thread_state;
+  }
+
+  // We only reach this line after InitializeThreadStateStorage() ran, i.e,
+  // after TSan (and therefore libpthread) have been initialized.
+  ThreadState *thr = (ThreadState *)pthread_getspecific(thread_state_key);
+  if (UNLIKELY(!thr)) {
+    thr = (ThreadState *)MmapOrDie(sizeof(ThreadState), "ThreadState");
+    int res = pthread_setspecific(thread_state_key, thr);
+    CHECK_EQ(res, 0);
+  }
+  return thr;
+}
+
+void set_cur_thread(ThreadState *thr) {
+  int res = pthread_setspecific(thread_state_key, thr);
+  CHECK_EQ(res, 0);
+}
+
+void cur_thread_finalize() {
+  ThreadState *thr = (ThreadState *)pthread_getspecific(thread_state_key);
+  CHECK(thr);
+  if (thr == (ThreadState *)main_thread_state) {
+    // Calling dispatch_main() or xpc_main() actually invokes pthread_exit to
+    // exit the main thread. Let's keep the main thread's ThreadState.
+    return;
+  }
+  // Intercepted functions can still get called after cur_thread_finalize()
+  // (called from DestroyThreadState()), so put a fake thread state for "dead"
+  // threads.  An alternative solution would be to release the ThreadState
+  // object from THREAD_DESTROY (which is delivered later and on the parent
+  // thread) instead of THREAD_TERMINATE.
+  int res = pthread_setspecific(thread_state_key, dead_thread_state);
+  CHECK_EQ(res, 0);
+  UnmapOrDie(thr, sizeof(ThreadState));
+}
+#endif
+
+void FlushShadowMemory() {
+}
+
+static void RegionMemUsage(uptr start, uptr end, uptr *res, uptr *dirty) {
+  vm_address_t address = start;
+  vm_address_t end_address = end;
+  uptr resident_pages = 0;
+  uptr dirty_pages = 0;
+  while (address < end_address) {
+    vm_size_t vm_region_size;
+    mach_msg_type_number_t count = VM_REGION_EXTENDED_INFO_COUNT;
+    vm_region_extended_info_data_t vm_region_info;
+    mach_port_t object_name;
+    kern_return_t ret = vm_region_64(
+        mach_task_self(), &address, &vm_region_size, VM_REGION_EXTENDED_INFO,
+        (vm_region_info_t)&vm_region_info, &count, &object_name);
+    if (ret != KERN_SUCCESS) break;
+
+    resident_pages += vm_region_info.pages_resident;
+    dirty_pages += vm_region_info.pages_dirtied;
+
+    address += vm_region_size;
+  }
+  *res = resident_pages * GetPageSizeCached();
+  *dirty = dirty_pages * GetPageSizeCached();
+}
+
+void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns) {
+  uptr shadow_res, shadow_dirty;
+  uptr meta_res, meta_dirty;
+  uptr trace_res, trace_dirty;
+  RegionMemUsage(ShadowBeg(), ShadowEnd(), &shadow_res, &shadow_dirty);
+  RegionMemUsage(MetaShadowBeg(), MetaShadowEnd(), &meta_res, &meta_dirty);
+  RegionMemUsage(TraceMemBeg(), TraceMemEnd(), &trace_res, &trace_dirty);
+
+#if !SANITIZER_GO
+  uptr low_res, low_dirty;
+  uptr high_res, high_dirty;
+  uptr heap_res, heap_dirty;
+  RegionMemUsage(LoAppMemBeg(), LoAppMemEnd(), &low_res, &low_dirty);
+  RegionMemUsage(HiAppMemBeg(), HiAppMemEnd(), &high_res, &high_dirty);
+  RegionMemUsage(HeapMemBeg(), HeapMemEnd(), &heap_res, &heap_dirty);
+#else  // !SANITIZER_GO
+  uptr app_res, app_dirty;
+  RegionMemUsage(LoAppMemBeg(), LoAppMemEnd(), &app_res, &app_dirty);
+#endif
+
+  StackDepotStats stacks = StackDepotGetStats();
+  uptr nthread, nlive;
+  ctx->thread_registry.GetNumberOfThreads(&nthread, &nlive);
+  internal_snprintf(
+      buf, buf_size,
+      "shadow   (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
+      "meta     (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
+      "traces   (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
+#  if !SANITIZER_GO
+      "low app  (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
+      "high app (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
+      "heap     (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
+#  else  // !SANITIZER_GO
+      "app      (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
+#  endif
+      "stacks: %zd unique IDs, %zd kB allocated\n"
+      "threads: %zd total, %zd live\n"
+      "------------------------------\n",
+      ShadowBeg(), ShadowEnd(), shadow_res / 1024, shadow_dirty / 1024,
+      MetaShadowBeg(), MetaShadowEnd(), meta_res / 1024, meta_dirty / 1024,
+      TraceMemBeg(), TraceMemEnd(), trace_res / 1024, trace_dirty / 1024,
+#  if !SANITIZER_GO
+      LoAppMemBeg(), LoAppMemEnd(), low_res / 1024, low_dirty / 1024,
+      HiAppMemBeg(), HiAppMemEnd(), high_res / 1024, high_dirty / 1024,
+      HeapMemBeg(), HeapMemEnd(), heap_res / 1024, heap_dirty / 1024,
+#  else  // !SANITIZER_GO
+      LoAppMemBeg(), LoAppMemEnd(), app_res / 1024, app_dirty / 1024,
+#  endif
+      stacks.n_uniq_ids, stacks.allocated / 1024, nthread, nlive);
+}
+
+#  if !SANITIZER_GO
+void InitializeShadowMemoryPlatform() { }
+
+// On OS X, GCD worker threads are created without a call to pthread_create. We
+// need to properly register these threads with ThreadCreate and ThreadStart.
+// These threads don't have a parent thread, as they are created "spuriously".
+// We're using a libpthread API that notifies us about a newly created thread.
+// The `thread == pthread_self()` check indicates this is actually a worker
+// thread. If it's just a regular thread, this hook is called on the parent
+// thread.
+typedef void (*pthread_introspection_hook_t)(unsigned int event,
+                                             pthread_t thread, void *addr,
+                                             size_t size);
+extern "C" pthread_introspection_hook_t pthread_introspection_hook_install(
+    pthread_introspection_hook_t hook);
+static const uptr PTHREAD_INTROSPECTION_THREAD_CREATE = 1;
+static const uptr PTHREAD_INTROSPECTION_THREAD_TERMINATE = 3;
+static pthread_introspection_hook_t prev_pthread_introspection_hook;
+static void my_pthread_introspection_hook(unsigned int event, pthread_t thread,
+                                          void *addr, size_t size) {
+  if (event == PTHREAD_INTROSPECTION_THREAD_CREATE) {
+    if (thread == pthread_self()) {
+      // The current thread is a newly created GCD worker thread.
+      ThreadState *thr = cur_thread();
+      Processor *proc = ProcCreate();
+      ProcWire(proc, thr);
+      ThreadState *parent_thread_state = nullptr;  // No parent.
+      Tid tid = ThreadCreate(parent_thread_state, 0, (uptr)thread, true);
+      CHECK_NE(tid, kMainTid);
+      ThreadStart(thr, tid, GetTid(), ThreadType::Worker);
+    }
+  } else if (event == PTHREAD_INTROSPECTION_THREAD_TERMINATE) {
+    CHECK_EQ(thread, pthread_self());
+    ThreadState *thr = cur_thread();
+    if (thr->tctx) {
+      DestroyThreadState();
+    }
+  }
+
+  if (prev_pthread_introspection_hook != nullptr)
+    prev_pthread_introspection_hook(event, thread, addr, size);
+}
+#endif
+
+void InitializePlatformEarly() {
+#  if !SANITIZER_GO && SANITIZER_IOS
+  uptr max_vm = GetMaxUserVirtualAddress() + 1;
+  if (max_vm != HiAppMemEnd()) {
+    Printf("ThreadSanitizer: unsupported vm address limit %p, expected %p.\n",
+           (void *)max_vm, (void *)HiAppMemEnd());
+    Die();
+  }
+#endif
+}
+
+static uptr longjmp_xor_key = 0;
+
+void InitializePlatform() {
+  DisableCoreDumperIfNecessary();
+#if !SANITIZER_GO
+  CheckAndProtect();
+
+  InitializeThreadStateStorage();
+
+  prev_pthread_introspection_hook =
+      pthread_introspection_hook_install(&my_pthread_introspection_hook);
+#endif
+
+  if (GetMacosAlignedVersion() >= MacosVersion(10, 14)) {
+    // Libsystem currently uses a process-global key; this might change.
+    const unsigned kTLSLongjmpXorKeySlot = 0x7;
+    longjmp_xor_key = (uptr)pthread_getspecific(kTLSLongjmpXorKeySlot);
+  }
+}
+
+#ifdef __aarch64__
+# define LONG_JMP_SP_ENV_SLOT \
+    ((GetMacosAlignedVersion() >= MacosVersion(10, 14)) ? 12 : 13)
+#else
+# define LONG_JMP_SP_ENV_SLOT 2
+#endif
+
+uptr ExtractLongJmpSp(uptr *env) {
+  uptr mangled_sp = env[LONG_JMP_SP_ENV_SLOT];
+  uptr sp = mangled_sp ^ longjmp_xor_key;
+  sp = (uptr)ptrauth_auth_data((void *)sp, ptrauth_key_asdb,
+                               ptrauth_string_discriminator("sp"));
+  return sp;
+}
+
+#if !SANITIZER_GO
+extern "C" void __tsan_tls_initialization() {}
+
+void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size) {
+  const uptr pc = StackTrace::GetNextInstructionPc(
+      reinterpret_cast<uptr>(__tsan_tls_initialization));
+  // Unlike Linux, we only store a pointer to the ThreadState object in TLS;
+  // just mark the entire range as written to.
+  MemoryRangeImitateWrite(thr, pc, tls_addr, tls_size);
+}
+#endif
+
+#if !SANITIZER_GO
+// Note: this function runs with async signals enabled,
+// so it must not touch any tsan state.
+int call_pthread_cancel_with_cleanup(int (*fn)(void *arg),
+                                     void (*cleanup)(void *arg), void *arg) {
+  // pthread_cleanup_push/pop are hardcore macros mess.
+  // We can't intercept nor call them w/o including pthread.h.
+  int res;
+  pthread_cleanup_push(cleanup, arg);
+  res = fn(arg);
+  pthread_cleanup_pop(0);
+  return res;
+}
+#endif
+
+}  // namespace __tsan
+
+#endif  // SANITIZER_MAC
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_platform_posix.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_platform_posix.cpp
new file mode 100644
index 000000000000..763ac444377e
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_platform_posix.cpp
@@ -0,0 +1,147 @@
+//===-- tsan_platform_posix.cpp -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// POSIX-specific code.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_platform.h"
+#if SANITIZER_POSIX
+
+#  include <dlfcn.h>
+
+#  include "sanitizer_common/sanitizer_common.h"
+#  include "sanitizer_common/sanitizer_errno.h"
+#  include "sanitizer_common/sanitizer_libc.h"
+#  include "sanitizer_common/sanitizer_procmaps.h"
+#  include "tsan_platform.h"
+#  include "tsan_rtl.h"
+
+namespace __tsan {
+
+static const char kShadowMemoryMappingWarning[] =
+    "FATAL: %s can not madvise shadow region [%zx, %zx] with %s (errno: %d)\n";
+static const char kShadowMemoryMappingHint[] =
+    "HINT: if %s is not supported in your environment, you may set "
+    "TSAN_OPTIONS=%s=0\n";
+
+#  if !SANITIZER_GO
+static void DontDumpShadow(uptr addr, uptr size) {
+  if (common_flags()->use_madv_dontdump)
+    if (!DontDumpShadowMemory(addr, size)) {
+      Printf(kShadowMemoryMappingWarning, SanitizerToolName, addr, addr + size,
+             "MADV_DONTDUMP", errno);
+      Printf(kShadowMemoryMappingHint, "MADV_DONTDUMP", "use_madv_dontdump");
+      Die();
+    }
+}
+
+void InitializeShadowMemory() {
+  // Map memory shadow.
+  if (!MmapFixedSuperNoReserve(ShadowBeg(), ShadowEnd() - ShadowBeg(),
+                               "shadow")) {
+    Printf("FATAL: ThreadSanitizer can not mmap the shadow memory\n");
+    Printf("FATAL: Make sure to compile with -fPIE and to link with -pie.\n");
+    Die();
+  }
+  // This memory range is used for thread stacks and large user mmaps.
+  // Frequently a thread uses only a small part of stack and similarly
+  // a program uses a small part of large mmap. On some programs
+  // we see 20% memory usage reduction without huge pages for this range.
+  DontDumpShadow(ShadowBeg(), ShadowEnd() - ShadowBeg());
+  DPrintf("memory shadow: %zx-%zx (%zuGB)\n",
+      ShadowBeg(), ShadowEnd(),
+      (ShadowEnd() - ShadowBeg()) >> 30);
+
+  // Map meta shadow.
+  const uptr meta = MetaShadowBeg();
+  const uptr meta_size = MetaShadowEnd() - meta;
+  if (!MmapFixedSuperNoReserve(meta, meta_size, "meta shadow")) {
+    Printf("FATAL: ThreadSanitizer can not mmap the shadow memory\n");
+    Printf("FATAL: Make sure to compile with -fPIE and to link with -pie.\n");
+    Die();
+  }
+  DontDumpShadow(meta, meta_size);
+  DPrintf("meta shadow: %zx-%zx (%zuGB)\n",
+      meta, meta + meta_size, meta_size >> 30);
+
+  InitializeShadowMemoryPlatform();
+
+  on_initialize = reinterpret_cast<void (*)(void)>(
+      dlsym(RTLD_DEFAULT, "__tsan_on_initialize"));
+  on_finalize =
+      reinterpret_cast<int (*)(int)>(dlsym(RTLD_DEFAULT, "__tsan_on_finalize"));
+}
+
+static bool TryProtectRange(uptr beg, uptr end) {
+  CHECK_LE(beg, end);
+  if (beg == end)
+    return true;
+  return beg == (uptr)MmapFixedNoAccess(beg, end - beg);
+}
+
+static void ProtectRange(uptr beg, uptr end) {
+  if (!TryProtectRange(beg, end)) {
+    Printf("FATAL: ThreadSanitizer can not protect [%zx,%zx]\n", beg, end);
+    Printf("FATAL: Make sure you are not using unlimited stack\n");
+    Die();
+  }
+}
+
+void CheckAndProtect() {
+  // Ensure that the binary is indeed compiled with -pie.
+  MemoryMappingLayout proc_maps(true);
+  MemoryMappedSegment segment;
+  while (proc_maps.Next(&segment)) {
+    if (IsAppMem(segment.start)) continue;
+    if (segment.start >= HeapMemEnd() && segment.start < HeapEnd()) continue;
+    if (segment.protection == 0)  // Zero page or mprotected.
+      continue;
+    if (segment.start >= VdsoBeg())  // vdso
+      break;
+    Printf("FATAL: ThreadSanitizer: unexpected memory mapping 0x%zx-0x%zx\n",
+           segment.start, segment.end);
+    Die();
+  }
+
+#    if defined(__aarch64__) && defined(__APPLE__) && SANITIZER_IOS
+  ProtectRange(HeapMemEnd(), ShadowBeg());
+  ProtectRange(ShadowEnd(), MetaShadowBeg());
+  ProtectRange(MetaShadowEnd(), TraceMemBeg());
+#else
+  ProtectRange(LoAppMemEnd(), ShadowBeg());
+  ProtectRange(ShadowEnd(), MetaShadowBeg());
+  if (MidAppMemBeg()) {
+    ProtectRange(MetaShadowEnd(), MidAppMemBeg());
+    ProtectRange(MidAppMemEnd(), TraceMemBeg());
+  } else {
+    ProtectRange(MetaShadowEnd(), TraceMemBeg());
+  }
+  // Memory for traces is mapped lazily in MapThreadTrace.
+  // Protect the whole range for now, so that user does not map something here.
+  ProtectRange(TraceMemBeg(), TraceMemEnd());
+  ProtectRange(TraceMemEnd(), HeapMemBeg());
+  ProtectRange(HeapEnd(), HiAppMemBeg());
+#endif
+
+#if defined(__s390x__)
+  // Protect the rest of the address space.
+  const uptr user_addr_max_l4 = 0x0020000000000000ull;
+  const uptr user_addr_max_l5 = 0xfffffffffffff000ull;
+  // All the maintained s390x kernels support at least 4-level page tables.
+  ProtectRange(HiAppMemEnd(), user_addr_max_l4);
+  // Older s390x kernels may not support 5-level page tables.
+  TryProtectRange(user_addr_max_l4, user_addr_max_l5);
+#endif
+}
+#endif
+
+}  // namespace __tsan
+
+#endif  // SANITIZER_POSIX
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_platform_windows.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_platform_windows.cpp
new file mode 100644
index 000000000000..fea893768c79
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_platform_windows.cpp
@@ -0,0 +1,36 @@
+//===-- tsan_platform_windows.cpp -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// Windows-specific code.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_platform.h"
+#if SANITIZER_WINDOWS
+
+#include "tsan_platform.h"
+
+#include <stdlib.h>
+
+namespace __tsan {
+
+void FlushShadowMemory() {
+}
+
+void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns) {}
+
+void InitializePlatformEarly() {
+}
+
+void InitializePlatform() {
+}
+
+}  // namespace __tsan
+
+#endif  // SANITIZER_WINDOWS
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_ppc_regs.h b/compiler-rt/lib/tsan/rtl-old/tsan_ppc_regs.h
new file mode 100644
index 000000000000..5b43f3ddada3
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_ppc_regs.h
@@ -0,0 +1,96 @@
+#define r0 0
+#define r1 1
+#define r2 2
+#define r3 3
+#define r4 4
+#define r5 5
+#define r6 6
+#define r7 7
+#define r8 8
+#define r9 9
+#define r10 10
+#define r11 11
+#define r12 12
+#define r13 13
+#define r14 14
+#define r15 15
+#define r16 16
+#define r17 17
+#define r18 18
+#define r19 19
+#define r20 20
+#define r21 21
+#define r22 22
+#define r23 23
+#define r24 24
+#define r25 25
+#define r26 26
+#define r27 27
+#define r28 28
+#define r29 29
+#define r30 30
+#define r31 31
+#define f0 0
+#define f1 1
+#define f2 2
+#define f3 3
+#define f4 4
+#define f5 5
+#define f6 6
+#define f7 7
+#define f8 8
+#define f9 9
+#define f10 10
+#define f11 11
+#define f12 12
+#define f13 13
+#define f14 14
+#define f15 15
+#define f16 16
+#define f17 17
+#define f18 18
+#define f19 19
+#define f20 20
+#define f21 21
+#define f22 22
+#define f23 23
+#define f24 24
+#define f25 25
+#define f26 26
+#define f27 27
+#define f28 28
+#define f29 29
+#define f30 30
+#define f31 31
+#define v0 0
+#define v1 1
+#define v2 2
+#define v3 3
+#define v4 4
+#define v5 5
+#define v6 6
+#define v7 7
+#define v8 8
+#define v9 9
+#define v10 10
+#define v11 11
+#define v12 12
+#define v13 13
+#define v14 14
+#define v15 15
+#define v16 16
+#define v17 17
+#define v18 18
+#define v19 19
+#define v20 20
+#define v21 21
+#define v22 22
+#define v23 23
+#define v24 24
+#define v25 25
+#define v26 26
+#define v27 27
+#define v28 28
+#define v29 29
+#define v30 30
+#define v31 31
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_preinit.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_preinit.cpp
new file mode 100644
index 000000000000..205bdbf93b20
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_preinit.cpp
@@ -0,0 +1,26 @@
+//===-- tsan_preinit.cpp --------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer.
+//
+// Call __tsan_init at the very early stage of process startup.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_internal_defs.h"
+#include "tsan_interface.h"
+
+#if SANITIZER_CAN_USE_PREINIT_ARRAY
+
+// The symbol is called __local_tsan_preinit, because it's not intended to be
+// exported.
+// This code linked into the main executable when -fsanitize=thread is in
+// the link flags. It can only use exported interface functions.
+__attribute__((section(".preinit_array"), used))
+void (*__local_tsan_preinit)(void) = __tsan_init;
+
+#endif
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_report.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_report.cpp
new file mode 100644
index 000000000000..a926c3761ccf
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_report.cpp
@@ -0,0 +1,479 @@
+//===-- tsan_report.cpp ---------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#include "tsan_report.h"
+#include "tsan_platform.h"
+#include "tsan_rtl.h"
+#include "sanitizer_common/sanitizer_file.h"
+#include "sanitizer_common/sanitizer_placement_new.h"
+#include "sanitizer_common/sanitizer_report_decorator.h"
+#include "sanitizer_common/sanitizer_stacktrace_printer.h"
+
+namespace __tsan {
+
+class Decorator: public __sanitizer::SanitizerCommonDecorator {
+ public:
+  Decorator() : SanitizerCommonDecorator() { }
+  const char *Access()     { return Blue(); }
+  const char *ThreadDescription()    { return Cyan(); }
+  const char *Location()   { return Green(); }
+  const char *Sleep()   { return Yellow(); }
+  const char *Mutex()   { return Magenta(); }
+};
+
+ReportDesc::ReportDesc()
+    : tag(kExternalTagNone)
+    , stacks()
+    , mops()
+    , locs()
+    , mutexes()
+    , threads()
+    , unique_tids()
+    , sleep()
+    , count() {
+}
+
+ReportMop::ReportMop()
+    : mset() {
+}
+
+ReportDesc::~ReportDesc() {
+  // FIXME(dvyukov): it must be leaking a lot of memory.
+}
+
+#if !SANITIZER_GO
+
+const int kThreadBufSize = 32;
+const char *thread_name(char *buf, Tid tid) {
+  if (tid == kMainTid)
+    return "main thread";
+  internal_snprintf(buf, kThreadBufSize, "thread T%d", tid);
+  return buf;
+}
+
+static const char *ReportTypeString(ReportType typ, uptr tag) {
+  switch (typ) {
+    case ReportTypeRace:
+      return "data race";
+    case ReportTypeVptrRace:
+      return "data race on vptr (ctor/dtor vs virtual call)";
+    case ReportTypeUseAfterFree:
+      return "heap-use-after-free";
+    case ReportTypeVptrUseAfterFree:
+      return "heap-use-after-free (virtual call vs free)";
+    case ReportTypeExternalRace: {
+      const char *str = GetReportHeaderFromTag(tag);
+      return str ? str : "race on external object";
+    }
+    case ReportTypeThreadLeak:
+      return "thread leak";
+    case ReportTypeMutexDestroyLocked:
+      return "destroy of a locked mutex";
+    case ReportTypeMutexDoubleLock:
+      return "double lock of a mutex";
+    case ReportTypeMutexInvalidAccess:
+      return "use of an invalid mutex (e.g. uninitialized or destroyed)";
+    case ReportTypeMutexBadUnlock:
+      return "unlock of an unlocked mutex (or by a wrong thread)";
+    case ReportTypeMutexBadReadLock:
+      return "read lock of a write locked mutex";
+    case ReportTypeMutexBadReadUnlock:
+      return "read unlock of a write locked mutex";
+    case ReportTypeSignalUnsafe:
+      return "signal-unsafe call inside of a signal";
+    case ReportTypeErrnoInSignal:
+      return "signal handler spoils errno";
+    case ReportTypeDeadlock:
+      return "lock-order-inversion (potential deadlock)";
+    // No default case so compiler warns us if we miss one
+  }
+  UNREACHABLE("missing case");
+}
+
+#if SANITIZER_MAC
+static const char *const kInterposedFunctionPrefix = "wrap_";
+#else
+static const char *const kInterposedFunctionPrefix = "__interceptor_";
+#endif
+
+void PrintStack(const ReportStack *ent) {
+  if (ent == 0 || ent->frames == 0) {
+    Printf("    [failed to restore the stack]\n\n");
+    return;
+  }
+  SymbolizedStack *frame = ent->frames;
+  for (int i = 0; frame && frame->info.address; frame = frame->next, i++) {
+    InternalScopedString res;
+    RenderFrame(&res, common_flags()->stack_trace_format, i,
+                frame->info.address, &frame->info,
+                common_flags()->symbolize_vs_style,
+                common_flags()->strip_path_prefix, kInterposedFunctionPrefix);
+    Printf("%s\n", res.data());
+  }
+  Printf("\n");
+}
+
+static void PrintMutexSet(Vector<ReportMopMutex> const& mset) {
+  for (uptr i = 0; i < mset.Size(); i++) {
+    if (i == 0)
+      Printf(" (mutexes:");
+    const ReportMopMutex m = mset[i];
+    Printf(" %s M%llu", m.write ? "write" : "read", m.id);
+    Printf(i == mset.Size() - 1 ? ")" : ",");
+  }
+}
+
+static const char *MopDesc(bool first, bool write, bool atomic) {
+  return atomic ? (first ? (write ? "Atomic write" : "Atomic read")
+                : (write ? "Previous atomic write" : "Previous atomic read"))
+                : (first ? (write ? "Write" : "Read")
+                : (write ? "Previous write" : "Previous read"));
+}
+
+static const char *ExternalMopDesc(bool first, bool write) {
+  return first ? (write ? "Modifying" : "Read-only")
+               : (write ? "Previous modifying" : "Previous read-only");
+}
+
+static void PrintMop(const ReportMop *mop, bool first) {
+  Decorator d;
+  char thrbuf[kThreadBufSize];
+  Printf("%s", d.Access());
+  if (mop->external_tag == kExternalTagNone) {
+    Printf("  %s of size %d at %p by %s",
+           MopDesc(first, mop->write, mop->atomic), mop->size,
+           (void *)mop->addr, thread_name(thrbuf, mop->tid));
+  } else {
+    const char *object_type = GetObjectTypeFromTag(mop->external_tag);
+    if (object_type == nullptr)
+        object_type = "external object";
+    Printf("  %s access of %s at %p by %s",
+           ExternalMopDesc(first, mop->write), object_type,
+           (void *)mop->addr, thread_name(thrbuf, mop->tid));
+  }
+  PrintMutexSet(mop->mset);
+  Printf(":\n");
+  Printf("%s", d.Default());
+  PrintStack(mop->stack);
+}
+
+static void PrintLocation(const ReportLocation *loc) {
+  Decorator d;
+  char thrbuf[kThreadBufSize];
+  bool print_stack = false;
+  Printf("%s", d.Location());
+  if (loc->type == ReportLocationGlobal) {
+    const DataInfo &global = loc->global;
+    if (global.size != 0)
+      Printf("  Location is global '%s' of size %zu at %p (%s+0x%zx)\n\n",
+             global.name, global.size, reinterpret_cast<void *>(global.start),
+             StripModuleName(global.module), global.module_offset);
+    else
+      Printf("  Location is global '%s' at %p (%s+0x%zx)\n\n", global.name,
+             reinterpret_cast<void *>(global.start),
+             StripModuleName(global.module), global.module_offset);
+  } else if (loc->type == ReportLocationHeap) {
+    char thrbuf[kThreadBufSize];
+    const char *object_type = GetObjectTypeFromTag(loc->external_tag);
+    if (!object_type) {
+      Printf("  Location is heap block of size %zu at %p allocated by %s:\n",
+             loc->heap_chunk_size,
+             reinterpret_cast<void *>(loc->heap_chunk_start),
+             thread_name(thrbuf, loc->tid));
+    } else {
+      Printf("  Location is %s of size %zu at %p allocated by %s:\n",
+             object_type, loc->heap_chunk_size,
+             reinterpret_cast<void *>(loc->heap_chunk_start),
+             thread_name(thrbuf, loc->tid));
+    }
+    print_stack = true;
+  } else if (loc->type == ReportLocationStack) {
+    Printf("  Location is stack of %s.\n\n", thread_name(thrbuf, loc->tid));
+  } else if (loc->type == ReportLocationTLS) {
+    Printf("  Location is TLS of %s.\n\n", thread_name(thrbuf, loc->tid));
+  } else if (loc->type == ReportLocationFD) {
+    Printf("  Location is file descriptor %d created by %s at:\n",
+        loc->fd, thread_name(thrbuf, loc->tid));
+    print_stack = true;
+  }
+  Printf("%s", d.Default());
+  if (print_stack)
+    PrintStack(loc->stack);
+}
+
+static void PrintMutexShort(const ReportMutex *rm, const char *after) {
+  Decorator d;
+  Printf("%sM%lld%s%s", d.Mutex(), rm->id, d.Default(), after);
+}
+
+static void PrintMutexShortWithAddress(const ReportMutex *rm,
+                                       const char *after) {
+  Decorator d;
+  Printf("%sM%lld (%p)%s%s", d.Mutex(), rm->id,
+         reinterpret_cast<void *>(rm->addr), d.Default(), after);
+}
+
+static void PrintMutex(const ReportMutex *rm) {
+  Decorator d;
+  if (rm->destroyed) {
+    Printf("%s", d.Mutex());
+    Printf("  Mutex M%llu is already destroyed.\n\n", rm->id);
+    Printf("%s", d.Default());
+  } else {
+    Printf("%s", d.Mutex());
+    Printf("  Mutex M%llu (%p) created at:\n", rm->id,
+           reinterpret_cast<void *>(rm->addr));
+    Printf("%s", d.Default());
+    PrintStack(rm->stack);
+  }
+}
+
+static void PrintThread(const ReportThread *rt) {
+  Decorator d;
+  if (rt->id == kMainTid)  // Little sense in describing the main thread.
+    return;
+  Printf("%s", d.ThreadDescription());
+  Printf("  Thread T%d", rt->id);
+  if (rt->name && rt->name[0] != '\0')
+    Printf(" '%s'", rt->name);
+  char thrbuf[kThreadBufSize];
+  const char *thread_status = rt->running ? "running" : "finished";
+  if (rt->thread_type == ThreadType::Worker) {
+    Printf(" (tid=%llu, %s) is a GCD worker thread\n", rt->os_id,
+           thread_status);
+    Printf("\n");
+    Printf("%s", d.Default());
+    return;
+  }
+  Printf(" (tid=%llu, %s) created by %s", rt->os_id, thread_status,
+         thread_name(thrbuf, rt->parent_tid));
+  if (rt->stack)
+    Printf(" at:");
+  Printf("\n");
+  Printf("%s", d.Default());
+  PrintStack(rt->stack);
+}
+
+static void PrintSleep(const ReportStack *s) {
+  Decorator d;
+  Printf("%s", d.Sleep());
+  Printf("  As if synchronized via sleep:\n");
+  Printf("%s", d.Default());
+  PrintStack(s);
+}
+
+static ReportStack *ChooseSummaryStack(const ReportDesc *rep) {
+  if (rep->mops.Size())
+    return rep->mops[0]->stack;
+  if (rep->stacks.Size())
+    return rep->stacks[0];
+  if (rep->mutexes.Size())
+    return rep->mutexes[0]->stack;
+  if (rep->threads.Size())
+    return rep->threads[0]->stack;
+  return 0;
+}
+
+static bool FrameIsInternal(const SymbolizedStack *frame) {
+  if (frame == 0)
+    return false;
+  const char *file = frame->info.file;
+  const char *module = frame->info.module;
+  if (file != 0 &&
+      (internal_strstr(file, "tsan_interceptors_posix.cpp") ||
+       internal_strstr(file, "sanitizer_common_interceptors.inc") ||
+       internal_strstr(file, "tsan_interface_")))
+    return true;
+  if (module != 0 && (internal_strstr(module, "libclang_rt.tsan_")))
+    return true;
+  return false;
+}
+
+static SymbolizedStack *SkipTsanInternalFrames(SymbolizedStack *frames) {
+  while (FrameIsInternal(frames) && frames->next)
+    frames = frames->next;
+  return frames;
+}
+
+void PrintReport(const ReportDesc *rep) {
+  Decorator d;
+  Printf("==================\n");
+  const char *rep_typ_str = ReportTypeString(rep->typ, rep->tag);
+  Printf("%s", d.Warning());
+  Printf("WARNING: ThreadSanitizer: %s (pid=%d)\n", rep_typ_str,
+         (int)internal_getpid());
+  Printf("%s", d.Default());
+
+  if (rep->typ == ReportTypeDeadlock) {
+    char thrbuf[kThreadBufSize];
+    Printf("  Cycle in lock order graph: ");
+    for (uptr i = 0; i < rep->mutexes.Size(); i++)
+      PrintMutexShortWithAddress(rep->mutexes[i], " => ");
+    PrintMutexShort(rep->mutexes[0], "\n\n");
+    CHECK_GT(rep->mutexes.Size(), 0U);
+    CHECK_EQ(rep->mutexes.Size() * (flags()->second_deadlock_stack ? 2 : 1),
+             rep->stacks.Size());
+    for (uptr i = 0; i < rep->mutexes.Size(); i++) {
+      Printf("  Mutex ");
+      PrintMutexShort(rep->mutexes[(i + 1) % rep->mutexes.Size()],
+                      " acquired here while holding mutex ");
+      PrintMutexShort(rep->mutexes[i], " in ");
+      Printf("%s", d.ThreadDescription());
+      Printf("%s:\n", thread_name(thrbuf, rep->unique_tids[i]));
+      Printf("%s", d.Default());
+      if (flags()->second_deadlock_stack) {
+        PrintStack(rep->stacks[2*i]);
+        Printf("  Mutex ");
+        PrintMutexShort(rep->mutexes[i],
+                        " previously acquired by the same thread here:\n");
+        PrintStack(rep->stacks[2*i+1]);
+      } else {
+        PrintStack(rep->stacks[i]);
+        if (i == 0)
+          Printf("    Hint: use TSAN_OPTIONS=second_deadlock_stack=1 "
+                 "to get more informative warning message\n\n");
+      }
+    }
+  } else {
+    for (uptr i = 0; i < rep->stacks.Size(); i++) {
+      if (i)
+        Printf("  and:\n");
+      PrintStack(rep->stacks[i]);
+    }
+  }
+
+  for (uptr i = 0; i < rep->mops.Size(); i++)
+    PrintMop(rep->mops[i], i == 0);
+
+  if (rep->sleep)
+    PrintSleep(rep->sleep);
+
+  for (uptr i = 0; i < rep->locs.Size(); i++)
+    PrintLocation(rep->locs[i]);
+
+  if (rep->typ != ReportTypeDeadlock) {
+    for (uptr i = 0; i < rep->mutexes.Size(); i++)
+      PrintMutex(rep->mutexes[i]);
+  }
+
+  for (uptr i = 0; i < rep->threads.Size(); i++)
+    PrintThread(rep->threads[i]);
+
+  if (rep->typ == ReportTypeThreadLeak && rep->count > 1)
+    Printf("  And %d more similar thread leaks.\n\n", rep->count - 1);
+
+  if (ReportStack *stack = ChooseSummaryStack(rep)) {
+    if (SymbolizedStack *frame = SkipTsanInternalFrames(stack->frames))
+      ReportErrorSummary(rep_typ_str, frame->info);
+  }
+
+  if (common_flags()->print_module_map == 2)
+    DumpProcessMap();
+
+  Printf("==================\n");
+}
+
+#else  // #if !SANITIZER_GO
+
+const Tid kMainGoroutineId = 1;
+
+void PrintStack(const ReportStack *ent) {
+  if (ent == 0 || ent->frames == 0) {
+    Printf("  [failed to restore the stack]\n");
+    return;
+  }
+  SymbolizedStack *frame = ent->frames;
+  for (int i = 0; frame; frame = frame->next, i++) {
+    const AddressInfo &info = frame->info;
+    Printf("  %s()\n      %s:%d +0x%zx\n", info.function,
+           StripPathPrefix(info.file, common_flags()->strip_path_prefix),
+           info.line, info.module_offset);
+  }
+}
+
+static void PrintMop(const ReportMop *mop, bool first) {
+  Printf("\n");
+  Printf("%s at %p by ",
+         (first ? (mop->write ? "Write" : "Read")
+                : (mop->write ? "Previous write" : "Previous read")),
+         reinterpret_cast<void *>(mop->addr));
+  if (mop->tid == kMainGoroutineId)
+    Printf("main goroutine:\n");
+  else
+    Printf("goroutine %d:\n", mop->tid);
+  PrintStack(mop->stack);
+}
+
+static void PrintLocation(const ReportLocation *loc) {
+  switch (loc->type) {
+  case ReportLocationHeap: {
+    Printf("\n");
+    Printf("Heap block of size %zu at %p allocated by ", loc->heap_chunk_size,
+           reinterpret_cast<void *>(loc->heap_chunk_start));
+    if (loc->tid == kMainGoroutineId)
+      Printf("main goroutine:\n");
+    else
+      Printf("goroutine %d:\n", loc->tid);
+    PrintStack(loc->stack);
+    break;
+  }
+  case ReportLocationGlobal: {
+    Printf("\n");
+    Printf("Global var %s of size %zu at %p declared at %s:%zu\n",
+           loc->global.name, loc->global.size,
+           reinterpret_cast<void *>(loc->global.start), loc->global.file,
+           loc->global.line);
+    break;
+  }
+  default:
+    break;
+  }
+}
+
+static void PrintThread(const ReportThread *rt) {
+  if (rt->id == kMainGoroutineId)
+    return;
+  Printf("\n");
+  Printf("Goroutine %d (%s) created at:\n",
+    rt->id, rt->running ? "running" : "finished");
+  PrintStack(rt->stack);
+}
+
+void PrintReport(const ReportDesc *rep) {
+  Printf("==================\n");
+  if (rep->typ == ReportTypeRace) {
+    Printf("WARNING: DATA RACE");
+    for (uptr i = 0; i < rep->mops.Size(); i++)
+      PrintMop(rep->mops[i], i == 0);
+    for (uptr i = 0; i < rep->locs.Size(); i++)
+      PrintLocation(rep->locs[i]);
+    for (uptr i = 0; i < rep->threads.Size(); i++)
+      PrintThread(rep->threads[i]);
+  } else if (rep->typ == ReportTypeDeadlock) {
+    Printf("WARNING: DEADLOCK\n");
+    for (uptr i = 0; i < rep->mutexes.Size(); i++) {
+      Printf("Goroutine %d lock mutex %llu while holding mutex %llu:\n", 999,
+             rep->mutexes[i]->id,
+             rep->mutexes[(i + 1) % rep->mutexes.Size()]->id);
+      PrintStack(rep->stacks[2*i]);
+      Printf("\n");
+      Printf("Mutex %llu was previously locked here:\n",
+             rep->mutexes[(i + 1) % rep->mutexes.Size()]->id);
+      PrintStack(rep->stacks[2*i + 1]);
+      Printf("\n");
+    }
+  }
+  Printf("==================\n");
+}
+
+#endif
+
+}  // namespace __tsan
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_report.h b/compiler-rt/lib/tsan/rtl-old/tsan_report.h
new file mode 100644
index 000000000000..d68c2db88828
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_report.h
@@ -0,0 +1,127 @@
+//===-- tsan_report.h -------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_REPORT_H
+#define TSAN_REPORT_H
+
+#include "sanitizer_common/sanitizer_symbolizer.h"
+#include "sanitizer_common/sanitizer_thread_registry.h"
+#include "sanitizer_common/sanitizer_vector.h"
+#include "tsan_defs.h"
+
+namespace __tsan {
+
+enum ReportType {
+  ReportTypeRace,
+  ReportTypeVptrRace,
+  ReportTypeUseAfterFree,
+  ReportTypeVptrUseAfterFree,
+  ReportTypeExternalRace,
+  ReportTypeThreadLeak,
+  ReportTypeMutexDestroyLocked,
+  ReportTypeMutexDoubleLock,
+  ReportTypeMutexInvalidAccess,
+  ReportTypeMutexBadUnlock,
+  ReportTypeMutexBadReadLock,
+  ReportTypeMutexBadReadUnlock,
+  ReportTypeSignalUnsafe,
+  ReportTypeErrnoInSignal,
+  ReportTypeDeadlock
+};
+
+struct ReportStack {
+  SymbolizedStack *frames = nullptr;
+  bool suppressable = false;
+};
+
+struct ReportMopMutex {
+  u64 id;
+  bool write;
+};
+
+struct ReportMop {
+  int tid;
+  uptr addr;
+  int size;
+  bool write;
+  bool atomic;
+  uptr external_tag;
+  Vector<ReportMopMutex> mset;
+  ReportStack *stack;
+
+  ReportMop();
+};
+
+enum ReportLocationType {
+  ReportLocationGlobal,
+  ReportLocationHeap,
+  ReportLocationStack,
+  ReportLocationTLS,
+  ReportLocationFD
+};
+
+struct ReportLocation {
+  ReportLocationType type = ReportLocationGlobal;
+  DataInfo global = {};
+  uptr heap_chunk_start = 0;
+  uptr heap_chunk_size = 0;
+  uptr external_tag = 0;
+  Tid tid = kInvalidTid;
+  int fd = 0;
+  bool suppressable = false;
+  ReportStack *stack = nullptr;
+};
+
+struct ReportThread {
+  Tid id;
+  tid_t os_id;
+  bool running;
+  ThreadType thread_type;
+  char *name;
+  Tid parent_tid;
+  ReportStack *stack;
+};
+
+struct ReportMutex {
+  u64 id;
+  uptr addr;
+  bool destroyed;
+  ReportStack *stack;
+};
+
+class ReportDesc {
+ public:
+  ReportType typ;
+  uptr tag;
+  Vector<ReportStack*> stacks;
+  Vector<ReportMop*> mops;
+  Vector<ReportLocation*> locs;
+  Vector<ReportMutex*> mutexes;
+  Vector<ReportThread*> threads;
+  Vector<Tid> unique_tids;
+  ReportStack *sleep;
+  int count;
+
+  ReportDesc();
+  ~ReportDesc();
+
+ private:
+  ReportDesc(const ReportDesc&);
+  void operator = (const ReportDesc&);
+};
+
+// Format and output the report to the console/log. No additional logic.
+void PrintReport(const ReportDesc *rep);
+void PrintStack(const ReportStack *stack);
+
+}  // namespace __tsan
+
+#endif  // TSAN_REPORT_H
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_rtl.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_rtl.cpp
new file mode 100644
index 000000000000..c14af9788e32
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_rtl.cpp
@@ -0,0 +1,811 @@
+//===-- tsan_rtl.cpp ------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// Main file (entry points) for the TSan run-time.
+//===----------------------------------------------------------------------===//
+
+#include "tsan_rtl.h"
+
+#include "sanitizer_common/sanitizer_atomic.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_file.h"
+#include "sanitizer_common/sanitizer_libc.h"
+#include "sanitizer_common/sanitizer_placement_new.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
+#include "sanitizer_common/sanitizer_symbolizer.h"
+#include "tsan_defs.h"
+#include "tsan_interface.h"
+#include "tsan_mman.h"
+#include "tsan_platform.h"
+#include "tsan_suppressions.h"
+#include "tsan_symbolize.h"
+#include "ubsan/ubsan_init.h"
+
+volatile int __tsan_resumed = 0;
+
+extern "C" void __tsan_resume() {
+  __tsan_resumed = 1;
+}
+
+SANITIZER_WEAK_DEFAULT_IMPL
+void __tsan_test_only_on_fork() {}
+
+namespace __tsan {
+
+#if !SANITIZER_GO
+void (*on_initialize)(void);
+int (*on_finalize)(int);
+#endif
+
+#if !SANITIZER_GO && !SANITIZER_MAC
+__attribute__((tls_model("initial-exec")))
+THREADLOCAL char cur_thread_placeholder[sizeof(ThreadState)] ALIGNED(
+    SANITIZER_CACHE_LINE_SIZE);
+#endif
+static char ctx_placeholder[sizeof(Context)] ALIGNED(SANITIZER_CACHE_LINE_SIZE);
+Context *ctx;
+
+// Can be overriden by a front-end.
+#ifdef TSAN_EXTERNAL_HOOKS
+bool OnFinalize(bool failed);
+void OnInitialize();
+#else
+#include <dlfcn.h>
+SANITIZER_WEAK_CXX_DEFAULT_IMPL
+bool OnFinalize(bool failed) {
+#if !SANITIZER_GO
+  if (on_finalize)
+    return on_finalize(failed);
+#endif
+  return failed;
+}
+SANITIZER_WEAK_CXX_DEFAULT_IMPL
+void OnInitialize() {
+#if !SANITIZER_GO
+  if (on_initialize)
+    on_initialize();
+#endif
+}
+#endif
+
+static ThreadContextBase *CreateThreadContext(Tid tid) {
+  // Map thread trace when context is created.
+  char name[50];
+  internal_snprintf(name, sizeof(name), "trace %u", tid);
+  MapThreadTrace(GetThreadTrace(tid), TraceSize() * sizeof(Event), name);
+  const uptr hdr = GetThreadTraceHeader(tid);
+  internal_snprintf(name, sizeof(name), "trace header %u", tid);
+  MapThreadTrace(hdr, sizeof(Trace), name);
+  new((void*)hdr) Trace();
+  // We are going to use only a small part of the trace with the default
+  // value of history_size. However, the constructor writes to the whole trace.
+  // Release the unused part.
+  uptr hdr_end = hdr + sizeof(Trace);
+  hdr_end -= sizeof(TraceHeader) * (kTraceParts - TraceParts());
+  hdr_end = RoundUp(hdr_end, GetPageSizeCached());
+  if (hdr_end < hdr + sizeof(Trace)) {
+    ReleaseMemoryPagesToOS(hdr_end, hdr + sizeof(Trace));
+    uptr unused = hdr + sizeof(Trace) - hdr_end;
+    if (hdr_end != (uptr)MmapFixedNoAccess(hdr_end, unused)) {
+      Report("ThreadSanitizer: failed to mprotect [0x%zx-0x%zx) \n", hdr_end,
+             unused);
+      CHECK("unable to mprotect" && 0);
+    }
+  }
+  return New<ThreadContext>(tid);
+}
+
+#if !SANITIZER_GO
+static const u32 kThreadQuarantineSize = 16;
+#else
+static const u32 kThreadQuarantineSize = 64;
+#endif
+
+Context::Context()
+    : initialized(),
+      report_mtx(MutexTypeReport),
+      nreported(),
+      thread_registry(CreateThreadContext, kMaxTid, kThreadQuarantineSize,
+                      kMaxTidReuse),
+      racy_mtx(MutexTypeRacy),
+      racy_stacks(),
+      racy_addresses(),
+      fired_suppressions_mtx(MutexTypeFired),
+      clock_alloc(LINKER_INITIALIZED, "clock allocator") {
+  fired_suppressions.reserve(8);
+}
+
+// The objects are allocated in TLS, so one may rely on zero-initialization.
+ThreadState::ThreadState(Context *ctx, Tid tid, int unique_id, u64 epoch,
+                         unsigned reuse_count, uptr stk_addr, uptr stk_size,
+                         uptr tls_addr, uptr tls_size)
+    : fast_state(tid, epoch)
+      // Do not touch these, rely on zero initialization,
+      // they may be accessed before the ctor.
+      // , ignore_reads_and_writes()
+      // , ignore_interceptors()
+      ,
+      clock(tid, reuse_count)
+#if !SANITIZER_GO
+      ,
+      jmp_bufs()
+#endif
+      ,
+      tid(tid),
+      unique_id(unique_id),
+      stk_addr(stk_addr),
+      stk_size(stk_size),
+      tls_addr(tls_addr),
+      tls_size(tls_size)
+#if !SANITIZER_GO
+      ,
+      last_sleep_clock(tid)
+#endif
+{
+  CHECK_EQ(reinterpret_cast<uptr>(this) % SANITIZER_CACHE_LINE_SIZE, 0);
+#if !SANITIZER_GO
+  // C/C++ uses fixed size shadow stack.
+  const int kInitStackSize = kShadowStackSize;
+  shadow_stack = static_cast<uptr *>(
+      MmapNoReserveOrDie(kInitStackSize * sizeof(uptr), "shadow stack"));
+  SetShadowRegionHugePageMode(reinterpret_cast<uptr>(shadow_stack),
+                              kInitStackSize * sizeof(uptr));
+#else
+  // Go uses malloc-allocated shadow stack with dynamic size.
+  const int kInitStackSize = 8;
+  shadow_stack = static_cast<uptr *>(Alloc(kInitStackSize * sizeof(uptr)));
+#endif
+  shadow_stack_pos = shadow_stack;
+  shadow_stack_end = shadow_stack + kInitStackSize;
+}
+
+#if !SANITIZER_GO
+void MemoryProfiler(u64 uptime) {
+  if (ctx->memprof_fd == kInvalidFd)
+    return;
+  InternalMmapVector<char> buf(4096);
+  WriteMemoryProfile(buf.data(), buf.size(), uptime);
+  WriteToFile(ctx->memprof_fd, buf.data(), internal_strlen(buf.data()));
+}
+
+void InitializeMemoryProfiler() {
+  ctx->memprof_fd = kInvalidFd;
+  const char *fname = flags()->profile_memory;
+  if (!fname || !fname[0])
+    return;
+  if (internal_strcmp(fname, "stdout") == 0) {
+    ctx->memprof_fd = 1;
+  } else if (internal_strcmp(fname, "stderr") == 0) {
+    ctx->memprof_fd = 2;
+  } else {
+    InternalScopedString filename;
+    filename.append("%s.%d", fname, (int)internal_getpid());
+    ctx->memprof_fd = OpenFile(filename.data(), WrOnly);
+    if (ctx->memprof_fd == kInvalidFd) {
+      Printf("ThreadSanitizer: failed to open memory profile file '%s'\n",
+             filename.data());
+      return;
+    }
+  }
+  MemoryProfiler(0);
+  MaybeSpawnBackgroundThread();
+}
+
+static void *BackgroundThread(void *arg) {
+  // This is a non-initialized non-user thread, nothing to see here.
+  // We don't use ScopedIgnoreInterceptors, because we want ignores to be
+  // enabled even when the thread function exits (e.g. during pthread thread
+  // shutdown code).
+  cur_thread_init()->ignore_interceptors++;
+  const u64 kMs2Ns = 1000 * 1000;
+  const u64 start = NanoTime();
+
+  u64 last_flush = NanoTime();
+  uptr last_rss = 0;
+  for (int i = 0;
+      atomic_load(&ctx->stop_background_thread, memory_order_relaxed) == 0;
+      i++) {
+    SleepForMillis(100);
+    u64 now = NanoTime();
+
+    // Flush memory if requested.
+    if (flags()->flush_memory_ms > 0) {
+      if (last_flush + flags()->flush_memory_ms * kMs2Ns < now) {
+        VPrintf(1, "ThreadSanitizer: periodic memory flush\n");
+        FlushShadowMemory();
+        last_flush = NanoTime();
+      }
+    }
+    if (flags()->memory_limit_mb > 0) {
+      uptr rss = GetRSS();
+      uptr limit = uptr(flags()->memory_limit_mb) << 20;
+      VPrintf(1, "ThreadSanitizer: memory flush check"
+                 " RSS=%llu LAST=%llu LIMIT=%llu\n",
+              (u64)rss >> 20, (u64)last_rss >> 20, (u64)limit >> 20);
+      if (2 * rss > limit + last_rss) {
+        VPrintf(1, "ThreadSanitizer: flushing memory due to RSS\n");
+        FlushShadowMemory();
+        rss = GetRSS();
+        VPrintf(1, "ThreadSanitizer: memory flushed RSS=%llu\n", (u64)rss>>20);
+      }
+      last_rss = rss;
+    }
+
+    MemoryProfiler(now - start);
+
+    // Flush symbolizer cache if requested.
+    if (flags()->flush_symbolizer_ms > 0) {
+      u64 last = atomic_load(&ctx->last_symbolize_time_ns,
+                             memory_order_relaxed);
+      if (last != 0 && last + flags()->flush_symbolizer_ms * kMs2Ns < now) {
+        Lock l(&ctx->report_mtx);
+        ScopedErrorReportLock l2;
+        SymbolizeFlush();
+        atomic_store(&ctx->last_symbolize_time_ns, 0, memory_order_relaxed);
+      }
+    }
+  }
+  return nullptr;
+}
+
+static void StartBackgroundThread() {
+  ctx->background_thread = internal_start_thread(&BackgroundThread, 0);
+}
+
+#ifndef __mips__
+static void StopBackgroundThread() {
+  atomic_store(&ctx->stop_background_thread, 1, memory_order_relaxed);
+  internal_join_thread(ctx->background_thread);
+  ctx->background_thread = 0;
+}
+#endif
+#endif
+
+void DontNeedShadowFor(uptr addr, uptr size) {
+  ReleaseMemoryPagesToOS(reinterpret_cast<uptr>(MemToShadow(addr)),
+                         reinterpret_cast<uptr>(MemToShadow(addr + size)));
+}
+
+#if !SANITIZER_GO
+// We call UnmapShadow before the actual munmap, at that point we don't yet
+// know if the provided address/size are sane. We can't call UnmapShadow
+// after the actual munmap becuase at that point the memory range can
+// already be reused for something else, so we can't rely on the munmap
+// return value to understand is the values are sane.
+// While calling munmap with insane values (non-canonical address, negative
+// size, etc) is an error, the kernel won't crash. We must also try to not
+// crash as the failure mode is very confusing (paging fault inside of the
+// runtime on some derived shadow address).
+static bool IsValidMmapRange(uptr addr, uptr size) {
+  if (size == 0)
+    return true;
+  if (static_cast<sptr>(size) < 0)
+    return false;
+  if (!IsAppMem(addr) || !IsAppMem(addr + size - 1))
+    return false;
+  // Check that if the start of the region belongs to one of app ranges,
+  // end of the region belongs to the same region.
+  const uptr ranges[][2] = {
+      {LoAppMemBeg(), LoAppMemEnd()},
+      {MidAppMemBeg(), MidAppMemEnd()},
+      {HiAppMemBeg(), HiAppMemEnd()},
+  };
+  for (auto range : ranges) {
+    if (addr >= range[0] && addr < range[1])
+      return addr + size <= range[1];
+  }
+  return false;
+}
+
+void UnmapShadow(ThreadState *thr, uptr addr, uptr size) {
+  if (size == 0 || !IsValidMmapRange(addr, size))
+    return;
+  DontNeedShadowFor(addr, size);
+  ScopedGlobalProcessor sgp;
+  ctx->metamap.ResetRange(thr->proc(), addr, size);
+}
+#endif
+
+void MapShadow(uptr addr, uptr size) {
+  // Global data is not 64K aligned, but there are no adjacent mappings,
+  // so we can get away with unaligned mapping.
+  // CHECK_EQ(addr, addr & ~((64 << 10) - 1));  // windows wants 64K alignment
+  const uptr kPageSize = GetPageSizeCached();
+  uptr shadow_begin = RoundDownTo((uptr)MemToShadow(addr), kPageSize);
+  uptr shadow_end = RoundUpTo((uptr)MemToShadow(addr + size), kPageSize);
+  if (!MmapFixedSuperNoReserve(shadow_begin, shadow_end - shadow_begin,
+                               "shadow"))
+    Die();
+
+  // Meta shadow is 2:1, so tread carefully.
+  static bool data_mapped = false;
+  static uptr mapped_meta_end = 0;
+  uptr meta_begin = (uptr)MemToMeta(addr);
+  uptr meta_end = (uptr)MemToMeta(addr + size);
+  meta_begin = RoundDownTo(meta_begin, 64 << 10);
+  meta_end = RoundUpTo(meta_end, 64 << 10);
+  if (!data_mapped) {
+    // First call maps data+bss.
+    data_mapped = true;
+    if (!MmapFixedSuperNoReserve(meta_begin, meta_end - meta_begin,
+                                 "meta shadow"))
+      Die();
+  } else {
+    // Mapping continuous heap.
+    // Windows wants 64K alignment.
+    meta_begin = RoundDownTo(meta_begin, 64 << 10);
+    meta_end = RoundUpTo(meta_end, 64 << 10);
+    if (meta_end <= mapped_meta_end)
+      return;
+    if (meta_begin < mapped_meta_end)
+      meta_begin = mapped_meta_end;
+    if (!MmapFixedSuperNoReserve(meta_begin, meta_end - meta_begin,
+                                 "meta shadow"))
+      Die();
+    mapped_meta_end = meta_end;
+  }
+  VPrintf(2, "mapped meta shadow for (0x%zx-0x%zx) at (0x%zx-0x%zx)\n", addr,
+          addr + size, meta_begin, meta_end);
+}
+
+void MapThreadTrace(uptr addr, uptr size, const char *name) {
+  DPrintf("#0: Mapping trace at 0x%zx-0x%zx(0x%zx)\n", addr, addr + size, size);
+  CHECK_GE(addr, TraceMemBeg());
+  CHECK_LE(addr + size, TraceMemEnd());
+  CHECK_EQ(addr, addr & ~((64 << 10) - 1));  // windows wants 64K alignment
+  if (!MmapFixedSuperNoReserve(addr, size, name)) {
+    Printf("FATAL: ThreadSanitizer can not mmap thread trace (0x%zx/0x%zx)\n",
+           addr, size);
+    Die();
+  }
+}
+
+#if !SANITIZER_GO
+static void OnStackUnwind(const SignalContext &sig, const void *,
+                          BufferedStackTrace *stack) {
+  stack->Unwind(StackTrace::GetNextInstructionPc(sig.pc), sig.bp, sig.context,
+                common_flags()->fast_unwind_on_fatal);
+}
+
+static void TsanOnDeadlySignal(int signo, void *siginfo, void *context) {
+  HandleDeadlySignal(siginfo, context, GetTid(), &OnStackUnwind, nullptr);
+}
+#endif
+
+void CheckUnwind() {
+  // There is high probability that interceptors will check-fail as well,
+  // on the other hand there is no sense in processing interceptors
+  // since we are going to die soon.
+  ScopedIgnoreInterceptors ignore;
+#if !SANITIZER_GO
+  cur_thread()->ignore_sync++;
+  cur_thread()->ignore_reads_and_writes++;
+#endif
+  PrintCurrentStackSlow(StackTrace::GetCurrentPc());
+}
+
+bool is_initialized;
+
+void Initialize(ThreadState *thr) {
+  // Thread safe because done before all threads exist.
+  if (is_initialized)
+    return;
+  is_initialized = true;
+  // We are not ready to handle interceptors yet.
+  ScopedIgnoreInterceptors ignore;
+  SanitizerToolName = "ThreadSanitizer";
+  // Install tool-specific callbacks in sanitizer_common.
+  SetCheckUnwindCallback(CheckUnwind);
+
+  ctx = new(ctx_placeholder) Context;
+  const char *env_name = SANITIZER_GO ? "GORACE" : "TSAN_OPTIONS";
+  const char *options = GetEnv(env_name);
+  CacheBinaryName();
+  CheckASLR();
+  InitializeFlags(&ctx->flags, options, env_name);
+  AvoidCVE_2016_2143();
+  __sanitizer::InitializePlatformEarly();
+  __tsan::InitializePlatformEarly();
+
+#if !SANITIZER_GO
+  // Re-exec ourselves if we need to set additional env or command line args.
+  MaybeReexec();
+
+  InitializeAllocator();
+  ReplaceSystemMalloc();
+#endif
+  if (common_flags()->detect_deadlocks)
+    ctx->dd = DDetector::Create(flags());
+  Processor *proc = ProcCreate();
+  ProcWire(proc, thr);
+  InitializeInterceptors();
+  InitializePlatform();
+  InitializeDynamicAnnotations();
+#if !SANITIZER_GO
+  InitializeShadowMemory();
+  InitializeAllocatorLate();
+  InstallDeadlySignalHandlers(TsanOnDeadlySignal);
+#endif
+  // Setup correct file descriptor for error reports.
+  __sanitizer_set_report_path(common_flags()->log_path);
+  InitializeSuppressions();
+#if !SANITIZER_GO
+  InitializeLibIgnore();
+  Symbolizer::GetOrInit()->AddHooks(EnterSymbolizer, ExitSymbolizer);
+#endif
+
+  VPrintf(1, "***** Running under ThreadSanitizer v2 (pid %d) *****\n",
+          (int)internal_getpid());
+
+  // Initialize thread 0.
+  Tid tid = ThreadCreate(thr, 0, 0, true);
+  CHECK_EQ(tid, kMainTid);
+  ThreadStart(thr, tid, GetTid(), ThreadType::Regular);
+#if TSAN_CONTAINS_UBSAN
+  __ubsan::InitAsPlugin();
+#endif
+  ctx->initialized = true;
+
+#if !SANITIZER_GO
+  Symbolizer::LateInitialize();
+  InitializeMemoryProfiler();
+#endif
+
+  if (flags()->stop_on_start) {
+    Printf("ThreadSanitizer is suspended at startup (pid %d)."
+           " Call __tsan_resume().\n",
+           (int)internal_getpid());
+    while (__tsan_resumed == 0) {}
+  }
+
+  OnInitialize();
+}
+
+void MaybeSpawnBackgroundThread() {
+  // On MIPS, TSan initialization is run before
+  // __pthread_initialize_minimal_internal() is finished, so we can not spawn
+  // new threads.
+#if !SANITIZER_GO && !defined(__mips__)
+  static atomic_uint32_t bg_thread = {};
+  if (atomic_load(&bg_thread, memory_order_relaxed) == 0 &&
+      atomic_exchange(&bg_thread, 1, memory_order_relaxed) == 0) {
+    StartBackgroundThread();
+    SetSandboxingCallback(StopBackgroundThread);
+  }
+#endif
+}
+
+
+int Finalize(ThreadState *thr) {
+  bool failed = false;
+
+  if (common_flags()->print_module_map == 1)
+    DumpProcessMap();
+
+  if (flags()->atexit_sleep_ms > 0 && ThreadCount(thr) > 1)
+    SleepForMillis(flags()->atexit_sleep_ms);
+
+  // Wait for pending reports.
+  ctx->report_mtx.Lock();
+  { ScopedErrorReportLock l; }
+  ctx->report_mtx.Unlock();
+
+#if !SANITIZER_GO
+  if (Verbosity()) AllocatorPrintStats();
+#endif
+
+  ThreadFinalize(thr);
+
+  if (ctx->nreported) {
+    failed = true;
+#if !SANITIZER_GO
+    Printf("ThreadSanitizer: reported %d warnings\n", ctx->nreported);
+#else
+    Printf("Found %d data race(s)\n", ctx->nreported);
+#endif
+  }
+
+  if (common_flags()->print_suppressions)
+    PrintMatchedSuppressions();
+
+  failed = OnFinalize(failed);
+
+  return failed ? common_flags()->exitcode : 0;
+}
+
+#if !SANITIZER_GO
+void ForkBefore(ThreadState *thr, uptr pc) NO_THREAD_SAFETY_ANALYSIS {
+  ctx->thread_registry.Lock();
+  ctx->report_mtx.Lock();
+  ScopedErrorReportLock::Lock();
+  AllocatorLock();
+  // Suppress all reports in the pthread_atfork callbacks.
+  // Reports will deadlock on the report_mtx.
+  // We could ignore sync operations as well,
+  // but so far it's unclear if it will do more good or harm.
+  // Unnecessarily ignoring things can lead to false positives later.
+  thr->suppress_reports++;
+  // On OS X, REAL(fork) can call intercepted functions (OSSpinLockLock), and
+  // we'll assert in CheckNoLocks() unless we ignore interceptors.
+  // On OS X libSystem_atfork_prepare/parent/child callbacks are called
+  // after/before our callbacks and they call free.
+  thr->ignore_interceptors++;
+  // Disables memory write in OnUserAlloc/Free.
+  thr->ignore_reads_and_writes++;
+
+  __tsan_test_only_on_fork();
+}
+
+void ForkParentAfter(ThreadState *thr, uptr pc) NO_THREAD_SAFETY_ANALYSIS {
+  thr->suppress_reports--;  // Enabled in ForkBefore.
+  thr->ignore_interceptors--;
+  thr->ignore_reads_and_writes--;
+  AllocatorUnlock();
+  ScopedErrorReportLock::Unlock();
+  ctx->report_mtx.Unlock();
+  ctx->thread_registry.Unlock();
+}
+
+void ForkChildAfter(ThreadState *thr, uptr pc,
+                    bool start_thread) NO_THREAD_SAFETY_ANALYSIS {
+  thr->suppress_reports--;  // Enabled in ForkBefore.
+  thr->ignore_interceptors--;
+  thr->ignore_reads_and_writes--;
+  AllocatorUnlock();
+  ScopedErrorReportLock::Unlock();
+  ctx->report_mtx.Unlock();
+  ctx->thread_registry.Unlock();
+
+  uptr nthread = 0;
+  ctx->thread_registry.GetNumberOfThreads(0, 0, &nthread /* alive threads */);
+  VPrintf(1, "ThreadSanitizer: forked new process with pid %d,"
+      " parent had %d threads\n", (int)internal_getpid(), (int)nthread);
+  if (nthread == 1) {
+    if (start_thread)
+      StartBackgroundThread();
+  } else {
+    // We've just forked a multi-threaded process. We cannot reasonably function
+    // after that (some mutexes may be locked before fork). So just enable
+    // ignores for everything in the hope that we will exec soon.
+    ctx->after_multithreaded_fork = true;
+    thr->ignore_interceptors++;
+    ThreadIgnoreBegin(thr, pc);
+    ThreadIgnoreSyncBegin(thr, pc);
+  }
+}
+#endif
+
+#if SANITIZER_GO
+NOINLINE
+void GrowShadowStack(ThreadState *thr) {
+  const int sz = thr->shadow_stack_end - thr->shadow_stack;
+  const int newsz = 2 * sz;
+  auto *newstack = (uptr *)Alloc(newsz * sizeof(uptr));
+  internal_memcpy(newstack, thr->shadow_stack, sz * sizeof(uptr));
+  Free(thr->shadow_stack);
+  thr->shadow_stack = newstack;
+  thr->shadow_stack_pos = newstack + sz;
+  thr->shadow_stack_end = newstack + newsz;
+}
+#endif
+
+StackID CurrentStackId(ThreadState *thr, uptr pc) {
+  if (!thr->is_inited)  // May happen during bootstrap.
+    return kInvalidStackID;
+  if (pc != 0) {
+#if !SANITIZER_GO
+    DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
+#else
+    if (thr->shadow_stack_pos == thr->shadow_stack_end)
+      GrowShadowStack(thr);
+#endif
+    thr->shadow_stack_pos[0] = pc;
+    thr->shadow_stack_pos++;
+  }
+  StackID id = StackDepotPut(
+      StackTrace(thr->shadow_stack, thr->shadow_stack_pos - thr->shadow_stack));
+  if (pc != 0)
+    thr->shadow_stack_pos--;
+  return id;
+}
+
+namespace v3 {
+
+NOINLINE
+void TraceSwitchPart(ThreadState *thr) {
+  Trace *trace = &thr->tctx->trace;
+  Event *pos = reinterpret_cast<Event *>(atomic_load_relaxed(&thr->trace_pos));
+  DCHECK_EQ(reinterpret_cast<uptr>(pos + 1) & TracePart::kAlignment, 0);
+  auto *part = trace->parts.Back();
+  DPrintf("TraceSwitchPart part=%p pos=%p\n", part, pos);
+  if (part) {
+    // We can get here when we still have space in the current trace part.
+    // The fast-path check in TraceAcquire has false positives in the middle of
+    // the part. Check if we are indeed at the end of the current part or not,
+    // and fill any gaps with NopEvent's.
+    Event *end = &part->events[TracePart::kSize];
+    DCHECK_GE(pos, &part->events[0]);
+    DCHECK_LE(pos, end);
+    if (pos + 1 < end) {
+      if ((reinterpret_cast<uptr>(pos) & TracePart::kAlignment) ==
+          TracePart::kAlignment)
+        *pos++ = NopEvent;
+      *pos++ = NopEvent;
+      DCHECK_LE(pos + 2, end);
+      atomic_store_relaxed(&thr->trace_pos, reinterpret_cast<uptr>(pos));
+      // Ensure we setup trace so that the next TraceAcquire
+      // won't detect trace part end.
+      Event *ev;
+      CHECK(TraceAcquire(thr, &ev));
+      return;
+    }
+    // We are indeed at the end.
+    for (; pos < end; pos++) *pos = NopEvent;
+  }
+#if !SANITIZER_GO
+  if (ctx->after_multithreaded_fork) {
+    // We just need to survive till exec.
+    CHECK(part);
+    atomic_store_relaxed(&thr->trace_pos,
+                         reinterpret_cast<uptr>(&part->events[0]));
+    return;
+  }
+#endif
+  part = new (MmapOrDie(sizeof(TracePart), "TracePart")) TracePart();
+  part->trace = trace;
+  thr->trace_prev_pc = 0;
+  {
+    Lock lock(&trace->mtx);
+    trace->parts.PushBack(part);
+    atomic_store_relaxed(&thr->trace_pos,
+                         reinterpret_cast<uptr>(&part->events[0]));
+  }
+  // Make this part self-sufficient by restoring the current stack
+  // and mutex set in the beginning of the trace.
+  TraceTime(thr);
+  for (uptr *pos = &thr->shadow_stack[0]; pos < thr->shadow_stack_pos; pos++)
+    CHECK(TryTraceFunc(thr, *pos));
+  for (uptr i = 0; i < thr->mset.Size(); i++) {
+    MutexSet::Desc d = thr->mset.Get(i);
+    TraceMutexLock(thr, d.write ? EventType::kLock : EventType::kRLock, 0,
+                   d.addr, d.stack_id);
+  }
+}
+
+}  // namespace v3
+
+void TraceSwitch(ThreadState *thr) {
+#if !SANITIZER_GO
+  if (ctx->after_multithreaded_fork)
+    return;
+#endif
+  thr->nomalloc++;
+  Trace *thr_trace = ThreadTrace(thr->tid);
+  Lock l(&thr_trace->mtx);
+  unsigned trace = (thr->fast_state.epoch() / kTracePartSize) % TraceParts();
+  TraceHeader *hdr = &thr_trace->headers[trace];
+  hdr->epoch0 = thr->fast_state.epoch();
+  ObtainCurrentStack(thr, 0, &hdr->stack0);
+  hdr->mset0 = thr->mset;
+  thr->nomalloc--;
+}
+
+Trace *ThreadTrace(Tid tid) { return (Trace *)GetThreadTraceHeader(tid); }
+
+uptr TraceTopPC(ThreadState *thr) {
+  Event *events = (Event*)GetThreadTrace(thr->tid);
+  uptr pc = events[thr->fast_state.GetTracePos()];
+  return pc;
+}
+
+uptr TraceSize() {
+  return (uptr)(1ull << (kTracePartSizeBits + flags()->history_size + 1));
+}
+
+uptr TraceParts() {
+  return TraceSize() / kTracePartSize;
+}
+
+#if !SANITIZER_GO
+extern "C" void __tsan_trace_switch() {
+  TraceSwitch(cur_thread());
+}
+
+extern "C" void __tsan_report_race() {
+  ReportRace(cur_thread());
+}
+#endif
+
+void ThreadIgnoreBegin(ThreadState *thr, uptr pc) {
+  DPrintf("#%d: ThreadIgnoreBegin\n", thr->tid);
+  thr->ignore_reads_and_writes++;
+  CHECK_GT(thr->ignore_reads_and_writes, 0);
+  thr->fast_state.SetIgnoreBit();
+#if !SANITIZER_GO
+  if (pc && !ctx->after_multithreaded_fork)
+    thr->mop_ignore_set.Add(CurrentStackId(thr, pc));
+#endif
+}
+
+void ThreadIgnoreEnd(ThreadState *thr) {
+  DPrintf("#%d: ThreadIgnoreEnd\n", thr->tid);
+  CHECK_GT(thr->ignore_reads_and_writes, 0);
+  thr->ignore_reads_and_writes--;
+  if (thr->ignore_reads_and_writes == 0) {
+    thr->fast_state.ClearIgnoreBit();
+#if !SANITIZER_GO
+    thr->mop_ignore_set.Reset();
+#endif
+  }
+}
+
+#if !SANITIZER_GO
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE
+uptr __tsan_testonly_shadow_stack_current_size() {
+  ThreadState *thr = cur_thread();
+  return thr->shadow_stack_pos - thr->shadow_stack;
+}
+#endif
+
+void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc) {
+  DPrintf("#%d: ThreadIgnoreSyncBegin\n", thr->tid);
+  thr->ignore_sync++;
+  CHECK_GT(thr->ignore_sync, 0);
+#if !SANITIZER_GO
+  if (pc && !ctx->after_multithreaded_fork)
+    thr->sync_ignore_set.Add(CurrentStackId(thr, pc));
+#endif
+}
+
+void ThreadIgnoreSyncEnd(ThreadState *thr) {
+  DPrintf("#%d: ThreadIgnoreSyncEnd\n", thr->tid);
+  CHECK_GT(thr->ignore_sync, 0);
+  thr->ignore_sync--;
+#if !SANITIZER_GO
+  if (thr->ignore_sync == 0)
+    thr->sync_ignore_set.Reset();
+#endif
+}
+
+bool MD5Hash::operator==(const MD5Hash &other) const {
+  return hash[0] == other.hash[0] && hash[1] == other.hash[1];
+}
+
+#if SANITIZER_DEBUG
+void build_consistency_debug() {}
+#else
+void build_consistency_release() {}
+#endif
+
+}  // namespace __tsan
+
+#if SANITIZER_CHECK_DEADLOCKS
+namespace __sanitizer {
+using namespace __tsan;
+MutexMeta mutex_meta[] = {
+    {MutexInvalid, "Invalid", {}},
+    {MutexThreadRegistry, "ThreadRegistry", {}},
+    {MutexTypeTrace, "Trace", {}},
+    {MutexTypeReport,
+     "Report",
+     {MutexTypeSyncVar, MutexTypeGlobalProc, MutexTypeTrace}},
+    {MutexTypeSyncVar, "SyncVar", {MutexTypeTrace}},
+    {MutexTypeAnnotations, "Annotations", {}},
+    {MutexTypeAtExit, "AtExit", {MutexTypeSyncVar}},
+    {MutexTypeFired, "Fired", {MutexLeaf}},
+    {MutexTypeRacy, "Racy", {MutexLeaf}},
+    {MutexTypeGlobalProc, "GlobalProc", {}},
+    {MutexTypeInternalAlloc, "InternalAlloc", {MutexLeaf}},
+    {},
+};
+
+void PrintMutexPC(uptr pc) { StackTrace(&pc, 1).Print(); }
+}  // namespace __sanitizer
+#endif
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_rtl.h b/compiler-rt/lib/tsan/rtl-old/tsan_rtl.h
new file mode 100644
index 000000000000..c71b27e1cbf5
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_rtl.h
@@ -0,0 +1,796 @@
+//===-- tsan_rtl.h ----------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// Main internal TSan header file.
+//
+// Ground rules:
+//   - C++ run-time should not be used (static CTORs, RTTI, exceptions, static
+//     function-scope locals)
+//   - All functions/classes/etc reside in namespace __tsan, except for those
+//     declared in tsan_interface.h.
+//   - Platform-specific files should be used instead of ifdefs (*).
+//   - No system headers included in header files (*).
+//   - Platform specific headres included only into platform-specific files (*).
+//
+//  (*) Except when inlining is critical for performance.
+//===----------------------------------------------------------------------===//
+
+#ifndef TSAN_RTL_H
+#define TSAN_RTL_H
+
+#include "sanitizer_common/sanitizer_allocator.h"
+#include "sanitizer_common/sanitizer_allocator_internal.h"
+#include "sanitizer_common/sanitizer_asm.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_deadlock_detector_interface.h"
+#include "sanitizer_common/sanitizer_libignore.h"
+#include "sanitizer_common/sanitizer_suppressions.h"
+#include "sanitizer_common/sanitizer_thread_registry.h"
+#include "sanitizer_common/sanitizer_vector.h"
+#include "tsan_clock.h"
+#include "tsan_defs.h"
+#include "tsan_flags.h"
+#include "tsan_ignoreset.h"
+#include "tsan_mman.h"
+#include "tsan_mutexset.h"
+#include "tsan_platform.h"
+#include "tsan_report.h"
+#include "tsan_shadow.h"
+#include "tsan_stack_trace.h"
+#include "tsan_sync.h"
+#include "tsan_trace.h"
+
+#if SANITIZER_WORDSIZE != 64
+# error "ThreadSanitizer is supported only on 64-bit platforms"
+#endif
+
+namespace __tsan {
+
+#if !SANITIZER_GO
+struct MapUnmapCallback;
+#if defined(__mips64) || defined(__aarch64__) || defined(__powerpc__)
+
+struct AP32 {
+  static const uptr kSpaceBeg = 0;
+  static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE;
+  static const uptr kMetadataSize = 0;
+  typedef __sanitizer::CompactSizeClassMap SizeClassMap;
+  static const uptr kRegionSizeLog = 20;
+  using AddressSpaceView = LocalAddressSpaceView;
+  typedef __tsan::MapUnmapCallback MapUnmapCallback;
+  static const uptr kFlags = 0;
+};
+typedef SizeClassAllocator32<AP32> PrimaryAllocator;
+#else
+struct AP64 {  // Allocator64 parameters. Deliberately using a short name.
+#    if defined(__s390x__)
+  typedef MappingS390x Mapping;
+#    else
+  typedef Mapping48AddressSpace Mapping;
+#    endif
+  static const uptr kSpaceBeg = Mapping::kHeapMemBeg;
+  static const uptr kSpaceSize = Mapping::kHeapMemEnd - Mapping::kHeapMemBeg;
+  static const uptr kMetadataSize = 0;
+  typedef DefaultSizeClassMap SizeClassMap;
+  typedef __tsan::MapUnmapCallback MapUnmapCallback;
+  static const uptr kFlags = 0;
+  using AddressSpaceView = LocalAddressSpaceView;
+};
+typedef SizeClassAllocator64<AP64> PrimaryAllocator;
+#endif
+typedef CombinedAllocator<PrimaryAllocator> Allocator;
+typedef Allocator::AllocatorCache AllocatorCache;
+Allocator *allocator();
+#endif
+
+struct ThreadSignalContext;
+
+struct JmpBuf {
+  uptr sp;
+  int int_signal_send;
+  bool in_blocking_func;
+  uptr in_signal_handler;
+  uptr *shadow_stack_pos;
+};
+
+// A Processor represents a physical thread, or a P for Go.
+// It is used to store internal resources like allocate cache, and does not
+// participate in race-detection logic (invisible to end user).
+// In C++ it is tied to an OS thread just like ThreadState, however ideally
+// it should be tied to a CPU (this way we will have fewer allocator caches).
+// In Go it is tied to a P, so there are significantly fewer Processor's than
+// ThreadState's (which are tied to Gs).
+// A ThreadState must be wired with a Processor to handle events.
+struct Processor {
+  ThreadState *thr; // currently wired thread, or nullptr
+#if !SANITIZER_GO
+  AllocatorCache alloc_cache;
+  InternalAllocatorCache internal_alloc_cache;
+#endif
+  DenseSlabAllocCache block_cache;
+  DenseSlabAllocCache sync_cache;
+  DenseSlabAllocCache clock_cache;
+  DDPhysicalThread *dd_pt;
+};
+
+#if !SANITIZER_GO
+// ScopedGlobalProcessor temporary setups a global processor for the current
+// thread, if it does not have one. Intended for interceptors that can run
+// at the very thread end, when we already destroyed the thread processor.
+struct ScopedGlobalProcessor {
+  ScopedGlobalProcessor();
+  ~ScopedGlobalProcessor();
+};
+#endif
+
+// This struct is stored in TLS.
+struct ThreadState {
+  FastState fast_state;
+  // Synch epoch represents the threads's epoch before the last synchronization
+  // action. It allows to reduce number of shadow state updates.
+  // For example, fast_synch_epoch=100, last write to addr X was at epoch=150,
+  // if we are processing write to X from the same thread at epoch=200,
+  // we do nothing, because both writes happen in the same 'synch epoch'.
+  // That is, if another memory access does not race with the former write,
+  // it does not race with the latter as well.
+  // QUESTION: can we can squeeze this into ThreadState::Fast?
+  // E.g. ThreadState::Fast is a 44-bit, 32 are taken by synch_epoch and 12 are
+  // taken by epoch between synchs.
+  // This way we can save one load from tls.
+  u64 fast_synch_epoch;
+  // Technically `current` should be a separate THREADLOCAL variable;
+  // but it is placed here in order to share cache line with previous fields.
+  ThreadState* current;
+  // This is a slow path flag. On fast path, fast_state.GetIgnoreBit() is read.
+  // We do not distinguish beteween ignoring reads and writes
+  // for better performance.
+  int ignore_reads_and_writes;
+  atomic_sint32_t pending_signals;
+  int ignore_sync;
+  int suppress_reports;
+  // Go does not support ignores.
+#if !SANITIZER_GO
+  IgnoreSet mop_ignore_set;
+  IgnoreSet sync_ignore_set;
+#endif
+  uptr *shadow_stack;
+  uptr *shadow_stack_end;
+  uptr *shadow_stack_pos;
+  RawShadow *racy_shadow_addr;
+  RawShadow racy_state[2];
+  MutexSet mset;
+  ThreadClock clock;
+#if !SANITIZER_GO
+  Vector<JmpBuf> jmp_bufs;
+  int ignore_interceptors;
+#endif
+  const Tid tid;
+  const int unique_id;
+  bool in_symbolizer;
+  bool in_ignored_lib;
+  bool is_inited;
+  bool is_dead;
+  bool is_freeing;
+  bool is_vptr_access;
+  const uptr stk_addr;
+  const uptr stk_size;
+  const uptr tls_addr;
+  const uptr tls_size;
+  ThreadContext *tctx;
+
+  DDLogicalThread *dd_lt;
+
+  // Current wired Processor, or nullptr. Required to handle any events.
+  Processor *proc1;
+#if !SANITIZER_GO
+  Processor *proc() { return proc1; }
+#else
+  Processor *proc();
+#endif
+
+  atomic_uintptr_t in_signal_handler;
+  ThreadSignalContext *signal_ctx;
+
+#if !SANITIZER_GO
+  StackID last_sleep_stack_id;
+  ThreadClock last_sleep_clock;
+#endif
+
+  // Set in regions of runtime that must be signal-safe and fork-safe.
+  // If set, malloc must not be called.
+  int nomalloc;
+
+  const ReportDesc *current_report;
+
+  // Current position in tctx->trace.Back()->events (Event*).
+  atomic_uintptr_t trace_pos;
+  // PC of the last memory access, used to compute PC deltas in the trace.
+  uptr trace_prev_pc;
+  Sid sid;
+  Epoch epoch;
+
+  explicit ThreadState(Context *ctx, Tid tid, int unique_id, u64 epoch,
+                       unsigned reuse_count, uptr stk_addr, uptr stk_size,
+                       uptr tls_addr, uptr tls_size);
+} ALIGNED(SANITIZER_CACHE_LINE_SIZE);
+
+#if !SANITIZER_GO
+#if SANITIZER_MAC || SANITIZER_ANDROID
+ThreadState *cur_thread();
+void set_cur_thread(ThreadState *thr);
+void cur_thread_finalize();
+inline ThreadState *cur_thread_init() { return cur_thread(); }
+#  else
+__attribute__((tls_model("initial-exec")))
+extern THREADLOCAL char cur_thread_placeholder[];
+inline ThreadState *cur_thread() {
+  return reinterpret_cast<ThreadState *>(cur_thread_placeholder)->current;
+}
+inline ThreadState *cur_thread_init() {
+  ThreadState *thr = reinterpret_cast<ThreadState *>(cur_thread_placeholder);
+  if (UNLIKELY(!thr->current))
+    thr->current = thr;
+  return thr->current;
+}
+inline void set_cur_thread(ThreadState *thr) {
+  reinterpret_cast<ThreadState *>(cur_thread_placeholder)->current = thr;
+}
+inline void cur_thread_finalize() { }
+#  endif  // SANITIZER_MAC || SANITIZER_ANDROID
+#endif  // SANITIZER_GO
+
+class ThreadContext final : public ThreadContextBase {
+ public:
+  explicit ThreadContext(Tid tid);
+  ~ThreadContext();
+  ThreadState *thr;
+  StackID creation_stack_id;
+  SyncClock sync;
+  // Epoch at which the thread had started.
+  // If we see an event from the thread stamped by an older epoch,
+  // the event is from a dead thread that shared tid with this thread.
+  u64 epoch0;
+  u64 epoch1;
+
+  v3::Trace trace;
+
+  // Override superclass callbacks.
+  void OnDead() override;
+  void OnJoined(void *arg) override;
+  void OnFinished() override;
+  void OnStarted(void *arg) override;
+  void OnCreated(void *arg) override;
+  void OnReset() override;
+  void OnDetached(void *arg) override;
+};
+
+struct RacyStacks {
+  MD5Hash hash[2];
+  bool operator==(const RacyStacks &other) const;
+};
+
+struct RacyAddress {
+  uptr addr_min;
+  uptr addr_max;
+};
+
+struct FiredSuppression {
+  ReportType type;
+  uptr pc_or_addr;
+  Suppression *supp;
+};
+
+struct Context {
+  Context();
+
+  bool initialized;
+#if !SANITIZER_GO
+  bool after_multithreaded_fork;
+#endif
+
+  MetaMap metamap;
+
+  Mutex report_mtx;
+  int nreported;
+  atomic_uint64_t last_symbolize_time_ns;
+
+  void *background_thread;
+  atomic_uint32_t stop_background_thread;
+
+  ThreadRegistry thread_registry;
+
+  Mutex racy_mtx;
+  Vector<RacyStacks> racy_stacks;
+  Vector<RacyAddress> racy_addresses;
+  // Number of fired suppressions may be large enough.
+  Mutex fired_suppressions_mtx;
+  InternalMmapVector<FiredSuppression> fired_suppressions;
+  DDetector *dd;
+
+  ClockAlloc clock_alloc;
+
+  Flags flags;
+  fd_t memprof_fd;
+
+  Mutex slot_mtx;
+};
+
+extern Context *ctx;  // The one and the only global runtime context.
+
+ALWAYS_INLINE Flags *flags() {
+  return &ctx->flags;
+}
+
+struct ScopedIgnoreInterceptors {
+  ScopedIgnoreInterceptors() {
+#if !SANITIZER_GO
+    cur_thread()->ignore_interceptors++;
+#endif
+  }
+
+  ~ScopedIgnoreInterceptors() {
+#if !SANITIZER_GO
+    cur_thread()->ignore_interceptors--;
+#endif
+  }
+};
+
+const char *GetObjectTypeFromTag(uptr tag);
+const char *GetReportHeaderFromTag(uptr tag);
+uptr TagFromShadowStackFrame(uptr pc);
+
+class ScopedReportBase {
+ public:
+  void AddMemoryAccess(uptr addr, uptr external_tag, Shadow s, StackTrace stack,
+                       const MutexSet *mset);
+  void AddStack(StackTrace stack, bool suppressable = false);
+  void AddThread(const ThreadContext *tctx, bool suppressable = false);
+  void AddThread(Tid unique_tid, bool suppressable = false);
+  void AddUniqueTid(Tid unique_tid);
+  void AddMutex(const SyncVar *s);
+  u64 AddMutex(u64 id);
+  void AddLocation(uptr addr, uptr size);
+  void AddSleep(StackID stack_id);
+  void SetCount(int count);
+
+  const ReportDesc *GetReport() const;
+
+ protected:
+  ScopedReportBase(ReportType typ, uptr tag);
+  ~ScopedReportBase();
+
+ private:
+  ReportDesc *rep_;
+  // Symbolizer makes lots of intercepted calls. If we try to process them,
+  // at best it will cause deadlocks on internal mutexes.
+  ScopedIgnoreInterceptors ignore_interceptors_;
+
+  void AddDeadMutex(u64 id);
+
+  ScopedReportBase(const ScopedReportBase &) = delete;
+  void operator=(const ScopedReportBase &) = delete;
+};
+
+class ScopedReport : public ScopedReportBase {
+ public:
+  explicit ScopedReport(ReportType typ, uptr tag = kExternalTagNone);
+  ~ScopedReport();
+
+ private:
+  ScopedErrorReportLock lock_;
+};
+
+bool ShouldReport(ThreadState *thr, ReportType typ);
+ThreadContext *IsThreadStackOrTls(uptr addr, bool *is_stack);
+void RestoreStack(Tid tid, const u64 epoch, VarSizeStackTrace *stk,
+                  MutexSet *mset, uptr *tag = nullptr);
+
+// The stack could look like:
+//   <start> | <main> | <foo> | tag | <bar>
+// This will extract the tag and keep:
+//   <start> | <main> | <foo> | <bar>
+template<typename StackTraceTy>
+void ExtractTagFromStack(StackTraceTy *stack, uptr *tag = nullptr) {
+  if (stack->size < 2) return;
+  uptr possible_tag_pc = stack->trace[stack->size - 2];
+  uptr possible_tag = TagFromShadowStackFrame(possible_tag_pc);
+  if (possible_tag == kExternalTagNone) return;
+  stack->trace_buffer[stack->size - 2] = stack->trace_buffer[stack->size - 1];
+  stack->size -= 1;
+  if (tag) *tag = possible_tag;
+}
+
+template<typename StackTraceTy>
+void ObtainCurrentStack(ThreadState *thr, uptr toppc, StackTraceTy *stack,
+                        uptr *tag = nullptr) {
+  uptr size = thr->shadow_stack_pos - thr->shadow_stack;
+  uptr start = 0;
+  if (size + !!toppc > kStackTraceMax) {
+    start = size + !!toppc - kStackTraceMax;
+    size = kStackTraceMax - !!toppc;
+  }
+  stack->Init(&thr->shadow_stack[start], size, toppc);
+  ExtractTagFromStack(stack, tag);
+}
+
+#define GET_STACK_TRACE_FATAL(thr, pc) \
+  VarSizeStackTrace stack; \
+  ObtainCurrentStack(thr, pc, &stack); \
+  stack.ReverseOrder();
+
+void MapShadow(uptr addr, uptr size);
+void MapThreadTrace(uptr addr, uptr size, const char *name);
+void DontNeedShadowFor(uptr addr, uptr size);
+void UnmapShadow(ThreadState *thr, uptr addr, uptr size);
+void InitializeShadowMemory();
+void InitializeInterceptors();
+void InitializeLibIgnore();
+void InitializeDynamicAnnotations();
+
+void ForkBefore(ThreadState *thr, uptr pc);
+void ForkParentAfter(ThreadState *thr, uptr pc);
+void ForkChildAfter(ThreadState *thr, uptr pc, bool start_thread);
+
+void ReportRace(ThreadState *thr);
+bool OutputReport(ThreadState *thr, const ScopedReport &srep);
+bool IsFiredSuppression(Context *ctx, ReportType type, StackTrace trace);
+bool IsExpectedReport(uptr addr, uptr size);
+
+#if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 1
+# define DPrintf Printf
+#else
+# define DPrintf(...)
+#endif
+
+#if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 2
+# define DPrintf2 Printf
+#else
+# define DPrintf2(...)
+#endif
+
+StackID CurrentStackId(ThreadState *thr, uptr pc);
+ReportStack *SymbolizeStackId(StackID stack_id);
+void PrintCurrentStack(ThreadState *thr, uptr pc);
+void PrintCurrentStackSlow(uptr pc);  // uses libunwind
+MBlock *JavaHeapBlock(uptr addr, uptr *start);
+
+void Initialize(ThreadState *thr);
+void MaybeSpawnBackgroundThread();
+int Finalize(ThreadState *thr);
+
+void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write);
+void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write);
+
+void MemoryAccess(ThreadState *thr, uptr pc, uptr addr,
+    int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic);
+void MemoryAccessImpl(ThreadState *thr, uptr addr,
+    int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic,
+    u64 *shadow_mem, Shadow cur);
+void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr,
+    uptr size, bool is_write);
+void UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size,
+                           AccessType typ);
+
+const int kSizeLog1 = 0;
+const int kSizeLog2 = 1;
+const int kSizeLog4 = 2;
+const int kSizeLog8 = 3;
+
+ALWAYS_INLINE
+void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size,
+                  AccessType typ) {
+  int size_log;
+  switch (size) {
+    case 1:
+      size_log = kSizeLog1;
+      break;
+    case 2:
+      size_log = kSizeLog2;
+      break;
+    case 4:
+      size_log = kSizeLog4;
+      break;
+    default:
+      DCHECK_EQ(size, 8);
+      size_log = kSizeLog8;
+      break;
+  }
+  bool is_write = !(typ & kAccessRead);
+  bool is_atomic = typ & kAccessAtomic;
+  if (typ & kAccessVptr)
+    thr->is_vptr_access = true;
+  if (typ & kAccessFree)
+    thr->is_freeing = true;
+  MemoryAccess(thr, pc, addr, size_log, is_write, is_atomic);
+  if (typ & kAccessVptr)
+    thr->is_vptr_access = false;
+  if (typ & kAccessFree)
+    thr->is_freeing = false;
+}
+
+void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size);
+void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size);
+void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size);
+void MemoryRangeImitateWriteOrResetRange(ThreadState *thr, uptr pc, uptr addr,
+                                         uptr size);
+
+void ThreadIgnoreBegin(ThreadState *thr, uptr pc);
+void ThreadIgnoreEnd(ThreadState *thr);
+void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc);
+void ThreadIgnoreSyncEnd(ThreadState *thr);
+
+void FuncEntry(ThreadState *thr, uptr pc);
+void FuncExit(ThreadState *thr);
+
+Tid ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached);
+void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id,
+                 ThreadType thread_type);
+void ThreadFinish(ThreadState *thr);
+Tid ThreadConsumeTid(ThreadState *thr, uptr pc, uptr uid);
+void ThreadJoin(ThreadState *thr, uptr pc, Tid tid);
+void ThreadDetach(ThreadState *thr, uptr pc, Tid tid);
+void ThreadFinalize(ThreadState *thr);
+void ThreadSetName(ThreadState *thr, const char *name);
+int ThreadCount(ThreadState *thr);
+void ProcessPendingSignalsImpl(ThreadState *thr);
+void ThreadNotJoined(ThreadState *thr, uptr pc, Tid tid, uptr uid);
+
+Processor *ProcCreate();
+void ProcDestroy(Processor *proc);
+void ProcWire(Processor *proc, ThreadState *thr);
+void ProcUnwire(Processor *proc, ThreadState *thr);
+
+// Note: the parameter is called flagz, because flags is already taken
+// by the global function that returns flags.
+void MutexCreate(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
+void MutexDestroy(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
+void MutexPreLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
+void MutexPostLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0,
+    int rec = 1);
+int  MutexUnlock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
+void MutexPreReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
+void MutexPostReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
+void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr);
+void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr);
+void MutexRepair(ThreadState *thr, uptr pc, uptr addr);  // call on EOWNERDEAD
+void MutexInvalidAccess(ThreadState *thr, uptr pc, uptr addr);
+
+void Acquire(ThreadState *thr, uptr pc, uptr addr);
+// AcquireGlobal synchronizes the current thread with all other threads.
+// In terms of happens-before relation, it draws a HB edge from all threads
+// (where they happen to execute right now) to the current thread. We use it to
+// handle Go finalizers. Namely, finalizer goroutine executes AcquireGlobal
+// right before executing finalizers. This provides a coarse, but simple
+// approximation of the actual required synchronization.
+void AcquireGlobal(ThreadState *thr);
+void Release(ThreadState *thr, uptr pc, uptr addr);
+void ReleaseStoreAcquire(ThreadState *thr, uptr pc, uptr addr);
+void ReleaseStore(ThreadState *thr, uptr pc, uptr addr);
+void AfterSleep(ThreadState *thr, uptr pc);
+void AcquireImpl(ThreadState *thr, uptr pc, SyncClock *c);
+void ReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c);
+void ReleaseStoreAcquireImpl(ThreadState *thr, uptr pc, SyncClock *c);
+void ReleaseStoreImpl(ThreadState *thr, uptr pc, SyncClock *c);
+void AcquireReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c);
+
+// The hacky call uses custom calling convention and an assembly thunk.
+// It is considerably faster that a normal call for the caller
+// if it is not executed (it is intended for slow paths from hot functions).
+// The trick is that the call preserves all registers and the compiler
+// does not treat it as a call.
+// If it does not work for you, use normal call.
+#if !SANITIZER_DEBUG && defined(__x86_64__) && !SANITIZER_MAC
+// The caller may not create the stack frame for itself at all,
+// so we create a reserve stack frame for it (1024b must be enough).
+#define HACKY_CALL(f) \
+  __asm__ __volatile__("sub $1024, %%rsp;" \
+                       CFI_INL_ADJUST_CFA_OFFSET(1024) \
+                       ".hidden " #f "_thunk;" \
+                       "call " #f "_thunk;" \
+                       "add $1024, %%rsp;" \
+                       CFI_INL_ADJUST_CFA_OFFSET(-1024) \
+                       ::: "memory", "cc");
+#else
+#define HACKY_CALL(f) f()
+#endif
+
+void TraceSwitch(ThreadState *thr);
+uptr TraceTopPC(ThreadState *thr);
+uptr TraceSize();
+uptr TraceParts();
+Trace *ThreadTrace(Tid tid);
+
+extern "C" void __tsan_trace_switch();
+void ALWAYS_INLINE TraceAddEvent(ThreadState *thr, FastState fs,
+                                        EventType typ, u64 addr) {
+  if (!kCollectHistory)
+    return;
+  // TraceSwitch accesses shadow_stack, but it's called infrequently,
+  // so we check it here proactively.
+  DCHECK(thr->shadow_stack);
+  DCHECK_GE((int)typ, 0);
+  DCHECK_LE((int)typ, 7);
+  DCHECK_EQ(GetLsb(addr, kEventPCBits), addr);
+  u64 pos = fs.GetTracePos();
+  if (UNLIKELY((pos % kTracePartSize) == 0)) {
+#if !SANITIZER_GO
+    HACKY_CALL(__tsan_trace_switch);
+#else
+    TraceSwitch(thr);
+#endif
+  }
+  Event *trace = (Event*)GetThreadTrace(fs.tid());
+  Event *evp = &trace[pos];
+  Event ev = (u64)addr | ((u64)typ << kEventPCBits);
+  *evp = ev;
+}
+
+#if !SANITIZER_GO
+uptr ALWAYS_INLINE HeapEnd() {
+  return HeapMemEnd() + PrimaryAllocator::AdditionalSize();
+}
+#endif
+
+ThreadState *FiberCreate(ThreadState *thr, uptr pc, unsigned flags);
+void FiberDestroy(ThreadState *thr, uptr pc, ThreadState *fiber);
+void FiberSwitch(ThreadState *thr, uptr pc, ThreadState *fiber, unsigned flags);
+
+// These need to match __tsan_switch_to_fiber_* flags defined in
+// tsan_interface.h. See documentation there as well.
+enum FiberSwitchFlags {
+  FiberSwitchFlagNoSync = 1 << 0, // __tsan_switch_to_fiber_no_sync
+};
+
+ALWAYS_INLINE void ProcessPendingSignals(ThreadState *thr) {
+  if (UNLIKELY(atomic_load_relaxed(&thr->pending_signals)))
+    ProcessPendingSignalsImpl(thr);
+}
+
+extern bool is_initialized;
+
+ALWAYS_INLINE
+void LazyInitialize(ThreadState *thr) {
+  // If we can use .preinit_array, assume that __tsan_init
+  // called from .preinit_array initializes runtime before
+  // any instrumented code.
+#if !SANITIZER_CAN_USE_PREINIT_ARRAY
+  if (UNLIKELY(!is_initialized))
+    Initialize(thr);
+#endif
+}
+
+namespace v3 {
+
+void TraceSwitchPart(ThreadState *thr);
+bool RestoreStack(Tid tid, EventType type, Sid sid, Epoch epoch, uptr addr,
+                  uptr size, AccessType typ, VarSizeStackTrace *pstk,
+                  MutexSet *pmset, uptr *ptag);
+
+template <typename EventT>
+ALWAYS_INLINE WARN_UNUSED_RESULT bool TraceAcquire(ThreadState *thr,
+                                                   EventT **ev) {
+  Event *pos = reinterpret_cast<Event *>(atomic_load_relaxed(&thr->trace_pos));
+#if SANITIZER_DEBUG
+  // TraceSwitch acquires these mutexes,
+  // so we lock them here to detect deadlocks more reliably.
+  { Lock lock(&ctx->slot_mtx); }
+  { Lock lock(&thr->tctx->trace.mtx); }
+  TracePart *current = thr->tctx->trace.parts.Back();
+  if (current) {
+    DCHECK_GE(pos, &current->events[0]);
+    DCHECK_LE(pos, &current->events[TracePart::kSize]);
+  } else {
+    DCHECK_EQ(pos, nullptr);
+  }
+#endif
+  // TracePart is allocated with mmap and is at least 4K aligned.
+  // So the following check is a faster way to check for part end.
+  // It may have false positives in the middle of the trace,
+  // they are filtered out in TraceSwitch.
+  if (UNLIKELY(((uptr)(pos + 1) & TracePart::kAlignment) == 0))
+    return false;
+  *ev = reinterpret_cast<EventT *>(pos);
+  return true;
+}
+
+template <typename EventT>
+ALWAYS_INLINE void TraceRelease(ThreadState *thr, EventT *evp) {
+  DCHECK_LE(evp + 1, &thr->tctx->trace.parts.Back()->events[TracePart::kSize]);
+  atomic_store_relaxed(&thr->trace_pos, (uptr)(evp + 1));
+}
+
+template <typename EventT>
+void TraceEvent(ThreadState *thr, EventT ev) {
+  EventT *evp;
+  if (!TraceAcquire(thr, &evp)) {
+    TraceSwitchPart(thr);
+    UNUSED bool res = TraceAcquire(thr, &evp);
+    DCHECK(res);
+  }
+  *evp = ev;
+  TraceRelease(thr, evp);
+}
+
+ALWAYS_INLINE WARN_UNUSED_RESULT bool TryTraceFunc(ThreadState *thr,
+                                                   uptr pc = 0) {
+  if (!kCollectHistory)
+    return true;
+  EventFunc *ev;
+  if (UNLIKELY(!TraceAcquire(thr, &ev)))
+    return false;
+  ev->is_access = 0;
+  ev->is_func = 1;
+  ev->pc = pc;
+  TraceRelease(thr, ev);
+  return true;
+}
+
+WARN_UNUSED_RESULT
+bool TryTraceMemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size,
+                          AccessType typ);
+WARN_UNUSED_RESULT
+bool TryTraceMemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size,
+                               AccessType typ);
+void TraceMemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size,
+                            AccessType typ);
+void TraceFunc(ThreadState *thr, uptr pc = 0);
+void TraceMutexLock(ThreadState *thr, EventType type, uptr pc, uptr addr,
+                    StackID stk);
+void TraceMutexUnlock(ThreadState *thr, uptr addr);
+void TraceTime(ThreadState *thr);
+
+}  // namespace v3
+
+void GrowShadowStack(ThreadState *thr);
+
+ALWAYS_INLINE
+void FuncEntry(ThreadState *thr, uptr pc) {
+  DPrintf2("#%d: FuncEntry %p\n", (int)thr->fast_state.tid(), (void *)pc);
+  if (kCollectHistory) {
+    thr->fast_state.IncrementEpoch();
+    TraceAddEvent(thr, thr->fast_state, EventTypeFuncEnter, pc);
+  }
+
+  // Shadow stack maintenance can be replaced with
+  // stack unwinding during trace switch (which presumably must be faster).
+  DCHECK_GE(thr->shadow_stack_pos, thr->shadow_stack);
+#if !SANITIZER_GO
+  DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
+#else
+  if (thr->shadow_stack_pos == thr->shadow_stack_end)
+    GrowShadowStack(thr);
+#endif
+  thr->shadow_stack_pos[0] = pc;
+  thr->shadow_stack_pos++;
+}
+
+ALWAYS_INLINE
+void FuncExit(ThreadState *thr) {
+  DPrintf2("#%d: FuncExit\n", (int)thr->fast_state.tid());
+  if (kCollectHistory) {
+    thr->fast_state.IncrementEpoch();
+    TraceAddEvent(thr, thr->fast_state, EventTypeFuncExit, 0);
+  }
+
+  DCHECK_GT(thr->shadow_stack_pos, thr->shadow_stack);
+#if !SANITIZER_GO
+  DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
+#endif
+  thr->shadow_stack_pos--;
+}
+
+#if !SANITIZER_GO
+extern void (*on_initialize)(void);
+extern int (*on_finalize)(int);
+#endif
+
+}  // namespace __tsan
+
+#endif  // TSAN_RTL_H
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_rtl_aarch64.S b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_aarch64.S
new file mode 100644
index 000000000000..e0b4c71dfed9
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_aarch64.S
@@ -0,0 +1,245 @@
+// The content of this file is AArch64-only:
+#if defined(__aarch64__)
+
+#include "sanitizer_common/sanitizer_asm.h"
+
+#if defined(__APPLE__)
+.align  2
+
+.section  __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers
+.long _setjmp$non_lazy_ptr
+_setjmp$non_lazy_ptr:
+.indirect_symbol _setjmp
+.long 0
+
+.section  __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers
+.long __setjmp$non_lazy_ptr
+__setjmp$non_lazy_ptr:
+.indirect_symbol __setjmp
+.long 0
+
+.section  __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers
+.long _sigsetjmp$non_lazy_ptr
+_sigsetjmp$non_lazy_ptr:
+.indirect_symbol _sigsetjmp
+.long 0
+#endif
+
+#if !defined(__APPLE__)
+.section .text
+#else
+.section __TEXT,__text
+.align 3
+#endif
+
+ASM_HIDDEN(__tsan_setjmp)
+.comm _ZN14__interception11real_setjmpE,8,8
+.globl ASM_SYMBOL_INTERCEPTOR(setjmp)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(setjmp))
+ASM_SYMBOL_INTERCEPTOR(setjmp):
+  CFI_STARTPROC
+
+  // Save frame/link register
+  stp     x29, x30, [sp, -32]!
+  CFI_DEF_CFA_OFFSET (32)
+  CFI_OFFSET (29, -32)
+  CFI_OFFSET (30, -24)
+
+  // Adjust the SP for previous frame
+  add     x29, sp, 0
+  CFI_DEF_CFA_REGISTER (29)
+
+  // Save env parameter
+  str     x0, [sp, 16]
+  CFI_OFFSET (0, -16)
+
+  // Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
+  add     x0, x29, 32
+
+  // call tsan interceptor
+  bl      ASM_SYMBOL(__tsan_setjmp)
+
+  // Restore env parameter
+  ldr     x0, [sp, 16]
+  CFI_RESTORE (0)
+
+  // Restore frame/link register
+  ldp     x29, x30, [sp], 32
+  CFI_RESTORE (29)
+  CFI_RESTORE (30)
+  CFI_DEF_CFA (31, 0)
+
+  // tail jump to libc setjmp
+#if !defined(__APPLE__)
+  adrp    x1, :got:_ZN14__interception11real_setjmpE
+  ldr     x1, [x1, #:got_lo12:_ZN14__interception11real_setjmpE]
+  ldr     x1, [x1]
+#else
+  adrp    x1, _setjmp$non_lazy_ptr@page
+  add     x1, x1, _setjmp$non_lazy_ptr@pageoff
+  ldr     x1, [x1]
+#endif
+  br      x1
+
+  CFI_ENDPROC
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(setjmp))
+
+.comm _ZN14__interception12real__setjmpE,8,8
+.globl ASM_SYMBOL_INTERCEPTOR(_setjmp)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_setjmp))
+ASM_SYMBOL_INTERCEPTOR(_setjmp):
+  CFI_STARTPROC
+
+  // Save frame/link register
+  stp     x29, x30, [sp, -32]!
+  CFI_DEF_CFA_OFFSET (32)
+  CFI_OFFSET (29, -32)
+  CFI_OFFSET (30, -24)
+
+  // Adjust the SP for previous frame
+  add     x29, sp, 0
+  CFI_DEF_CFA_REGISTER (29)
+
+  // Save env parameter
+  str     x0, [sp, 16]
+  CFI_OFFSET (0, -16)
+
+  // Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
+  add     x0, x29, 32
+
+  // call tsan interceptor
+  bl      ASM_SYMBOL(__tsan_setjmp)
+
+  // Restore env parameter
+  ldr     x0, [sp, 16]
+  CFI_RESTORE (0)
+
+  // Restore frame/link register
+  ldp     x29, x30, [sp], 32
+  CFI_RESTORE (29)
+  CFI_RESTORE (30)
+  CFI_DEF_CFA (31, 0)
+
+  // tail jump to libc setjmp
+#if !defined(__APPLE__)
+  adrp    x1, :got:_ZN14__interception12real__setjmpE
+  ldr     x1, [x1, #:got_lo12:_ZN14__interception12real__setjmpE]
+  ldr     x1, [x1]
+#else
+  adrp    x1, __setjmp$non_lazy_ptr@page
+  add     x1, x1, __setjmp$non_lazy_ptr@pageoff
+  ldr     x1, [x1]
+#endif
+  br      x1
+
+  CFI_ENDPROC
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(_setjmp))
+
+.comm _ZN14__interception14real_sigsetjmpE,8,8
+.globl ASM_SYMBOL_INTERCEPTOR(sigsetjmp)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
+ASM_SYMBOL_INTERCEPTOR(sigsetjmp):
+  CFI_STARTPROC
+
+  // Save frame/link register
+  stp     x29, x30, [sp, -32]!
+  CFI_DEF_CFA_OFFSET (32)
+  CFI_OFFSET (29, -32)
+  CFI_OFFSET (30, -24)
+
+  // Adjust the SP for previous frame
+  add     x29, sp, 0
+  CFI_DEF_CFA_REGISTER (29)
+
+  // Save env and savesigs parameter
+  stp     x0, x1, [sp, 16]
+  CFI_OFFSET (0, -16)
+  CFI_OFFSET (1, -8)
+
+  // Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
+  add     x0, x29, 32
+
+  // call tsan interceptor
+  bl      ASM_SYMBOL(__tsan_setjmp)
+
+  // Restore env and savesigs parameter
+  ldp     x0, x1, [sp, 16]
+  CFI_RESTORE (0)
+  CFI_RESTORE (1)
+
+  // Restore frame/link register
+  ldp     x29, x30, [sp], 32
+  CFI_RESTORE (29)
+  CFI_RESTORE (30)
+  CFI_DEF_CFA (31, 0)
+
+  // tail jump to libc sigsetjmp
+#if !defined(__APPLE__)
+  adrp    x2, :got:_ZN14__interception14real_sigsetjmpE
+  ldr     x2, [x2, #:got_lo12:_ZN14__interception14real_sigsetjmpE]
+  ldr     x2, [x2]
+#else
+  adrp    x2, _sigsetjmp$non_lazy_ptr@page
+  add     x2, x2, _sigsetjmp$non_lazy_ptr@pageoff
+  ldr     x2, [x2]
+#endif
+  br      x2
+  CFI_ENDPROC
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
+
+#if !defined(__APPLE__)
+.comm _ZN14__interception16real___sigsetjmpE,8,8
+.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
+ASM_SYMBOL_INTERCEPTOR(__sigsetjmp):
+  CFI_STARTPROC
+
+  // Save frame/link register
+  stp     x29, x30, [sp, -32]!
+  CFI_DEF_CFA_OFFSET (32)
+  CFI_OFFSET (29, -32)
+  CFI_OFFSET (30, -24)
+
+  // Adjust the SP for previous frame
+  add     x29, sp, 0
+  CFI_DEF_CFA_REGISTER (29)
+
+  // Save env and savesigs parameter
+  stp     x0, x1, [sp, 16]
+  CFI_OFFSET (0, -16)
+  CFI_OFFSET (1, -8)
+
+  // Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
+  add     x0, x29, 32
+
+  // call tsan interceptor
+  bl      ASM_SYMBOL(__tsan_setjmp)
+
+  // Restore env and savesigs parameter
+  ldp     x0, x1, [sp, 16]
+  CFI_RESTORE (0)
+  CFI_RESTORE (1)
+
+  // Restore frame/link register
+  ldp     x29, x30, [sp], 32
+  CFI_RESTORE (29)
+  CFI_RESTORE (30)
+  CFI_DEF_CFA (31, 0)
+
+  // tail jump to libc __sigsetjmp
+#if !defined(__APPLE__)
+  adrp    x2, :got:_ZN14__interception16real___sigsetjmpE
+  ldr     x2, [x2, #:got_lo12:_ZN14__interception16real___sigsetjmpE]
+  ldr     x2, [x2]
+#else
+  adrp    x2, ASM_SYMBOL(__sigsetjmp)@page
+  add     x2, x2, ASM_SYMBOL(__sigsetjmp)@pageoff
+#endif
+  br      x2
+  CFI_ENDPROC
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
+#endif
+
+NO_EXEC_STACK_DIRECTIVE
+
+#endif
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_rtl_access.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_access.cpp
new file mode 100644
index 000000000000..7365fdaa3038
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_access.cpp
@@ -0,0 +1,604 @@
+//===-- tsan_rtl_access.cpp -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// Definitions of memory access and function entry/exit entry points.
+//===----------------------------------------------------------------------===//
+
+#include "tsan_rtl.h"
+
+namespace __tsan {
+
+namespace v3 {
+
+ALWAYS_INLINE USED bool TryTraceMemoryAccess(ThreadState *thr, uptr pc,
+                                             uptr addr, uptr size,
+                                             AccessType typ) {
+  DCHECK(size == 1 || size == 2 || size == 4 || size == 8);
+  if (!kCollectHistory)
+    return true;
+  EventAccess *ev;
+  if (UNLIKELY(!TraceAcquire(thr, &ev)))
+    return false;
+  u64 size_log = size == 1 ? 0 : size == 2 ? 1 : size == 4 ? 2 : 3;
+  uptr pc_delta = pc - thr->trace_prev_pc + (1 << (EventAccess::kPCBits - 1));
+  thr->trace_prev_pc = pc;
+  if (LIKELY(pc_delta < (1 << EventAccess::kPCBits))) {
+    ev->is_access = 1;
+    ev->is_read = !!(typ & kAccessRead);
+    ev->is_atomic = !!(typ & kAccessAtomic);
+    ev->size_log = size_log;
+    ev->pc_delta = pc_delta;
+    DCHECK_EQ(ev->pc_delta, pc_delta);
+    ev->addr = CompressAddr(addr);
+    TraceRelease(thr, ev);
+    return true;
+  }
+  auto *evex = reinterpret_cast<EventAccessExt *>(ev);
+  evex->is_access = 0;
+  evex->is_func = 0;
+  evex->type = EventType::kAccessExt;
+  evex->is_read = !!(typ & kAccessRead);
+  evex->is_atomic = !!(typ & kAccessAtomic);
+  evex->size_log = size_log;
+  evex->addr = CompressAddr(addr);
+  evex->pc = pc;
+  TraceRelease(thr, evex);
+  return true;
+}
+
+ALWAYS_INLINE USED bool TryTraceMemoryAccessRange(ThreadState *thr, uptr pc,
+                                                  uptr addr, uptr size,
+                                                  AccessType typ) {
+  if (!kCollectHistory)
+    return true;
+  EventAccessRange *ev;
+  if (UNLIKELY(!TraceAcquire(thr, &ev)))
+    return false;
+  thr->trace_prev_pc = pc;
+  ev->is_access = 0;
+  ev->is_func = 0;
+  ev->type = EventType::kAccessRange;
+  ev->is_read = !!(typ & kAccessRead);
+  ev->is_free = !!(typ & kAccessFree);
+  ev->size_lo = size;
+  ev->pc = CompressAddr(pc);
+  ev->addr = CompressAddr(addr);
+  ev->size_hi = size >> EventAccessRange::kSizeLoBits;
+  TraceRelease(thr, ev);
+  return true;
+}
+
+void TraceMemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size,
+                            AccessType typ) {
+  if (LIKELY(TryTraceMemoryAccessRange(thr, pc, addr, size, typ)))
+    return;
+  TraceSwitchPart(thr);
+  UNUSED bool res = TryTraceMemoryAccessRange(thr, pc, addr, size, typ);
+  DCHECK(res);
+}
+
+void TraceFunc(ThreadState *thr, uptr pc) {
+  if (LIKELY(TryTraceFunc(thr, pc)))
+    return;
+  TraceSwitchPart(thr);
+  UNUSED bool res = TryTraceFunc(thr, pc);
+  DCHECK(res);
+}
+
+void TraceMutexLock(ThreadState *thr, EventType type, uptr pc, uptr addr,
+                    StackID stk) {
+  DCHECK(type == EventType::kLock || type == EventType::kRLock);
+  if (!kCollectHistory)
+    return;
+  EventLock ev;
+  ev.is_access = 0;
+  ev.is_func = 0;
+  ev.type = type;
+  ev.pc = CompressAddr(pc);
+  ev.stack_lo = stk;
+  ev.stack_hi = stk >> EventLock::kStackIDLoBits;
+  ev._ = 0;
+  ev.addr = CompressAddr(addr);
+  TraceEvent(thr, ev);
+}
+
+void TraceMutexUnlock(ThreadState *thr, uptr addr) {
+  if (!kCollectHistory)
+    return;
+  EventUnlock ev;
+  ev.is_access = 0;
+  ev.is_func = 0;
+  ev.type = EventType::kUnlock;
+  ev._ = 0;
+  ev.addr = CompressAddr(addr);
+  TraceEvent(thr, ev);
+}
+
+void TraceTime(ThreadState *thr) {
+  if (!kCollectHistory)
+    return;
+  EventTime ev;
+  ev.is_access = 0;
+  ev.is_func = 0;
+  ev.type = EventType::kTime;
+  ev.sid = static_cast<u64>(thr->sid);
+  ev.epoch = static_cast<u64>(thr->epoch);
+  ev._ = 0;
+  TraceEvent(thr, ev);
+}
+
+}  // namespace v3
+
+ALWAYS_INLINE
+Shadow LoadShadow(u64 *p) {
+  u64 raw = atomic_load((atomic_uint64_t *)p, memory_order_relaxed);
+  return Shadow(raw);
+}
+
+ALWAYS_INLINE
+void StoreShadow(u64 *sp, u64 s) {
+  atomic_store((atomic_uint64_t *)sp, s, memory_order_relaxed);
+}
+
+ALWAYS_INLINE
+void StoreIfNotYetStored(u64 *sp, u64 *s) {
+  StoreShadow(sp, *s);
+  *s = 0;
+}
+
+extern "C" void __tsan_report_race();
+
+ALWAYS_INLINE
+void HandleRace(ThreadState *thr, u64 *shadow_mem, Shadow cur, Shadow old) {
+  thr->racy_state[0] = cur.raw();
+  thr->racy_state[1] = old.raw();
+  thr->racy_shadow_addr = shadow_mem;
+#if !SANITIZER_GO
+  HACKY_CALL(__tsan_report_race);
+#else
+  ReportRace(thr);
+#endif
+}
+
+static inline bool HappensBefore(Shadow old, ThreadState *thr) {
+  return thr->clock.get(old.TidWithIgnore()) >= old.epoch();
+}
+
+ALWAYS_INLINE
+void MemoryAccessImpl1(ThreadState *thr, uptr addr, int kAccessSizeLog,
+                       bool kAccessIsWrite, bool kIsAtomic, u64 *shadow_mem,
+                       Shadow cur) {
+  // This potentially can live in an MMX/SSE scratch register.
+  // The required intrinsics are:
+  // __m128i _mm_move_epi64(__m128i*);
+  // _mm_storel_epi64(u64*, __m128i);
+  u64 store_word = cur.raw();
+  bool stored = false;
+
+  // scan all the shadow values and dispatch to 4 categories:
+  // same, replace, candidate and race (see comments below).
+  // we consider only 3 cases regarding access sizes:
+  // equal, intersect and not intersect. initially I considered
+  // larger and smaller as well, it allowed to replace some
+  // 'candidates' with 'same' or 'replace', but I think
+  // it's just not worth it (performance- and complexity-wise).
+
+  Shadow old(0);
+
+  // It release mode we manually unroll the loop,
+  // because empirically gcc generates better code this way.
+  // However, we can't afford unrolling in debug mode, because the function
+  // consumes almost 4K of stack. Gtest gives only 4K of stack to death test
+  // threads, which is not enough for the unrolled loop.
+#if SANITIZER_DEBUG
+  for (int idx = 0; idx < 4; idx++) {
+#  include "tsan_update_shadow_word.inc"
+  }
+#else
+  int idx = 0;
+#  include "tsan_update_shadow_word.inc"
+  idx = 1;
+  if (stored) {
+#  include "tsan_update_shadow_word.inc"
+  } else {
+#  include "tsan_update_shadow_word.inc"
+  }
+  idx = 2;
+  if (stored) {
+#  include "tsan_update_shadow_word.inc"
+  } else {
+#  include "tsan_update_shadow_word.inc"
+  }
+  idx = 3;
+  if (stored) {
+#  include "tsan_update_shadow_word.inc"
+  } else {
+#  include "tsan_update_shadow_word.inc"
+  }
+#endif
+
+  // we did not find any races and had already stored
+  // the current access info, so we are done
+  if (LIKELY(stored))
+    return;
+  // choose a random candidate slot and replace it
+  StoreShadow(shadow_mem + (cur.epoch() % kShadowCnt), store_word);
+  return;
+RACE:
+  HandleRace(thr, shadow_mem, cur, old);
+  return;
+}
+
+void UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size,
+                           AccessType typ) {
+  DCHECK(!(typ & kAccessAtomic));
+  const bool kAccessIsWrite = !(typ & kAccessRead);
+  const bool kIsAtomic = false;
+  while (size) {
+    int size1 = 1;
+    int kAccessSizeLog = kSizeLog1;
+    if (size >= 8 && (addr & ~7) == ((addr + 7) & ~7)) {
+      size1 = 8;
+      kAccessSizeLog = kSizeLog8;
+    } else if (size >= 4 && (addr & ~7) == ((addr + 3) & ~7)) {
+      size1 = 4;
+      kAccessSizeLog = kSizeLog4;
+    } else if (size >= 2 && (addr & ~7) == ((addr + 1) & ~7)) {
+      size1 = 2;
+      kAccessSizeLog = kSizeLog2;
+    }
+    MemoryAccess(thr, pc, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic);
+    addr += size1;
+    size -= size1;
+  }
+}
+
+ALWAYS_INLINE
+bool ContainsSameAccessSlow(u64 *s, u64 a, u64 sync_epoch, bool is_write) {
+  Shadow cur(a);
+  for (uptr i = 0; i < kShadowCnt; i++) {
+    Shadow old(LoadShadow(&s[i]));
+    if (Shadow::Addr0AndSizeAreEqual(cur, old) &&
+        old.TidWithIgnore() == cur.TidWithIgnore() &&
+        old.epoch() > sync_epoch && old.IsAtomic() == cur.IsAtomic() &&
+        old.IsRead() <= cur.IsRead())
+      return true;
+  }
+  return false;
+}
+
+#if TSAN_VECTORIZE
+#  define SHUF(v0, v1, i0, i1, i2, i3)                    \
+    _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(v0), \
+                                    _mm_castsi128_ps(v1), \
+                                    (i0)*1 + (i1)*4 + (i2)*16 + (i3)*64))
+ALWAYS_INLINE
+bool ContainsSameAccessFast(u64 *s, u64 a, u64 sync_epoch, bool is_write) {
+  // This is an optimized version of ContainsSameAccessSlow.
+  // load current access into access[0:63]
+  const m128 access = _mm_cvtsi64_si128(a);
+  // duplicate high part of access in addr0:
+  // addr0[0:31]        = access[32:63]
+  // addr0[32:63]       = access[32:63]
+  // addr0[64:95]       = access[32:63]
+  // addr0[96:127]      = access[32:63]
+  const m128 addr0 = SHUF(access, access, 1, 1, 1, 1);
+  // load 4 shadow slots
+  const m128 shadow0 = _mm_load_si128((__m128i *)s);
+  const m128 shadow1 = _mm_load_si128((__m128i *)s + 1);
+  // load high parts of 4 shadow slots into addr_vect:
+  // addr_vect[0:31]    = shadow0[32:63]
+  // addr_vect[32:63]   = shadow0[96:127]
+  // addr_vect[64:95]   = shadow1[32:63]
+  // addr_vect[96:127]  = shadow1[96:127]
+  m128 addr_vect = SHUF(shadow0, shadow1, 1, 3, 1, 3);
+  if (!is_write) {
+    // set IsRead bit in addr_vect
+    const m128 rw_mask1 = _mm_cvtsi64_si128(1 << 15);
+    const m128 rw_mask = SHUF(rw_mask1, rw_mask1, 0, 0, 0, 0);
+    addr_vect = _mm_or_si128(addr_vect, rw_mask);
+  }
+  // addr0 == addr_vect?
+  const m128 addr_res = _mm_cmpeq_epi32(addr0, addr_vect);
+  // epoch1[0:63]       = sync_epoch
+  const m128 epoch1 = _mm_cvtsi64_si128(sync_epoch);
+  // epoch[0:31]        = sync_epoch[0:31]
+  // epoch[32:63]       = sync_epoch[0:31]
+  // epoch[64:95]       = sync_epoch[0:31]
+  // epoch[96:127]      = sync_epoch[0:31]
+  const m128 epoch = SHUF(epoch1, epoch1, 0, 0, 0, 0);
+  // load low parts of shadow cell epochs into epoch_vect:
+  // epoch_vect[0:31]   = shadow0[0:31]
+  // epoch_vect[32:63]  = shadow0[64:95]
+  // epoch_vect[64:95]  = shadow1[0:31]
+  // epoch_vect[96:127] = shadow1[64:95]
+  const m128 epoch_vect = SHUF(shadow0, shadow1, 0, 2, 0, 2);
+  // epoch_vect >= sync_epoch?
+  const m128 epoch_res = _mm_cmpgt_epi32(epoch_vect, epoch);
+  // addr_res & epoch_res
+  const m128 res = _mm_and_si128(addr_res, epoch_res);
+  // mask[0] = res[7]
+  // mask[1] = res[15]
+  // ...
+  // mask[15] = res[127]
+  const int mask = _mm_movemask_epi8(res);
+  return mask != 0;
+}
+#endif
+
+ALWAYS_INLINE
+bool ContainsSameAccess(u64 *s, u64 a, u64 sync_epoch, bool is_write) {
+#if TSAN_VECTORIZE
+  bool res = ContainsSameAccessFast(s, a, sync_epoch, is_write);
+  // NOTE: this check can fail if the shadow is concurrently mutated
+  // by other threads. But it still can be useful if you modify
+  // ContainsSameAccessFast and want to ensure that it's not completely broken.
+  // DCHECK_EQ(res, ContainsSameAccessSlow(s, a, sync_epoch, is_write));
+  return res;
+#else
+  return ContainsSameAccessSlow(s, a, sync_epoch, is_write);
+#endif
+}
+
+ALWAYS_INLINE USED void MemoryAccess(ThreadState *thr, uptr pc, uptr addr,
+                                     int kAccessSizeLog, bool kAccessIsWrite,
+                                     bool kIsAtomic) {
+  RawShadow *shadow_mem = MemToShadow(addr);
+  DPrintf2(
+      "#%d: MemoryAccess: @%p %p size=%d"
+      " is_write=%d shadow_mem=%p {%zx, %zx, %zx, %zx}\n",
+      (int)thr->fast_state.tid(), (void *)pc, (void *)addr,
+      (int)(1 << kAccessSizeLog), kAccessIsWrite, shadow_mem,
+      (uptr)shadow_mem[0], (uptr)shadow_mem[1], (uptr)shadow_mem[2],
+      (uptr)shadow_mem[3]);
+#if SANITIZER_DEBUG
+  if (!IsAppMem(addr)) {
+    Printf("Access to non app mem %zx\n", addr);
+    DCHECK(IsAppMem(addr));
+  }
+  if (!IsShadowMem(shadow_mem)) {
+    Printf("Bad shadow addr %p (%zx)\n", shadow_mem, addr);
+    DCHECK(IsShadowMem(shadow_mem));
+  }
+#endif
+
+  if (!SANITIZER_GO && !kAccessIsWrite && *shadow_mem == kShadowRodata) {
+    // Access to .rodata section, no races here.
+    // Measurements show that it can be 10-20% of all memory accesses.
+    return;
+  }
+
+  FastState fast_state = thr->fast_state;
+  if (UNLIKELY(fast_state.GetIgnoreBit())) {
+    return;
+  }
+
+  Shadow cur(fast_state);
+  cur.SetAddr0AndSizeLog(addr & 7, kAccessSizeLog);
+  cur.SetWrite(kAccessIsWrite);
+  cur.SetAtomic(kIsAtomic);
+
+  if (LIKELY(ContainsSameAccess(shadow_mem, cur.raw(), thr->fast_synch_epoch,
+                                kAccessIsWrite))) {
+    return;
+  }
+
+  if (kCollectHistory) {
+    fast_state.IncrementEpoch();
+    thr->fast_state = fast_state;
+    TraceAddEvent(thr, fast_state, EventTypeMop, pc);
+    cur.IncrementEpoch();
+  }
+
+  MemoryAccessImpl1(thr, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic,
+                    shadow_mem, cur);
+}
+
+// Called by MemoryAccessRange in tsan_rtl_thread.cpp
+ALWAYS_INLINE USED void MemoryAccessImpl(ThreadState *thr, uptr addr,
+                                         int kAccessSizeLog,
+                                         bool kAccessIsWrite, bool kIsAtomic,
+                                         u64 *shadow_mem, Shadow cur) {
+  if (LIKELY(ContainsSameAccess(shadow_mem, cur.raw(), thr->fast_synch_epoch,
+                                kAccessIsWrite))) {
+    return;
+  }
+
+  MemoryAccessImpl1(thr, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic,
+                    shadow_mem, cur);
+}
+
+static void MemoryRangeSet(ThreadState *thr, uptr pc, uptr addr, uptr size,
+                           u64 val) {
+  (void)thr;
+  (void)pc;
+  if (size == 0)
+    return;
+  // FIXME: fix me.
+  uptr offset = addr % kShadowCell;
+  if (offset) {
+    offset = kShadowCell - offset;
+    if (size <= offset)
+      return;
+    addr += offset;
+    size -= offset;
+  }
+  DCHECK_EQ(addr % 8, 0);
+  // If a user passes some insane arguments (memset(0)),
+  // let it just crash as usual.
+  if (!IsAppMem(addr) || !IsAppMem(addr + size - 1))
+    return;
+  // Don't want to touch lots of shadow memory.
+  // If a program maps 10MB stack, there is no need reset the whole range.
+  size = (size + (kShadowCell - 1)) & ~(kShadowCell - 1);
+  // UnmapOrDie/MmapFixedNoReserve does not work on Windows.
+  if (SANITIZER_WINDOWS || size < common_flags()->clear_shadow_mmap_threshold) {
+    RawShadow *p = MemToShadow(addr);
+    CHECK(IsShadowMem(p));
+    CHECK(IsShadowMem(p + size * kShadowCnt / kShadowCell - 1));
+    // FIXME: may overwrite a part outside the region
+    for (uptr i = 0; i < size / kShadowCell * kShadowCnt;) {
+      p[i++] = val;
+      for (uptr j = 1; j < kShadowCnt; j++) p[i++] = 0;
+    }
+  } else {
+    // The region is big, reset only beginning and end.
+    const uptr kPageSize = GetPageSizeCached();
+    RawShadow *begin = MemToShadow(addr);
+    RawShadow *end = begin + size / kShadowCell * kShadowCnt;
+    RawShadow *p = begin;
+    // Set at least first kPageSize/2 to page boundary.
+    while ((p < begin + kPageSize / kShadowSize / 2) || ((uptr)p % kPageSize)) {
+      *p++ = val;
+      for (uptr j = 1; j < kShadowCnt; j++) *p++ = 0;
+    }
+    // Reset middle part.
+    RawShadow *p1 = p;
+    p = RoundDown(end, kPageSize);
+    if (!MmapFixedSuperNoReserve((uptr)p1, (uptr)p - (uptr)p1))
+      Die();
+    // Set the ending.
+    while (p < end) {
+      *p++ = val;
+      for (uptr j = 1; j < kShadowCnt; j++) *p++ = 0;
+    }
+  }
+}
+
+void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size) {
+  MemoryRangeSet(thr, pc, addr, size, 0);
+}
+
+void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size) {
+  // Processing more than 1k (4k of shadow) is expensive,
+  // can cause excessive memory consumption (user does not necessary touch
+  // the whole range) and most likely unnecessary.
+  if (size > 1024)
+    size = 1024;
+  CHECK_EQ(thr->is_freeing, false);
+  thr->is_freeing = true;
+  MemoryAccessRange(thr, pc, addr, size, true);
+  thr->is_freeing = false;
+  if (kCollectHistory) {
+    thr->fast_state.IncrementEpoch();
+    TraceAddEvent(thr, thr->fast_state, EventTypeMop, pc);
+  }
+  Shadow s(thr->fast_state);
+  s.ClearIgnoreBit();
+  s.MarkAsFreed();
+  s.SetWrite(true);
+  s.SetAddr0AndSizeLog(0, 3);
+  MemoryRangeSet(thr, pc, addr, size, s.raw());
+}
+
+void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size) {
+  if (kCollectHistory) {
+    thr->fast_state.IncrementEpoch();
+    TraceAddEvent(thr, thr->fast_state, EventTypeMop, pc);
+  }
+  Shadow s(thr->fast_state);
+  s.ClearIgnoreBit();
+  s.SetWrite(true);
+  s.SetAddr0AndSizeLog(0, 3);
+  MemoryRangeSet(thr, pc, addr, size, s.raw());
+}
+
+void MemoryRangeImitateWriteOrResetRange(ThreadState *thr, uptr pc, uptr addr,
+                                         uptr size) {
+  if (thr->ignore_reads_and_writes == 0)
+    MemoryRangeImitateWrite(thr, pc, addr, size);
+  else
+    MemoryResetRange(thr, pc, addr, size);
+}
+
+void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size,
+                       bool is_write) {
+  if (size == 0)
+    return;
+
+  RawShadow *shadow_mem = MemToShadow(addr);
+  DPrintf2("#%d: MemoryAccessRange: @%p %p size=%d is_write=%d\n", thr->tid,
+           (void *)pc, (void *)addr, (int)size, is_write);
+
+#if SANITIZER_DEBUG
+  if (!IsAppMem(addr)) {
+    Printf("Access to non app mem %zx\n", addr);
+    DCHECK(IsAppMem(addr));
+  }
+  if (!IsAppMem(addr + size - 1)) {
+    Printf("Access to non app mem %zx\n", addr + size - 1);
+    DCHECK(IsAppMem(addr + size - 1));
+  }
+  if (!IsShadowMem(shadow_mem)) {
+    Printf("Bad shadow addr %p (%zx)\n", shadow_mem, addr);
+    DCHECK(IsShadowMem(shadow_mem));
+  }
+  if (!IsShadowMem(shadow_mem + size * kShadowCnt / 8 - 1)) {
+    Printf("Bad shadow addr %p (%zx)\n", shadow_mem + size * kShadowCnt / 8 - 1,
+           addr + size - 1);
+    DCHECK(IsShadowMem(shadow_mem + size * kShadowCnt / 8 - 1));
+  }
+#endif
+
+  if (*shadow_mem == kShadowRodata) {
+    DCHECK(!is_write);
+    // Access to .rodata section, no races here.
+    // Measurements show that it can be 10-20% of all memory accesses.
+    return;
+  }
+
+  FastState fast_state = thr->fast_state;
+  if (fast_state.GetIgnoreBit())
+    return;
+
+  fast_state.IncrementEpoch();
+  thr->fast_state = fast_state;
+  TraceAddEvent(thr, fast_state, EventTypeMop, pc);
+
+  bool unaligned = (addr % kShadowCell) != 0;
+
+  // Handle unaligned beginning, if any.
+  for (; addr % kShadowCell && size; addr++, size--) {
+    int const kAccessSizeLog = 0;
+    Shadow cur(fast_state);
+    cur.SetWrite(is_write);
+    cur.SetAddr0AndSizeLog(addr & (kShadowCell - 1), kAccessSizeLog);
+    MemoryAccessImpl(thr, addr, kAccessSizeLog, is_write, false, shadow_mem,
+                     cur);
+  }
+  if (unaligned)
+    shadow_mem += kShadowCnt;
+  // Handle middle part, if any.
+  for (; size >= kShadowCell; addr += kShadowCell, size -= kShadowCell) {
+    int const kAccessSizeLog = 3;
+    Shadow cur(fast_state);
+    cur.SetWrite(is_write);
+    cur.SetAddr0AndSizeLog(0, kAccessSizeLog);
+    MemoryAccessImpl(thr, addr, kAccessSizeLog, is_write, false, shadow_mem,
+                     cur);
+    shadow_mem += kShadowCnt;
+  }
+  // Handle ending, if any.
+  for (; size; addr++, size--) {
+    int const kAccessSizeLog = 0;
+    Shadow cur(fast_state);
+    cur.SetWrite(is_write);
+    cur.SetAddr0AndSizeLog(addr & (kShadowCell - 1), kAccessSizeLog);
+    MemoryAccessImpl(thr, addr, kAccessSizeLog, is_write, false, shadow_mem,
+                     cur);
+  }
+}
+
+}  // namespace __tsan
+
+#if !SANITIZER_GO
+// Must be included in this file to make sure everything is inlined.
+#  include "tsan_interface.inc"
+#endif
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_rtl_amd64.S b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_amd64.S
new file mode 100644
index 000000000000..632b19d18158
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_amd64.S
@@ -0,0 +1,446 @@
+// The content of this file is x86_64-only:
+#if defined(__x86_64__)
+
+#include "sanitizer_common/sanitizer_asm.h"
+
+#if !defined(__APPLE__)
+.section .text
+#else
+.section __TEXT,__text
+#endif
+
+ASM_HIDDEN(__tsan_trace_switch)
+.globl ASM_SYMBOL(__tsan_trace_switch_thunk)
+ASM_SYMBOL(__tsan_trace_switch_thunk):
+  CFI_STARTPROC
+  _CET_ENDBR
+  # Save scratch registers.
+  push %rax
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%rax, 0)
+  push %rcx
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%rcx, 0)
+  push %rdx
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%rdx, 0)
+  push %rsi
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%rsi, 0)
+  push %rdi
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%rdi, 0)
+  push %r8
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%r8, 0)
+  push %r9
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%r9, 0)
+  push %r10
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%r10, 0)
+  push %r11
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%r11, 0)
+  # All XMM registers are caller-saved.
+  sub $0x100, %rsp
+  CFI_ADJUST_CFA_OFFSET(0x100)
+  vmovdqu %xmm0, 0x0(%rsp)
+  vmovdqu %xmm1, 0x10(%rsp)
+  vmovdqu %xmm2, 0x20(%rsp)
+  vmovdqu %xmm3, 0x30(%rsp)
+  vmovdqu %xmm4, 0x40(%rsp)
+  vmovdqu %xmm5, 0x50(%rsp)
+  vmovdqu %xmm6, 0x60(%rsp)
+  vmovdqu %xmm7, 0x70(%rsp)
+  vmovdqu %xmm8, 0x80(%rsp)
+  vmovdqu %xmm9, 0x90(%rsp)
+  vmovdqu %xmm10, 0xa0(%rsp)
+  vmovdqu %xmm11, 0xb0(%rsp)
+  vmovdqu %xmm12, 0xc0(%rsp)
+  vmovdqu %xmm13, 0xd0(%rsp)
+  vmovdqu %xmm14, 0xe0(%rsp)
+  vmovdqu %xmm15, 0xf0(%rsp)
+  # Align stack frame.
+  push %rbx  # non-scratch
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%rbx, 0)
+  mov %rsp, %rbx  # save current rsp
+  CFI_DEF_CFA_REGISTER(%rbx)
+  shr $4, %rsp  # clear 4 lsb, align to 16
+  shl $4, %rsp
+
+  call ASM_SYMBOL(__tsan_trace_switch)
+
+  # Unalign stack frame back.
+  mov %rbx, %rsp  # restore the original rsp
+  CFI_DEF_CFA_REGISTER(%rsp)
+  pop %rbx
+  CFI_ADJUST_CFA_OFFSET(-8)
+  # Restore scratch registers.
+  vmovdqu 0x0(%rsp), %xmm0
+  vmovdqu 0x10(%rsp), %xmm1
+  vmovdqu 0x20(%rsp), %xmm2
+  vmovdqu 0x30(%rsp), %xmm3
+  vmovdqu 0x40(%rsp), %xmm4
+  vmovdqu 0x50(%rsp), %xmm5
+  vmovdqu 0x60(%rsp), %xmm6
+  vmovdqu 0x70(%rsp), %xmm7
+  vmovdqu 0x80(%rsp), %xmm8
+  vmovdqu 0x90(%rsp), %xmm9
+  vmovdqu 0xa0(%rsp), %xmm10
+  vmovdqu 0xb0(%rsp), %xmm11
+  vmovdqu 0xc0(%rsp), %xmm12
+  vmovdqu 0xd0(%rsp), %xmm13
+  vmovdqu 0xe0(%rsp), %xmm14
+  vmovdqu 0xf0(%rsp), %xmm15
+  add $0x100, %rsp
+  CFI_ADJUST_CFA_OFFSET(-0x100)
+  pop %r11
+  CFI_ADJUST_CFA_OFFSET(-8)
+  pop %r10
+  CFI_ADJUST_CFA_OFFSET(-8)
+  pop %r9
+  CFI_ADJUST_CFA_OFFSET(-8)
+  pop %r8
+  CFI_ADJUST_CFA_OFFSET(-8)
+  pop %rdi
+  CFI_ADJUST_CFA_OFFSET(-8)
+  pop %rsi
+  CFI_ADJUST_CFA_OFFSET(-8)
+  pop %rdx
+  CFI_ADJUST_CFA_OFFSET(-8)
+  pop %rcx
+  CFI_ADJUST_CFA_OFFSET(-8)
+  pop %rax
+  CFI_ADJUST_CFA_OFFSET(-8)
+  CFI_RESTORE(%rax)
+  CFI_RESTORE(%rbx)
+  CFI_RESTORE(%rcx)
+  CFI_RESTORE(%rdx)
+  CFI_RESTORE(%rsi)
+  CFI_RESTORE(%rdi)
+  CFI_RESTORE(%r8)
+  CFI_RESTORE(%r9)
+  CFI_RESTORE(%r10)
+  CFI_RESTORE(%r11)
+  ret
+  CFI_ENDPROC
+
+ASM_HIDDEN(__tsan_report_race)
+.globl ASM_SYMBOL(__tsan_report_race_thunk)
+ASM_SYMBOL(__tsan_report_race_thunk):
+  CFI_STARTPROC
+  _CET_ENDBR
+  # Save scratch registers.
+  push %rax
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%rax, 0)
+  push %rcx
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%rcx, 0)
+  push %rdx
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%rdx, 0)
+  push %rsi
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%rsi, 0)
+  push %rdi
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%rdi, 0)
+  push %r8
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%r8, 0)
+  push %r9
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%r9, 0)
+  push %r10
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%r10, 0)
+  push %r11
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%r11, 0)
+  # All XMM registers are caller-saved.
+  sub $0x100, %rsp
+  CFI_ADJUST_CFA_OFFSET(0x100)
+  vmovdqu %xmm0, 0x0(%rsp)
+  vmovdqu %xmm1, 0x10(%rsp)
+  vmovdqu %xmm2, 0x20(%rsp)
+  vmovdqu %xmm3, 0x30(%rsp)
+  vmovdqu %xmm4, 0x40(%rsp)
+  vmovdqu %xmm5, 0x50(%rsp)
+  vmovdqu %xmm6, 0x60(%rsp)
+  vmovdqu %xmm7, 0x70(%rsp)
+  vmovdqu %xmm8, 0x80(%rsp)
+  vmovdqu %xmm9, 0x90(%rsp)
+  vmovdqu %xmm10, 0xa0(%rsp)
+  vmovdqu %xmm11, 0xb0(%rsp)
+  vmovdqu %xmm12, 0xc0(%rsp)
+  vmovdqu %xmm13, 0xd0(%rsp)
+  vmovdqu %xmm14, 0xe0(%rsp)
+  vmovdqu %xmm15, 0xf0(%rsp)
+  # Align stack frame.
+  push %rbx  # non-scratch
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%rbx, 0)
+  mov %rsp, %rbx  # save current rsp
+  CFI_DEF_CFA_REGISTER(%rbx)
+  shr $4, %rsp  # clear 4 lsb, align to 16
+  shl $4, %rsp
+
+  call ASM_SYMBOL(__tsan_report_race)
+
+  # Unalign stack frame back.
+  mov %rbx, %rsp  # restore the original rsp
+  CFI_DEF_CFA_REGISTER(%rsp)
+  pop %rbx
+  CFI_ADJUST_CFA_OFFSET(-8)
+  # Restore scratch registers.
+  vmovdqu 0x0(%rsp), %xmm0
+  vmovdqu 0x10(%rsp), %xmm1
+  vmovdqu 0x20(%rsp), %xmm2
+  vmovdqu 0x30(%rsp), %xmm3
+  vmovdqu 0x40(%rsp), %xmm4
+  vmovdqu 0x50(%rsp), %xmm5
+  vmovdqu 0x60(%rsp), %xmm6
+  vmovdqu 0x70(%rsp), %xmm7
+  vmovdqu 0x80(%rsp), %xmm8
+  vmovdqu 0x90(%rsp), %xmm9
+  vmovdqu 0xa0(%rsp), %xmm10
+  vmovdqu 0xb0(%rsp), %xmm11
+  vmovdqu 0xc0(%rsp), %xmm12
+  vmovdqu 0xd0(%rsp), %xmm13
+  vmovdqu 0xe0(%rsp), %xmm14
+  vmovdqu 0xf0(%rsp), %xmm15
+  add $0x100, %rsp
+  CFI_ADJUST_CFA_OFFSET(-0x100)
+  pop %r11
+  CFI_ADJUST_CFA_OFFSET(-8)
+  pop %r10
+  CFI_ADJUST_CFA_OFFSET(-8)
+  pop %r9
+  CFI_ADJUST_CFA_OFFSET(-8)
+  pop %r8
+  CFI_ADJUST_CFA_OFFSET(-8)
+  pop %rdi
+  CFI_ADJUST_CFA_OFFSET(-8)
+  pop %rsi
+  CFI_ADJUST_CFA_OFFSET(-8)
+  pop %rdx
+  CFI_ADJUST_CFA_OFFSET(-8)
+  pop %rcx
+  CFI_ADJUST_CFA_OFFSET(-8)
+  pop %rax
+  CFI_ADJUST_CFA_OFFSET(-8)
+  CFI_RESTORE(%rax)
+  CFI_RESTORE(%rbx)
+  CFI_RESTORE(%rcx)
+  CFI_RESTORE(%rdx)
+  CFI_RESTORE(%rsi)
+  CFI_RESTORE(%rdi)
+  CFI_RESTORE(%r8)
+  CFI_RESTORE(%r9)
+  CFI_RESTORE(%r10)
+  CFI_RESTORE(%r11)
+  ret
+  CFI_ENDPROC
+
+ASM_HIDDEN(__tsan_setjmp)
+#if defined(__NetBSD__)
+.comm _ZN14__interception15real___setjmp14E,8,8
+#elif !defined(__APPLE__)
+.comm _ZN14__interception11real_setjmpE,8,8
+#endif
+#if defined(__NetBSD__)
+.globl ASM_SYMBOL_INTERCEPTOR(__setjmp14)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__setjmp14))
+ASM_SYMBOL_INTERCEPTOR(__setjmp14):
+#else
+.globl ASM_SYMBOL_INTERCEPTOR(setjmp)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(setjmp))
+ASM_SYMBOL_INTERCEPTOR(setjmp):
+#endif
+  CFI_STARTPROC
+  _CET_ENDBR
+  // save env parameter
+  push %rdi
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%rdi, 0)
+  // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)`
+#if defined(__FreeBSD__) || defined(__NetBSD__)
+  lea 8(%rsp), %rdi
+#elif defined(__linux__) || defined(__APPLE__)
+  lea 16(%rsp), %rdi
+#else
+# error "Unknown platform"
+#endif
+  // call tsan interceptor
+  call ASM_SYMBOL(__tsan_setjmp)
+  // restore env parameter
+  pop %rdi
+  CFI_ADJUST_CFA_OFFSET(-8)
+  CFI_RESTORE(%rdi)
+  // tail jump to libc setjmp
+  movl $0, %eax
+#if defined(__NetBSD__)
+  movq _ZN14__interception15real___setjmp14E@GOTPCREL(%rip), %rdx
+  jmp *(%rdx)
+#elif !defined(__APPLE__)
+  movq _ZN14__interception11real_setjmpE@GOTPCREL(%rip), %rdx
+  jmp *(%rdx)
+#else
+  jmp ASM_SYMBOL(setjmp)
+#endif
+  CFI_ENDPROC
+#if defined(__NetBSD__)
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__setjmp14))
+#else
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(setjmp))
+#endif
+
+.comm _ZN14__interception12real__setjmpE,8,8
+.globl ASM_SYMBOL_INTERCEPTOR(_setjmp)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_setjmp))
+ASM_SYMBOL_INTERCEPTOR(_setjmp):
+  CFI_STARTPROC
+  _CET_ENDBR
+  // save env parameter
+  push %rdi
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%rdi, 0)
+  // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)`
+#if defined(__FreeBSD__) || defined(__NetBSD__)
+  lea 8(%rsp), %rdi
+#elif defined(__linux__) || defined(__APPLE__)
+  lea 16(%rsp), %rdi
+#else
+# error "Unknown platform"
+#endif
+  // call tsan interceptor
+  call ASM_SYMBOL(__tsan_setjmp)
+  // restore env parameter
+  pop %rdi
+  CFI_ADJUST_CFA_OFFSET(-8)
+  CFI_RESTORE(%rdi)
+  // tail jump to libc setjmp
+  movl $0, %eax
+#if !defined(__APPLE__)
+  movq _ZN14__interception12real__setjmpE@GOTPCREL(%rip), %rdx
+  jmp *(%rdx)
+#else
+  jmp ASM_SYMBOL(_setjmp)
+#endif
+  CFI_ENDPROC
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(_setjmp))
+
+#if defined(__NetBSD__)
+.comm _ZN14__interception18real___sigsetjmp14E,8,8
+.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14))
+ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14):
+#else
+.comm _ZN14__interception14real_sigsetjmpE,8,8
+.globl ASM_SYMBOL_INTERCEPTOR(sigsetjmp)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
+ASM_SYMBOL_INTERCEPTOR(sigsetjmp):
+#endif
+  CFI_STARTPROC
+  _CET_ENDBR
+  // save env parameter
+  push %rdi
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%rdi, 0)
+  // save savesigs parameter
+  push %rsi
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%rsi, 0)
+  // align stack frame
+  sub $8, %rsp
+  CFI_ADJUST_CFA_OFFSET(8)
+  // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)`
+#if defined(__FreeBSD__) || defined(__NetBSD__)
+  lea 24(%rsp), %rdi
+#elif defined(__linux__) || defined(__APPLE__)
+  lea 32(%rsp), %rdi
+#else
+# error "Unknown platform"
+#endif
+  // call tsan interceptor
+  call ASM_SYMBOL(__tsan_setjmp)
+  // unalign stack frame
+  add $8, %rsp
+  CFI_ADJUST_CFA_OFFSET(-8)
+  // restore savesigs parameter
+  pop %rsi
+  CFI_ADJUST_CFA_OFFSET(-8)
+  CFI_RESTORE(%rsi)
+  // restore env parameter
+  pop %rdi
+  CFI_ADJUST_CFA_OFFSET(-8)
+  CFI_RESTORE(%rdi)
+  // tail jump to libc sigsetjmp
+  movl $0, %eax
+#if defined(__NetBSD__)
+  movq _ZN14__interception18real___sigsetjmp14E@GOTPCREL(%rip), %rdx
+  jmp *(%rdx)
+#elif !defined(__APPLE__)
+  movq _ZN14__interception14real_sigsetjmpE@GOTPCREL(%rip), %rdx
+  jmp *(%rdx)
+#else
+  jmp ASM_SYMBOL(sigsetjmp)
+#endif
+  CFI_ENDPROC
+#if defined(__NetBSD__)
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14))
+#else
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
+#endif
+
+#if !defined(__APPLE__) && !defined(__NetBSD__)
+.comm _ZN14__interception16real___sigsetjmpE,8,8
+.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
+ASM_SYMBOL_INTERCEPTOR(__sigsetjmp):
+  CFI_STARTPROC
+  _CET_ENDBR
+  // save env parameter
+  push %rdi
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%rdi, 0)
+  // save savesigs parameter
+  push %rsi
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%rsi, 0)
+  // align stack frame
+  sub $8, %rsp
+  CFI_ADJUST_CFA_OFFSET(8)
+  // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)`
+#if defined(__FreeBSD__)
+  lea 24(%rsp), %rdi
+#else
+  lea 32(%rsp), %rdi
+#endif
+  // call tsan interceptor
+  call ASM_SYMBOL(__tsan_setjmp)
+  // unalign stack frame
+  add $8, %rsp
+  CFI_ADJUST_CFA_OFFSET(-8)
+  // restore savesigs parameter
+  pop %rsi
+  CFI_ADJUST_CFA_OFFSET(-8)
+  CFI_RESTORE(%rsi)
+  // restore env parameter
+  pop %rdi
+  CFI_ADJUST_CFA_OFFSET(-8)
+  CFI_RESTORE(%rdi)
+  // tail jump to libc sigsetjmp
+  movl $0, %eax
+  movq _ZN14__interception16real___sigsetjmpE@GOTPCREL(%rip), %rdx
+  jmp *(%rdx)
+  CFI_ENDPROC
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
+#endif  // !defined(__APPLE__) && !defined(__NetBSD__)
+
+NO_EXEC_STACK_DIRECTIVE
+
+#endif
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_rtl_mips64.S b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_mips64.S
new file mode 100644
index 000000000000..d0f7a3f9af98
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_mips64.S
@@ -0,0 +1,214 @@
+.section .text
+.set noreorder
+
+.hidden __tsan_setjmp
+.comm _ZN14__interception11real_setjmpE,8,8
+.globl setjmp
+.type setjmp, @function
+setjmp:
+
+  // save env parameters
+  daddiu $sp,$sp,-40
+  sd $s0,32($sp)
+  sd $ra,24($sp)
+  sd $fp,16($sp)
+  sd $gp,8($sp)
+
+  // calculate and save pointer to GOT
+  lui $gp,%hi(%neg(%gp_rel(setjmp)))
+  daddu $gp,$gp,$t9
+  daddiu $gp,$gp,%lo(%neg(%gp_rel(setjmp)))
+  move $s0,$gp
+
+  // save jmp_buf
+  sd $a0,0($sp)
+
+  // obtain $sp
+  dadd $a0,$zero,$sp
+
+  // call tsan interceptor
+  jal __tsan_setjmp
+  daddiu $a1,$a0,40
+
+  // restore jmp_buf
+  ld $a0,0($sp)
+
+  // restore gp
+  move $gp,$s0
+
+  // load pointer of libc setjmp to t9
+  dla $t9,(_ZN14__interception11real_setjmpE) 
+
+  // restore env parameters
+  ld $gp,8($sp)
+  ld $fp,16($sp)
+  ld $ra,24($sp)
+  ld $s0,32($sp)
+  daddiu $sp,$sp,40
+
+  // tail jump to libc setjmp
+  ld $t9,0($t9)
+  jr $t9
+  nop
+
+.size setjmp, .-setjmp
+
+.hidden __tsan_setjmp
+.globl _setjmp
+.comm _ZN14__interception12real__setjmpE,8,8
+.type _setjmp, @function
+_setjmp:
+
+  // Save env parameters
+  daddiu $sp,$sp,-40
+  sd $s0,32($sp)
+  sd $ra,24($sp)
+  sd $fp,16($sp)
+  sd $gp,8($sp)
+
+  // calculate and save pointer to GOT
+  lui $gp,%hi(%neg(%gp_rel(_setjmp)))
+  daddu $gp,$gp,$t9
+  daddiu $gp,$gp,%lo(%neg(%gp_rel(_setjmp)))
+  move $s0,$gp
+
+  // save jmp_buf
+  sd $a0,0($sp)
+
+  // obtain $sp
+  dadd $a0,$zero,$sp
+
+  // call tsan interceptor
+  jal __tsan_setjmp
+  daddiu $a1,$a0,40
+
+  // restore jmp_buf
+  ld $a0,0($sp)
+
+  // restore gp
+  move $gp,$s0
+
+  // load pointer of libc _setjmp to t9
+  dla $t9,(_ZN14__interception12real__setjmpE)
+
+  // restore env parameters
+  ld $gp,8($sp)
+  ld $fp,16($sp)
+  ld $ra,24($sp)
+  ld $s0,32($sp)
+  daddiu $sp,$sp,40
+
+  // tail jump to libc _setjmp
+  ld $t9,0($t9)
+  jr $t9
+  nop
+
+.size _setjmp, .-_setjmp
+
+.hidden __tsan_setjmp
+.globl sigsetjmp
+.comm _ZN14__interception14real_sigsetjmpE,8,8
+.type sigsetjmp, @function
+sigsetjmp:
+
+  // Save env parameters
+  daddiu $sp,$sp,-48
+  sd $s0,40($sp)
+  sd $ra,32($sp)
+  sd $fp,24($sp)
+  sd $gp,16($sp)
+
+  // calculate and save pointer to GOT
+  lui $gp,%hi(%neg(%gp_rel(sigsetjmp)))
+  daddu $gp,$gp,$t9
+  daddiu $gp,$gp,%lo(%neg(%gp_rel(sigsetjmp)))
+  move $s0,$gp
+
+  // save jmp_buf and savesig
+  sd $a0,0($sp)
+  sd $a1,8($sp)
+
+  // obtain $sp
+  dadd $a0,$zero,$sp
+
+  // call tsan interceptor
+  jal __tsan_setjmp
+  daddiu $a1,$a0,48
+
+  // restore jmp_buf and savesig
+  ld $a0,0($sp)
+  ld $a1,8($sp)
+
+  // restore gp
+  move $gp,$s0
+
+  // load pointer of libc sigsetjmp to t9
+  dla $t9,(_ZN14__interception14real_sigsetjmpE) 
+
+  // restore env parameters
+  ld $gp,16($sp)
+  ld $fp,24($sp)
+  ld $ra,32($sp)
+  ld $s0,40($sp)
+  daddiu $sp,$sp,48
+
+  // tail jump to libc sigsetjmp
+  ld $t9,0($t9)
+  jr $t9
+  nop
+
+.size sigsetjmp, .-sigsetjmp
+
+.hidden __tsan_setjmp
+.comm _ZN14__interception16real___sigsetjmpE,8,8
+.globl __sigsetjmp
+.type __sigsetjmp, @function
+__sigsetjmp:
+
+  // Save env parameters
+  daddiu $sp,$sp,-48
+  sd $s0,40($sp)
+  sd $ra,32($sp)
+  sd $fp,24($sp)
+  sd $gp,16($sp)
+
+  // calculate and save pointer to GOT
+  lui $gp,%hi(%neg(%gp_rel(__sigsetjmp)))
+  daddu $gp,$gp,$t9
+  daddiu $gp,$gp,%lo(%neg(%gp_rel(__sigsetjmp)))
+  move $s0,$gp
+
+  // save jmp_buf and savesig
+  sd $a0,0($sp)
+  sd $a1,8($sp)
+
+  // obtain $sp
+  dadd $a0,$zero,$sp
+
+  // call tsan interceptor
+  jal __tsan_setjmp
+  daddiu $a1,$a0,48
+
+  // restore jmp_buf and savesig
+  ld $a0,0($sp)
+  ld $a1,8($sp)
+
+  // restore gp
+  move $gp,$s0
+
+  // load pointer to libc __sigsetjmp in t9
+  dla $t9,(_ZN14__interception16real___sigsetjmpE)
+
+  // restore env parameters
+  ld $gp,16($sp)
+  ld $fp,24($sp)
+  ld $ra,32($sp)
+  ld $s0,40($sp)
+  daddiu $sp,$sp,48
+
+  // tail jump to libc __sigsetjmp
+  ld $t9,0($t9)
+  jr $t9
+  nop
+
+.size __sigsetjmp, .-__sigsetjmp
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_rtl_mutex.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_mutex.cpp
new file mode 100644
index 000000000000..7d6b41116aa6
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_mutex.cpp
@@ -0,0 +1,555 @@
+//===-- tsan_rtl_mutex.cpp ------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+
+#include <sanitizer_common/sanitizer_deadlock_detector_interface.h>
+#include <sanitizer_common/sanitizer_stackdepot.h>
+
+#include "tsan_rtl.h"
+#include "tsan_flags.h"
+#include "tsan_sync.h"
+#include "tsan_report.h"
+#include "tsan_symbolize.h"
+#include "tsan_platform.h"
+
+namespace __tsan {
+
+void ReportDeadlock(ThreadState *thr, uptr pc, DDReport *r);
+
+struct Callback final : public DDCallback {
+  ThreadState *thr;
+  uptr pc;
+
+  Callback(ThreadState *thr, uptr pc)
+      : thr(thr)
+      , pc(pc) {
+    DDCallback::pt = thr->proc()->dd_pt;
+    DDCallback::lt = thr->dd_lt;
+  }
+
+  StackID Unwind() override { return CurrentStackId(thr, pc); }
+  int UniqueTid() override { return thr->unique_id; }
+};
+
+void DDMutexInit(ThreadState *thr, uptr pc, SyncVar *s) {
+  Callback cb(thr, pc);
+  ctx->dd->MutexInit(&cb, &s->dd);
+  s->dd.ctx = s->GetId();
+}
+
+static void ReportMutexMisuse(ThreadState *thr, uptr pc, ReportType typ,
+    uptr addr, u64 mid) {
+  // In Go, these misuses are either impossible, or detected by std lib,
+  // or false positives (e.g. unlock in a different thread).
+  if (SANITIZER_GO)
+    return;
+  if (!ShouldReport(thr, typ))
+    return;
+  ThreadRegistryLock l(&ctx->thread_registry);
+  ScopedReport rep(typ);
+  rep.AddMutex(mid);
+  VarSizeStackTrace trace;
+  ObtainCurrentStack(thr, pc, &trace);
+  rep.AddStack(trace, true);
+  rep.AddLocation(addr, 1);
+  OutputReport(thr, rep);
+}
+
+void MutexCreate(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
+  DPrintf("#%d: MutexCreate %zx flagz=0x%x\n", thr->tid, addr, flagz);
+  if (!(flagz & MutexFlagLinkerInit) && IsAppMem(addr)) {
+    CHECK(!thr->is_freeing);
+    thr->is_freeing = true;
+    MemoryAccess(thr, pc, addr, 1, kAccessWrite);
+    thr->is_freeing = false;
+  }
+  SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
+  Lock l(&s->mtx);
+  s->SetFlags(flagz & MutexCreationFlagMask);
+  // Save stack in the case the sync object was created before as atomic.
+  if (!SANITIZER_GO && s->creation_stack_id == 0)
+    s->creation_stack_id = CurrentStackId(thr, pc);
+}
+
+void MutexDestroy(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
+  DPrintf("#%d: MutexDestroy %zx\n", thr->tid, addr);
+  bool unlock_locked = false;
+  u64 mid = 0;
+  u64 last_lock = 0;
+  {
+    SyncVar *s = ctx->metamap.GetSyncIfExists(addr);
+    if (s == 0)
+      return;
+    Lock l(&s->mtx);
+    if ((flagz & MutexFlagLinkerInit) || s->IsFlagSet(MutexFlagLinkerInit) ||
+        ((flagz & MutexFlagNotStatic) && !s->IsFlagSet(MutexFlagNotStatic))) {
+      // Destroy is no-op for linker-initialized mutexes.
+      return;
+    }
+    if (common_flags()->detect_deadlocks) {
+      Callback cb(thr, pc);
+      ctx->dd->MutexDestroy(&cb, &s->dd);
+      ctx->dd->MutexInit(&cb, &s->dd);
+    }
+    if (flags()->report_destroy_locked && s->owner_tid != kInvalidTid &&
+        !s->IsFlagSet(MutexFlagBroken)) {
+      s->SetFlags(MutexFlagBroken);
+      unlock_locked = true;
+    }
+    mid = s->GetId();
+    last_lock = s->last_lock;
+    if (!unlock_locked)
+      s->Reset(thr->proc());  // must not reset it before the report is printed
+  }
+  if (unlock_locked && ShouldReport(thr, ReportTypeMutexDestroyLocked)) {
+    ThreadRegistryLock l(&ctx->thread_registry);
+    ScopedReport rep(ReportTypeMutexDestroyLocked);
+    rep.AddMutex(mid);
+    VarSizeStackTrace trace;
+    ObtainCurrentStack(thr, pc, &trace);
+    rep.AddStack(trace, true);
+    FastState last(last_lock);
+    RestoreStack(last.tid(), last.epoch(), &trace, 0);
+    rep.AddStack(trace, true);
+    rep.AddLocation(addr, 1);
+    OutputReport(thr, rep);
+
+    SyncVar *s = ctx->metamap.GetSyncIfExists(addr);
+    if (s != 0) {
+      Lock l(&s->mtx);
+      s->Reset(thr->proc());
+    }
+  }
+  thr->mset.Remove(mid);
+  // Imitate a memory write to catch unlock-destroy races.
+  // Do this outside of sync mutex, because it can report a race which locks
+  // sync mutexes.
+  if (IsAppMem(addr))
+    MemoryAccess(thr, pc, addr, 1, kAccessWrite | kAccessFree);
+  // s will be destroyed and freed in MetaMap::FreeBlock.
+}
+
+void MutexPreLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
+  DPrintf("#%d: MutexPreLock %zx flagz=0x%x\n", thr->tid, addr, flagz);
+  if (!(flagz & MutexFlagTryLock) && common_flags()->detect_deadlocks) {
+    SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
+    {
+      ReadLock l(&s->mtx);
+      s->UpdateFlags(flagz);
+      if (s->owner_tid != thr->tid) {
+        Callback cb(thr, pc);
+        ctx->dd->MutexBeforeLock(&cb, &s->dd, true);
+      }
+    }
+    Callback cb(thr, pc);
+    ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
+  }
+}
+
+void MutexPostLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz, int rec) {
+  DPrintf("#%d: MutexPostLock %zx flag=0x%x rec=%d\n",
+      thr->tid, addr, flagz, rec);
+  if (flagz & MutexFlagRecursiveLock)
+    CHECK_GT(rec, 0);
+  else
+    rec = 1;
+  if (IsAppMem(addr))
+    MemoryAccess(thr, pc, addr, 1, kAccessRead | kAccessAtomic);
+  u64 mid = 0;
+  bool pre_lock = false;
+  bool first = false;
+  bool report_double_lock = false;
+  {
+    SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
+    Lock l(&s->mtx);
+    s->UpdateFlags(flagz);
+    thr->fast_state.IncrementEpoch();
+    TraceAddEvent(thr, thr->fast_state, EventTypeLock, s->GetId());
+    if (s->owner_tid == kInvalidTid) {
+      CHECK_EQ(s->recursion, 0);
+      s->owner_tid = thr->tid;
+      s->last_lock = thr->fast_state.raw();
+    } else if (s->owner_tid == thr->tid) {
+      CHECK_GT(s->recursion, 0);
+    } else if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
+      s->SetFlags(MutexFlagBroken);
+      report_double_lock = true;
+    }
+    first = s->recursion == 0;
+    s->recursion += rec;
+    if (first) {
+      AcquireImpl(thr, pc, &s->clock);
+      AcquireImpl(thr, pc, &s->read_clock);
+    } else if (!s->IsFlagSet(MutexFlagWriteReentrant)) {
+    }
+    thr->mset.Add(s->GetId(), true, thr->fast_state.epoch());
+    if (first && common_flags()->detect_deadlocks) {
+      pre_lock =
+          (flagz & MutexFlagDoPreLockOnPostLock) && !(flagz & MutexFlagTryLock);
+      Callback cb(thr, pc);
+      if (pre_lock)
+        ctx->dd->MutexBeforeLock(&cb, &s->dd, true);
+      ctx->dd->MutexAfterLock(&cb, &s->dd, true, flagz & MutexFlagTryLock);
+    }
+    mid = s->GetId();
+  }
+  if (report_double_lock)
+    ReportMutexMisuse(thr, pc, ReportTypeMutexDoubleLock, addr, mid);
+  if (first && pre_lock && common_flags()->detect_deadlocks) {
+    Callback cb(thr, pc);
+    ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
+  }
+}
+
+int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
+  DPrintf("#%d: MutexUnlock %zx flagz=0x%x\n", thr->tid, addr, flagz);
+  if (IsAppMem(addr))
+    MemoryAccess(thr, pc, addr, 1, kAccessRead | kAccessAtomic);
+  u64 mid = 0;
+  bool report_bad_unlock = false;
+  int rec = 0;
+  {
+    SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
+    Lock l(&s->mtx);
+    thr->fast_state.IncrementEpoch();
+    TraceAddEvent(thr, thr->fast_state, EventTypeUnlock, s->GetId());
+    if (!SANITIZER_GO && (s->recursion == 0 || s->owner_tid != thr->tid)) {
+      if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
+        s->SetFlags(MutexFlagBroken);
+        report_bad_unlock = true;
+      }
+    } else {
+      rec = (flagz & MutexFlagRecursiveUnlock) ? s->recursion : 1;
+      s->recursion -= rec;
+      if (s->recursion == 0) {
+        s->owner_tid = kInvalidTid;
+        ReleaseStoreImpl(thr, pc, &s->clock);
+      } else {
+      }
+    }
+    thr->mset.Del(s->GetId(), true);
+    if (common_flags()->detect_deadlocks && s->recursion == 0 &&
+        !report_bad_unlock) {
+      Callback cb(thr, pc);
+      ctx->dd->MutexBeforeUnlock(&cb, &s->dd, true);
+    }
+    mid = s->GetId();
+  }
+  if (report_bad_unlock)
+    ReportMutexMisuse(thr, pc, ReportTypeMutexBadUnlock, addr, mid);
+  if (common_flags()->detect_deadlocks && !report_bad_unlock) {
+    Callback cb(thr, pc);
+    ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
+  }
+  return rec;
+}
+
+void MutexPreReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
+  DPrintf("#%d: MutexPreReadLock %zx flagz=0x%x\n", thr->tid, addr, flagz);
+  if (!(flagz & MutexFlagTryLock) && common_flags()->detect_deadlocks) {
+    {
+      SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
+      ReadLock l(&s->mtx);
+      s->UpdateFlags(flagz);
+      Callback cb(thr, pc);
+      ctx->dd->MutexBeforeLock(&cb, &s->dd, false);
+    }
+    Callback cb(thr, pc);
+    ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
+  }
+}
+
+void MutexPostReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
+  DPrintf("#%d: MutexPostReadLock %zx flagz=0x%x\n", thr->tid, addr, flagz);
+  if (IsAppMem(addr))
+    MemoryAccess(thr, pc, addr, 1, kAccessRead | kAccessAtomic);
+  u64 mid = 0;
+  bool report_bad_lock = false;
+  bool pre_lock = false;
+  {
+    SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
+    ReadLock l(&s->mtx);
+    s->UpdateFlags(flagz);
+    thr->fast_state.IncrementEpoch();
+    TraceAddEvent(thr, thr->fast_state, EventTypeRLock, s->GetId());
+    if (s->owner_tid != kInvalidTid) {
+      if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
+        s->SetFlags(MutexFlagBroken);
+        report_bad_lock = true;
+      }
+    }
+    AcquireImpl(thr, pc, &s->clock);
+    s->last_lock = thr->fast_state.raw();
+    thr->mset.Add(s->GetId(), false, thr->fast_state.epoch());
+    if (common_flags()->detect_deadlocks) {
+      pre_lock =
+          (flagz & MutexFlagDoPreLockOnPostLock) && !(flagz & MutexFlagTryLock);
+      Callback cb(thr, pc);
+      if (pre_lock)
+        ctx->dd->MutexBeforeLock(&cb, &s->dd, false);
+      ctx->dd->MutexAfterLock(&cb, &s->dd, false, flagz & MutexFlagTryLock);
+    }
+    mid = s->GetId();
+  }
+  if (report_bad_lock)
+    ReportMutexMisuse(thr, pc, ReportTypeMutexBadReadLock, addr, mid);
+  if (pre_lock  && common_flags()->detect_deadlocks) {
+    Callback cb(thr, pc);
+    ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
+  }
+}
+
+void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr) {
+  DPrintf("#%d: MutexReadUnlock %zx\n", thr->tid, addr);
+  if (IsAppMem(addr))
+    MemoryAccess(thr, pc, addr, 1, kAccessRead | kAccessAtomic);
+  u64 mid = 0;
+  bool report_bad_unlock = false;
+  {
+    SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
+    Lock l(&s->mtx);
+    thr->fast_state.IncrementEpoch();
+    TraceAddEvent(thr, thr->fast_state, EventTypeRUnlock, s->GetId());
+    if (s->owner_tid != kInvalidTid) {
+      if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
+        s->SetFlags(MutexFlagBroken);
+        report_bad_unlock = true;
+      }
+    }
+    ReleaseImpl(thr, pc, &s->read_clock);
+    if (common_flags()->detect_deadlocks && s->recursion == 0) {
+      Callback cb(thr, pc);
+      ctx->dd->MutexBeforeUnlock(&cb, &s->dd, false);
+    }
+    mid = s->GetId();
+  }
+  thr->mset.Del(mid, false);
+  if (report_bad_unlock)
+    ReportMutexMisuse(thr, pc, ReportTypeMutexBadReadUnlock, addr, mid);
+  if (common_flags()->detect_deadlocks) {
+    Callback cb(thr, pc);
+    ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
+  }
+}
+
+void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr) {
+  DPrintf("#%d: MutexReadOrWriteUnlock %zx\n", thr->tid, addr);
+  if (IsAppMem(addr))
+    MemoryAccess(thr, pc, addr, 1, kAccessRead | kAccessAtomic);
+  u64 mid = 0;
+  bool report_bad_unlock = false;
+  {
+    SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
+    Lock l(&s->mtx);
+    bool write = true;
+    if (s->owner_tid == kInvalidTid) {
+      // Seems to be read unlock.
+      write = false;
+      thr->fast_state.IncrementEpoch();
+      TraceAddEvent(thr, thr->fast_state, EventTypeRUnlock, s->GetId());
+      ReleaseImpl(thr, pc, &s->read_clock);
+    } else if (s->owner_tid == thr->tid) {
+      // Seems to be write unlock.
+      thr->fast_state.IncrementEpoch();
+      TraceAddEvent(thr, thr->fast_state, EventTypeUnlock, s->GetId());
+      CHECK_GT(s->recursion, 0);
+      s->recursion--;
+      if (s->recursion == 0) {
+        s->owner_tid = kInvalidTid;
+        ReleaseStoreImpl(thr, pc, &s->clock);
+      } else {
+      }
+    } else if (!s->IsFlagSet(MutexFlagBroken)) {
+      s->SetFlags(MutexFlagBroken);
+      report_bad_unlock = true;
+    }
+    thr->mset.Del(s->GetId(), write);
+    if (common_flags()->detect_deadlocks && s->recursion == 0) {
+      Callback cb(thr, pc);
+      ctx->dd->MutexBeforeUnlock(&cb, &s->dd, write);
+    }
+    mid = s->GetId();
+  }
+  if (report_bad_unlock)
+    ReportMutexMisuse(thr, pc, ReportTypeMutexBadUnlock, addr, mid);
+  if (common_flags()->detect_deadlocks) {
+    Callback cb(thr, pc);
+    ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
+  }
+}
+
+void MutexRepair(ThreadState *thr, uptr pc, uptr addr) {
+  DPrintf("#%d: MutexRepair %zx\n", thr->tid, addr);
+  SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
+  Lock l(&s->mtx);
+  s->owner_tid = kInvalidTid;
+  s->recursion = 0;
+}
+
+void MutexInvalidAccess(ThreadState *thr, uptr pc, uptr addr) {
+  DPrintf("#%d: MutexInvalidAccess %zx\n", thr->tid, addr);
+  SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
+  ReportMutexMisuse(thr, pc, ReportTypeMutexInvalidAccess, addr, s->GetId());
+}
+
+void Acquire(ThreadState *thr, uptr pc, uptr addr) {
+  DPrintf("#%d: Acquire %zx\n", thr->tid, addr);
+  if (thr->ignore_sync)
+    return;
+  SyncVar *s = ctx->metamap.GetSyncIfExists(addr);
+  if (!s)
+    return;
+  ReadLock l(&s->mtx);
+  AcquireImpl(thr, pc, &s->clock);
+}
+
+static void UpdateClockCallback(ThreadContextBase *tctx_base, void *arg) {
+  ThreadState *thr = reinterpret_cast<ThreadState*>(arg);
+  ThreadContext *tctx = static_cast<ThreadContext*>(tctx_base);
+  u64 epoch = tctx->epoch1;
+  if (tctx->status == ThreadStatusRunning) {
+    epoch = tctx->thr->fast_state.epoch();
+    tctx->thr->clock.NoteGlobalAcquire(epoch);
+  }
+  thr->clock.set(&thr->proc()->clock_cache, tctx->tid, epoch);
+}
+
+void AcquireGlobal(ThreadState *thr) {
+  DPrintf("#%d: AcquireGlobal\n", thr->tid);
+  if (thr->ignore_sync)
+    return;
+  ThreadRegistryLock l(&ctx->thread_registry);
+  ctx->thread_registry.RunCallbackForEachThreadLocked(UpdateClockCallback, thr);
+}
+
+void ReleaseStoreAcquire(ThreadState *thr, uptr pc, uptr addr) {
+  DPrintf("#%d: ReleaseStoreAcquire %zx\n", thr->tid, addr);
+  if (thr->ignore_sync)
+    return;
+  SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, false);
+  Lock l(&s->mtx);
+  thr->fast_state.IncrementEpoch();
+  // Can't increment epoch w/o writing to the trace as well.
+  TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
+  ReleaseStoreAcquireImpl(thr, pc, &s->clock);
+}
+
+void Release(ThreadState *thr, uptr pc, uptr addr) {
+  DPrintf("#%d: Release %zx\n", thr->tid, addr);
+  if (thr->ignore_sync)
+    return;
+  SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, false);
+  Lock l(&s->mtx);
+  thr->fast_state.IncrementEpoch();
+  // Can't increment epoch w/o writing to the trace as well.
+  TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
+  ReleaseImpl(thr, pc, &s->clock);
+}
+
+void ReleaseStore(ThreadState *thr, uptr pc, uptr addr) {
+  DPrintf("#%d: ReleaseStore %zx\n", thr->tid, addr);
+  if (thr->ignore_sync)
+    return;
+  SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, false);
+  Lock l(&s->mtx);
+  thr->fast_state.IncrementEpoch();
+  // Can't increment epoch w/o writing to the trace as well.
+  TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
+  ReleaseStoreImpl(thr, pc, &s->clock);
+}
+
+#if !SANITIZER_GO
+static void UpdateSleepClockCallback(ThreadContextBase *tctx_base, void *arg) {
+  ThreadState *thr = reinterpret_cast<ThreadState*>(arg);
+  ThreadContext *tctx = static_cast<ThreadContext*>(tctx_base);
+  u64 epoch = tctx->epoch1;
+  if (tctx->status == ThreadStatusRunning)
+    epoch = tctx->thr->fast_state.epoch();
+  thr->last_sleep_clock.set(&thr->proc()->clock_cache, tctx->tid, epoch);
+}
+
+void AfterSleep(ThreadState *thr, uptr pc) {
+  DPrintf("#%d: AfterSleep\n", thr->tid);
+  if (thr->ignore_sync)
+    return;
+  thr->last_sleep_stack_id = CurrentStackId(thr, pc);
+  ThreadRegistryLock l(&ctx->thread_registry);
+  ctx->thread_registry.RunCallbackForEachThreadLocked(UpdateSleepClockCallback,
+                                                      thr);
+}
+#endif
+
+void AcquireImpl(ThreadState *thr, uptr pc, SyncClock *c) {
+  if (thr->ignore_sync)
+    return;
+  thr->clock.set(thr->fast_state.epoch());
+  thr->clock.acquire(&thr->proc()->clock_cache, c);
+}
+
+void ReleaseStoreAcquireImpl(ThreadState *thr, uptr pc, SyncClock *c) {
+  if (thr->ignore_sync)
+    return;
+  thr->clock.set(thr->fast_state.epoch());
+  thr->fast_synch_epoch = thr->fast_state.epoch();
+  thr->clock.releaseStoreAcquire(&thr->proc()->clock_cache, c);
+}
+
+void ReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c) {
+  if (thr->ignore_sync)
+    return;
+  thr->clock.set(thr->fast_state.epoch());
+  thr->fast_synch_epoch = thr->fast_state.epoch();
+  thr->clock.release(&thr->proc()->clock_cache, c);
+}
+
+void ReleaseStoreImpl(ThreadState *thr, uptr pc, SyncClock *c) {
+  if (thr->ignore_sync)
+    return;
+  thr->clock.set(thr->fast_state.epoch());
+  thr->fast_synch_epoch = thr->fast_state.epoch();
+  thr->clock.ReleaseStore(&thr->proc()->clock_cache, c);
+}
+
+void AcquireReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c) {
+  if (thr->ignore_sync)
+    return;
+  thr->clock.set(thr->fast_state.epoch());
+  thr->fast_synch_epoch = thr->fast_state.epoch();
+  thr->clock.acq_rel(&thr->proc()->clock_cache, c);
+}
+
+void ReportDeadlock(ThreadState *thr, uptr pc, DDReport *r) {
+  if (r == 0 || !ShouldReport(thr, ReportTypeDeadlock))
+    return;
+  ThreadRegistryLock l(&ctx->thread_registry);
+  ScopedReport rep(ReportTypeDeadlock);
+  for (int i = 0; i < r->n; i++) {
+    rep.AddMutex(r->loop[i].mtx_ctx0);
+    rep.AddUniqueTid((int)r->loop[i].thr_ctx);
+    rep.AddThread((int)r->loop[i].thr_ctx);
+  }
+  uptr dummy_pc = 0x42;
+  for (int i = 0; i < r->n; i++) {
+    for (int j = 0; j < (flags()->second_deadlock_stack ? 2 : 1); j++) {
+      u32 stk = r->loop[i].stk[j];
+      if (stk && stk != 0xffffffff) {
+        rep.AddStack(StackDepotGet(stk), true);
+      } else {
+        // Sometimes we fail to extract the stack trace (FIXME: investigate),
+        // but we should still produce some stack trace in the report.
+        rep.AddStack(StackTrace(&dummy_pc, 1), true);
+      }
+    }
+  }
+  OutputReport(thr, rep);
+}
+
+}  // namespace __tsan
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_rtl_ppc64.S b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_ppc64.S
new file mode 100644
index 000000000000..8285e21aa1ec
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_ppc64.S
@@ -0,0 +1,288 @@
+#include "tsan_ppc_regs.h"
+
+        .section .text
+        .hidden __tsan_setjmp
+        .globl _setjmp
+        .type _setjmp, @function
+        .align 4
+#if _CALL_ELF == 2
+_setjmp:
+#else
+	.section ".opd","aw"
+	.align 3
+_setjmp:
+	.quad   .L._setjmp,.TOC.@tocbase,0
+	.previous
+#endif
+.L._setjmp:
+        mflr    r0
+        stdu    r1,-48(r1)
+        std     r2,24(r1)
+        std     r3,32(r1)
+        std     r0,40(r1)
+        // r3 is the original stack pointer.
+        addi    r3,r1,48
+        // r4 is the mangled stack pointer (see glibc)
+        ld      r4,-28696(r13)
+        xor     r4,r3,r4
+        // Materialize a TOC in case we were called from libc.
+        // For big-endian, we load the TOC from the OPD.  For little-
+        // endian, we use the .TOC. symbol to find it.
+        nop
+        bcl     20,31,0f
+0:
+        mflr    r2
+#if _CALL_ELF == 2
+        addis   r2,r2,.TOC.-0b@ha
+        addi    r2,r2,.TOC.-0b@l
+#else
+        addis   r2,r2,_setjmp-0b@ha
+        addi    r2,r2,_setjmp-0b@l
+        ld      r2,8(r2)
+#endif
+        // Call the interceptor.
+        bl      __tsan_setjmp
+        nop
+        // Restore regs needed for setjmp.
+        ld      r3,32(r1)
+        ld      r0,40(r1)
+        // Emulate the real setjmp function.  We do this because we can't
+        // perform a sibcall:  The real setjmp function trashes the TOC
+        // pointer, and with a sibcall we have no way to restore it.
+        // This way we can make sure our caller's stack pointer and
+        // link register are saved correctly in the jmpbuf.
+        ld      r6,-28696(r13)
+        addi    r5,r1,48  // original stack ptr of caller
+        xor     r5,r6,r5
+        std     r5,0(r3)  // mangled stack ptr of caller
+        ld      r5,24(r1)
+        std     r5,8(r3)  // caller's saved TOC pointer
+        xor     r0,r6,r0
+        std     r0,16(r3) // caller's mangled return address
+        mfcr    r0
+        // Nonvolatiles.
+        std     r14,24(r3)
+        stfd    f14,176(r3)
+        stw     r0,172(r3) // CR
+        std     r15,32(r3)
+        stfd    f15,184(r3)
+        std     r16,40(r3)
+        stfd    f16,192(r3)
+        std     r17,48(r3)
+        stfd    f17,200(r3)
+        std     r18,56(r3)
+        stfd    f18,208(r3)
+        std     r19,64(r3)
+        stfd    f19,216(r3)
+        std     r20,72(r3)
+        stfd    f20,224(r3)
+        std     r21,80(r3)
+        stfd    f21,232(r3)
+        std     r22,88(r3)
+        stfd    f22,240(r3)
+        std     r23,96(r3)
+        stfd    f23,248(r3)
+        std     r24,104(r3)
+        stfd    f24,256(r3)
+        std     r25,112(r3)
+        stfd    f25,264(r3)
+        std     r26,120(r3)
+        stfd    f26,272(r3)
+        std     r27,128(r3)
+        stfd    f27,280(r3)
+        std     r28,136(r3)
+        stfd    f28,288(r3)
+        std     r29,144(r3)
+        stfd    f29,296(r3)
+        std     r30,152(r3)
+        stfd    f30,304(r3)
+        std     r31,160(r3)
+        stfd    f31,312(r3)
+        addi    r5,r3,320
+        mfspr   r0,256
+        stw     r0,168(r3)  // VRSAVE
+        addi    r6,r5,16
+        stvx    v20,0,r5
+        addi    r5,r5,32
+        stvx    v21,0,r6
+        addi    r6,r6,32
+        stvx    v22,0,r5
+        addi    r5,r5,32
+        stvx    v23,0,r6
+        addi    r6,r6,32
+        stvx    v24,0,r5
+        addi    r5,r5,32
+        stvx    v25,0,r6
+        addi    r6,r6,32
+        stvx    v26,0,r5
+        addi    r5,r5,32
+        stvx    v27,0,r6
+        addi    r6,r6,32
+        stvx    v28,0,r5
+        addi    r5,r5,32
+        stvx    v29,0,r6
+        addi    r6,r6,32
+        stvx    v30,0,r5
+        stvx    v31,0,r6
+        // Clear the "mask-saved" slot.
+        li      r4,0
+        stw     r4,512(r3)
+        // Restore TOC, LR, and stack and return to caller.
+        ld      r2,24(r1)
+        ld      r0,40(r1)
+        addi    r1,r1,48
+        li      r3,0  // This is the setjmp return path
+        mtlr    r0
+        blr
+        .size _setjmp, .-.L._setjmp
+
+        .globl setjmp
+        .type setjmp, @function
+        .align 4
+setjmp:
+        b       _setjmp
+        .size setjmp, .-setjmp
+
+        // sigsetjmp is like setjmp, except that the mask in r4 needs
+        // to be saved at offset 512 of the jump buffer.
+        .globl __sigsetjmp
+        .type __sigsetjmp, @function
+        .align 4
+#if _CALL_ELF == 2
+__sigsetjmp:
+#else
+	.section ".opd","aw"
+	.align 3
+__sigsetjmp:
+	.quad   .L.__sigsetjmp,.TOC.@tocbase,0
+	.previous
+#endif
+.L.__sigsetjmp:
+        mflr    r0
+        stdu    r1,-64(r1)
+        std     r2,24(r1)
+        std     r3,32(r1)
+        std     r4,40(r1)
+        std     r0,48(r1)
+        // r3 is the original stack pointer.
+        addi    r3,r1,64
+        // r4 is the mangled stack pointer (see glibc)
+        ld      r4,-28696(r13)
+        xor     r4,r3,r4
+        // Materialize a TOC in case we were called from libc.
+        // For big-endian, we load the TOC from the OPD.  For little-
+        // endian, we use the .TOC. symbol to find it.
+        nop
+        bcl     20,31,1f
+1:
+        mflr    r2
+#if _CALL_ELF == 2
+        addis   r2,r2,.TOC.-1b@ha
+        addi    r2,r2,.TOC.-1b@l
+#else
+        addis   r2,r2,_setjmp-1b@ha
+        addi    r2,r2,_setjmp-1b@l
+        ld      r2,8(r2)
+#endif
+        // Call the interceptor.
+        bl      __tsan_setjmp
+        nop
+        // Restore regs needed for __sigsetjmp.
+        ld      r3,32(r1)
+        ld      r4,40(r1)
+        ld      r0,48(r1)
+        // Emulate the real sigsetjmp function.  We do this because we can't
+        // perform a sibcall:  The real sigsetjmp function trashes the TOC
+        // pointer, and with a sibcall we have no way to restore it.
+        // This way we can make sure our caller's stack pointer and
+        // link register are saved correctly in the jmpbuf.
+        ld      r6,-28696(r13)
+        addi    r5,r1,64  // original stack ptr of caller
+        xor     r5,r6,r5
+        std     r5,0(r3)  // mangled stack ptr of caller
+        ld      r5,24(r1)
+        std     r5,8(r3)  // caller's saved TOC pointer
+        xor     r0,r6,r0
+        std     r0,16(r3) // caller's mangled return address
+        mfcr    r0
+        // Nonvolatiles.
+        std     r14,24(r3)
+        stfd    f14,176(r3)
+        stw     r0,172(r3) // CR
+        std     r15,32(r3)
+        stfd    f15,184(r3)
+        std     r16,40(r3)
+        stfd    f16,192(r3)
+        std     r17,48(r3)
+        stfd    f17,200(r3)
+        std     r18,56(r3)
+        stfd    f18,208(r3)
+        std     r19,64(r3)
+        stfd    f19,216(r3)
+        std     r20,72(r3)
+        stfd    f20,224(r3)
+        std     r21,80(r3)
+        stfd    f21,232(r3)
+        std     r22,88(r3)
+        stfd    f22,240(r3)
+        std     r23,96(r3)
+        stfd    f23,248(r3)
+        std     r24,104(r3)
+        stfd    f24,256(r3)
+        std     r25,112(r3)
+        stfd    f25,264(r3)
+        std     r26,120(r3)
+        stfd    f26,272(r3)
+        std     r27,128(r3)
+        stfd    f27,280(r3)
+        std     r28,136(r3)
+        stfd    f28,288(r3)
+        std     r29,144(r3)
+        stfd    f29,296(r3)
+        std     r30,152(r3)
+        stfd    f30,304(r3)
+        std     r31,160(r3)
+        stfd    f31,312(r3)
+        addi    r5,r3,320
+        mfspr   r0,256
+        stw     r0,168(r3) // VRSAVE
+        addi    r6,r5,16
+        stvx    v20,0,r5
+        addi    r5,r5,32
+        stvx    v21,0,r6
+        addi    r6,r6,32
+        stvx    v22,0,r5
+        addi    r5,r5,32
+        stvx    v23,0,r6
+        addi    r6,r6,32
+        stvx    v24,0,r5
+        addi    r5,r5,32
+        stvx    v25,0,r6
+        addi    r6,r6,32
+        stvx    v26,0,r5
+        addi    r5,r5,32
+        stvx    v27,0,r6
+        addi    r6,r6,32
+        stvx    v28,0,r5
+        addi    r5,r5,32
+        stvx    v29,0,r6
+        addi    r6,r6,32
+        stvx    v30,0,r5
+        stvx    v31,0,r6
+        // Save into the "mask-saved" slot.
+        stw     r4,512(r3)
+        // Restore TOC, LR, and stack and return to caller.
+        ld      r2,24(r1)
+        ld      r0,48(r1)
+        addi    r1,r1,64
+        li      r3,0  // This is the sigsetjmp return path
+        mtlr    r0
+        blr
+        .size __sigsetjmp, .-.L.__sigsetjmp
+
+        .globl sigsetjmp
+        .type sigsetjmp, @function
+        .align 4
+sigsetjmp:
+        b       __sigsetjmp
+        .size sigsetjmp, .-sigsetjmp
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_rtl_proc.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_proc.cpp
new file mode 100644
index 000000000000..def61cca14d5
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_proc.cpp
@@ -0,0 +1,60 @@
+//===-- tsan_rtl_proc.cpp -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_placement_new.h"
+#include "tsan_rtl.h"
+#include "tsan_mman.h"
+#include "tsan_flags.h"
+
+namespace __tsan {
+
+Processor *ProcCreate() {
+  void *mem = InternalAlloc(sizeof(Processor));
+  internal_memset(mem, 0, sizeof(Processor));
+  Processor *proc = new(mem) Processor;
+  proc->thr = nullptr;
+#if !SANITIZER_GO
+  AllocatorProcStart(proc);
+#endif
+  if (common_flags()->detect_deadlocks)
+    proc->dd_pt = ctx->dd->CreatePhysicalThread();
+  return proc;
+}
+
+void ProcDestroy(Processor *proc) {
+  CHECK_EQ(proc->thr, nullptr);
+#if !SANITIZER_GO
+  AllocatorProcFinish(proc);
+#endif
+  ctx->clock_alloc.FlushCache(&proc->clock_cache);
+  ctx->metamap.OnProcIdle(proc);
+  if (common_flags()->detect_deadlocks)
+     ctx->dd->DestroyPhysicalThread(proc->dd_pt);
+  proc->~Processor();
+  InternalFree(proc);
+}
+
+void ProcWire(Processor *proc, ThreadState *thr) {
+  CHECK_EQ(thr->proc1, nullptr);
+  CHECK_EQ(proc->thr, nullptr);
+  thr->proc1 = proc;
+  proc->thr = thr;
+}
+
+void ProcUnwire(Processor *proc, ThreadState *thr) {
+  CHECK_EQ(thr->proc1, proc);
+  CHECK_EQ(proc->thr, thr);
+  thr->proc1 = nullptr;
+  proc->thr = nullptr;
+}
+
+}  // namespace __tsan
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_rtl_report.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_report.cpp
new file mode 100644
index 000000000000..f332a6a8d1d8
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_report.cpp
@@ -0,0 +1,984 @@
+//===-- tsan_rtl_report.cpp -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_libc.h"
+#include "sanitizer_common/sanitizer_placement_new.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_stacktrace.h"
+#include "tsan_platform.h"
+#include "tsan_rtl.h"
+#include "tsan_suppressions.h"
+#include "tsan_symbolize.h"
+#include "tsan_report.h"
+#include "tsan_sync.h"
+#include "tsan_mman.h"
+#include "tsan_flags.h"
+#include "tsan_fd.h"
+
+namespace __tsan {
+
+using namespace __sanitizer;
+
+static ReportStack *SymbolizeStack(StackTrace trace);
+
+// Can be overriden by an application/test to intercept reports.
+#ifdef TSAN_EXTERNAL_HOOKS
+bool OnReport(const ReportDesc *rep, bool suppressed);
+#else
+SANITIZER_WEAK_CXX_DEFAULT_IMPL
+bool OnReport(const ReportDesc *rep, bool suppressed) {
+  (void)rep;
+  return suppressed;
+}
+#endif
+
+SANITIZER_WEAK_DEFAULT_IMPL
+void __tsan_on_report(const ReportDesc *rep) {
+  (void)rep;
+}
+
+static void StackStripMain(SymbolizedStack *frames) {
+  SymbolizedStack *last_frame = nullptr;
+  SymbolizedStack *last_frame2 = nullptr;
+  for (SymbolizedStack *cur = frames; cur; cur = cur->next) {
+    last_frame2 = last_frame;
+    last_frame = cur;
+  }
+
+  if (last_frame2 == 0)
+    return;
+#if !SANITIZER_GO
+  const char *last = last_frame->info.function;
+  const char *last2 = last_frame2->info.function;
+  // Strip frame above 'main'
+  if (last2 && 0 == internal_strcmp(last2, "main")) {
+    last_frame->ClearAll();
+    last_frame2->next = nullptr;
+  // Strip our internal thread start routine.
+  } else if (last && 0 == internal_strcmp(last, "__tsan_thread_start_func")) {
+    last_frame->ClearAll();
+    last_frame2->next = nullptr;
+    // Strip global ctors init, .preinit_array and main caller.
+  } else if (last && (0 == internal_strcmp(last, "__do_global_ctors_aux") ||
+                      0 == internal_strcmp(last, "__libc_csu_init") ||
+                      0 == internal_strcmp(last, "__libc_start_main"))) {
+    last_frame->ClearAll();
+    last_frame2->next = nullptr;
+  // If both are 0, then we probably just failed to symbolize.
+  } else if (last || last2) {
+    // Ensure that we recovered stack completely. Trimmed stack
+    // can actually happen if we do not instrument some code,
+    // so it's only a debug print. However we must try hard to not miss it
+    // due to our fault.
+    DPrintf("Bottom stack frame is missed\n");
+  }
+#else
+  // The last frame always point into runtime (gosched0, goexit0, runtime.main).
+  last_frame->ClearAll();
+  last_frame2->next = nullptr;
+#endif
+}
+
+ReportStack *SymbolizeStackId(u32 stack_id) {
+  if (stack_id == 0)
+    return 0;
+  StackTrace stack = StackDepotGet(stack_id);
+  if (stack.trace == nullptr)
+    return nullptr;
+  return SymbolizeStack(stack);
+}
+
+static ReportStack *SymbolizeStack(StackTrace trace) {
+  if (trace.size == 0)
+    return 0;
+  SymbolizedStack *top = nullptr;
+  for (uptr si = 0; si < trace.size; si++) {
+    const uptr pc = trace.trace[si];
+    uptr pc1 = pc;
+    // We obtain the return address, but we're interested in the previous
+    // instruction.
+    if ((pc & kExternalPCBit) == 0)
+      pc1 = StackTrace::GetPreviousInstructionPc(pc);
+    SymbolizedStack *ent = SymbolizeCode(pc1);
+    CHECK_NE(ent, 0);
+    SymbolizedStack *last = ent;
+    while (last->next) {
+      last->info.address = pc;  // restore original pc for report
+      last = last->next;
+    }
+    last->info.address = pc;  // restore original pc for report
+    last->next = top;
+    top = ent;
+  }
+  StackStripMain(top);
+
+  auto *stack = New<ReportStack>();
+  stack->frames = top;
+  return stack;
+}
+
+bool ShouldReport(ThreadState *thr, ReportType typ) {
+  // We set thr->suppress_reports in the fork context.
+  // Taking any locking in the fork context can lead to deadlocks.
+  // If any locks are already taken, it's too late to do this check.
+  CheckedMutex::CheckNoLocks();
+  // For the same reason check we didn't lock thread_registry yet.
+  if (SANITIZER_DEBUG)
+    ThreadRegistryLock l(&ctx->thread_registry);
+  if (!flags()->report_bugs || thr->suppress_reports)
+    return false;
+  switch (typ) {
+    case ReportTypeSignalUnsafe:
+      return flags()->report_signal_unsafe;
+    case ReportTypeThreadLeak:
+#if !SANITIZER_GO
+      // It's impossible to join phantom threads
+      // in the child after fork.
+      if (ctx->after_multithreaded_fork)
+        return false;
+#endif
+      return flags()->report_thread_leaks;
+    case ReportTypeMutexDestroyLocked:
+      return flags()->report_destroy_locked;
+    default:
+      return true;
+  }
+}
+
+ScopedReportBase::ScopedReportBase(ReportType typ, uptr tag) {
+  ctx->thread_registry.CheckLocked();
+  rep_ = New<ReportDesc>();
+  rep_->typ = typ;
+  rep_->tag = tag;
+  ctx->report_mtx.Lock();
+}
+
+ScopedReportBase::~ScopedReportBase() {
+  ctx->report_mtx.Unlock();
+  DestroyAndFree(rep_);
+}
+
+void ScopedReportBase::AddStack(StackTrace stack, bool suppressable) {
+  ReportStack **rs = rep_->stacks.PushBack();
+  *rs = SymbolizeStack(stack);
+  (*rs)->suppressable = suppressable;
+}
+
+void ScopedReportBase::AddMemoryAccess(uptr addr, uptr external_tag, Shadow s,
+                                       StackTrace stack, const MutexSet *mset) {
+  auto *mop = New<ReportMop>();
+  rep_->mops.PushBack(mop);
+  mop->tid = s.tid();
+  mop->addr = addr + s.addr0();
+  mop->size = s.size();
+  mop->write = s.IsWrite();
+  mop->atomic = s.IsAtomic();
+  mop->stack = SymbolizeStack(stack);
+  mop->external_tag = external_tag;
+  if (mop->stack)
+    mop->stack->suppressable = true;
+  for (uptr i = 0; i < mset->Size(); i++) {
+    MutexSet::Desc d = mset->Get(i);
+    u64 mid = this->AddMutex(d.id);
+    ReportMopMutex mtx = {mid, d.write};
+    mop->mset.PushBack(mtx);
+  }
+}
+
+void ScopedReportBase::AddUniqueTid(Tid unique_tid) {
+  rep_->unique_tids.PushBack(unique_tid);
+}
+
+void ScopedReportBase::AddThread(const ThreadContext *tctx, bool suppressable) {
+  for (uptr i = 0; i < rep_->threads.Size(); i++) {
+    if ((u32)rep_->threads[i]->id == tctx->tid)
+      return;
+  }
+  auto *rt = New<ReportThread>();
+  rep_->threads.PushBack(rt);
+  rt->id = tctx->tid;
+  rt->os_id = tctx->os_id;
+  rt->running = (tctx->status == ThreadStatusRunning);
+  rt->name = internal_strdup(tctx->name);
+  rt->parent_tid = tctx->parent_tid;
+  rt->thread_type = tctx->thread_type;
+  rt->stack = 0;
+  rt->stack = SymbolizeStackId(tctx->creation_stack_id);
+  if (rt->stack)
+    rt->stack->suppressable = suppressable;
+}
+
+#if !SANITIZER_GO
+static bool FindThreadByUidLockedCallback(ThreadContextBase *tctx, void *arg) {
+  int unique_id = *(int *)arg;
+  return tctx->unique_id == (u32)unique_id;
+}
+
+static ThreadContext *FindThreadByUidLocked(Tid unique_id) {
+  ctx->thread_registry.CheckLocked();
+  return static_cast<ThreadContext *>(
+      ctx->thread_registry.FindThreadContextLocked(
+          FindThreadByUidLockedCallback, &unique_id));
+}
+
+static ThreadContext *FindThreadByTidLocked(Tid tid) {
+  ctx->thread_registry.CheckLocked();
+  return static_cast<ThreadContext *>(
+      ctx->thread_registry.GetThreadLocked(tid));
+}
+
+static bool IsInStackOrTls(ThreadContextBase *tctx_base, void *arg) {
+  uptr addr = (uptr)arg;
+  ThreadContext *tctx = static_cast<ThreadContext*>(tctx_base);
+  if (tctx->status != ThreadStatusRunning)
+    return false;
+  ThreadState *thr = tctx->thr;
+  CHECK(thr);
+  return ((addr >= thr->stk_addr && addr < thr->stk_addr + thr->stk_size) ||
+          (addr >= thr->tls_addr && addr < thr->tls_addr + thr->tls_size));
+}
+
+ThreadContext *IsThreadStackOrTls(uptr addr, bool *is_stack) {
+  ctx->thread_registry.CheckLocked();
+  ThreadContext *tctx =
+      static_cast<ThreadContext *>(ctx->thread_registry.FindThreadContextLocked(
+          IsInStackOrTls, (void *)addr));
+  if (!tctx)
+    return 0;
+  ThreadState *thr = tctx->thr;
+  CHECK(thr);
+  *is_stack = (addr >= thr->stk_addr && addr < thr->stk_addr + thr->stk_size);
+  return tctx;
+}
+#endif
+
+void ScopedReportBase::AddThread(Tid unique_tid, bool suppressable) {
+#if !SANITIZER_GO
+  if (const ThreadContext *tctx = FindThreadByUidLocked(unique_tid))
+    AddThread(tctx, suppressable);
+#endif
+}
+
+void ScopedReportBase::AddMutex(const SyncVar *s) {
+  for (uptr i = 0; i < rep_->mutexes.Size(); i++) {
+    if (rep_->mutexes[i]->id == s->uid)
+      return;
+  }
+  auto *rm = New<ReportMutex>();
+  rep_->mutexes.PushBack(rm);
+  rm->id = s->uid;
+  rm->addr = s->addr;
+  rm->destroyed = false;
+  rm->stack = SymbolizeStackId(s->creation_stack_id);
+}
+
+u64 ScopedReportBase::AddMutex(u64 id) {
+  u64 uid = 0;
+  u64 mid = id;
+  uptr addr = SyncVar::SplitId(id, &uid);
+  SyncVar *s = ctx->metamap.GetSyncIfExists(addr);
+  // Check that the mutex is still alive.
+  // Another mutex can be created at the same address,
+  // so check uid as well.
+  if (s && s->CheckId(uid)) {
+    Lock l(&s->mtx);
+    mid = s->uid;
+    AddMutex(s);
+  } else {
+    AddDeadMutex(id);
+  }
+  return mid;
+}
+
+void ScopedReportBase::AddDeadMutex(u64 id) {
+  for (uptr i = 0; i < rep_->mutexes.Size(); i++) {
+    if (rep_->mutexes[i]->id == id)
+      return;
+  }
+  auto *rm = New<ReportMutex>();
+  rep_->mutexes.PushBack(rm);
+  rm->id = id;
+  rm->addr = 0;
+  rm->destroyed = true;
+  rm->stack = 0;
+}
+
+void ScopedReportBase::AddLocation(uptr addr, uptr size) {
+  if (addr == 0)
+    return;
+#if !SANITIZER_GO
+  int fd = -1;
+  Tid creat_tid = kInvalidTid;
+  StackID creat_stack = 0;
+  if (FdLocation(addr, &fd, &creat_tid, &creat_stack)) {
+    auto *loc = New<ReportLocation>();
+    loc->type = ReportLocationFD;
+    loc->fd = fd;
+    loc->tid = creat_tid;
+    loc->stack = SymbolizeStackId(creat_stack);
+    rep_->locs.PushBack(loc);
+    ThreadContext *tctx = FindThreadByUidLocked(creat_tid);
+    if (tctx)
+      AddThread(tctx);
+    return;
+  }
+  MBlock *b = 0;
+  uptr block_begin = 0;
+  Allocator *a = allocator();
+  if (a->PointerIsMine((void*)addr)) {
+    block_begin = (uptr)a->GetBlockBegin((void *)addr);
+    if (block_begin)
+      b = ctx->metamap.GetBlock(block_begin);
+  }
+  if (!b)
+    b = JavaHeapBlock(addr, &block_begin);
+  if (b != 0) {
+    ThreadContext *tctx = FindThreadByTidLocked(b->tid);
+    auto *loc = New<ReportLocation>();
+    loc->type = ReportLocationHeap;
+    loc->heap_chunk_start = block_begin;
+    loc->heap_chunk_size = b->siz;
+    loc->external_tag = b->tag;
+    loc->tid = tctx ? tctx->tid : b->tid;
+    loc->stack = SymbolizeStackId(b->stk);
+    rep_->locs.PushBack(loc);
+    if (tctx)
+      AddThread(tctx);
+    return;
+  }
+  bool is_stack = false;
+  if (ThreadContext *tctx = IsThreadStackOrTls(addr, &is_stack)) {
+    auto *loc = New<ReportLocation>();
+    loc->type = is_stack ? ReportLocationStack : ReportLocationTLS;
+    loc->tid = tctx->tid;
+    rep_->locs.PushBack(loc);
+    AddThread(tctx);
+  }
+#endif
+  if (ReportLocation *loc = SymbolizeData(addr)) {
+    loc->suppressable = true;
+    rep_->locs.PushBack(loc);
+    return;
+  }
+}
+
+#if !SANITIZER_GO
+void ScopedReportBase::AddSleep(StackID stack_id) {
+  rep_->sleep = SymbolizeStackId(stack_id);
+}
+#endif
+
+void ScopedReportBase::SetCount(int count) { rep_->count = count; }
+
+const ReportDesc *ScopedReportBase::GetReport() const { return rep_; }
+
+ScopedReport::ScopedReport(ReportType typ, uptr tag)
+    : ScopedReportBase(typ, tag) {}
+
+ScopedReport::~ScopedReport() {}
+
+void RestoreStack(Tid tid, const u64 epoch, VarSizeStackTrace *stk,
+                  MutexSet *mset, uptr *tag) {
+  // This function restores stack trace and mutex set for the thread/epoch.
+  // It does so by getting stack trace and mutex set at the beginning of
+  // trace part, and then replaying the trace till the given epoch.
+  Trace* trace = ThreadTrace(tid);
+  ReadLock l(&trace->mtx);
+  const int partidx = (epoch / kTracePartSize) % TraceParts();
+  TraceHeader* hdr = &trace->headers[partidx];
+  if (epoch < hdr->epoch0 || epoch >= hdr->epoch0 + kTracePartSize)
+    return;
+  CHECK_EQ(RoundDown(epoch, kTracePartSize), hdr->epoch0);
+  const u64 epoch0 = RoundDown(epoch, TraceSize());
+  const u64 eend = epoch % TraceSize();
+  const u64 ebegin = RoundDown(eend, kTracePartSize);
+  DPrintf("#%d: RestoreStack epoch=%zu ebegin=%zu eend=%zu partidx=%d\n",
+          tid, (uptr)epoch, (uptr)ebegin, (uptr)eend, partidx);
+  Vector<uptr> stack;
+  stack.Resize(hdr->stack0.size + 64);
+  for (uptr i = 0; i < hdr->stack0.size; i++) {
+    stack[i] = hdr->stack0.trace[i];
+    DPrintf2("  #%02zu: pc=%zx\n", i, stack[i]);
+  }
+  if (mset)
+    *mset = hdr->mset0;
+  uptr pos = hdr->stack0.size;
+  Event *events = (Event*)GetThreadTrace(tid);
+  for (uptr i = ebegin; i <= eend; i++) {
+    Event ev = events[i];
+    EventType typ = (EventType)(ev >> kEventPCBits);
+    uptr pc = (uptr)(ev & ((1ull << kEventPCBits) - 1));
+    DPrintf2("  %zu typ=%d pc=%zx\n", i, typ, pc);
+    if (typ == EventTypeMop) {
+      stack[pos] = pc;
+    } else if (typ == EventTypeFuncEnter) {
+      if (stack.Size() < pos + 2)
+        stack.Resize(pos + 2);
+      stack[pos++] = pc;
+    } else if (typ == EventTypeFuncExit) {
+      if (pos > 0)
+        pos--;
+    }
+    if (mset) {
+      if (typ == EventTypeLock) {
+        mset->Add(pc, true, epoch0 + i);
+      } else if (typ == EventTypeUnlock) {
+        mset->Del(pc, true);
+      } else if (typ == EventTypeRLock) {
+        mset->Add(pc, false, epoch0 + i);
+      } else if (typ == EventTypeRUnlock) {
+        mset->Del(pc, false);
+      }
+    }
+    for (uptr j = 0; j <= pos; j++)
+      DPrintf2("      #%zu: %zx\n", j, stack[j]);
+  }
+  if (pos == 0 && stack[0] == 0)
+    return;
+  pos++;
+  stk->Init(&stack[0], pos);
+  ExtractTagFromStack(stk, tag);
+}
+
+namespace v3 {
+
+// Replays the trace up to last_pos position in the last part
+// or up to the provided epoch/sid (whichever is earlier)
+// and calls the provided function f for each event.
+template <typename Func>
+void TraceReplay(Trace *trace, TracePart *last, Event *last_pos, Sid sid,
+                 Epoch epoch, Func f) {
+  TracePart *part = trace->parts.Front();
+  Sid ev_sid = kFreeSid;
+  Epoch ev_epoch = kEpochOver;
+  for (;;) {
+    DCHECK_EQ(part->trace, trace);
+    // Note: an event can't start in the last element.
+    // Since an event can take up to 2 elements,
+    // we ensure we have at least 2 before adding an event.
+    Event *end = &part->events[TracePart::kSize - 1];
+    if (part == last)
+      end = last_pos;
+    for (Event *evp = &part->events[0]; evp < end; evp++) {
+      Event *evp0 = evp;
+      if (!evp->is_access && !evp->is_func) {
+        switch (evp->type) {
+          case EventType::kTime: {
+            auto *ev = reinterpret_cast<EventTime *>(evp);
+            ev_sid = static_cast<Sid>(ev->sid);
+            ev_epoch = static_cast<Epoch>(ev->epoch);
+            if (ev_sid == sid && ev_epoch > epoch)
+              return;
+            break;
+          }
+          case EventType::kAccessExt:
+            FALLTHROUGH;
+          case EventType::kAccessRange:
+            FALLTHROUGH;
+          case EventType::kLock:
+            FALLTHROUGH;
+          case EventType::kRLock:
+            // These take 2 Event elements.
+            evp++;
+            break;
+          case EventType::kUnlock:
+            // This takes 1 Event element.
+            break;
+        }
+      }
+      CHECK_NE(ev_sid, kFreeSid);
+      CHECK_NE(ev_epoch, kEpochOver);
+      f(ev_sid, ev_epoch, evp0);
+    }
+    if (part == last)
+      return;
+    part = trace->parts.Next(part);
+    CHECK(part);
+  }
+  CHECK(0);
+}
+
+static void RestoreStackMatch(VarSizeStackTrace *pstk, MutexSet *pmset,
+                              Vector<uptr> *stack, MutexSet *mset, uptr pc,
+                              bool *found) {
+  DPrintf2("    MATCHED\n");
+  *pmset = *mset;
+  stack->PushBack(pc);
+  pstk->Init(&(*stack)[0], stack->Size());
+  stack->PopBack();
+  *found = true;
+}
+
+// Checks if addr1|size1 is fully contained in addr2|size2.
+// We check for fully contained instread of just overlapping
+// because a memory access is always traced once, but can be
+// split into multiple accesses in the shadow.
+static constexpr bool IsWithinAccess(uptr addr1, uptr size1, uptr addr2,
+                                     uptr size2) {
+  return addr1 >= addr2 && addr1 + size1 <= addr2 + size2;
+}
+
+// Replays the trace of thread tid up to the target event identified
+// by sid/epoch/addr/size/typ and restores and returns stack, mutex set
+// and tag for that event. If there are multiple such events, it returns
+// the last one. Returns false if the event is not present in the trace.
+bool RestoreStack(Tid tid, EventType type, Sid sid, Epoch epoch, uptr addr,
+                  uptr size, AccessType typ, VarSizeStackTrace *pstk,
+                  MutexSet *pmset, uptr *ptag) {
+  // This function restores stack trace and mutex set for the thread/epoch.
+  // It does so by getting stack trace and mutex set at the beginning of
+  // trace part, and then replaying the trace till the given epoch.
+  DPrintf2("RestoreStack: tid=%u sid=%u@%u addr=0x%zx/%zu typ=%x\n", tid,
+           static_cast<int>(sid), static_cast<int>(epoch), addr, size,
+           static_cast<int>(typ));
+  ctx->slot_mtx.CheckLocked();  // needed to prevent trace part recycling
+  ctx->thread_registry.CheckLocked();
+  ThreadContext *tctx =
+      static_cast<ThreadContext *>(ctx->thread_registry.GetThreadLocked(tid));
+  Trace *trace = &tctx->trace;
+  // Snapshot first/last parts and the current position in the last part.
+  TracePart *first_part;
+  TracePart *last_part;
+  Event *last_pos;
+  {
+    Lock lock(&trace->mtx);
+    first_part = trace->parts.Front();
+    if (!first_part)
+      return false;
+    last_part = trace->parts.Back();
+    last_pos = trace->final_pos;
+    if (tctx->thr)
+      last_pos = (Event *)atomic_load_relaxed(&tctx->thr->trace_pos);
+  }
+  DynamicMutexSet mset;
+  Vector<uptr> stack;
+  uptr prev_pc = 0;
+  bool found = false;
+  bool is_read = typ & kAccessRead;
+  bool is_atomic = typ & kAccessAtomic;
+  bool is_free = typ & kAccessFree;
+  TraceReplay(
+      trace, last_part, last_pos, sid, epoch,
+      [&](Sid ev_sid, Epoch ev_epoch, Event *evp) {
+        bool match = ev_sid == sid && ev_epoch == epoch;
+        if (evp->is_access) {
+          if (evp->is_func == 0 && evp->type == EventType::kAccessExt &&
+              evp->_ == 0)  // NopEvent
+            return;
+          auto *ev = reinterpret_cast<EventAccess *>(evp);
+          uptr ev_addr = RestoreAddr(ev->addr);
+          uptr ev_size = 1 << ev->size_log;
+          uptr ev_pc =
+              prev_pc + ev->pc_delta - (1 << (EventAccess::kPCBits - 1));
+          prev_pc = ev_pc;
+          DPrintf2("  Access: pc=0x%zx addr=0x%zx/%zu type=%u/%u\n", ev_pc,
+                   ev_addr, ev_size, ev->is_read, ev->is_atomic);
+          if (match && type == EventType::kAccessExt &&
+              IsWithinAccess(addr, size, ev_addr, ev_size) &&
+              is_read == ev->is_read && is_atomic == ev->is_atomic && !is_free)
+            RestoreStackMatch(pstk, pmset, &stack, mset, ev_pc, &found);
+          return;
+        }
+        if (evp->is_func) {
+          auto *ev = reinterpret_cast<EventFunc *>(evp);
+          if (ev->pc) {
+            DPrintf2("  FuncEnter: pc=0x%llx\n", ev->pc);
+            stack.PushBack(ev->pc);
+          } else {
+            DPrintf2("  FuncExit\n");
+            CHECK(stack.Size());
+            stack.PopBack();
+          }
+          return;
+        }
+        switch (evp->type) {
+          case EventType::kAccessExt: {
+            auto *ev = reinterpret_cast<EventAccessExt *>(evp);
+            uptr ev_addr = RestoreAddr(ev->addr);
+            uptr ev_size = 1 << ev->size_log;
+            prev_pc = ev->pc;
+            DPrintf2("  AccessExt: pc=0x%llx addr=0x%zx/%zu type=%u/%u\n",
+                     ev->pc, ev_addr, ev_size, ev->is_read, ev->is_atomic);
+            if (match && type == EventType::kAccessExt &&
+                IsWithinAccess(addr, size, ev_addr, ev_size) &&
+                is_read == ev->is_read && is_atomic == ev->is_atomic &&
+                !is_free)
+              RestoreStackMatch(pstk, pmset, &stack, mset, ev->pc, &found);
+            break;
+          }
+          case EventType::kAccessRange: {
+            auto *ev = reinterpret_cast<EventAccessRange *>(evp);
+            uptr ev_addr = RestoreAddr(ev->addr);
+            uptr ev_size =
+                (ev->size_hi << EventAccessRange::kSizeLoBits) + ev->size_lo;
+            uptr ev_pc = RestoreAddr(ev->pc);
+            prev_pc = ev_pc;
+            DPrintf2("  Range: pc=0x%zx addr=0x%zx/%zu type=%u/%u\n", ev_pc,
+                     ev_addr, ev_size, ev->is_read, ev->is_free);
+            if (match && type == EventType::kAccessExt &&
+                IsWithinAccess(addr, size, ev_addr, ev_size) &&
+                is_read == ev->is_read && !is_atomic && is_free == ev->is_free)
+              RestoreStackMatch(pstk, pmset, &stack, mset, ev_pc, &found);
+            break;
+          }
+          case EventType::kLock:
+            FALLTHROUGH;
+          case EventType::kRLock: {
+            auto *ev = reinterpret_cast<EventLock *>(evp);
+            bool is_write = ev->type == EventType::kLock;
+            uptr ev_addr = RestoreAddr(ev->addr);
+            uptr ev_pc = RestoreAddr(ev->pc);
+            StackID stack_id =
+                (ev->stack_hi << EventLock::kStackIDLoBits) + ev->stack_lo;
+            DPrintf2("  Lock: pc=0x%zx addr=0x%zx stack=%u write=%d\n", ev_pc,
+                     ev_addr, stack_id, is_write);
+            mset->AddAddr(ev_addr, stack_id, is_write);
+            // Events with ev_pc == 0 are written to the beginning of trace
+            // part as initial mutex set (are not real).
+            if (match && type == EventType::kLock && addr == ev_addr && ev_pc)
+              RestoreStackMatch(pstk, pmset, &stack, mset, ev_pc, &found);
+            break;
+          }
+          case EventType::kUnlock: {
+            auto *ev = reinterpret_cast<EventUnlock *>(evp);
+            uptr ev_addr = RestoreAddr(ev->addr);
+            DPrintf2("  Unlock: addr=0x%zx\n", ev_addr);
+            mset->DelAddr(ev_addr);
+            break;
+          }
+          case EventType::kTime:
+            // TraceReplay already extracted sid/epoch from it,
+            // nothing else to do here.
+            break;
+        }
+      });
+  ExtractTagFromStack(pstk, ptag);
+  return found;
+}
+
+}  // namespace v3
+
+bool RacyStacks::operator==(const RacyStacks &other) const {
+  if (hash[0] == other.hash[0] && hash[1] == other.hash[1])
+    return true;
+  if (hash[0] == other.hash[1] && hash[1] == other.hash[0])
+    return true;
+  return false;
+}
+
+static bool FindRacyStacks(const RacyStacks &hash) {
+  for (uptr i = 0; i < ctx->racy_stacks.Size(); i++) {
+    if (hash == ctx->racy_stacks[i]) {
+      VPrintf(2, "ThreadSanitizer: suppressing report as doubled (stack)\n");
+      return true;
+    }
+  }
+  return false;
+}
+
+static bool HandleRacyStacks(ThreadState *thr, VarSizeStackTrace traces[2]) {
+  if (!flags()->suppress_equal_stacks)
+    return false;
+  RacyStacks hash;
+  hash.hash[0] = md5_hash(traces[0].trace, traces[0].size * sizeof(uptr));
+  hash.hash[1] = md5_hash(traces[1].trace, traces[1].size * sizeof(uptr));
+  {
+    ReadLock lock(&ctx->racy_mtx);
+    if (FindRacyStacks(hash))
+      return true;
+  }
+  Lock lock(&ctx->racy_mtx);
+  if (FindRacyStacks(hash))
+    return true;
+  ctx->racy_stacks.PushBack(hash);
+  return false;
+}
+
+static bool FindRacyAddress(const RacyAddress &ra0) {
+  for (uptr i = 0; i < ctx->racy_addresses.Size(); i++) {
+    RacyAddress ra2 = ctx->racy_addresses[i];
+    uptr maxbeg = max(ra0.addr_min, ra2.addr_min);
+    uptr minend = min(ra0.addr_max, ra2.addr_max);
+    if (maxbeg < minend) {
+      VPrintf(2, "ThreadSanitizer: suppressing report as doubled (addr)\n");
+      return true;
+    }
+  }
+  return false;
+}
+
+static bool HandleRacyAddress(ThreadState *thr, uptr addr_min, uptr addr_max) {
+  if (!flags()->suppress_equal_addresses)
+    return false;
+  RacyAddress ra0 = {addr_min, addr_max};
+  {
+    ReadLock lock(&ctx->racy_mtx);
+    if (FindRacyAddress(ra0))
+      return true;
+  }
+  Lock lock(&ctx->racy_mtx);
+  if (FindRacyAddress(ra0))
+    return true;
+  ctx->racy_addresses.PushBack(ra0);
+  return false;
+}
+
+bool OutputReport(ThreadState *thr, const ScopedReport &srep) {
+  // These should have been checked in ShouldReport.
+  // It's too late to check them here, we have already taken locks.
+  CHECK(flags()->report_bugs);
+  CHECK(!thr->suppress_reports);
+  atomic_store_relaxed(&ctx->last_symbolize_time_ns, NanoTime());
+  const ReportDesc *rep = srep.GetReport();
+  CHECK_EQ(thr->current_report, nullptr);
+  thr->current_report = rep;
+  Suppression *supp = 0;
+  uptr pc_or_addr = 0;
+  for (uptr i = 0; pc_or_addr == 0 && i < rep->mops.Size(); i++)
+    pc_or_addr = IsSuppressed(rep->typ, rep->mops[i]->stack, &supp);
+  for (uptr i = 0; pc_or_addr == 0 && i < rep->stacks.Size(); i++)
+    pc_or_addr = IsSuppressed(rep->typ, rep->stacks[i], &supp);
+  for (uptr i = 0; pc_or_addr == 0 && i < rep->threads.Size(); i++)
+    pc_or_addr = IsSuppressed(rep->typ, rep->threads[i]->stack, &supp);
+  for (uptr i = 0; pc_or_addr == 0 && i < rep->locs.Size(); i++)
+    pc_or_addr = IsSuppressed(rep->typ, rep->locs[i], &supp);
+  if (pc_or_addr != 0) {
+    Lock lock(&ctx->fired_suppressions_mtx);
+    FiredSuppression s = {srep.GetReport()->typ, pc_or_addr, supp};
+    ctx->fired_suppressions.push_back(s);
+  }
+  {
+    bool old_is_freeing = thr->is_freeing;
+    thr->is_freeing = false;
+    bool suppressed = OnReport(rep, pc_or_addr != 0);
+    thr->is_freeing = old_is_freeing;
+    if (suppressed) {
+      thr->current_report = nullptr;
+      return false;
+    }
+  }
+  PrintReport(rep);
+  __tsan_on_report(rep);
+  ctx->nreported++;
+  if (flags()->halt_on_error)
+    Die();
+  thr->current_report = nullptr;
+  return true;
+}
+
+bool IsFiredSuppression(Context *ctx, ReportType type, StackTrace trace) {
+  ReadLock lock(&ctx->fired_suppressions_mtx);
+  for (uptr k = 0; k < ctx->fired_suppressions.size(); k++) {
+    if (ctx->fired_suppressions[k].type != type)
+      continue;
+    for (uptr j = 0; j < trace.size; j++) {
+      FiredSuppression *s = &ctx->fired_suppressions[k];
+      if (trace.trace[j] == s->pc_or_addr) {
+        if (s->supp)
+          atomic_fetch_add(&s->supp->hit_count, 1, memory_order_relaxed);
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
+static bool IsFiredSuppression(Context *ctx, ReportType type, uptr addr) {
+  ReadLock lock(&ctx->fired_suppressions_mtx);
+  for (uptr k = 0; k < ctx->fired_suppressions.size(); k++) {
+    if (ctx->fired_suppressions[k].type != type)
+      continue;
+    FiredSuppression *s = &ctx->fired_suppressions[k];
+    if (addr == s->pc_or_addr) {
+      if (s->supp)
+        atomic_fetch_add(&s->supp->hit_count, 1, memory_order_relaxed);
+      return true;
+    }
+  }
+  return false;
+}
+
+static bool RaceBetweenAtomicAndFree(ThreadState *thr) {
+  Shadow s0(thr->racy_state[0]);
+  Shadow s1(thr->racy_state[1]);
+  CHECK(!(s0.IsAtomic() && s1.IsAtomic()));
+  if (!s0.IsAtomic() && !s1.IsAtomic())
+    return true;
+  if (s0.IsAtomic() && s1.IsFreed())
+    return true;
+  if (s1.IsAtomic() && thr->is_freeing)
+    return true;
+  return false;
+}
+
+void ReportRace(ThreadState *thr) {
+  CheckedMutex::CheckNoLocks();
+
+  // Symbolizer makes lots of intercepted calls. If we try to process them,
+  // at best it will cause deadlocks on internal mutexes.
+  ScopedIgnoreInterceptors ignore;
+
+  if (!ShouldReport(thr, ReportTypeRace))
+    return;
+  if (!flags()->report_atomic_races && !RaceBetweenAtomicAndFree(thr))
+    return;
+
+  bool freed = false;
+  {
+    Shadow s(thr->racy_state[1]);
+    freed = s.GetFreedAndReset();
+    thr->racy_state[1] = s.raw();
+  }
+
+  uptr addr = ShadowToMem(thr->racy_shadow_addr);
+  uptr addr_min = 0;
+  uptr addr_max = 0;
+  {
+    uptr a0 = addr + Shadow(thr->racy_state[0]).addr0();
+    uptr a1 = addr + Shadow(thr->racy_state[1]).addr0();
+    uptr e0 = a0 + Shadow(thr->racy_state[0]).size();
+    uptr e1 = a1 + Shadow(thr->racy_state[1]).size();
+    addr_min = min(a0, a1);
+    addr_max = max(e0, e1);
+    if (IsExpectedReport(addr_min, addr_max - addr_min))
+      return;
+  }
+  if (HandleRacyAddress(thr, addr_min, addr_max))
+    return;
+
+  ReportType typ = ReportTypeRace;
+  if (thr->is_vptr_access && freed)
+    typ = ReportTypeVptrUseAfterFree;
+  else if (thr->is_vptr_access)
+    typ = ReportTypeVptrRace;
+  else if (freed)
+    typ = ReportTypeUseAfterFree;
+
+  if (IsFiredSuppression(ctx, typ, addr))
+    return;
+
+  const uptr kMop = 2;
+  VarSizeStackTrace traces[kMop];
+  uptr tags[kMop] = {kExternalTagNone};
+  uptr toppc = TraceTopPC(thr);
+  if (toppc >> kEventPCBits) {
+    // This is a work-around for a known issue.
+    // The scenario where this happens is rather elaborate and requires
+    // an instrumented __sanitizer_report_error_summary callback and
+    // a __tsan_symbolize_external callback and a race during a range memory
+    // access larger than 8 bytes. MemoryAccessRange adds the current PC to
+    // the trace and starts processing memory accesses. A first memory access
+    // triggers a race, we report it and call the instrumented
+    // __sanitizer_report_error_summary, which adds more stuff to the trace
+    // since it is intrumented. Then a second memory access in MemoryAccessRange
+    // also triggers a race and we get here and call TraceTopPC to get the
+    // current PC, however now it contains some unrelated events from the
+    // callback. Most likely, TraceTopPC will now return a EventTypeFuncExit
+    // event. Later we subtract -1 from it (in GetPreviousInstructionPc)
+    // and the resulting PC has kExternalPCBit set, so we pass it to
+    // __tsan_symbolize_external_ex. __tsan_symbolize_external_ex is within its
+    // rights to crash since the PC is completely bogus.
+    // test/tsan/double_race.cpp contains a test case for this.
+    toppc = 0;
+  }
+  ObtainCurrentStack(thr, toppc, &traces[0], &tags[0]);
+  if (IsFiredSuppression(ctx, typ, traces[0]))
+    return;
+
+  DynamicMutexSet mset2;
+  Shadow s2(thr->racy_state[1]);
+  RestoreStack(s2.tid(), s2.epoch(), &traces[1], mset2, &tags[1]);
+  if (IsFiredSuppression(ctx, typ, traces[1]))
+    return;
+
+  if (HandleRacyStacks(thr, traces))
+    return;
+
+  // If any of the accesses has a tag, treat this as an "external" race.
+  uptr tag = kExternalTagNone;
+  for (uptr i = 0; i < kMop; i++) {
+    if (tags[i] != kExternalTagNone) {
+      typ = ReportTypeExternalRace;
+      tag = tags[i];
+      break;
+    }
+  }
+
+  ThreadRegistryLock l0(&ctx->thread_registry);
+  ScopedReport rep(typ, tag);
+  for (uptr i = 0; i < kMop; i++) {
+    Shadow s(thr->racy_state[i]);
+    rep.AddMemoryAccess(addr, tags[i], s, traces[i],
+                        i == 0 ? &thr->mset : mset2);
+  }
+
+  for (uptr i = 0; i < kMop; i++) {
+    FastState s(thr->racy_state[i]);
+    ThreadContext *tctx = static_cast<ThreadContext *>(
+        ctx->thread_registry.GetThreadLocked(s.tid()));
+    if (s.epoch() < tctx->epoch0 || s.epoch() > tctx->epoch1)
+      continue;
+    rep.AddThread(tctx);
+  }
+
+  rep.AddLocation(addr_min, addr_max - addr_min);
+
+#if !SANITIZER_GO
+  {
+    Shadow s(thr->racy_state[1]);
+    if (s.epoch() <= thr->last_sleep_clock.get(s.tid()))
+      rep.AddSleep(thr->last_sleep_stack_id);
+  }
+#endif
+
+  OutputReport(thr, rep);
+}
+
+void PrintCurrentStack(ThreadState *thr, uptr pc) {
+  VarSizeStackTrace trace;
+  ObtainCurrentStack(thr, pc, &trace);
+  PrintStack(SymbolizeStack(trace));
+}
+
+// Always inlining PrintCurrentStackSlow, because LocatePcInTrace assumes
+// __sanitizer_print_stack_trace exists in the actual unwinded stack, but
+// tail-call to PrintCurrentStackSlow breaks this assumption because
+// __sanitizer_print_stack_trace disappears after tail-call.
+// However, this solution is not reliable enough, please see dvyukov's comment
+// http://reviews.llvm.org/D19148#406208
+// Also see PR27280 comment 2 and 3 for breaking examples and analysis.
+ALWAYS_INLINE USED void PrintCurrentStackSlow(uptr pc) {
+#if !SANITIZER_GO
+  uptr bp = GET_CURRENT_FRAME();
+  auto *ptrace = New<BufferedStackTrace>();
+  ptrace->Unwind(pc, bp, nullptr, false);
+
+  for (uptr i = 0; i < ptrace->size / 2; i++) {
+    uptr tmp = ptrace->trace_buffer[i];
+    ptrace->trace_buffer[i] = ptrace->trace_buffer[ptrace->size - i - 1];
+    ptrace->trace_buffer[ptrace->size - i - 1] = tmp;
+  }
+  PrintStack(SymbolizeStack(*ptrace));
+#endif
+}
+
+}  // namespace __tsan
+
+using namespace __tsan;
+
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_print_stack_trace() {
+  PrintCurrentStackSlow(StackTrace::GetCurrentPc());
+}
+}  // extern "C"
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_rtl_s390x.S b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_s390x.S
new file mode 100644
index 000000000000..fcff35fbc7e0
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_s390x.S
@@ -0,0 +1,47 @@
+#include "sanitizer_common/sanitizer_asm.h"
+
+#define CFA_OFFSET 160
+#define R2_REL_OFFSET 16
+#define R3_REL_OFFSET 24
+#define R14_REL_OFFSET 112
+#define R15_REL_OFFSET 120
+#define FRAME_SIZE 160
+
+.text
+
+ASM_HIDDEN(__tsan_setjmp)
+
+.macro intercept symbol, real
+.comm \real, 8, 8
+.globl ASM_SYMBOL_INTERCEPTOR(\symbol)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(\symbol))
+ASM_SYMBOL_INTERCEPTOR(\symbol):
+  CFI_STARTPROC
+  stmg %r2, %r3, R2_REL_OFFSET(%r15)
+  CFI_REL_OFFSET(%r2, R2_REL_OFFSET)
+  CFI_REL_OFFSET(%r3, R3_REL_OFFSET)
+  stmg %r14, %r15, R14_REL_OFFSET(%r15)
+  CFI_REL_OFFSET(%r14, R14_REL_OFFSET)
+  CFI_REL_OFFSET(%r15, R15_REL_OFFSET)
+  aghi %r15, -FRAME_SIZE
+  CFI_ADJUST_CFA_OFFSET(FRAME_SIZE)
+  la %r2, FRAME_SIZE(%r15)
+  brasl %r14, ASM_SYMBOL(__tsan_setjmp)
+  lmg %r14, %r15, FRAME_SIZE + R14_REL_OFFSET(%r15)
+  CFI_RESTORE(%r14)
+  CFI_RESTORE(%r15)
+  CFI_DEF_CFA_OFFSET(CFA_OFFSET)
+  lmg %r2, %r3, R2_REL_OFFSET(%r15)
+  CFI_RESTORE(%r2)
+  CFI_RESTORE(%r3)
+  larl %r1, \real
+  lg %r1, 0(%r1)
+  br %r1
+  CFI_ENDPROC
+  ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(\symbol))
+.endm
+
+intercept setjmp, _ZN14__interception11real_setjmpE
+intercept _setjmp, _ZN14__interception12real__setjmpE
+intercept sigsetjmp, _ZN14__interception14real_sigsetjmpE
+intercept __sigsetjmp, _ZN14__interception16real___sigsetjmpE
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_rtl_thread.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_thread.cpp
new file mode 100644
index 000000000000..c8f7124c009d
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_rtl_thread.cpp
@@ -0,0 +1,349 @@
+//===-- tsan_rtl_thread.cpp -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_placement_new.h"
+#include "tsan_rtl.h"
+#include "tsan_mman.h"
+#include "tsan_platform.h"
+#include "tsan_report.h"
+#include "tsan_sync.h"
+
+namespace __tsan {
+
+// ThreadContext implementation.
+
+ThreadContext::ThreadContext(Tid tid)
+    : ThreadContextBase(tid), thr(), sync(), epoch0(), epoch1() {}
+
+#if !SANITIZER_GO
+ThreadContext::~ThreadContext() {
+}
+#endif
+
+void ThreadContext::OnReset() {
+  CHECK_EQ(sync.size(), 0);
+  uptr trace_p = GetThreadTrace(tid);
+  ReleaseMemoryPagesToOS(trace_p, trace_p + TraceSize() * sizeof(Event));
+  //!!! ReleaseMemoryToOS(GetThreadTraceHeader(tid), sizeof(Trace));
+}
+
+#if !SANITIZER_GO
+struct ThreadLeak {
+  ThreadContext *tctx;
+  int count;
+};
+
+static void CollectThreadLeaks(ThreadContextBase *tctx_base, void *arg) {
+  auto &leaks = *static_cast<Vector<ThreadLeak> *>(arg);
+  auto *tctx = static_cast<ThreadContext *>(tctx_base);
+  if (tctx->detached || tctx->status != ThreadStatusFinished)
+    return;
+  for (uptr i = 0; i < leaks.Size(); i++) {
+    if (leaks[i].tctx->creation_stack_id == tctx->creation_stack_id) {
+      leaks[i].count++;
+      return;
+    }
+  }
+  leaks.PushBack({tctx, 1});
+}
+#endif
+
+#if !SANITIZER_GO
+static void ReportIgnoresEnabled(ThreadContext *tctx, IgnoreSet *set) {
+  if (tctx->tid == kMainTid) {
+    Printf("ThreadSanitizer: main thread finished with ignores enabled\n");
+  } else {
+    Printf("ThreadSanitizer: thread T%d %s finished with ignores enabled,"
+      " created at:\n", tctx->tid, tctx->name);
+    PrintStack(SymbolizeStackId(tctx->creation_stack_id));
+  }
+  Printf("  One of the following ignores was not ended"
+      " (in order of probability)\n");
+  for (uptr i = 0; i < set->Size(); i++) {
+    Printf("  Ignore was enabled at:\n");
+    PrintStack(SymbolizeStackId(set->At(i)));
+  }
+  Die();
+}
+
+static void ThreadCheckIgnore(ThreadState *thr) {
+  if (ctx->after_multithreaded_fork)
+    return;
+  if (thr->ignore_reads_and_writes)
+    ReportIgnoresEnabled(thr->tctx, &thr->mop_ignore_set);
+  if (thr->ignore_sync)
+    ReportIgnoresEnabled(thr->tctx, &thr->sync_ignore_set);
+}
+#else
+static void ThreadCheckIgnore(ThreadState *thr) {}
+#endif
+
+void ThreadFinalize(ThreadState *thr) {
+  ThreadCheckIgnore(thr);
+#if !SANITIZER_GO
+  if (!ShouldReport(thr, ReportTypeThreadLeak))
+    return;
+  ThreadRegistryLock l(&ctx->thread_registry);
+  Vector<ThreadLeak> leaks;
+  ctx->thread_registry.RunCallbackForEachThreadLocked(CollectThreadLeaks,
+                                                      &leaks);
+  for (uptr i = 0; i < leaks.Size(); i++) {
+    ScopedReport rep(ReportTypeThreadLeak);
+    rep.AddThread(leaks[i].tctx, true);
+    rep.SetCount(leaks[i].count);
+    OutputReport(thr, rep);
+  }
+#endif
+}
+
+int ThreadCount(ThreadState *thr) {
+  uptr result;
+  ctx->thread_registry.GetNumberOfThreads(0, 0, &result);
+  return (int)result;
+}
+
+struct OnCreatedArgs {
+  ThreadState *thr;
+  uptr pc;
+};
+
+Tid ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached) {
+  OnCreatedArgs args = { thr, pc };
+  u32 parent_tid = thr ? thr->tid : kInvalidTid;  // No parent for GCD workers.
+  Tid tid = ctx->thread_registry.CreateThread(uid, detached, parent_tid, &args);
+  DPrintf("#%d: ThreadCreate tid=%d uid=%zu\n", parent_tid, tid, uid);
+  return tid;
+}
+
+void ThreadContext::OnCreated(void *arg) {
+  thr = 0;
+  if (tid == kMainTid)
+    return;
+  OnCreatedArgs *args = static_cast<OnCreatedArgs *>(arg);
+  if (!args->thr)  // GCD workers don't have a parent thread.
+    return;
+  args->thr->fast_state.IncrementEpoch();
+  // Can't increment epoch w/o writing to the trace as well.
+  TraceAddEvent(args->thr, args->thr->fast_state, EventTypeMop, 0);
+  ReleaseImpl(args->thr, 0, &sync);
+  creation_stack_id = CurrentStackId(args->thr, args->pc);
+}
+
+extern "C" void __tsan_stack_initialization() {}
+
+struct OnStartedArgs {
+  ThreadState *thr;
+  uptr stk_addr;
+  uptr stk_size;
+  uptr tls_addr;
+  uptr tls_size;
+};
+
+void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id,
+                 ThreadType thread_type) {
+  uptr stk_addr = 0;
+  uptr stk_size = 0;
+  uptr tls_addr = 0;
+  uptr tls_size = 0;
+#if !SANITIZER_GO
+  if (thread_type != ThreadType::Fiber)
+    GetThreadStackAndTls(tid == kMainTid, &stk_addr, &stk_size, &tls_addr,
+                         &tls_size);
+#endif
+
+  ThreadRegistry *tr = &ctx->thread_registry;
+  OnStartedArgs args = { thr, stk_addr, stk_size, tls_addr, tls_size };
+  tr->StartThread(tid, os_id, thread_type, &args);
+
+  while (!thr->tctx->trace.parts.Empty()) thr->tctx->trace.parts.PopBack();
+
+#if !SANITIZER_GO
+  if (ctx->after_multithreaded_fork) {
+    thr->ignore_interceptors++;
+    ThreadIgnoreBegin(thr, 0);
+    ThreadIgnoreSyncBegin(thr, 0);
+  }
+#endif
+
+#if !SANITIZER_GO
+  // Don't imitate stack/TLS writes for the main thread,
+  // because its initialization is synchronized with all
+  // subsequent threads anyway.
+  if (tid != kMainTid) {
+    if (stk_addr && stk_size) {
+      const uptr pc = StackTrace::GetNextInstructionPc(
+          reinterpret_cast<uptr>(__tsan_stack_initialization));
+      MemoryRangeImitateWrite(thr, pc, stk_addr, stk_size);
+    }
+
+    if (tls_addr && tls_size)
+      ImitateTlsWrite(thr, tls_addr, tls_size);
+  }
+#endif
+}
+
+void ThreadContext::OnStarted(void *arg) {
+  OnStartedArgs *args = static_cast<OnStartedArgs *>(arg);
+  thr = args->thr;
+  // RoundUp so that one trace part does not contain events
+  // from different threads.
+  epoch0 = RoundUp(epoch1 + 1, kTracePartSize);
+  epoch1 = (u64)-1;
+  new (thr)
+      ThreadState(ctx, tid, unique_id, epoch0, reuse_count, args->stk_addr,
+                  args->stk_size, args->tls_addr, args->tls_size);
+  if (common_flags()->detect_deadlocks)
+    thr->dd_lt = ctx->dd->CreateLogicalThread(unique_id);
+  thr->fast_state.SetHistorySize(flags()->history_size);
+  // Commit switch to the new part of the trace.
+  // TraceAddEvent will reset stack0/mset0 in the new part for us.
+  TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
+
+  thr->fast_synch_epoch = epoch0;
+  AcquireImpl(thr, 0, &sync);
+  sync.Reset(&thr->proc()->clock_cache);
+  thr->tctx = this;
+  thr->is_inited = true;
+  DPrintf(
+      "#%d: ThreadStart epoch=%zu stk_addr=%zx stk_size=%zx "
+      "tls_addr=%zx tls_size=%zx\n",
+      tid, (uptr)epoch0, args->stk_addr, args->stk_size, args->tls_addr,
+      args->tls_size);
+}
+
+void ThreadFinish(ThreadState *thr) {
+  ThreadCheckIgnore(thr);
+  if (thr->stk_addr && thr->stk_size)
+    DontNeedShadowFor(thr->stk_addr, thr->stk_size);
+  if (thr->tls_addr && thr->tls_size)
+    DontNeedShadowFor(thr->tls_addr, thr->tls_size);
+  thr->is_dead = true;
+  thr->is_inited = false;
+#if !SANITIZER_GO
+  thr->ignore_interceptors++;
+#endif
+  ctx->thread_registry.FinishThread(thr->tid);
+}
+
+void ThreadContext::OnFinished() {
+  if (!detached) {
+    thr->fast_state.IncrementEpoch();
+    // Can't increment epoch w/o writing to the trace as well.
+    TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
+    ReleaseImpl(thr, 0, &sync);
+  }
+  epoch1 = thr->fast_state.epoch();
+
+#if !SANITIZER_GO
+  UnmapOrDie(thr->shadow_stack, kShadowStackSize * sizeof(uptr));
+#else
+  Free(thr->shadow_stack);
+#endif
+  thr->shadow_stack = nullptr;
+  thr->shadow_stack_pos = nullptr;
+  thr->shadow_stack_end = nullptr;
+
+  if (common_flags()->detect_deadlocks)
+    ctx->dd->DestroyLogicalThread(thr->dd_lt);
+  thr->clock.ResetCached(&thr->proc()->clock_cache);
+#if !SANITIZER_GO
+  thr->last_sleep_clock.ResetCached(&thr->proc()->clock_cache);
+#endif
+#if !SANITIZER_GO
+  PlatformCleanUpThreadState(thr);
+#endif
+  thr->~ThreadState();
+  thr = 0;
+}
+
+struct ConsumeThreadContext {
+  uptr uid;
+  ThreadContextBase *tctx;
+};
+
+Tid ThreadConsumeTid(ThreadState *thr, uptr pc, uptr uid) {
+  return ctx->thread_registry.ConsumeThreadUserId(uid);
+}
+
+void ThreadJoin(ThreadState *thr, uptr pc, Tid tid) {
+  CHECK_GT(tid, 0);
+  CHECK_LT(tid, kMaxTid);
+  DPrintf("#%d: ThreadJoin tid=%d\n", thr->tid, tid);
+  ctx->thread_registry.JoinThread(tid, thr);
+}
+
+void ThreadContext::OnJoined(void *arg) {
+  ThreadState *caller_thr = static_cast<ThreadState *>(arg);
+  AcquireImpl(caller_thr, 0, &sync);
+  sync.Reset(&caller_thr->proc()->clock_cache);
+}
+
+void ThreadContext::OnDead() { CHECK_EQ(sync.size(), 0); }
+
+void ThreadDetach(ThreadState *thr, uptr pc, Tid tid) {
+  CHECK_GT(tid, 0);
+  CHECK_LT(tid, kMaxTid);
+  ctx->thread_registry.DetachThread(tid, thr);
+}
+
+void ThreadContext::OnDetached(void *arg) {
+  ThreadState *thr1 = static_cast<ThreadState *>(arg);
+  sync.Reset(&thr1->proc()->clock_cache);
+}
+
+void ThreadNotJoined(ThreadState *thr, uptr pc, Tid tid, uptr uid) {
+  CHECK_GT(tid, 0);
+  CHECK_LT(tid, kMaxTid);
+  ctx->thread_registry.SetThreadUserId(tid, uid);
+}
+
+void ThreadSetName(ThreadState *thr, const char *name) {
+  ctx->thread_registry.SetThreadName(thr->tid, name);
+}
+
+#if !SANITIZER_GO
+void FiberSwitchImpl(ThreadState *from, ThreadState *to) {
+  Processor *proc = from->proc();
+  ProcUnwire(proc, from);
+  ProcWire(proc, to);
+  set_cur_thread(to);
+}
+
+ThreadState *FiberCreate(ThreadState *thr, uptr pc, unsigned flags) {
+  void *mem = Alloc(sizeof(ThreadState));
+  ThreadState *fiber = static_cast<ThreadState *>(mem);
+  internal_memset(fiber, 0, sizeof(*fiber));
+  Tid tid = ThreadCreate(thr, pc, 0, true);
+  FiberSwitchImpl(thr, fiber);
+  ThreadStart(fiber, tid, 0, ThreadType::Fiber);
+  FiberSwitchImpl(fiber, thr);
+  return fiber;
+}
+
+void FiberDestroy(ThreadState *thr, uptr pc, ThreadState *fiber) {
+  FiberSwitchImpl(thr, fiber);
+  ThreadFinish(fiber);
+  FiberSwitchImpl(fiber, thr);
+  Free(fiber);
+}
+
+void FiberSwitch(ThreadState *thr, uptr pc,
+                 ThreadState *fiber, unsigned flags) {
+  if (!(flags & FiberSwitchFlagNoSync))
+    Release(thr, pc, (uptr)fiber);
+  FiberSwitchImpl(thr, fiber);
+  if (!(flags & FiberSwitchFlagNoSync))
+    Acquire(fiber, pc, (uptr)fiber);
+}
+#endif
+
+}  // namespace __tsan
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_shadow.h b/compiler-rt/lib/tsan/rtl-old/tsan_shadow.h
new file mode 100644
index 000000000000..8b7bc341713e
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_shadow.h
@@ -0,0 +1,233 @@
+//===-- tsan_shadow.h -------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef TSAN_SHADOW_H
+#define TSAN_SHADOW_H
+
+#include "tsan_defs.h"
+#include "tsan_trace.h"
+
+namespace __tsan {
+
+// FastState (from most significant bit):
+//   ignore          : 1
+//   tid             : kTidBits
+//   unused          : -
+//   history_size    : 3
+//   epoch           : kClkBits
+class FastState {
+ public:
+  FastState(u64 tid, u64 epoch) {
+    x_ = tid << kTidShift;
+    x_ |= epoch;
+    DCHECK_EQ(tid, this->tid());
+    DCHECK_EQ(epoch, this->epoch());
+    DCHECK_EQ(GetIgnoreBit(), false);
+  }
+
+  explicit FastState(u64 x) : x_(x) {}
+
+  u64 raw() const { return x_; }
+
+  u64 tid() const {
+    u64 res = (x_ & ~kIgnoreBit) >> kTidShift;
+    return res;
+  }
+
+  u64 TidWithIgnore() const {
+    u64 res = x_ >> kTidShift;
+    return res;
+  }
+
+  u64 epoch() const {
+    u64 res = x_ & ((1ull << kClkBits) - 1);
+    return res;
+  }
+
+  void IncrementEpoch() {
+    u64 old_epoch = epoch();
+    x_ += 1;
+    DCHECK_EQ(old_epoch + 1, epoch());
+    (void)old_epoch;
+  }
+
+  void SetIgnoreBit() { x_ |= kIgnoreBit; }
+  void ClearIgnoreBit() { x_ &= ~kIgnoreBit; }
+  bool GetIgnoreBit() const { return (s64)x_ < 0; }
+
+  void SetHistorySize(int hs) {
+    CHECK_GE(hs, 0);
+    CHECK_LE(hs, 7);
+    x_ = (x_ & ~(kHistoryMask << kHistoryShift)) | (u64(hs) << kHistoryShift);
+  }
+
+  ALWAYS_INLINE
+  int GetHistorySize() const {
+    return (int)((x_ >> kHistoryShift) & kHistoryMask);
+  }
+
+  void ClearHistorySize() { SetHistorySize(0); }
+
+  ALWAYS_INLINE
+  u64 GetTracePos() const {
+    const int hs = GetHistorySize();
+    // When hs == 0, the trace consists of 2 parts.
+    const u64 mask = (1ull << (kTracePartSizeBits + hs + 1)) - 1;
+    return epoch() & mask;
+  }
+
+ private:
+  friend class Shadow;
+  static const int kTidShift = 64 - kTidBits - 1;
+  static const u64 kIgnoreBit = 1ull << 63;
+  static const u64 kFreedBit = 1ull << 63;
+  static const u64 kHistoryShift = kClkBits;
+  static const u64 kHistoryMask = 7;
+  u64 x_;
+};
+
+// Shadow (from most significant bit):
+//   freed           : 1
+//   tid             : kTidBits
+//   is_atomic       : 1
+//   is_read         : 1
+//   size_log        : 2
+//   addr0           : 3
+//   epoch           : kClkBits
+class Shadow : public FastState {
+ public:
+  explicit Shadow(u64 x) : FastState(x) {}
+
+  explicit Shadow(const FastState &s) : FastState(s.x_) { ClearHistorySize(); }
+
+  void SetAddr0AndSizeLog(u64 addr0, unsigned kAccessSizeLog) {
+    DCHECK_EQ((x_ >> kClkBits) & 31, 0);
+    DCHECK_LE(addr0, 7);
+    DCHECK_LE(kAccessSizeLog, 3);
+    x_ |= ((kAccessSizeLog << 3) | addr0) << kClkBits;
+    DCHECK_EQ(kAccessSizeLog, size_log());
+    DCHECK_EQ(addr0, this->addr0());
+  }
+
+  void SetWrite(unsigned kAccessIsWrite) {
+    DCHECK_EQ(x_ & kReadBit, 0);
+    if (!kAccessIsWrite)
+      x_ |= kReadBit;
+    DCHECK_EQ(kAccessIsWrite, IsWrite());
+  }
+
+  void SetAtomic(bool kIsAtomic) {
+    DCHECK(!IsAtomic());
+    if (kIsAtomic)
+      x_ |= kAtomicBit;
+    DCHECK_EQ(IsAtomic(), kIsAtomic);
+  }
+
+  bool IsAtomic() const { return x_ & kAtomicBit; }
+
+  bool IsZero() const { return x_ == 0; }
+
+  static inline bool TidsAreEqual(const Shadow s1, const Shadow s2) {
+    u64 shifted_xor = (s1.x_ ^ s2.x_) >> kTidShift;
+    DCHECK_EQ(shifted_xor == 0, s1.TidWithIgnore() == s2.TidWithIgnore());
+    return shifted_xor == 0;
+  }
+
+  static ALWAYS_INLINE bool Addr0AndSizeAreEqual(const Shadow s1,
+                                                 const Shadow s2) {
+    u64 masked_xor = ((s1.x_ ^ s2.x_) >> kClkBits) & 31;
+    return masked_xor == 0;
+  }
+
+  static ALWAYS_INLINE bool TwoRangesIntersect(Shadow s1, Shadow s2,
+                                               unsigned kS2AccessSize) {
+    bool res = false;
+    u64 diff = s1.addr0() - s2.addr0();
+    if ((s64)diff < 0) {  // s1.addr0 < s2.addr0
+      // if (s1.addr0() + size1) > s2.addr0()) return true;
+      if (s1.size() > -diff)
+        res = true;
+    } else {
+      // if (s2.addr0() + kS2AccessSize > s1.addr0()) return true;
+      if (kS2AccessSize > diff)
+        res = true;
+    }
+    DCHECK_EQ(res, TwoRangesIntersectSlow(s1, s2));
+    DCHECK_EQ(res, TwoRangesIntersectSlow(s2, s1));
+    return res;
+  }
+
+  u64 ALWAYS_INLINE addr0() const { return (x_ >> kClkBits) & 7; }
+  u64 ALWAYS_INLINE size() const { return 1ull << size_log(); }
+  bool ALWAYS_INLINE IsWrite() const { return !IsRead(); }
+  bool ALWAYS_INLINE IsRead() const { return x_ & kReadBit; }
+
+  // The idea behind the freed bit is as follows.
+  // When the memory is freed (or otherwise unaccessible) we write to the shadow
+  // values with tid/epoch related to the free and the freed bit set.
+  // During memory accesses processing the freed bit is considered
+  // as msb of tid. So any access races with shadow with freed bit set
+  // (it is as if write from a thread with which we never synchronized before).
+  // This allows us to detect accesses to freed memory w/o additional
+  // overheads in memory access processing and at the same time restore
+  // tid/epoch of free.
+  void MarkAsFreed() { x_ |= kFreedBit; }
+
+  bool IsFreed() const { return x_ & kFreedBit; }
+
+  bool GetFreedAndReset() {
+    bool res = x_ & kFreedBit;
+    x_ &= ~kFreedBit;
+    return res;
+  }
+
+  bool ALWAYS_INLINE IsBothReadsOrAtomic(bool kIsWrite, bool kIsAtomic) const {
+    bool v = x_ & ((u64(kIsWrite ^ 1) << kReadShift) |
+                   (u64(kIsAtomic) << kAtomicShift));
+    DCHECK_EQ(v, (!IsWrite() && !kIsWrite) || (IsAtomic() && kIsAtomic));
+    return v;
+  }
+
+  bool ALWAYS_INLINE IsRWNotWeaker(bool kIsWrite, bool kIsAtomic) const {
+    bool v = ((x_ >> kReadShift) & 3) <= u64((kIsWrite ^ 1) | (kIsAtomic << 1));
+    DCHECK_EQ(v, (IsAtomic() < kIsAtomic) ||
+                     (IsAtomic() == kIsAtomic && !IsWrite() <= !kIsWrite));
+    return v;
+  }
+
+  bool ALWAYS_INLINE IsRWWeakerOrEqual(bool kIsWrite, bool kIsAtomic) const {
+    bool v = ((x_ >> kReadShift) & 3) >= u64((kIsWrite ^ 1) | (kIsAtomic << 1));
+    DCHECK_EQ(v, (IsAtomic() > kIsAtomic) ||
+                     (IsAtomic() == kIsAtomic && !IsWrite() >= !kIsWrite));
+    return v;
+  }
+
+ private:
+  static const u64 kReadShift = 5 + kClkBits;
+  static const u64 kReadBit = 1ull << kReadShift;
+  static const u64 kAtomicShift = 6 + kClkBits;
+  static const u64 kAtomicBit = 1ull << kAtomicShift;
+
+  u64 size_log() const { return (x_ >> (3 + kClkBits)) & 3; }
+
+  static bool TwoRangesIntersectSlow(const Shadow s1, const Shadow s2) {
+    if (s1.addr0() == s2.addr0())
+      return true;
+    if (s1.addr0() < s2.addr0() && s1.addr0() + s1.size() > s2.addr0())
+      return true;
+    if (s2.addr0() < s1.addr0() && s2.addr0() + s2.size() > s1.addr0())
+      return true;
+    return false;
+  }
+};
+
+const RawShadow kShadowRodata = (RawShadow)-1;  // .rodata shadow marker
+
+}  // namespace __tsan
+
+#endif
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_stack_trace.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_stack_trace.cpp
new file mode 100644
index 000000000000..9bbaafb3a85f
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_stack_trace.cpp
@@ -0,0 +1,57 @@
+//===-- tsan_stack_trace.cpp ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#include "tsan_stack_trace.h"
+#include "tsan_rtl.h"
+#include "tsan_mman.h"
+
+namespace __tsan {
+
+VarSizeStackTrace::VarSizeStackTrace()
+    : StackTrace(nullptr, 0), trace_buffer(nullptr) {}
+
+VarSizeStackTrace::~VarSizeStackTrace() {
+  ResizeBuffer(0);
+}
+
+void VarSizeStackTrace::ResizeBuffer(uptr new_size) {
+  Free(trace_buffer);
+  trace_buffer = (new_size > 0)
+                     ? (uptr *)Alloc(new_size * sizeof(trace_buffer[0]))
+                     : nullptr;
+  trace = trace_buffer;
+  size = new_size;
+}
+
+void VarSizeStackTrace::Init(const uptr *pcs, uptr cnt, uptr extra_top_pc) {
+  ResizeBuffer(cnt + !!extra_top_pc);
+  internal_memcpy(trace_buffer, pcs, cnt * sizeof(trace_buffer[0]));
+  if (extra_top_pc)
+    trace_buffer[cnt] = extra_top_pc;
+}
+
+void VarSizeStackTrace::ReverseOrder() {
+  for (u32 i = 0; i < (size >> 1); i++)
+    Swap(trace_buffer[i], trace_buffer[size - 1 - i]);
+}
+
+}  // namespace __tsan
+
+#if !SANITIZER_GO
+void __sanitizer::BufferedStackTrace::UnwindImpl(
+    uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) {
+  uptr top = 0;
+  uptr bottom = 0;
+  GetThreadStackTopAndBottom(false, &top, &bottom);
+  bool fast = StackTrace::WillUseFastUnwind(request_fast);
+  Unwind(max_depth, pc, bp, context, top, bottom, fast);
+}
+#endif  // SANITIZER_GO
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_stack_trace.h b/compiler-rt/lib/tsan/rtl-old/tsan_stack_trace.h
new file mode 100644
index 000000000000..3eb8ce156e83
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_stack_trace.h
@@ -0,0 +1,42 @@
+//===-- tsan_stack_trace.h --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_STACK_TRACE_H
+#define TSAN_STACK_TRACE_H
+
+#include "sanitizer_common/sanitizer_stacktrace.h"
+#include "tsan_defs.h"
+
+namespace __tsan {
+
+// StackTrace which calls malloc/free to allocate the buffer for
+// addresses in stack traces.
+struct VarSizeStackTrace : public StackTrace {
+  uptr *trace_buffer;  // Owned.
+
+  VarSizeStackTrace();
+  ~VarSizeStackTrace();
+  void Init(const uptr *pcs, uptr cnt, uptr extra_top_pc = 0);
+
+  // Reverses the current stack trace order, the top frame goes to the bottom,
+  // the last frame goes to the top.
+  void ReverseOrder();
+
+ private:
+  void ResizeBuffer(uptr new_size);
+
+  VarSizeStackTrace(const VarSizeStackTrace &);
+  void operator=(const VarSizeStackTrace &);
+};
+
+}  // namespace __tsan
+
+#endif  // TSAN_STACK_TRACE_H
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_suppressions.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_suppressions.cpp
new file mode 100644
index 000000000000..a1c1bf81bf67
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_suppressions.cpp
@@ -0,0 +1,161 @@
+//===-- tsan_suppressions.cpp ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_libc.h"
+#include "sanitizer_common/sanitizer_placement_new.h"
+#include "sanitizer_common/sanitizer_suppressions.h"
+#include "tsan_suppressions.h"
+#include "tsan_rtl.h"
+#include "tsan_flags.h"
+#include "tsan_mman.h"
+#include "tsan_platform.h"
+
+#if !SANITIZER_GO
+// Suppressions for true/false positives in standard libraries.
+static const char *const std_suppressions =
+// Libstdc++ 4.4 has data races in std::string.
+// See http://crbug.com/181502 for an example.
+"race:^_M_rep$\n"
+"race:^_M_is_leaked$\n"
+// False positive when using std <thread>.
+// Happens because we miss atomic synchronization in libstdc++.
+// See http://llvm.org/bugs/show_bug.cgi?id=17066 for details.
+"race:std::_Sp_counted_ptr_inplace<std::thread::_Impl\n";
+
+// Can be overriden in frontend.
+SANITIZER_WEAK_DEFAULT_IMPL
+const char *__tsan_default_suppressions() {
+  return 0;
+}
+#endif
+
+namespace __tsan {
+
+ALIGNED(64) static char suppression_placeholder[sizeof(SuppressionContext)];
+static SuppressionContext *suppression_ctx = nullptr;
+static const char *kSuppressionTypes[] = {
+    kSuppressionRace,   kSuppressionRaceTop, kSuppressionMutex,
+    kSuppressionThread, kSuppressionSignal, kSuppressionLib,
+    kSuppressionDeadlock};
+
+void InitializeSuppressions() {
+  CHECK_EQ(nullptr, suppression_ctx);
+  suppression_ctx = new (suppression_placeholder)
+      SuppressionContext(kSuppressionTypes, ARRAY_SIZE(kSuppressionTypes));
+  suppression_ctx->ParseFromFile(flags()->suppressions);
+#if !SANITIZER_GO
+  suppression_ctx->Parse(__tsan_default_suppressions());
+  suppression_ctx->Parse(std_suppressions);
+#endif
+}
+
+SuppressionContext *Suppressions() {
+  CHECK(suppression_ctx);
+  return suppression_ctx;
+}
+
+static const char *conv(ReportType typ) {
+  switch (typ) {
+    case ReportTypeRace:
+    case ReportTypeVptrRace:
+    case ReportTypeUseAfterFree:
+    case ReportTypeVptrUseAfterFree:
+    case ReportTypeExternalRace:
+      return kSuppressionRace;
+    case ReportTypeThreadLeak:
+      return kSuppressionThread;
+    case ReportTypeMutexDestroyLocked:
+    case ReportTypeMutexDoubleLock:
+    case ReportTypeMutexInvalidAccess:
+    case ReportTypeMutexBadUnlock:
+    case ReportTypeMutexBadReadLock:
+    case ReportTypeMutexBadReadUnlock:
+      return kSuppressionMutex;
+    case ReportTypeSignalUnsafe:
+    case ReportTypeErrnoInSignal:
+      return kSuppressionSignal;
+    case ReportTypeDeadlock:
+      return kSuppressionDeadlock;
+    // No default case so compiler warns us if we miss one
+  }
+  UNREACHABLE("missing case");
+}
+
+static uptr IsSuppressed(const char *stype, const AddressInfo &info,
+    Suppression **sp) {
+  if (suppression_ctx->Match(info.function, stype, sp) ||
+      suppression_ctx->Match(info.file, stype, sp) ||
+      suppression_ctx->Match(info.module, stype, sp)) {
+    VPrintf(2, "ThreadSanitizer: matched suppression '%s'\n", (*sp)->templ);
+    atomic_fetch_add(&(*sp)->hit_count, 1, memory_order_relaxed);
+    return info.address;
+  }
+  return 0;
+}
+
+uptr IsSuppressed(ReportType typ, const ReportStack *stack, Suppression **sp) {
+  CHECK(suppression_ctx);
+  if (!suppression_ctx->SuppressionCount() || stack == 0 ||
+      !stack->suppressable)
+    return 0;
+  const char *stype = conv(typ);
+  if (0 == internal_strcmp(stype, kSuppressionNone))
+    return 0;
+  for (const SymbolizedStack *frame = stack->frames; frame;
+      frame = frame->next) {
+    uptr pc = IsSuppressed(stype, frame->info, sp);
+    if (pc != 0)
+      return pc;
+  }
+  if (0 == internal_strcmp(stype, kSuppressionRace) && stack->frames != nullptr)
+    return IsSuppressed(kSuppressionRaceTop, stack->frames->info, sp);
+  return 0;
+}
+
+uptr IsSuppressed(ReportType typ, const ReportLocation *loc, Suppression **sp) {
+  CHECK(suppression_ctx);
+  if (!suppression_ctx->SuppressionCount() || loc == 0 ||
+      loc->type != ReportLocationGlobal || !loc->suppressable)
+    return 0;
+  const char *stype = conv(typ);
+  if (0 == internal_strcmp(stype, kSuppressionNone))
+    return 0;
+  Suppression *s;
+  const DataInfo &global = loc->global;
+  if (suppression_ctx->Match(global.name, stype, &s) ||
+      suppression_ctx->Match(global.module, stype, &s)) {
+      VPrintf(2, "ThreadSanitizer: matched suppression '%s'\n", s->templ);
+      atomic_fetch_add(&s->hit_count, 1, memory_order_relaxed);
+      *sp = s;
+      return global.start;
+  }
+  return 0;
+}
+
+void PrintMatchedSuppressions() {
+  InternalMmapVector<Suppression *> matched;
+  CHECK(suppression_ctx);
+  suppression_ctx->GetMatched(&matched);
+  if (!matched.size())
+    return;
+  int hit_count = 0;
+  for (uptr i = 0; i < matched.size(); i++)
+    hit_count += atomic_load_relaxed(&matched[i]->hit_count);
+  Printf("ThreadSanitizer: Matched %d suppressions (pid=%d):\n", hit_count,
+         (int)internal_getpid());
+  for (uptr i = 0; i < matched.size(); i++) {
+    Printf("%d %s:%s\n", atomic_load_relaxed(&matched[i]->hit_count),
+           matched[i]->type, matched[i]->templ);
+  }
+}
+}  // namespace __tsan
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_suppressions.h b/compiler-rt/lib/tsan/rtl-old/tsan_suppressions.h
new file mode 100644
index 000000000000..f430aeb6c4cf
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_suppressions.h
@@ -0,0 +1,37 @@
+//===-- tsan_suppressions.h -------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_SUPPRESSIONS_H
+#define TSAN_SUPPRESSIONS_H
+
+#include "sanitizer_common/sanitizer_suppressions.h"
+#include "tsan_report.h"
+
+namespace __tsan {
+
+const char kSuppressionNone[] = "none";
+const char kSuppressionRace[] = "race";
+const char kSuppressionRaceTop[] = "race_top";
+const char kSuppressionMutex[] = "mutex";
+const char kSuppressionThread[] = "thread";
+const char kSuppressionSignal[] = "signal";
+const char kSuppressionLib[] = "called_from_lib";
+const char kSuppressionDeadlock[] = "deadlock";
+
+void InitializeSuppressions();
+SuppressionContext *Suppressions();
+void PrintMatchedSuppressions();
+uptr IsSuppressed(ReportType typ, const ReportStack *stack, Suppression **sp);
+uptr IsSuppressed(ReportType typ, const ReportLocation *loc, Suppression **sp);
+
+}  // namespace __tsan
+
+#endif  // TSAN_SUPPRESSIONS_H
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_symbolize.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_symbolize.cpp
new file mode 100644
index 000000000000..2e2744d2eae7
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_symbolize.cpp
@@ -0,0 +1,123 @@
+//===-- tsan_symbolize.cpp ------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+
+#include "tsan_symbolize.h"
+
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_placement_new.h"
+#include "sanitizer_common/sanitizer_symbolizer.h"
+#include "tsan_flags.h"
+#include "tsan_report.h"
+#include "tsan_rtl.h"
+
+namespace __tsan {
+
+void EnterSymbolizer() {
+  ThreadState *thr = cur_thread();
+  CHECK(!thr->in_symbolizer);
+  thr->in_symbolizer = true;
+  thr->ignore_interceptors++;
+}
+
+void ExitSymbolizer() {
+  ThreadState *thr = cur_thread();
+  CHECK(thr->in_symbolizer);
+  thr->in_symbolizer = false;
+  thr->ignore_interceptors--;
+}
+
+// Legacy API.
+// May be overriden by JIT/JAVA/etc,
+// whatever produces PCs marked with kExternalPCBit.
+SANITIZER_WEAK_DEFAULT_IMPL
+bool __tsan_symbolize_external(uptr pc, char *func_buf, uptr func_siz,
+                               char *file_buf, uptr file_siz, int *line,
+                               int *col) {
+  return false;
+}
+
+// New API: call __tsan_symbolize_external_ex only when it exists.
+// Once old clients are gone, provide dummy implementation.
+SANITIZER_WEAK_DEFAULT_IMPL
+void __tsan_symbolize_external_ex(uptr pc,
+                                  void (*add_frame)(void *, const char *,
+                                                    const char *, int, int),
+                                  void *ctx) {}
+
+struct SymbolizedStackBuilder {
+  SymbolizedStack *head;
+  SymbolizedStack *tail;
+  uptr addr;
+};
+
+static void AddFrame(void *ctx, const char *function_name, const char *file,
+                     int line, int column) {
+  SymbolizedStackBuilder *ssb = (struct SymbolizedStackBuilder *)ctx;
+  if (ssb->tail) {
+    ssb->tail->next = SymbolizedStack::New(ssb->addr);
+    ssb->tail = ssb->tail->next;
+  } else {
+    ssb->head = ssb->tail = SymbolizedStack::New(ssb->addr);
+  }
+  AddressInfo *info = &ssb->tail->info;
+  if (function_name) {
+    info->function = internal_strdup(function_name);
+  }
+  if (file) {
+    info->file = internal_strdup(file);
+  }
+  info->line = line;
+  info->column = column;
+}
+
+SymbolizedStack *SymbolizeCode(uptr addr) {
+  // Check if PC comes from non-native land.
+  if (addr & kExternalPCBit) {
+    SymbolizedStackBuilder ssb = {nullptr, nullptr, addr};
+    __tsan_symbolize_external_ex(addr, AddFrame, &ssb);
+    if (ssb.head)
+      return ssb.head;
+    // Legacy code: remove along with the declaration above
+    // once all clients using this API are gone.
+    // Declare static to not consume too much stack space.
+    // We symbolize reports in a single thread, so this is fine.
+    static char func_buf[1024];
+    static char file_buf[1024];
+    int line, col;
+    SymbolizedStack *frame = SymbolizedStack::New(addr);
+    if (__tsan_symbolize_external(addr, func_buf, sizeof(func_buf), file_buf,
+                                  sizeof(file_buf), &line, &col)) {
+      frame->info.function = internal_strdup(func_buf);
+      frame->info.file = internal_strdup(file_buf);
+      frame->info.line = line;
+      frame->info.column = col;
+    }
+    return frame;
+  }
+  return Symbolizer::GetOrInit()->SymbolizePC(addr);
+}
+
+ReportLocation *SymbolizeData(uptr addr) {
+  DataInfo info;
+  if (!Symbolizer::GetOrInit()->SymbolizeData(addr, &info))
+    return 0;
+  auto *ent = New<ReportLocation>();
+  ent->type = ReportLocationGlobal;
+  internal_memcpy(&ent->global, &info, sizeof(info));
+  return ent;
+}
+
+void SymbolizeFlush() {
+  Symbolizer::GetOrInit()->Flush();
+}
+
+}  // namespace __tsan
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_symbolize.h b/compiler-rt/lib/tsan/rtl-old/tsan_symbolize.h
new file mode 100644
index 000000000000..7adaa04dc273
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_symbolize.h
@@ -0,0 +1,30 @@
+//===-- tsan_symbolize.h ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_SYMBOLIZE_H
+#define TSAN_SYMBOLIZE_H
+
+#include "tsan_defs.h"
+#include "tsan_report.h"
+
+namespace __tsan {
+
+void EnterSymbolizer();
+void ExitSymbolizer();
+SymbolizedStack *SymbolizeCode(uptr addr);
+ReportLocation *SymbolizeData(uptr addr);
+void SymbolizeFlush();
+
+ReportStack *NewReportStackEntry(uptr addr);
+
+}  // namespace __tsan
+
+#endif  // TSAN_SYMBOLIZE_H
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_sync.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_sync.cpp
new file mode 100644
index 000000000000..f042abab74e5
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_sync.cpp
@@ -0,0 +1,279 @@
+//===-- tsan_sync.cpp -----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#include "sanitizer_common/sanitizer_placement_new.h"
+#include "tsan_sync.h"
+#include "tsan_rtl.h"
+#include "tsan_mman.h"
+
+namespace __tsan {
+
+void DDMutexInit(ThreadState *thr, uptr pc, SyncVar *s);
+
+SyncVar::SyncVar() : mtx(MutexTypeSyncVar) { Reset(0); }
+
+void SyncVar::Init(ThreadState *thr, uptr pc, uptr addr, u64 uid,
+                   bool save_stack) {
+  this->addr = addr;
+  this->uid = uid;
+  this->next = 0;
+
+  creation_stack_id = kInvalidStackID;
+  if (save_stack && !SANITIZER_GO)  // Go does not use them
+    creation_stack_id = CurrentStackId(thr, pc);
+  if (common_flags()->detect_deadlocks)
+    DDMutexInit(thr, pc, this);
+}
+
+void SyncVar::Reset(Processor *proc) {
+  uid = 0;
+  creation_stack_id = kInvalidStackID;
+  owner_tid = kInvalidTid;
+  last_lock = 0;
+  recursion = 0;
+  atomic_store_relaxed(&flags, 0);
+
+  if (proc == 0) {
+    CHECK_EQ(clock.size(), 0);
+    CHECK_EQ(read_clock.size(), 0);
+  } else {
+    clock.Reset(&proc->clock_cache);
+    read_clock.Reset(&proc->clock_cache);
+  }
+}
+
+MetaMap::MetaMap()
+    : block_alloc_(LINKER_INITIALIZED, "heap block allocator"),
+      sync_alloc_(LINKER_INITIALIZED, "sync allocator") {
+  atomic_store(&uid_gen_, 0, memory_order_relaxed);
+}
+
+void MetaMap::AllocBlock(ThreadState *thr, uptr pc, uptr p, uptr sz) {
+  u32 idx = block_alloc_.Alloc(&thr->proc()->block_cache);
+  MBlock *b = block_alloc_.Map(idx);
+  b->siz = sz;
+  b->tag = 0;
+  b->tid = thr->tid;
+  b->stk = CurrentStackId(thr, pc);
+  u32 *meta = MemToMeta(p);
+  DCHECK_EQ(*meta, 0);
+  *meta = idx | kFlagBlock;
+}
+
+uptr MetaMap::FreeBlock(Processor *proc, uptr p) {
+  MBlock* b = GetBlock(p);
+  if (b == 0)
+    return 0;
+  uptr sz = RoundUpTo(b->siz, kMetaShadowCell);
+  FreeRange(proc, p, sz);
+  return sz;
+}
+
+bool MetaMap::FreeRange(Processor *proc, uptr p, uptr sz) {
+  bool has_something = false;
+  u32 *meta = MemToMeta(p);
+  u32 *end = MemToMeta(p + sz);
+  if (end == meta)
+    end++;
+  for (; meta < end; meta++) {
+    u32 idx = *meta;
+    if (idx == 0) {
+      // Note: don't write to meta in this case -- the block can be huge.
+      continue;
+    }
+    *meta = 0;
+    has_something = true;
+    while (idx != 0) {
+      if (idx & kFlagBlock) {
+        block_alloc_.Free(&proc->block_cache, idx & ~kFlagMask);
+        break;
+      } else if (idx & kFlagSync) {
+        DCHECK(idx & kFlagSync);
+        SyncVar *s = sync_alloc_.Map(idx & ~kFlagMask);
+        u32 next = s->next;
+        s->Reset(proc);
+        sync_alloc_.Free(&proc->sync_cache, idx & ~kFlagMask);
+        idx = next;
+      } else {
+        CHECK(0);
+      }
+    }
+  }
+  return has_something;
+}
+
+// ResetRange removes all meta objects from the range.
+// It is called for large mmap-ed regions. The function is best-effort wrt
+// freeing of meta objects, because we don't want to page in the whole range
+// which can be huge. The function probes pages one-by-one until it finds a page
+// without meta objects, at this point it stops freeing meta objects. Because
+// thread stacks grow top-down, we do the same starting from end as well.
+void MetaMap::ResetRange(Processor *proc, uptr p, uptr sz) {
+  if (SANITIZER_GO) {
+    // UnmapOrDie/MmapFixedNoReserve does not work on Windows,
+    // so we do the optimization only for C/C++.
+    FreeRange(proc, p, sz);
+    return;
+  }
+  const uptr kMetaRatio = kMetaShadowCell / kMetaShadowSize;
+  const uptr kPageSize = GetPageSizeCached() * kMetaRatio;
+  if (sz <= 4 * kPageSize) {
+    // If the range is small, just do the normal free procedure.
+    FreeRange(proc, p, sz);
+    return;
+  }
+  // First, round both ends of the range to page size.
+  uptr diff = RoundUp(p, kPageSize) - p;
+  if (diff != 0) {
+    FreeRange(proc, p, diff);
+    p += diff;
+    sz -= diff;
+  }
+  diff = p + sz - RoundDown(p + sz, kPageSize);
+  if (diff != 0) {
+    FreeRange(proc, p + sz - diff, diff);
+    sz -= diff;
+  }
+  // Now we must have a non-empty page-aligned range.
+  CHECK_GT(sz, 0);
+  CHECK_EQ(p, RoundUp(p, kPageSize));
+  CHECK_EQ(sz, RoundUp(sz, kPageSize));
+  const uptr p0 = p;
+  const uptr sz0 = sz;
+  // Probe start of the range.
+  for (uptr checked = 0; sz > 0; checked += kPageSize) {
+    bool has_something = FreeRange(proc, p, kPageSize);
+    p += kPageSize;
+    sz -= kPageSize;
+    if (!has_something && checked > (128 << 10))
+      break;
+  }
+  // Probe end of the range.
+  for (uptr checked = 0; sz > 0; checked += kPageSize) {
+    bool has_something = FreeRange(proc, p + sz - kPageSize, kPageSize);
+    sz -= kPageSize;
+    // Stacks grow down, so sync object are most likely at the end of the region
+    // (if it is a stack). The very end of the stack is TLS and tsan increases
+    // TLS by at least 256K, so check at least 512K.
+    if (!has_something && checked > (512 << 10))
+      break;
+  }
+  // Finally, page out the whole range (including the parts that we've just
+  // freed). Note: we can't simply madvise, because we need to leave a zeroed
+  // range (otherwise __tsan_java_move can crash if it encounters a left-over
+  // meta objects in java heap).
+  uptr metap = (uptr)MemToMeta(p0);
+  uptr metasz = sz0 / kMetaRatio;
+  UnmapOrDie((void*)metap, metasz);
+  if (!MmapFixedSuperNoReserve(metap, metasz))
+    Die();
+}
+
+MBlock* MetaMap::GetBlock(uptr p) {
+  u32 *meta = MemToMeta(p);
+  u32 idx = *meta;
+  for (;;) {
+    if (idx == 0)
+      return 0;
+    if (idx & kFlagBlock)
+      return block_alloc_.Map(idx & ~kFlagMask);
+    DCHECK(idx & kFlagSync);
+    SyncVar * s = sync_alloc_.Map(idx & ~kFlagMask);
+    idx = s->next;
+  }
+}
+
+SyncVar *MetaMap::GetSync(ThreadState *thr, uptr pc, uptr addr, bool create,
+                          bool save_stack) {
+  u32 *meta = MemToMeta(addr);
+  u32 idx0 = *meta;
+  u32 myidx = 0;
+  SyncVar *mys = nullptr;
+  for (;;) {
+    for (u32 idx = idx0; idx && !(idx & kFlagBlock);) {
+      DCHECK(idx & kFlagSync);
+      SyncVar * s = sync_alloc_.Map(idx & ~kFlagMask);
+      if (LIKELY(s->addr == addr)) {
+        if (UNLIKELY(myidx != 0)) {
+          mys->Reset(thr->proc());
+          sync_alloc_.Free(&thr->proc()->sync_cache, myidx);
+        }
+        return s;
+      }
+      idx = s->next;
+    }
+    if (!create)
+      return nullptr;
+    if (UNLIKELY(*meta != idx0)) {
+      idx0 = *meta;
+      continue;
+    }
+
+    if (LIKELY(myidx == 0)) {
+      const u64 uid = atomic_fetch_add(&uid_gen_, 1, memory_order_relaxed);
+      myidx = sync_alloc_.Alloc(&thr->proc()->sync_cache);
+      mys = sync_alloc_.Map(myidx);
+      mys->Init(thr, pc, addr, uid, save_stack);
+    }
+    mys->next = idx0;
+    if (atomic_compare_exchange_strong((atomic_uint32_t*)meta, &idx0,
+        myidx | kFlagSync, memory_order_release)) {
+      return mys;
+    }
+  }
+}
+
+void MetaMap::MoveMemory(uptr src, uptr dst, uptr sz) {
+  // src and dst can overlap,
+  // there are no concurrent accesses to the regions (e.g. stop-the-world).
+  CHECK_NE(src, dst);
+  CHECK_NE(sz, 0);
+  uptr diff = dst - src;
+  u32 *src_meta = MemToMeta(src);
+  u32 *dst_meta = MemToMeta(dst);
+  u32 *src_meta_end = MemToMeta(src + sz);
+  uptr inc = 1;
+  if (dst > src) {
+    src_meta = MemToMeta(src + sz) - 1;
+    dst_meta = MemToMeta(dst + sz) - 1;
+    src_meta_end = MemToMeta(src) - 1;
+    inc = -1;
+  }
+  for (; src_meta != src_meta_end; src_meta += inc, dst_meta += inc) {
+    CHECK_EQ(*dst_meta, 0);
+    u32 idx = *src_meta;
+    *src_meta = 0;
+    *dst_meta = idx;
+    // Patch the addresses in sync objects.
+    while (idx != 0) {
+      if (idx & kFlagBlock)
+        break;
+      CHECK(idx & kFlagSync);
+      SyncVar *s = sync_alloc_.Map(idx & ~kFlagMask);
+      s->addr += diff;
+      idx = s->next;
+    }
+  }
+}
+
+void MetaMap::OnProcIdle(Processor *proc) {
+  block_alloc_.FlushCache(&proc->block_cache);
+  sync_alloc_.FlushCache(&proc->sync_cache);
+}
+
+MetaMap::MemoryStats MetaMap::GetMemoryStats() const {
+  MemoryStats stats;
+  stats.mem_block = block_alloc_.AllocatedMemory();
+  stats.sync_obj = sync_alloc_.AllocatedMemory();
+  return stats;
+}
+
+}  // namespace __tsan
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_sync.h b/compiler-rt/lib/tsan/rtl-old/tsan_sync.h
new file mode 100644
index 000000000000..fc8fa288a841
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_sync.h
@@ -0,0 +1,153 @@
+//===-- tsan_sync.h ---------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_SYNC_H
+#define TSAN_SYNC_H
+
+#include "sanitizer_common/sanitizer_atomic.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_deadlock_detector_interface.h"
+#include "tsan_defs.h"
+#include "tsan_clock.h"
+#include "tsan_dense_alloc.h"
+
+namespace __tsan {
+
+// These need to match __tsan_mutex_* flags defined in tsan_interface.h.
+// See documentation there as well.
+enum MutexFlags {
+  MutexFlagLinkerInit          = 1 << 0, // __tsan_mutex_linker_init
+  MutexFlagWriteReentrant      = 1 << 1, // __tsan_mutex_write_reentrant
+  MutexFlagReadReentrant       = 1 << 2, // __tsan_mutex_read_reentrant
+  MutexFlagReadLock            = 1 << 3, // __tsan_mutex_read_lock
+  MutexFlagTryLock             = 1 << 4, // __tsan_mutex_try_lock
+  MutexFlagTryLockFailed       = 1 << 5, // __tsan_mutex_try_lock_failed
+  MutexFlagRecursiveLock       = 1 << 6, // __tsan_mutex_recursive_lock
+  MutexFlagRecursiveUnlock     = 1 << 7, // __tsan_mutex_recursive_unlock
+  MutexFlagNotStatic           = 1 << 8, // __tsan_mutex_not_static
+
+  // The following flags are runtime private.
+  // Mutex API misuse was detected, so don't report any more.
+  MutexFlagBroken              = 1 << 30,
+  // We did not intercept pre lock event, so handle it on post lock.
+  MutexFlagDoPreLockOnPostLock = 1 << 29,
+  // Must list all mutex creation flags.
+  MutexCreationFlagMask        = MutexFlagLinkerInit |
+                                 MutexFlagWriteReentrant |
+                                 MutexFlagReadReentrant |
+                                 MutexFlagNotStatic,
+};
+
+// SyncVar is a descriptor of a user synchronization object
+// (mutex or an atomic variable).
+struct SyncVar {
+  SyncVar();
+
+  uptr addr;  // overwritten by DenseSlabAlloc freelist
+  Mutex mtx;
+  u64 uid;  // Globally unique id.
+  StackID creation_stack_id;
+  Tid owner_tid;  // Set only by exclusive owners.
+  u64 last_lock;
+  int recursion;
+  atomic_uint32_t flags;
+  u32 next;  // in MetaMap
+  DDMutex dd;
+  SyncClock read_clock;  // Used for rw mutexes only.
+  // The clock is placed last, so that it is situated on a different cache line
+  // with the mtx. This reduces contention for hot sync objects.
+  SyncClock clock;
+
+  void Init(ThreadState *thr, uptr pc, uptr addr, u64 uid, bool save_stack);
+  void Reset(Processor *proc);
+
+  u64 GetId() const {
+    // 48 lsb is addr, then 14 bits is low part of uid, then 2 zero bits.
+    return GetLsb((u64)addr | (uid << 48), 60);
+  }
+  bool CheckId(u64 uid) const {
+    CHECK_EQ(uid, GetLsb(uid, 14));
+    return GetLsb(this->uid, 14) == uid;
+  }
+  static uptr SplitId(u64 id, u64 *uid) {
+    *uid = id >> 48;
+    return (uptr)GetLsb(id, 48);
+  }
+
+  bool IsFlagSet(u32 f) const {
+    return atomic_load_relaxed(&flags) & f;
+  }
+
+  void SetFlags(u32 f) {
+    atomic_store_relaxed(&flags, atomic_load_relaxed(&flags) | f);
+  }
+
+  void UpdateFlags(u32 flagz) {
+    // Filter out operation flags.
+    if (!(flagz & MutexCreationFlagMask))
+      return;
+    u32 current = atomic_load_relaxed(&flags);
+    if (current & MutexCreationFlagMask)
+      return;
+    // Note: this can be called from MutexPostReadLock which holds only read
+    // lock on the SyncVar.
+    atomic_store_relaxed(&flags, current | (flagz & MutexCreationFlagMask));
+  }
+};
+
+// MetaMap maps app addresses to heap block (MBlock) and sync var (SyncVar)
+// descriptors. It uses 1/2 direct shadow, see tsan_platform.h for the mapping.
+class MetaMap {
+ public:
+  MetaMap();
+
+  void AllocBlock(ThreadState *thr, uptr pc, uptr p, uptr sz);
+  uptr FreeBlock(Processor *proc, uptr p);
+  bool FreeRange(Processor *proc, uptr p, uptr sz);
+  void ResetRange(Processor *proc, uptr p, uptr sz);
+  MBlock* GetBlock(uptr p);
+
+  SyncVar *GetSyncOrCreate(ThreadState *thr, uptr pc, uptr addr,
+                           bool save_stack) {
+    return GetSync(thr, pc, addr, true, save_stack);
+  }
+  SyncVar *GetSyncIfExists(uptr addr) {
+    return GetSync(nullptr, 0, addr, false, false);
+  }
+
+  void MoveMemory(uptr src, uptr dst, uptr sz);
+
+  void OnProcIdle(Processor *proc);
+
+  struct MemoryStats {
+    uptr mem_block;
+    uptr sync_obj;
+  };
+
+  MemoryStats GetMemoryStats() const;
+
+ private:
+  static const u32 kFlagMask  = 3u << 30;
+  static const u32 kFlagBlock = 1u << 30;
+  static const u32 kFlagSync  = 2u << 30;
+  typedef DenseSlabAlloc<MBlock, 1 << 18, 1 << 12, kFlagMask> BlockAlloc;
+  typedef DenseSlabAlloc<SyncVar, 1 << 20, 1 << 10, kFlagMask> SyncAlloc;
+  BlockAlloc block_alloc_;
+  SyncAlloc sync_alloc_;
+  atomic_uint64_t uid_gen_;
+
+  SyncVar *GetSync(ThreadState *thr, uptr pc, uptr addr, bool create,
+                   bool save_stack);
+};
+
+}  // namespace __tsan
+
+#endif  // TSAN_SYNC_H
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_trace.h b/compiler-rt/lib/tsan/rtl-old/tsan_trace.h
new file mode 100644
index 000000000000..ffc8c991ece0
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_trace.h
@@ -0,0 +1,252 @@
+//===-- tsan_trace.h --------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_TRACE_H
+#define TSAN_TRACE_H
+
+#include "tsan_defs.h"
+#include "tsan_ilist.h"
+#include "tsan_mutexset.h"
+#include "tsan_stack_trace.h"
+
+namespace __tsan {
+
+const int kTracePartSizeBits = 13;
+const int kTracePartSize = 1 << kTracePartSizeBits;
+const int kTraceParts = 2 * 1024 * 1024 / kTracePartSize;
+const int kTraceSize = kTracePartSize * kTraceParts;
+
+// Must fit into 3 bits.
+enum EventType {
+  EventTypeMop,
+  EventTypeFuncEnter,
+  EventTypeFuncExit,
+  EventTypeLock,
+  EventTypeUnlock,
+  EventTypeRLock,
+  EventTypeRUnlock
+};
+
+// Represents a thread event (from most significant bit):
+// u64 typ  : 3;   // EventType.
+// u64 addr : 61;  // Associated pc.
+typedef u64 Event;
+
+const uptr kEventPCBits = 61;
+
+struct TraceHeader {
+#if !SANITIZER_GO
+  BufferedStackTrace stack0;  // Start stack for the trace.
+#else
+  VarSizeStackTrace stack0;
+#endif
+  u64        epoch0;  // Start epoch for the trace.
+  MutexSet   mset0;
+
+  TraceHeader() : stack0(), epoch0() {}
+};
+
+struct Trace {
+  Mutex mtx;
+#if !SANITIZER_GO
+  // Must be last to catch overflow as paging fault.
+  // Go shadow stack is dynamically allocated.
+  uptr shadow_stack[kShadowStackSize];
+#endif
+  // Must be the last field, because we unmap the unused part in
+  // CreateThreadContext.
+  TraceHeader headers[kTraceParts];
+
+  Trace() : mtx(MutexTypeTrace) {}
+};
+
+namespace v3 {
+
+enum class EventType : u64 {
+  kAccessExt,
+  kAccessRange,
+  kLock,
+  kRLock,
+  kUnlock,
+  kTime,
+};
+
+// "Base" type for all events for type dispatch.
+struct Event {
+  // We use variable-length type encoding to give more bits to some event
+  // types that need them. If is_access is set, this is EventAccess.
+  // Otherwise, if is_func is set, this is EventFunc.
+  // Otherwise type denotes the type.
+  u64 is_access : 1;
+  u64 is_func : 1;
+  EventType type : 3;
+  u64 _ : 59;
+};
+static_assert(sizeof(Event) == 8, "bad Event size");
+
+// Nop event used as padding and does not affect state during replay.
+static constexpr Event NopEvent = {1, 0, EventType::kAccessExt, 0};
+
+// Compressed memory access can represent only some events with PCs
+// close enough to each other. Otherwise we fall back to EventAccessExt.
+struct EventAccess {
+  static constexpr uptr kPCBits = 15;
+  static_assert(kPCBits + kCompressedAddrBits + 5 == 64,
+                "unused bits in EventAccess");
+
+  u64 is_access : 1;  // = 1
+  u64 is_read : 1;
+  u64 is_atomic : 1;
+  u64 size_log : 2;
+  u64 pc_delta : kPCBits;  // signed delta from the previous memory access PC
+  u64 addr : kCompressedAddrBits;
+};
+static_assert(sizeof(EventAccess) == 8, "bad EventAccess size");
+
+// Function entry (pc != 0) or exit (pc == 0).
+struct EventFunc {
+  u64 is_access : 1;  // = 0
+  u64 is_func : 1;    // = 1
+  u64 pc : 62;
+};
+static_assert(sizeof(EventFunc) == 8, "bad EventFunc size");
+
+// Extended memory access with full PC.
+struct EventAccessExt {
+  // Note: precisely specifying the unused parts of the bitfield is critical for
+  // performance. If we don't specify them, compiler will generate code to load
+  // the old value and shuffle it to extract the unused bits to apply to the new
+  // value. If we specify the unused part and store 0 in there, all that
+  // unnecessary code goes away (store of the 0 const is combined with other
+  // constant parts).
+  static constexpr uptr kUnusedBits = 11;
+  static_assert(kCompressedAddrBits + kUnusedBits + 9 == 64,
+                "unused bits in EventAccessExt");
+
+  u64 is_access : 1;   // = 0
+  u64 is_func : 1;     // = 0
+  EventType type : 3;  // = EventType::kAccessExt
+  u64 is_read : 1;
+  u64 is_atomic : 1;
+  u64 size_log : 2;
+  u64 _ : kUnusedBits;
+  u64 addr : kCompressedAddrBits;
+  u64 pc;
+};
+static_assert(sizeof(EventAccessExt) == 16, "bad EventAccessExt size");
+
+// Access to a memory range.
+struct EventAccessRange {
+  static constexpr uptr kSizeLoBits = 13;
+  static_assert(kCompressedAddrBits + kSizeLoBits + 7 == 64,
+                "unused bits in EventAccessRange");
+
+  u64 is_access : 1;   // = 0
+  u64 is_func : 1;     // = 0
+  EventType type : 3;  // = EventType::kAccessRange
+  u64 is_read : 1;
+  u64 is_free : 1;
+  u64 size_lo : kSizeLoBits;
+  u64 pc : kCompressedAddrBits;
+  u64 addr : kCompressedAddrBits;
+  u64 size_hi : 64 - kCompressedAddrBits;
+};
+static_assert(sizeof(EventAccessRange) == 16, "bad EventAccessRange size");
+
+// Mutex lock.
+struct EventLock {
+  static constexpr uptr kStackIDLoBits = 15;
+  static constexpr uptr kStackIDHiBits =
+      sizeof(StackID) * kByteBits - kStackIDLoBits;
+  static constexpr uptr kUnusedBits = 3;
+  static_assert(kCompressedAddrBits + kStackIDLoBits + 5 == 64,
+                "unused bits in EventLock");
+  static_assert(kCompressedAddrBits + kStackIDHiBits + kUnusedBits == 64,
+                "unused bits in EventLock");
+
+  u64 is_access : 1;   // = 0
+  u64 is_func : 1;     // = 0
+  EventType type : 3;  // = EventType::kLock or EventType::kRLock
+  u64 pc : kCompressedAddrBits;
+  u64 stack_lo : kStackIDLoBits;
+  u64 stack_hi : sizeof(StackID) * kByteBits - kStackIDLoBits;
+  u64 _ : kUnusedBits;
+  u64 addr : kCompressedAddrBits;
+};
+static_assert(sizeof(EventLock) == 16, "bad EventLock size");
+
+// Mutex unlock.
+struct EventUnlock {
+  static constexpr uptr kUnusedBits = 15;
+  static_assert(kCompressedAddrBits + kUnusedBits + 5 == 64,
+                "unused bits in EventUnlock");
+
+  u64 is_access : 1;   // = 0
+  u64 is_func : 1;     // = 0
+  EventType type : 3;  // = EventType::kUnlock
+  u64 _ : kUnusedBits;
+  u64 addr : kCompressedAddrBits;
+};
+static_assert(sizeof(EventUnlock) == 8, "bad EventUnlock size");
+
+// Time change event.
+struct EventTime {
+  static constexpr uptr kUnusedBits = 37;
+  static_assert(kUnusedBits + sizeof(Sid) * kByteBits + kEpochBits + 5 == 64,
+                "unused bits in EventTime");
+
+  u64 is_access : 1;   // = 0
+  u64 is_func : 1;     // = 0
+  EventType type : 3;  // = EventType::kTime
+  u64 sid : sizeof(Sid) * kByteBits;
+  u64 epoch : kEpochBits;
+  u64 _ : kUnusedBits;
+};
+static_assert(sizeof(EventTime) == 8, "bad EventTime size");
+
+struct Trace;
+
+struct TraceHeader {
+  Trace* trace = nullptr;  // back-pointer to Trace containing this part
+  INode trace_parts;       // in Trace::parts
+};
+
+struct TracePart : TraceHeader {
+  // There are a lot of goroutines in Go, so we use smaller parts.
+  static constexpr uptr kByteSize = (SANITIZER_GO ? 128 : 256) << 10;
+  static constexpr uptr kSize =
+      (kByteSize - sizeof(TraceHeader)) / sizeof(Event);
+  // TraceAcquire does a fast event pointer overflow check by comparing
+  // pointer into TracePart::events with kAlignment mask. Since TracePart's
+  // are allocated page-aligned, this check detects end of the array
+  // (it also have false positives in the middle that are filtered separately).
+  // This also requires events to be the last field.
+  static constexpr uptr kAlignment = 0xff0;
+  Event events[kSize];
+
+  TracePart() {}
+};
+static_assert(sizeof(TracePart) == TracePart::kByteSize, "bad TracePart size");
+
+struct Trace {
+  Mutex mtx;
+  IList<TraceHeader, &TraceHeader::trace_parts, TracePart> parts;
+  Event* final_pos =
+      nullptr;  // final position in the last part for finished threads
+
+  Trace() : mtx(MutexTypeTrace) {}
+};
+
+}  // namespace v3
+
+}  // namespace __tsan
+
+#endif  // TSAN_TRACE_H
diff --git a/compiler-rt/lib/tsan/rtl/tsan_update_shadow_word.inc b/compiler-rt/lib/tsan/rtl-old/tsan_update_shadow_word.inc
index a58ef0f17efa..a58ef0f17efa 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_update_shadow_word.inc
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_update_shadow_word.inc
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_vector_clock.cpp b/compiler-rt/lib/tsan/rtl-old/tsan_vector_clock.cpp
new file mode 100644
index 000000000000..278298565d3f
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_vector_clock.cpp
@@ -0,0 +1,126 @@
+//===-- tsan_vector_clock.cpp ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#include "tsan_vector_clock.h"
+
+#include "sanitizer_common/sanitizer_placement_new.h"
+#include "tsan_mman.h"
+
+namespace __tsan {
+
+#if TSAN_VECTORIZE
+const uptr kVectorClockSize = kThreadSlotCount * sizeof(Epoch) / sizeof(m128);
+#endif
+
+VectorClock::VectorClock() { Reset(); }
+
+void VectorClock::Reset() {
+#if !TSAN_VECTORIZE
+  for (uptr i = 0; i < kThreadSlotCount; i++)
+    clk_[i] = kEpochZero;
+#else
+  m128 z = _mm_setzero_si128();
+  m128* vclk = reinterpret_cast<m128*>(clk_);
+  for (uptr i = 0; i < kVectorClockSize; i++) _mm_store_si128(&vclk[i], z);
+#endif
+}
+
+void VectorClock::Acquire(const VectorClock* src) {
+  if (!src)
+    return;
+#if !TSAN_VECTORIZE
+  for (uptr i = 0; i < kThreadSlotCount; i++)
+    clk_[i] = max(clk_[i], src->clk_[i]);
+#else
+  m128* __restrict vdst = reinterpret_cast<m128*>(clk_);
+  m128 const* __restrict vsrc = reinterpret_cast<m128 const*>(src->clk_);
+  for (uptr i = 0; i < kVectorClockSize; i++) {
+    m128 s = _mm_load_si128(&vsrc[i]);
+    m128 d = _mm_load_si128(&vdst[i]);
+    m128 m = _mm_max_epu16(s, d);
+    _mm_store_si128(&vdst[i], m);
+  }
+#endif
+}
+
+static VectorClock* AllocClock(VectorClock** dstp) {
+  if (UNLIKELY(!*dstp))
+    *dstp = New<VectorClock>();
+  return *dstp;
+}
+
+void VectorClock::Release(VectorClock** dstp) const {
+  VectorClock* dst = AllocClock(dstp);
+  dst->Acquire(this);
+}
+
+void VectorClock::ReleaseStore(VectorClock** dstp) const {
+  VectorClock* dst = AllocClock(dstp);
+  *dst = *this;
+}
+
+VectorClock& VectorClock::operator=(const VectorClock& other) {
+#if !TSAN_VECTORIZE
+  for (uptr i = 0; i < kThreadSlotCount; i++)
+    clk_[i] = other.clk_[i];
+#else
+  m128* __restrict vdst = reinterpret_cast<m128*>(clk_);
+  m128 const* __restrict vsrc = reinterpret_cast<m128 const*>(other.clk_);
+  for (uptr i = 0; i < kVectorClockSize; i++) {
+    m128 s = _mm_load_si128(&vsrc[i]);
+    _mm_store_si128(&vdst[i], s);
+  }
+#endif
+  return *this;
+}
+
+void VectorClock::ReleaseStoreAcquire(VectorClock** dstp) {
+  VectorClock* dst = AllocClock(dstp);
+#if !TSAN_VECTORIZE
+  for (uptr i = 0; i < kThreadSlotCount; i++) {
+    Epoch tmp = dst->clk_[i];
+    dst->clk_[i] = clk_[i];
+    clk_[i] = max(clk_[i], tmp);
+  }
+#else
+  m128* __restrict vdst = reinterpret_cast<m128*>(dst->clk_);
+  m128* __restrict vclk = reinterpret_cast<m128*>(clk_);
+  for (uptr i = 0; i < kVectorClockSize; i++) {
+    m128 t = _mm_load_si128(&vdst[i]);
+    m128 c = _mm_load_si128(&vclk[i]);
+    m128 m = _mm_max_epu16(c, t);
+    _mm_store_si128(&vdst[i], c);
+    _mm_store_si128(&vclk[i], m);
+  }
+#endif
+}
+
+void VectorClock::ReleaseAcquire(VectorClock** dstp) {
+  VectorClock* dst = AllocClock(dstp);
+#if !TSAN_VECTORIZE
+  for (uptr i = 0; i < kThreadSlotCount; i++) {
+    dst->clk_[i] = max(dst->clk_[i], clk_[i]);
+    clk_[i] = dst->clk_[i];
+  }
+#else
+  m128* __restrict vdst = reinterpret_cast<m128*>(dst->clk_);
+  m128* __restrict vclk = reinterpret_cast<m128*>(clk_);
+  for (uptr i = 0; i < kVectorClockSize; i++) {
+    m128 c = _mm_load_si128(&vclk[i]);
+    m128 d = _mm_load_si128(&vdst[i]);
+    m128 m = _mm_max_epu16(c, d);
+    _mm_store_si128(&vdst[i], m);
+    _mm_store_si128(&vclk[i], m);
+  }
+#endif
+}
+
+}  // namespace __tsan
diff --git a/compiler-rt/lib/tsan/rtl-old/tsan_vector_clock.h b/compiler-rt/lib/tsan/rtl-old/tsan_vector_clock.h
new file mode 100644
index 000000000000..63b206302190
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl-old/tsan_vector_clock.h
@@ -0,0 +1,51 @@
+//===-- tsan_vector_clock.h -------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_VECTOR_CLOCK_H
+#define TSAN_VECTOR_CLOCK_H
+
+#include "tsan_defs.h"
+
+namespace __tsan {
+
+// Fixed-size vector clock, used both for threads and sync objects.
+class VectorClock {
+ public:
+  VectorClock();
+
+  Epoch Get(Sid sid) const;
+  void Set(Sid sid, Epoch v);
+
+  void Reset();
+  void Acquire(const VectorClock* src);
+  void Release(VectorClock** dstp) const;
+  void ReleaseStore(VectorClock** dstp) const;
+  void ReleaseStoreAcquire(VectorClock** dstp);
+  void ReleaseAcquire(VectorClock** dstp);
+
+  VectorClock& operator=(const VectorClock& other);
+
+ private:
+  Epoch clk_[kThreadSlotCount] VECTOR_ALIGNED;
+};
+
+ALWAYS_INLINE Epoch VectorClock::Get(Sid sid) const {
+  return clk_[static_cast<u8>(sid)];
+}
+
+ALWAYS_INLINE void VectorClock::Set(Sid sid, Epoch v) {
+  DCHECK_GE(v, clk_[static_cast<u8>(sid)]);
+  clk_[static_cast<u8>(sid)] = v;
+}
+
+}  // namespace __tsan
+
+#endif  // TSAN_VECTOR_CLOCK_H
diff --git a/compiler-rt/lib/tsan/rtl/tsan_debugging.cpp b/compiler-rt/lib/tsan/rtl/tsan_debugging.cpp
index 1d3c3849a446..1e61c31c5a97 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_debugging.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_debugging.cpp
@@ -157,7 +157,7 @@ int __tsan_get_report_mutex(void *report, uptr idx, uptr *mutex_id, void **addr,
   ReportMutex *mutex = rep->mutexes[idx];
   *mutex_id = mutex->id;
   *addr = (void *)mutex->addr;
-  *destroyed = mutex->destroyed;
+  *destroyed = false;
   if (mutex->stack) CopyTrace(mutex->stack->frames, trace, trace_size);
   return 1;
 }
diff --git a/compiler-rt/lib/tsan/rtl/tsan_defs.h b/compiler-rt/lib/tsan/rtl/tsan_defs.h
index 4712c2be1813..2e13e0e5486b 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_defs.h
+++ b/compiler-rt/lib/tsan/rtl/tsan_defs.h
@@ -63,41 +63,14 @@ enum class Epoch : u16 {};
 constexpr uptr kEpochBits = 14;
 constexpr Epoch kEpochZero = static_cast<Epoch>(0);
 constexpr Epoch kEpochOver = static_cast<Epoch>(1 << kEpochBits);
+constexpr Epoch kEpochLast = static_cast<Epoch>((1 << kEpochBits) - 1);
 
-const int kClkBits = 42;
-const unsigned kMaxTidReuse = (1 << (64 - kClkBits)) - 1;
-
-struct ClockElem {
-  u64 epoch  : kClkBits;
-  u64 reused : 64 - kClkBits;  // tid reuse count
-};
-
-struct ClockBlock {
-  static const uptr kSize = 512;
-  static const uptr kTableSize = kSize / sizeof(u32);
-  static const uptr kClockCount = kSize / sizeof(ClockElem);
-  static const uptr kRefIdx = kTableSize - 1;
-  static const uptr kBlockIdx = kTableSize - 2;
-
-  union {
-    u32       table[kTableSize];
-    ClockElem clock[kClockCount];
-  };
+inline Epoch EpochInc(Epoch epoch) {
+  return static_cast<Epoch>(static_cast<u16>(epoch) + 1);
+}
 
-  ClockBlock() {
-  }
-};
+inline bool EpochOverflow(Epoch epoch) { return epoch == kEpochOver; }
 
-const int kTidBits = 13;
-// Reduce kMaxTid by kClockCount because one slot in ClockBlock table is
-// occupied by reference counter, so total number of elements we can store
-// in SyncClock is kClockCount * (kTableSize - 1).
-const unsigned kMaxTid = (1 << kTidBits) - ClockBlock::kClockCount;
-#if !SANITIZER_GO
-const unsigned kMaxTidInClock = kMaxTid * 2;  // This includes msb 'freed' bit.
-#else
-const unsigned kMaxTidInClock = kMaxTid;  // Go does not track freed memory.
-#endif
 const uptr kShadowStackSize = 64 * 1024;
 
 // Count of shadow values in a shadow cell.
@@ -107,7 +80,7 @@ const uptr kShadowCnt = 4;
 const uptr kShadowCell = 8;
 
 // Single shadow value.
-typedef u64 RawShadow;
+enum class RawShadow : u32 {};
 const uptr kShadowSize = sizeof(RawShadow);
 
 // Shadow memory is kShadowMultiplier times larger than user memory.
@@ -184,10 +157,13 @@ MD5Hash md5_hash(const void *data, uptr size);
 struct Processor;
 struct ThreadState;
 class ThreadContext;
+struct TidSlot;
 struct Context;
 struct ReportStack;
 class ReportDesc;
 class RegionAlloc;
+struct Trace;
+struct TracePart;
 
 typedef uptr AccessType;
 
@@ -198,6 +174,8 @@ enum : AccessType {
   kAccessVptr = 1 << 2,  // read or write of an object virtual table pointer
   kAccessFree = 1 << 3,  // synthetic memory access during memory freeing
   kAccessExternalPC = 1 << 4,  // access PC can have kExternalPCBit set
+  kAccessCheckOnly = 1 << 5,   // check for races, but don't store
+  kAccessNoRodata = 1 << 6,    // don't check for .rodata marker
 };
 
 // Descriptor of user's memory block.
@@ -219,9 +197,8 @@ enum ExternalTag : uptr {
   // as 16-bit values, see tsan_defs.h.
 };
 
-enum MutexType {
-  MutexTypeTrace = MutexLastCommon,
-  MutexTypeReport,
+enum {
+  MutexTypeReport = MutexLastCommon,
   MutexTypeSyncVar,
   MutexTypeAnnotations,
   MutexTypeAtExit,
@@ -229,6 +206,9 @@ enum MutexType {
   MutexTypeRacy,
   MutexTypeGlobalProc,
   MutexTypeInternalAlloc,
+  MutexTypeTrace,
+  MutexTypeSlot,
+  MutexTypeSlots,
 };
 
 }  // namespace __tsan
diff --git a/compiler-rt/lib/tsan/rtl/tsan_dense_alloc.h b/compiler-rt/lib/tsan/rtl/tsan_dense_alloc.h
index 9e15f74a0615..7a39a39d51de 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_dense_alloc.h
+++ b/compiler-rt/lib/tsan/rtl/tsan_dense_alloc.h
@@ -104,6 +104,15 @@ class DenseSlabAlloc {
     return atomic_load_relaxed(&fillpos_) * kL2Size * sizeof(T);
   }
 
+  template <typename Func>
+  void ForEach(Func func) {
+    SpinMutexLock lock(&mtx_);
+    uptr fillpos = atomic_load_relaxed(&fillpos_);
+    for (uptr l1 = 0; l1 < fillpos; l1++) {
+      for (IndexT l2 = l1 == 0 ? 1 : 0; l2 < kL2Size; l2++) func(&map_[l1][l2]);
+    }
+  }
+
  private:
   T *map_[kL1Size];
   SpinMutex mtx_;
diff --git a/compiler-rt/lib/tsan/rtl/tsan_fd.cpp b/compiler-rt/lib/tsan/rtl/tsan_fd.cpp
index 255ffa8daf76..9a6400c2e9f9 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_fd.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_fd.cpp
@@ -11,9 +11,12 @@
 //===----------------------------------------------------------------------===//
 
 #include "tsan_fd.h"
-#include "tsan_rtl.h"
+
 #include <sanitizer_common/sanitizer_atomic.h>
 
+#include "tsan_interceptors.h"
+#include "tsan_rtl.h"
+
 namespace __tsan {
 
 const int kTableSizeL1 = 1024;
@@ -192,19 +195,21 @@ void FdClose(ThreadState *thr, uptr pc, int fd, bool write) {
   if (bogusfd(fd))
     return;
   FdDesc *d = fddesc(thr, pc, fd);
-  if (write) {
-    // To catch races between fd usage and close.
-    MemoryAccess(thr, pc, (uptr)d, 8, kAccessWrite);
-  } else {
-    // This path is used only by dup2/dup3 calls.
-    // We do read instead of write because there is a number of legitimate
-    // cases where write would lead to false positives:
-    // 1. Some software dups a closed pipe in place of a socket before closing
-    //    the socket (to prevent races actually).
-    // 2. Some daemons dup /dev/null in place of stdin/stdout.
-    // On the other hand we have not seen cases when write here catches real
-    // bugs.
-    MemoryAccess(thr, pc, (uptr)d, 8, kAccessRead);
+  if (!MustIgnoreInterceptor(thr)) {
+    if (write) {
+      // To catch races between fd usage and close.
+      MemoryAccess(thr, pc, (uptr)d, 8, kAccessWrite);
+    } else {
+      // This path is used only by dup2/dup3 calls.
+      // We do read instead of write because there is a number of legitimate
+      // cases where write would lead to false positives:
+      // 1. Some software dups a closed pipe in place of a socket before closing
+      //    the socket (to prevent races actually).
+      // 2. Some daemons dup /dev/null in place of stdin/stdout.
+      // On the other hand we have not seen cases when write here catches real
+      // bugs.
+      MemoryAccess(thr, pc, (uptr)d, 8, kAccessRead);
+    }
   }
   // We need to clear it, because if we do not intercept any call out there
   // that creates fd, we will hit false postives.
diff --git a/compiler-rt/lib/tsan/rtl/tsan_flags.cpp b/compiler-rt/lib/tsan/rtl/tsan_flags.cpp
index ee89862d17bd..54bed9f9a6be 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_flags.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_flags.cpp
@@ -110,12 +110,6 @@ void InitializeFlags(Flags *f, const char *env, const char *env_option_name) {
 
   if (common_flags()->help) parser.PrintFlagDescriptions();
 
-  if (f->history_size < 0 || f->history_size > 7) {
-    Printf("ThreadSanitizer: incorrect value for history_size"
-           " (must be [0..7])\n");
-    Die();
-  }
-
   if (f->io_sync < 0 || f->io_sync > 2) {
     Printf("ThreadSanitizer: incorrect value for io_sync"
            " (must be [0..2])\n");
diff --git a/compiler-rt/lib/tsan/rtl/tsan_flags.inc b/compiler-rt/lib/tsan/rtl/tsan_flags.inc
index 7954a4307fa1..b1691452d022 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_flags.inc
+++ b/compiler-rt/lib/tsan/rtl/tsan_flags.inc
@@ -43,6 +43,9 @@ TSAN_FLAG(
     bool, force_seq_cst_atomics, false,
     "If set, all atomics are effectively sequentially consistent (seq_cst), "
     "regardless of what user actually specified.")
+TSAN_FLAG(bool, force_background_thread, false,
+          "If set, eagerly launch a background thread for memory reclamation "
+          "instead of waiting for a user call to pthread_create.")
 TSAN_FLAG(bool, halt_on_error, false, "Exit after first reported error.")
 TSAN_FLAG(int, atexit_sleep_ms, 1000,
           "Sleep in main thread before exiting for that many ms "
@@ -59,14 +62,10 @@ TSAN_FLAG(bool, stop_on_start, false,
           "Stops on start until __tsan_resume() is called (for debugging).")
 TSAN_FLAG(bool, running_on_valgrind, false,
           "Controls whether RunningOnValgrind() returns true or false.")
-// There are a lot of goroutines in Go, so we use smaller history.
 TSAN_FLAG(
-    int, history_size, SANITIZER_GO ? 1 : 3,
-    "Per-thread history size, controls how many previous memory accesses "
-    "are remembered per thread.  Possible values are [0..7]. "
-    "history_size=0 amounts to 32K memory accesses.  Each next value doubles "
-    "the amount of memory accesses, up to history_size=7 that amounts to "
-    "4M memory accesses.  The default value is 2 (128K memory accesses).")
+    uptr, history_size, 0,
+    "Per-thread history size,"
+    " controls how many extra previous memory accesses are remembered per thread.")
 TSAN_FLAG(int, io_sync, 1,
           "Controls level of synchronization implied by IO operations. "
           "0 - no synchronization "
diff --git a/compiler-rt/lib/tsan/rtl/tsan_interceptors.h b/compiler-rt/lib/tsan/rtl/tsan_interceptors.h
index 61dbb81ffec4..88a54b554421 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_interceptors.h
+++ b/compiler-rt/lib/tsan/rtl/tsan_interceptors.h
@@ -36,6 +36,10 @@ inline bool in_symbolizer() {
 }
 #endif
 
+inline bool MustIgnoreInterceptor(ThreadState *thr) {
+  return !thr->is_inited || thr->ignore_interceptors || thr->in_ignored_lib;
+}
+
 }  // namespace __tsan
 
 #define SCOPED_INTERCEPTOR_RAW(func, ...)            \
@@ -60,10 +64,10 @@ inline bool in_symbolizer() {
 #  define CHECK_REAL_FUNC(func) DCHECK(REAL(func))
 #endif
 
-#define SCOPED_TSAN_INTERCEPTOR(func, ...)                                \
-  SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__);                              \
-  CHECK_REAL_FUNC(func);                                                  \
-  if (!thr->is_inited || thr->ignore_interceptors || thr->in_ignored_lib) \
+#define SCOPED_TSAN_INTERCEPTOR(func, ...)   \
+  SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \
+  CHECK_REAL_FUNC(func);                     \
+  if (MustIgnoreInterceptor(thr))            \
     return REAL(func)(__VA_ARGS__);
 
 #define SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START() \
diff --git a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
index cf3dc90d96a1..c4f43d8171ab 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
@@ -1681,11 +1681,10 @@ TSAN_INTERCEPTOR(int, eventfd, unsigned initval, int flags) {
 
 #if SANITIZER_LINUX
 TSAN_INTERCEPTOR(int, signalfd, int fd, void *mask, int flags) {
-  SCOPED_TSAN_INTERCEPTOR(signalfd, fd, mask, flags);
-  if (fd >= 0)
-    FdClose(thr, pc, fd);
+  SCOPED_INTERCEPTOR_RAW(signalfd, fd, mask, flags);
+  FdClose(thr, pc, fd);
   fd = REAL(signalfd)(fd, mask, flags);
-  if (fd >= 0)
+  if (!MustIgnoreInterceptor(thr))
     FdSignalCreate(thr, pc, fd);
   return fd;
 }
@@ -1762,17 +1761,15 @@ TSAN_INTERCEPTOR(int, listen, int fd, int backlog) {
 }
 
 TSAN_INTERCEPTOR(int, close, int fd) {
-  SCOPED_TSAN_INTERCEPTOR(close, fd);
-  if (fd >= 0)
-    FdClose(thr, pc, fd);
+  SCOPED_INTERCEPTOR_RAW(close, fd);
+  FdClose(thr, pc, fd);
   return REAL(close)(fd);
 }
 
 #if SANITIZER_LINUX
 TSAN_INTERCEPTOR(int, __close, int fd) {
-  SCOPED_TSAN_INTERCEPTOR(__close, fd);
-  if (fd >= 0)
-    FdClose(thr, pc, fd);
+  SCOPED_INTERCEPTOR_RAW(__close, fd);
+  FdClose(thr, pc, fd);
   return REAL(__close)(fd);
 }
 #define TSAN_MAYBE_INTERCEPT___CLOSE TSAN_INTERCEPT(__close)
@@ -1783,13 +1780,10 @@ TSAN_INTERCEPTOR(int, __close, int fd) {
 // glibc guts
 #if SANITIZER_LINUX && !SANITIZER_ANDROID
 TSAN_INTERCEPTOR(void, __res_iclose, void *state, bool free_addr) {
-  SCOPED_TSAN_INTERCEPTOR(__res_iclose, state, free_addr);
+  SCOPED_INTERCEPTOR_RAW(__res_iclose, state, free_addr);
   int fds[64];
   int cnt = ExtractResolvFDs(state, fds, ARRAY_SIZE(fds));
-  for (int i = 0; i < cnt; i++) {
-    if (fds[i] > 0)
-      FdClose(thr, pc, fds[i]);
-  }
+  for (int i = 0; i < cnt; i++) FdClose(thr, pc, fds[i]);
   REAL(__res_iclose)(state, free_addr);
 }
 #define TSAN_MAYBE_INTERCEPT___RES_ICLOSE TSAN_INTERCEPT(__res_iclose)
@@ -1870,7 +1864,7 @@ TSAN_INTERCEPTOR(int, rmdir, char *path) {
 }
 
 TSAN_INTERCEPTOR(int, closedir, void *dirp) {
-  SCOPED_TSAN_INTERCEPTOR(closedir, dirp);
+  SCOPED_INTERCEPTOR_RAW(closedir, dirp);
   if (dirp) {
     int fd = dirfd(dirp);
     FdClose(thr, pc, fd);
@@ -1981,6 +1975,7 @@ static void ReportErrnoSpoiling(ThreadState *thr, uptr pc) {
 static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire,
                                   int sig, __sanitizer_siginfo *info,
                                   void *uctx) {
+  CHECK(thr->slot);
   __sanitizer_sigaction *sigactions = interceptor_ctx()->sigactions;
   if (acquire)
     Acquire(thr, 0, (uptr)&sigactions[sig]);
@@ -2268,7 +2263,7 @@ struct dl_iterate_phdr_data {
 };
 
 static bool IsAppNotRodata(uptr addr) {
-  return IsAppMem(addr) && *MemToShadow(addr) != kShadowRodata;
+  return IsAppMem(addr) && *MemToShadow(addr) != Shadow::kRodata;
 }
 
 static int dl_iterate_phdr_cb(__sanitizer_dl_phdr_info *info, SIZE_T size,
@@ -2374,7 +2369,7 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc,
 #define COMMON_INTERCEPTOR_FILE_CLOSE(ctx, file) \
   if (file) {                                    \
     int fd = fileno_unlocked(file);              \
-    if (fd >= 0) FdClose(thr, pc, fd);           \
+    FdClose(thr, pc, fd);                        \
   }
 
 #define COMMON_INTERCEPTOR_DLOPEN(filename, flag) \
@@ -2581,7 +2576,7 @@ static USED void syscall_release(uptr pc, uptr addr) {
 }
 
 static void syscall_fd_close(uptr pc, int fd) {
-  TSAN_SYSCALL();
+  auto *thr = cur_thread();
   FdClose(thr, pc, fd);
 }
 
diff --git a/compiler-rt/lib/tsan/rtl/tsan_interface.cpp b/compiler-rt/lib/tsan/rtl/tsan_interface.cpp
index 048715185151..e6c4bf2e60a7 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_interface.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_interface.cpp
@@ -26,20 +26,6 @@ void __tsan_flush_memory() {
   FlushShadowMemory();
 }
 
-void __tsan_read16(void *addr) {
-  uptr pc = CALLERPC;
-  ThreadState *thr = cur_thread();
-  MemoryAccess(thr, pc, (uptr)addr, 8, kAccessRead);
-  MemoryAccess(thr, pc, (uptr)addr + 8, 8, kAccessRead);
-}
-
-void __tsan_write16(void *addr) {
-  uptr pc = CALLERPC;
-  ThreadState *thr = cur_thread();
-  MemoryAccess(thr, pc, (uptr)addr, 8, kAccessWrite);
-  MemoryAccess(thr, pc, (uptr)addr + 8, 8, kAccessWrite);
-}
-
 void __tsan_read16_pc(void *addr, void *pc) {
   uptr pc_no_pac = STRIP_PAC_PC(pc);
   ThreadState *thr = cur_thread();
diff --git a/compiler-rt/lib/tsan/rtl/tsan_interface.inc b/compiler-rt/lib/tsan/rtl/tsan_interface.inc
index 0031800e851f..b0a424ff9c25 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_interface.inc
+++ b/compiler-rt/lib/tsan/rtl/tsan_interface.inc
@@ -34,6 +34,10 @@ void __tsan_read8(void *addr) {
   MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, kAccessRead);
 }
 
+void __tsan_read16(void *addr) {
+  MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr, kAccessRead);
+}
+
 void __tsan_write1(void *addr) {
   MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 1, kAccessWrite);
 }
@@ -50,6 +54,10 @@ void __tsan_write8(void *addr) {
   MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, kAccessWrite);
 }
 
+void __tsan_write16(void *addr) {
+  MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr, kAccessWrite);
+}
+
 void __tsan_read1_pc(void *addr, void *pc) {
   MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 1, kAccessRead | kAccessExternalPC);
 }
diff --git a/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp b/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp
index 24ba3bb1f65d..f794a2fcdd0d 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp
@@ -235,8 +235,9 @@ static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a, morder mo) {
   T v = NoTsanAtomicLoad(a, mo);
   SyncVar *s = ctx->metamap.GetSyncIfExists((uptr)a);
   if (s) {
-    ReadLock l(&s->mtx);
-    AcquireImpl(thr, pc, &s->clock);
+    SlotLocker locker(thr);
+    ReadLock lock(&s->mtx);
+    thr->clock.Acquire(s->clock);
     // Re-read under sync mutex because we need a consistent snapshot
     // of the value and the clock we acquire.
     v = NoTsanAtomicLoad(a, mo);
@@ -270,14 +271,14 @@ static void AtomicStore(ThreadState *thr, uptr pc, volatile T *a, T v,
     NoTsanAtomicStore(a, v, mo);
     return;
   }
-  __sync_synchronize();
-  SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false);
-  Lock l(&s->mtx);
-  thr->fast_state.IncrementEpoch();
-  // Can't increment epoch w/o writing to the trace as well.
-  TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
-  ReleaseStoreImpl(thr, pc, &s->clock);
-  NoTsanAtomicStore(a, v, mo);
+  SlotLocker locker(thr);
+  {
+    auto s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false);
+    Lock lock(&s->mtx);
+    thr->clock.ReleaseStore(&s->clock);
+    NoTsanAtomicStore(a, v, mo);
+  }
+  IncrementEpoch(thr);
 }
 
 template <typename T, T (*F)(volatile T *v, T op)>
@@ -285,18 +286,21 @@ static T AtomicRMW(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) {
   MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), kAccessWrite | kAccessAtomic);
   if (LIKELY(mo == mo_relaxed))
     return F(a, v);
-  SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false);
-  Lock l(&s->mtx);
-  thr->fast_state.IncrementEpoch();
-  // Can't increment epoch w/o writing to the trace as well.
-  TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
-  if (IsAcqRelOrder(mo))
-    AcquireReleaseImpl(thr, pc, &s->clock);
-  else if (IsReleaseOrder(mo))
-    ReleaseImpl(thr, pc, &s->clock);
-  else if (IsAcquireOrder(mo))
-    AcquireImpl(thr, pc, &s->clock);
-  return F(a, v);
+  SlotLocker locker(thr);
+  {
+    auto s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false);
+    RWLock lock(&s->mtx, IsReleaseOrder(mo));
+    if (IsAcqRelOrder(mo))
+      thr->clock.ReleaseAcquire(&s->clock);
+    else if (IsReleaseOrder(mo))
+      thr->clock.Release(&s->clock);
+    else if (IsAcquireOrder(mo))
+      thr->clock.Acquire(s->clock);
+    v = F(a, v);
+  }
+  if (IsReleaseOrder(mo))
+    IncrementEpoch(thr);
+  return v;
 }
 
 template<typename T>
@@ -416,27 +420,28 @@ static bool AtomicCAS(ThreadState *thr, uptr pc, volatile T *a, T *c, T v,
     *c = pr;
     return false;
   }
-
+  SlotLocker locker(thr);
   bool release = IsReleaseOrder(mo);
-  SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false);
-  RWLock l(&s->mtx, release);
-  T cc = *c;
-  T pr = func_cas(a, cc, v);
-  bool success = pr == cc;
-  if (!success) {
-    *c = pr;
-    mo = fmo;
+  bool success;
+  {
+    auto s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false);
+    RWLock lock(&s->mtx, release);
+    T cc = *c;
+    T pr = func_cas(a, cc, v);
+    success = pr == cc;
+    if (!success) {
+      *c = pr;
+      mo = fmo;
+    }
+    if (success && IsAcqRelOrder(mo))
+      thr->clock.ReleaseAcquire(&s->clock);
+    else if (success && IsReleaseOrder(mo))
+      thr->clock.Release(&s->clock);
+    else if (IsAcquireOrder(mo))
+      thr->clock.Acquire(s->clock);
   }
-  thr->fast_state.IncrementEpoch();
-  // Can't increment epoch w/o writing to the trace as well.
-  TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
-
-  if (success && IsAcqRelOrder(mo))
-    AcquireReleaseImpl(thr, pc, &s->clock);
-  else if (success && IsReleaseOrder(mo))
-    ReleaseImpl(thr, pc, &s->clock);
-  else if (IsAcquireOrder(mo))
-    AcquireImpl(thr, pc, &s->clock);
+  if (success && release)
+    IncrementEpoch(thr);
   return success;
 }
 
diff --git a/compiler-rt/lib/tsan/rtl/tsan_interface_java.cpp b/compiler-rt/lib/tsan/rtl/tsan_interface_java.cpp
index c090c1f08cbe..7c15a1638826 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_interface_java.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_interface_java.cpp
@@ -106,7 +106,7 @@ void __tsan_java_free(jptr ptr, jptr size) {
   DCHECK_GE(ptr, jctx->heap_begin);
   DCHECK_LE(ptr + size, jctx->heap_begin + jctx->heap_size);
 
-  ctx->metamap.FreeRange(thr->proc(), ptr, size);
+  ctx->metamap.FreeRange(thr->proc(), ptr, size, false);
 }
 
 void __tsan_java_move(jptr src, jptr dst, jptr size) {
@@ -133,7 +133,7 @@ void __tsan_java_move(jptr src, jptr dst, jptr size) {
   // support that anymore as it contains addresses of accesses.
   RawShadow *d = MemToShadow(dst);
   RawShadow *dend = MemToShadow(dst + size);
-  internal_memset(d, 0, (dend - d) * sizeof(*d));
+  ShadowSet(d, dend, Shadow::kEmpty);
 }
 
 jptr __tsan_java_find(jptr *from_ptr, jptr to) {
diff --git a/compiler-rt/lib/tsan/rtl/tsan_mman.cpp b/compiler-rt/lib/tsan/rtl/tsan_mman.cpp
index a31bebcb6ba9..7a72efb12263 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_mman.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_mman.cpp
@@ -125,7 +125,6 @@ ScopedGlobalProcessor::~ScopedGlobalProcessor() {
 }
 
 void AllocatorLock() NO_THREAD_SAFETY_ANALYSIS {
-  global_proc()->mtx.Lock();
   global_proc()->internal_alloc_mtx.Lock();
   InternalAllocatorLock();
 }
@@ -133,6 +132,13 @@ void AllocatorLock() NO_THREAD_SAFETY_ANALYSIS {
 void AllocatorUnlock() NO_THREAD_SAFETY_ANALYSIS {
   InternalAllocatorUnlock();
   global_proc()->internal_alloc_mtx.Unlock();
+}
+
+void GlobalProcessorLock() NO_THREAD_SAFETY_ANALYSIS {
+  global_proc()->mtx.Lock();
+}
+
+void GlobalProcessorUnlock() NO_THREAD_SAFETY_ANALYSIS {
   global_proc()->mtx.Unlock();
 }
 
@@ -192,6 +198,12 @@ void *user_alloc_internal(ThreadState *thr, uptr pc, uptr sz, uptr align,
     GET_STACK_TRACE_FATAL(thr, pc);
     ReportAllocationSizeTooBig(sz, malloc_limit, &stack);
   }
+  if (UNLIKELY(IsRssLimitExceeded())) {
+    if (AllocatorMayReturnNull())
+      return nullptr;
+    GET_STACK_TRACE_FATAL(thr, pc);
+    ReportRssLimitExceeded(&stack);
+  }
   void *p = allocator()->Allocate(&thr->proc()->alloc_cache, sz, align);
   if (UNLIKELY(!p)) {
     SetAllocatorOutOfMemory();
@@ -245,8 +257,17 @@ void *user_reallocarray(ThreadState *thr, uptr pc, void *p, uptr size, uptr n) {
 
 void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write) {
   DPrintf("#%d: alloc(%zu) = 0x%zx\n", thr->tid, sz, p);
+  // Note: this can run before thread initialization/after finalization.
+  // As a result this is not necessarily synchronized with DoReset,
+  // which iterates over and resets all sync objects,
+  // but it is fine to create new MBlocks in this context.
   ctx->metamap.AllocBlock(thr, pc, p, sz);
-  if (write && thr->ignore_reads_and_writes == 0 && thr->is_inited)
+  // If this runs before thread initialization/after finalization
+  // and we don't have trace initialized, we can't imitate writes.
+  // In such case just reset the shadow range, it is fine since
+  // it affects only a small fraction of special objects.
+  if (write && thr->ignore_reads_and_writes == 0 &&
+      atomic_load_relaxed(&thr->trace_pos))
     MemoryRangeImitateWrite(thr, pc, (uptr)p, sz);
   else
     MemoryResetRange(thr, pc, (uptr)p, sz);
@@ -254,9 +275,16 @@ void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write) {
 
 void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write) {
   CHECK_NE(p, (void*)0);
-  uptr sz = ctx->metamap.FreeBlock(thr->proc(), p);
+  if (!thr->slot) {
+    // Very early/late in thread lifetime, or during fork.
+    UNUSED uptr sz = ctx->metamap.FreeBlock(thr->proc(), p, false);
+    DPrintf("#%d: free(0x%zx, %zu) (no slot)\n", thr->tid, p, sz);
+    return;
+  }
+  SlotLocker locker(thr);
+  uptr sz = ctx->metamap.FreeBlock(thr->proc(), p, true);
   DPrintf("#%d: free(0x%zx, %zu)\n", thr->tid, p, sz);
-  if (write && thr->ignore_reads_and_writes == 0 && thr->is_inited)
+  if (write && thr->ignore_reads_and_writes == 0)
     MemoryRangeFreed(thr, pc, (uptr)p, sz);
 }
 
@@ -336,7 +364,7 @@ void *user_pvalloc(ThreadState *thr, uptr pc, uptr sz) {
 }
 
 uptr user_alloc_usable_size(const void *p) {
-  if (p == 0)
+  if (p == 0 || !IsAppMem((uptr)p))
     return 0;
   MBlock *b = ctx->metamap.GetBlock((uptr)p);
   if (!b)
@@ -421,8 +449,6 @@ uptr __sanitizer_get_allocated_size(const void *p) {
 
 void __tsan_on_thread_idle() {
   ThreadState *thr = cur_thread();
-  thr->clock.ResetCached(&thr->proc()->clock_cache);
-  thr->last_sleep_clock.ResetCached(&thr->proc()->clock_cache);
   allocator()->SwallowCache(&thr->proc()->alloc_cache);
   internal_allocator()->SwallowCache(&thr->proc()->internal_alloc_cache);
   ctx->metamap.OnProcIdle(thr->proc());
diff --git a/compiler-rt/lib/tsan/rtl/tsan_mman.h b/compiler-rt/lib/tsan/rtl/tsan_mman.h
index db8488eabbe2..2095f28c0253 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_mman.h
+++ b/compiler-rt/lib/tsan/rtl/tsan_mman.h
@@ -26,6 +26,8 @@ void AllocatorProcFinish(Processor *proc);
 void AllocatorPrintStats();
 void AllocatorLock();
 void AllocatorUnlock();
+void GlobalProcessorLock();
+void GlobalProcessorUnlock();
 
 // For user allocations.
 void *user_alloc_internal(ThreadState *thr, uptr pc, uptr sz,
diff --git a/compiler-rt/lib/tsan/rtl/tsan_mutexset.cpp b/compiler-rt/lib/tsan/rtl/tsan_mutexset.cpp
index 735179686ba9..3a75b80ac30f 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_mutexset.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_mutexset.cpp
@@ -19,57 +19,7 @@ namespace __tsan {
 MutexSet::MutexSet() {
 }
 
-void MutexSet::Add(u64 id, bool write, u64 epoch) {
-  // Look up existing mutex with the same id.
-  for (uptr i = 0; i < size_; i++) {
-    if (descs_[i].id == id) {
-      descs_[i].count++;
-      descs_[i].epoch = epoch;
-      return;
-    }
-  }
-  // On overflow, find the oldest mutex and drop it.
-  if (size_ == kMaxSize) {
-    u64 minepoch = (u64)-1;
-    u64 mini = (u64)-1;
-    for (uptr i = 0; i < size_; i++) {
-      if (descs_[i].epoch < minepoch) {
-        minepoch = descs_[i].epoch;
-        mini = i;
-      }
-    }
-    RemovePos(mini);
-    CHECK_EQ(size_, kMaxSize - 1);
-  }
-  // Add new mutex descriptor.
-  descs_[size_].addr = 0;
-  descs_[size_].stack_id = kInvalidStackID;
-  descs_[size_].id = id;
-  descs_[size_].write = write;
-  descs_[size_].epoch = epoch;
-  descs_[size_].seq = seq_++;
-  descs_[size_].count = 1;
-  size_++;
-}
-
-void MutexSet::Del(u64 id, bool write) {
-  for (uptr i = 0; i < size_; i++) {
-    if (descs_[i].id == id) {
-      if (--descs_[i].count == 0)
-        RemovePos(i);
-      return;
-    }
-  }
-}
-
-void MutexSet::Remove(u64 id) {
-  for (uptr i = 0; i < size_; i++) {
-    if (descs_[i].id == id) {
-      RemovePos(i);
-      return;
-    }
-  }
-}
+void MutexSet::Reset() { internal_memset(this, 0, sizeof(*this)); }
 
 void MutexSet::AddAddr(uptr addr, StackID stack_id, bool write) {
   // Look up existing mutex with the same id.
@@ -93,9 +43,7 @@ void MutexSet::AddAddr(uptr addr, StackID stack_id, bool write) {
   // Add new mutex descriptor.
   descs_[size_].addr = addr;
   descs_[size_].stack_id = stack_id;
-  descs_[size_].id = 0;
   descs_[size_].write = write;
-  descs_[size_].epoch = 0;
   descs_[size_].seq = seq_++;
   descs_[size_].count = 1;
   size_++;
diff --git a/compiler-rt/lib/tsan/rtl/tsan_mutexset.h b/compiler-rt/lib/tsan/rtl/tsan_mutexset.h
index 93776a664135..aabd361e6afd 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_mutexset.h
+++ b/compiler-rt/lib/tsan/rtl/tsan_mutexset.h
@@ -25,8 +25,6 @@ class MutexSet {
   struct Desc {
     uptr addr;
     StackID stack_id;
-    u64 id;
-    u64 epoch;
     u32 seq;
     u32 count;
     bool write;
@@ -40,10 +38,7 @@ class MutexSet {
   };
 
   MutexSet();
-  // The 'id' is obtained from SyncVar::GetId().
-  void Add(u64 id, bool write, u64 epoch);
-  void Del(u64 id, bool write);
-  void Remove(u64 id);  // Removes the mutex completely (if it's destroyed).
+  void Reset();
   void AddAddr(uptr addr, StackID stack_id, bool write);
   void DelAddr(uptr addr, bool destroy = false);
   uptr Size() const;
@@ -82,9 +77,7 @@ class DynamicMutexSet {
 // in different goroutine).
 #if SANITIZER_GO
 MutexSet::MutexSet() {}
-void MutexSet::Add(u64 id, bool write, u64 epoch) {}
-void MutexSet::Del(u64 id, bool write) {}
-void MutexSet::Remove(u64 id) {}
+void MutexSet::Reset() {}
 void MutexSet::AddAddr(uptr addr, StackID stack_id, bool write) {}
 void MutexSet::DelAddr(uptr addr, bool destroy) {}
 uptr MutexSet::Size() const { return 0; }
diff --git a/compiler-rt/lib/tsan/rtl/tsan_platform.h b/compiler-rt/lib/tsan/rtl/tsan_platform.h
index 7ff0acace8f6..233bf0a39df0 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_platform.h
+++ b/compiler-rt/lib/tsan/rtl/tsan_platform.h
@@ -18,8 +18,8 @@
 # error "Only 64-bit is supported"
 #endif
 
+#include "sanitizer_common/sanitizer_common.h"
 #include "tsan_defs.h"
-#include "tsan_trace.h"
 
 namespace __tsan {
 
@@ -40,14 +40,12 @@ enum {
 C/C++ on linux/x86_64 and freebsd/x86_64
 0000 0000 1000 - 0080 0000 0000: main binary and/or MAP_32BIT mappings (512GB)
 0040 0000 0000 - 0100 0000 0000: -
-0100 0000 0000 - 2000 0000 0000: shadow
-2000 0000 0000 - 3000 0000 0000: -
+0100 0000 0000 - 1000 0000 0000: shadow
+1000 0000 0000 - 3000 0000 0000: -
 3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
 4000 0000 0000 - 5500 0000 0000: -
 5500 0000 0000 - 5680 0000 0000: pie binaries without ASLR or on 4.1+ kernels
-5680 0000 0000 - 6000 0000 0000: -
-6000 0000 0000 - 6200 0000 0000: traces
-6200 0000 0000 - 7d00 0000 0000: -
+5680 0000 0000 - 7d00 0000 0000: -
 7b00 0000 0000 - 7c00 0000 0000: heap
 7c00 0000 0000 - 7e80 0000 0000: -
 7e80 0000 0000 - 8000 0000 0000: modules and main thread stack
@@ -67,10 +65,8 @@ C/C++ on netbsd/amd64 can reuse the same mapping:
 struct Mapping48AddressSpace {
   static const uptr kMetaShadowBeg = 0x300000000000ull;
   static const uptr kMetaShadowEnd = 0x340000000000ull;
-  static const uptr kTraceMemBeg   = 0x600000000000ull;
-  static const uptr kTraceMemEnd   = 0x620000000000ull;
   static const uptr kShadowBeg     = 0x010000000000ull;
-  static const uptr kShadowEnd     = 0x200000000000ull;
+  static const uptr kShadowEnd = 0x100000000000ull;
   static const uptr kHeapMemBeg    = 0x7b0000000000ull;
   static const uptr kHeapMemEnd    = 0x7c0000000000ull;
   static const uptr kLoAppMemBeg   = 0x000000001000ull;
@@ -89,14 +85,13 @@ struct Mapping48AddressSpace {
 C/C++ on linux/mips64 (40-bit VMA)
 0000 0000 00 - 0100 0000 00: -                                           (4 GB)
 0100 0000 00 - 0200 0000 00: main binary                                 (4 GB)
-0200 0000 00 - 2000 0000 00: -                                         (120 GB)
-2000 0000 00 - 4000 0000 00: shadow                                    (128 GB)
+0200 0000 00 - 1200 0000 00: -                                          (64 GB)
+1200 0000 00 - 2200 0000 00: shadow                                     (64 GB)
+2200 0000 00 - 4000 0000 00: -                                         (120 GB)
 4000 0000 00 - 5000 0000 00: metainfo (memory blocks and sync objects)  (64 GB)
 5000 0000 00 - aa00 0000 00: -                                         (360 GB)
 aa00 0000 00 - ab00 0000 00: main binary (PIE)                           (4 GB)
-ab00 0000 00 - b000 0000 00: -                                          (20 GB)
-b000 0000 00 - b200 0000 00: traces                                      (8 GB)
-b200 0000 00 - fe00 0000 00: -                                         (304 GB)
+ab00 0000 00 - fe00 0000 00: -                                         (332 GB)
 fe00 0000 00 - ff00 0000 00: heap                                        (4 GB)
 ff00 0000 00 - ff80 0000 00: -                                           (2 GB)
 ff80 0000 00 - ffff ffff ff: modules and main thread stack              (<2 GB)
@@ -104,10 +99,8 @@ ff80 0000 00 - ffff ffff ff: modules and main thread stack              (<2 GB)
 struct MappingMips64_40 {
   static const uptr kMetaShadowBeg = 0x4000000000ull;
   static const uptr kMetaShadowEnd = 0x5000000000ull;
-  static const uptr kTraceMemBeg   = 0xb000000000ull;
-  static const uptr kTraceMemEnd   = 0xb200000000ull;
-  static const uptr kShadowBeg     = 0x2000000000ull;
-  static const uptr kShadowEnd     = 0x4000000000ull;
+  static const uptr kShadowBeg = 0x1200000000ull;
+  static const uptr kShadowEnd = 0x2200000000ull;
   static const uptr kHeapMemBeg    = 0xfe00000000ull;
   static const uptr kHeapMemEnd    = 0xff00000000ull;
   static const uptr kLoAppMemBeg   = 0x0100000000ull;
@@ -128,12 +121,10 @@ C/C++ on Darwin/iOS/ARM64 (36-bit VMA, 64 GB VM)
 0100 0000 00 - 0200 0000 00: main binary, modules, thread stacks  (4 GB)
 0200 0000 00 - 0300 0000 00: heap                                 (4 GB)
 0300 0000 00 - 0400 0000 00: -                                    (4 GB)
-0400 0000 00 - 0c00 0000 00: shadow memory                       (32 GB)
-0c00 0000 00 - 0d00 0000 00: -                                    (4 GB)
+0400 0000 00 - 0800 0000 00: shadow memory                       (16 GB)
+0800 0000 00 - 0d00 0000 00: -                                   (20 GB)
 0d00 0000 00 - 0e00 0000 00: metainfo                             (4 GB)
-0e00 0000 00 - 0f00 0000 00: -                                    (4 GB)
-0f00 0000 00 - 0fc0 0000 00: traces                               (3 GB)
-0fc0 0000 00 - 1000 0000 00: -
+0e00 0000 00 - 1000 0000 00: -
 */
 struct MappingAppleAarch64 {
   static const uptr kLoAppMemBeg   = 0x0100000000ull;
@@ -141,16 +132,14 @@ struct MappingAppleAarch64 {
   static const uptr kHeapMemBeg    = 0x0200000000ull;
   static const uptr kHeapMemEnd    = 0x0300000000ull;
   static const uptr kShadowBeg     = 0x0400000000ull;
-  static const uptr kShadowEnd     = 0x0c00000000ull;
+  static const uptr kShadowEnd = 0x0800000000ull;
   static const uptr kMetaShadowBeg = 0x0d00000000ull;
   static const uptr kMetaShadowEnd = 0x0e00000000ull;
-  static const uptr kTraceMemBeg   = 0x0f00000000ull;
-  static const uptr kTraceMemEnd   = 0x0fc0000000ull;
   static const uptr kHiAppMemBeg   = 0x0fc0000000ull;
   static const uptr kHiAppMemEnd   = 0x0fc0000000ull;
   static const uptr kShadowMsk = 0x0ull;
   static const uptr kShadowXor = 0x0ull;
-  static const uptr kShadowAdd = 0x0ull;
+  static const uptr kShadowAdd = 0x0200000000ull;
   static const uptr kVdsoBeg       = 0x7000000000000000ull;
   static const uptr kMidAppMemBeg = 0;
   static const uptr kMidAppMemEnd = 0;
@@ -159,29 +148,25 @@ struct MappingAppleAarch64 {
 /*
 C/C++ on linux/aarch64 (39-bit VMA)
 0000 0010 00 - 0100 0000 00: main binary
-0100 0000 00 - 0800 0000 00: -
-0800 0000 00 - 2000 0000 00: shadow memory
+0100 0000 00 - 0400 0000 00: -
+0400 0000 00 - 1000 0000 00: shadow memory
 2000 0000 00 - 3100 0000 00: -
 3100 0000 00 - 3400 0000 00: metainfo
 3400 0000 00 - 5500 0000 00: -
 5500 0000 00 - 5600 0000 00: main binary (PIE)
-5600 0000 00 - 6000 0000 00: -
-6000 0000 00 - 6200 0000 00: traces
-6200 0000 00 - 7d00 0000 00: -
+5600 0000 00 - 7c00 0000 00: -
 7c00 0000 00 - 7d00 0000 00: heap
 7d00 0000 00 - 7fff ffff ff: modules and main thread stack
 */
 struct MappingAarch64_39 {
   static const uptr kLoAppMemBeg   = 0x0000001000ull;
   static const uptr kLoAppMemEnd   = 0x0100000000ull;
-  static const uptr kShadowBeg     = 0x0800000000ull;
-  static const uptr kShadowEnd     = 0x2000000000ull;
+  static const uptr kShadowBeg = 0x0400000000ull;
+  static const uptr kShadowEnd = 0x1000000000ull;
   static const uptr kMetaShadowBeg = 0x3100000000ull;
   static const uptr kMetaShadowEnd = 0x3400000000ull;
   static const uptr kMidAppMemBeg  = 0x5500000000ull;
-  static const uptr kMidAppMemEnd  = 0x5600000000ull;
-  static const uptr kTraceMemBeg   = 0x6000000000ull;
-  static const uptr kTraceMemEnd   = 0x6200000000ull;
+  static const uptr kMidAppMemEnd = 0x5600000000ull;
   static const uptr kHeapMemBeg    = 0x7c00000000ull;
   static const uptr kHeapMemEnd    = 0x7d00000000ull;
   static const uptr kHiAppMemBeg   = 0x7e00000000ull;
@@ -195,15 +180,13 @@ struct MappingAarch64_39 {
 /*
 C/C++ on linux/aarch64 (42-bit VMA)
 00000 0010 00 - 01000 0000 00: main binary
-01000 0000 00 - 10000 0000 00: -
-10000 0000 00 - 20000 0000 00: shadow memory
-20000 0000 00 - 26000 0000 00: -
+01000 0000 00 - 08000 0000 00: -
+08000 0000 00 - 10000 0000 00: shadow memory
+10000 0000 00 - 26000 0000 00: -
 26000 0000 00 - 28000 0000 00: metainfo
 28000 0000 00 - 2aa00 0000 00: -
 2aa00 0000 00 - 2ab00 0000 00: main binary (PIE)
-2ab00 0000 00 - 36200 0000 00: -
-36200 0000 00 - 36240 0000 00: traces
-36240 0000 00 - 3e000 0000 00: -
+2ab00 0000 00 - 3e000 0000 00: -
 3e000 0000 00 - 3f000 0000 00: heap
 3f000 0000 00 - 3ffff ffff ff: modules and main thread stack
 */
@@ -211,14 +194,12 @@ struct MappingAarch64_42 {
   static const uptr kBroken = kBrokenReverseMapping;
   static const uptr kLoAppMemBeg   = 0x00000001000ull;
   static const uptr kLoAppMemEnd   = 0x01000000000ull;
-  static const uptr kShadowBeg     = 0x10000000000ull;
-  static const uptr kShadowEnd     = 0x20000000000ull;
+  static const uptr kShadowBeg = 0x08000000000ull;
+  static const uptr kShadowEnd = 0x10000000000ull;
   static const uptr kMetaShadowBeg = 0x26000000000ull;
   static const uptr kMetaShadowEnd = 0x28000000000ull;
   static const uptr kMidAppMemBeg  = 0x2aa00000000ull;
-  static const uptr kMidAppMemEnd  = 0x2ab00000000ull;
-  static const uptr kTraceMemBeg   = 0x36200000000ull;
-  static const uptr kTraceMemEnd   = 0x36400000000ull;
+  static const uptr kMidAppMemEnd = 0x2ab00000000ull;
   static const uptr kHeapMemBeg    = 0x3e000000000ull;
   static const uptr kHeapMemEnd    = 0x3f000000000ull;
   static const uptr kHiAppMemBeg   = 0x3f000000000ull;
@@ -232,14 +213,12 @@ struct MappingAarch64_42 {
 struct MappingAarch64_48 {
   static const uptr kLoAppMemBeg   = 0x0000000001000ull;
   static const uptr kLoAppMemEnd   = 0x0000200000000ull;
-  static const uptr kShadowBeg     = 0x0002000000000ull;
-  static const uptr kShadowEnd     = 0x0004000000000ull;
+  static const uptr kShadowBeg = 0x0001000000000ull;
+  static const uptr kShadowEnd = 0x0002000000000ull;
   static const uptr kMetaShadowBeg = 0x0005000000000ull;
   static const uptr kMetaShadowEnd = 0x0006000000000ull;
   static const uptr kMidAppMemBeg  = 0x0aaaa00000000ull;
-  static const uptr kMidAppMemEnd  = 0x0aaaf00000000ull;
-  static const uptr kTraceMemBeg   = 0x0f06000000000ull;
-  static const uptr kTraceMemEnd   = 0x0f06200000000ull;
+  static const uptr kMidAppMemEnd = 0x0aaaf00000000ull;
   static const uptr kHeapMemBeg    = 0x0ffff00000000ull;
   static const uptr kHeapMemEnd    = 0x0ffff00000000ull;
   static const uptr kHiAppMemBeg   = 0x0ffff00000000ull;
@@ -257,9 +236,7 @@ C/C++ on linux/powerpc64 (44-bit VMA)
 0001 0000 0000 - 0b00 0000 0000: shadow
 0b00 0000 0000 - 0b00 0000 0000: -
 0b00 0000 0000 - 0d00 0000 0000: metainfo (memory blocks and sync objects)
-0d00 0000 0000 - 0d00 0000 0000: -
-0d00 0000 0000 - 0f00 0000 0000: traces
-0f00 0000 0000 - 0f00 0000 0000: -
+0d00 0000 0000 - 0f00 0000 0000: -
 0f00 0000 0000 - 0f50 0000 0000: heap
 0f50 0000 0000 - 0f60 0000 0000: -
 0f60 0000 0000 - 1000 0000 0000: modules and main thread stack
@@ -269,8 +246,6 @@ struct MappingPPC64_44 {
       kBrokenMapping | kBrokenReverseMapping | kBrokenLinearity;
   static const uptr kMetaShadowBeg = 0x0b0000000000ull;
   static const uptr kMetaShadowEnd = 0x0d0000000000ull;
-  static const uptr kTraceMemBeg   = 0x0d0000000000ull;
-  static const uptr kTraceMemEnd   = 0x0f0000000000ull;
   static const uptr kShadowBeg     = 0x000100000000ull;
   static const uptr kShadowEnd     = 0x0b0000000000ull;
   static const uptr kLoAppMemBeg   = 0x000000000100ull;
@@ -291,23 +266,19 @@ struct MappingPPC64_44 {
 C/C++ on linux/powerpc64 (46-bit VMA)
 0000 0000 1000 - 0100 0000 0000: main binary
 0100 0000 0000 - 0200 0000 0000: -
-0100 0000 0000 - 1000 0000 0000: shadow
-1000 0000 0000 - 1000 0000 0000: -
-1000 0000 0000 - 2000 0000 0000: metainfo (memory blocks and sync objects)
-2000 0000 0000 - 2000 0000 0000: -
-2000 0000 0000 - 2200 0000 0000: traces
-2200 0000 0000 - 3d00 0000 0000: -
+0100 0000 0000 - 0800 0000 0000: shadow
+0800 0000 0000 - 1000 0000 0000: -
+1000 0000 0000 - 1200 0000 0000: metainfo (memory blocks and sync objects)
+1200 0000 0000 - 3d00 0000 0000: -
 3d00 0000 0000 - 3e00 0000 0000: heap
 3e00 0000 0000 - 3e80 0000 0000: -
 3e80 0000 0000 - 4000 0000 0000: modules and main thread stack
 */
 struct MappingPPC64_46 {
   static const uptr kMetaShadowBeg = 0x100000000000ull;
-  static const uptr kMetaShadowEnd = 0x200000000000ull;
-  static const uptr kTraceMemBeg   = 0x200000000000ull;
-  static const uptr kTraceMemEnd   = 0x220000000000ull;
+  static const uptr kMetaShadowEnd = 0x120000000000ull;
   static const uptr kShadowBeg     = 0x010000000000ull;
-  static const uptr kShadowEnd     = 0x100000000000ull;
+  static const uptr kShadowEnd = 0x080000000000ull;
   static const uptr kHeapMemBeg    = 0x3d0000000000ull;
   static const uptr kHeapMemEnd    = 0x3e0000000000ull;
   static const uptr kLoAppMemBeg   = 0x000000001000ull;
@@ -326,23 +297,19 @@ struct MappingPPC64_46 {
 C/C++ on linux/powerpc64 (47-bit VMA)
 0000 0000 1000 - 0100 0000 0000: main binary
 0100 0000 0000 - 0200 0000 0000: -
-0100 0000 0000 - 1000 0000 0000: shadow
-1000 0000 0000 - 1000 0000 0000: -
-1000 0000 0000 - 2000 0000 0000: metainfo (memory blocks and sync objects)
-2000 0000 0000 - 2000 0000 0000: -
-2000 0000 0000 - 2200 0000 0000: traces
-2200 0000 0000 - 7d00 0000 0000: -
+0100 0000 0000 - 0800 0000 0000: shadow
+0800 0000 0000 - 1000 0000 0000: -
+1000 0000 0000 - 1200 0000 0000: metainfo (memory blocks and sync objects)
+1200 0000 0000 - 7d00 0000 0000: -
 7d00 0000 0000 - 7e00 0000 0000: heap
 7e00 0000 0000 - 7e80 0000 0000: -
 7e80 0000 0000 - 8000 0000 0000: modules and main thread stack
 */
 struct MappingPPC64_47 {
   static const uptr kMetaShadowBeg = 0x100000000000ull;
-  static const uptr kMetaShadowEnd = 0x200000000000ull;
-  static const uptr kTraceMemBeg   = 0x200000000000ull;
-  static const uptr kTraceMemEnd   = 0x220000000000ull;
+  static const uptr kMetaShadowEnd = 0x120000000000ull;
   static const uptr kShadowBeg     = 0x010000000000ull;
-  static const uptr kShadowEnd     = 0x100000000000ull;
+  static const uptr kShadowEnd = 0x080000000000ull;
   static const uptr kHeapMemBeg    = 0x7d0000000000ull;
   static const uptr kHeapMemEnd    = 0x7e0000000000ull;
   static const uptr kLoAppMemBeg   = 0x000000001000ull;
@@ -362,22 +329,18 @@ C/C++ on linux/s390x
 While the kernel provides a 64-bit address space, we have to restrict ourselves
 to 48 bits due to how e.g. SyncVar::GetId() works.
 0000 0000 1000 - 0e00 0000 0000: binary, modules, stacks - 14 TiB
-0e00 0000 0000 - 4000 0000 0000: -
-4000 0000 0000 - 8000 0000 0000: shadow - 64TiB (4 * app)
-8000 0000 0000 - 9000 0000 0000: -
+0e00 0000 0000 - 2000 0000 0000: -
+2000 0000 0000 - 4000 0000 0000: shadow - 32TiB (2 * app)
+4000 0000 0000 - 9000 0000 0000: -
 9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app)
-9800 0000 0000 - a000 0000 0000: -
-a000 0000 0000 - b000 0000 0000: traces - 16TiB (max history * 128k threads)
-b000 0000 0000 - be00 0000 0000: -
+9800 0000 0000 - be00 0000 0000: -
 be00 0000 0000 - c000 0000 0000: heap - 2TiB (max supported by the allocator)
 */
 struct MappingS390x {
   static const uptr kMetaShadowBeg = 0x900000000000ull;
   static const uptr kMetaShadowEnd = 0x980000000000ull;
-  static const uptr kTraceMemBeg   = 0xa00000000000ull;
-  static const uptr kTraceMemEnd   = 0xb00000000000ull;
-  static const uptr kShadowBeg     = 0x400000000000ull;
-  static const uptr kShadowEnd     = 0x800000000000ull;
+  static const uptr kShadowBeg = 0x200000000000ull;
+  static const uptr kShadowEnd = 0x400000000000ull;
   static const uptr kHeapMemBeg    = 0xbe0000000000ull;
   static const uptr kHeapMemEnd    = 0xc00000000000ull;
   static const uptr kLoAppMemBeg   = 0x000000001000ull;
@@ -397,21 +360,17 @@ struct MappingS390x {
 0000 1000 0000 - 00c0 0000 0000: -
 00c0 0000 0000 - 00e0 0000 0000: heap
 00e0 0000 0000 - 2000 0000 0000: -
-2000 0000 0000 - 2380 0000 0000: shadow
-2380 0000 0000 - 3000 0000 0000: -
+2000 0000 0000 - 21c0 0000 0000: shadow
+21c0 0000 0000 - 3000 0000 0000: -
 3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
-4000 0000 0000 - 6000 0000 0000: -
-6000 0000 0000 - 6200 0000 0000: traces
-6200 0000 0000 - 8000 0000 0000: -
+4000 0000 0000 - 8000 0000 0000: -
 */
 
 struct MappingGo48 {
   static const uptr kMetaShadowBeg = 0x300000000000ull;
   static const uptr kMetaShadowEnd = 0x400000000000ull;
-  static const uptr kTraceMemBeg   = 0x600000000000ull;
-  static const uptr kTraceMemEnd   = 0x620000000000ull;
   static const uptr kShadowBeg     = 0x200000000000ull;
-  static const uptr kShadowEnd     = 0x238000000000ull;
+  static const uptr kShadowEnd = 0x21c000000000ull;
   static const uptr kLoAppMemBeg = 0x000000001000ull;
   static const uptr kLoAppMemEnd = 0x00e000000000ull;
   static const uptr kMidAppMemBeg = 0;
@@ -431,8 +390,8 @@ struct MappingGo48 {
 0000 1000 0000 - 00f8 0000 0000: -
 00c0 0000 0000 - 00e0 0000 0000: heap
 00e0 0000 0000 - 0100 0000 0000: -
-0100 0000 0000 - 0500 0000 0000: shadow
-0500 0000 0000 - 0700 0000 0000: traces
+0100 0000 0000 - 0300 0000 0000: shadow
+0300 0000 0000 - 0700 0000 0000: -
 0700 0000 0000 - 0770 0000 0000: metainfo (memory blocks and sync objects)
 07d0 0000 0000 - 8000 0000 0000: -
 */
@@ -440,10 +399,8 @@ struct MappingGo48 {
 struct MappingGoWindows {
   static const uptr kMetaShadowBeg = 0x070000000000ull;
   static const uptr kMetaShadowEnd = 0x077000000000ull;
-  static const uptr kTraceMemBeg = 0x050000000000ull;
-  static const uptr kTraceMemEnd = 0x070000000000ull;
   static const uptr kShadowBeg     = 0x010000000000ull;
-  static const uptr kShadowEnd     = 0x050000000000ull;
+  static const uptr kShadowEnd = 0x030000000000ull;
   static const uptr kLoAppMemBeg = 0x000000001000ull;
   static const uptr kLoAppMemEnd = 0x00e000000000ull;
   static const uptr kMidAppMemBeg = 0;
@@ -463,21 +420,17 @@ struct MappingGoWindows {
 0000 1000 0000 - 00c0 0000 0000: -
 00c0 0000 0000 - 00e0 0000 0000: heap
 00e0 0000 0000 - 2000 0000 0000: -
-2000 0000 0000 - 2380 0000 0000: shadow
-2380 0000 0000 - 2400 0000 0000: -
-2400 0000 0000 - 3400 0000 0000: metainfo (memory blocks and sync objects)
-3400 0000 0000 - 3600 0000 0000: -
-3600 0000 0000 - 3800 0000 0000: traces
-3800 0000 0000 - 4000 0000 0000: -
+2000 0000 0000 - 21c0 0000 0000: shadow
+21c0 0000 0000 - 2400 0000 0000: -
+2400 0000 0000 - 2470 0000 0000: metainfo (memory blocks and sync objects)
+2470 0000 0000 - 4000 0000 0000: -
 */
 
 struct MappingGoPPC64_46 {
   static const uptr kMetaShadowBeg = 0x240000000000ull;
-  static const uptr kMetaShadowEnd = 0x340000000000ull;
-  static const uptr kTraceMemBeg   = 0x360000000000ull;
-  static const uptr kTraceMemEnd   = 0x380000000000ull;
+  static const uptr kMetaShadowEnd = 0x247000000000ull;
   static const uptr kShadowBeg     = 0x200000000000ull;
-  static const uptr kShadowEnd     = 0x238000000000ull;
+  static const uptr kShadowEnd = 0x21c000000000ull;
   static const uptr kLoAppMemBeg = 0x000000001000ull;
   static const uptr kLoAppMemEnd = 0x00e000000000ull;
   static const uptr kMidAppMemBeg = 0;
@@ -497,21 +450,17 @@ struct MappingGoPPC64_46 {
 0000 1000 0000 - 00c0 0000 0000: -
 00c0 0000 0000 - 00e0 0000 0000: heap
 00e0 0000 0000 - 2000 0000 0000: -
-2000 0000 0000 - 3000 0000 0000: shadow
-3000 0000 0000 - 3000 0000 0000: -
-3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
-4000 0000 0000 - 6000 0000 0000: -
-6000 0000 0000 - 6200 0000 0000: traces
-6200 0000 0000 - 8000 0000 0000: -
+2000 0000 0000 - 2800 0000 0000: shadow
+2800 0000 0000 - 3000 0000 0000: -
+3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
+3200 0000 0000 - 8000 0000 0000: -
 */
 
 struct MappingGoPPC64_47 {
   static const uptr kMetaShadowBeg = 0x300000000000ull;
-  static const uptr kMetaShadowEnd = 0x400000000000ull;
-  static const uptr kTraceMemBeg   = 0x600000000000ull;
-  static const uptr kTraceMemEnd   = 0x620000000000ull;
+  static const uptr kMetaShadowEnd = 0x320000000000ull;
   static const uptr kShadowBeg     = 0x200000000000ull;
-  static const uptr kShadowEnd     = 0x300000000000ull;
+  static const uptr kShadowEnd = 0x280000000000ull;
   static const uptr kLoAppMemBeg = 0x000000001000ull;
   static const uptr kLoAppMemEnd = 0x00e000000000ull;
   static const uptr kMidAppMemBeg = 0;
@@ -531,20 +480,16 @@ struct MappingGoPPC64_47 {
 0000 1000 0000 - 00c0 0000 0000: -
 00c0 0000 0000 - 00e0 0000 0000: heap
 00e0 0000 0000 - 2000 0000 0000: -
-2000 0000 0000 - 3000 0000 0000: shadow
-3000 0000 0000 - 3000 0000 0000: -
-3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
-4000 0000 0000 - 6000 0000 0000: -
-6000 0000 0000 - 6200 0000 0000: traces
-6200 0000 0000 - 8000 0000 0000: -
+2000 0000 0000 - 2800 0000 0000: shadow
+2800 0000 0000 - 3000 0000 0000: -
+3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
+3200 0000 0000 - 8000 0000 0000: -
 */
 struct MappingGoAarch64 {
   static const uptr kMetaShadowBeg = 0x300000000000ull;
-  static const uptr kMetaShadowEnd = 0x400000000000ull;
-  static const uptr kTraceMemBeg   = 0x600000000000ull;
-  static const uptr kTraceMemEnd   = 0x620000000000ull;
+  static const uptr kMetaShadowEnd = 0x320000000000ull;
   static const uptr kShadowBeg     = 0x200000000000ull;
-  static const uptr kShadowEnd     = 0x300000000000ull;
+  static const uptr kShadowEnd = 0x280000000000ull;
   static const uptr kLoAppMemBeg = 0x000000001000ull;
   static const uptr kLoAppMemEnd = 0x00e000000000ull;
   static const uptr kMidAppMemBeg = 0;
@@ -565,20 +510,16 @@ Go on linux/mips64 (47-bit VMA)
 0000 1000 0000 - 00c0 0000 0000: -
 00c0 0000 0000 - 00e0 0000 0000: heap
 00e0 0000 0000 - 2000 0000 0000: -
-2000 0000 0000 - 3000 0000 0000: shadow
-3000 0000 0000 - 3000 0000 0000: -
-3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
-4000 0000 0000 - 6000 0000 0000: -
-6000 0000 0000 - 6200 0000 0000: traces
-6200 0000 0000 - 8000 0000 0000: -
+2000 0000 0000 - 2800 0000 0000: shadow
+2800 0000 0000 - 3000 0000 0000: -
+3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
+3200 0000 0000 - 8000 0000 0000: -
 */
 struct MappingGoMips64_47 {
   static const uptr kMetaShadowBeg = 0x300000000000ull;
-  static const uptr kMetaShadowEnd = 0x400000000000ull;
-  static const uptr kTraceMemBeg = 0x600000000000ull;
-  static const uptr kTraceMemEnd = 0x620000000000ull;
+  static const uptr kMetaShadowEnd = 0x320000000000ull;
   static const uptr kShadowBeg = 0x200000000000ull;
-  static const uptr kShadowEnd = 0x300000000000ull;
+  static const uptr kShadowEnd = 0x280000000000ull;
   static const uptr kLoAppMemBeg = 0x000000001000ull;
   static const uptr kLoAppMemEnd = 0x00e000000000ull;
   static const uptr kMidAppMemBeg = 0;
@@ -597,19 +538,15 @@ struct MappingGoMips64_47 {
 Go on linux/s390x
 0000 0000 1000 - 1000 0000 0000: executable and heap - 16 TiB
 1000 0000 0000 - 4000 0000 0000: -
-4000 0000 0000 - 8000 0000 0000: shadow - 64TiB (4 * app)
-8000 0000 0000 - 9000 0000 0000: -
+4000 0000 0000 - 6000 0000 0000: shadow - 64TiB (4 * app)
+6000 0000 0000 - 9000 0000 0000: -
 9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app)
-9800 0000 0000 - a000 0000 0000: -
-a000 0000 0000 - b000 0000 0000: traces - 16TiB (max history * 128k threads)
 */
 struct MappingGoS390x {
   static const uptr kMetaShadowBeg = 0x900000000000ull;
   static const uptr kMetaShadowEnd = 0x980000000000ull;
-  static const uptr kTraceMemBeg   = 0xa00000000000ull;
-  static const uptr kTraceMemEnd   = 0xb00000000000ull;
   static const uptr kShadowBeg     = 0x400000000000ull;
-  static const uptr kShadowEnd     = 0x800000000000ull;
+  static const uptr kShadowEnd = 0x600000000000ull;
   static const uptr kLoAppMemBeg = 0x000000001000ull;
   static const uptr kLoAppMemEnd = 0x100000000000ull;
   static const uptr kMidAppMemBeg = 0;
@@ -648,11 +585,11 @@ ALWAYS_INLINE auto SelectMapping(Arg arg) {
   return Func::template Apply<MappingGo48>(arg);
 #  endif
 #else  // SANITIZER_GO
-#  if defined(__x86_64__) || SANITIZER_IOSSIM || SANITIZER_MAC && !SANITIZER_IOS
-  return Func::template Apply<Mapping48AddressSpace>(arg);
-#  elif defined(__aarch64__) && defined(__APPLE__)
+#  if SANITIZER_IOS && !SANITIZER_IOSSIM
   return Func::template Apply<MappingAppleAarch64>(arg);
-#  elif defined(__aarch64__) && !defined(__APPLE__)
+#  elif defined(__x86_64__) || SANITIZER_MAC
+  return Func::template Apply<Mapping48AddressSpace>(arg);
+#  elif defined(__aarch64__)
   switch (vmaSize) {
     case 39:
       return Func::template Apply<MappingAarch64_39>(arg);
@@ -715,8 +652,6 @@ enum MappingType {
   kShadowEnd,
   kMetaShadowBeg,
   kMetaShadowEnd,
-  kTraceMemBeg,
-  kTraceMemEnd,
   kVdsoBeg,
 };
 
@@ -750,10 +685,6 @@ struct MappingField {
         return Mapping::kMetaShadowBeg;
       case kMetaShadowEnd:
         return Mapping::kMetaShadowEnd;
-      case kTraceMemBeg:
-        return Mapping::kTraceMemBeg;
-      case kTraceMemEnd:
-        return Mapping::kTraceMemEnd;
     }
     Die();
   }
@@ -792,11 +723,6 @@ uptr MetaShadowBeg(void) { return SelectMapping<MappingField>(kMetaShadowBeg); }
 ALWAYS_INLINE
 uptr MetaShadowEnd(void) { return SelectMapping<MappingField>(kMetaShadowEnd); }
 
-ALWAYS_INLINE
-uptr TraceMemBeg(void) { return SelectMapping<MappingField>(kTraceMemBeg); }
-ALWAYS_INLINE
-uptr TraceMemEnd(void) { return SelectMapping<MappingField>(kTraceMemEnd); }
-
 struct IsAppMemImpl {
   template <typename Mapping>
   static bool Apply(uptr mem) {
@@ -934,43 +860,10 @@ inline uptr RestoreAddr(uptr addr) {
   return SelectMapping<RestoreAddrImpl>(addr);
 }
 
-// The additional page is to catch shadow stack overflow as paging fault.
-// Windows wants 64K alignment for mmaps.
-const uptr kTotalTraceSize = (kTraceSize * sizeof(Event) + sizeof(Trace)
-    + (64 << 10) + (64 << 10) - 1) & ~((64 << 10) - 1);
-
-struct GetThreadTraceImpl {
-  template <typename Mapping>
-  static uptr Apply(uptr tid) {
-    uptr p = Mapping::kTraceMemBeg + tid * kTotalTraceSize;
-    DCHECK_LT(p, Mapping::kTraceMemEnd);
-    return p;
-  }
-};
-
-ALWAYS_INLINE
-uptr GetThreadTrace(int tid) { return SelectMapping<GetThreadTraceImpl>(tid); }
-
-struct GetThreadTraceHeaderImpl {
-  template <typename Mapping>
-  static uptr Apply(uptr tid) {
-    uptr p = Mapping::kTraceMemBeg + tid * kTotalTraceSize +
-             kTraceSize * sizeof(Event);
-    DCHECK_LT(p, Mapping::kTraceMemEnd);
-    return p;
-  }
-};
-
-ALWAYS_INLINE
-uptr GetThreadTraceHeader(int tid) {
-  return SelectMapping<GetThreadTraceHeaderImpl>(tid);
-}
-
 void InitializePlatform();
 void InitializePlatformEarly();
 void CheckAndProtect();
 void InitializeShadowMemoryPlatform();
-void FlushShadowMemory();
 void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns);
 int ExtractResolvFDs(void *state, int *fds, int nfd);
 int ExtractRecvmsgFDs(void *msg, int *fds, int nfd);
diff --git a/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cpp b/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cpp
index 73ec14892d28..17dbdff8a539 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cpp
@@ -94,7 +94,6 @@ enum {
   MemMeta,
   MemFile,
   MemMmap,
-  MemTrace,
   MemHeap,
   MemOther,
   MemCount,
@@ -112,8 +111,6 @@ void FillProfileCallback(uptr p, uptr rss, bool file, uptr *mem) {
     mem[file ? MemFile : MemMmap] += rss;
   else if (p >= HeapMemBeg() && p < HeapMemEnd())
     mem[MemHeap] += rss;
-  else if (p >= TraceMemBeg() && p < TraceMemEnd())
-    mem[MemTrace] += rss;
   else
     mem[MemOther] += rss;
 }
@@ -126,42 +123,33 @@ void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns) {
   StackDepotStats stacks = StackDepotGetStats();
   uptr nthread, nlive;
   ctx->thread_registry.GetNumberOfThreads(&nthread, &nlive);
+  uptr trace_mem;
+  {
+    Lock l(&ctx->slot_mtx);
+    trace_mem = ctx->trace_part_total_allocated * sizeof(TracePart);
+  }
   uptr internal_stats[AllocatorStatCount];
   internal_allocator()->GetStats(internal_stats);
   // All these are allocated from the common mmap region.
-  mem[MemMmap] -= meta.mem_block + meta.sync_obj + stacks.allocated +
-                  internal_stats[AllocatorStatMapped];
+  mem[MemMmap] -= meta.mem_block + meta.sync_obj + trace_mem +
+                  stacks.allocated + internal_stats[AllocatorStatMapped];
   if (s64(mem[MemMmap]) < 0)
     mem[MemMmap] = 0;
   internal_snprintf(
       buf, buf_size,
-      "%llus: RSS %zd MB: shadow:%zd meta:%zd file:%zd mmap:%zd"
-      " trace:%zd heap:%zd other:%zd intalloc:%zd memblocks:%zd syncobj:%zu"
-      " stacks=%zd[%zd] nthr=%zd/%zd\n",
-      uptime_ns / (1000 * 1000 * 1000), mem[MemTotal] >> 20,
-      mem[MemShadow] >> 20, mem[MemMeta] >> 20, mem[MemFile] >> 20,
-      mem[MemMmap] >> 20, mem[MemTrace] >> 20, mem[MemHeap] >> 20,
+      "==%zu== %llus [%zu]: RSS %zd MB: shadow:%zd meta:%zd file:%zd"
+      " mmap:%zd heap:%zd other:%zd intalloc:%zd memblocks:%zd syncobj:%zu"
+      " trace:%zu stacks=%zd threads=%zu/%zu\n",
+      internal_getpid(), uptime_ns / (1000 * 1000 * 1000), ctx->global_epoch,
+      mem[MemTotal] >> 20, mem[MemShadow] >> 20, mem[MemMeta] >> 20,
+      mem[MemFile] >> 20, mem[MemMmap] >> 20, mem[MemHeap] >> 20,
       mem[MemOther] >> 20, internal_stats[AllocatorStatMapped] >> 20,
-      meta.mem_block >> 20, meta.sync_obj >> 20, stacks.allocated >> 20,
-      stacks.n_uniq_ids, nlive, nthread);
-}
-
-#  if SANITIZER_LINUX
-void FlushShadowMemoryCallback(
-    const SuspendedThreadsList &suspended_threads_list,
-    void *argument) {
-  ReleaseMemoryPagesToOS(ShadowBeg(), ShadowEnd());
-}
-#endif
-
-void FlushShadowMemory() {
-#if SANITIZER_LINUX
-  StopTheWorld(FlushShadowMemoryCallback, 0);
-#endif
+      meta.mem_block >> 20, meta.sync_obj >> 20, trace_mem >> 20,
+      stacks.allocated >> 20, nlive, nthread);
 }
 
 #if !SANITIZER_GO
-// Mark shadow for .rodata sections with the special kShadowRodata marker.
+// Mark shadow for .rodata sections with the special Shadow::kRodata marker.
 // Accesses to .rodata can't race, so this saves time, memory and trace space.
 static void MapRodata() {
   // First create temp file.
@@ -182,13 +170,13 @@ static void MapRodata() {
     return;
   internal_unlink(name);  // Unlink it now, so that we can reuse the buffer.
   fd_t fd = openrv;
-  // Fill the file with kShadowRodata.
+  // Fill the file with Shadow::kRodata.
   const uptr kMarkerSize = 512 * 1024 / sizeof(RawShadow);
   InternalMmapVector<RawShadow> marker(kMarkerSize);
   // volatile to prevent insertion of memset
   for (volatile RawShadow *p = marker.data(); p < marker.data() + kMarkerSize;
        p++)
-    *p = kShadowRodata;
+    *p = Shadow::kRodata;
   internal_write(fd, marker.data(), marker.size() * sizeof(RawShadow));
   // Map the file into memory.
   uptr page = internal_mmap(0, GetPageSizeCached(), PROT_READ | PROT_WRITE,
diff --git a/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cpp b/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cpp
index 1465f9953c19..44b98d46cfbc 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cpp
@@ -126,9 +126,6 @@ void cur_thread_finalize() {
 }
 #endif
 
-void FlushShadowMemory() {
-}
-
 static void RegionMemUsage(uptr start, uptr end, uptr *res, uptr *dirty) {
   vm_address_t address = start;
   vm_address_t end_address = end;
@@ -156,12 +153,10 @@ static void RegionMemUsage(uptr start, uptr end, uptr *res, uptr *dirty) {
 void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns) {
   uptr shadow_res, shadow_dirty;
   uptr meta_res, meta_dirty;
-  uptr trace_res, trace_dirty;
   RegionMemUsage(ShadowBeg(), ShadowEnd(), &shadow_res, &shadow_dirty);
   RegionMemUsage(MetaShadowBeg(), MetaShadowEnd(), &meta_res, &meta_dirty);
-  RegionMemUsage(TraceMemBeg(), TraceMemEnd(), &trace_res, &trace_dirty);
 
-#if !SANITIZER_GO
+#  if !SANITIZER_GO
   uptr low_res, low_dirty;
   uptr high_res, high_dirty;
   uptr heap_res, heap_dirty;
@@ -180,7 +175,6 @@ void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns) {
       buf, buf_size,
       "shadow   (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
       "meta     (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
-      "traces   (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
 #  if !SANITIZER_GO
       "low app  (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
       "high app (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
@@ -193,7 +187,6 @@ void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns) {
       "------------------------------\n",
       ShadowBeg(), ShadowEnd(), shadow_res / 1024, shadow_dirty / 1024,
       MetaShadowBeg(), MetaShadowEnd(), meta_res / 1024, meta_dirty / 1024,
-      TraceMemBeg(), TraceMemEnd(), trace_res / 1024, trace_dirty / 1024,
 #  if !SANITIZER_GO
       LoAppMemBeg(), LoAppMemEnd(), low_res / 1024, low_dirty / 1024,
       HiAppMemBeg(), HiAppMemEnd(), high_res / 1024, high_dirty / 1024,
diff --git a/compiler-rt/lib/tsan/rtl/tsan_platform_posix.cpp b/compiler-rt/lib/tsan/rtl/tsan_platform_posix.cpp
index 763ac444377e..71874aad8dc5 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_platform_posix.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_platform_posix.cpp
@@ -110,27 +110,23 @@ void CheckAndProtect() {
     Die();
   }
 
-#    if defined(__aarch64__) && defined(__APPLE__) && SANITIZER_IOS
+#    if SANITIZER_IOS && !SANITIZER_IOSSIM
   ProtectRange(HeapMemEnd(), ShadowBeg());
   ProtectRange(ShadowEnd(), MetaShadowBeg());
-  ProtectRange(MetaShadowEnd(), TraceMemBeg());
-#else
+  ProtectRange(MetaShadowEnd(), HiAppMemBeg());
+#    else
   ProtectRange(LoAppMemEnd(), ShadowBeg());
   ProtectRange(ShadowEnd(), MetaShadowBeg());
   if (MidAppMemBeg()) {
     ProtectRange(MetaShadowEnd(), MidAppMemBeg());
-    ProtectRange(MidAppMemEnd(), TraceMemBeg());
+    ProtectRange(MidAppMemEnd(), HeapMemBeg());
   } else {
-    ProtectRange(MetaShadowEnd(), TraceMemBeg());
+    ProtectRange(MetaShadowEnd(), HeapMemBeg());
   }
-  // Memory for traces is mapped lazily in MapThreadTrace.
-  // Protect the whole range for now, so that user does not map something here.
-  ProtectRange(TraceMemBeg(), TraceMemEnd());
-  ProtectRange(TraceMemEnd(), HeapMemBeg());
   ProtectRange(HeapEnd(), HiAppMemBeg());
-#endif
+#    endif
 
-#if defined(__s390x__)
+#    if defined(__s390x__)
   // Protect the rest of the address space.
   const uptr user_addr_max_l4 = 0x0020000000000000ull;
   const uptr user_addr_max_l5 = 0xfffffffffffff000ull;
diff --git a/compiler-rt/lib/tsan/rtl/tsan_platform_windows.cpp b/compiler-rt/lib/tsan/rtl/tsan_platform_windows.cpp
index fea893768c79..eb8f354742f4 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_platform_windows.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_platform_windows.cpp
@@ -20,9 +20,6 @@
 
 namespace __tsan {
 
-void FlushShadowMemory() {
-}
-
 void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns) {}
 
 void InitializePlatformEarly() {
diff --git a/compiler-rt/lib/tsan/rtl/tsan_report.cpp b/compiler-rt/lib/tsan/rtl/tsan_report.cpp
index a926c3761ccf..10d9c761b8ee 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_report.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_report.cpp
@@ -126,7 +126,7 @@ static void PrintMutexSet(Vector<ReportMopMutex> const& mset) {
     if (i == 0)
       Printf(" (mutexes:");
     const ReportMopMutex m = mset[i];
-    Printf(" %s M%llu", m.write ? "write" : "read", m.id);
+    Printf(" %s M%u", m.write ? "write" : "read", m.id);
     Printf(i == mset.Size() - 1 ? ")" : ",");
   }
 }
@@ -211,29 +211,23 @@ static void PrintLocation(const ReportLocation *loc) {
 
 static void PrintMutexShort(const ReportMutex *rm, const char *after) {
   Decorator d;
-  Printf("%sM%lld%s%s", d.Mutex(), rm->id, d.Default(), after);
+  Printf("%sM%d%s%s", d.Mutex(), rm->id, d.Default(), after);
 }
 
 static void PrintMutexShortWithAddress(const ReportMutex *rm,
                                        const char *after) {
   Decorator d;
-  Printf("%sM%lld (%p)%s%s", d.Mutex(), rm->id,
+  Printf("%sM%d (%p)%s%s", d.Mutex(), rm->id,
          reinterpret_cast<void *>(rm->addr), d.Default(), after);
 }
 
 static void PrintMutex(const ReportMutex *rm) {
   Decorator d;
-  if (rm->destroyed) {
-    Printf("%s", d.Mutex());
-    Printf("  Mutex M%llu is already destroyed.\n\n", rm->id);
-    Printf("%s", d.Default());
-  } else {
-    Printf("%s", d.Mutex());
-    Printf("  Mutex M%llu (%p) created at:\n", rm->id,
-           reinterpret_cast<void *>(rm->addr));
-    Printf("%s", d.Default());
-    PrintStack(rm->stack);
-  }
+  Printf("%s", d.Mutex());
+  Printf("  Mutex M%u (%p) created at:\n", rm->id,
+         reinterpret_cast<void *>(rm->addr));
+  Printf("%s", d.Default());
+  PrintStack(rm->stack);
 }
 
 static void PrintThread(const ReportThread *rt) {
@@ -460,12 +454,12 @@ void PrintReport(const ReportDesc *rep) {
   } else if (rep->typ == ReportTypeDeadlock) {
     Printf("WARNING: DEADLOCK\n");
     for (uptr i = 0; i < rep->mutexes.Size(); i++) {
-      Printf("Goroutine %d lock mutex %llu while holding mutex %llu:\n", 999,
+      Printf("Goroutine %d lock mutex %u while holding mutex %u:\n", 999,
              rep->mutexes[i]->id,
              rep->mutexes[(i + 1) % rep->mutexes.Size()]->id);
       PrintStack(rep->stacks[2*i]);
       Printf("\n");
-      Printf("Mutex %llu was previously locked here:\n",
+      Printf("Mutex %u was previously locked here:\n",
              rep->mutexes[(i + 1) % rep->mutexes.Size()]->id);
       PrintStack(rep->stacks[2*i + 1]);
       Printf("\n");
diff --git a/compiler-rt/lib/tsan/rtl/tsan_report.h b/compiler-rt/lib/tsan/rtl/tsan_report.h
index d68c2db88828..3b367f38e266 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_report.h
+++ b/compiler-rt/lib/tsan/rtl/tsan_report.h
@@ -43,7 +43,7 @@ struct ReportStack {
 };
 
 struct ReportMopMutex {
-  u64 id;
+  int id;
   bool write;
 };
 
@@ -91,9 +91,8 @@ struct ReportThread {
 };
 
 struct ReportMutex {
-  u64 id;
+  int id;
   uptr addr;
-  bool destroyed;
   ReportStack *stack;
 };
 
diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp b/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp
index c14af9788e32..ed60e250cff8 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp
@@ -57,110 +57,352 @@ Context *ctx;
 bool OnFinalize(bool failed);
 void OnInitialize();
 #else
-#include <dlfcn.h>
 SANITIZER_WEAK_CXX_DEFAULT_IMPL
 bool OnFinalize(bool failed) {
-#if !SANITIZER_GO
+#  if !SANITIZER_GO
   if (on_finalize)
     return on_finalize(failed);
-#endif
+#  endif
   return failed;
 }
+
 SANITIZER_WEAK_CXX_DEFAULT_IMPL
 void OnInitialize() {
-#if !SANITIZER_GO
+#  if !SANITIZER_GO
   if (on_initialize)
     on_initialize();
-#endif
+#  endif
 }
 #endif
 
-static ThreadContextBase *CreateThreadContext(Tid tid) {
-  // Map thread trace when context is created.
-  char name[50];
-  internal_snprintf(name, sizeof(name), "trace %u", tid);
-  MapThreadTrace(GetThreadTrace(tid), TraceSize() * sizeof(Event), name);
-  const uptr hdr = GetThreadTraceHeader(tid);
-  internal_snprintf(name, sizeof(name), "trace header %u", tid);
-  MapThreadTrace(hdr, sizeof(Trace), name);
-  new((void*)hdr) Trace();
-  // We are going to use only a small part of the trace with the default
-  // value of history_size. However, the constructor writes to the whole trace.
-  // Release the unused part.
-  uptr hdr_end = hdr + sizeof(Trace);
-  hdr_end -= sizeof(TraceHeader) * (kTraceParts - TraceParts());
-  hdr_end = RoundUp(hdr_end, GetPageSizeCached());
-  if (hdr_end < hdr + sizeof(Trace)) {
-    ReleaseMemoryPagesToOS(hdr_end, hdr + sizeof(Trace));
-    uptr unused = hdr + sizeof(Trace) - hdr_end;
-    if (hdr_end != (uptr)MmapFixedNoAccess(hdr_end, unused)) {
-      Report("ThreadSanitizer: failed to mprotect [0x%zx-0x%zx) \n", hdr_end,
-             unused);
-      CHECK("unable to mprotect" && 0);
+static TracePart* TracePartAlloc(ThreadState* thr) {
+  TracePart* part = nullptr;
+  {
+    Lock lock(&ctx->slot_mtx);
+    uptr max_parts = Trace::kMinParts + flags()->history_size;
+    Trace* trace = &thr->tctx->trace;
+    if (trace->parts_allocated == max_parts ||
+        ctx->trace_part_finished_excess) {
+      part = ctx->trace_part_recycle.PopFront();
+      DPrintf("#%d: TracePartAlloc: part=%p\n", thr->tid, part);
+      if (part && part->trace) {
+        Trace* trace1 = part->trace;
+        Lock trace_lock(&trace1->mtx);
+        part->trace = nullptr;
+        TracePart* part1 = trace1->parts.PopFront();
+        CHECK_EQ(part, part1);
+        if (trace1->parts_allocated > trace1->parts.Size()) {
+          ctx->trace_part_finished_excess +=
+              trace1->parts_allocated - trace1->parts.Size();
+          trace1->parts_allocated = trace1->parts.Size();
+        }
+      }
+    }
+    if (trace->parts_allocated < max_parts) {
+      trace->parts_allocated++;
+      if (ctx->trace_part_finished_excess)
+        ctx->trace_part_finished_excess--;
+    }
+    if (!part)
+      ctx->trace_part_total_allocated++;
+    else if (ctx->trace_part_recycle_finished)
+      ctx->trace_part_recycle_finished--;
+  }
+  if (!part)
+    part = new (MmapOrDie(sizeof(*part), "TracePart")) TracePart();
+  return part;
+}
+
+static void TracePartFree(TracePart* part) REQUIRES(ctx->slot_mtx) {
+  DCHECK(part->trace);
+  part->trace = nullptr;
+  ctx->trace_part_recycle.PushFront(part);
+}
+
+void TraceResetForTesting() {
+  Lock lock(&ctx->slot_mtx);
+  while (auto* part = ctx->trace_part_recycle.PopFront()) {
+    if (auto trace = part->trace)
+      CHECK_EQ(trace->parts.PopFront(), part);
+    UnmapOrDie(part, sizeof(*part));
+  }
+  ctx->trace_part_total_allocated = 0;
+  ctx->trace_part_recycle_finished = 0;
+  ctx->trace_part_finished_excess = 0;
+}
+
+static void DoResetImpl(uptr epoch) {
+  ThreadRegistryLock lock0(&ctx->thread_registry);
+  Lock lock1(&ctx->slot_mtx);
+  CHECK_EQ(ctx->global_epoch, epoch);
+  ctx->global_epoch++;
+  CHECK(!ctx->resetting);
+  ctx->resetting = true;
+  for (u32 i = ctx->thread_registry.NumThreadsLocked(); i--;) {
+    ThreadContext* tctx = (ThreadContext*)ctx->thread_registry.GetThreadLocked(
+        static_cast<Tid>(i));
+    // Potentially we could purge all ThreadStatusDead threads from the
+    // registry. Since we reset all shadow, they can't race with anything
+    // anymore. However, their tid's can still be stored in some aux places
+    // (e.g. tid of thread that created something).
+    auto trace = &tctx->trace;
+    Lock lock(&trace->mtx);
+    bool attached = tctx->thr && tctx->thr->slot;
+    auto parts = &trace->parts;
+    bool local = false;
+    while (!parts->Empty()) {
+      auto part = parts->Front();
+      local = local || part == trace->local_head;
+      if (local)
+        CHECK(!ctx->trace_part_recycle.Queued(part));
+      else
+        ctx->trace_part_recycle.Remove(part);
+      if (attached && parts->Size() == 1) {
+        // The thread is running and this is the last/current part.
+        // Set the trace position to the end of the current part
+        // to force the thread to call SwitchTracePart and re-attach
+        // to a new slot and allocate a new trace part.
+        // Note: the thread is concurrently modifying the position as well,
+        // so this is only best-effort. The thread can only modify position
+        // within this part, because switching parts is protected by
+        // slot/trace mutexes that we hold here.
+        atomic_store_relaxed(
+            &tctx->thr->trace_pos,
+            reinterpret_cast<uptr>(&part->events[TracePart::kSize]));
+        break;
+      }
+      parts->Remove(part);
+      TracePartFree(part);
+    }
+    CHECK_LE(parts->Size(), 1);
+    trace->local_head = parts->Front();
+    if (tctx->thr && !tctx->thr->slot) {
+      atomic_store_relaxed(&tctx->thr->trace_pos, 0);
+      tctx->thr->trace_prev_pc = 0;
+    }
+    if (trace->parts_allocated > trace->parts.Size()) {
+      ctx->trace_part_finished_excess +=
+          trace->parts_allocated - trace->parts.Size();
+      trace->parts_allocated = trace->parts.Size();
+    }
+  }
+  while (ctx->slot_queue.PopFront()) {
+  }
+  for (auto& slot : ctx->slots) {
+    slot.SetEpoch(kEpochZero);
+    slot.journal.Reset();
+    slot.thr = nullptr;
+    ctx->slot_queue.PushBack(&slot);
+  }
+
+  DPrintf("Resetting shadow...\n");
+  if (!MmapFixedSuperNoReserve(ShadowBeg(), ShadowEnd() - ShadowBeg(),
+                               "shadow")) {
+    Printf("failed to reset shadow memory\n");
+    Die();
+  }
+  DPrintf("Resetting meta shadow...\n");
+  ctx->metamap.ResetClocks();
+  ctx->resetting = false;
+}
+
+// Clang does not understand locking all slots in the loop:
+// error: expecting mutex 'slot.mtx' to be held at start of each loop
+void DoReset(ThreadState* thr, uptr epoch) NO_THREAD_SAFETY_ANALYSIS {
+  {
+    for (auto& slot : ctx->slots) {
+      slot.mtx.Lock();
+      if (UNLIKELY(epoch == 0))
+        epoch = ctx->global_epoch;
+      if (UNLIKELY(epoch != ctx->global_epoch)) {
+        // Epoch can't change once we've locked the first slot.
+        CHECK_EQ(slot.sid, 0);
+        slot.mtx.Unlock();
+        return;
+      }
+    }
+  }
+  DPrintf("#%d: DoReset epoch=%lu\n", thr ? thr->tid : -1, epoch);
+  DoResetImpl(epoch);
+  for (auto& slot : ctx->slots) slot.mtx.Unlock();
+}
+
+void FlushShadowMemory() { DoReset(nullptr, 0); }
+
+static TidSlot* FindSlotAndLock(ThreadState* thr)
+    ACQUIRE(thr->slot->mtx) NO_THREAD_SAFETY_ANALYSIS {
+  CHECK(!thr->slot);
+  TidSlot* slot = nullptr;
+  for (;;) {
+    uptr epoch;
+    {
+      Lock lock(&ctx->slot_mtx);
+      epoch = ctx->global_epoch;
+      if (slot) {
+        // This is an exhausted slot from the previous iteration.
+        if (ctx->slot_queue.Queued(slot))
+          ctx->slot_queue.Remove(slot);
+        thr->slot_locked = false;
+        slot->mtx.Unlock();
+      }
+      for (;;) {
+        slot = ctx->slot_queue.PopFront();
+        if (!slot)
+          break;
+        if (slot->epoch() != kEpochLast) {
+          ctx->slot_queue.PushBack(slot);
+          break;
+        }
+      }
+    }
+    if (!slot) {
+      DoReset(thr, epoch);
+      continue;
     }
+    slot->mtx.Lock();
+    CHECK(!thr->slot_locked);
+    thr->slot_locked = true;
+    if (slot->thr) {
+      DPrintf("#%d: preempting sid=%d tid=%d\n", thr->tid, (u32)slot->sid,
+              slot->thr->tid);
+      slot->SetEpoch(slot->thr->fast_state.epoch());
+      slot->thr = nullptr;
+    }
+    if (slot->epoch() != kEpochLast)
+      return slot;
   }
-  return New<ThreadContext>(tid);
 }
 
+void SlotAttachAndLock(ThreadState* thr) {
+  TidSlot* slot = FindSlotAndLock(thr);
+  DPrintf("#%d: SlotAttach: slot=%u\n", thr->tid, static_cast<int>(slot->sid));
+  CHECK(!slot->thr);
+  CHECK(!thr->slot);
+  slot->thr = thr;
+  thr->slot = slot;
+  Epoch epoch = EpochInc(slot->epoch());
+  CHECK(!EpochOverflow(epoch));
+  slot->SetEpoch(epoch);
+  thr->fast_state.SetSid(slot->sid);
+  thr->fast_state.SetEpoch(epoch);
+  if (thr->slot_epoch != ctx->global_epoch) {
+    thr->slot_epoch = ctx->global_epoch;
+    thr->clock.Reset();
 #if !SANITIZER_GO
-static const u32 kThreadQuarantineSize = 16;
-#else
-static const u32 kThreadQuarantineSize = 64;
+    thr->last_sleep_stack_id = kInvalidStackID;
+    thr->last_sleep_clock.Reset();
+#endif
+  }
+  thr->clock.Set(slot->sid, epoch);
+  slot->journal.PushBack({thr->tid, epoch});
+}
+
+static void SlotDetachImpl(ThreadState* thr, bool exiting) {
+  TidSlot* slot = thr->slot;
+  thr->slot = nullptr;
+  if (thr != slot->thr) {
+    slot = nullptr;  // we don't own the slot anymore
+    if (thr->slot_epoch != ctx->global_epoch) {
+      TracePart* part = nullptr;
+      auto* trace = &thr->tctx->trace;
+      {
+        Lock l(&trace->mtx);
+        auto* parts = &trace->parts;
+        // The trace can be completely empty in an unlikely event
+        // the thread is preempted right after it acquired the slot
+        // in ThreadStart and did not trace any events yet.
+        CHECK_LE(parts->Size(), 1);
+        part = parts->PopFront();
+        thr->tctx->trace.local_head = nullptr;
+        atomic_store_relaxed(&thr->trace_pos, 0);
+        thr->trace_prev_pc = 0;
+      }
+      if (part) {
+        Lock l(&ctx->slot_mtx);
+        TracePartFree(part);
+      }
+    }
+    return;
+  }
+  CHECK(exiting || thr->fast_state.epoch() == kEpochLast);
+  slot->SetEpoch(thr->fast_state.epoch());
+  slot->thr = nullptr;
+}
+
+void SlotDetach(ThreadState* thr) {
+  Lock lock(&thr->slot->mtx);
+  SlotDetachImpl(thr, true);
+}
+
+void SlotLock(ThreadState* thr) NO_THREAD_SAFETY_ANALYSIS {
+  DCHECK(!thr->slot_locked);
+#if SANITIZER_DEBUG
+  // Check these mutexes are not locked.
+  // We can call DoReset from SlotAttachAndLock, which will lock
+  // these mutexes, but it happens only every once in a while.
+  { ThreadRegistryLock lock(&ctx->thread_registry); }
+  { Lock lock(&ctx->slot_mtx); }
 #endif
+  TidSlot* slot = thr->slot;
+  slot->mtx.Lock();
+  thr->slot_locked = true;
+  if (LIKELY(thr == slot->thr && thr->fast_state.epoch() != kEpochLast))
+    return;
+  SlotDetachImpl(thr, false);
+  thr->slot_locked = false;
+  slot->mtx.Unlock();
+  SlotAttachAndLock(thr);
+}
+
+void SlotUnlock(ThreadState* thr) {
+  DCHECK(thr->slot_locked);
+  thr->slot_locked = false;
+  thr->slot->mtx.Unlock();
+}
 
 Context::Context()
     : initialized(),
       report_mtx(MutexTypeReport),
       nreported(),
-      thread_registry(CreateThreadContext, kMaxTid, kThreadQuarantineSize,
-                      kMaxTidReuse),
+      thread_registry([](Tid tid) -> ThreadContextBase* {
+        return new (Alloc(sizeof(ThreadContext))) ThreadContext(tid);
+      }),
       racy_mtx(MutexTypeRacy),
       racy_stacks(),
       racy_addresses(),
       fired_suppressions_mtx(MutexTypeFired),
-      clock_alloc(LINKER_INITIALIZED, "clock allocator") {
+      slot_mtx(MutexTypeSlots),
+      resetting() {
   fired_suppressions.reserve(8);
+  for (uptr i = 0; i < ARRAY_SIZE(slots); i++) {
+    TidSlot* slot = &slots[i];
+    slot->sid = static_cast<Sid>(i);
+    slot_queue.PushBack(slot);
+  }
+  global_epoch = 1;
 }
 
+TidSlot::TidSlot() : mtx(MutexTypeSlot) {}
+
 // The objects are allocated in TLS, so one may rely on zero-initialization.
-ThreadState::ThreadState(Context *ctx, Tid tid, int unique_id, u64 epoch,
-                         unsigned reuse_count, uptr stk_addr, uptr stk_size,
-                         uptr tls_addr, uptr tls_size)
-    : fast_state(tid, epoch)
-      // Do not touch these, rely on zero initialization,
-      // they may be accessed before the ctor.
-      // , ignore_reads_and_writes()
-      // , ignore_interceptors()
-      ,
-      clock(tid, reuse_count)
-#if !SANITIZER_GO
-      ,
-      jmp_bufs()
-#endif
-      ,
-      tid(tid),
-      unique_id(unique_id),
-      stk_addr(stk_addr),
-      stk_size(stk_size),
-      tls_addr(tls_addr),
-      tls_size(tls_size)
-#if !SANITIZER_GO
-      ,
-      last_sleep_clock(tid)
-#endif
-{
+ThreadState::ThreadState(Tid tid)
+    // Do not touch these, rely on zero initialization,
+    // they may be accessed before the ctor.
+    // ignore_reads_and_writes()
+    // ignore_interceptors()
+    : tid(tid) {
   CHECK_EQ(reinterpret_cast<uptr>(this) % SANITIZER_CACHE_LINE_SIZE, 0);
 #if !SANITIZER_GO
   // C/C++ uses fixed size shadow stack.
   const int kInitStackSize = kShadowStackSize;
-  shadow_stack = static_cast<uptr *>(
+  shadow_stack = static_cast<uptr*>(
       MmapNoReserveOrDie(kInitStackSize * sizeof(uptr), "shadow stack"));
   SetShadowRegionHugePageMode(reinterpret_cast<uptr>(shadow_stack),
                               kInitStackSize * sizeof(uptr));
 #else
   // Go uses malloc-allocated shadow stack with dynamic size.
   const int kInitStackSize = 8;
-  shadow_stack = static_cast<uptr *>(Alloc(kInitStackSize * sizeof(uptr)));
+  shadow_stack = static_cast<uptr*>(Alloc(kInitStackSize * sizeof(uptr)));
 #endif
   shadow_stack_pos = shadow_stack;
   shadow_stack_end = shadow_stack + kInitStackSize;
@@ -175,11 +417,11 @@ void MemoryProfiler(u64 uptime) {
   WriteToFile(ctx->memprof_fd, buf.data(), internal_strlen(buf.data()));
 }
 
-void InitializeMemoryProfiler() {
+static bool InitializeMemoryProfiler() {
   ctx->memprof_fd = kInvalidFd;
   const char *fname = flags()->profile_memory;
   if (!fname || !fname[0])
-    return;
+    return false;
   if (internal_strcmp(fname, "stdout") == 0) {
     ctx->memprof_fd = 1;
   } else if (internal_strcmp(fname, "stderr") == 0) {
@@ -191,11 +433,11 @@ void InitializeMemoryProfiler() {
     if (ctx->memprof_fd == kInvalidFd) {
       Printf("ThreadSanitizer: failed to open memory profile file '%s'\n",
              filename.data());
-      return;
+      return false;
     }
   }
   MemoryProfiler(0);
-  MaybeSpawnBackgroundThread();
+  return true;
 }
 
 static void *BackgroundThread(void *arg) {
@@ -207,33 +449,34 @@ static void *BackgroundThread(void *arg) {
   const u64 kMs2Ns = 1000 * 1000;
   const u64 start = NanoTime();
 
-  u64 last_flush = NanoTime();
+  u64 last_flush = start;
   uptr last_rss = 0;
-  for (int i = 0;
-      atomic_load(&ctx->stop_background_thread, memory_order_relaxed) == 0;
-      i++) {
+  while (!atomic_load_relaxed(&ctx->stop_background_thread)) {
     SleepForMillis(100);
     u64 now = NanoTime();
 
     // Flush memory if requested.
     if (flags()->flush_memory_ms > 0) {
       if (last_flush + flags()->flush_memory_ms * kMs2Ns < now) {
-        VPrintf(1, "ThreadSanitizer: periodic memory flush\n");
+        VReport(1, "ThreadSanitizer: periodic memory flush\n");
         FlushShadowMemory();
-        last_flush = NanoTime();
+        now = last_flush = NanoTime();
       }
     }
     if (flags()->memory_limit_mb > 0) {
       uptr rss = GetRSS();
       uptr limit = uptr(flags()->memory_limit_mb) << 20;
-      VPrintf(1, "ThreadSanitizer: memory flush check"
-                 " RSS=%llu LAST=%llu LIMIT=%llu\n",
+      VReport(1,
+              "ThreadSanitizer: memory flush check"
+              " RSS=%llu LAST=%llu LIMIT=%llu\n",
               (u64)rss >> 20, (u64)last_rss >> 20, (u64)limit >> 20);
       if (2 * rss > limit + last_rss) {
-        VPrintf(1, "ThreadSanitizer: flushing memory due to RSS\n");
+        VReport(1, "ThreadSanitizer: flushing memory due to RSS\n");
         FlushShadowMemory();
         rss = GetRSS();
-        VPrintf(1, "ThreadSanitizer: memory flushed RSS=%llu\n", (u64)rss>>20);
+        now = NanoTime();
+        VReport(1, "ThreadSanitizer: memory flushed RSS=%llu\n",
+                (u64)rss >> 20);
       }
       last_rss = rss;
     }
@@ -309,7 +552,8 @@ void UnmapShadow(ThreadState *thr, uptr addr, uptr size) {
     return;
   DontNeedShadowFor(addr, size);
   ScopedGlobalProcessor sgp;
-  ctx->metamap.ResetRange(thr->proc(), addr, size);
+  SlotLocker locker(thr, true);
+  ctx->metamap.ResetRange(thr->proc(), addr, size, true);
 }
 #endif
 
@@ -355,18 +599,6 @@ void MapShadow(uptr addr, uptr size) {
           addr + size, meta_begin, meta_end);
 }
 
-void MapThreadTrace(uptr addr, uptr size, const char *name) {
-  DPrintf("#0: Mapping trace at 0x%zx-0x%zx(0x%zx)\n", addr, addr + size, size);
-  CHECK_GE(addr, TraceMemBeg());
-  CHECK_LE(addr + size, TraceMemEnd());
-  CHECK_EQ(addr, addr & ~((64 << 10) - 1));  // windows wants 64K alignment
-  if (!MmapFixedSuperNoReserve(addr, size, name)) {
-    Printf("FATAL: ThreadSanitizer can not mmap thread trace (0x%zx/0x%zx)\n",
-           addr, size);
-    Die();
-  }
-}
-
 #if !SANITIZER_GO
 static void OnStackUnwind(const SignalContext &sig, const void *,
                           BufferedStackTrace *stack) {
@@ -385,8 +617,11 @@ void CheckUnwind() {
   // since we are going to die soon.
   ScopedIgnoreInterceptors ignore;
 #if !SANITIZER_GO
-  cur_thread()->ignore_sync++;
-  cur_thread()->ignore_reads_and_writes++;
+  ThreadState* thr = cur_thread();
+  thr->nomalloc = false;
+  thr->ignore_sync++;
+  thr->ignore_reads_and_writes++;
+  atomic_store_relaxed(&thr->in_signal_handler, 0);
 #endif
   PrintCurrentStackSlow(StackTrace::GetCurrentPc());
 }
@@ -441,22 +676,23 @@ void Initialize(ThreadState *thr) {
   Symbolizer::GetOrInit()->AddHooks(EnterSymbolizer, ExitSymbolizer);
 #endif
 
-  VPrintf(1, "***** Running under ThreadSanitizer v2 (pid %d) *****\n",
+  VPrintf(1, "***** Running under ThreadSanitizer v3 (pid %d) *****\n",
           (int)internal_getpid());
 
   // Initialize thread 0.
-  Tid tid = ThreadCreate(thr, 0, 0, true);
+  Tid tid = ThreadCreate(nullptr, 0, 0, true);
   CHECK_EQ(tid, kMainTid);
   ThreadStart(thr, tid, GetTid(), ThreadType::Regular);
 #if TSAN_CONTAINS_UBSAN
   __ubsan::InitAsPlugin();
 #endif
-  ctx->initialized = true;
 
 #if !SANITIZER_GO
   Symbolizer::LateInitialize();
-  InitializeMemoryProfiler();
+  if (InitializeMemoryProfiler() || flags()->force_background_thread)
+    MaybeSpawnBackgroundThread();
 #endif
+  ctx->initialized = true;
 
   if (flags()->stop_on_start) {
     Printf("ThreadSanitizer is suspended at startup (pid %d)."
@@ -482,7 +718,6 @@ void MaybeSpawnBackgroundThread() {
 #endif
 }
 
-
 int Finalize(ThreadState *thr) {
   bool failed = false;
 
@@ -490,12 +725,12 @@ int Finalize(ThreadState *thr) {
     DumpProcessMap();
 
   if (flags()->atexit_sleep_ms > 0 && ThreadCount(thr) > 1)
-    SleepForMillis(flags()->atexit_sleep_ms);
+    internal_usleep(u64(flags()->atexit_sleep_ms) * 1000);
 
-  // Wait for pending reports.
-  ctx->report_mtx.Lock();
-  { ScopedErrorReportLock l; }
-  ctx->report_mtx.Unlock();
+  {
+    // Wait for pending reports.
+    ScopedErrorReportLock lock;
+  }
 
 #if !SANITIZER_GO
   if (Verbosity()) AllocatorPrintStats();
@@ -522,8 +757,13 @@ int Finalize(ThreadState *thr) {
 
 #if !SANITIZER_GO
 void ForkBefore(ThreadState *thr, uptr pc) NO_THREAD_SAFETY_ANALYSIS {
+  GlobalProcessorLock();
+  // Detaching from the slot makes OnUserFree skip writing to the shadow.
+  // The slot will be locked so any attempts to use it will deadlock anyway.
+  SlotDetach(thr);
+  for (auto& slot : ctx->slots) slot.mtx.Lock();
   ctx->thread_registry.Lock();
-  ctx->report_mtx.Lock();
+  ctx->slot_mtx.Lock();
   ScopedErrorReportLock::Lock();
   AllocatorLock();
   // Suppress all reports in the pthread_atfork callbacks.
@@ -543,30 +783,29 @@ void ForkBefore(ThreadState *thr, uptr pc) NO_THREAD_SAFETY_ANALYSIS {
   __tsan_test_only_on_fork();
 }
 
-void ForkParentAfter(ThreadState *thr, uptr pc) NO_THREAD_SAFETY_ANALYSIS {
+static void ForkAfter(ThreadState* thr) NO_THREAD_SAFETY_ANALYSIS {
   thr->suppress_reports--;  // Enabled in ForkBefore.
   thr->ignore_interceptors--;
   thr->ignore_reads_and_writes--;
   AllocatorUnlock();
   ScopedErrorReportLock::Unlock();
-  ctx->report_mtx.Unlock();
+  ctx->slot_mtx.Unlock();
   ctx->thread_registry.Unlock();
+  for (auto& slot : ctx->slots) slot.mtx.Unlock();
+  SlotAttachAndLock(thr);
+  SlotUnlock(thr);
+  GlobalProcessorUnlock();
 }
 
-void ForkChildAfter(ThreadState *thr, uptr pc,
-                    bool start_thread) NO_THREAD_SAFETY_ANALYSIS {
-  thr->suppress_reports--;  // Enabled in ForkBefore.
-  thr->ignore_interceptors--;
-  thr->ignore_reads_and_writes--;
-  AllocatorUnlock();
-  ScopedErrorReportLock::Unlock();
-  ctx->report_mtx.Unlock();
-  ctx->thread_registry.Unlock();
+void ForkParentAfter(ThreadState* thr, uptr pc) { ForkAfter(thr); }
 
-  uptr nthread = 0;
-  ctx->thread_registry.GetNumberOfThreads(0, 0, &nthread /* alive threads */);
-  VPrintf(1, "ThreadSanitizer: forked new process with pid %d,"
-      " parent had %d threads\n", (int)internal_getpid(), (int)nthread);
+void ForkChildAfter(ThreadState* thr, uptr pc, bool start_thread) {
+  ForkAfter(thr);
+  u32 nthread = ctx->thread_registry.OnFork(thr->tid);
+  VPrintf(1,
+          "ThreadSanitizer: forked new process with pid %d,"
+          " parent had %d threads\n",
+          (int)internal_getpid(), (int)nthread);
   if (nthread == 1) {
     if (start_thread)
       StartBackgroundThread();
@@ -576,6 +815,7 @@ void ForkChildAfter(ThreadState *thr, uptr pc,
     // ignores for everything in the hope that we will exec soon.
     ctx->after_multithreaded_fork = true;
     thr->ignore_interceptors++;
+    thr->suppress_reports++;
     ThreadIgnoreBegin(thr, pc);
     ThreadIgnoreSyncBegin(thr, pc);
   }
@@ -597,8 +837,10 @@ void GrowShadowStack(ThreadState *thr) {
 #endif
 
 StackID CurrentStackId(ThreadState *thr, uptr pc) {
+#if !SANITIZER_GO
   if (!thr->is_inited)  // May happen during bootstrap.
     return kInvalidStackID;
+#endif
   if (pc != 0) {
 #if !SANITIZER_GO
     DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
@@ -616,53 +858,72 @@ StackID CurrentStackId(ThreadState *thr, uptr pc) {
   return id;
 }
 
-namespace v3 {
-
-NOINLINE
-void TraceSwitchPart(ThreadState *thr) {
+static bool TraceSkipGap(ThreadState* thr) {
   Trace *trace = &thr->tctx->trace;
   Event *pos = reinterpret_cast<Event *>(atomic_load_relaxed(&thr->trace_pos));
   DCHECK_EQ(reinterpret_cast<uptr>(pos + 1) & TracePart::kAlignment, 0);
   auto *part = trace->parts.Back();
-  DPrintf("TraceSwitchPart part=%p pos=%p\n", part, pos);
-  if (part) {
-    // We can get here when we still have space in the current trace part.
-    // The fast-path check in TraceAcquire has false positives in the middle of
-    // the part. Check if we are indeed at the end of the current part or not,
-    // and fill any gaps with NopEvent's.
-    Event *end = &part->events[TracePart::kSize];
-    DCHECK_GE(pos, &part->events[0]);
-    DCHECK_LE(pos, end);
-    if (pos + 1 < end) {
-      if ((reinterpret_cast<uptr>(pos) & TracePart::kAlignment) ==
-          TracePart::kAlignment)
-        *pos++ = NopEvent;
+  DPrintf("#%d: TraceSwitchPart enter trace=%p parts=%p-%p pos=%p\n", thr->tid,
+          trace, trace->parts.Front(), part, pos);
+  if (!part)
+    return false;
+  // We can get here when we still have space in the current trace part.
+  // The fast-path check in TraceAcquire has false positives in the middle of
+  // the part. Check if we are indeed at the end of the current part or not,
+  // and fill any gaps with NopEvent's.
+  Event* end = &part->events[TracePart::kSize];
+  DCHECK_GE(pos, &part->events[0]);
+  DCHECK_LE(pos, end);
+  if (pos + 1 < end) {
+    if ((reinterpret_cast<uptr>(pos) & TracePart::kAlignment) ==
+        TracePart::kAlignment)
       *pos++ = NopEvent;
-      DCHECK_LE(pos + 2, end);
-      atomic_store_relaxed(&thr->trace_pos, reinterpret_cast<uptr>(pos));
-      // Ensure we setup trace so that the next TraceAcquire
-      // won't detect trace part end.
-      Event *ev;
-      CHECK(TraceAcquire(thr, &ev));
-      return;
-    }
-    // We are indeed at the end.
-    for (; pos < end; pos++) *pos = NopEvent;
+    *pos++ = NopEvent;
+    DCHECK_LE(pos + 2, end);
+    atomic_store_relaxed(&thr->trace_pos, reinterpret_cast<uptr>(pos));
+    return true;
   }
+  // We are indeed at the end.
+  for (; pos < end; pos++) *pos = NopEvent;
+  return false;
+}
+
+NOINLINE
+void TraceSwitchPart(ThreadState* thr) {
+  if (TraceSkipGap(thr))
+    return;
 #if !SANITIZER_GO
   if (ctx->after_multithreaded_fork) {
     // We just need to survive till exec.
-    CHECK(part);
-    atomic_store_relaxed(&thr->trace_pos,
-                         reinterpret_cast<uptr>(&part->events[0]));
-    return;
+    TracePart* part = thr->tctx->trace.parts.Back();
+    if (part) {
+      atomic_store_relaxed(&thr->trace_pos,
+                           reinterpret_cast<uptr>(&part->events[0]));
+      return;
+    }
   }
 #endif
-  part = new (MmapOrDie(sizeof(TracePart), "TracePart")) TracePart();
+  TraceSwitchPartImpl(thr);
+}
+
+void TraceSwitchPartImpl(ThreadState* thr) {
+  SlotLocker locker(thr, true);
+  Trace* trace = &thr->tctx->trace;
+  TracePart* part = TracePartAlloc(thr);
   part->trace = trace;
   thr->trace_prev_pc = 0;
+  TracePart* recycle = nullptr;
+  // Keep roughly half of parts local to the thread
+  // (not queued into the recycle queue).
+  uptr local_parts = (Trace::kMinParts + flags()->history_size + 1) / 2;
   {
     Lock lock(&trace->mtx);
+    if (trace->parts.Empty())
+      trace->local_head = part;
+    if (trace->parts.Size() >= local_parts) {
+      recycle = trace->local_head;
+      trace->local_head = trace->parts.Next(recycle);
+    }
     trace->parts.PushBack(part);
     atomic_store_relaxed(&thr->trace_pos,
                          reinterpret_cast<uptr>(&part->events[0]));
@@ -670,60 +931,49 @@ void TraceSwitchPart(ThreadState *thr) {
   // Make this part self-sufficient by restoring the current stack
   // and mutex set in the beginning of the trace.
   TraceTime(thr);
-  for (uptr *pos = &thr->shadow_stack[0]; pos < thr->shadow_stack_pos; pos++)
-    CHECK(TryTraceFunc(thr, *pos));
+  {
+    // Pathologically large stacks may not fit into the part.
+    // In these cases we log only fixed number of top frames.
+    const uptr kMaxFrames = 1000;
+    // Sanity check that kMaxFrames won't consume the whole part.
+    static_assert(kMaxFrames < TracePart::kSize / 2, "kMaxFrames is too big");
+    uptr* pos = Max(&thr->shadow_stack[0], thr->shadow_stack_pos - kMaxFrames);
+    for (; pos < thr->shadow_stack_pos; pos++) {
+      if (TryTraceFunc(thr, *pos))
+        continue;
+      CHECK(TraceSkipGap(thr));
+      CHECK(TryTraceFunc(thr, *pos));
+    }
+  }
   for (uptr i = 0; i < thr->mset.Size(); i++) {
     MutexSet::Desc d = thr->mset.Get(i);
-    TraceMutexLock(thr, d.write ? EventType::kLock : EventType::kRLock, 0,
-                   d.addr, d.stack_id);
+    for (uptr i = 0; i < d.count; i++)
+      TraceMutexLock(thr, d.write ? EventType::kLock : EventType::kRLock, 0,
+                     d.addr, d.stack_id);
   }
+  {
+    Lock lock(&ctx->slot_mtx);
+    // There is a small chance that the slot may be not queued at this point.
+    // This can happen if the slot has kEpochLast epoch and another thread
+    // in FindSlotAndLock discovered that it's exhausted and removed it from
+    // the slot queue. kEpochLast can happen in 2 cases: (1) if TraceSwitchPart
+    // was called with the slot locked and epoch already at kEpochLast,
+    // or (2) if we've acquired a new slot in SlotLock in the beginning
+    // of the function and the slot was at kEpochLast - 1, so after increment
+    // in SlotAttachAndLock it become kEpochLast.
+    if (ctx->slot_queue.Queued(thr->slot)) {
+      ctx->slot_queue.Remove(thr->slot);
+      ctx->slot_queue.PushBack(thr->slot);
+    }
+    if (recycle)
+      ctx->trace_part_recycle.PushBack(recycle);
+  }
+  DPrintf("#%d: TraceSwitchPart exit parts=%p-%p pos=0x%zx\n", thr->tid,
+          trace->parts.Front(), trace->parts.Back(),
+          atomic_load_relaxed(&thr->trace_pos));
 }
 
-}  // namespace v3
-
-void TraceSwitch(ThreadState *thr) {
-#if !SANITIZER_GO
-  if (ctx->after_multithreaded_fork)
-    return;
-#endif
-  thr->nomalloc++;
-  Trace *thr_trace = ThreadTrace(thr->tid);
-  Lock l(&thr_trace->mtx);
-  unsigned trace = (thr->fast_state.epoch() / kTracePartSize) % TraceParts();
-  TraceHeader *hdr = &thr_trace->headers[trace];
-  hdr->epoch0 = thr->fast_state.epoch();
-  ObtainCurrentStack(thr, 0, &hdr->stack0);
-  hdr->mset0 = thr->mset;
-  thr->nomalloc--;
-}
-
-Trace *ThreadTrace(Tid tid) { return (Trace *)GetThreadTraceHeader(tid); }
-
-uptr TraceTopPC(ThreadState *thr) {
-  Event *events = (Event*)GetThreadTrace(thr->tid);
-  uptr pc = events[thr->fast_state.GetTracePos()];
-  return pc;
-}
-
-uptr TraceSize() {
-  return (uptr)(1ull << (kTracePartSizeBits + flags()->history_size + 1));
-}
-
-uptr TraceParts() {
-  return TraceSize() / kTracePartSize;
-}
-
-#if !SANITIZER_GO
-extern "C" void __tsan_trace_switch() {
-  TraceSwitch(cur_thread());
-}
-
-extern "C" void __tsan_report_race() {
-  ReportRace(cur_thread());
-}
-#endif
-
-void ThreadIgnoreBegin(ThreadState *thr, uptr pc) {
+void ThreadIgnoreBegin(ThreadState* thr, uptr pc) {
   DPrintf("#%d: ThreadIgnoreBegin\n", thr->tid);
   thr->ignore_reads_and_writes++;
   CHECK_GT(thr->ignore_reads_and_writes, 0);
@@ -783,7 +1033,6 @@ void build_consistency_debug() {}
 #else
 void build_consistency_release() {}
 #endif
-
 }  // namespace __tsan
 
 #if SANITIZER_CHECK_DEADLOCKS
@@ -791,21 +1040,27 @@ namespace __sanitizer {
 using namespace __tsan;
 MutexMeta mutex_meta[] = {
     {MutexInvalid, "Invalid", {}},
-    {MutexThreadRegistry, "ThreadRegistry", {}},
-    {MutexTypeTrace, "Trace", {}},
-    {MutexTypeReport,
-     "Report",
-     {MutexTypeSyncVar, MutexTypeGlobalProc, MutexTypeTrace}},
-    {MutexTypeSyncVar, "SyncVar", {MutexTypeTrace}},
+    {MutexThreadRegistry,
+     "ThreadRegistry",
+     {MutexTypeSlots, MutexTypeTrace, MutexTypeReport}},
+    {MutexTypeReport, "Report", {MutexTypeTrace}},
+    {MutexTypeSyncVar, "SyncVar", {MutexTypeReport, MutexTypeTrace}},
     {MutexTypeAnnotations, "Annotations", {}},
-    {MutexTypeAtExit, "AtExit", {MutexTypeSyncVar}},
+    {MutexTypeAtExit, "AtExit", {}},
     {MutexTypeFired, "Fired", {MutexLeaf}},
     {MutexTypeRacy, "Racy", {MutexLeaf}},
-    {MutexTypeGlobalProc, "GlobalProc", {}},
+    {MutexTypeGlobalProc, "GlobalProc", {MutexTypeSlot, MutexTypeSlots}},
     {MutexTypeInternalAlloc, "InternalAlloc", {MutexLeaf}},
+    {MutexTypeTrace, "Trace", {}},
+    {MutexTypeSlot,
+     "Slot",
+     {MutexMulti, MutexTypeTrace, MutexTypeSyncVar, MutexThreadRegistry,
+      MutexTypeSlots}},
+    {MutexTypeSlots, "Slots", {MutexTypeTrace, MutexTypeReport}},
     {},
 };
 
 void PrintMutexPC(uptr pc) { StackTrace(&pc, 1).Print(); }
+
 }  // namespace __sanitizer
 #endif
diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl.h b/compiler-rt/lib/tsan/rtl/tsan_rtl.h
index c71b27e1cbf5..d06358b462eb 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_rtl.h
+++ b/compiler-rt/lib/tsan/rtl/tsan_rtl.h
@@ -34,10 +34,10 @@
 #include "sanitizer_common/sanitizer_suppressions.h"
 #include "sanitizer_common/sanitizer_thread_registry.h"
 #include "sanitizer_common/sanitizer_vector.h"
-#include "tsan_clock.h"
 #include "tsan_defs.h"
 #include "tsan_flags.h"
 #include "tsan_ignoreset.h"
+#include "tsan_ilist.h"
 #include "tsan_mman.h"
 #include "tsan_mutexset.h"
 #include "tsan_platform.h"
@@ -46,6 +46,7 @@
 #include "tsan_stack_trace.h"
 #include "tsan_sync.h"
 #include "tsan_trace.h"
+#include "tsan_vector_clock.h"
 
 #if SANITIZER_WORDSIZE != 64
 # error "ThreadSanitizer is supported only on 64-bit platforms"
@@ -116,7 +117,6 @@ struct Processor {
 #endif
   DenseSlabAllocCache block_cache;
   DenseSlabAllocCache sync_cache;
-  DenseSlabAllocCache clock_cache;
   DDPhysicalThread *dd_pt;
 };
 
@@ -130,30 +130,56 @@ struct ScopedGlobalProcessor {
 };
 #endif
 
+struct TidEpoch {
+  Tid tid;
+  Epoch epoch;
+};
+
+struct TidSlot {
+  Mutex mtx;
+  Sid sid;
+  atomic_uint32_t raw_epoch;
+  ThreadState *thr;
+  Vector<TidEpoch> journal;
+  INode node;
+
+  Epoch epoch() const {
+    return static_cast<Epoch>(atomic_load(&raw_epoch, memory_order_relaxed));
+  }
+
+  void SetEpoch(Epoch v) {
+    atomic_store(&raw_epoch, static_cast<u32>(v), memory_order_relaxed);
+  }
+
+  TidSlot();
+} ALIGNED(SANITIZER_CACHE_LINE_SIZE);
+
 // This struct is stored in TLS.
 struct ThreadState {
   FastState fast_state;
-  // Synch epoch represents the threads's epoch before the last synchronization
-  // action. It allows to reduce number of shadow state updates.
-  // For example, fast_synch_epoch=100, last write to addr X was at epoch=150,
-  // if we are processing write to X from the same thread at epoch=200,
-  // we do nothing, because both writes happen in the same 'synch epoch'.
-  // That is, if another memory access does not race with the former write,
-  // it does not race with the latter as well.
-  // QUESTION: can we can squeeze this into ThreadState::Fast?
-  // E.g. ThreadState::Fast is a 44-bit, 32 are taken by synch_epoch and 12 are
-  // taken by epoch between synchs.
-  // This way we can save one load from tls.
-  u64 fast_synch_epoch;
+  int ignore_sync;
+#if !SANITIZER_GO
+  int ignore_interceptors;
+#endif
+  uptr *shadow_stack_pos;
+
+  // Current position in tctx->trace.Back()->events (Event*).
+  atomic_uintptr_t trace_pos;
+  // PC of the last memory access, used to compute PC deltas in the trace.
+  uptr trace_prev_pc;
+
   // Technically `current` should be a separate THREADLOCAL variable;
   // but it is placed here in order to share cache line with previous fields.
   ThreadState* current;
+
+  atomic_sint32_t pending_signals;
+
+  VectorClock clock;
+
   // This is a slow path flag. On fast path, fast_state.GetIgnoreBit() is read.
   // We do not distinguish beteween ignoring reads and writes
   // for better performance.
   int ignore_reads_and_writes;
-  atomic_sint32_t pending_signals;
-  int ignore_sync;
   int suppress_reports;
   // Go does not support ignores.
 #if !SANITIZER_GO
@@ -162,31 +188,27 @@ struct ThreadState {
 #endif
   uptr *shadow_stack;
   uptr *shadow_stack_end;
-  uptr *shadow_stack_pos;
-  RawShadow *racy_shadow_addr;
-  RawShadow racy_state[2];
-  MutexSet mset;
-  ThreadClock clock;
 #if !SANITIZER_GO
   Vector<JmpBuf> jmp_bufs;
-  int ignore_interceptors;
-#endif
-  const Tid tid;
-  const int unique_id;
-  bool in_symbolizer;
+  int in_symbolizer;
   bool in_ignored_lib;
   bool is_inited;
+#endif
+  MutexSet mset;
   bool is_dead;
-  bool is_freeing;
-  bool is_vptr_access;
-  const uptr stk_addr;
-  const uptr stk_size;
-  const uptr tls_addr;
-  const uptr tls_size;
+  const Tid tid;
+  uptr stk_addr;
+  uptr stk_size;
+  uptr tls_addr;
+  uptr tls_size;
   ThreadContext *tctx;
 
   DDLogicalThread *dd_lt;
 
+  TidSlot *slot;
+  uptr slot_epoch;
+  bool slot_locked;
+
   // Current wired Processor, or nullptr. Required to handle any events.
   Processor *proc1;
 #if !SANITIZER_GO
@@ -200,7 +222,7 @@ struct ThreadState {
 
 #if !SANITIZER_GO
   StackID last_sleep_stack_id;
-  ThreadClock last_sleep_clock;
+  VectorClock last_sleep_clock;
 #endif
 
   // Set in regions of runtime that must be signal-safe and fork-safe.
@@ -209,16 +231,7 @@ struct ThreadState {
 
   const ReportDesc *current_report;
 
-  // Current position in tctx->trace.Back()->events (Event*).
-  atomic_uintptr_t trace_pos;
-  // PC of the last memory access, used to compute PC deltas in the trace.
-  uptr trace_prev_pc;
-  Sid sid;
-  Epoch epoch;
-
-  explicit ThreadState(Context *ctx, Tid tid, int unique_id, u64 epoch,
-                       unsigned reuse_count, uptr stk_addr, uptr stk_size,
-                       uptr tls_addr, uptr tls_size);
+  explicit ThreadState(Tid tid);
 } ALIGNED(SANITIZER_CACHE_LINE_SIZE);
 
 #if !SANITIZER_GO
@@ -252,14 +265,9 @@ class ThreadContext final : public ThreadContextBase {
   ~ThreadContext();
   ThreadState *thr;
   StackID creation_stack_id;
-  SyncClock sync;
-  // Epoch at which the thread had started.
-  // If we see an event from the thread stamped by an older epoch,
-  // the event is from a dead thread that shared tid with this thread.
-  u64 epoch0;
-  u64 epoch1;
-
-  v3::Trace trace;
+  VectorClock *sync;
+  uptr sync_epoch;
+  Trace trace;
 
   // Override superclass callbacks.
   void OnDead() override;
@@ -314,12 +322,22 @@ struct Context {
   InternalMmapVector<FiredSuppression> fired_suppressions;
   DDetector *dd;
 
-  ClockAlloc clock_alloc;
-
   Flags flags;
   fd_t memprof_fd;
 
+  // The last slot index (kFreeSid) is used to denote freed memory.
+  TidSlot slots[kThreadSlotCount - 1];
+
+  // Protects global_epoch, slot_queue, trace_part_recycle.
   Mutex slot_mtx;
+  uptr global_epoch;  // guarded by slot_mtx and by all slot mutexes
+  bool resetting;     // global reset is in progress
+  IList<TidSlot, &TidSlot::node> slot_queue GUARDED_BY(slot_mtx);
+  IList<TraceHeader, &TraceHeader::global, TracePart> trace_part_recycle
+      GUARDED_BY(slot_mtx);
+  uptr trace_part_total_allocated GUARDED_BY(slot_mtx);
+  uptr trace_part_recycle_finished GUARDED_BY(slot_mtx);
+  uptr trace_part_finished_excess GUARDED_BY(slot_mtx);
 };
 
 extern Context *ctx;  // The one and the only global runtime context.
@@ -348,14 +366,13 @@ uptr TagFromShadowStackFrame(uptr pc);
 
 class ScopedReportBase {
  public:
-  void AddMemoryAccess(uptr addr, uptr external_tag, Shadow s, StackTrace stack,
-                       const MutexSet *mset);
+  void AddMemoryAccess(uptr addr, uptr external_tag, Shadow s, Tid tid,
+                       StackTrace stack, const MutexSet *mset);
   void AddStack(StackTrace stack, bool suppressable = false);
   void AddThread(const ThreadContext *tctx, bool suppressable = false);
-  void AddThread(Tid unique_tid, bool suppressable = false);
+  void AddThread(Tid tid, bool suppressable = false);
   void AddUniqueTid(Tid unique_tid);
-  void AddMutex(const SyncVar *s);
-  u64 AddMutex(u64 id);
+  int AddMutex(uptr addr, StackID creation_stack_id);
   void AddLocation(uptr addr, uptr size);
   void AddSleep(StackID stack_id);
   void SetCount(int count);
@@ -372,8 +389,6 @@ class ScopedReportBase {
   // at best it will cause deadlocks on internal mutexes.
   ScopedIgnoreInterceptors ignore_interceptors_;
 
-  void AddDeadMutex(u64 id);
-
   ScopedReportBase(const ScopedReportBase &) = delete;
   void operator=(const ScopedReportBase &) = delete;
 };
@@ -389,8 +404,6 @@ class ScopedReport : public ScopedReportBase {
 
 bool ShouldReport(ThreadState *thr, ReportType typ);
 ThreadContext *IsThreadStackOrTls(uptr addr, bool *is_stack);
-void RestoreStack(Tid tid, const u64 epoch, VarSizeStackTrace *stk,
-                  MutexSet *mset, uptr *tag = nullptr);
 
 // The stack could look like:
 //   <start> | <main> | <foo> | tag | <bar>
@@ -438,7 +451,8 @@ void ForkBefore(ThreadState *thr, uptr pc);
 void ForkParentAfter(ThreadState *thr, uptr pc);
 void ForkChildAfter(ThreadState *thr, uptr pc, bool start_thread);
 
-void ReportRace(ThreadState *thr);
+void ReportRace(ThreadState *thr, RawShadow *shadow_mem, Shadow cur, Shadow old,
+                AccessType typ);
 bool OutputReport(ThreadState *thr, const ScopedReport &srep);
 bool IsFiredSuppression(Context *ctx, ReportType type, StackTrace trace);
 bool IsExpectedReport(uptr addr, uptr size);
@@ -468,55 +482,28 @@ int Finalize(ThreadState *thr);
 void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write);
 void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write);
 
-void MemoryAccess(ThreadState *thr, uptr pc, uptr addr,
-    int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic);
-void MemoryAccessImpl(ThreadState *thr, uptr addr,
-    int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic,
-    u64 *shadow_mem, Shadow cur);
-void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr,
-    uptr size, bool is_write);
+void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size,
+                  AccessType typ);
 void UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size,
                            AccessType typ);
-
-const int kSizeLog1 = 0;
-const int kSizeLog2 = 1;
-const int kSizeLog4 = 2;
-const int kSizeLog8 = 3;
+// This creates 2 non-inlined specialized versions of MemoryAccessRange.
+template <bool is_read>
+void MemoryAccessRangeT(ThreadState *thr, uptr pc, uptr addr, uptr size);
 
 ALWAYS_INLINE
-void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size,
-                  AccessType typ) {
-  int size_log;
-  switch (size) {
-    case 1:
-      size_log = kSizeLog1;
-      break;
-    case 2:
-      size_log = kSizeLog2;
-      break;
-    case 4:
-      size_log = kSizeLog4;
-      break;
-    default:
-      DCHECK_EQ(size, 8);
-      size_log = kSizeLog8;
-      break;
-  }
-  bool is_write = !(typ & kAccessRead);
-  bool is_atomic = typ & kAccessAtomic;
-  if (typ & kAccessVptr)
-    thr->is_vptr_access = true;
-  if (typ & kAccessFree)
-    thr->is_freeing = true;
-  MemoryAccess(thr, pc, addr, size_log, is_write, is_atomic);
-  if (typ & kAccessVptr)
-    thr->is_vptr_access = false;
-  if (typ & kAccessFree)
-    thr->is_freeing = false;
+void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size,
+                       bool is_write) {
+  if (size == 0)
+    return;
+  if (is_write)
+    MemoryAccessRangeT<false>(thr, pc, addr, size);
+  else
+    MemoryAccessRangeT<true>(thr, pc, addr, size);
 }
 
-void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size);
+void ShadowSet(RawShadow *p, RawShadow *end, RawShadow v);
 void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size);
+void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size);
 void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size);
 void MemoryRangeImitateWriteOrResetRange(ThreadState *thr, uptr pc, uptr addr,
                                          uptr size);
@@ -526,9 +513,6 @@ void ThreadIgnoreEnd(ThreadState *thr);
 void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc);
 void ThreadIgnoreSyncEnd(ThreadState *thr);
 
-void FuncEntry(ThreadState *thr, uptr pc);
-void FuncExit(ThreadState *thr);
-
 Tid ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached);
 void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id,
                  ThreadType thread_type);
@@ -574,63 +558,7 @@ void Release(ThreadState *thr, uptr pc, uptr addr);
 void ReleaseStoreAcquire(ThreadState *thr, uptr pc, uptr addr);
 void ReleaseStore(ThreadState *thr, uptr pc, uptr addr);
 void AfterSleep(ThreadState *thr, uptr pc);
-void AcquireImpl(ThreadState *thr, uptr pc, SyncClock *c);
-void ReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c);
-void ReleaseStoreAcquireImpl(ThreadState *thr, uptr pc, SyncClock *c);
-void ReleaseStoreImpl(ThreadState *thr, uptr pc, SyncClock *c);
-void AcquireReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c);
-
-// The hacky call uses custom calling convention and an assembly thunk.
-// It is considerably faster that a normal call for the caller
-// if it is not executed (it is intended for slow paths from hot functions).
-// The trick is that the call preserves all registers and the compiler
-// does not treat it as a call.
-// If it does not work for you, use normal call.
-#if !SANITIZER_DEBUG && defined(__x86_64__) && !SANITIZER_MAC
-// The caller may not create the stack frame for itself at all,
-// so we create a reserve stack frame for it (1024b must be enough).
-#define HACKY_CALL(f) \
-  __asm__ __volatile__("sub $1024, %%rsp;" \
-                       CFI_INL_ADJUST_CFA_OFFSET(1024) \
-                       ".hidden " #f "_thunk;" \
-                       "call " #f "_thunk;" \
-                       "add $1024, %%rsp;" \
-                       CFI_INL_ADJUST_CFA_OFFSET(-1024) \
-                       ::: "memory", "cc");
-#else
-#define HACKY_CALL(f) f()
-#endif
-
-void TraceSwitch(ThreadState *thr);
-uptr TraceTopPC(ThreadState *thr);
-uptr TraceSize();
-uptr TraceParts();
-Trace *ThreadTrace(Tid tid);
-
-extern "C" void __tsan_trace_switch();
-void ALWAYS_INLINE TraceAddEvent(ThreadState *thr, FastState fs,
-                                        EventType typ, u64 addr) {
-  if (!kCollectHistory)
-    return;
-  // TraceSwitch accesses shadow_stack, but it's called infrequently,
-  // so we check it here proactively.
-  DCHECK(thr->shadow_stack);
-  DCHECK_GE((int)typ, 0);
-  DCHECK_LE((int)typ, 7);
-  DCHECK_EQ(GetLsb(addr, kEventPCBits), addr);
-  u64 pos = fs.GetTracePos();
-  if (UNLIKELY((pos % kTracePartSize) == 0)) {
-#if !SANITIZER_GO
-    HACKY_CALL(__tsan_trace_switch);
-#else
-    TraceSwitch(thr);
-#endif
-  }
-  Event *trace = (Event*)GetThreadTrace(fs.tid());
-  Event *evp = &trace[pos];
-  Event ev = (u64)addr | ((u64)typ << kEventPCBits);
-  *evp = ev;
-}
+void IncrementEpoch(ThreadState *thr);
 
 #if !SANITIZER_GO
 uptr ALWAYS_INLINE HeapEnd() {
@@ -638,6 +566,13 @@ uptr ALWAYS_INLINE HeapEnd() {
 }
 #endif
 
+void SlotAttachAndLock(ThreadState *thr) ACQUIRE(thr->slot->mtx);
+void SlotDetach(ThreadState *thr);
+void SlotLock(ThreadState *thr) ACQUIRE(thr->slot->mtx);
+void SlotUnlock(ThreadState *thr) RELEASE(thr->slot->mtx);
+void DoReset(ThreadState *thr, uptr epoch);
+void FlushShadowMemory();
+
 ThreadState *FiberCreate(ThreadState *thr, uptr pc, unsigned flags);
 void FiberDestroy(ThreadState *thr, uptr pc, ThreadState *fiber);
 void FiberSwitch(ThreadState *thr, uptr pc, ThreadState *fiber, unsigned flags);
@@ -648,6 +583,43 @@ enum FiberSwitchFlags {
   FiberSwitchFlagNoSync = 1 << 0, // __tsan_switch_to_fiber_no_sync
 };
 
+class SlotLocker {
+ public:
+  ALWAYS_INLINE
+  SlotLocker(ThreadState *thr, bool recursive = false)
+      : thr_(thr), locked_(recursive ? thr->slot_locked : false) {
+    if (!locked_)
+      SlotLock(thr_);
+  }
+
+  ALWAYS_INLINE
+  ~SlotLocker() {
+    if (!locked_)
+      SlotUnlock(thr_);
+  }
+
+ private:
+  ThreadState *thr_;
+  bool locked_;
+};
+
+class SlotUnlocker {
+ public:
+  SlotUnlocker(ThreadState *thr) : thr_(thr), locked_(thr->slot_locked) {
+    if (locked_)
+      SlotUnlock(thr_);
+  }
+
+  ~SlotUnlocker() {
+    if (locked_)
+      SlotLock(thr_);
+  }
+
+ private:
+  ThreadState *thr_;
+  bool locked_;
+};
+
 ALWAYS_INLINE void ProcessPendingSignals(ThreadState *thr) {
   if (UNLIKELY(atomic_load_relaxed(&thr->pending_signals)))
     ProcessPendingSignalsImpl(thr);
@@ -666,16 +638,19 @@ void LazyInitialize(ThreadState *thr) {
 #endif
 }
 
-namespace v3 {
-
+void TraceResetForTesting();
 void TraceSwitchPart(ThreadState *thr);
-bool RestoreStack(Tid tid, EventType type, Sid sid, Epoch epoch, uptr addr,
-                  uptr size, AccessType typ, VarSizeStackTrace *pstk,
+void TraceSwitchPartImpl(ThreadState *thr);
+bool RestoreStack(EventType type, Sid sid, Epoch epoch, uptr addr, uptr size,
+                  AccessType typ, Tid *ptid, VarSizeStackTrace *pstk,
                   MutexSet *pmset, uptr *ptag);
 
 template <typename EventT>
 ALWAYS_INLINE WARN_UNUSED_RESULT bool TraceAcquire(ThreadState *thr,
                                                    EventT **ev) {
+  // TraceSwitchPart accesses shadow_stack, but it's called infrequently,
+  // so we check it here proactively.
+  DCHECK(thr->shadow_stack);
   Event *pos = reinterpret_cast<Event *>(atomic_load_relaxed(&thr->trace_pos));
 #if SANITIZER_DEBUG
   // TraceSwitch acquires these mutexes,
@@ -746,20 +721,16 @@ void TraceMutexLock(ThreadState *thr, EventType type, uptr pc, uptr addr,
 void TraceMutexUnlock(ThreadState *thr, uptr addr);
 void TraceTime(ThreadState *thr);
 
-}  // namespace v3
+void TraceRestartFuncExit(ThreadState *thr);
+void TraceRestartFuncEntry(ThreadState *thr, uptr pc);
 
 void GrowShadowStack(ThreadState *thr);
 
 ALWAYS_INLINE
 void FuncEntry(ThreadState *thr, uptr pc) {
-  DPrintf2("#%d: FuncEntry %p\n", (int)thr->fast_state.tid(), (void *)pc);
-  if (kCollectHistory) {
-    thr->fast_state.IncrementEpoch();
-    TraceAddEvent(thr, thr->fast_state, EventTypeFuncEnter, pc);
-  }
-
-  // Shadow stack maintenance can be replaced with
-  // stack unwinding during trace switch (which presumably must be faster).
+  DPrintf2("#%d: FuncEntry %p\n", (int)thr->fast_state.sid(), (void *)pc);
+  if (UNLIKELY(!TryTraceFunc(thr, pc)))
+    return TraceRestartFuncEntry(thr, pc);
   DCHECK_GE(thr->shadow_stack_pos, thr->shadow_stack);
 #if !SANITIZER_GO
   DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
@@ -773,12 +744,9 @@ void FuncEntry(ThreadState *thr, uptr pc) {
 
 ALWAYS_INLINE
 void FuncExit(ThreadState *thr) {
-  DPrintf2("#%d: FuncExit\n", (int)thr->fast_state.tid());
-  if (kCollectHistory) {
-    thr->fast_state.IncrementEpoch();
-    TraceAddEvent(thr, thr->fast_state, EventTypeFuncExit, 0);
-  }
-
+  DPrintf2("#%d: FuncExit\n", (int)thr->fast_state.sid());
+  if (UNLIKELY(!TryTraceFunc(thr, 0)))
+    return TraceRestartFuncExit(thr);
   DCHECK_GT(thr->shadow_stack_pos, thr->shadow_stack);
 #if !SANITIZER_GO
   DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
@@ -790,7 +758,6 @@ void FuncExit(ThreadState *thr) {
 extern void (*on_initialize)(void);
 extern int (*on_finalize)(int);
 #endif
-
 }  // namespace __tsan
 
 #endif  // TSAN_RTL_H
diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl_access.cpp b/compiler-rt/lib/tsan/rtl/tsan_rtl_access.cpp
index 7365fdaa3038..940c20fcfa1a 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_rtl_access.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_rtl_access.cpp
@@ -15,15 +15,13 @@
 
 namespace __tsan {
 
-namespace v3 {
-
-ALWAYS_INLINE USED bool TryTraceMemoryAccess(ThreadState *thr, uptr pc,
+ALWAYS_INLINE USED bool TryTraceMemoryAccess(ThreadState* thr, uptr pc,
                                              uptr addr, uptr size,
                                              AccessType typ) {
   DCHECK(size == 1 || size == 2 || size == 4 || size == 8);
   if (!kCollectHistory)
     return true;
-  EventAccess *ev;
+  EventAccess* ev;
   if (UNLIKELY(!TraceAcquire(thr, &ev)))
     return false;
   u64 size_log = size == 1 ? 0 : size == 2 ? 1 : size == 4 ? 2 : 3;
@@ -40,25 +38,27 @@ ALWAYS_INLINE USED bool TryTraceMemoryAccess(ThreadState *thr, uptr pc,
     TraceRelease(thr, ev);
     return true;
   }
-  auto *evex = reinterpret_cast<EventAccessExt *>(ev);
+  auto* evex = reinterpret_cast<EventAccessExt*>(ev);
   evex->is_access = 0;
   evex->is_func = 0;
   evex->type = EventType::kAccessExt;
   evex->is_read = !!(typ & kAccessRead);
   evex->is_atomic = !!(typ & kAccessAtomic);
   evex->size_log = size_log;
+  // Note: this is important, see comment in EventAccessExt.
+  evex->_ = 0;
   evex->addr = CompressAddr(addr);
   evex->pc = pc;
   TraceRelease(thr, evex);
   return true;
 }
 
-ALWAYS_INLINE USED bool TryTraceMemoryAccessRange(ThreadState *thr, uptr pc,
-                                                  uptr addr, uptr size,
-                                                  AccessType typ) {
+ALWAYS_INLINE
+bool TryTraceMemoryAccessRange(ThreadState* thr, uptr pc, uptr addr, uptr size,
+                               AccessType typ) {
   if (!kCollectHistory)
     return true;
-  EventAccessRange *ev;
+  EventAccessRange* ev;
   if (UNLIKELY(!TraceAcquire(thr, &ev)))
     return false;
   thr->trace_prev_pc = pc;
@@ -75,7 +75,7 @@ ALWAYS_INLINE USED bool TryTraceMemoryAccessRange(ThreadState *thr, uptr pc,
   return true;
 }
 
-void TraceMemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size,
+void TraceMemoryAccessRange(ThreadState* thr, uptr pc, uptr addr, uptr size,
                             AccessType typ) {
   if (LIKELY(TryTraceMemoryAccessRange(thr, pc, addr, size, typ)))
     return;
@@ -84,7 +84,7 @@ void TraceMemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size,
   DCHECK(res);
 }
 
-void TraceFunc(ThreadState *thr, uptr pc) {
+void TraceFunc(ThreadState* thr, uptr pc) {
   if (LIKELY(TryTraceFunc(thr, pc)))
     return;
   TraceSwitchPart(thr);
@@ -92,7 +92,17 @@ void TraceFunc(ThreadState *thr, uptr pc) {
   DCHECK(res);
 }
 
-void TraceMutexLock(ThreadState *thr, EventType type, uptr pc, uptr addr,
+NOINLINE void TraceRestartFuncEntry(ThreadState* thr, uptr pc) {
+  TraceSwitchPart(thr);
+  FuncEntry(thr, pc);
+}
+
+NOINLINE void TraceRestartFuncExit(ThreadState* thr) {
+  TraceSwitchPart(thr);
+  FuncExit(thr);
+}
+
+void TraceMutexLock(ThreadState* thr, EventType type, uptr pc, uptr addr,
                     StackID stk) {
   DCHECK(type == EventType::kLock || type == EventType::kRLock);
   if (!kCollectHistory)
@@ -109,7 +119,7 @@ void TraceMutexLock(ThreadState *thr, EventType type, uptr pc, uptr addr,
   TraceEvent(thr, ev);
 }
 
-void TraceMutexUnlock(ThreadState *thr, uptr addr) {
+void TraceMutexUnlock(ThreadState* thr, uptr addr) {
   if (!kCollectHistory)
     return;
   EventUnlock ev;
@@ -121,396 +131,523 @@ void TraceMutexUnlock(ThreadState *thr, uptr addr) {
   TraceEvent(thr, ev);
 }
 
-void TraceTime(ThreadState *thr) {
+void TraceTime(ThreadState* thr) {
   if (!kCollectHistory)
     return;
+  FastState fast_state = thr->fast_state;
   EventTime ev;
   ev.is_access = 0;
   ev.is_func = 0;
   ev.type = EventType::kTime;
-  ev.sid = static_cast<u64>(thr->sid);
-  ev.epoch = static_cast<u64>(thr->epoch);
+  ev.sid = static_cast<u64>(fast_state.sid());
+  ev.epoch = static_cast<u64>(fast_state.epoch());
   ev._ = 0;
   TraceEvent(thr, ev);
 }
 
-}  // namespace v3
+ALWAYS_INLINE RawShadow LoadShadow(RawShadow* p) {
+  return static_cast<RawShadow>(
+      atomic_load((atomic_uint32_t*)p, memory_order_relaxed));
+}
 
-ALWAYS_INLINE
-Shadow LoadShadow(u64 *p) {
-  u64 raw = atomic_load((atomic_uint64_t *)p, memory_order_relaxed);
-  return Shadow(raw);
+ALWAYS_INLINE void StoreShadow(RawShadow* sp, RawShadow s) {
+  atomic_store((atomic_uint32_t*)sp, static_cast<u32>(s), memory_order_relaxed);
 }
 
-ALWAYS_INLINE
-void StoreShadow(u64 *sp, u64 s) {
-  atomic_store((atomic_uint64_t *)sp, s, memory_order_relaxed);
+NOINLINE void DoReportRace(ThreadState* thr, RawShadow* shadow_mem, Shadow cur,
+                           Shadow old,
+                           AccessType typ) NO_THREAD_SAFETY_ANALYSIS {
+  // For the free shadow markers the first element (that contains kFreeSid)
+  // triggers the race, but the second element contains info about the freeing
+  // thread, take it.
+  if (old.sid() == kFreeSid)
+    old = Shadow(LoadShadow(&shadow_mem[1]));
+  // This prevents trapping on this address in future.
+  for (uptr i = 0; i < kShadowCnt; i++)
+    StoreShadow(&shadow_mem[i], i == 0 ? Shadow::kRodata : Shadow::kEmpty);
+  // See the comment in MemoryRangeFreed as to why the slot is locked
+  // for free memory accesses. ReportRace must not be called with
+  // the slot locked because of the fork. But MemoryRangeFreed is not
+  // called during fork because fork sets ignore_reads_and_writes,
+  // so simply unlocking the slot should be fine.
+  if (typ & kAccessFree)
+    SlotUnlock(thr);
+  ReportRace(thr, shadow_mem, cur, Shadow(old), typ);
+  if (typ & kAccessFree)
+    SlotLock(thr);
 }
 
+#if !TSAN_VECTORIZE
 ALWAYS_INLINE
-void StoreIfNotYetStored(u64 *sp, u64 *s) {
-  StoreShadow(sp, *s);
-  *s = 0;
+bool ContainsSameAccess(RawShadow* s, Shadow cur, int unused0, int unused1,
+                        AccessType typ) {
+  for (uptr i = 0; i < kShadowCnt; i++) {
+    auto old = LoadShadow(&s[i]);
+    if (!(typ & kAccessRead)) {
+      if (old == cur.raw())
+        return true;
+      continue;
+    }
+    auto masked = static_cast<RawShadow>(static_cast<u32>(old) |
+                                         static_cast<u32>(Shadow::kRodata));
+    if (masked == cur.raw())
+      return true;
+    if (!(typ & kAccessNoRodata) && !SANITIZER_GO) {
+      if (old == Shadow::kRodata)
+        return true;
+    }
+  }
+  return false;
 }
 
-extern "C" void __tsan_report_race();
-
 ALWAYS_INLINE
-void HandleRace(ThreadState *thr, u64 *shadow_mem, Shadow cur, Shadow old) {
-  thr->racy_state[0] = cur.raw();
-  thr->racy_state[1] = old.raw();
-  thr->racy_shadow_addr = shadow_mem;
-#if !SANITIZER_GO
-  HACKY_CALL(__tsan_report_race);
-#else
-  ReportRace(thr);
-#endif
+bool CheckRaces(ThreadState* thr, RawShadow* shadow_mem, Shadow cur,
+                int unused0, int unused1, AccessType typ) {
+  bool stored = false;
+  for (uptr idx = 0; idx < kShadowCnt; idx++) {
+    RawShadow* sp = &shadow_mem[idx];
+    Shadow old(LoadShadow(sp));
+    if (LIKELY(old.raw() == Shadow::kEmpty)) {
+      if (!(typ & kAccessCheckOnly) && !stored)
+        StoreShadow(sp, cur.raw());
+      return false;
+    }
+    if (LIKELY(!(cur.access() & old.access())))
+      continue;
+    if (LIKELY(cur.sid() == old.sid())) {
+      if (!(typ & kAccessCheckOnly) &&
+          LIKELY(cur.access() == old.access() && old.IsRWWeakerOrEqual(typ))) {
+        StoreShadow(sp, cur.raw());
+        stored = true;
+      }
+      continue;
+    }
+    if (LIKELY(old.IsBothReadsOrAtomic(typ)))
+      continue;
+    if (LIKELY(thr->clock.Get(old.sid()) >= old.epoch()))
+      continue;
+    DoReportRace(thr, shadow_mem, cur, old, typ);
+    return true;
+  }
+  // We did not find any races and had already stored
+  // the current access info, so we are done.
+  if (LIKELY(stored))
+    return false;
+  // Choose a random candidate slot and replace it.
+  uptr index =
+      atomic_load_relaxed(&thr->trace_pos) / sizeof(Event) % kShadowCnt;
+  StoreShadow(&shadow_mem[index], cur.raw());
+  return false;
 }
 
-static inline bool HappensBefore(Shadow old, ThreadState *thr) {
-  return thr->clock.get(old.TidWithIgnore()) >= old.epoch();
-}
+#  define LOAD_CURRENT_SHADOW(cur, shadow_mem) UNUSED int access = 0, shadow = 0
 
-ALWAYS_INLINE
-void MemoryAccessImpl1(ThreadState *thr, uptr addr, int kAccessSizeLog,
-                       bool kAccessIsWrite, bool kIsAtomic, u64 *shadow_mem,
-                       Shadow cur) {
-  // This potentially can live in an MMX/SSE scratch register.
-  // The required intrinsics are:
-  // __m128i _mm_move_epi64(__m128i*);
-  // _mm_storel_epi64(u64*, __m128i);
-  u64 store_word = cur.raw();
-  bool stored = false;
+#else /* !TSAN_VECTORIZE */
 
-  // scan all the shadow values and dispatch to 4 categories:
-  // same, replace, candidate and race (see comments below).
-  // we consider only 3 cases regarding access sizes:
-  // equal, intersect and not intersect. initially I considered
-  // larger and smaller as well, it allowed to replace some
-  // 'candidates' with 'same' or 'replace', but I think
-  // it's just not worth it (performance- and complexity-wise).
-
-  Shadow old(0);
-
-  // It release mode we manually unroll the loop,
-  // because empirically gcc generates better code this way.
-  // However, we can't afford unrolling in debug mode, because the function
-  // consumes almost 4K of stack. Gtest gives only 4K of stack to death test
-  // threads, which is not enough for the unrolled loop.
-#if SANITIZER_DEBUG
-  for (int idx = 0; idx < 4; idx++) {
-#  include "tsan_update_shadow_word.inc"
-  }
-#else
-  int idx = 0;
-#  include "tsan_update_shadow_word.inc"
-  idx = 1;
-  if (stored) {
-#  include "tsan_update_shadow_word.inc"
-  } else {
-#  include "tsan_update_shadow_word.inc"
-  }
-  idx = 2;
-  if (stored) {
-#  include "tsan_update_shadow_word.inc"
-  } else {
-#  include "tsan_update_shadow_word.inc"
+ALWAYS_INLINE
+bool ContainsSameAccess(RawShadow* unused0, Shadow unused1, m128 shadow,
+                        m128 access, AccessType typ) {
+  // Note: we could check if there is a larger access of the same type,
+  // e.g. we just allocated/memset-ed a block (so it contains 8 byte writes)
+  // and now do smaller reads/writes, these can also be considered as "same
+  // access". However, it will make the check more expensive, so it's unclear
+  // if it's worth it. But this would conserve trace space, so it's useful
+  // besides potential speed up.
+  if (!(typ & kAccessRead)) {
+    const m128 same = _mm_cmpeq_epi32(shadow, access);
+    return _mm_movemask_epi8(same);
   }
-  idx = 3;
-  if (stored) {
-#  include "tsan_update_shadow_word.inc"
-  } else {
-#  include "tsan_update_shadow_word.inc"
+  // For reads we need to reset read bit in the shadow,
+  // because we need to match read with both reads and writes.
+  // Shadow::kRodata has only read bit set, so it does what we want.
+  // We also abuse it for rodata check to save few cycles
+  // since we already loaded Shadow::kRodata into a register.
+  // Reads from rodata can't race.
+  // Measurements show that they can be 10-20% of all memory accesses.
+  // Shadow::kRodata has epoch 0 which cannot appear in shadow normally
+  // (thread epochs start from 1). So the same read bit mask
+  // serves as rodata indicator.
+  const m128 read_mask = _mm_set1_epi32(static_cast<u32>(Shadow::kRodata));
+  const m128 masked_shadow = _mm_or_si128(shadow, read_mask);
+  m128 same = _mm_cmpeq_epi32(masked_shadow, access);
+  // Range memory accesses check Shadow::kRodata before calling this,
+  // Shadow::kRodatas is not possible for free memory access
+  // and Go does not use Shadow::kRodata.
+  if (!(typ & kAccessNoRodata) && !SANITIZER_GO) {
+    const m128 ro = _mm_cmpeq_epi32(shadow, read_mask);
+    same = _mm_or_si128(ro, same);
   }
-#endif
-
-  // we did not find any races and had already stored
-  // the current access info, so we are done
-  if (LIKELY(stored))
-    return;
-  // choose a random candidate slot and replace it
-  StoreShadow(shadow_mem + (cur.epoch() % kShadowCnt), store_word);
-  return;
-RACE:
-  HandleRace(thr, shadow_mem, cur, old);
-  return;
+  return _mm_movemask_epi8(same);
 }
 
-void UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size,
-                           AccessType typ) {
-  DCHECK(!(typ & kAccessAtomic));
-  const bool kAccessIsWrite = !(typ & kAccessRead);
-  const bool kIsAtomic = false;
-  while (size) {
-    int size1 = 1;
-    int kAccessSizeLog = kSizeLog1;
-    if (size >= 8 && (addr & ~7) == ((addr + 7) & ~7)) {
-      size1 = 8;
-      kAccessSizeLog = kSizeLog8;
-    } else if (size >= 4 && (addr & ~7) == ((addr + 3) & ~7)) {
-      size1 = 4;
-      kAccessSizeLog = kSizeLog4;
-    } else if (size >= 2 && (addr & ~7) == ((addr + 1) & ~7)) {
-      size1 = 2;
-      kAccessSizeLog = kSizeLog2;
-    }
-    MemoryAccess(thr, pc, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic);
-    addr += size1;
-    size -= size1;
+NOINLINE void DoReportRaceV(ThreadState* thr, RawShadow* shadow_mem, Shadow cur,
+                            u32 race_mask, m128 shadow, AccessType typ) {
+  // race_mask points which of the shadow elements raced with the current
+  // access. Extract that element.
+  CHECK_NE(race_mask, 0);
+  u32 old;
+  // Note: _mm_extract_epi32 index must be a constant value.
+  switch (__builtin_ffs(race_mask) / 4) {
+    case 0:
+      old = _mm_extract_epi32(shadow, 0);
+      break;
+    case 1:
+      old = _mm_extract_epi32(shadow, 1);
+      break;
+    case 2:
+      old = _mm_extract_epi32(shadow, 2);
+      break;
+    case 3:
+      old = _mm_extract_epi32(shadow, 3);
+      break;
   }
+  Shadow prev(static_cast<RawShadow>(old));
+  // For the free shadow markers the first element (that contains kFreeSid)
+  // triggers the race, but the second element contains info about the freeing
+  // thread, take it.
+  if (prev.sid() == kFreeSid)
+    prev = Shadow(static_cast<RawShadow>(_mm_extract_epi32(shadow, 1)));
+  DoReportRace(thr, shadow_mem, cur, prev, typ);
 }
 
 ALWAYS_INLINE
-bool ContainsSameAccessSlow(u64 *s, u64 a, u64 sync_epoch, bool is_write) {
-  Shadow cur(a);
-  for (uptr i = 0; i < kShadowCnt; i++) {
-    Shadow old(LoadShadow(&s[i]));
-    if (Shadow::Addr0AndSizeAreEqual(cur, old) &&
-        old.TidWithIgnore() == cur.TidWithIgnore() &&
-        old.epoch() > sync_epoch && old.IsAtomic() == cur.IsAtomic() &&
-        old.IsRead() <= cur.IsRead())
-      return true;
+bool CheckRaces(ThreadState* thr, RawShadow* shadow_mem, Shadow cur,
+                m128 shadow, m128 access, AccessType typ) {
+  // Note: empty/zero slots don't intersect with any access.
+  const m128 zero = _mm_setzero_si128();
+  const m128 mask_access = _mm_set1_epi32(0x000000ff);
+  const m128 mask_sid = _mm_set1_epi32(0x0000ff00);
+  const m128 mask_read_atomic = _mm_set1_epi32(0xc0000000);
+  const m128 access_and = _mm_and_si128(access, shadow);
+  const m128 access_xor = _mm_xor_si128(access, shadow);
+  const m128 intersect = _mm_and_si128(access_and, mask_access);
+  const m128 not_intersect = _mm_cmpeq_epi32(intersect, zero);
+  const m128 not_same_sid = _mm_and_si128(access_xor, mask_sid);
+  const m128 same_sid = _mm_cmpeq_epi32(not_same_sid, zero);
+  const m128 both_read_or_atomic = _mm_and_si128(access_and, mask_read_atomic);
+  const m128 no_race =
+      _mm_or_si128(_mm_or_si128(not_intersect, same_sid), both_read_or_atomic);
+  const int race_mask = _mm_movemask_epi8(_mm_cmpeq_epi32(no_race, zero));
+  if (UNLIKELY(race_mask))
+    goto SHARED;
+
+STORE : {
+  if (typ & kAccessCheckOnly)
+    return false;
+  // We could also replace different sid's if access is the same,
+  // rw weaker and happens before. However, just checking access below
+  // is not enough because we also need to check that !both_read_or_atomic
+  // (reads from different sids can be concurrent).
+  // Theoretically we could replace smaller accesses with larger accesses,
+  // but it's unclear if it's worth doing.
+  const m128 mask_access_sid = _mm_set1_epi32(0x0000ffff);
+  const m128 not_same_sid_access = _mm_and_si128(access_xor, mask_access_sid);
+  const m128 same_sid_access = _mm_cmpeq_epi32(not_same_sid_access, zero);
+  const m128 access_read_atomic =
+      _mm_set1_epi32((typ & (kAccessRead | kAccessAtomic)) << 30);
+  const m128 rw_weaker =
+      _mm_cmpeq_epi32(_mm_max_epu32(shadow, access_read_atomic), shadow);
+  const m128 rewrite = _mm_and_si128(same_sid_access, rw_weaker);
+  const int rewrite_mask = _mm_movemask_epi8(rewrite);
+  int index = __builtin_ffs(rewrite_mask);
+  if (UNLIKELY(index == 0)) {
+    const m128 empty = _mm_cmpeq_epi32(shadow, zero);
+    const int empty_mask = _mm_movemask_epi8(empty);
+    index = __builtin_ffs(empty_mask);
+    if (UNLIKELY(index == 0))
+      index = (atomic_load_relaxed(&thr->trace_pos) / 2) % 16;
   }
+  StoreShadow(&shadow_mem[index / 4], cur.raw());
+  // We could zero other slots determined by rewrite_mask.
+  // That would help other threads to evict better slots,
+  // but it's unclear if it's worth it.
   return false;
 }
 
-#if TSAN_VECTORIZE
-#  define SHUF(v0, v1, i0, i1, i2, i3)                    \
-    _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(v0), \
-                                    _mm_castsi128_ps(v1), \
-                                    (i0)*1 + (i1)*4 + (i2)*16 + (i3)*64))
-ALWAYS_INLINE
-bool ContainsSameAccessFast(u64 *s, u64 a, u64 sync_epoch, bool is_write) {
-  // This is an optimized version of ContainsSameAccessSlow.
-  // load current access into access[0:63]
-  const m128 access = _mm_cvtsi64_si128(a);
-  // duplicate high part of access in addr0:
-  // addr0[0:31]        = access[32:63]
-  // addr0[32:63]       = access[32:63]
-  // addr0[64:95]       = access[32:63]
-  // addr0[96:127]      = access[32:63]
-  const m128 addr0 = SHUF(access, access, 1, 1, 1, 1);
-  // load 4 shadow slots
-  const m128 shadow0 = _mm_load_si128((__m128i *)s);
-  const m128 shadow1 = _mm_load_si128((__m128i *)s + 1);
-  // load high parts of 4 shadow slots into addr_vect:
-  // addr_vect[0:31]    = shadow0[32:63]
-  // addr_vect[32:63]   = shadow0[96:127]
-  // addr_vect[64:95]   = shadow1[32:63]
-  // addr_vect[96:127]  = shadow1[96:127]
-  m128 addr_vect = SHUF(shadow0, shadow1, 1, 3, 1, 3);
-  if (!is_write) {
-    // set IsRead bit in addr_vect
-    const m128 rw_mask1 = _mm_cvtsi64_si128(1 << 15);
-    const m128 rw_mask = SHUF(rw_mask1, rw_mask1, 0, 0, 0, 0);
-    addr_vect = _mm_or_si128(addr_vect, rw_mask);
-  }
-  // addr0 == addr_vect?
-  const m128 addr_res = _mm_cmpeq_epi32(addr0, addr_vect);
-  // epoch1[0:63]       = sync_epoch
-  const m128 epoch1 = _mm_cvtsi64_si128(sync_epoch);
-  // epoch[0:31]        = sync_epoch[0:31]
-  // epoch[32:63]       = sync_epoch[0:31]
-  // epoch[64:95]       = sync_epoch[0:31]
-  // epoch[96:127]      = sync_epoch[0:31]
-  const m128 epoch = SHUF(epoch1, epoch1, 0, 0, 0, 0);
-  // load low parts of shadow cell epochs into epoch_vect:
-  // epoch_vect[0:31]   = shadow0[0:31]
-  // epoch_vect[32:63]  = shadow0[64:95]
-  // epoch_vect[64:95]  = shadow1[0:31]
-  // epoch_vect[96:127] = shadow1[64:95]
-  const m128 epoch_vect = SHUF(shadow0, shadow1, 0, 2, 0, 2);
-  // epoch_vect >= sync_epoch?
-  const m128 epoch_res = _mm_cmpgt_epi32(epoch_vect, epoch);
-  // addr_res & epoch_res
-  const m128 res = _mm_and_si128(addr_res, epoch_res);
-  // mask[0] = res[7]
-  // mask[1] = res[15]
-  // ...
-  // mask[15] = res[127]
-  const int mask = _mm_movemask_epi8(res);
-  return mask != 0;
+SHARED:
+  m128 thread_epochs = _mm_set1_epi32(0x7fffffff);
+  // Need to unwind this because _mm_extract_epi8/_mm_insert_epi32
+  // indexes must be constants.
+#  define LOAD_EPOCH(idx)                                                     \
+    if (LIKELY(race_mask & (1 << (idx * 4)))) {                               \
+      u8 sid = _mm_extract_epi8(shadow, idx * 4 + 1);                         \
+      u16 epoch = static_cast<u16>(thr->clock.Get(static_cast<Sid>(sid)));    \
+      thread_epochs = _mm_insert_epi32(thread_epochs, u32(epoch) << 16, idx); \
+    }
+  LOAD_EPOCH(0);
+  LOAD_EPOCH(1);
+  LOAD_EPOCH(2);
+  LOAD_EPOCH(3);
+#  undef LOAD_EPOCH
+  const m128 mask_epoch = _mm_set1_epi32(0x3fff0000);
+  const m128 shadow_epochs = _mm_and_si128(shadow, mask_epoch);
+  const m128 concurrent = _mm_cmplt_epi32(thread_epochs, shadow_epochs);
+  const int concurrent_mask = _mm_movemask_epi8(concurrent);
+  if (LIKELY(concurrent_mask == 0))
+    goto STORE;
+
+  DoReportRaceV(thr, shadow_mem, cur, concurrent_mask, shadow, typ);
+  return true;
 }
-#endif
 
-ALWAYS_INLINE
-bool ContainsSameAccess(u64 *s, u64 a, u64 sync_epoch, bool is_write) {
-#if TSAN_VECTORIZE
-  bool res = ContainsSameAccessFast(s, a, sync_epoch, is_write);
-  // NOTE: this check can fail if the shadow is concurrently mutated
-  // by other threads. But it still can be useful if you modify
-  // ContainsSameAccessFast and want to ensure that it's not completely broken.
-  // DCHECK_EQ(res, ContainsSameAccessSlow(s, a, sync_epoch, is_write));
-  return res;
-#else
-  return ContainsSameAccessSlow(s, a, sync_epoch, is_write);
+#  define LOAD_CURRENT_SHADOW(cur, shadow_mem)                         \
+    const m128 access = _mm_set1_epi32(static_cast<u32>((cur).raw())); \
+    const m128 shadow = _mm_load_si128(reinterpret_cast<m128*>(shadow_mem))
 #endif
-}
 
-ALWAYS_INLINE USED void MemoryAccess(ThreadState *thr, uptr pc, uptr addr,
-                                     int kAccessSizeLog, bool kAccessIsWrite,
-                                     bool kIsAtomic) {
-  RawShadow *shadow_mem = MemToShadow(addr);
-  DPrintf2(
-      "#%d: MemoryAccess: @%p %p size=%d"
-      " is_write=%d shadow_mem=%p {%zx, %zx, %zx, %zx}\n",
-      (int)thr->fast_state.tid(), (void *)pc, (void *)addr,
-      (int)(1 << kAccessSizeLog), kAccessIsWrite, shadow_mem,
-      (uptr)shadow_mem[0], (uptr)shadow_mem[1], (uptr)shadow_mem[2],
-      (uptr)shadow_mem[3]);
-#if SANITIZER_DEBUG
-  if (!IsAppMem(addr)) {
-    Printf("Access to non app mem %zx\n", addr);
-    DCHECK(IsAppMem(addr));
+char* DumpShadow(char* buf, RawShadow raw) {
+  if (raw == Shadow::kEmpty) {
+    internal_snprintf(buf, 64, "0");
+    return buf;
   }
-  if (!IsShadowMem(shadow_mem)) {
-    Printf("Bad shadow addr %p (%zx)\n", shadow_mem, addr);
-    DCHECK(IsShadowMem(shadow_mem));
-  }
-#endif
+  Shadow s(raw);
+  AccessType typ;
+  s.GetAccess(nullptr, nullptr, &typ);
+  internal_snprintf(buf, 64, "{tid=%u@%u access=0x%x typ=%x}",
+                    static_cast<u32>(s.sid()), static_cast<u32>(s.epoch()),
+                    s.access(), static_cast<u32>(typ));
+  return buf;
+}
 
-  if (!SANITIZER_GO && !kAccessIsWrite && *shadow_mem == kShadowRodata) {
-    // Access to .rodata section, no races here.
-    // Measurements show that it can be 10-20% of all memory accesses.
-    return;
-  }
+// TryTrace* and TraceRestart* functions allow to turn memory access and func
+// entry/exit callbacks into leaf functions with all associated performance
+// benefits. These hottest callbacks do only 2 slow path calls: report a race
+// and trace part switching. Race reporting is easy to turn into a tail call, we
+// just always return from the runtime after reporting a race. But trace part
+// switching is harder because it needs to be in the middle of callbacks. To
+// turn it into a tail call we immidiately return after TraceRestart* functions,
+// but TraceRestart* functions themselves recurse into the callback after
+// switching trace part. As the result the hottest callbacks contain only tail
+// calls, which effectively makes them leaf functions (can use all registers,
+// no frame setup, etc).
+NOINLINE void TraceRestartMemoryAccess(ThreadState* thr, uptr pc, uptr addr,
+                                       uptr size, AccessType typ) {
+  TraceSwitchPart(thr);
+  MemoryAccess(thr, pc, addr, size, typ);
+}
+
+ALWAYS_INLINE USED void MemoryAccess(ThreadState* thr, uptr pc, uptr addr,
+                                     uptr size, AccessType typ) {
+  RawShadow* shadow_mem = MemToShadow(addr);
+  UNUSED char memBuf[4][64];
+  DPrintf2("#%d: Access: %d@%d %p/%zd typ=0x%x {%s, %s, %s, %s}\n", thr->tid,
+           static_cast<int>(thr->fast_state.sid()),
+           static_cast<int>(thr->fast_state.epoch()), (void*)addr, size,
+           static_cast<int>(typ), DumpShadow(memBuf[0], shadow_mem[0]),
+           DumpShadow(memBuf[1], shadow_mem[1]),
+           DumpShadow(memBuf[2], shadow_mem[2]),
+           DumpShadow(memBuf[3], shadow_mem[3]));
 
   FastState fast_state = thr->fast_state;
-  if (UNLIKELY(fast_state.GetIgnoreBit())) {
+  Shadow cur(fast_state, addr, size, typ);
+
+  LOAD_CURRENT_SHADOW(cur, shadow_mem);
+  if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))
     return;
-  }
+  if (UNLIKELY(fast_state.GetIgnoreBit()))
+    return;
+  if (!TryTraceMemoryAccess(thr, pc, addr, size, typ))
+    return TraceRestartMemoryAccess(thr, pc, addr, size, typ);
+  CheckRaces(thr, shadow_mem, cur, shadow, access, typ);
+}
 
-  Shadow cur(fast_state);
-  cur.SetAddr0AndSizeLog(addr & 7, kAccessSizeLog);
-  cur.SetWrite(kAccessIsWrite);
-  cur.SetAtomic(kIsAtomic);
+void MemoryAccess16(ThreadState* thr, uptr pc, uptr addr, AccessType typ);
 
-  if (LIKELY(ContainsSameAccess(shadow_mem, cur.raw(), thr->fast_synch_epoch,
-                                kAccessIsWrite))) {
-    return;
-  }
+NOINLINE
+void RestartMemoryAccess16(ThreadState* thr, uptr pc, uptr addr,
+                           AccessType typ) {
+  TraceSwitchPart(thr);
+  MemoryAccess16(thr, pc, addr, typ);
+}
 
-  if (kCollectHistory) {
-    fast_state.IncrementEpoch();
-    thr->fast_state = fast_state;
-    TraceAddEvent(thr, fast_state, EventTypeMop, pc);
-    cur.IncrementEpoch();
+ALWAYS_INLINE USED void MemoryAccess16(ThreadState* thr, uptr pc, uptr addr,
+                                       AccessType typ) {
+  const uptr size = 16;
+  FastState fast_state = thr->fast_state;
+  if (UNLIKELY(fast_state.GetIgnoreBit()))
+    return;
+  Shadow cur(fast_state, 0, 8, typ);
+  RawShadow* shadow_mem = MemToShadow(addr);
+  bool traced = false;
+  {
+    LOAD_CURRENT_SHADOW(cur, shadow_mem);
+    if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))
+      goto SECOND;
+    if (!TryTraceMemoryAccessRange(thr, pc, addr, size, typ))
+      return RestartMemoryAccess16(thr, pc, addr, typ);
+    traced = true;
+    if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, shadow, access, typ)))
+      return;
   }
+SECOND:
+  shadow_mem += kShadowCnt;
+  LOAD_CURRENT_SHADOW(cur, shadow_mem);
+  if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))
+    return;
+  if (!traced && !TryTraceMemoryAccessRange(thr, pc, addr, size, typ))
+    return RestartMemoryAccess16(thr, pc, addr, typ);
+  CheckRaces(thr, shadow_mem, cur, shadow, access, typ);
+}
 
-  MemoryAccessImpl1(thr, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic,
-                    shadow_mem, cur);
+NOINLINE
+void RestartUnalignedMemoryAccess(ThreadState* thr, uptr pc, uptr addr,
+                                  uptr size, AccessType typ) {
+  TraceSwitchPart(thr);
+  UnalignedMemoryAccess(thr, pc, addr, size, typ);
 }
 
-// Called by MemoryAccessRange in tsan_rtl_thread.cpp
-ALWAYS_INLINE USED void MemoryAccessImpl(ThreadState *thr, uptr addr,
-                                         int kAccessSizeLog,
-                                         bool kAccessIsWrite, bool kIsAtomic,
-                                         u64 *shadow_mem, Shadow cur) {
-  if (LIKELY(ContainsSameAccess(shadow_mem, cur.raw(), thr->fast_synch_epoch,
-                                kAccessIsWrite))) {
+ALWAYS_INLINE USED void UnalignedMemoryAccess(ThreadState* thr, uptr pc,
+                                              uptr addr, uptr size,
+                                              AccessType typ) {
+  DCHECK_LE(size, 8);
+  FastState fast_state = thr->fast_state;
+  if (UNLIKELY(fast_state.GetIgnoreBit()))
     return;
+  RawShadow* shadow_mem = MemToShadow(addr);
+  bool traced = false;
+  uptr size1 = Min<uptr>(size, RoundUp(addr + 1, kShadowCell) - addr);
+  {
+    Shadow cur(fast_state, addr, size1, typ);
+    LOAD_CURRENT_SHADOW(cur, shadow_mem);
+    if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))
+      goto SECOND;
+    if (!TryTraceMemoryAccessRange(thr, pc, addr, size, typ))
+      return RestartUnalignedMemoryAccess(thr, pc, addr, size, typ);
+    traced = true;
+    if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, shadow, access, typ)))
+      return;
   }
+SECOND:
+  uptr size2 = size - size1;
+  if (LIKELY(size2 == 0))
+    return;
+  shadow_mem += kShadowCnt;
+  Shadow cur(fast_state, 0, size2, typ);
+  LOAD_CURRENT_SHADOW(cur, shadow_mem);
+  if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))
+    return;
+  if (!traced && !TryTraceMemoryAccessRange(thr, pc, addr, size, typ))
+    return RestartUnalignedMemoryAccess(thr, pc, addr, size, typ);
+  CheckRaces(thr, shadow_mem, cur, shadow, access, typ);
+}
 
-  MemoryAccessImpl1(thr, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic,
-                    shadow_mem, cur);
+void ShadowSet(RawShadow* p, RawShadow* end, RawShadow v) {
+  DCHECK_LE(p, end);
+  DCHECK(IsShadowMem(p));
+  DCHECK(IsShadowMem(end));
+  UNUSED const uptr kAlign = kShadowCnt * kShadowSize;
+  DCHECK_EQ(reinterpret_cast<uptr>(p) % kAlign, 0);
+  DCHECK_EQ(reinterpret_cast<uptr>(end) % kAlign, 0);
+#if !TSAN_VECTORIZE
+  for (; p < end; p += kShadowCnt) {
+    p[0] = v;
+    for (uptr i = 1; i < kShadowCnt; i++) p[i] = Shadow::kEmpty;
+  }
+#else
+  m128 vv = _mm_setr_epi32(
+      static_cast<u32>(v), static_cast<u32>(Shadow::kEmpty),
+      static_cast<u32>(Shadow::kEmpty), static_cast<u32>(Shadow::kEmpty));
+  m128* vp = reinterpret_cast<m128*>(p);
+  m128* vend = reinterpret_cast<m128*>(end);
+  for (; vp < vend; vp++) _mm_store_si128(vp, vv);
+#endif
 }
 
-static void MemoryRangeSet(ThreadState *thr, uptr pc, uptr addr, uptr size,
-                           u64 val) {
-  (void)thr;
-  (void)pc;
+static void MemoryRangeSet(uptr addr, uptr size, RawShadow val) {
   if (size == 0)
     return;
-  // FIXME: fix me.
-  uptr offset = addr % kShadowCell;
-  if (offset) {
-    offset = kShadowCell - offset;
-    if (size <= offset)
-      return;
-    addr += offset;
-    size -= offset;
-  }
-  DCHECK_EQ(addr % 8, 0);
+  DCHECK_EQ(addr % kShadowCell, 0);
+  DCHECK_EQ(size % kShadowCell, 0);
   // If a user passes some insane arguments (memset(0)),
   // let it just crash as usual.
   if (!IsAppMem(addr) || !IsAppMem(addr + size - 1))
     return;
+  RawShadow* begin = MemToShadow(addr);
+  RawShadow* end = begin + size / kShadowCell * kShadowCnt;
   // Don't want to touch lots of shadow memory.
   // If a program maps 10MB stack, there is no need reset the whole range.
-  size = (size + (kShadowCell - 1)) & ~(kShadowCell - 1);
   // UnmapOrDie/MmapFixedNoReserve does not work on Windows.
-  if (SANITIZER_WINDOWS || size < common_flags()->clear_shadow_mmap_threshold) {
-    RawShadow *p = MemToShadow(addr);
-    CHECK(IsShadowMem(p));
-    CHECK(IsShadowMem(p + size * kShadowCnt / kShadowCell - 1));
-    // FIXME: may overwrite a part outside the region
-    for (uptr i = 0; i < size / kShadowCell * kShadowCnt;) {
-      p[i++] = val;
-      for (uptr j = 1; j < kShadowCnt; j++) p[i++] = 0;
-    }
-  } else {
-    // The region is big, reset only beginning and end.
-    const uptr kPageSize = GetPageSizeCached();
-    RawShadow *begin = MemToShadow(addr);
-    RawShadow *end = begin + size / kShadowCell * kShadowCnt;
-    RawShadow *p = begin;
-    // Set at least first kPageSize/2 to page boundary.
-    while ((p < begin + kPageSize / kShadowSize / 2) || ((uptr)p % kPageSize)) {
-      *p++ = val;
-      for (uptr j = 1; j < kShadowCnt; j++) *p++ = 0;
-    }
-    // Reset middle part.
-    RawShadow *p1 = p;
-    p = RoundDown(end, kPageSize);
-    if (!MmapFixedSuperNoReserve((uptr)p1, (uptr)p - (uptr)p1))
+  if (SANITIZER_WINDOWS ||
+      size <= common_flags()->clear_shadow_mmap_threshold) {
+    ShadowSet(begin, end, val);
+    return;
+  }
+  // The region is big, reset only beginning and end.
+  const uptr kPageSize = GetPageSizeCached();
+  // Set at least first kPageSize/2 to page boundary.
+  RawShadow* mid1 =
+      Min(end, reinterpret_cast<RawShadow*>(RoundUp(
+                   reinterpret_cast<uptr>(begin) + kPageSize / 2, kPageSize)));
+  ShadowSet(begin, mid1, val);
+  // Reset middle part.
+  RawShadow* mid2 = RoundDown(end, kPageSize);
+  if (mid2 > mid1) {
+    if (!MmapFixedSuperNoReserve((uptr)mid1, (uptr)mid2 - (uptr)mid1))
       Die();
-    // Set the ending.
-    while (p < end) {
-      *p++ = val;
-      for (uptr j = 1; j < kShadowCnt; j++) *p++ = 0;
-    }
   }
+  // Set the ending.
+  ShadowSet(mid2, end, val);
 }
 
-void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size) {
-  MemoryRangeSet(thr, pc, addr, size, 0);
+void MemoryResetRange(ThreadState* thr, uptr pc, uptr addr, uptr size) {
+  uptr addr1 = RoundDown(addr, kShadowCell);
+  uptr size1 = RoundUp(size + addr - addr1, kShadowCell);
+  MemoryRangeSet(addr1, size1, Shadow::kEmpty);
 }
 
-void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size) {
-  // Processing more than 1k (4k of shadow) is expensive,
+void MemoryRangeFreed(ThreadState* thr, uptr pc, uptr addr, uptr size) {
+  // Callers must lock the slot to ensure synchronization with the reset.
+  // The problem with "freed" memory is that it's not "monotonic"
+  // with respect to bug detection: freed memory is bad to access,
+  // but then if the heap block is reallocated later, it's good to access.
+  // As the result a garbage "freed" shadow can lead to a false positive
+  // if it happens to match a real free in the thread trace,
+  // but the heap block was reallocated before the current memory access,
+  // so it's still good to access. It's not the case with data races.
+  DCHECK(thr->slot_locked);
+  DCHECK_EQ(addr % kShadowCell, 0);
+  size = RoundUp(size, kShadowCell);
+  // Processing more than 1k (2k of shadow) is expensive,
   // can cause excessive memory consumption (user does not necessary touch
   // the whole range) and most likely unnecessary.
-  if (size > 1024)
-    size = 1024;
-  CHECK_EQ(thr->is_freeing, false);
-  thr->is_freeing = true;
-  MemoryAccessRange(thr, pc, addr, size, true);
-  thr->is_freeing = false;
-  if (kCollectHistory) {
-    thr->fast_state.IncrementEpoch();
-    TraceAddEvent(thr, thr->fast_state, EventTypeMop, pc);
+  size = Min<uptr>(size, 1024);
+  const AccessType typ =
+      kAccessWrite | kAccessFree | kAccessCheckOnly | kAccessNoRodata;
+  TraceMemoryAccessRange(thr, pc, addr, size, typ);
+  RawShadow* shadow_mem = MemToShadow(addr);
+  Shadow cur(thr->fast_state, 0, kShadowCell, typ);
+#if TSAN_VECTORIZE
+  const m128 access = _mm_set1_epi32(static_cast<u32>(cur.raw()));
+  const m128 freed = _mm_setr_epi32(
+      static_cast<u32>(Shadow::FreedMarker()),
+      static_cast<u32>(Shadow::FreedInfo(cur.sid(), cur.epoch())), 0, 0);
+  for (; size; size -= kShadowCell, shadow_mem += kShadowCnt) {
+    const m128 shadow = _mm_load_si128((m128*)shadow_mem);
+    if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, shadow, access, typ)))
+      return;
+    _mm_store_si128((m128*)shadow_mem, freed);
   }
-  Shadow s(thr->fast_state);
-  s.ClearIgnoreBit();
-  s.MarkAsFreed();
-  s.SetWrite(true);
-  s.SetAddr0AndSizeLog(0, 3);
-  MemoryRangeSet(thr, pc, addr, size, s.raw());
-}
-
-void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size) {
-  if (kCollectHistory) {
-    thr->fast_state.IncrementEpoch();
-    TraceAddEvent(thr, thr->fast_state, EventTypeMop, pc);
+#else
+  for (; size; size -= kShadowCell, shadow_mem += kShadowCnt) {
+    if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, 0, 0, typ)))
+      return;
+    StoreShadow(&shadow_mem[0], Shadow::FreedMarker());
+    StoreShadow(&shadow_mem[1], Shadow::FreedInfo(cur.sid(), cur.epoch()));
+    StoreShadow(&shadow_mem[2], Shadow::kEmpty);
+    StoreShadow(&shadow_mem[3], Shadow::kEmpty);
   }
-  Shadow s(thr->fast_state);
-  s.ClearIgnoreBit();
-  s.SetWrite(true);
-  s.SetAddr0AndSizeLog(0, 3);
-  MemoryRangeSet(thr, pc, addr, size, s.raw());
+#endif
+}
+
+void MemoryRangeImitateWrite(ThreadState* thr, uptr pc, uptr addr, uptr size) {
+  DCHECK_EQ(addr % kShadowCell, 0);
+  size = RoundUp(size, kShadowCell);
+  TraceMemoryAccessRange(thr, pc, addr, size, kAccessWrite);
+  Shadow cur(thr->fast_state, 0, 8, kAccessWrite);
+  MemoryRangeSet(addr, size, cur.raw());
 }
 
-void MemoryRangeImitateWriteOrResetRange(ThreadState *thr, uptr pc, uptr addr,
+void MemoryRangeImitateWriteOrResetRange(ThreadState* thr, uptr pc, uptr addr,
                                          uptr size) {
   if (thr->ignore_reads_and_writes == 0)
     MemoryRangeImitateWrite(thr, pc, addr, size);
@@ -518,14 +655,29 @@ void MemoryRangeImitateWriteOrResetRange(ThreadState *thr, uptr pc, uptr addr,
     MemoryResetRange(thr, pc, addr, size);
 }
 
-void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size,
-                       bool is_write) {
-  if (size == 0)
-    return;
+ALWAYS_INLINE
+bool MemoryAccessRangeOne(ThreadState* thr, RawShadow* shadow_mem, Shadow cur,
+                          AccessType typ) {
+  LOAD_CURRENT_SHADOW(cur, shadow_mem);
+  if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))
+    return false;
+  return CheckRaces(thr, shadow_mem, cur, shadow, access, typ);
+}
+
+template <bool is_read>
+NOINLINE void RestartMemoryAccessRange(ThreadState* thr, uptr pc, uptr addr,
+                                       uptr size) {
+  TraceSwitchPart(thr);
+  MemoryAccessRangeT<is_read>(thr, pc, addr, size);
+}
 
-  RawShadow *shadow_mem = MemToShadow(addr);
-  DPrintf2("#%d: MemoryAccessRange: @%p %p size=%d is_write=%d\n", thr->tid,
-           (void *)pc, (void *)addr, (int)size, is_write);
+template <bool is_read>
+void MemoryAccessRangeT(ThreadState* thr, uptr pc, uptr addr, uptr size) {
+  const AccessType typ =
+      (is_read ? kAccessRead : kAccessWrite) | kAccessNoRodata;
+  RawShadow* shadow_mem = MemToShadow(addr);
+  DPrintf2("#%d: MemoryAccessRange: @%p %p size=%d is_read=%d\n", thr->tid,
+           (void*)pc, (void*)addr, (int)size, is_read);
 
 #if SANITIZER_DEBUG
   if (!IsAppMem(addr)) {
@@ -537,65 +689,62 @@ void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size,
     DCHECK(IsAppMem(addr + size - 1));
   }
   if (!IsShadowMem(shadow_mem)) {
-    Printf("Bad shadow addr %p (%zx)\n", shadow_mem, addr);
+    Printf("Bad shadow addr %p (%zx)\n", static_cast<void*>(shadow_mem), addr);
     DCHECK(IsShadowMem(shadow_mem));
   }
-  if (!IsShadowMem(shadow_mem + size * kShadowCnt / 8 - 1)) {
-    Printf("Bad shadow addr %p (%zx)\n", shadow_mem + size * kShadowCnt / 8 - 1,
+  if (!IsShadowMem(shadow_mem + size * kShadowCnt - 1)) {
+    Printf("Bad shadow addr %p (%zx)\n",
+           static_cast<void*>(shadow_mem + size * kShadowCnt - 1),
            addr + size - 1);
-    DCHECK(IsShadowMem(shadow_mem + size * kShadowCnt / 8 - 1));
+    DCHECK(IsShadowMem(shadow_mem + size * kShadowCnt - 1));
   }
 #endif
 
-  if (*shadow_mem == kShadowRodata) {
-    DCHECK(!is_write);
-    // Access to .rodata section, no races here.
-    // Measurements show that it can be 10-20% of all memory accesses.
+  // Access to .rodata section, no races here.
+  // Measurements show that it can be 10-20% of all memory accesses.
+  // Check here once to not check for every access separately.
+  // Note: we could (and should) do this only for the is_read case
+  // (writes shouldn't go to .rodata). But it happens in Chromium tests:
+  // https://bugs.chromium.org/p/chromium/issues/detail?id=1275581#c19
+  // Details are unknown since it happens only on CI machines.
+  if (*shadow_mem == Shadow::kRodata)
     return;
-  }
 
   FastState fast_state = thr->fast_state;
-  if (fast_state.GetIgnoreBit())
+  if (UNLIKELY(fast_state.GetIgnoreBit()))
     return;
 
-  fast_state.IncrementEpoch();
-  thr->fast_state = fast_state;
-  TraceAddEvent(thr, fast_state, EventTypeMop, pc);
+  if (!TryTraceMemoryAccessRange(thr, pc, addr, size, typ))
+    return RestartMemoryAccessRange<is_read>(thr, pc, addr, size);
 
-  bool unaligned = (addr % kShadowCell) != 0;
-
-  // Handle unaligned beginning, if any.
-  for (; addr % kShadowCell && size; addr++, size--) {
-    int const kAccessSizeLog = 0;
-    Shadow cur(fast_state);
-    cur.SetWrite(is_write);
-    cur.SetAddr0AndSizeLog(addr & (kShadowCell - 1), kAccessSizeLog);
-    MemoryAccessImpl(thr, addr, kAccessSizeLog, is_write, false, shadow_mem,
-                     cur);
-  }
-  if (unaligned)
+  if (UNLIKELY(addr % kShadowCell)) {
+    // Handle unaligned beginning, if any.
+    uptr size1 = Min(size, RoundUp(addr, kShadowCell) - addr);
+    size -= size1;
+    Shadow cur(fast_state, addr, size1, typ);
+    if (UNLIKELY(MemoryAccessRangeOne(thr, shadow_mem, cur, typ)))
+      return;
     shadow_mem += kShadowCnt;
+  }
   // Handle middle part, if any.
-  for (; size >= kShadowCell; addr += kShadowCell, size -= kShadowCell) {
-    int const kAccessSizeLog = 3;
-    Shadow cur(fast_state);
-    cur.SetWrite(is_write);
-    cur.SetAddr0AndSizeLog(0, kAccessSizeLog);
-    MemoryAccessImpl(thr, addr, kAccessSizeLog, is_write, false, shadow_mem,
-                     cur);
-    shadow_mem += kShadowCnt;
+  Shadow cur(fast_state, 0, kShadowCell, typ);
+  for (; size >= kShadowCell; size -= kShadowCell, shadow_mem += kShadowCnt) {
+    if (UNLIKELY(MemoryAccessRangeOne(thr, shadow_mem, cur, typ)))
+      return;
   }
   // Handle ending, if any.
-  for (; size; addr++, size--) {
-    int const kAccessSizeLog = 0;
-    Shadow cur(fast_state);
-    cur.SetWrite(is_write);
-    cur.SetAddr0AndSizeLog(addr & (kShadowCell - 1), kAccessSizeLog);
-    MemoryAccessImpl(thr, addr, kAccessSizeLog, is_write, false, shadow_mem,
-                     cur);
+  if (UNLIKELY(size)) {
+    Shadow cur(fast_state, 0, size, typ);
+    if (UNLIKELY(MemoryAccessRangeOne(thr, shadow_mem, cur, typ)))
+      return;
   }
 }
 
+template void MemoryAccessRangeT<true>(ThreadState* thr, uptr pc, uptr addr,
+                                       uptr size);
+template void MemoryAccessRangeT<false>(ThreadState* thr, uptr pc, uptr addr,
+                                        uptr size);
+
 }  // namespace __tsan
 
 #if !SANITIZER_GO
diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl_amd64.S b/compiler-rt/lib/tsan/rtl/tsan_rtl_amd64.S
index 632b19d18158..f848be9dd46c 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_rtl_amd64.S
+++ b/compiler-rt/lib/tsan/rtl/tsan_rtl_amd64.S
@@ -9,242 +9,6 @@
 .section __TEXT,__text
 #endif
 
-ASM_HIDDEN(__tsan_trace_switch)
-.globl ASM_SYMBOL(__tsan_trace_switch_thunk)
-ASM_SYMBOL(__tsan_trace_switch_thunk):
-  CFI_STARTPROC
-  _CET_ENDBR
-  # Save scratch registers.
-  push %rax
-  CFI_ADJUST_CFA_OFFSET(8)
-  CFI_REL_OFFSET(%rax, 0)
-  push %rcx
-  CFI_ADJUST_CFA_OFFSET(8)
-  CFI_REL_OFFSET(%rcx, 0)
-  push %rdx
-  CFI_ADJUST_CFA_OFFSET(8)
-  CFI_REL_OFFSET(%rdx, 0)
-  push %rsi
-  CFI_ADJUST_CFA_OFFSET(8)
-  CFI_REL_OFFSET(%rsi, 0)
-  push %rdi
-  CFI_ADJUST_CFA_OFFSET(8)
-  CFI_REL_OFFSET(%rdi, 0)
-  push %r8
-  CFI_ADJUST_CFA_OFFSET(8)
-  CFI_REL_OFFSET(%r8, 0)
-  push %r9
-  CFI_ADJUST_CFA_OFFSET(8)
-  CFI_REL_OFFSET(%r9, 0)
-  push %r10
-  CFI_ADJUST_CFA_OFFSET(8)
-  CFI_REL_OFFSET(%r10, 0)
-  push %r11
-  CFI_ADJUST_CFA_OFFSET(8)
-  CFI_REL_OFFSET(%r11, 0)
-  # All XMM registers are caller-saved.
-  sub $0x100, %rsp
-  CFI_ADJUST_CFA_OFFSET(0x100)
-  vmovdqu %xmm0, 0x0(%rsp)
-  vmovdqu %xmm1, 0x10(%rsp)
-  vmovdqu %xmm2, 0x20(%rsp)
-  vmovdqu %xmm3, 0x30(%rsp)
-  vmovdqu %xmm4, 0x40(%rsp)
-  vmovdqu %xmm5, 0x50(%rsp)
-  vmovdqu %xmm6, 0x60(%rsp)
-  vmovdqu %xmm7, 0x70(%rsp)
-  vmovdqu %xmm8, 0x80(%rsp)
-  vmovdqu %xmm9, 0x90(%rsp)
-  vmovdqu %xmm10, 0xa0(%rsp)
-  vmovdqu %xmm11, 0xb0(%rsp)
-  vmovdqu %xmm12, 0xc0(%rsp)
-  vmovdqu %xmm13, 0xd0(%rsp)
-  vmovdqu %xmm14, 0xe0(%rsp)
-  vmovdqu %xmm15, 0xf0(%rsp)
-  # Align stack frame.
-  push %rbx  # non-scratch
-  CFI_ADJUST_CFA_OFFSET(8)
-  CFI_REL_OFFSET(%rbx, 0)
-  mov %rsp, %rbx  # save current rsp
-  CFI_DEF_CFA_REGISTER(%rbx)
-  shr $4, %rsp  # clear 4 lsb, align to 16
-  shl $4, %rsp
-
-  call ASM_SYMBOL(__tsan_trace_switch)
-
-  # Unalign stack frame back.
-  mov %rbx, %rsp  # restore the original rsp
-  CFI_DEF_CFA_REGISTER(%rsp)
-  pop %rbx
-  CFI_ADJUST_CFA_OFFSET(-8)
-  # Restore scratch registers.
-  vmovdqu 0x0(%rsp), %xmm0
-  vmovdqu 0x10(%rsp), %xmm1
-  vmovdqu 0x20(%rsp), %xmm2
-  vmovdqu 0x30(%rsp), %xmm3
-  vmovdqu 0x40(%rsp), %xmm4
-  vmovdqu 0x50(%rsp), %xmm5
-  vmovdqu 0x60(%rsp), %xmm6
-  vmovdqu 0x70(%rsp), %xmm7
-  vmovdqu 0x80(%rsp), %xmm8
-  vmovdqu 0x90(%rsp), %xmm9
-  vmovdqu 0xa0(%rsp), %xmm10
-  vmovdqu 0xb0(%rsp), %xmm11
-  vmovdqu 0xc0(%rsp), %xmm12
-  vmovdqu 0xd0(%rsp), %xmm13
-  vmovdqu 0xe0(%rsp), %xmm14
-  vmovdqu 0xf0(%rsp), %xmm15
-  add $0x100, %rsp
-  CFI_ADJUST_CFA_OFFSET(-0x100)
-  pop %r11
-  CFI_ADJUST_CFA_OFFSET(-8)
-  pop %r10
-  CFI_ADJUST_CFA_OFFSET(-8)
-  pop %r9
-  CFI_ADJUST_CFA_OFFSET(-8)
-  pop %r8
-  CFI_ADJUST_CFA_OFFSET(-8)
-  pop %rdi
-  CFI_ADJUST_CFA_OFFSET(-8)
-  pop %rsi
-  CFI_ADJUST_CFA_OFFSET(-8)
-  pop %rdx
-  CFI_ADJUST_CFA_OFFSET(-8)
-  pop %rcx
-  CFI_ADJUST_CFA_OFFSET(-8)
-  pop %rax
-  CFI_ADJUST_CFA_OFFSET(-8)
-  CFI_RESTORE(%rax)
-  CFI_RESTORE(%rbx)
-  CFI_RESTORE(%rcx)
-  CFI_RESTORE(%rdx)
-  CFI_RESTORE(%rsi)
-  CFI_RESTORE(%rdi)
-  CFI_RESTORE(%r8)
-  CFI_RESTORE(%r9)
-  CFI_RESTORE(%r10)
-  CFI_RESTORE(%r11)
-  ret
-  CFI_ENDPROC
-
-ASM_HIDDEN(__tsan_report_race)
-.globl ASM_SYMBOL(__tsan_report_race_thunk)
-ASM_SYMBOL(__tsan_report_race_thunk):
-  CFI_STARTPROC
-  _CET_ENDBR
-  # Save scratch registers.
-  push %rax
-  CFI_ADJUST_CFA_OFFSET(8)
-  CFI_REL_OFFSET(%rax, 0)
-  push %rcx
-  CFI_ADJUST_CFA_OFFSET(8)
-  CFI_REL_OFFSET(%rcx, 0)
-  push %rdx
-  CFI_ADJUST_CFA_OFFSET(8)
-  CFI_REL_OFFSET(%rdx, 0)
-  push %rsi
-  CFI_ADJUST_CFA_OFFSET(8)
-  CFI_REL_OFFSET(%rsi, 0)
-  push %rdi
-  CFI_ADJUST_CFA_OFFSET(8)
-  CFI_REL_OFFSET(%rdi, 0)
-  push %r8
-  CFI_ADJUST_CFA_OFFSET(8)
-  CFI_REL_OFFSET(%r8, 0)
-  push %r9
-  CFI_ADJUST_CFA_OFFSET(8)
-  CFI_REL_OFFSET(%r9, 0)
-  push %r10
-  CFI_ADJUST_CFA_OFFSET(8)
-  CFI_REL_OFFSET(%r10, 0)
-  push %r11
-  CFI_ADJUST_CFA_OFFSET(8)
-  CFI_REL_OFFSET(%r11, 0)
-  # All XMM registers are caller-saved.
-  sub $0x100, %rsp
-  CFI_ADJUST_CFA_OFFSET(0x100)
-  vmovdqu %xmm0, 0x0(%rsp)
-  vmovdqu %xmm1, 0x10(%rsp)
-  vmovdqu %xmm2, 0x20(%rsp)
-  vmovdqu %xmm3, 0x30(%rsp)
-  vmovdqu %xmm4, 0x40(%rsp)
-  vmovdqu %xmm5, 0x50(%rsp)
-  vmovdqu %xmm6, 0x60(%rsp)
-  vmovdqu %xmm7, 0x70(%rsp)
-  vmovdqu %xmm8, 0x80(%rsp)
-  vmovdqu %xmm9, 0x90(%rsp)
-  vmovdqu %xmm10, 0xa0(%rsp)
-  vmovdqu %xmm11, 0xb0(%rsp)
-  vmovdqu %xmm12, 0xc0(%rsp)
-  vmovdqu %xmm13, 0xd0(%rsp)
-  vmovdqu %xmm14, 0xe0(%rsp)
-  vmovdqu %xmm15, 0xf0(%rsp)
-  # Align stack frame.
-  push %rbx  # non-scratch
-  CFI_ADJUST_CFA_OFFSET(8)
-  CFI_REL_OFFSET(%rbx, 0)
-  mov %rsp, %rbx  # save current rsp
-  CFI_DEF_CFA_REGISTER(%rbx)
-  shr $4, %rsp  # clear 4 lsb, align to 16
-  shl $4, %rsp
-
-  call ASM_SYMBOL(__tsan_report_race)
-
-  # Unalign stack frame back.
-  mov %rbx, %rsp  # restore the original rsp
-  CFI_DEF_CFA_REGISTER(%rsp)
-  pop %rbx
-  CFI_ADJUST_CFA_OFFSET(-8)
-  # Restore scratch registers.
-  vmovdqu 0x0(%rsp), %xmm0
-  vmovdqu 0x10(%rsp), %xmm1
-  vmovdqu 0x20(%rsp), %xmm2
-  vmovdqu 0x30(%rsp), %xmm3
-  vmovdqu 0x40(%rsp), %xmm4
-  vmovdqu 0x50(%rsp), %xmm5
-  vmovdqu 0x60(%rsp), %xmm6
-  vmovdqu 0x70(%rsp), %xmm7
-  vmovdqu 0x80(%rsp), %xmm8
-  vmovdqu 0x90(%rsp), %xmm9
-  vmovdqu 0xa0(%rsp), %xmm10
-  vmovdqu 0xb0(%rsp), %xmm11
-  vmovdqu 0xc0(%rsp), %xmm12
-  vmovdqu 0xd0(%rsp), %xmm13
-  vmovdqu 0xe0(%rsp), %xmm14
-  vmovdqu 0xf0(%rsp), %xmm15
-  add $0x100, %rsp
-  CFI_ADJUST_CFA_OFFSET(-0x100)
-  pop %r11
-  CFI_ADJUST_CFA_OFFSET(-8)
-  pop %r10
-  CFI_ADJUST_CFA_OFFSET(-8)
-  pop %r9
-  CFI_ADJUST_CFA_OFFSET(-8)
-  pop %r8
-  CFI_ADJUST_CFA_OFFSET(-8)
-  pop %rdi
-  CFI_ADJUST_CFA_OFFSET(-8)
-  pop %rsi
-  CFI_ADJUST_CFA_OFFSET(-8)
-  pop %rdx
-  CFI_ADJUST_CFA_OFFSET(-8)
-  pop %rcx
-  CFI_ADJUST_CFA_OFFSET(-8)
-  pop %rax
-  CFI_ADJUST_CFA_OFFSET(-8)
-  CFI_RESTORE(%rax)
-  CFI_RESTORE(%rbx)
-  CFI_RESTORE(%rcx)
-  CFI_RESTORE(%rdx)
-  CFI_RESTORE(%rsi)
-  CFI_RESTORE(%rdi)
-  CFI_RESTORE(%r8)
-  CFI_RESTORE(%r9)
-  CFI_RESTORE(%r10)
-  CFI_RESTORE(%r11)
-  ret
-  CFI_ENDPROC
-
 ASM_HIDDEN(__tsan_setjmp)
 #if defined(__NetBSD__)
 .comm _ZN14__interception15real___setjmp14E,8,8
diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl_mutex.cpp b/compiler-rt/lib/tsan/rtl/tsan_rtl_mutex.cpp
index 7d6b41116aa6..5d31005c2af0 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_rtl_mutex.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_rtl_mutex.cpp
@@ -23,6 +23,8 @@
 namespace __tsan {
 
 void ReportDeadlock(ThreadState *thr, uptr pc, DDReport *r);
+void ReportDestroyLocked(ThreadState *thr, uptr pc, uptr addr,
+                         FastState last_lock, StackID creation_stack_id);
 
 struct Callback final : public DDCallback {
   ThreadState *thr;
@@ -36,17 +38,17 @@ struct Callback final : public DDCallback {
   }
 
   StackID Unwind() override { return CurrentStackId(thr, pc); }
-  int UniqueTid() override { return thr->unique_id; }
+  int UniqueTid() override { return thr->tid; }
 };
 
 void DDMutexInit(ThreadState *thr, uptr pc, SyncVar *s) {
   Callback cb(thr, pc);
   ctx->dd->MutexInit(&cb, &s->dd);
-  s->dd.ctx = s->GetId();
+  s->dd.ctx = s->addr;
 }
 
 static void ReportMutexMisuse(ThreadState *thr, uptr pc, ReportType typ,
-    uptr addr, u64 mid) {
+                              uptr addr, StackID creation_stack_id) {
   // In Go, these misuses are either impossible, or detected by std lib,
   // or false positives (e.g. unlock in a different thread).
   if (SANITIZER_GO)
@@ -55,7 +57,7 @@ static void ReportMutexMisuse(ThreadState *thr, uptr pc, ReportType typ,
     return;
   ThreadRegistryLock l(&ctx->thread_registry);
   ScopedReport rep(typ);
-  rep.AddMutex(mid);
+  rep.AddMutex(addr, creation_stack_id);
   VarSizeStackTrace trace;
   ObtainCurrentStack(thr, pc, &trace);
   rep.AddStack(trace, true);
@@ -63,95 +65,93 @@ static void ReportMutexMisuse(ThreadState *thr, uptr pc, ReportType typ,
   OutputReport(thr, rep);
 }
 
+static void RecordMutexLock(ThreadState *thr, uptr pc, uptr addr,
+                            StackID stack_id, bool write) {
+  auto typ = write ? EventType::kLock : EventType::kRLock;
+  // Note: it's important to trace before modifying mutex set
+  // because tracing can switch trace part and we write the current
+  // mutex set in the beginning of each part.
+  // If we do it in the opposite order, we will write already reduced
+  // mutex set in the beginning of the part and then trace unlock again.
+  TraceMutexLock(thr, typ, pc, addr, stack_id);
+  thr->mset.AddAddr(addr, stack_id, write);
+}
+
+static void RecordMutexUnlock(ThreadState *thr, uptr addr) {
+  // See the comment in RecordMutexLock re order of operations.
+  TraceMutexUnlock(thr, addr);
+  thr->mset.DelAddr(addr);
+}
+
 void MutexCreate(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
   DPrintf("#%d: MutexCreate %zx flagz=0x%x\n", thr->tid, addr, flagz);
-  if (!(flagz & MutexFlagLinkerInit) && IsAppMem(addr)) {
-    CHECK(!thr->is_freeing);
-    thr->is_freeing = true;
+  if (!(flagz & MutexFlagLinkerInit) && pc && IsAppMem(addr))
     MemoryAccess(thr, pc, addr, 1, kAccessWrite);
-    thr->is_freeing = false;
-  }
-  SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
-  Lock l(&s->mtx);
+  SlotLocker locker(thr);
+  auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
   s->SetFlags(flagz & MutexCreationFlagMask);
   // Save stack in the case the sync object was created before as atomic.
-  if (!SANITIZER_GO && s->creation_stack_id == 0)
+  if (!SANITIZER_GO && s->creation_stack_id == kInvalidStackID)
     s->creation_stack_id = CurrentStackId(thr, pc);
 }
 
 void MutexDestroy(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
   DPrintf("#%d: MutexDestroy %zx\n", thr->tid, addr);
   bool unlock_locked = false;
-  u64 mid = 0;
-  u64 last_lock = 0;
+  StackID creation_stack_id;
+  FastState last_lock;
   {
-    SyncVar *s = ctx->metamap.GetSyncIfExists(addr);
-    if (s == 0)
+    auto s = ctx->metamap.GetSyncIfExists(addr);
+    if (!s)
       return;
-    Lock l(&s->mtx);
-    if ((flagz & MutexFlagLinkerInit) || s->IsFlagSet(MutexFlagLinkerInit) ||
-        ((flagz & MutexFlagNotStatic) && !s->IsFlagSet(MutexFlagNotStatic))) {
-      // Destroy is no-op for linker-initialized mutexes.
-      return;
-    }
-    if (common_flags()->detect_deadlocks) {
-      Callback cb(thr, pc);
-      ctx->dd->MutexDestroy(&cb, &s->dd);
-      ctx->dd->MutexInit(&cb, &s->dd);
-    }
-    if (flags()->report_destroy_locked && s->owner_tid != kInvalidTid &&
-        !s->IsFlagSet(MutexFlagBroken)) {
-      s->SetFlags(MutexFlagBroken);
-      unlock_locked = true;
-    }
-    mid = s->GetId();
-    last_lock = s->last_lock;
-    if (!unlock_locked)
-      s->Reset(thr->proc());  // must not reset it before the report is printed
-  }
-  if (unlock_locked && ShouldReport(thr, ReportTypeMutexDestroyLocked)) {
-    ThreadRegistryLock l(&ctx->thread_registry);
-    ScopedReport rep(ReportTypeMutexDestroyLocked);
-    rep.AddMutex(mid);
-    VarSizeStackTrace trace;
-    ObtainCurrentStack(thr, pc, &trace);
-    rep.AddStack(trace, true);
-    FastState last(last_lock);
-    RestoreStack(last.tid(), last.epoch(), &trace, 0);
-    rep.AddStack(trace, true);
-    rep.AddLocation(addr, 1);
-    OutputReport(thr, rep);
-
-    SyncVar *s = ctx->metamap.GetSyncIfExists(addr);
-    if (s != 0) {
-      Lock l(&s->mtx);
-      s->Reset(thr->proc());
+    SlotLocker locker(thr);
+    {
+      Lock lock(&s->mtx);
+      creation_stack_id = s->creation_stack_id;
+      last_lock = s->last_lock;
+      if ((flagz & MutexFlagLinkerInit) || s->IsFlagSet(MutexFlagLinkerInit) ||
+          ((flagz & MutexFlagNotStatic) && !s->IsFlagSet(MutexFlagNotStatic))) {
+        // Destroy is no-op for linker-initialized mutexes.
+        return;
+      }
+      if (common_flags()->detect_deadlocks) {
+        Callback cb(thr, pc);
+        ctx->dd->MutexDestroy(&cb, &s->dd);
+        ctx->dd->MutexInit(&cb, &s->dd);
+      }
+      if (flags()->report_destroy_locked && s->owner_tid != kInvalidTid &&
+          !s->IsFlagSet(MutexFlagBroken)) {
+        s->SetFlags(MutexFlagBroken);
+        unlock_locked = true;
+      }
+      s->Reset();
     }
+    // Imitate a memory write to catch unlock-destroy races.
+    if (pc && IsAppMem(addr))
+      MemoryAccess(thr, pc, addr, 1, kAccessWrite | kAccessFree);
   }
-  thr->mset.Remove(mid);
-  // Imitate a memory write to catch unlock-destroy races.
-  // Do this outside of sync mutex, because it can report a race which locks
-  // sync mutexes.
-  if (IsAppMem(addr))
-    MemoryAccess(thr, pc, addr, 1, kAccessWrite | kAccessFree);
+  if (unlock_locked && ShouldReport(thr, ReportTypeMutexDestroyLocked))
+    ReportDestroyLocked(thr, pc, addr, last_lock, creation_stack_id);
+  thr->mset.DelAddr(addr, true);
   // s will be destroyed and freed in MetaMap::FreeBlock.
 }
 
 void MutexPreLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
   DPrintf("#%d: MutexPreLock %zx flagz=0x%x\n", thr->tid, addr, flagz);
-  if (!(flagz & MutexFlagTryLock) && common_flags()->detect_deadlocks) {
-    SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
-    {
-      ReadLock l(&s->mtx);
-      s->UpdateFlags(flagz);
-      if (s->owner_tid != thr->tid) {
-        Callback cb(thr, pc);
-        ctx->dd->MutexBeforeLock(&cb, &s->dd, true);
-      }
-    }
-    Callback cb(thr, pc);
-    ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
+  if (flagz & MutexFlagTryLock)
+    return;
+  if (!common_flags()->detect_deadlocks)
+    return;
+  Callback cb(thr, pc);
+  {
+    SlotLocker locker(thr);
+    auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
+    ReadLock lock(&s->mtx);
+    s->UpdateFlags(flagz);
+    if (s->owner_tid != thr->tid)
+      ctx->dd->MutexBeforeLock(&cb, &s->dd, true);
   }
+  ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
 }
 
 void MutexPostLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz, int rec) {
@@ -161,48 +161,51 @@ void MutexPostLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz, int rec) {
     CHECK_GT(rec, 0);
   else
     rec = 1;
-  if (IsAppMem(addr))
+  if (pc && IsAppMem(addr))
     MemoryAccess(thr, pc, addr, 1, kAccessRead | kAccessAtomic);
-  u64 mid = 0;
+  bool report_double_lock = false;
   bool pre_lock = false;
   bool first = false;
-  bool report_double_lock = false;
+  StackID creation_stack_id = kInvalidStackID;
   {
-    SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
-    Lock l(&s->mtx);
-    s->UpdateFlags(flagz);
-    thr->fast_state.IncrementEpoch();
-    TraceAddEvent(thr, thr->fast_state, EventTypeLock, s->GetId());
-    if (s->owner_tid == kInvalidTid) {
-      CHECK_EQ(s->recursion, 0);
-      s->owner_tid = thr->tid;
-      s->last_lock = thr->fast_state.raw();
-    } else if (s->owner_tid == thr->tid) {
-      CHECK_GT(s->recursion, 0);
-    } else if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
-      s->SetFlags(MutexFlagBroken);
-      report_double_lock = true;
-    }
-    first = s->recursion == 0;
-    s->recursion += rec;
-    if (first) {
-      AcquireImpl(thr, pc, &s->clock);
-      AcquireImpl(thr, pc, &s->read_clock);
-    } else if (!s->IsFlagSet(MutexFlagWriteReentrant)) {
-    }
-    thr->mset.Add(s->GetId(), true, thr->fast_state.epoch());
-    if (first && common_flags()->detect_deadlocks) {
-      pre_lock =
-          (flagz & MutexFlagDoPreLockOnPostLock) && !(flagz & MutexFlagTryLock);
-      Callback cb(thr, pc);
-      if (pre_lock)
-        ctx->dd->MutexBeforeLock(&cb, &s->dd, true);
-      ctx->dd->MutexAfterLock(&cb, &s->dd, true, flagz & MutexFlagTryLock);
+    SlotLocker locker(thr);
+    auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
+    creation_stack_id = s->creation_stack_id;
+    RecordMutexLock(thr, pc, addr, creation_stack_id, true);
+    {
+      Lock lock(&s->mtx);
+      first = s->recursion == 0;
+      s->UpdateFlags(flagz);
+      if (s->owner_tid == kInvalidTid) {
+        CHECK_EQ(s->recursion, 0);
+        s->owner_tid = thr->tid;
+        s->last_lock = thr->fast_state;
+      } else if (s->owner_tid == thr->tid) {
+        CHECK_GT(s->recursion, 0);
+      } else if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
+        s->SetFlags(MutexFlagBroken);
+        report_double_lock = true;
+      }
+      s->recursion += rec;
+      if (first) {
+        if (!thr->ignore_sync) {
+          thr->clock.Acquire(s->clock);
+          thr->clock.Acquire(s->read_clock);
+        }
+      }
+      if (first && common_flags()->detect_deadlocks) {
+        pre_lock = (flagz & MutexFlagDoPreLockOnPostLock) &&
+                   !(flagz & MutexFlagTryLock);
+        Callback cb(thr, pc);
+        if (pre_lock)
+          ctx->dd->MutexBeforeLock(&cb, &s->dd, true);
+        ctx->dd->MutexAfterLock(&cb, &s->dd, true, flagz & MutexFlagTryLock);
+      }
     }
-    mid = s->GetId();
   }
   if (report_double_lock)
-    ReportMutexMisuse(thr, pc, ReportTypeMutexDoubleLock, addr, mid);
+    ReportMutexMisuse(thr, pc, ReportTypeMutexDoubleLock, addr,
+                      creation_stack_id);
   if (first && pre_lock && common_flags()->detect_deadlocks) {
     Callback cb(thr, pc);
     ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
@@ -211,40 +214,47 @@ void MutexPostLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz, int rec) {
 
 int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
   DPrintf("#%d: MutexUnlock %zx flagz=0x%x\n", thr->tid, addr, flagz);
-  if (IsAppMem(addr))
+  if (pc && IsAppMem(addr))
     MemoryAccess(thr, pc, addr, 1, kAccessRead | kAccessAtomic);
-  u64 mid = 0;
+  StackID creation_stack_id;
+  RecordMutexUnlock(thr, addr);
   bool report_bad_unlock = false;
   int rec = 0;
   {
-    SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
-    Lock l(&s->mtx);
-    thr->fast_state.IncrementEpoch();
-    TraceAddEvent(thr, thr->fast_state, EventTypeUnlock, s->GetId());
-    if (!SANITIZER_GO && (s->recursion == 0 || s->owner_tid != thr->tid)) {
-      if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
-        s->SetFlags(MutexFlagBroken);
-        report_bad_unlock = true;
-      }
-    } else {
-      rec = (flagz & MutexFlagRecursiveUnlock) ? s->recursion : 1;
-      s->recursion -= rec;
-      if (s->recursion == 0) {
-        s->owner_tid = kInvalidTid;
-        ReleaseStoreImpl(thr, pc, &s->clock);
+    SlotLocker locker(thr);
+    auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
+    bool released = false;
+    {
+      Lock lock(&s->mtx);
+      creation_stack_id = s->creation_stack_id;
+      if (!SANITIZER_GO && (s->recursion == 0 || s->owner_tid != thr->tid)) {
+        if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
+          s->SetFlags(MutexFlagBroken);
+          report_bad_unlock = true;
+        }
       } else {
+        rec = (flagz & MutexFlagRecursiveUnlock) ? s->recursion : 1;
+        s->recursion -= rec;
+        if (s->recursion == 0) {
+          s->owner_tid = kInvalidTid;
+          if (!thr->ignore_sync) {
+            thr->clock.ReleaseStore(&s->clock);
+            released = true;
+          }
+        }
+      }
+      if (common_flags()->detect_deadlocks && s->recursion == 0 &&
+          !report_bad_unlock) {
+        Callback cb(thr, pc);
+        ctx->dd->MutexBeforeUnlock(&cb, &s->dd, true);
       }
     }
-    thr->mset.Del(s->GetId(), true);
-    if (common_flags()->detect_deadlocks && s->recursion == 0 &&
-        !report_bad_unlock) {
-      Callback cb(thr, pc);
-      ctx->dd->MutexBeforeUnlock(&cb, &s->dd, true);
-    }
-    mid = s->GetId();
+    if (released)
+      IncrementEpoch(thr);
   }
   if (report_bad_unlock)
-    ReportMutexMisuse(thr, pc, ReportTypeMutexBadUnlock, addr, mid);
+    ReportMutexMisuse(thr, pc, ReportTypeMutexBadUnlock, addr,
+                      creation_stack_id);
   if (common_flags()->detect_deadlocks && !report_bad_unlock) {
     Callback cb(thr, pc);
     ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
@@ -254,53 +264,56 @@ int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
 
 void MutexPreReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
   DPrintf("#%d: MutexPreReadLock %zx flagz=0x%x\n", thr->tid, addr, flagz);
-  if (!(flagz & MutexFlagTryLock) && common_flags()->detect_deadlocks) {
-    {
-      SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
-      ReadLock l(&s->mtx);
-      s->UpdateFlags(flagz);
-      Callback cb(thr, pc);
-      ctx->dd->MutexBeforeLock(&cb, &s->dd, false);
-    }
-    Callback cb(thr, pc);
-    ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
+  if ((flagz & MutexFlagTryLock) || !common_flags()->detect_deadlocks)
+    return;
+  Callback cb(thr, pc);
+  {
+    SlotLocker locker(thr);
+    auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
+    ReadLock lock(&s->mtx);
+    s->UpdateFlags(flagz);
+    ctx->dd->MutexBeforeLock(&cb, &s->dd, false);
   }
+  ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
 }
 
 void MutexPostReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
   DPrintf("#%d: MutexPostReadLock %zx flagz=0x%x\n", thr->tid, addr, flagz);
-  if (IsAppMem(addr))
+  if (pc && IsAppMem(addr))
     MemoryAccess(thr, pc, addr, 1, kAccessRead | kAccessAtomic);
-  u64 mid = 0;
   bool report_bad_lock = false;
   bool pre_lock = false;
+  StackID creation_stack_id = kInvalidStackID;
   {
-    SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
-    ReadLock l(&s->mtx);
-    s->UpdateFlags(flagz);
-    thr->fast_state.IncrementEpoch();
-    TraceAddEvent(thr, thr->fast_state, EventTypeRLock, s->GetId());
-    if (s->owner_tid != kInvalidTid) {
-      if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
-        s->SetFlags(MutexFlagBroken);
-        report_bad_lock = true;
+    SlotLocker locker(thr);
+    auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
+    creation_stack_id = s->creation_stack_id;
+    RecordMutexLock(thr, pc, addr, creation_stack_id, false);
+    {
+      ReadLock lock(&s->mtx);
+      s->UpdateFlags(flagz);
+      if (s->owner_tid != kInvalidTid) {
+        if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
+          s->SetFlags(MutexFlagBroken);
+          report_bad_lock = true;
+        }
+      }
+      if (!thr->ignore_sync)
+        thr->clock.Acquire(s->clock);
+      s->last_lock = thr->fast_state;
+      if (common_flags()->detect_deadlocks) {
+        pre_lock = (flagz & MutexFlagDoPreLockOnPostLock) &&
+                   !(flagz & MutexFlagTryLock);
+        Callback cb(thr, pc);
+        if (pre_lock)
+          ctx->dd->MutexBeforeLock(&cb, &s->dd, false);
+        ctx->dd->MutexAfterLock(&cb, &s->dd, false, flagz & MutexFlagTryLock);
       }
     }
-    AcquireImpl(thr, pc, &s->clock);
-    s->last_lock = thr->fast_state.raw();
-    thr->mset.Add(s->GetId(), false, thr->fast_state.epoch());
-    if (common_flags()->detect_deadlocks) {
-      pre_lock =
-          (flagz & MutexFlagDoPreLockOnPostLock) && !(flagz & MutexFlagTryLock);
-      Callback cb(thr, pc);
-      if (pre_lock)
-        ctx->dd->MutexBeforeLock(&cb, &s->dd, false);
-      ctx->dd->MutexAfterLock(&cb, &s->dd, false, flagz & MutexFlagTryLock);
-    }
-    mid = s->GetId();
   }
   if (report_bad_lock)
-    ReportMutexMisuse(thr, pc, ReportTypeMutexBadReadLock, addr, mid);
+    ReportMutexMisuse(thr, pc, ReportTypeMutexBadReadLock, addr,
+                      creation_stack_id);
   if (pre_lock  && common_flags()->detect_deadlocks) {
     Callback cb(thr, pc);
     ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
@@ -309,31 +322,39 @@ void MutexPostReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
 
 void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr) {
   DPrintf("#%d: MutexReadUnlock %zx\n", thr->tid, addr);
-  if (IsAppMem(addr))
+  if (pc && IsAppMem(addr))
     MemoryAccess(thr, pc, addr, 1, kAccessRead | kAccessAtomic);
-  u64 mid = 0;
+  RecordMutexUnlock(thr, addr);
+  StackID creation_stack_id;
   bool report_bad_unlock = false;
   {
-    SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
-    Lock l(&s->mtx);
-    thr->fast_state.IncrementEpoch();
-    TraceAddEvent(thr, thr->fast_state, EventTypeRUnlock, s->GetId());
-    if (s->owner_tid != kInvalidTid) {
-      if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
-        s->SetFlags(MutexFlagBroken);
-        report_bad_unlock = true;
+    SlotLocker locker(thr);
+    auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
+    bool released = false;
+    {
+      Lock lock(&s->mtx);
+      creation_stack_id = s->creation_stack_id;
+      if (s->owner_tid != kInvalidTid) {
+        if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
+          s->SetFlags(MutexFlagBroken);
+          report_bad_unlock = true;
+        }
+      }
+      if (!thr->ignore_sync) {
+        thr->clock.Release(&s->read_clock);
+        released = true;
+      }
+      if (common_flags()->detect_deadlocks && s->recursion == 0) {
+        Callback cb(thr, pc);
+        ctx->dd->MutexBeforeUnlock(&cb, &s->dd, false);
       }
     }
-    ReleaseImpl(thr, pc, &s->read_clock);
-    if (common_flags()->detect_deadlocks && s->recursion == 0) {
-      Callback cb(thr, pc);
-      ctx->dd->MutexBeforeUnlock(&cb, &s->dd, false);
-    }
-    mid = s->GetId();
+    if (released)
+      IncrementEpoch(thr);
   }
-  thr->mset.Del(mid, false);
   if (report_bad_unlock)
-    ReportMutexMisuse(thr, pc, ReportTypeMutexBadReadUnlock, addr, mid);
+    ReportMutexMisuse(thr, pc, ReportTypeMutexBadReadUnlock, addr,
+                      creation_stack_id);
   if (common_flags()->detect_deadlocks) {
     Callback cb(thr, pc);
     ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
@@ -342,44 +363,52 @@ void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr) {
 
 void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr) {
   DPrintf("#%d: MutexReadOrWriteUnlock %zx\n", thr->tid, addr);
-  if (IsAppMem(addr))
+  if (pc && IsAppMem(addr))
     MemoryAccess(thr, pc, addr, 1, kAccessRead | kAccessAtomic);
-  u64 mid = 0;
+  RecordMutexUnlock(thr, addr);
+  StackID creation_stack_id;
   bool report_bad_unlock = false;
+  bool write = true;
   {
-    SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
-    Lock l(&s->mtx);
-    bool write = true;
-    if (s->owner_tid == kInvalidTid) {
-      // Seems to be read unlock.
-      write = false;
-      thr->fast_state.IncrementEpoch();
-      TraceAddEvent(thr, thr->fast_state, EventTypeRUnlock, s->GetId());
-      ReleaseImpl(thr, pc, &s->read_clock);
-    } else if (s->owner_tid == thr->tid) {
-      // Seems to be write unlock.
-      thr->fast_state.IncrementEpoch();
-      TraceAddEvent(thr, thr->fast_state, EventTypeUnlock, s->GetId());
-      CHECK_GT(s->recursion, 0);
-      s->recursion--;
-      if (s->recursion == 0) {
-        s->owner_tid = kInvalidTid;
-        ReleaseStoreImpl(thr, pc, &s->clock);
-      } else {
+    SlotLocker locker(thr);
+    auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
+    bool released = false;
+    {
+      Lock lock(&s->mtx);
+      creation_stack_id = s->creation_stack_id;
+      if (s->owner_tid == kInvalidTid) {
+        // Seems to be read unlock.
+        write = false;
+        if (!thr->ignore_sync) {
+          thr->clock.Release(&s->read_clock);
+          released = true;
+        }
+      } else if (s->owner_tid == thr->tid) {
+        // Seems to be write unlock.
+        CHECK_GT(s->recursion, 0);
+        s->recursion--;
+        if (s->recursion == 0) {
+          s->owner_tid = kInvalidTid;
+          if (!thr->ignore_sync) {
+            thr->clock.ReleaseStore(&s->clock);
+            released = true;
+          }
+        }
+      } else if (!s->IsFlagSet(MutexFlagBroken)) {
+        s->SetFlags(MutexFlagBroken);
+        report_bad_unlock = true;
+      }
+      if (common_flags()->detect_deadlocks && s->recursion == 0) {
+        Callback cb(thr, pc);
+        ctx->dd->MutexBeforeUnlock(&cb, &s->dd, write);
       }
-    } else if (!s->IsFlagSet(MutexFlagBroken)) {
-      s->SetFlags(MutexFlagBroken);
-      report_bad_unlock = true;
-    }
-    thr->mset.Del(s->GetId(), write);
-    if (common_flags()->detect_deadlocks && s->recursion == 0) {
-      Callback cb(thr, pc);
-      ctx->dd->MutexBeforeUnlock(&cb, &s->dd, write);
     }
-    mid = s->GetId();
+    if (released)
+      IncrementEpoch(thr);
   }
   if (report_bad_unlock)
-    ReportMutexMisuse(thr, pc, ReportTypeMutexBadUnlock, addr, mid);
+    ReportMutexMisuse(thr, pc, ReportTypeMutexBadUnlock, addr,
+                      creation_stack_id);
   if (common_flags()->detect_deadlocks) {
     Callback cb(thr, pc);
     ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
@@ -388,143 +417,112 @@ void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr) {
 
 void MutexRepair(ThreadState *thr, uptr pc, uptr addr) {
   DPrintf("#%d: MutexRepair %zx\n", thr->tid, addr);
-  SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
-  Lock l(&s->mtx);
+  SlotLocker locker(thr);
+  auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
+  Lock lock(&s->mtx);
   s->owner_tid = kInvalidTid;
   s->recursion = 0;
 }
 
 void MutexInvalidAccess(ThreadState *thr, uptr pc, uptr addr) {
   DPrintf("#%d: MutexInvalidAccess %zx\n", thr->tid, addr);
-  SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
-  ReportMutexMisuse(thr, pc, ReportTypeMutexInvalidAccess, addr, s->GetId());
+  StackID creation_stack_id = kInvalidStackID;
+  {
+    SlotLocker locker(thr);
+    auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
+    if (s)
+      creation_stack_id = s->creation_stack_id;
+  }
+  ReportMutexMisuse(thr, pc, ReportTypeMutexInvalidAccess, addr,
+                    creation_stack_id);
 }
 
 void Acquire(ThreadState *thr, uptr pc, uptr addr) {
   DPrintf("#%d: Acquire %zx\n", thr->tid, addr);
   if (thr->ignore_sync)
     return;
-  SyncVar *s = ctx->metamap.GetSyncIfExists(addr);
+  auto s = ctx->metamap.GetSyncIfExists(addr);
   if (!s)
     return;
-  ReadLock l(&s->mtx);
-  AcquireImpl(thr, pc, &s->clock);
-}
-
-static void UpdateClockCallback(ThreadContextBase *tctx_base, void *arg) {
-  ThreadState *thr = reinterpret_cast<ThreadState*>(arg);
-  ThreadContext *tctx = static_cast<ThreadContext*>(tctx_base);
-  u64 epoch = tctx->epoch1;
-  if (tctx->status == ThreadStatusRunning) {
-    epoch = tctx->thr->fast_state.epoch();
-    tctx->thr->clock.NoteGlobalAcquire(epoch);
-  }
-  thr->clock.set(&thr->proc()->clock_cache, tctx->tid, epoch);
+  SlotLocker locker(thr);
+  if (!s->clock)
+    return;
+  ReadLock lock(&s->mtx);
+  thr->clock.Acquire(s->clock);
 }
 
 void AcquireGlobal(ThreadState *thr) {
   DPrintf("#%d: AcquireGlobal\n", thr->tid);
   if (thr->ignore_sync)
     return;
-  ThreadRegistryLock l(&ctx->thread_registry);
-  ctx->thread_registry.RunCallbackForEachThreadLocked(UpdateClockCallback, thr);
-}
-
-void ReleaseStoreAcquire(ThreadState *thr, uptr pc, uptr addr) {
-  DPrintf("#%d: ReleaseStoreAcquire %zx\n", thr->tid, addr);
-  if (thr->ignore_sync)
-    return;
-  SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, false);
-  Lock l(&s->mtx);
-  thr->fast_state.IncrementEpoch();
-  // Can't increment epoch w/o writing to the trace as well.
-  TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
-  ReleaseStoreAcquireImpl(thr, pc, &s->clock);
+  SlotLocker locker(thr);
+  for (auto &slot : ctx->slots) thr->clock.Set(slot.sid, slot.epoch());
 }
 
 void Release(ThreadState *thr, uptr pc, uptr addr) {
   DPrintf("#%d: Release %zx\n", thr->tid, addr);
   if (thr->ignore_sync)
     return;
-  SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, false);
-  Lock l(&s->mtx);
-  thr->fast_state.IncrementEpoch();
-  // Can't increment epoch w/o writing to the trace as well.
-  TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
-  ReleaseImpl(thr, pc, &s->clock);
+  SlotLocker locker(thr);
+  {
+    auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, false);
+    Lock lock(&s->mtx);
+    thr->clock.Release(&s->clock);
+  }
+  IncrementEpoch(thr);
 }
 
 void ReleaseStore(ThreadState *thr, uptr pc, uptr addr) {
   DPrintf("#%d: ReleaseStore %zx\n", thr->tid, addr);
   if (thr->ignore_sync)
     return;
-  SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, false);
-  Lock l(&s->mtx);
-  thr->fast_state.IncrementEpoch();
-  // Can't increment epoch w/o writing to the trace as well.
-  TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
-  ReleaseStoreImpl(thr, pc, &s->clock);
-}
-
-#if !SANITIZER_GO
-static void UpdateSleepClockCallback(ThreadContextBase *tctx_base, void *arg) {
-  ThreadState *thr = reinterpret_cast<ThreadState*>(arg);
-  ThreadContext *tctx = static_cast<ThreadContext*>(tctx_base);
-  u64 epoch = tctx->epoch1;
-  if (tctx->status == ThreadStatusRunning)
-    epoch = tctx->thr->fast_state.epoch();
-  thr->last_sleep_clock.set(&thr->proc()->clock_cache, tctx->tid, epoch);
-}
-
-void AfterSleep(ThreadState *thr, uptr pc) {
-  DPrintf("#%d: AfterSleep\n", thr->tid);
-  if (thr->ignore_sync)
-    return;
-  thr->last_sleep_stack_id = CurrentStackId(thr, pc);
-  ThreadRegistryLock l(&ctx->thread_registry);
-  ctx->thread_registry.RunCallbackForEachThreadLocked(UpdateSleepClockCallback,
-                                                      thr);
-}
-#endif
-
-void AcquireImpl(ThreadState *thr, uptr pc, SyncClock *c) {
-  if (thr->ignore_sync)
-    return;
-  thr->clock.set(thr->fast_state.epoch());
-  thr->clock.acquire(&thr->proc()->clock_cache, c);
-}
-
-void ReleaseStoreAcquireImpl(ThreadState *thr, uptr pc, SyncClock *c) {
-  if (thr->ignore_sync)
-    return;
-  thr->clock.set(thr->fast_state.epoch());
-  thr->fast_synch_epoch = thr->fast_state.epoch();
-  thr->clock.releaseStoreAcquire(&thr->proc()->clock_cache, c);
+  SlotLocker locker(thr);
+  {
+    auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, false);
+    Lock lock(&s->mtx);
+    thr->clock.ReleaseStore(&s->clock);
+  }
+  IncrementEpoch(thr);
 }
 
-void ReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c) {
+void ReleaseStoreAcquire(ThreadState *thr, uptr pc, uptr addr) {
+  DPrintf("#%d: ReleaseStoreAcquire %zx\n", thr->tid, addr);
   if (thr->ignore_sync)
     return;
-  thr->clock.set(thr->fast_state.epoch());
-  thr->fast_synch_epoch = thr->fast_state.epoch();
-  thr->clock.release(&thr->proc()->clock_cache, c);
+  SlotLocker locker(thr);
+  {
+    auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, false);
+    Lock lock(&s->mtx);
+    thr->clock.ReleaseStoreAcquire(&s->clock);
+  }
+  IncrementEpoch(thr);
 }
 
-void ReleaseStoreImpl(ThreadState *thr, uptr pc, SyncClock *c) {
-  if (thr->ignore_sync)
-    return;
-  thr->clock.set(thr->fast_state.epoch());
-  thr->fast_synch_epoch = thr->fast_state.epoch();
-  thr->clock.ReleaseStore(&thr->proc()->clock_cache, c);
+void IncrementEpoch(ThreadState *thr) {
+  DCHECK(!thr->ignore_sync);
+  DCHECK(thr->slot_locked);
+  Epoch epoch = EpochInc(thr->fast_state.epoch());
+  if (!EpochOverflow(epoch)) {
+    Sid sid = thr->fast_state.sid();
+    thr->clock.Set(sid, epoch);
+    thr->fast_state.SetEpoch(epoch);
+    thr->slot->SetEpoch(epoch);
+    TraceTime(thr);
+  }
 }
 
-void AcquireReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c) {
+#if !SANITIZER_GO
+void AfterSleep(ThreadState *thr, uptr pc) {
+  DPrintf("#%d: AfterSleep\n", thr->tid);
   if (thr->ignore_sync)
     return;
-  thr->clock.set(thr->fast_state.epoch());
-  thr->fast_synch_epoch = thr->fast_state.epoch();
-  thr->clock.acq_rel(&thr->proc()->clock_cache, c);
+  thr->last_sleep_stack_id = CurrentStackId(thr, pc);
+  thr->last_sleep_clock.Reset();
+  SlotLocker locker(thr);
+  for (auto &slot : ctx->slots)
+    thr->last_sleep_clock.Set(slot.sid, slot.epoch());
 }
+#endif
 
 void ReportDeadlock(ThreadState *thr, uptr pc, DDReport *r) {
   if (r == 0 || !ShouldReport(thr, ReportTypeDeadlock))
@@ -532,7 +530,7 @@ void ReportDeadlock(ThreadState *thr, uptr pc, DDReport *r) {
   ThreadRegistryLock l(&ctx->thread_registry);
   ScopedReport rep(ReportTypeDeadlock);
   for (int i = 0; i < r->n; i++) {
-    rep.AddMutex(r->loop[i].mtx_ctx0);
+    rep.AddMutex(r->loop[i].mtx_ctx0, r->loop[i].stk[0]);
     rep.AddUniqueTid((int)r->loop[i].thr_ctx);
     rep.AddThread((int)r->loop[i].thr_ctx);
   }
@@ -540,7 +538,7 @@ void ReportDeadlock(ThreadState *thr, uptr pc, DDReport *r) {
   for (int i = 0; i < r->n; i++) {
     for (int j = 0; j < (flags()->second_deadlock_stack ? 2 : 1); j++) {
       u32 stk = r->loop[i].stk[j];
-      if (stk && stk != 0xffffffff) {
+      if (stk && stk != kInvalidStackID) {
         rep.AddStack(StackDepotGet(stk), true);
       } else {
         // Sometimes we fail to extract the stack trace (FIXME: investigate),
@@ -552,4 +550,28 @@ void ReportDeadlock(ThreadState *thr, uptr pc, DDReport *r) {
   OutputReport(thr, rep);
 }
 
+void ReportDestroyLocked(ThreadState *thr, uptr pc, uptr addr,
+                         FastState last_lock, StackID creation_stack_id) {
+  // We need to lock the slot during RestoreStack because it protects
+  // the slot journal.
+  Lock slot_lock(&ctx->slots[static_cast<uptr>(last_lock.sid())].mtx);
+  ThreadRegistryLock l0(&ctx->thread_registry);
+  Lock slots_lock(&ctx->slot_mtx);
+  ScopedReport rep(ReportTypeMutexDestroyLocked);
+  rep.AddMutex(addr, creation_stack_id);
+  VarSizeStackTrace trace;
+  ObtainCurrentStack(thr, pc, &trace);
+  rep.AddStack(trace, true);
+
+  Tid tid;
+  DynamicMutexSet mset;
+  uptr tag;
+  if (!RestoreStack(EventType::kLock, last_lock.sid(), last_lock.epoch(), addr,
+                    0, kAccessWrite, &tid, &trace, mset, &tag))
+    return;
+  rep.AddStack(trace, true);
+  rep.AddLocation(addr, 1);
+  OutputReport(thr, rep);
+}
+
 }  // namespace __tsan
diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl_proc.cpp b/compiler-rt/lib/tsan/rtl/tsan_rtl_proc.cpp
index def61cca14d5..5acc3967208e 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_rtl_proc.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_rtl_proc.cpp
@@ -35,7 +35,6 @@ void ProcDestroy(Processor *proc) {
 #if !SANITIZER_GO
   AllocatorProcFinish(proc);
 #endif
-  ctx->clock_alloc.FlushCache(&proc->clock_cache);
   ctx->metamap.OnProcIdle(proc);
   if (common_flags()->detect_deadlocks)
      ctx->dd->DestroyPhysicalThread(proc->dd_pt);
diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cpp b/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cpp
index f332a6a8d1d8..58949ead07b3 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cpp
@@ -175,22 +175,26 @@ void ScopedReportBase::AddStack(StackTrace stack, bool suppressable) {
 }
 
 void ScopedReportBase::AddMemoryAccess(uptr addr, uptr external_tag, Shadow s,
-                                       StackTrace stack, const MutexSet *mset) {
+                                       Tid tid, StackTrace stack,
+                                       const MutexSet *mset) {
+  uptr addr0, size;
+  AccessType typ;
+  s.GetAccess(&addr0, &size, &typ);
   auto *mop = New<ReportMop>();
   rep_->mops.PushBack(mop);
-  mop->tid = s.tid();
-  mop->addr = addr + s.addr0();
-  mop->size = s.size();
-  mop->write = s.IsWrite();
-  mop->atomic = s.IsAtomic();
+  mop->tid = tid;
+  mop->addr = addr + addr0;
+  mop->size = size;
+  mop->write = !(typ & kAccessRead);
+  mop->atomic = typ & kAccessAtomic;
   mop->stack = SymbolizeStack(stack);
   mop->external_tag = external_tag;
   if (mop->stack)
     mop->stack->suppressable = true;
   for (uptr i = 0; i < mset->Size(); i++) {
     MutexSet::Desc d = mset->Get(i);
-    u64 mid = this->AddMutex(d.id);
-    ReportMopMutex mtx = {mid, d.write};
+    int id = this->AddMutex(d.addr, d.stack_id);
+    ReportMopMutex mtx = {id, d.write};
     mop->mset.PushBack(mtx);
   }
 }
@@ -219,18 +223,6 @@ void ScopedReportBase::AddThread(const ThreadContext *tctx, bool suppressable) {
 }
 
 #if !SANITIZER_GO
-static bool FindThreadByUidLockedCallback(ThreadContextBase *tctx, void *arg) {
-  int unique_id = *(int *)arg;
-  return tctx->unique_id == (u32)unique_id;
-}
-
-static ThreadContext *FindThreadByUidLocked(Tid unique_id) {
-  ctx->thread_registry.CheckLocked();
-  return static_cast<ThreadContext *>(
-      ctx->thread_registry.FindThreadContextLocked(
-          FindThreadByUidLockedCallback, &unique_id));
-}
-
 static ThreadContext *FindThreadByTidLocked(Tid tid) {
   ctx->thread_registry.CheckLocked();
   return static_cast<ThreadContext *>(
@@ -262,55 +254,24 @@ ThreadContext *IsThreadStackOrTls(uptr addr, bool *is_stack) {
 }
 #endif
 
-void ScopedReportBase::AddThread(Tid unique_tid, bool suppressable) {
+void ScopedReportBase::AddThread(Tid tid, bool suppressable) {
 #if !SANITIZER_GO
-  if (const ThreadContext *tctx = FindThreadByUidLocked(unique_tid))
+  if (const ThreadContext *tctx = FindThreadByTidLocked(tid))
     AddThread(tctx, suppressable);
 #endif
 }
 
-void ScopedReportBase::AddMutex(const SyncVar *s) {
+int ScopedReportBase::AddMutex(uptr addr, StackID creation_stack_id) {
   for (uptr i = 0; i < rep_->mutexes.Size(); i++) {
-    if (rep_->mutexes[i]->id == s->uid)
-      return;
+    if (rep_->mutexes[i]->addr == addr)
+      return rep_->mutexes[i]->id;
   }
   auto *rm = New<ReportMutex>();
   rep_->mutexes.PushBack(rm);
-  rm->id = s->uid;
-  rm->addr = s->addr;
-  rm->destroyed = false;
-  rm->stack = SymbolizeStackId(s->creation_stack_id);
-}
-
-u64 ScopedReportBase::AddMutex(u64 id) {
-  u64 uid = 0;
-  u64 mid = id;
-  uptr addr = SyncVar::SplitId(id, &uid);
-  SyncVar *s = ctx->metamap.GetSyncIfExists(addr);
-  // Check that the mutex is still alive.
-  // Another mutex can be created at the same address,
-  // so check uid as well.
-  if (s && s->CheckId(uid)) {
-    Lock l(&s->mtx);
-    mid = s->uid;
-    AddMutex(s);
-  } else {
-    AddDeadMutex(id);
-  }
-  return mid;
-}
-
-void ScopedReportBase::AddDeadMutex(u64 id) {
-  for (uptr i = 0; i < rep_->mutexes.Size(); i++) {
-    if (rep_->mutexes[i]->id == id)
-      return;
-  }
-  auto *rm = New<ReportMutex>();
-  rep_->mutexes.PushBack(rm);
-  rm->id = id;
-  rm->addr = 0;
-  rm->destroyed = true;
-  rm->stack = 0;
+  rm->id = rep_->mutexes.Size() - 1;
+  rm->addr = addr;
+  rm->stack = SymbolizeStackId(creation_stack_id);
+  return rm->id;
 }
 
 void ScopedReportBase::AddLocation(uptr addr, uptr size) {
@@ -327,7 +288,7 @@ void ScopedReportBase::AddLocation(uptr addr, uptr size) {
     loc->tid = creat_tid;
     loc->stack = SymbolizeStackId(creat_stack);
     rep_->locs.PushBack(loc);
-    ThreadContext *tctx = FindThreadByUidLocked(creat_tid);
+    ThreadContext *tctx = FindThreadByTidLocked(creat_tid);
     if (tctx)
       AddThread(tctx);
     return;
@@ -343,16 +304,15 @@ void ScopedReportBase::AddLocation(uptr addr, uptr size) {
   if (!b)
     b = JavaHeapBlock(addr, &block_begin);
   if (b != 0) {
-    ThreadContext *tctx = FindThreadByTidLocked(b->tid);
     auto *loc = New<ReportLocation>();
     loc->type = ReportLocationHeap;
     loc->heap_chunk_start = block_begin;
     loc->heap_chunk_size = b->siz;
     loc->external_tag = b->tag;
-    loc->tid = tctx ? tctx->tid : b->tid;
+    loc->tid = b->tid;
     loc->stack = SymbolizeStackId(b->stk);
     rep_->locs.PushBack(loc);
-    if (tctx)
+    if (ThreadContext *tctx = FindThreadByTidLocked(b->tid))
       AddThread(tctx);
     return;
   }
@@ -387,71 +347,6 @@ ScopedReport::ScopedReport(ReportType typ, uptr tag)
 
 ScopedReport::~ScopedReport() {}
 
-void RestoreStack(Tid tid, const u64 epoch, VarSizeStackTrace *stk,
-                  MutexSet *mset, uptr *tag) {
-  // This function restores stack trace and mutex set for the thread/epoch.
-  // It does so by getting stack trace and mutex set at the beginning of
-  // trace part, and then replaying the trace till the given epoch.
-  Trace* trace = ThreadTrace(tid);
-  ReadLock l(&trace->mtx);
-  const int partidx = (epoch / kTracePartSize) % TraceParts();
-  TraceHeader* hdr = &trace->headers[partidx];
-  if (epoch < hdr->epoch0 || epoch >= hdr->epoch0 + kTracePartSize)
-    return;
-  CHECK_EQ(RoundDown(epoch, kTracePartSize), hdr->epoch0);
-  const u64 epoch0 = RoundDown(epoch, TraceSize());
-  const u64 eend = epoch % TraceSize();
-  const u64 ebegin = RoundDown(eend, kTracePartSize);
-  DPrintf("#%d: RestoreStack epoch=%zu ebegin=%zu eend=%zu partidx=%d\n",
-          tid, (uptr)epoch, (uptr)ebegin, (uptr)eend, partidx);
-  Vector<uptr> stack;
-  stack.Resize(hdr->stack0.size + 64);
-  for (uptr i = 0; i < hdr->stack0.size; i++) {
-    stack[i] = hdr->stack0.trace[i];
-    DPrintf2("  #%02zu: pc=%zx\n", i, stack[i]);
-  }
-  if (mset)
-    *mset = hdr->mset0;
-  uptr pos = hdr->stack0.size;
-  Event *events = (Event*)GetThreadTrace(tid);
-  for (uptr i = ebegin; i <= eend; i++) {
-    Event ev = events[i];
-    EventType typ = (EventType)(ev >> kEventPCBits);
-    uptr pc = (uptr)(ev & ((1ull << kEventPCBits) - 1));
-    DPrintf2("  %zu typ=%d pc=%zx\n", i, typ, pc);
-    if (typ == EventTypeMop) {
-      stack[pos] = pc;
-    } else if (typ == EventTypeFuncEnter) {
-      if (stack.Size() < pos + 2)
-        stack.Resize(pos + 2);
-      stack[pos++] = pc;
-    } else if (typ == EventTypeFuncExit) {
-      if (pos > 0)
-        pos--;
-    }
-    if (mset) {
-      if (typ == EventTypeLock) {
-        mset->Add(pc, true, epoch0 + i);
-      } else if (typ == EventTypeUnlock) {
-        mset->Del(pc, true);
-      } else if (typ == EventTypeRLock) {
-        mset->Add(pc, false, epoch0 + i);
-      } else if (typ == EventTypeRUnlock) {
-        mset->Del(pc, false);
-      }
-    }
-    for (uptr j = 0; j <= pos; j++)
-      DPrintf2("      #%zu: %zx\n", j, stack[j]);
-  }
-  if (pos == 0 && stack[0] == 0)
-    return;
-  pos++;
-  stk->Init(&stack[0], pos);
-  ExtractTagFromStack(stk, tag);
-}
-
-namespace v3 {
-
 // Replays the trace up to last_pos position in the last part
 // or up to the provided epoch/sid (whichever is earlier)
 // and calls the provided function f for each event.
@@ -469,6 +364,7 @@ void TraceReplay(Trace *trace, TracePart *last, Event *last_pos, Sid sid,
     Event *end = &part->events[TracePart::kSize - 1];
     if (part == last)
       end = last_pos;
+    f(kFreeSid, kEpochOver, nullptr);  // notify about part start
     for (Event *evp = &part->events[0]; evp < end; evp++) {
       Event *evp0 = evp;
       if (!evp->is_access && !evp->is_func) {
@@ -528,21 +424,36 @@ static constexpr bool IsWithinAccess(uptr addr1, uptr size1, uptr addr2,
   return addr1 >= addr2 && addr1 + size1 <= addr2 + size2;
 }
 
-// Replays the trace of thread tid up to the target event identified
-// by sid/epoch/addr/size/typ and restores and returns stack, mutex set
+// Replays the trace of slot sid up to the target event identified
+// by epoch/addr/size/typ and restores and returns tid, stack, mutex set
 // and tag for that event. If there are multiple such events, it returns
 // the last one. Returns false if the event is not present in the trace.
-bool RestoreStack(Tid tid, EventType type, Sid sid, Epoch epoch, uptr addr,
-                  uptr size, AccessType typ, VarSizeStackTrace *pstk,
+bool RestoreStack(EventType type, Sid sid, Epoch epoch, uptr addr, uptr size,
+                  AccessType typ, Tid *ptid, VarSizeStackTrace *pstk,
                   MutexSet *pmset, uptr *ptag) {
   // This function restores stack trace and mutex set for the thread/epoch.
   // It does so by getting stack trace and mutex set at the beginning of
   // trace part, and then replaying the trace till the given epoch.
-  DPrintf2("RestoreStack: tid=%u sid=%u@%u addr=0x%zx/%zu typ=%x\n", tid,
+  DPrintf2("RestoreStack: sid=%u@%u addr=0x%zx/%zu typ=%x\n",
            static_cast<int>(sid), static_cast<int>(epoch), addr, size,
            static_cast<int>(typ));
   ctx->slot_mtx.CheckLocked();  // needed to prevent trace part recycling
   ctx->thread_registry.CheckLocked();
+  TidSlot *slot = &ctx->slots[static_cast<uptr>(sid)];
+  Tid tid = kInvalidTid;
+  // Need to lock the slot mutex as it protects slot->journal.
+  slot->mtx.CheckLocked();
+  for (uptr i = 0; i < slot->journal.Size(); i++) {
+    DPrintf2("  journal: epoch=%d tid=%d\n",
+             static_cast<int>(slot->journal[i].epoch), slot->journal[i].tid);
+    if (i == slot->journal.Size() - 1 || slot->journal[i + 1].epoch > epoch) {
+      tid = slot->journal[i].tid;
+      break;
+    }
+  }
+  if (tid == kInvalidTid)
+    return false;
+  *ptid = tid;
   ThreadContext *tctx =
       static_cast<ThreadContext *>(ctx->thread_registry.GetThreadLocked(tid));
   Trace *trace = &tctx->trace;
@@ -553,8 +464,10 @@ bool RestoreStack(Tid tid, EventType type, Sid sid, Epoch epoch, uptr addr,
   {
     Lock lock(&trace->mtx);
     first_part = trace->parts.Front();
-    if (!first_part)
+    if (!first_part) {
+      DPrintf2("RestoreStack: tid=%d trace=%p no trace parts\n", tid, trace);
       return false;
+    }
     last_part = trace->parts.Back();
     last_pos = trace->final_pos;
     if (tctx->thr)
@@ -567,9 +480,18 @@ bool RestoreStack(Tid tid, EventType type, Sid sid, Epoch epoch, uptr addr,
   bool is_read = typ & kAccessRead;
   bool is_atomic = typ & kAccessAtomic;
   bool is_free = typ & kAccessFree;
+  DPrintf2("RestoreStack: tid=%d parts=[%p-%p] last_pos=%p\n", tid,
+           trace->parts.Front(), last_part, last_pos);
   TraceReplay(
       trace, last_part, last_pos, sid, epoch,
       [&](Sid ev_sid, Epoch ev_epoch, Event *evp) {
+        if (evp == nullptr) {
+          // Each trace part is self-consistent, so we reset state.
+          stack.Resize(0);
+          mset->Reset();
+          prev_pc = 0;
+          return;
+        }
         bool match = ev_sid == sid && ev_epoch == epoch;
         if (evp->is_access) {
           if (evp->is_func == 0 && evp->type == EventType::kAccessExt &&
@@ -592,12 +514,15 @@ bool RestoreStack(Tid tid, EventType type, Sid sid, Epoch epoch, uptr addr,
         if (evp->is_func) {
           auto *ev = reinterpret_cast<EventFunc *>(evp);
           if (ev->pc) {
-            DPrintf2("  FuncEnter: pc=0x%llx\n", ev->pc);
+            DPrintf2(" FuncEnter: pc=0x%llx\n", ev->pc);
             stack.PushBack(ev->pc);
           } else {
-            DPrintf2("  FuncExit\n");
-            CHECK(stack.Size());
-            stack.PopBack();
+            DPrintf2(" FuncExit\n");
+            // We don't log pathologically large stacks in each part,
+            // if the stack was truncated we can have more func exits than
+            // entries.
+            if (stack.Size())
+              stack.PopBack();
           }
           return;
         }
@@ -666,8 +591,6 @@ bool RestoreStack(Tid tid, EventType type, Sid sid, Epoch epoch, uptr addr,
   return found;
 }
 
-}  // namespace v3
-
 bool RacyStacks::operator==(const RacyStacks &other) const {
   if (hash[0] == other.hash[0] && hash[1] == other.hash[1])
     return true;
@@ -758,10 +681,7 @@ bool OutputReport(ThreadState *thr, const ScopedReport &srep) {
     ctx->fired_suppressions.push_back(s);
   }
   {
-    bool old_is_freeing = thr->is_freeing;
-    thr->is_freeing = false;
     bool suppressed = OnReport(rep, pc_or_addr != 0);
-    thr->is_freeing = old_is_freeing;
     if (suppressed) {
       thr->current_report = nullptr;
       return false;
@@ -808,97 +728,72 @@ static bool IsFiredSuppression(Context *ctx, ReportType type, uptr addr) {
   return false;
 }
 
-static bool RaceBetweenAtomicAndFree(ThreadState *thr) {
-  Shadow s0(thr->racy_state[0]);
-  Shadow s1(thr->racy_state[1]);
-  CHECK(!(s0.IsAtomic() && s1.IsAtomic()));
-  if (!s0.IsAtomic() && !s1.IsAtomic())
-    return true;
-  if (s0.IsAtomic() && s1.IsFreed())
-    return true;
-  if (s1.IsAtomic() && thr->is_freeing)
-    return true;
-  return false;
-}
-
-void ReportRace(ThreadState *thr) {
+void ReportRace(ThreadState *thr, RawShadow *shadow_mem, Shadow cur, Shadow old,
+                AccessType typ0) {
   CheckedMutex::CheckNoLocks();
 
   // Symbolizer makes lots of intercepted calls. If we try to process them,
   // at best it will cause deadlocks on internal mutexes.
   ScopedIgnoreInterceptors ignore;
 
+  uptr addr = ShadowToMem(shadow_mem);
+  DPrintf("#%d: ReportRace %p\n", thr->tid, (void *)addr);
   if (!ShouldReport(thr, ReportTypeRace))
     return;
-  if (!flags()->report_atomic_races && !RaceBetweenAtomicAndFree(thr))
+  uptr addr_off0, size0;
+  cur.GetAccess(&addr_off0, &size0, nullptr);
+  uptr addr_off1, size1, typ1;
+  old.GetAccess(&addr_off1, &size1, &typ1);
+  if (!flags()->report_atomic_races &&
+      ((typ0 & kAccessAtomic) || (typ1 & kAccessAtomic)) &&
+      !(typ0 & kAccessFree) && !(typ1 & kAccessFree))
     return;
 
-  bool freed = false;
-  {
-    Shadow s(thr->racy_state[1]);
-    freed = s.GetFreedAndReset();
-    thr->racy_state[1] = s.raw();
-  }
-
-  uptr addr = ShadowToMem(thr->racy_shadow_addr);
-  uptr addr_min = 0;
-  uptr addr_max = 0;
-  {
-    uptr a0 = addr + Shadow(thr->racy_state[0]).addr0();
-    uptr a1 = addr + Shadow(thr->racy_state[1]).addr0();
-    uptr e0 = a0 + Shadow(thr->racy_state[0]).size();
-    uptr e1 = a1 + Shadow(thr->racy_state[1]).size();
-    addr_min = min(a0, a1);
-    addr_max = max(e0, e1);
-    if (IsExpectedReport(addr_min, addr_max - addr_min))
-      return;
-  }
+  const uptr kMop = 2;
+  Shadow s[kMop] = {cur, old};
+  uptr addr0 = addr + addr_off0;
+  uptr addr1 = addr + addr_off1;
+  uptr end0 = addr0 + size0;
+  uptr end1 = addr1 + size1;
+  uptr addr_min = min(addr0, addr1);
+  uptr addr_max = max(end0, end1);
+  if (IsExpectedReport(addr_min, addr_max - addr_min))
+    return;
   if (HandleRacyAddress(thr, addr_min, addr_max))
     return;
 
-  ReportType typ = ReportTypeRace;
-  if (thr->is_vptr_access && freed)
-    typ = ReportTypeVptrUseAfterFree;
-  else if (thr->is_vptr_access)
-    typ = ReportTypeVptrRace;
-  else if (freed)
-    typ = ReportTypeUseAfterFree;
+  ReportType rep_typ = ReportTypeRace;
+  if ((typ0 & kAccessVptr) && (typ1 & kAccessFree))
+    rep_typ = ReportTypeVptrUseAfterFree;
+  else if (typ0 & kAccessVptr)
+    rep_typ = ReportTypeVptrRace;
+  else if (typ1 & kAccessFree)
+    rep_typ = ReportTypeUseAfterFree;
 
-  if (IsFiredSuppression(ctx, typ, addr))
+  if (IsFiredSuppression(ctx, rep_typ, addr))
     return;
 
-  const uptr kMop = 2;
   VarSizeStackTrace traces[kMop];
-  uptr tags[kMop] = {kExternalTagNone};
-  uptr toppc = TraceTopPC(thr);
-  if (toppc >> kEventPCBits) {
-    // This is a work-around for a known issue.
-    // The scenario where this happens is rather elaborate and requires
-    // an instrumented __sanitizer_report_error_summary callback and
-    // a __tsan_symbolize_external callback and a race during a range memory
-    // access larger than 8 bytes. MemoryAccessRange adds the current PC to
-    // the trace and starts processing memory accesses. A first memory access
-    // triggers a race, we report it and call the instrumented
-    // __sanitizer_report_error_summary, which adds more stuff to the trace
-    // since it is intrumented. Then a second memory access in MemoryAccessRange
-    // also triggers a race and we get here and call TraceTopPC to get the
-    // current PC, however now it contains some unrelated events from the
-    // callback. Most likely, TraceTopPC will now return a EventTypeFuncExit
-    // event. Later we subtract -1 from it (in GetPreviousInstructionPc)
-    // and the resulting PC has kExternalPCBit set, so we pass it to
-    // __tsan_symbolize_external_ex. __tsan_symbolize_external_ex is within its
-    // rights to crash since the PC is completely bogus.
-    // test/tsan/double_race.cpp contains a test case for this.
-    toppc = 0;
-  }
-  ObtainCurrentStack(thr, toppc, &traces[0], &tags[0]);
-  if (IsFiredSuppression(ctx, typ, traces[0]))
+  Tid tids[kMop] = {thr->tid, kInvalidTid};
+  uptr tags[kMop] = {kExternalTagNone, kExternalTagNone};
+
+  ObtainCurrentStack(thr, thr->trace_prev_pc, &traces[0], &tags[0]);
+  if (IsFiredSuppression(ctx, rep_typ, traces[0]))
+    return;
+
+  DynamicMutexSet mset1;
+  MutexSet *mset[kMop] = {&thr->mset, mset1};
+
+  // We need to lock the slot during RestoreStack because it protects
+  // the slot journal.
+  Lock slot_lock(&ctx->slots[static_cast<uptr>(s[1].sid())].mtx);
+  ThreadRegistryLock l0(&ctx->thread_registry);
+  Lock slots_lock(&ctx->slot_mtx);
+  if (!RestoreStack(EventType::kAccessExt, s[1].sid(), s[1].epoch(), addr1,
+                    size1, typ1, &tids[1], &traces[1], mset[1], &tags[1]))
     return;
 
-  DynamicMutexSet mset2;
-  Shadow s2(thr->racy_state[1]);
-  RestoreStack(s2.tid(), s2.epoch(), &traces[1], mset2, &tags[1]);
-  if (IsFiredSuppression(ctx, typ, traces[1]))
+  if (IsFiredSuppression(ctx, rep_typ, traces[1]))
     return;
 
   if (HandleRacyStacks(thr, traces))
@@ -908,39 +803,29 @@ void ReportRace(ThreadState *thr) {
   uptr tag = kExternalTagNone;
   for (uptr i = 0; i < kMop; i++) {
     if (tags[i] != kExternalTagNone) {
-      typ = ReportTypeExternalRace;
+      rep_typ = ReportTypeExternalRace;
       tag = tags[i];
       break;
     }
   }
 
-  ThreadRegistryLock l0(&ctx->thread_registry);
-  ScopedReport rep(typ, tag);
-  for (uptr i = 0; i < kMop; i++) {
-    Shadow s(thr->racy_state[i]);
-    rep.AddMemoryAccess(addr, tags[i], s, traces[i],
-                        i == 0 ? &thr->mset : mset2);
-  }
+  ScopedReport rep(rep_typ, tag);
+  for (uptr i = 0; i < kMop; i++)
+    rep.AddMemoryAccess(addr, tags[i], s[i], tids[i], traces[i], mset[i]);
 
   for (uptr i = 0; i < kMop; i++) {
-    FastState s(thr->racy_state[i]);
     ThreadContext *tctx = static_cast<ThreadContext *>(
-        ctx->thread_registry.GetThreadLocked(s.tid()));
-    if (s.epoch() < tctx->epoch0 || s.epoch() > tctx->epoch1)
-      continue;
+        ctx->thread_registry.GetThreadLocked(tids[i]));
     rep.AddThread(tctx);
   }
 
   rep.AddLocation(addr_min, addr_max - addr_min);
 
 #if !SANITIZER_GO
-  {
-    Shadow s(thr->racy_state[1]);
-    if (s.epoch() <= thr->last_sleep_clock.get(s.tid()))
-      rep.AddSleep(thr->last_sleep_stack_id);
-  }
+  if (!((typ0 | typ1) & kAccessFree) &&
+      s[1].epoch() <= thr->last_sleep_clock.Get(s[1].sid()))
+    rep.AddSleep(thr->last_sleep_stack_id);
 #endif
-
   OutputReport(thr, rep);
 }
 
diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp b/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp
index c8f7124c009d..86c8b3764cc7 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp
@@ -21,20 +21,14 @@ namespace __tsan {
 
 // ThreadContext implementation.
 
-ThreadContext::ThreadContext(Tid tid)
-    : ThreadContextBase(tid), thr(), sync(), epoch0(), epoch1() {}
+ThreadContext::ThreadContext(Tid tid) : ThreadContextBase(tid), thr(), sync() {}
 
 #if !SANITIZER_GO
 ThreadContext::~ThreadContext() {
 }
 #endif
 
-void ThreadContext::OnReset() {
-  CHECK_EQ(sync.size(), 0);
-  uptr trace_p = GetThreadTrace(tid);
-  ReleaseMemoryPagesToOS(trace_p, trace_p + TraceSize() * sizeof(Event));
-  //!!! ReleaseMemoryToOS(GetThreadTraceHeader(tid), sizeof(Trace));
-}
+void ThreadContext::OnReset() { CHECK(!sync); }
 
 #if !SANITIZER_GO
 struct ThreadLeak {
@@ -57,7 +51,9 @@ static void CollectThreadLeaks(ThreadContextBase *tctx_base, void *arg) {
 }
 #endif
 
-#if !SANITIZER_GO
+// Disabled on Mac because lldb test TestTsanBasic fails:
+// https://reviews.llvm.org/D112603#3163158
+#if !SANITIZER_GO && !SANITIZER_MAC
 static void ReportIgnoresEnabled(ThreadContext *tctx, IgnoreSet *set) {
   if (tctx->tid == kMainTid) {
     Printf("ThreadSanitizer: main thread finished with ignores enabled\n");
@@ -112,30 +108,35 @@ int ThreadCount(ThreadState *thr) {
 }
 
 struct OnCreatedArgs {
-  ThreadState *thr;
-  uptr pc;
+  VectorClock *sync;
+  uptr sync_epoch;
+  StackID stack;
 };
 
 Tid ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached) {
-  OnCreatedArgs args = { thr, pc };
-  u32 parent_tid = thr ? thr->tid : kInvalidTid;  // No parent for GCD workers.
-  Tid tid = ctx->thread_registry.CreateThread(uid, detached, parent_tid, &args);
-  DPrintf("#%d: ThreadCreate tid=%d uid=%zu\n", parent_tid, tid, uid);
+  // The main thread and GCD workers don't have a parent thread.
+  Tid parent = kInvalidTid;
+  OnCreatedArgs arg = {nullptr, 0, kInvalidStackID};
+  if (thr) {
+    parent = thr->tid;
+    arg.stack = CurrentStackId(thr, pc);
+    if (!thr->ignore_sync) {
+      SlotLocker locker(thr);
+      thr->clock.ReleaseStore(&arg.sync);
+      arg.sync_epoch = ctx->global_epoch;
+      IncrementEpoch(thr);
+    }
+  }
+  Tid tid = ctx->thread_registry.CreateThread(uid, detached, parent, &arg);
+  DPrintf("#%d: ThreadCreate tid=%d uid=%zu\n", parent, tid, uid);
   return tid;
 }
 
 void ThreadContext::OnCreated(void *arg) {
-  thr = 0;
-  if (tid == kMainTid)
-    return;
   OnCreatedArgs *args = static_cast<OnCreatedArgs *>(arg);
-  if (!args->thr)  // GCD workers don't have a parent thread.
-    return;
-  args->thr->fast_state.IncrementEpoch();
-  // Can't increment epoch w/o writing to the trace as well.
-  TraceAddEvent(args->thr, args->thr->fast_state, EventTypeMop, 0);
-  ReleaseImpl(args->thr, 0, &sync);
-  creation_stack_id = CurrentStackId(args->thr, args->pc);
+  sync = args->sync;
+  sync_epoch = args->sync_epoch;
+  creation_stack_id = args->stack;
 }
 
 extern "C" void __tsan_stack_initialization() {}
@@ -150,6 +151,15 @@ struct OnStartedArgs {
 
 void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id,
                  ThreadType thread_type) {
+  ctx->thread_registry.StartThread(tid, os_id, thread_type, thr);
+  if (!thr->ignore_sync) {
+    SlotAttachAndLock(thr);
+    if (thr->tctx->sync_epoch == ctx->global_epoch)
+      thr->clock.Acquire(thr->tctx->sync);
+    SlotUnlock(thr);
+  }
+  Free(thr->tctx->sync);
+
   uptr stk_addr = 0;
   uptr stk_size = 0;
   uptr tls_addr = 0;
@@ -159,12 +169,10 @@ void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id,
     GetThreadStackAndTls(tid == kMainTid, &stk_addr, &stk_size, &tls_addr,
                          &tls_size);
 #endif
-
-  ThreadRegistry *tr = &ctx->thread_registry;
-  OnStartedArgs args = { thr, stk_addr, stk_size, tls_addr, tls_size };
-  tr->StartThread(tid, os_id, thread_type, &args);
-
-  while (!thr->tctx->trace.parts.Empty()) thr->tctx->trace.parts.PopBack();
+  thr->stk_addr = stk_addr;
+  thr->stk_size = stk_size;
+  thr->tls_addr = tls_addr;
+  thr->tls_size = tls_size;
 
 #if !SANITIZER_GO
   if (ctx->after_multithreaded_fork) {
@@ -192,57 +200,41 @@ void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id,
 }
 
 void ThreadContext::OnStarted(void *arg) {
-  OnStartedArgs *args = static_cast<OnStartedArgs *>(arg);
-  thr = args->thr;
-  // RoundUp so that one trace part does not contain events
-  // from different threads.
-  epoch0 = RoundUp(epoch1 + 1, kTracePartSize);
-  epoch1 = (u64)-1;
-  new (thr)
-      ThreadState(ctx, tid, unique_id, epoch0, reuse_count, args->stk_addr,
-                  args->stk_size, args->tls_addr, args->tls_size);
+  thr = static_cast<ThreadState *>(arg);
+  DPrintf("#%d: ThreadStart\n", tid);
+  new (thr) ThreadState(tid);
   if (common_flags()->detect_deadlocks)
-    thr->dd_lt = ctx->dd->CreateLogicalThread(unique_id);
-  thr->fast_state.SetHistorySize(flags()->history_size);
-  // Commit switch to the new part of the trace.
-  // TraceAddEvent will reset stack0/mset0 in the new part for us.
-  TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
-
-  thr->fast_synch_epoch = epoch0;
-  AcquireImpl(thr, 0, &sync);
-  sync.Reset(&thr->proc()->clock_cache);
+    thr->dd_lt = ctx->dd->CreateLogicalThread(tid);
   thr->tctx = this;
+#if !SANITIZER_GO
   thr->is_inited = true;
-  DPrintf(
-      "#%d: ThreadStart epoch=%zu stk_addr=%zx stk_size=%zx "
-      "tls_addr=%zx tls_size=%zx\n",
-      tid, (uptr)epoch0, args->stk_addr, args->stk_size, args->tls_addr,
-      args->tls_size);
+#endif
 }
 
 void ThreadFinish(ThreadState *thr) {
+  DPrintf("#%d: ThreadFinish\n", thr->tid);
   ThreadCheckIgnore(thr);
   if (thr->stk_addr && thr->stk_size)
     DontNeedShadowFor(thr->stk_addr, thr->stk_size);
   if (thr->tls_addr && thr->tls_size)
     DontNeedShadowFor(thr->tls_addr, thr->tls_size);
   thr->is_dead = true;
-  thr->is_inited = false;
 #if !SANITIZER_GO
+  thr->is_inited = false;
   thr->ignore_interceptors++;
+  PlatformCleanUpThreadState(thr);
 #endif
-  ctx->thread_registry.FinishThread(thr->tid);
-}
-
-void ThreadContext::OnFinished() {
-  if (!detached) {
-    thr->fast_state.IncrementEpoch();
-    // Can't increment epoch w/o writing to the trace as well.
-    TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
-    ReleaseImpl(thr, 0, &sync);
+  if (!thr->ignore_sync) {
+    SlotLocker locker(thr);
+    ThreadRegistryLock lock(&ctx->thread_registry);
+    // Note: detached is protected by the thread registry mutex,
+    // the thread may be detaching concurrently in another thread.
+    if (!thr->tctx->detached) {
+      thr->clock.ReleaseStore(&thr->tctx->sync);
+      thr->tctx->sync_epoch = ctx->global_epoch;
+      IncrementEpoch(thr);
+    }
   }
-  epoch1 = thr->fast_state.epoch();
-
 #if !SANITIZER_GO
   UnmapOrDie(thr->shadow_stack, kShadowStackSize * sizeof(uptr));
 #else
@@ -251,18 +243,37 @@ void ThreadContext::OnFinished() {
   thr->shadow_stack = nullptr;
   thr->shadow_stack_pos = nullptr;
   thr->shadow_stack_end = nullptr;
-
   if (common_flags()->detect_deadlocks)
     ctx->dd->DestroyLogicalThread(thr->dd_lt);
-  thr->clock.ResetCached(&thr->proc()->clock_cache);
-#if !SANITIZER_GO
-  thr->last_sleep_clock.ResetCached(&thr->proc()->clock_cache);
-#endif
-#if !SANITIZER_GO
-  PlatformCleanUpThreadState(thr);
-#endif
+  SlotDetach(thr);
+  ctx->thread_registry.FinishThread(thr->tid);
   thr->~ThreadState();
-  thr = 0;
+}
+
+void ThreadContext::OnFinished() {
+  Lock lock(&ctx->slot_mtx);
+  Lock lock1(&trace.mtx);
+  // Queue all trace parts into the global recycle queue.
+  auto parts = &trace.parts;
+  while (trace.local_head) {
+    CHECK(parts->Queued(trace.local_head));
+    ctx->trace_part_recycle.PushBack(trace.local_head);
+    trace.local_head = parts->Next(trace.local_head);
+  }
+  ctx->trace_part_recycle_finished += parts->Size();
+  if (ctx->trace_part_recycle_finished > Trace::kFinishedThreadHi) {
+    ctx->trace_part_finished_excess += parts->Size();
+    trace.parts_allocated = 0;
+  } else if (ctx->trace_part_recycle_finished > Trace::kFinishedThreadLo &&
+             parts->Size() > 1) {
+    ctx->trace_part_finished_excess += parts->Size() - 1;
+    trace.parts_allocated = 1;
+  }
+  // From now on replay will use trace->final_pos.
+  trace.final_pos = (Event *)atomic_load_relaxed(&thr->trace_pos);
+  atomic_store_relaxed(&thr->trace_pos, 0);
+  thr->tctx = nullptr;
+  thr = nullptr;
 }
 
 struct ConsumeThreadContext {
@@ -274,35 +285,43 @@ Tid ThreadConsumeTid(ThreadState *thr, uptr pc, uptr uid) {
   return ctx->thread_registry.ConsumeThreadUserId(uid);
 }
 
+struct JoinArg {
+  VectorClock *sync;
+  uptr sync_epoch;
+};
+
 void ThreadJoin(ThreadState *thr, uptr pc, Tid tid) {
   CHECK_GT(tid, 0);
-  CHECK_LT(tid, kMaxTid);
   DPrintf("#%d: ThreadJoin tid=%d\n", thr->tid, tid);
-  ctx->thread_registry.JoinThread(tid, thr);
+  JoinArg arg = {};
+  ctx->thread_registry.JoinThread(tid, &arg);
+  if (!thr->ignore_sync) {
+    SlotLocker locker(thr);
+    if (arg.sync_epoch == ctx->global_epoch)
+      thr->clock.Acquire(arg.sync);
+  }
+  Free(arg.sync);
 }
 
-void ThreadContext::OnJoined(void *arg) {
-  ThreadState *caller_thr = static_cast<ThreadState *>(arg);
-  AcquireImpl(caller_thr, 0, &sync);
-  sync.Reset(&caller_thr->proc()->clock_cache);
+void ThreadContext::OnJoined(void *ptr) {
+  auto arg = static_cast<JoinArg *>(ptr);
+  arg->sync = sync;
+  arg->sync_epoch = sync_epoch;
+  sync = nullptr;
+  sync_epoch = 0;
 }
 
-void ThreadContext::OnDead() { CHECK_EQ(sync.size(), 0); }
+void ThreadContext::OnDead() { CHECK_EQ(sync, nullptr); }
 
 void ThreadDetach(ThreadState *thr, uptr pc, Tid tid) {
   CHECK_GT(tid, 0);
-  CHECK_LT(tid, kMaxTid);
   ctx->thread_registry.DetachThread(tid, thr);
 }
 
-void ThreadContext::OnDetached(void *arg) {
-  ThreadState *thr1 = static_cast<ThreadState *>(arg);
-  sync.Reset(&thr1->proc()->clock_cache);
-}
+void ThreadContext::OnDetached(void *arg) { Free(sync); }
 
 void ThreadNotJoined(ThreadState *thr, uptr pc, Tid tid, uptr uid) {
   CHECK_GT(tid, 0);
-  CHECK_LT(tid, kMaxTid);
   ctx->thread_registry.SetThreadUserId(tid, uid);
 }
 
diff --git a/compiler-rt/lib/tsan/rtl/tsan_shadow.h b/compiler-rt/lib/tsan/rtl/tsan_shadow.h
index 8b7bc341713e..843573ecf5d3 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_shadow.h
+++ b/compiler-rt/lib/tsan/rtl/tsan_shadow.h
@@ -10,223 +10,170 @@
 #define TSAN_SHADOW_H
 
 #include "tsan_defs.h"
-#include "tsan_trace.h"
 
 namespace __tsan {
 
-// FastState (from most significant bit):
-//   ignore          : 1
-//   tid             : kTidBits
-//   unused          : -
-//   history_size    : 3
-//   epoch           : kClkBits
 class FastState {
  public:
-  FastState(u64 tid, u64 epoch) {
-    x_ = tid << kTidShift;
-    x_ |= epoch;
-    DCHECK_EQ(tid, this->tid());
-    DCHECK_EQ(epoch, this->epoch());
-    DCHECK_EQ(GetIgnoreBit(), false);
-  }
-
-  explicit FastState(u64 x) : x_(x) {}
-
-  u64 raw() const { return x_; }
-
-  u64 tid() const {
-    u64 res = (x_ & ~kIgnoreBit) >> kTidShift;
-    return res;
-  }
-
-  u64 TidWithIgnore() const {
-    u64 res = x_ >> kTidShift;
-    return res;
-  }
-
-  u64 epoch() const {
-    u64 res = x_ & ((1ull << kClkBits) - 1);
-    return res;
-  }
+  FastState() { Reset(); }
 
-  void IncrementEpoch() {
-    u64 old_epoch = epoch();
-    x_ += 1;
-    DCHECK_EQ(old_epoch + 1, epoch());
-    (void)old_epoch;
+  void Reset() {
+    part_.unused0_ = 0;
+    part_.sid_ = static_cast<u8>(kFreeSid);
+    part_.epoch_ = static_cast<u16>(kEpochLast);
+    part_.unused1_ = 0;
+    part_.ignore_accesses_ = false;
   }
 
-  void SetIgnoreBit() { x_ |= kIgnoreBit; }
-  void ClearIgnoreBit() { x_ &= ~kIgnoreBit; }
-  bool GetIgnoreBit() const { return (s64)x_ < 0; }
+  void SetSid(Sid sid) { part_.sid_ = static_cast<u8>(sid); }
 
-  void SetHistorySize(int hs) {
-    CHECK_GE(hs, 0);
-    CHECK_LE(hs, 7);
-    x_ = (x_ & ~(kHistoryMask << kHistoryShift)) | (u64(hs) << kHistoryShift);
-  }
+  Sid sid() const { return static_cast<Sid>(part_.sid_); }
 
-  ALWAYS_INLINE
-  int GetHistorySize() const {
-    return (int)((x_ >> kHistoryShift) & kHistoryMask);
-  }
+  Epoch epoch() const { return static_cast<Epoch>(part_.epoch_); }
 
-  void ClearHistorySize() { SetHistorySize(0); }
+  void SetEpoch(Epoch epoch) { part_.epoch_ = static_cast<u16>(epoch); }
 
-  ALWAYS_INLINE
-  u64 GetTracePos() const {
-    const int hs = GetHistorySize();
-    // When hs == 0, the trace consists of 2 parts.
-    const u64 mask = (1ull << (kTracePartSizeBits + hs + 1)) - 1;
-    return epoch() & mask;
-  }
+  void SetIgnoreBit() { part_.ignore_accesses_ = 1; }
+  void ClearIgnoreBit() { part_.ignore_accesses_ = 0; }
+  bool GetIgnoreBit() const { return part_.ignore_accesses_; }
 
  private:
   friend class Shadow;
-  static const int kTidShift = 64 - kTidBits - 1;
-  static const u64 kIgnoreBit = 1ull << 63;
-  static const u64 kFreedBit = 1ull << 63;
-  static const u64 kHistoryShift = kClkBits;
-  static const u64 kHistoryMask = 7;
-  u64 x_;
+  struct Parts {
+    u32 unused0_ : 8;
+    u32 sid_ : 8;
+    u32 epoch_ : kEpochBits;
+    u32 unused1_ : 1;
+    u32 ignore_accesses_ : 1;
+  };
+  union {
+    Parts part_;
+    u32 raw_;
+  };
 };
 
-// Shadow (from most significant bit):
-//   freed           : 1
-//   tid             : kTidBits
-//   is_atomic       : 1
-//   is_read         : 1
-//   size_log        : 2
-//   addr0           : 3
-//   epoch           : kClkBits
-class Shadow : public FastState {
- public:
-  explicit Shadow(u64 x) : FastState(x) {}
+static_assert(sizeof(FastState) == kShadowSize, "bad FastState size");
 
-  explicit Shadow(const FastState &s) : FastState(s.x_) { ClearHistorySize(); }
-
-  void SetAddr0AndSizeLog(u64 addr0, unsigned kAccessSizeLog) {
-    DCHECK_EQ((x_ >> kClkBits) & 31, 0);
-    DCHECK_LE(addr0, 7);
-    DCHECK_LE(kAccessSizeLog, 3);
-    x_ |= ((kAccessSizeLog << 3) | addr0) << kClkBits;
-    DCHECK_EQ(kAccessSizeLog, size_log());
-    DCHECK_EQ(addr0, this->addr0());
-  }
-
-  void SetWrite(unsigned kAccessIsWrite) {
-    DCHECK_EQ(x_ & kReadBit, 0);
-    if (!kAccessIsWrite)
-      x_ |= kReadBit;
-    DCHECK_EQ(kAccessIsWrite, IsWrite());
-  }
-
-  void SetAtomic(bool kIsAtomic) {
-    DCHECK(!IsAtomic());
-    if (kIsAtomic)
-      x_ |= kAtomicBit;
-    DCHECK_EQ(IsAtomic(), kIsAtomic);
-  }
-
-  bool IsAtomic() const { return x_ & kAtomicBit; }
-
-  bool IsZero() const { return x_ == 0; }
-
-  static inline bool TidsAreEqual(const Shadow s1, const Shadow s2) {
-    u64 shifted_xor = (s1.x_ ^ s2.x_) >> kTidShift;
-    DCHECK_EQ(shifted_xor == 0, s1.TidWithIgnore() == s2.TidWithIgnore());
-    return shifted_xor == 0;
-  }
-
-  static ALWAYS_INLINE bool Addr0AndSizeAreEqual(const Shadow s1,
-                                                 const Shadow s2) {
-    u64 masked_xor = ((s1.x_ ^ s2.x_) >> kClkBits) & 31;
-    return masked_xor == 0;
+class Shadow {
+ public:
+  static constexpr RawShadow kEmpty = static_cast<RawShadow>(0);
+
+  Shadow(FastState state, u32 addr, u32 size, AccessType typ) {
+    raw_ = state.raw_;
+    DCHECK_GT(size, 0);
+    DCHECK_LE(size, 8);
+    UNUSED Sid sid0 = part_.sid_;
+    UNUSED u16 epoch0 = part_.epoch_;
+    raw_ |= (!!(typ & kAccessAtomic) << kIsAtomicShift) |
+            (!!(typ & kAccessRead) << kIsReadShift) |
+            (((((1u << size) - 1) << (addr & 0x7)) & 0xff) << kAccessShift);
+    // Note: we don't check kAccessAtomic because it overlaps with
+    // FastState::ignore_accesses_ and it may be set spuriously.
+    DCHECK_EQ(part_.is_read_, !!(typ & kAccessRead));
+    DCHECK_EQ(sid(), sid0);
+    DCHECK_EQ(epoch(), epoch0);
+  }
+
+  explicit Shadow(RawShadow x = Shadow::kEmpty) { raw_ = static_cast<u32>(x); }
+
+  RawShadow raw() const { return static_cast<RawShadow>(raw_); }
+  Sid sid() const { return part_.sid_; }
+  Epoch epoch() const { return static_cast<Epoch>(part_.epoch_); }
+  u8 access() const { return part_.access_; }
+
+  void GetAccess(uptr *addr, uptr *size, AccessType *typ) const {
+    DCHECK(part_.access_ != 0 || raw_ == static_cast<u32>(Shadow::kRodata));
+    if (addr)
+      *addr = part_.access_ ? __builtin_ffs(part_.access_) - 1 : 0;
+    if (size)
+      *size = part_.access_ == kFreeAccess ? kShadowCell
+                                           : __builtin_popcount(part_.access_);
+    if (typ)
+      *typ = (part_.is_read_ ? kAccessRead : kAccessWrite) |
+             (part_.is_atomic_ ? kAccessAtomic : 0) |
+             (part_.access_ == kFreeAccess ? kAccessFree : 0);
   }
 
-  static ALWAYS_INLINE bool TwoRangesIntersect(Shadow s1, Shadow s2,
-                                               unsigned kS2AccessSize) {
-    bool res = false;
-    u64 diff = s1.addr0() - s2.addr0();
-    if ((s64)diff < 0) {  // s1.addr0 < s2.addr0
-      // if (s1.addr0() + size1) > s2.addr0()) return true;
-      if (s1.size() > -diff)
-        res = true;
-    } else {
-      // if (s2.addr0() + kS2AccessSize > s1.addr0()) return true;
-      if (kS2AccessSize > diff)
-        res = true;
-    }
-    DCHECK_EQ(res, TwoRangesIntersectSlow(s1, s2));
-    DCHECK_EQ(res, TwoRangesIntersectSlow(s2, s1));
+  ALWAYS_INLINE
+  bool IsBothReadsOrAtomic(AccessType typ) const {
+    u32 is_read = !!(typ & kAccessRead);
+    u32 is_atomic = !!(typ & kAccessAtomic);
+    bool res =
+        raw_ & ((is_atomic << kIsAtomicShift) | (is_read << kIsReadShift));
+    DCHECK_EQ(res,
+              (part_.is_read_ && is_read) || (part_.is_atomic_ && is_atomic));
     return res;
   }
 
-  u64 ALWAYS_INLINE addr0() const { return (x_ >> kClkBits) & 7; }
-  u64 ALWAYS_INLINE size() const { return 1ull << size_log(); }
-  bool ALWAYS_INLINE IsWrite() const { return !IsRead(); }
-  bool ALWAYS_INLINE IsRead() const { return x_ & kReadBit; }
-
-  // The idea behind the freed bit is as follows.
-  // When the memory is freed (or otherwise unaccessible) we write to the shadow
-  // values with tid/epoch related to the free and the freed bit set.
-  // During memory accesses processing the freed bit is considered
-  // as msb of tid. So any access races with shadow with freed bit set
-  // (it is as if write from a thread with which we never synchronized before).
-  // This allows us to detect accesses to freed memory w/o additional
-  // overheads in memory access processing and at the same time restore
-  // tid/epoch of free.
-  void MarkAsFreed() { x_ |= kFreedBit; }
-
-  bool IsFreed() const { return x_ & kFreedBit; }
-
-  bool GetFreedAndReset() {
-    bool res = x_ & kFreedBit;
-    x_ &= ~kFreedBit;
+  ALWAYS_INLINE
+  bool IsRWWeakerOrEqual(AccessType typ) const {
+    u32 is_read = !!(typ & kAccessRead);
+    u32 is_atomic = !!(typ & kAccessAtomic);
+    UNUSED u32 res0 =
+        (part_.is_atomic_ > is_atomic) ||
+        (part_.is_atomic_ == is_atomic && part_.is_read_ >= is_read);
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+    const u32 kAtomicReadMask = (1 << kIsAtomicShift) | (1 << kIsReadShift);
+    bool res = (raw_ & kAtomicReadMask) >=
+               ((is_atomic << kIsAtomicShift) | (is_read << kIsReadShift));
+
+    DCHECK_EQ(res, res0);
     return res;
+#else
+    return res0;
+#endif
   }
 
-  bool ALWAYS_INLINE IsBothReadsOrAtomic(bool kIsWrite, bool kIsAtomic) const {
-    bool v = x_ & ((u64(kIsWrite ^ 1) << kReadShift) |
-                   (u64(kIsAtomic) << kAtomicShift));
-    DCHECK_EQ(v, (!IsWrite() && !kIsWrite) || (IsAtomic() && kIsAtomic));
-    return v;
-  }
-
-  bool ALWAYS_INLINE IsRWNotWeaker(bool kIsWrite, bool kIsAtomic) const {
-    bool v = ((x_ >> kReadShift) & 3) <= u64((kIsWrite ^ 1) | (kIsAtomic << 1));
-    DCHECK_EQ(v, (IsAtomic() < kIsAtomic) ||
-                     (IsAtomic() == kIsAtomic && !IsWrite() <= !kIsWrite));
-    return v;
+  // The FreedMarker must not pass "the same access check" so that we don't
+  // return from the race detection algorithm early.
+  static RawShadow FreedMarker() {
+    FastState fs;
+    fs.SetSid(kFreeSid);
+    fs.SetEpoch(kEpochLast);
+    Shadow s(fs, 0, 8, kAccessWrite);
+    return s.raw();
   }
 
-  bool ALWAYS_INLINE IsRWWeakerOrEqual(bool kIsWrite, bool kIsAtomic) const {
-    bool v = ((x_ >> kReadShift) & 3) >= u64((kIsWrite ^ 1) | (kIsAtomic << 1));
-    DCHECK_EQ(v, (IsAtomic() > kIsAtomic) ||
-                     (IsAtomic() == kIsAtomic && !IsWrite() >= !kIsWrite));
-    return v;
+  static RawShadow FreedInfo(Sid sid, Epoch epoch) {
+    Shadow s;
+    s.part_.sid_ = sid;
+    s.part_.epoch_ = static_cast<u16>(epoch);
+    s.part_.access_ = kFreeAccess;
+    return s.raw();
   }
 
  private:
-  static const u64 kReadShift = 5 + kClkBits;
-  static const u64 kReadBit = 1ull << kReadShift;
-  static const u64 kAtomicShift = 6 + kClkBits;
-  static const u64 kAtomicBit = 1ull << kAtomicShift;
-
-  u64 size_log() const { return (x_ >> (3 + kClkBits)) & 3; }
-
-  static bool TwoRangesIntersectSlow(const Shadow s1, const Shadow s2) {
-    if (s1.addr0() == s2.addr0())
-      return true;
-    if (s1.addr0() < s2.addr0() && s1.addr0() + s1.size() > s2.addr0())
-      return true;
-    if (s2.addr0() < s1.addr0() && s2.addr0() + s2.size() > s1.addr0())
-      return true;
-    return false;
-  }
+  struct Parts {
+    u8 access_;
+    Sid sid_;
+    u16 epoch_ : kEpochBits;
+    u16 is_read_ : 1;
+    u16 is_atomic_ : 1;
+  };
+  union {
+    Parts part_;
+    u32 raw_;
+  };
+
+  static constexpr u8 kFreeAccess = 0x81;
+
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+  static constexpr uptr kAccessShift = 0;
+  static constexpr uptr kIsReadShift = 30;
+  static constexpr uptr kIsAtomicShift = 31;
+#else
+  static constexpr uptr kAccessShift = 24;
+  static constexpr uptr kIsReadShift = 1;
+  static constexpr uptr kIsAtomicShift = 0;
+#endif
+
+ public:
+  // .rodata shadow marker, see MapRodata and ContainsSameAccessFast.
+  static constexpr RawShadow kRodata =
+      static_cast<RawShadow>(1 << kIsReadShift);
 };
 
-const RawShadow kShadowRodata = (RawShadow)-1;  // .rodata shadow marker
+static_assert(sizeof(Shadow) == kShadowSize, "bad Shadow size");
 
 }  // namespace __tsan
 
diff --git a/compiler-rt/lib/tsan/rtl/tsan_sync.cpp b/compiler-rt/lib/tsan/rtl/tsan_sync.cpp
index f042abab74e5..09d41780d188 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_sync.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_sync.cpp
@@ -18,43 +18,31 @@ namespace __tsan {
 
 void DDMutexInit(ThreadState *thr, uptr pc, SyncVar *s);
 
-SyncVar::SyncVar() : mtx(MutexTypeSyncVar) { Reset(0); }
+SyncVar::SyncVar() : mtx(MutexTypeSyncVar) { Reset(); }
 
-void SyncVar::Init(ThreadState *thr, uptr pc, uptr addr, u64 uid,
-                   bool save_stack) {
+void SyncVar::Init(ThreadState *thr, uptr pc, uptr addr, bool save_stack) {
+  Reset();
   this->addr = addr;
-  this->uid = uid;
-  this->next = 0;
-
-  creation_stack_id = kInvalidStackID;
+  next = 0;
   if (save_stack && !SANITIZER_GO)  // Go does not use them
     creation_stack_id = CurrentStackId(thr, pc);
   if (common_flags()->detect_deadlocks)
     DDMutexInit(thr, pc, this);
 }
 
-void SyncVar::Reset(Processor *proc) {
-  uid = 0;
+void SyncVar::Reset() {
+  CHECK(!ctx->resetting);
   creation_stack_id = kInvalidStackID;
   owner_tid = kInvalidTid;
-  last_lock = 0;
+  last_lock.Reset();
   recursion = 0;
   atomic_store_relaxed(&flags, 0);
-
-  if (proc == 0) {
-    CHECK_EQ(clock.size(), 0);
-    CHECK_EQ(read_clock.size(), 0);
-  } else {
-    clock.Reset(&proc->clock_cache);
-    read_clock.Reset(&proc->clock_cache);
-  }
+  Free(clock);
+  Free(read_clock);
 }
 
 MetaMap::MetaMap()
-    : block_alloc_(LINKER_INITIALIZED, "heap block allocator"),
-      sync_alloc_(LINKER_INITIALIZED, "sync allocator") {
-  atomic_store(&uid_gen_, 0, memory_order_relaxed);
-}
+    : block_alloc_("heap block allocator"), sync_alloc_("sync allocator") {}
 
 void MetaMap::AllocBlock(ThreadState *thr, uptr pc, uptr p, uptr sz) {
   u32 idx = block_alloc_.Alloc(&thr->proc()->block_cache);
@@ -68,16 +56,16 @@ void MetaMap::AllocBlock(ThreadState *thr, uptr pc, uptr p, uptr sz) {
   *meta = idx | kFlagBlock;
 }
 
-uptr MetaMap::FreeBlock(Processor *proc, uptr p) {
+uptr MetaMap::FreeBlock(Processor *proc, uptr p, bool reset) {
   MBlock* b = GetBlock(p);
   if (b == 0)
     return 0;
   uptr sz = RoundUpTo(b->siz, kMetaShadowCell);
-  FreeRange(proc, p, sz);
+  FreeRange(proc, p, sz, reset);
   return sz;
 }
 
-bool MetaMap::FreeRange(Processor *proc, uptr p, uptr sz) {
+bool MetaMap::FreeRange(Processor *proc, uptr p, uptr sz, bool reset) {
   bool has_something = false;
   u32 *meta = MemToMeta(p);
   u32 *end = MemToMeta(p + sz);
@@ -99,7 +87,8 @@ bool MetaMap::FreeRange(Processor *proc, uptr p, uptr sz) {
         DCHECK(idx & kFlagSync);
         SyncVar *s = sync_alloc_.Map(idx & ~kFlagMask);
         u32 next = s->next;
-        s->Reset(proc);
+        if (reset)
+          s->Reset();
         sync_alloc_.Free(&proc->sync_cache, idx & ~kFlagMask);
         idx = next;
       } else {
@@ -116,30 +105,30 @@ bool MetaMap::FreeRange(Processor *proc, uptr p, uptr sz) {
 // which can be huge. The function probes pages one-by-one until it finds a page
 // without meta objects, at this point it stops freeing meta objects. Because
 // thread stacks grow top-down, we do the same starting from end as well.
-void MetaMap::ResetRange(Processor *proc, uptr p, uptr sz) {
+void MetaMap::ResetRange(Processor *proc, uptr p, uptr sz, bool reset) {
   if (SANITIZER_GO) {
     // UnmapOrDie/MmapFixedNoReserve does not work on Windows,
     // so we do the optimization only for C/C++.
-    FreeRange(proc, p, sz);
+    FreeRange(proc, p, sz, reset);
     return;
   }
   const uptr kMetaRatio = kMetaShadowCell / kMetaShadowSize;
   const uptr kPageSize = GetPageSizeCached() * kMetaRatio;
   if (sz <= 4 * kPageSize) {
     // If the range is small, just do the normal free procedure.
-    FreeRange(proc, p, sz);
+    FreeRange(proc, p, sz, reset);
     return;
   }
   // First, round both ends of the range to page size.
   uptr diff = RoundUp(p, kPageSize) - p;
   if (diff != 0) {
-    FreeRange(proc, p, diff);
+    FreeRange(proc, p, diff, reset);
     p += diff;
     sz -= diff;
   }
   diff = p + sz - RoundDown(p + sz, kPageSize);
   if (diff != 0) {
-    FreeRange(proc, p + sz - diff, diff);
+    FreeRange(proc, p + sz - diff, diff, reset);
     sz -= diff;
   }
   // Now we must have a non-empty page-aligned range.
@@ -150,7 +139,7 @@ void MetaMap::ResetRange(Processor *proc, uptr p, uptr sz) {
   const uptr sz0 = sz;
   // Probe start of the range.
   for (uptr checked = 0; sz > 0; checked += kPageSize) {
-    bool has_something = FreeRange(proc, p, kPageSize);
+    bool has_something = FreeRange(proc, p, kPageSize, reset);
     p += kPageSize;
     sz -= kPageSize;
     if (!has_something && checked > (128 << 10))
@@ -158,7 +147,7 @@ void MetaMap::ResetRange(Processor *proc, uptr p, uptr sz) {
   }
   // Probe end of the range.
   for (uptr checked = 0; sz > 0; checked += kPageSize) {
-    bool has_something = FreeRange(proc, p + sz - kPageSize, kPageSize);
+    bool has_something = FreeRange(proc, p + sz - kPageSize, kPageSize, reset);
     sz -= kPageSize;
     // Stacks grow down, so sync object are most likely at the end of the region
     // (if it is a stack). The very end of the stack is TLS and tsan increases
@@ -177,6 +166,27 @@ void MetaMap::ResetRange(Processor *proc, uptr p, uptr sz) {
     Die();
 }
 
+void MetaMap::ResetClocks() {
+  // This can be called from the background thread
+  // which does not have proc/cache.
+  // The cache is too large for stack.
+  static InternalAllocatorCache cache;
+  internal_memset(&cache, 0, sizeof(cache));
+  internal_allocator()->InitCache(&cache);
+  sync_alloc_.ForEach([&](SyncVar *s) {
+    if (s->clock) {
+      InternalFree(s->clock, &cache);
+      s->clock = nullptr;
+    }
+    if (s->read_clock) {
+      InternalFree(s->read_clock, &cache);
+      s->read_clock = nullptr;
+    }
+    s->last_lock.Reset();
+  });
+  internal_allocator()->DestroyCache(&cache);
+}
+
 MBlock* MetaMap::GetBlock(uptr p) {
   u32 *meta = MemToMeta(p);
   u32 idx = *meta;
@@ -193,6 +203,7 @@ MBlock* MetaMap::GetBlock(uptr p) {
 
 SyncVar *MetaMap::GetSync(ThreadState *thr, uptr pc, uptr addr, bool create,
                           bool save_stack) {
+  DCHECK(!create || thr->slot_locked);
   u32 *meta = MemToMeta(addr);
   u32 idx0 = *meta;
   u32 myidx = 0;
@@ -203,7 +214,7 @@ SyncVar *MetaMap::GetSync(ThreadState *thr, uptr pc, uptr addr, bool create,
       SyncVar * s = sync_alloc_.Map(idx & ~kFlagMask);
       if (LIKELY(s->addr == addr)) {
         if (UNLIKELY(myidx != 0)) {
-          mys->Reset(thr->proc());
+          mys->Reset();
           sync_alloc_.Free(&thr->proc()->sync_cache, myidx);
         }
         return s;
@@ -218,10 +229,9 @@ SyncVar *MetaMap::GetSync(ThreadState *thr, uptr pc, uptr addr, bool create,
     }
 
     if (LIKELY(myidx == 0)) {
-      const u64 uid = atomic_fetch_add(&uid_gen_, 1, memory_order_relaxed);
       myidx = sync_alloc_.Alloc(&thr->proc()->sync_cache);
       mys = sync_alloc_.Map(myidx);
-      mys->Init(thr, pc, addr, uid, save_stack);
+      mys->Init(thr, pc, addr, save_stack);
     }
     mys->next = idx0;
     if (atomic_compare_exchange_strong((atomic_uint32_t*)meta, &idx0,
diff --git a/compiler-rt/lib/tsan/rtl/tsan_sync.h b/compiler-rt/lib/tsan/rtl/tsan_sync.h
index fc8fa288a841..67d3c0b5e7dd 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_sync.h
+++ b/compiler-rt/lib/tsan/rtl/tsan_sync.h
@@ -16,8 +16,9 @@
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_deadlock_detector_interface.h"
 #include "tsan_defs.h"
-#include "tsan_clock.h"
 #include "tsan_dense_alloc.h"
+#include "tsan_shadow.h"
+#include "tsan_vector_clock.h"
 
 namespace __tsan {
 
@@ -53,34 +54,18 @@ struct SyncVar {
 
   uptr addr;  // overwritten by DenseSlabAlloc freelist
   Mutex mtx;
-  u64 uid;  // Globally unique id.
   StackID creation_stack_id;
   Tid owner_tid;  // Set only by exclusive owners.
-  u64 last_lock;
+  FastState last_lock;
   int recursion;
   atomic_uint32_t flags;
   u32 next;  // in MetaMap
   DDMutex dd;
-  SyncClock read_clock;  // Used for rw mutexes only.
-  // The clock is placed last, so that it is situated on a different cache line
-  // with the mtx. This reduces contention for hot sync objects.
-  SyncClock clock;
+  VectorClock *read_clock;  // Used for rw mutexes only.
+  VectorClock *clock;
 
-  void Init(ThreadState *thr, uptr pc, uptr addr, u64 uid, bool save_stack);
-  void Reset(Processor *proc);
-
-  u64 GetId() const {
-    // 48 lsb is addr, then 14 bits is low part of uid, then 2 zero bits.
-    return GetLsb((u64)addr | (uid << 48), 60);
-  }
-  bool CheckId(u64 uid) const {
-    CHECK_EQ(uid, GetLsb(uid, 14));
-    return GetLsb(this->uid, 14) == uid;
-  }
-  static uptr SplitId(u64 id, u64 *uid) {
-    *uid = id >> 48;
-    return (uptr)GetLsb(id, 48);
-  }
+  void Init(ThreadState *thr, uptr pc, uptr addr, bool save_stack);
+  void Reset();
 
   bool IsFlagSet(u32 f) const {
     return atomic_load_relaxed(&flags) & f;
@@ -110,9 +95,20 @@ class MetaMap {
   MetaMap();
 
   void AllocBlock(ThreadState *thr, uptr pc, uptr p, uptr sz);
-  uptr FreeBlock(Processor *proc, uptr p);
-  bool FreeRange(Processor *proc, uptr p, uptr sz);
-  void ResetRange(Processor *proc, uptr p, uptr sz);
+
+  // FreeBlock resets all sync objects in the range if reset=true and must not
+  // run concurrently with ResetClocks which resets all sync objects
+  // w/o any synchronization (as part of DoReset).
+  // If we don't have a thread slot (very early/late in thread lifetime or
+  // Go/Java callbacks) or the slot is not locked, then reset must be set to
+  // false. In such case sync object clocks will be reset later (when it's
+  // reused or during the next ResetClocks).
+  uptr FreeBlock(Processor *proc, uptr p, bool reset);
+  bool FreeRange(Processor *proc, uptr p, uptr sz, bool reset);
+  void ResetRange(Processor *proc, uptr p, uptr sz, bool reset);
+  // Reset vector clocks of all sync objects.
+  // Must be called when no other threads access sync objects.
+  void ResetClocks();
   MBlock* GetBlock(uptr p);
 
   SyncVar *GetSyncOrCreate(ThreadState *thr, uptr pc, uptr addr,
@@ -142,7 +138,6 @@ class MetaMap {
   typedef DenseSlabAlloc<SyncVar, 1 << 20, 1 << 10, kFlagMask> SyncAlloc;
   BlockAlloc block_alloc_;
   SyncAlloc sync_alloc_;
-  atomic_uint64_t uid_gen_;
 
   SyncVar *GetSync(ThreadState *thr, uptr pc, uptr addr, bool create,
                    bool save_stack);
diff --git a/compiler-rt/lib/tsan/rtl/tsan_trace.h b/compiler-rt/lib/tsan/rtl/tsan_trace.h
index ffc8c991ece0..01bb7b34f43a 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_trace.h
+++ b/compiler-rt/lib/tsan/rtl/tsan_trace.h
@@ -19,57 +19,6 @@
 
 namespace __tsan {
 
-const int kTracePartSizeBits = 13;
-const int kTracePartSize = 1 << kTracePartSizeBits;
-const int kTraceParts = 2 * 1024 * 1024 / kTracePartSize;
-const int kTraceSize = kTracePartSize * kTraceParts;
-
-// Must fit into 3 bits.
-enum EventType {
-  EventTypeMop,
-  EventTypeFuncEnter,
-  EventTypeFuncExit,
-  EventTypeLock,
-  EventTypeUnlock,
-  EventTypeRLock,
-  EventTypeRUnlock
-};
-
-// Represents a thread event (from most significant bit):
-// u64 typ  : 3;   // EventType.
-// u64 addr : 61;  // Associated pc.
-typedef u64 Event;
-
-const uptr kEventPCBits = 61;
-
-struct TraceHeader {
-#if !SANITIZER_GO
-  BufferedStackTrace stack0;  // Start stack for the trace.
-#else
-  VarSizeStackTrace stack0;
-#endif
-  u64        epoch0;  // Start epoch for the trace.
-  MutexSet   mset0;
-
-  TraceHeader() : stack0(), epoch0() {}
-};
-
-struct Trace {
-  Mutex mtx;
-#if !SANITIZER_GO
-  // Must be last to catch overflow as paging fault.
-  // Go shadow stack is dynamically allocated.
-  uptr shadow_stack[kShadowStackSize];
-#endif
-  // Must be the last field, because we unmap the unused part in
-  // CreateThreadContext.
-  TraceHeader headers[kTraceParts];
-
-  Trace() : mtx(MutexTypeTrace) {}
-};
-
-namespace v3 {
-
 enum class EventType : u64 {
   kAccessExt,
   kAccessRange,
@@ -217,6 +166,7 @@ struct Trace;
 struct TraceHeader {
   Trace* trace = nullptr;  // back-pointer to Trace containing this part
   INode trace_parts;       // in Trace::parts
+  INode global;            // in Contex::trace_part_recycle
 };
 
 struct TracePart : TraceHeader {
@@ -239,13 +189,26 @@ static_assert(sizeof(TracePart) == TracePart::kByteSize, "bad TracePart size");
 struct Trace {
   Mutex mtx;
   IList<TraceHeader, &TraceHeader::trace_parts, TracePart> parts;
-  Event* final_pos =
-      nullptr;  // final position in the last part for finished threads
+  // First node non-queued into ctx->trace_part_recycle.
+  TracePart* local_head;
+  // Final position in the last part for finished threads.
+  Event* final_pos = nullptr;
+  // Number of trace parts allocated on behalf of this trace specifically.
+  // Total number of parts in this trace can be larger if we retake some
+  // parts from other traces.
+  uptr parts_allocated = 0;
 
   Trace() : mtx(MutexTypeTrace) {}
-};
 
-}  // namespace v3
+  // We need at least 3 parts per thread, because we want to keep at last
+  // 2 parts per thread that are not queued into ctx->trace_part_recycle
+  // (the current one being filled and one full part that ensures that
+  // we always have at least one part worth of previous memory accesses).
+  static constexpr uptr kMinParts = 3;
+
+  static constexpr uptr kFinishedThreadLo = 16;
+  static constexpr uptr kFinishedThreadHi = 64;
+};
 
 }  // namespace __tsan
 
diff --git a/compiler-rt/lib/xray/xray_allocator.h b/compiler-rt/lib/xray/xray_allocator.h
index 4b42c473261d..0284f4299fb1 100644
--- a/compiler-rt/lib/xray/xray_allocator.h
+++ b/compiler-rt/lib/xray/xray_allocator.h
@@ -65,9 +65,9 @@ template <class T> T *allocate() XRAY_NEVER_INSTRUMENT {
   int ErrNo = 0;
   if (UNLIKELY(internal_iserror(B, &ErrNo))) {
     if (Verbosity())
-      Report(
-          "XRay Profiling: Failed to allocate memory of size %d; Error = %d.\n",
-          RoundedSize, B);
+      Report("XRay Profiling: Failed to allocate memory of size %zu; Error = "
+             "%zu\n",
+             RoundedSize, B);
     return nullptr;
   }
 #endif
@@ -114,9 +114,9 @@ T *allocateBuffer(size_t S) XRAY_NEVER_INSTRUMENT {
   int ErrNo = 0;
   if (UNLIKELY(internal_iserror(B, &ErrNo))) {
     if (Verbosity())
-      Report(
-          "XRay Profiling: Failed to allocate memory of size %d; Error = %d.\n",
-          RoundedSize, B);
+      Report("XRay Profiling: Failed to allocate memory of size %zu; Error = "
+             "%zu\n",
+             RoundedSize, B);
     return nullptr;
   }
 #endif
@@ -183,7 +183,7 @@ private:
       BackingStore = allocateBuffer(MaxMemory);
       if (BackingStore == nullptr) {
         if (Verbosity())
-          Report("XRay Profiling: Failed to allocate memory for allocator.\n");
+          Report("XRay Profiling: Failed to allocate memory for allocator\n");
         return nullptr;
       }
 
@@ -198,7 +198,7 @@ private:
         AlignedNextBlock = BackingStore = nullptr;
         if (Verbosity())
           Report("XRay Profiling: Cannot obtain enough memory from "
-                 "preallocated region.\n");
+                 "preallocated region\n");
         return nullptr;
       }
 
diff --git a/compiler-rt/lib/xray/xray_basic_logging.cpp b/compiler-rt/lib/xray/xray_basic_logging.cpp
index a58ae9b5e267..6e83252a0516 100644
--- a/compiler-rt/lib/xray/xray_basic_logging.cpp
+++ b/compiler-rt/lib/xray/xray_basic_logging.cpp
@@ -345,12 +345,12 @@ static void TLDDestructor(void *P) XRAY_NEVER_INSTRUMENT {
     if (TLD.ShadowStack)
       InternalFree(TLD.ShadowStack);
     if (Verbosity())
-      Report("Cleaned up log for TID: %d\n", GetTid());
+      Report("Cleaned up log for TID: %llu\n", GetTid());
   });
 
   if (TLD.LogWriter == nullptr || TLD.BufferOffset == 0) {
     if (Verbosity())
-      Report("Skipping buffer for TID: %d; Offset = %llu\n", GetTid(),
+      Report("Skipping buffer for TID: %llu; Offset = %zu\n", GetTid(),
              TLD.BufferOffset);
     return;
   }
diff --git a/compiler-rt/lib/xray/xray_hexagon.cpp b/compiler-rt/lib/xray/xray_hexagon.cpp
new file mode 100644
index 000000000000..7f127b2b499c
--- /dev/null
+++ b/compiler-rt/lib/xray/xray_hexagon.cpp
@@ -0,0 +1,168 @@
+//===-- xray_hexagon.cpp --------------------------------------*- C++ ---*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of XRay, a dynamic runtime instrumentation system.
+//
+// Implementation of hexagon-specific routines (32-bit).
+//
+//===----------------------------------------------------------------------===//
+#include "sanitizer_common/sanitizer_common.h"
+#include "xray_defs.h"
+#include "xray_interface_internal.h"
+#include <assert.h>
+#include <atomic>
+
+namespace __xray {
+
+// The machine codes for some instructions used in runtime patching.
+enum PatchOpcodes : uint32_t {
+  PO_JUMPI_14 = 0x5800c00a, // jump #0x014 (PC + 0x014)
+  PO_CALLR_R6 = 0x50a6c000, // indirect call: callr r6
+  PO_TFR_IMM = 0x78000000,  // transfer immed
+                            // ICLASS 0x7 - S2-type A-type
+  PO_IMMEXT = 0x00000000, // constant extender
+};
+
+enum PacketWordParseBits : uint32_t {
+  PP_DUPLEX = 0x00 << 14,
+  PP_NOT_END = 0x01 << 14,
+  PP_PACKET_END = 0x03 << 14,
+};
+
+enum RegNum : uint32_t {
+  RN_R6 = 0x6,
+  RN_R7 = 0x7,
+};
+
+inline static uint32_t
+encodeExtendedTransferImmediate(uint32_t Imm, RegNum DestReg,
+                                bool PacketEnd = false) XRAY_NEVER_INSTRUMENT {
+  static const uint32_t REG_MASK = 0x1f;
+  assert((DestReg & (~REG_MASK)) == 0);
+  // The constant-extended register transfer encodes the 6 least
+  // significant bits of the effective constant:
+  Imm = Imm & 0x03f;
+  const PacketWordParseBits ParseBits = PacketEnd ? PP_PACKET_END : PP_NOT_END;
+
+  return PO_TFR_IMM | ParseBits | (Imm << 5) | (DestReg & REG_MASK);
+}
+
+inline static uint32_t
+encodeConstantExtender(uint32_t Imm) XRAY_NEVER_INSTRUMENT {
+  // Bits   Name      Description
+  // -----  -------   ------------------------------------------
+  // 31:28  ICLASS    Instruction class = 0000
+  // 27:16  high      High 12 bits of 26-bit constant extension
+  // 15:14  Parse     Parse bits
+  // 13:0   low       Low 14 bits of 26-bit constant extension
+  static const uint32_t IMM_MASK_LOW = 0x03fff;
+  static const uint32_t IMM_MASK_HIGH = 0x00fff << 14;
+
+  // The extender encodes the 26 most significant bits of the effective
+  // constant:
+  Imm = Imm >> 6;
+
+  const uint32_t high = (Imm & IMM_MASK_HIGH) << 16;
+  const uint32_t low = Imm & IMM_MASK_LOW;
+
+  return PO_IMMEXT | high | PP_NOT_END | low;
+}
+
+static void WriteInstFlushCache(void *Addr, uint32_t NewInstruction) {
+  asm volatile("icinva(%[inst_addr])\n\t"
+               "isync\n\t"
+               "memw(%[inst_addr]) = %[new_inst]\n\t"
+               "dccleaninva(%[inst_addr])\n\t"
+               "syncht\n\t"
+               :
+               : [ inst_addr ] "r"(Addr), [ new_inst ] "r"(NewInstruction)
+               : "memory");
+}
+
+inline static bool patchSled(const bool Enable, const uint32_t FuncId,
+                             const XRaySledEntry &Sled,
+                             void (*TracingHook)()) XRAY_NEVER_INSTRUMENT {
+  // When |Enable| == true,
+  // We replace the following compile-time stub (sled):
+  //
+  // .L_xray_sled_N:
+  // <xray_sled_base>:
+  // {  jump .Ltmp0 }
+  // {  nop
+  //    nop
+  //    nop
+  //    nop }
+  // .Ltmp0:
+
+  // With the following runtime patch:
+  //
+  // xray_sled_n (32-bit):
+  //
+  // <xray_sled_n>:
+  // {  immext(#...) // upper 26-bits of func id
+  //    r7 = ##...   // lower  6-bits of func id
+  //    immext(#...) // upper 26-bits of trampoline
+  //    r6 = ##... }  // lower 6 bits of trampoline
+  // {  callr r6 }
+  //
+  // When |Enable|==false, we set back the first instruction in the sled to be
+  // {  jump .Ltmp0 }
+
+  uint32_t *FirstAddress = reinterpret_cast<uint32_t *>(Sled.address());
+  if (Enable) {
+    uint32_t *CurAddress = FirstAddress + 1;
+    *CurAddress = encodeExtendedTransferImmediate(FuncId, RN_R7);
+    CurAddress++;
+    *CurAddress = encodeConstantExtender(reinterpret_cast<uint32_t>(TracingHook));
+    CurAddress++;
+    *CurAddress =
+        encodeExtendedTransferImmediate(reinterpret_cast<uint32_t>(TracingHook), RN_R6, true);
+    CurAddress++;
+
+    *CurAddress = uint32_t(PO_CALLR_R6);
+
+    WriteInstFlushCache(FirstAddress, uint32_t(encodeConstantExtender(FuncId)));
+  } else {
+    WriteInstFlushCache(FirstAddress, uint32_t(PatchOpcodes::PO_JUMPI_14));
+  }
+  return true;
+}
+
+bool patchFunctionEntry(const bool Enable, const uint32_t FuncId,
+                        const XRaySledEntry &Sled,
+                        void (*Trampoline)()) XRAY_NEVER_INSTRUMENT {
+  return patchSled(Enable, FuncId, Sled, Trampoline);
+}
+
+bool patchFunctionExit(const bool Enable, const uint32_t FuncId,
+                       const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT {
+  return patchSled(Enable, FuncId, Sled, __xray_FunctionExit);
+}
+
+bool patchFunctionTailExit(const bool Enable, const uint32_t FuncId,
+                           const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT {
+  return patchSled(Enable, FuncId, Sled, __xray_FunctionExit);
+}
+
+bool patchCustomEvent(const bool Enable, const uint32_t FuncId,
+                      const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT {
+  // FIXME: Implement in hexagon?
+  return false;
+}
+
+bool patchTypedEvent(const bool Enable, const uint32_t FuncId,
+                     const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT {
+  // FIXME: Implement in hexagon?
+  return false;
+}
+
+} // namespace __xray
+
+extern "C" void __xray_ArgLoggerEntry() XRAY_NEVER_INSTRUMENT {
+  // FIXME: this will have to be implemented in the trampoline assembly file
+}
diff --git a/compiler-rt/lib/xray/xray_interface.cpp b/compiler-rt/lib/xray/xray_interface.cpp
index ddf184c9b857..73e67618c9d5 100644
--- a/compiler-rt/lib/xray/xray_interface.cpp
+++ b/compiler-rt/lib/xray/xray_interface.cpp
@@ -14,7 +14,7 @@
 
 #include "xray_interface_internal.h"
 
-#include <cstdint>
+#include <cinttypes>
 #include <cstdio>
 #include <errno.h>
 #include <limits>
@@ -52,6 +52,8 @@ static const int16_t cSledLength = 48;
 static const int16_t cSledLength = 64;
 #elif defined(__powerpc64__)
 static const int16_t cSledLength = 8;
+#elif defined(__hexagon__)
+static const int16_t cSledLength = 20;
 #else
 #error "Unsupported CPU Architecture"
 #endif /* CPU architecture */
@@ -169,7 +171,8 @@ bool patchSled(const XRaySledEntry &Sled, bool Enable,
     Success = patchTypedEvent(Enable, FuncId, Sled);
     break;
   default:
-    Report("Unsupported sled kind '%d' @%04x\n", Sled.Address, int(Sled.Kind));
+    Report("Unsupported sled kind '%" PRIu64 "' @%04x\n", Sled.Address,
+           int(Sled.Kind));
     return false;
   }
   return Success;
@@ -305,7 +308,7 @@ XRayPatchingStatus controlPatching(bool Enable) XRAY_NEVER_INSTRUMENT {
                               ? flags()->xray_page_size_override
                               : GetPageSizeCached();
   if ((PageSize == 0) || ((PageSize & (PageSize - 1)) != 0)) {
-    Report("System page size is not a power of two: %lld\n", PageSize);
+    Report("System page size is not a power of two: %zu\n", PageSize);
     return XRayPatchingStatus::FAILED;
   }
 
@@ -356,7 +359,7 @@ XRayPatchingStatus mprotectAndPatchFunction(int32_t FuncId,
                               ? flags()->xray_page_size_override
                               : GetPageSizeCached();
   if ((PageSize == 0) || ((PageSize & (PageSize - 1)) != 0)) {
-    Report("Provided page size is not a power of two: %lld\n", PageSize);
+    Report("Provided page size is not a power of two: %zu\n", PageSize);
     return XRayPatchingStatus::FAILED;
   }
 
diff --git a/compiler-rt/lib/xray/xray_trampoline_hexagon.S b/compiler-rt/lib/xray/xray_trampoline_hexagon.S
new file mode 100644
index 000000000000..c87ec4bed1f9
--- /dev/null
+++ b/compiler-rt/lib/xray/xray_trampoline_hexagon.S
@@ -0,0 +1,99 @@
+//===-- xray_trampoline_hexagon.s -------------------------------*- ASM -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of XRay, a dynamic runtime instrumentation system.
+//
+// This implements the hexagon-specific assembler for the trampolines.
+//
+//===----------------------------------------------------------------------===//
+
+#include "../builtins/assembly.h"
+#include "../sanitizer_common/sanitizer_asm.h"
+
+.macro SAVE_REGISTERS
+memw(sp+#0)=r0
+memw(sp+#4)=r1
+memw(sp+#8)=r2
+memw(sp+#12)=r3
+memw(sp+#16)=r4
+.endm
+.macro RESTORE_REGISTERS
+r0=memw(sp+#0)
+r1=memw(sp+#4)
+r2=memw(sp+#8)
+r3=memw(sp+#12)
+r4=memw(sp+#16)
+.endm
+
+.macro CALL_PATCHED_FUNC entry_type
+	// if (xray::XRayPatchedFunctionE != NULL)
+	//     xray::XRayPatchedFunctionE(FuncType);
+
+	r8 = #ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)
+
+	// The patched sled puts the function type
+	// into r6.  Move it into r0 to pass it to
+	// the patched function.
+	{ r0 = r6
+          r1 = \entry_type
+          p0 = !cmp.eq(r8, #0)
+	  if (p0) callr r8 }
+.endm
+
+	.text
+	.globl ASM_SYMBOL(__xray_FunctionEntry)
+	ASM_HIDDEN(__xray_FunctionEntry)
+	ASM_TYPE_FUNCTION(__xray_FunctionEntry)
+# LLVM-MCA-BEGIN __xray_FunctionEntry
+ASM_SYMBOL(__xray_FunctionEntry):
+	CFI_STARTPROC
+	SAVE_REGISTERS
+
+	CALL_PATCHED_FUNC #0  // XRayEntryType::ENTRY
+.Ltmp0:
+	RESTORE_REGISTERS
+	// return
+# LLVM-MCA-END
+	ASM_SIZE(__xray_FunctionEntry)
+	CFI_ENDPROC
+
+
+	.globl ASM_SYMBOL(__xray_FunctionExit)
+	ASM_HIDDEN(__xray_FunctionExit)
+	ASM_TYPE_FUNCTION(__xray_FunctionExit)
+# LLVM-MCA-BEGIN __xray_FunctionExit
+ASM_SYMBOL(__xray_FunctionExit):
+	CFI_STARTPROC
+	SAVE_REGISTERS
+
+	CALL_PATCHED_FUNC #1  // XRayEntryType::EXIT
+.Ltmp1:
+	RESTORE_REGISTERS
+	// return
+	jumpr r31
+# LLVM-MCA-END
+	ASM_SIZE(__xray_FunctionExit)
+	CFI_ENDPROC
+
+
+	.globl ASM_SYMBOL(__xray_FunctionTailExit)
+	ASM_HIDDEN(__xray_FunctionTailExit)
+	ASM_TYPE_FUNCTION(__xray_FunctionTailExit)
+# LLVM-MCA-BEGIN __xray_FunctionTailExit
+ASM_SYMBOL(__xray_FunctionTailExit):
+	CFI_STARTPROC
+	SAVE_REGISTERS
+
+	CALL_PATCHED_FUNC #2  // XRayEntryType::TAIL
+.Ltmp2:
+	RESTORE_REGISTERS
+	// return
+	jumpr r31
+# LLVM-MCA-END
+	ASM_SIZE(__xray_FunctionTailExit)
+	CFI_ENDPROC
diff --git a/compiler-rt/lib/xray/xray_tsc.h b/compiler-rt/lib/xray/xray_tsc.h
index bd7e1911abb3..58347dca5f7a 100644
--- a/compiler-rt/lib/xray/xray_tsc.h
+++ b/compiler-rt/lib/xray/xray_tsc.h
@@ -42,7 +42,8 @@ inline uint64_t getTSCFrequency() XRAY_NEVER_INSTRUMENT {
 #include "xray_x86_64.inc"
 #elif defined(__powerpc64__)
 #include "xray_powerpc64.inc"
-#elif defined(__arm__) || defined(__aarch64__) || defined(__mips__)
+#elif defined(__arm__) || defined(__aarch64__) || defined(__mips__) ||         \
+    defined(__hexagon__)
 // Emulated TSC.
 // There is no instruction like RDTSCP in user mode on ARM. ARM's CP15 does
 //   not have a constant frequency like TSC on x86(_64), it may go faster
diff --git a/libcxx/CREDITS.TXT b/libcxx/CREDITS.TXT
index fc442f4db1a1..cd5bc08a60fc 100644
--- a/libcxx/CREDITS.TXT
+++ b/libcxx/CREDITS.TXT
@@ -12,6 +12,9 @@ N: Saleem Abdulrasool
 E: compnerd@compnerd.org
 D: Minor patches and Linux fixes.
 
+N: Ulf Adams
+D: Invented the Ryu and Ryu Printf algorithms used in floating-point to_chars, and wrote the initial code.
+
 N: Muiez Ahmed
 E: muiez@ibm.com
 D: z/OS port.
@@ -28,6 +31,9 @@ N: Holger Arnold
 E: holgerar@gmail.com
 D: Minor fix.
 
+N: Jorg Brown
+D: Ported floating-point to_chars from MSVC to libc++.
+
 N: David Chisnall
 E: theraven at theravensnest dot org
 D: FreeBSD and Solaris ports, libcxxrt support, some atomics work.
@@ -81,6 +87,14 @@ N: Argyrios Kyrtzidis
 E: kyrtzidis@apple.com
 D: Bug fixes.
 
+N: Stephan T. Lavavej
+E: stl@microsoft.com
+E: stl@nuwen.net
+D: Implemented floating-point to_chars.
+
+N: Microsoft Corporation
+D: Contributed floating-point to_chars.
+
 N: Bruce Mitchener, Jr.
 E: bruce.mitchener@gmail.com
 D: Emscripten-related changes.
@@ -152,6 +166,7 @@ D: Minor bug fix.
 N: Mark de Wever
 E: koraq at xs4all dot nl
 D: Format library support.
+D: Finalized the porting of MSVC's to_chars to libc++.
 
 N: Zhang Xiongpang
 E: zhangxiongpang@gmail.com
diff --git a/libcxx/include/__availability b/libcxx/include/__availability
index 87d43ed414bf..4652a6fd91b4 100644
--- a/libcxx/include/__availability
+++ b/libcxx/include/__availability
@@ -129,6 +129,10 @@
     // This controls the availability of std::to_chars.
 #   define _LIBCPP_AVAILABILITY_TO_CHARS
 
+    // This controls the availability of floating-point std::to_chars functions.
+    // These overloads were added later than the integer overloads.
+#   define _LIBCPP_AVAILABILITY_TO_CHARS_FLOATING_POINT
+
     // This controls the availability of the C++20 synchronization library,
     // which requires shared library support for various operations
     // (see libcxx/src/atomic.cpp).
@@ -222,6 +226,9 @@
 #   define _LIBCPP_AVAILABILITY_TO_CHARS                                        \
         _LIBCPP_AVAILABILITY_FILESYSTEM
 
+#   define _LIBCPP_AVAILABILITY_TO_CHARS_FLOATING_POINT                         \
+        __attribute__((unavailable))
+
 #   define _LIBCPP_AVAILABILITY_SYNC                                            \
         __attribute__((availability(macosx,strict,introduced=11.0)))            \
         __attribute__((availability(ios,strict,introduced=14.0)))               \
diff --git a/libcxx/include/__compare/strong_order.h b/libcxx/include/__compare/strong_order.h
index e49b2d45de45..42f060387d59 100644
--- a/libcxx/include/__compare/strong_order.h
+++ b/libcxx/include/__compare/strong_order.h
@@ -86,11 +86,11 @@ namespace __strong_order {
                 bool __u_is_nan = _VSTD::isnan(__u);
                 bool __t_is_negative = _VSTD::signbit(__t);
                 bool __u_is_negative = _VSTD::signbit(__u);
-                using _IntType = std::conditional_t<
-                    sizeof(__t) == sizeof(int32_t), int32_t, std::conditional_t<
+                using _IntType = conditional_t<
+                    sizeof(__t) == sizeof(int32_t), int32_t, conditional_t<
                     sizeof(__t) == sizeof(int64_t), int64_t, void>
                 >;
-                if constexpr (std::is_same_v<_IntType, void>) {
+                if constexpr (is_same_v<_IntType, void>) {
                     static_assert(sizeof(_Dp) == 0, "std::strong_order is unimplemented for this floating-point type");
                 } else if (__t_is_nan && __u_is_nan) {
                     // Order by sign bit, then by "payload bits" (we'll just use bit_cast).
diff --git a/libcxx/include/__compare/weak_order.h b/libcxx/include/__compare/weak_order.h
index f67416ed3ebe..ce914b232108 100644
--- a/libcxx/include/__compare/weak_order.h
+++ b/libcxx/include/__compare/weak_order.h
@@ -42,13 +42,13 @@ namespace __weak_order {
         _LIBCPP_HIDE_FROM_ABI static constexpr weak_ordering
         __go(_Tp&& __t, _Up&& __u, __priority_tag<2>) noexcept
         {
-            std::partial_ordering __po = (__t <=> __u);
-            if (__po == std::partial_ordering::less) {
-                return std::weak_ordering::less;
-            } else if (__po == std::partial_ordering::equivalent) {
-                return std::weak_ordering::equivalent;
-            } else if (__po == std::partial_ordering::greater) {
-                return std::weak_ordering::greater;
+            partial_ordering __po = (__t <=> __u);
+            if (__po == partial_ordering::less) {
+                return weak_ordering::less;
+            } else if (__po == partial_ordering::equivalent) {
+                return weak_ordering::equivalent;
+            } else if (__po == partial_ordering::greater) {
+                return weak_ordering::greater;
             } else {
                 // Otherwise, at least one of them is a NaN.
                 bool __t_is_nan = _VSTD::isnan(__t);
diff --git a/libcxx/include/__config b/libcxx/include/__config
index da03e877f753..720e12eac0dd 100644
--- a/libcxx/include/__config
+++ b/libcxx/include/__config
@@ -333,18 +333,43 @@
 #  define _LIBCPP_SHORT_WCHAR   1
 #endif
 
+// Libc++ supports various implementations of std::random_device.
+//
+// _LIBCPP_USING_DEV_RANDOM
+//      Read entropy from the given file, by default `/dev/urandom`.
+//      If a token is provided, it is assumed to be the path to a file
+//      to read entropy from. This is the default behavior if nothing
+//      else is specified. This implementation requires storing state
+//      inside `std::random_device`.
+//
+// _LIBCPP_USING_ARC4_RANDOM
+//      Use arc4random(). This allows obtaining random data even when
+//      using sandboxing mechanisms. On some platforms like Apple, this
+//      is the recommended source of entropy for user-space programs.
+//      When this option is used, the token passed to `std::random_device`'s
+//      constructor *must* be "/dev/urandom" -- anything else is an error.
+//
+// _LIBCPP_USING_GETENTROPY
+//      Use getentropy().
+//      When this option is used, the token passed to `std::random_device`'s
+//      constructor *must* be "/dev/urandom" -- anything else is an error.
+//
+// _LIBCPP_USING_NACL_RANDOM
+//      NaCl's sandbox (which PNaCl also runs in) doesn't allow filesystem access,
+//      including accesses to the special files under `/dev`. This implementation
+//      uses the NaCL syscall `nacl_secure_random_init()` to get entropy.
+//      When this option is used, the token passed to `std::random_device`'s
+//      constructor *must* be "/dev/urandom" -- anything else is an error.
+//
+// _LIBCPP_USING_WIN32_RANDOM
+//      Use rand_s(), for use on Windows.
+//      When this option is used, the token passed to `std::random_device`'s
+//      constructor *must* be "/dev/urandom" -- anything else is an error.
 #if defined(__OpenBSD__)
-   // Certain architectures provide arc4random(). Prefer using
-   // arc4random() over /dev/{u,}random to make it possible to obtain
-   // random data even when using sandboxing mechanisms such as chroots,
-   // Capsicum, etc.
 #  define _LIBCPP_USING_ARC4_RANDOM
 #elif defined(__Fuchsia__) || defined(__wasi__)
 #  define _LIBCPP_USING_GETENTROPY
 #elif defined(__native_client__)
-   // NaCl's sandbox (which PNaCl also runs in) doesn't allow filesystem access,
-   // including accesses to the special files under /dev. C++11's
-   // std::random_device is instead exposed through a NaCl syscall.
 #  define _LIBCPP_USING_NACL_RANDOM
 #elif defined(_LIBCPP_WIN32API)
 #  define _LIBCPP_USING_WIN32_RANDOM
@@ -837,12 +862,6 @@ typedef unsigned int   char32_t;
 #define _LIBCPP_HAS_NO_RANGES
 #endif
 
-#ifdef _LIBCPP_CXX03_LANG
-#  define _LIBCPP_DEFAULT {}
-#else
-#  define _LIBCPP_DEFAULT = default;
-#endif
-
 #ifdef __GNUC__
 #  define _LIBCPP_NOALIAS __attribute__((__malloc__))
 #else
diff --git a/libcxx/include/__debug b/libcxx/include/__debug
index e25039c088c6..42f6cef4c07f 100644
--- a/libcxx/include/__debug
+++ b/libcxx/include/__debug
@@ -34,7 +34,7 @@
 #   define _LIBCPP_DEBUG_ASSERT(x, m) ((void)0)
 #   define _LIBCPP_ASSERT_IMPL(x, m) ((x) ? (void)0 : _VSTD::__libcpp_debug_function(_VSTD::__libcpp_debug_info(__FILE__, __LINE__, #x, m)))
 #elif _LIBCPP_DEBUG_LEVEL == 2
-#   define _LIBCPP_DEBUG_ASSERT(x, m) _LIBCPP_ASSERT(x, m)
+#   define _LIBCPP_DEBUG_ASSERT(x, m) _LIBCPP_ASSERT(__libcpp_is_constant_evaluated() || (x), m)
 #   define _LIBCPP_ASSERT_IMPL(x, m) ((x) ? (void)0 : _VSTD::__libcpp_debug_function(_VSTD::__libcpp_debug_info(__FILE__, __LINE__, #x, m)))
 #else
 #   error _LIBCPP_DEBUG_LEVEL must be one of 0, 1, 2
diff --git a/libcxx/include/__filesystem/copy_options.h b/libcxx/include/__filesystem/copy_options.h
new file mode 100644
index 000000000000..c0140d45717b
--- /dev/null
+++ b/libcxx/include/__filesystem/copy_options.h
@@ -0,0 +1,80 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FILESYSTEM_COPY_OPTIONS_H
+#define _LIBCPP___FILESYSTEM_COPY_OPTIONS_H
+
+#include <__availability>
+#include <__config>
+
+#ifndef _LIBCPP_CXX03_LANG
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_PUSH
+
+enum class _LIBCPP_ENUM_VIS copy_options : unsigned short {
+  none = 0,
+  skip_existing = 1,
+  overwrite_existing = 2,
+  update_existing = 4,
+  recursive = 8,
+  copy_symlinks = 16,
+  skip_symlinks = 32,
+  directories_only = 64,
+  create_symlinks = 128,
+  create_hard_links = 256,
+  __in_recursive_copy = 512,
+};
+
+_LIBCPP_INLINE_VISIBILITY
+inline constexpr copy_options operator&(copy_options _LHS, copy_options _RHS) {
+  return static_cast<copy_options>(static_cast<unsigned short>(_LHS) &
+                                   static_cast<unsigned short>(_RHS));
+}
+
+_LIBCPP_INLINE_VISIBILITY
+inline constexpr copy_options operator|(copy_options _LHS, copy_options _RHS) {
+  return static_cast<copy_options>(static_cast<unsigned short>(_LHS) |
+                                   static_cast<unsigned short>(_RHS));
+}
+
+_LIBCPP_INLINE_VISIBILITY
+inline constexpr copy_options operator^(copy_options _LHS, copy_options _RHS) {
+  return static_cast<copy_options>(static_cast<unsigned short>(_LHS) ^
+                                   static_cast<unsigned short>(_RHS));
+}
+
+_LIBCPP_INLINE_VISIBILITY
+inline constexpr copy_options operator~(copy_options _LHS) {
+  return static_cast<copy_options>(~static_cast<unsigned short>(_LHS));
+}
+
+_LIBCPP_INLINE_VISIBILITY
+inline copy_options& operator&=(copy_options& _LHS, copy_options _RHS) {
+  return _LHS = _LHS & _RHS;
+}
+
+_LIBCPP_INLINE_VISIBILITY
+inline copy_options& operator|=(copy_options& _LHS, copy_options _RHS) {
+  return _LHS = _LHS | _RHS;
+}
+
+_LIBCPP_INLINE_VISIBILITY
+inline copy_options& operator^=(copy_options& _LHS, copy_options _RHS) {
+  return _LHS = _LHS ^ _RHS;
+}
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_POP
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
+
+#endif // _LIBCPP_CXX03_LANG
+
+#endif // _LIBCPP___FILESYSTEM_COPY_OPTIONS_H
diff --git a/libcxx/include/__filesystem/directory_entry.h b/libcxx/include/__filesystem/directory_entry.h
new file mode 100644
index 000000000000..9efe19465428
--- /dev/null
+++ b/libcxx/include/__filesystem/directory_entry.h
@@ -0,0 +1,504 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FILESYSTEM_DIRECTORY_ENTRY_H
+#define _LIBCPP___FILESYSTEM_DIRECTORY_ENTRY_H
+
+#include <__availability>
+#include <__config>
+#include <__filesystem/path.h>
+#include <__filesystem/file_time_type.h>
+#include <__filesystem/filesystem_error.h>
+#include <__filesystem/file_status.h>
+#include <__filesystem/file_type.h>
+#include <__filesystem/operations.h>
+#include <__filesystem/perms.h>
+#include <__errc>
+#include <chrono>
+#include <cstdint>
+#include <cstdlib>
+#include <system_error>
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#ifndef _LIBCPP_CXX03_LANG
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_PUSH
+
+
+class directory_entry {
+  typedef _VSTD_FS::path _Path;
+
+public:
+  // constructors and destructors
+  directory_entry() noexcept = default;
+  directory_entry(directory_entry const&) = default;
+  directory_entry(directory_entry&&) noexcept = default;
+
+  _LIBCPP_INLINE_VISIBILITY
+  explicit directory_entry(_Path const& __p) : __p_(__p) {
+    error_code __ec;
+    __refresh(&__ec);
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  directory_entry(_Path const& __p, error_code& __ec) : __p_(__p) {
+    __refresh(&__ec);
+  }
+
+  ~directory_entry() {}
+
+  directory_entry& operator=(directory_entry const&) = default;
+  directory_entry& operator=(directory_entry&&) noexcept = default;
+
+  _LIBCPP_INLINE_VISIBILITY
+  void assign(_Path const& __p) {
+    __p_ = __p;
+    error_code __ec;
+    __refresh(&__ec);
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  void assign(_Path const& __p, error_code& __ec) {
+    __p_ = __p;
+    __refresh(&__ec);
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  void replace_filename(_Path const& __p) {
+    __p_.replace_filename(__p);
+    error_code __ec;
+    __refresh(&__ec);
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  void replace_filename(_Path const& __p, error_code& __ec) {
+    __p_ = __p_.parent_path() / __p;
+    __refresh(&__ec);
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  void refresh() { __refresh(); }
+
+  _LIBCPP_INLINE_VISIBILITY
+  void refresh(error_code& __ec) noexcept { __refresh(&__ec); }
+
+  _LIBCPP_INLINE_VISIBILITY
+  _Path const& path() const noexcept { return __p_; }
+
+  _LIBCPP_INLINE_VISIBILITY
+  operator const _Path&() const noexcept { return __p_; }
+
+  _LIBCPP_INLINE_VISIBILITY
+  bool exists() const { return _VSTD_FS::exists(file_status{__get_ft()}); }
+
+  _LIBCPP_INLINE_VISIBILITY
+  bool exists(error_code& __ec) const noexcept {
+    return _VSTD_FS::exists(file_status{__get_ft(&__ec)});
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  bool is_block_file() const { return __get_ft() == file_type::block; }
+
+  _LIBCPP_INLINE_VISIBILITY
+  bool is_block_file(error_code& __ec) const noexcept {
+    return __get_ft(&__ec) == file_type::block;
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  bool is_character_file() const { return __get_ft() == file_type::character; }
+
+  _LIBCPP_INLINE_VISIBILITY
+  bool is_character_file(error_code& __ec) const noexcept {
+    return __get_ft(&__ec) == file_type::character;
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  bool is_directory() const { return __get_ft() == file_type::directory; }
+
+  _LIBCPP_INLINE_VISIBILITY
+  bool is_directory(error_code& __ec) const noexcept {
+    return __get_ft(&__ec) == file_type::directory;
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  bool is_fifo() const { return __get_ft() == file_type::fifo; }
+
+  _LIBCPP_INLINE_VISIBILITY
+  bool is_fifo(error_code& __ec) const noexcept {
+    return __get_ft(&__ec) == file_type::fifo;
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  bool is_other() const { return _VSTD_FS::is_other(file_status{__get_ft()}); }
+
+  _LIBCPP_INLINE_VISIBILITY
+  bool is_other(error_code& __ec) const noexcept {
+    return _VSTD_FS::is_other(file_status{__get_ft(&__ec)});
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  bool is_regular_file() const { return __get_ft() == file_type::regular; }
+
+  _LIBCPP_INLINE_VISIBILITY
+  bool is_regular_file(error_code& __ec) const noexcept {
+    return __get_ft(&__ec) == file_type::regular;
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  bool is_socket() const { return __get_ft() == file_type::socket; }
+
+  _LIBCPP_INLINE_VISIBILITY
+  bool is_socket(error_code& __ec) const noexcept {
+    return __get_ft(&__ec) == file_type::socket;
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  bool is_symlink() const { return __get_sym_ft() == file_type::symlink; }
+
+  _LIBCPP_INLINE_VISIBILITY
+  bool is_symlink(error_code& __ec) const noexcept {
+    return __get_sym_ft(&__ec) == file_type::symlink;
+  }
+  _LIBCPP_INLINE_VISIBILITY
+  uintmax_t file_size() const { return __get_size(); }
+
+  _LIBCPP_INLINE_VISIBILITY
+  uintmax_t file_size(error_code& __ec) const noexcept {
+    return __get_size(&__ec);
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  uintmax_t hard_link_count() const { return __get_nlink(); }
+
+  _LIBCPP_INLINE_VISIBILITY
+  uintmax_t hard_link_count(error_code& __ec) const noexcept {
+    return __get_nlink(&__ec);
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  file_time_type last_write_time() const { return __get_write_time(); }
+
+  _LIBCPP_INLINE_VISIBILITY
+  file_time_type last_write_time(error_code& __ec) const noexcept {
+    return __get_write_time(&__ec);
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  file_status status() const { return __get_status(); }
+
+  _LIBCPP_INLINE_VISIBILITY
+  file_status status(error_code& __ec) const noexcept {
+    return __get_status(&__ec);
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  file_status symlink_status() const { return __get_symlink_status(); }
+
+  _LIBCPP_INLINE_VISIBILITY
+  file_status symlink_status(error_code& __ec) const noexcept {
+    return __get_symlink_status(&__ec);
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  bool operator<(directory_entry const& __rhs) const noexcept {
+    return __p_ < __rhs.__p_;
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  bool operator==(directory_entry const& __rhs) const noexcept {
+    return __p_ == __rhs.__p_;
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  bool operator!=(directory_entry const& __rhs) const noexcept {
+    return __p_ != __rhs.__p_;
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  bool operator<=(directory_entry const& __rhs) const noexcept {
+    return __p_ <= __rhs.__p_;
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  bool operator>(directory_entry const& __rhs) const noexcept {
+    return __p_ > __rhs.__p_;
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  bool operator>=(directory_entry const& __rhs) const noexcept {
+    return __p_ >= __rhs.__p_;
+  }
+
+private:
+  friend class directory_iterator;
+  friend class recursive_directory_iterator;
+  friend class __dir_stream;
+
+  enum _CacheType : unsigned char {
+    _Empty,
+    _IterSymlink,
+    _IterNonSymlink,
+    _RefreshSymlink,
+    _RefreshSymlinkUnresolved,
+    _RefreshNonSymlink
+  };
+
+  struct __cached_data {
+    uintmax_t __size_;
+    uintmax_t __nlink_;
+    file_time_type __write_time_;
+    perms __sym_perms_;
+    perms __non_sym_perms_;
+    file_type __type_;
+    _CacheType __cache_type_;
+
+    _LIBCPP_INLINE_VISIBILITY
+    __cached_data() noexcept { __reset(); }
+
+    _LIBCPP_INLINE_VISIBILITY
+    void __reset() {
+      __cache_type_ = _Empty;
+      __type_ = file_type::none;
+      __sym_perms_ = __non_sym_perms_ = perms::unknown;
+      __size_ = __nlink_ = uintmax_t(-1);
+      __write_time_ = file_time_type::min();
+    }
+  };
+
+  _LIBCPP_INLINE_VISIBILITY
+  static __cached_data __create_iter_result(file_type __ft) {
+    __cached_data __data;
+    __data.__type_ = __ft;
+    __data.__cache_type_ = [&]() {
+      switch (__ft) {
+      case file_type::none:
+        return _Empty;
+      case file_type::symlink:
+        return _IterSymlink;
+      default:
+        return _IterNonSymlink;
+      }
+    }();
+    return __data;
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  void __assign_iter_entry(_Path&& __p, __cached_data __dt) {
+    __p_ = _VSTD::move(__p);
+    __data_ = __dt;
+  }
+
+  _LIBCPP_FUNC_VIS
+  error_code __do_refresh() noexcept;
+
+  _LIBCPP_INLINE_VISIBILITY
+  static bool __is_dne_error(error_code const& __ec) {
+    if (!__ec)
+      return true;
+    switch (static_cast<errc>(__ec.value())) {
+    case errc::no_such_file_or_directory:
+    case errc::not_a_directory:
+      return true;
+    default:
+      return false;
+    }
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  void __handle_error(const char* __msg, error_code* __dest_ec,
+                      error_code const& __ec, bool __allow_dne = false) const {
+    if (__dest_ec) {
+      *__dest_ec = __ec;
+      return;
+    }
+    if (__ec && (!__allow_dne || !__is_dne_error(__ec)))
+      __throw_filesystem_error(__msg, __p_, __ec);
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  void __refresh(error_code* __ec = nullptr) {
+    __handle_error("in directory_entry::refresh", __ec, __do_refresh(),
+                   /*allow_dne*/ true);
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  file_type __get_sym_ft(error_code* __ec = nullptr) const {
+    switch (__data_.__cache_type_) {
+    case _Empty:
+      return __symlink_status(__p_, __ec).type();
+    case _IterSymlink:
+    case _RefreshSymlink:
+    case _RefreshSymlinkUnresolved:
+      if (__ec)
+        __ec->clear();
+      return file_type::symlink;
+    case _IterNonSymlink:
+    case _RefreshNonSymlink:
+      file_status __st(__data_.__type_);
+      if (__ec && !_VSTD_FS::exists(__st))
+        *__ec = make_error_code(errc::no_such_file_or_directory);
+      else if (__ec)
+        __ec->clear();
+      return __data_.__type_;
+    }
+    _LIBCPP_UNREACHABLE();
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  file_type __get_ft(error_code* __ec = nullptr) const {
+    switch (__data_.__cache_type_) {
+    case _Empty:
+    case _IterSymlink:
+    case _RefreshSymlinkUnresolved:
+      return __status(__p_, __ec).type();
+    case _IterNonSymlink:
+    case _RefreshNonSymlink:
+    case _RefreshSymlink: {
+      file_status __st(__data_.__type_);
+      if (__ec && !_VSTD_FS::exists(__st))
+        *__ec = make_error_code(errc::no_such_file_or_directory);
+      else if (__ec)
+        __ec->clear();
+      return __data_.__type_;
+    }
+    }
+    _LIBCPP_UNREACHABLE();
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  file_status __get_status(error_code* __ec = nullptr) const {
+    switch (__data_.__cache_type_) {
+    case _Empty:
+    case _IterNonSymlink:
+    case _IterSymlink:
+    case _RefreshSymlinkUnresolved:
+      return __status(__p_, __ec);
+    case _RefreshNonSymlink:
+    case _RefreshSymlink:
+      return file_status(__get_ft(__ec), __data_.__non_sym_perms_);
+    }
+    _LIBCPP_UNREACHABLE();
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  file_status __get_symlink_status(error_code* __ec = nullptr) const {
+    switch (__data_.__cache_type_) {
+    case _Empty:
+    case _IterNonSymlink:
+    case _IterSymlink:
+      return __symlink_status(__p_, __ec);
+    case _RefreshNonSymlink:
+      return file_status(__get_sym_ft(__ec), __data_.__non_sym_perms_);
+    case _RefreshSymlink:
+    case _RefreshSymlinkUnresolved:
+      return file_status(__get_sym_ft(__ec), __data_.__sym_perms_);
+    }
+    _LIBCPP_UNREACHABLE();
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  uintmax_t __get_size(error_code* __ec = nullptr) const {
+    switch (__data_.__cache_type_) {
+    case _Empty:
+    case _IterNonSymlink:
+    case _IterSymlink:
+    case _RefreshSymlinkUnresolved:
+      return _VSTD_FS::__file_size(__p_, __ec);
+    case _RefreshSymlink:
+    case _RefreshNonSymlink: {
+      error_code __m_ec;
+      file_status __st(__get_ft(&__m_ec));
+      __handle_error("in directory_entry::file_size", __ec, __m_ec);
+      if (_VSTD_FS::exists(__st) && !_VSTD_FS::is_regular_file(__st)) {
+        errc __err_kind = _VSTD_FS::is_directory(__st) ? errc::is_a_directory
+                                                       : errc::not_supported;
+        __handle_error("in directory_entry::file_size", __ec,
+                       make_error_code(__err_kind));
+      }
+      return __data_.__size_;
+    }
+    }
+    _LIBCPP_UNREACHABLE();
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  uintmax_t __get_nlink(error_code* __ec = nullptr) const {
+    switch (__data_.__cache_type_) {
+    case _Empty:
+    case _IterNonSymlink:
+    case _IterSymlink:
+    case _RefreshSymlinkUnresolved:
+      return _VSTD_FS::__hard_link_count(__p_, __ec);
+    case _RefreshSymlink:
+    case _RefreshNonSymlink: {
+      error_code __m_ec;
+      (void)__get_ft(&__m_ec);
+      __handle_error("in directory_entry::hard_link_count", __ec, __m_ec);
+      return __data_.__nlink_;
+    }
+    }
+    _LIBCPP_UNREACHABLE();
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  file_time_type __get_write_time(error_code* __ec = nullptr) const {
+    switch (__data_.__cache_type_) {
+    case _Empty:
+    case _IterNonSymlink:
+    case _IterSymlink:
+    case _RefreshSymlinkUnresolved:
+      return _VSTD_FS::__last_write_time(__p_, __ec);
+    case _RefreshSymlink:
+    case _RefreshNonSymlink: {
+      error_code __m_ec;
+      file_status __st(__get_ft(&__m_ec));
+      __handle_error("in directory_entry::last_write_time", __ec, __m_ec);
+      if (_VSTD_FS::exists(__st) &&
+          __data_.__write_time_ == file_time_type::min())
+        __handle_error("in directory_entry::last_write_time", __ec,
+                       make_error_code(errc::value_too_large));
+      return __data_.__write_time_;
+    }
+    }
+    _LIBCPP_UNREACHABLE();
+  }
+
+private:
+  _Path __p_;
+  __cached_data __data_;
+};
+
+class __dir_element_proxy {
+public:
+  inline _LIBCPP_INLINE_VISIBILITY directory_entry operator*() {
+    return _VSTD::move(__elem_);
+  }
+
+private:
+  friend class directory_iterator;
+  friend class recursive_directory_iterator;
+  explicit __dir_element_proxy(directory_entry const& __e) : __elem_(__e) {}
+  __dir_element_proxy(__dir_element_proxy&& __o)
+      : __elem_(_VSTD::move(__o.__elem_)) {}
+  directory_entry __elem_;
+};
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_POP
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
+
+#endif // _LIBCPP_CXX03_LANG
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___FILESYSTEM_DIRECTORY_ENTRY_H
diff --git a/libcxx/include/__filesystem/directory_iterator.h b/libcxx/include/__filesystem/directory_iterator.h
new file mode 100644
index 000000000000..be958e0eb8de
--- /dev/null
+++ b/libcxx/include/__filesystem/directory_iterator.h
@@ -0,0 +1,150 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FILESYSTEM_DIRECTORY_ITERATOR_H
+#define _LIBCPP___FILESYSTEM_DIRECTORY_ITERATOR_H
+
+#include <__availability>
+#include <__config>
+#include <__filesystem/directory_entry.h>
+#include <__filesystem/directory_options.h>
+#include <__filesystem/path.h>
+#include <__iterator/iterator_traits.h>
+#include <__memory/shared_ptr.h>
+#include <__debug>
+#include <__ranges/enable_borrowed_range.h>
+#include <__ranges/enable_view.h>
+#include <cstddef>
+#include <system_error>
+
+#ifndef _LIBCPP_CXX03_LANG
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_PUSH
+
+class _LIBCPP_HIDDEN __dir_stream;
+class directory_iterator {
+public:
+  typedef directory_entry value_type;
+  typedef ptrdiff_t difference_type;
+  typedef value_type const* pointer;
+  typedef value_type const& reference;
+  typedef input_iterator_tag iterator_category;
+
+public:
+  //ctor & dtor
+  directory_iterator() noexcept {}
+
+  explicit directory_iterator(const path& __p)
+      : directory_iterator(__p, nullptr) {}
+
+  directory_iterator(const path& __p, directory_options __opts)
+      : directory_iterator(__p, nullptr, __opts) {}
+
+  directory_iterator(const path& __p, error_code& __ec)
+      : directory_iterator(__p, &__ec) {}
+
+  directory_iterator(const path& __p, directory_options __opts,
+                     error_code& __ec)
+      : directory_iterator(__p, &__ec, __opts) {}
+
+  directory_iterator(const directory_iterator&) = default;
+  directory_iterator(directory_iterator&&) = default;
+  directory_iterator& operator=(const directory_iterator&) = default;
+
+  directory_iterator& operator=(directory_iterator&& __o) noexcept {
+    // non-default implementation provided to support self-move assign.
+    if (this != &__o) {
+      __imp_ = _VSTD::move(__o.__imp_);
+    }
+    return *this;
+  }
+
+  ~directory_iterator() = default;
+
+  const directory_entry& operator*() const {
+    _LIBCPP_ASSERT(__imp_, "The end iterator cannot be dereferenced");
+    return __dereference();
+  }
+
+  const directory_entry* operator->() const { return &**this; }
+
+  directory_iterator& operator++() { return __increment(); }
+
+  __dir_element_proxy operator++(int) {
+    __dir_element_proxy __p(**this);
+    __increment();
+    return __p;
+  }
+
+  directory_iterator& increment(error_code& __ec) { return __increment(&__ec); }
+
+private:
+  inline _LIBCPP_INLINE_VISIBILITY friend bool
+  operator==(const directory_iterator& __lhs,
+             const directory_iterator& __rhs) noexcept;
+
+  // construct the dir_stream
+  _LIBCPP_FUNC_VIS
+  directory_iterator(const path&, error_code*,
+                     directory_options = directory_options::none);
+
+  _LIBCPP_FUNC_VIS
+  directory_iterator& __increment(error_code* __ec = nullptr);
+
+  _LIBCPP_FUNC_VIS
+  const directory_entry& __dereference() const;
+
+private:
+  shared_ptr<__dir_stream> __imp_;
+};
+
+inline _LIBCPP_INLINE_VISIBILITY bool
+operator==(const directory_iterator& __lhs,
+           const directory_iterator& __rhs) noexcept {
+  return __lhs.__imp_ == __rhs.__imp_;
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool
+operator!=(const directory_iterator& __lhs,
+           const directory_iterator& __rhs) noexcept {
+  return !(__lhs == __rhs);
+}
+
+// enable directory_iterator range-based for statements
+inline _LIBCPP_INLINE_VISIBILITY directory_iterator
+begin(directory_iterator __iter) noexcept {
+  return __iter;
+}
+
+inline _LIBCPP_INLINE_VISIBILITY directory_iterator
+end(directory_iterator) noexcept {
+  return directory_iterator();
+}
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_POP
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
+
+#if !defined(_LIBCPP_HAS_NO_RANGES)
+
+template <>
+_LIBCPP_AVAILABILITY_FILESYSTEM
+inline constexpr bool _VSTD::ranges::enable_borrowed_range<_VSTD_FS::directory_iterator> = true;
+
+template <>
+_LIBCPP_AVAILABILITY_FILESYSTEM
+inline constexpr bool _VSTD::ranges::enable_view<_VSTD_FS::directory_iterator> = true;
+
+#endif
+
+#endif // _LIBCPP_CXX03_LANG
+
+#endif // _LIBCPP___FILESYSTEM_DIRECTORY_ITERATOR_H
diff --git a/libcxx/include/__filesystem/directory_options.h b/libcxx/include/__filesystem/directory_options.h
new file mode 100644
index 000000000000..79c0c2cbaa55
--- /dev/null
+++ b/libcxx/include/__filesystem/directory_options.h
@@ -0,0 +1,78 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FILESYSTEM_DIRECTORY_OPTIONS_H
+#define _LIBCPP___FILESYSTEM_DIRECTORY_OPTIONS_H
+
+#include <__availability>
+#include <__config>
+
+#ifndef _LIBCPP_CXX03_LANG
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_PUSH
+
+enum class _LIBCPP_ENUM_VIS directory_options : unsigned char {
+  none = 0,
+  follow_directory_symlink = 1,
+  skip_permission_denied = 2
+};
+
+_LIBCPP_INLINE_VISIBILITY
+inline constexpr directory_options operator&(directory_options _LHS,
+                                             directory_options _RHS) {
+  return static_cast<directory_options>(static_cast<unsigned char>(_LHS) &
+                                        static_cast<unsigned char>(_RHS));
+}
+
+_LIBCPP_INLINE_VISIBILITY
+inline constexpr directory_options operator|(directory_options _LHS,
+                                             directory_options _RHS) {
+  return static_cast<directory_options>(static_cast<unsigned char>(_LHS) |
+                                        static_cast<unsigned char>(_RHS));
+}
+
+_LIBCPP_INLINE_VISIBILITY
+inline constexpr directory_options operator^(directory_options _LHS,
+                                             directory_options _RHS) {
+  return static_cast<directory_options>(static_cast<unsigned char>(_LHS) ^
+                                        static_cast<unsigned char>(_RHS));
+}
+
+_LIBCPP_INLINE_VISIBILITY
+inline constexpr directory_options operator~(directory_options _LHS) {
+  return static_cast<directory_options>(~static_cast<unsigned char>(_LHS));
+}
+
+_LIBCPP_INLINE_VISIBILITY
+inline directory_options& operator&=(directory_options& _LHS,
+                                     directory_options _RHS) {
+  return _LHS = _LHS & _RHS;
+}
+
+_LIBCPP_INLINE_VISIBILITY
+inline directory_options& operator|=(directory_options& _LHS,
+                                     directory_options _RHS) {
+  return _LHS = _LHS | _RHS;
+}
+
+_LIBCPP_INLINE_VISIBILITY
+inline directory_options& operator^=(directory_options& _LHS,
+                                     directory_options _RHS) {
+  return _LHS = _LHS ^ _RHS;
+}
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_POP
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
+
+#endif // _LIBCPP_CXX03_LANG
+
+#endif // _LIBCPP___FILESYSTEM_DIRECTORY_OPTIONS_H
diff --git a/libcxx/include/__filesystem/file_status.h b/libcxx/include/__filesystem/file_status.h
new file mode 100644
index 000000000000..a8f653ab44fc
--- /dev/null
+++ b/libcxx/include/__filesystem/file_status.h
@@ -0,0 +1,68 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FILESYSTEM_FILE_STATUS_H
+#define _LIBCPP___FILESYSTEM_FILE_STATUS_H
+
+#include <__availability>
+#include <__config>
+#include <__filesystem/file_type.h>
+#include <__filesystem/perms.h>
+
+#ifndef _LIBCPP_CXX03_LANG
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_PUSH
+
+class _LIBCPP_TYPE_VIS file_status {
+public:
+  // constructors
+  _LIBCPP_INLINE_VISIBILITY
+  file_status() noexcept : file_status(file_type::none) {}
+  _LIBCPP_INLINE_VISIBILITY
+  explicit file_status(file_type __ft, perms __prms = perms::unknown) noexcept
+      : __ft_(__ft),
+        __prms_(__prms) {}
+
+  file_status(const file_status&) noexcept = default;
+  file_status(file_status&&) noexcept = default;
+
+  _LIBCPP_INLINE_VISIBILITY
+  ~file_status() {}
+
+  file_status& operator=(const file_status&) noexcept = default;
+  file_status& operator=(file_status&&) noexcept = default;
+
+  // observers
+  _LIBCPP_INLINE_VISIBILITY
+  file_type type() const noexcept { return __ft_; }
+
+  _LIBCPP_INLINE_VISIBILITY
+  perms permissions() const noexcept { return __prms_; }
+
+  // modifiers
+  _LIBCPP_INLINE_VISIBILITY
+  void type(file_type __ft) noexcept { __ft_ = __ft; }
+
+  _LIBCPP_INLINE_VISIBILITY
+  void permissions(perms __p) noexcept { __prms_ = __p; }
+
+private:
+  file_type __ft_;
+  perms __prms_;
+};
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_POP
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
+
+#endif // _LIBCPP_CXX03_LANG
+
+#endif // _LIBCPP___FILESYSTEM_FILE_STATUS_H
diff --git a/libcxx/include/__filesystem/file_time_type.h b/libcxx/include/__filesystem/file_time_type.h
new file mode 100644
index 000000000000..590146a06600
--- /dev/null
+++ b/libcxx/include/__filesystem/file_time_type.h
@@ -0,0 +1,27 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FILESYSTEM_FILE_TIME_TYPE_H
+#define _LIBCPP___FILESYSTEM_FILE_TIME_TYPE_H
+
+#include <__availability>
+#include <__config>
+#include <chrono>
+
+#ifndef _LIBCPP_CXX03_LANG
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+typedef chrono::time_point<_FilesystemClock> file_time_type;
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
+
+#endif // _LIBCPP_CXX03_LANG
+
+#endif // _LIBCPP___FILESYSTEM_FILE_TIME_TYPE_H
diff --git a/libcxx/include/__filesystem/file_type.h b/libcxx/include/__filesystem/file_type.h
new file mode 100644
index 000000000000..93bee86ad635
--- /dev/null
+++ b/libcxx/include/__filesystem/file_type.h
@@ -0,0 +1,39 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FILESYSTEM_FILE_TYPE_H
+#define _LIBCPP___FILESYSTEM_FILE_TYPE_H
+
+#include <__availability>
+#include <__config>
+
+#ifndef _LIBCPP_CXX03_LANG
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+// On Windows, the library never identifies files as block, character, fifo
+// or socket.
+enum class _LIBCPP_ENUM_VIS file_type : signed char {
+  none = 0,
+  not_found = -1,
+  regular = 1,
+  directory = 2,
+  symlink = 3,
+  block = 4,
+  character = 5,
+  fifo = 6,
+  socket = 7,
+  unknown = 8
+};
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
+
+#endif // _LIBCPP_CXX03_LANG
+
+#endif // _LIBCPP___FILESYSTEM_FILE_TYPE_H
diff --git a/libcxx/include/__filesystem/filesystem_error.h b/libcxx/include/__filesystem/filesystem_error.h
new file mode 100644
index 000000000000..a8c6977b48e9
--- /dev/null
+++ b/libcxx/include/__filesystem/filesystem_error.h
@@ -0,0 +1,99 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FILESYSTEM_FILESYSTEM_ERROR_H
+#define _LIBCPP___FILESYSTEM_FILESYSTEM_ERROR_H
+
+#include <__availability>
+#include <__config>
+#include <__filesystem/path.h>
+#include <__memory/shared_ptr.h>
+#include <system_error>
+#include <iosfwd>
+#include <new>
+#include <type_traits>
+
+#ifndef _LIBCPP_CXX03_LANG
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+class _LIBCPP_AVAILABILITY_FILESYSTEM _LIBCPP_EXCEPTION_ABI filesystem_error : public system_error {
+public:
+  _LIBCPP_INLINE_VISIBILITY
+  filesystem_error(const string& __what, error_code __ec)
+      : system_error(__ec, __what),
+        __storage_(make_shared<_Storage>(path(), path())) {
+    __create_what(0);
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  filesystem_error(const string& __what, const path& __p1, error_code __ec)
+      : system_error(__ec, __what),
+        __storage_(make_shared<_Storage>(__p1, path())) {
+    __create_what(1);
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  filesystem_error(const string& __what, const path& __p1, const path& __p2,
+                   error_code __ec)
+      : system_error(__ec, __what),
+        __storage_(make_shared<_Storage>(__p1, __p2)) {
+    __create_what(2);
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  const path& path1() const noexcept { return __storage_->__p1_; }
+
+  _LIBCPP_INLINE_VISIBILITY
+  const path& path2() const noexcept { return __storage_->__p2_; }
+
+  filesystem_error(const filesystem_error&) = default;
+  ~filesystem_error() override; // key function
+
+  _LIBCPP_INLINE_VISIBILITY
+  const char* what() const noexcept override {
+    return __storage_->__what_.c_str();
+  }
+
+  void __create_what(int __num_paths);
+
+private:
+  struct _LIBCPP_HIDDEN _Storage {
+    _LIBCPP_INLINE_VISIBILITY
+    _Storage(const path& __p1, const path& __p2) : __p1_(__p1), __p2_(__p2) {}
+
+    path __p1_;
+    path __p2_;
+    string __what_;
+  };
+  shared_ptr<_Storage> __storage_;
+};
+
+// TODO(ldionne): We need to pop the pragma and push it again after
+//                filesystem_error to work around PR41078.
+_LIBCPP_AVAILABILITY_FILESYSTEM_PUSH
+
+template <class... _Args>
+_LIBCPP_NORETURN inline _LIBCPP_INLINE_VISIBILITY
+#ifndef _LIBCPP_NO_EXCEPTIONS
+void __throw_filesystem_error(_Args&&... __args) {
+  throw filesystem_error(_VSTD::forward<_Args>(__args)...);
+}
+#else
+void __throw_filesystem_error(_Args&&...) {
+  _VSTD::abort();
+}
+#endif
+_LIBCPP_AVAILABILITY_FILESYSTEM_POP
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
+
+#endif // _LIBCPP_CXX03_LANG
+
+#endif // _LIBCPP___FILESYSTEM_FILESYSTEM_ERROR_H
diff --git a/libcxx/include/__filesystem/operations.h b/libcxx/include/__filesystem/operations.h
new file mode 100644
index 000000000000..19d6c2d437f9
--- /dev/null
+++ b/libcxx/include/__filesystem/operations.h
@@ -0,0 +1,599 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FILESYSTEM_OPERATIONS_H
+#define _LIBCPP___FILESYSTEM_OPERATIONS_H
+
+#include <__availability>
+#include <__config>
+#include <__filesystem/copy_options.h>
+#include <__filesystem/file_status.h>
+#include <__filesystem/file_time_type.h>
+#include <__filesystem/file_type.h>
+#include <__filesystem/path.h>
+#include <__filesystem/perm_options.h>
+#include <__filesystem/perms.h>
+#include <__filesystem/space_info.h>
+#include <chrono>
+#include <cstdint>
+#include <system_error>
+
+#ifndef _LIBCPP_CXX03_LANG
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_PUSH
+
+_LIBCPP_FUNC_VIS
+path __absolute(const path&, error_code* __ec = nullptr);
+_LIBCPP_FUNC_VIS
+path __canonical(const path&, error_code* __ec = nullptr);
+_LIBCPP_FUNC_VIS
+bool __copy_file(const path& __from, const path& __to, copy_options __opt,
+                 error_code* __ec = nullptr);
+_LIBCPP_FUNC_VIS
+void __copy_symlink(const path& __existing_symlink, const path& __new_symlink,
+                    error_code* __ec = nullptr);
+_LIBCPP_FUNC_VIS
+void __copy(const path& __from, const path& __to, copy_options __opt,
+            error_code* __ec = nullptr);
+_LIBCPP_FUNC_VIS
+bool __create_directories(const path& p, error_code* ec = nullptr);
+_LIBCPP_FUNC_VIS
+void __create_directory_symlink(const path& __to, const path& __new_symlink,
+                                error_code* __ec = nullptr);
+_LIBCPP_FUNC_VIS
+bool __create_directory(const path& p, error_code* ec = nullptr);
+_LIBCPP_FUNC_VIS
+bool __create_directory(const path& p, const path& attributes,
+                        error_code* ec = nullptr);
+_LIBCPP_FUNC_VIS
+void __create_hard_link(const path& __to, const path& __new_hard_link,
+                        error_code* __ec = nullptr);
+_LIBCPP_FUNC_VIS
+void __create_symlink(const path& __to, const path& __new_symlink,
+                      error_code* __ec = nullptr);
+_LIBCPP_FUNC_VIS
+path __current_path(error_code* __ec = nullptr);
+_LIBCPP_FUNC_VIS
+void __current_path(const path&, error_code* __ec = nullptr);
+_LIBCPP_FUNC_VIS
+bool __equivalent(const path&, const path&, error_code* __ec = nullptr);
+_LIBCPP_FUNC_VIS
+file_status __status(const path&, error_code* __ec = nullptr);
+_LIBCPP_FUNC_VIS
+uintmax_t __file_size(const path&, error_code* __ec = nullptr);
+_LIBCPP_FUNC_VIS
+uintmax_t __hard_link_count(const path&, error_code* __ec = nullptr);
+_LIBCPP_FUNC_VIS
+file_status __symlink_status(const path&, error_code* __ec = nullptr);
+_LIBCPP_FUNC_VIS
+file_time_type __last_write_time(const path& p, error_code* ec = nullptr);
+_LIBCPP_FUNC_VIS
+void __last_write_time(const path& p, file_time_type new_time,
+                       error_code* ec = nullptr);
+_LIBCPP_FUNC_VIS
+path __weakly_canonical(path const& __p, error_code* __ec = nullptr);
+_LIBCPP_FUNC_VIS
+path __read_symlink(const path& p, error_code* ec = nullptr);
+_LIBCPP_FUNC_VIS
+uintmax_t __remove_all(const path& p, error_code* ec = nullptr);
+_LIBCPP_FUNC_VIS
+bool __remove(const path& p, error_code* ec = nullptr);
+_LIBCPP_FUNC_VIS
+void __rename(const path& from, const path& to, error_code* ec = nullptr);
+_LIBCPP_FUNC_VIS
+void __resize_file(const path& p, uintmax_t size, error_code* ec = nullptr);
+_LIBCPP_FUNC_VIS
+path __temp_directory_path(error_code* __ec = nullptr);
+
+inline _LIBCPP_INLINE_VISIBILITY path absolute(const path& __p) {
+  return __absolute(__p);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY path absolute(const path& __p,
+                                               error_code& __ec) {
+  return __absolute(__p, &__ec);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY path canonical(const path& __p) {
+  return __canonical(__p);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY path canonical(const path& __p,
+                                                error_code& __ec) {
+  return __canonical(__p, &__ec);
+}
+
+
+inline _LIBCPP_INLINE_VISIBILITY bool copy_file(const path& __from,
+                                                const path& __to) {
+  return __copy_file(__from, __to, copy_options::none);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool
+copy_file(const path& __from, const path& __to, error_code& __ec) {
+  return __copy_file(__from, __to, copy_options::none, &__ec);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool
+copy_file(const path& __from, const path& __to, copy_options __opt) {
+  return __copy_file(__from, __to, __opt);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool copy_file(const path& __from,
+                                                const path& __to,
+                                                copy_options __opt,
+                                                error_code& __ec) {
+  return __copy_file(__from, __to, __opt, &__ec);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY void copy_symlink(const path& __existing,
+                                                   const path& __new) {
+  __copy_symlink(__existing, __new);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY void
+copy_symlink(const path& __ext, const path& __new, error_code& __ec) noexcept {
+  __copy_symlink(__ext, __new, &__ec);
+}
+
+
+inline _LIBCPP_INLINE_VISIBILITY void copy(const path& __from,
+                                           const path& __to) {
+  __copy(__from, __to, copy_options::none);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY void copy(const path& __from, const path& __to,
+                                           error_code& __ec) {
+  __copy(__from, __to, copy_options::none, &__ec);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY void copy(const path& __from, const path& __to,
+                                           copy_options __opt) {
+  __copy(__from, __to, __opt);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY void copy(const path& __from, const path& __to,
+                                           copy_options __opt,
+                                           error_code& __ec) {
+  __copy(__from, __to, __opt, &__ec);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool create_directories(const path& __p) {
+  return __create_directories(__p);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool create_directories(const path& __p,
+                                                         error_code& __ec) {
+  return __create_directories(__p, &__ec);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY void
+create_directory_symlink(const path& __to, const path& __new) {
+  __create_directory_symlink(__to, __new);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY void
+create_directory_symlink(const path& __to, const path& __new,
+                         error_code& __ec) noexcept {
+  __create_directory_symlink(__to, __new, &__ec);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool create_directory(const path& __p) {
+  return __create_directory(__p);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool
+create_directory(const path& __p, error_code& __ec) noexcept {
+  return __create_directory(__p, &__ec);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool create_directory(const path& __p,
+                                                       const path& __attrs) {
+  return __create_directory(__p, __attrs);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool
+create_directory(const path& __p, const path& __attrs,
+                 error_code& __ec) noexcept {
+  return __create_directory(__p, __attrs, &__ec);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY void create_hard_link(const path& __to,
+                                                       const path& __new) {
+  __create_hard_link(__to, __new);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY void
+create_hard_link(const path& __to, const path& __new,
+                 error_code& __ec) noexcept {
+  __create_hard_link(__to, __new, &__ec);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY void create_symlink(const path& __to,
+                                                     const path& __new) {
+  __create_symlink(__to, __new);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY void
+create_symlink(const path& __to, const path& __new, error_code& __ec) noexcept {
+  return __create_symlink(__to, __new, &__ec);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY path current_path() {
+  return __current_path();
+}
+
+inline _LIBCPP_INLINE_VISIBILITY path current_path(error_code& __ec) {
+  return __current_path(&__ec);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY void current_path(const path& __p) {
+  __current_path(__p);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY void current_path(const path& __p,
+                                                   error_code& __ec) noexcept {
+  __current_path(__p, &__ec);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool equivalent(const path& __p1,
+                                                 const path& __p2) {
+  return __equivalent(__p1, __p2);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool
+equivalent(const path& __p1, const path& __p2, error_code& __ec) noexcept {
+  return __equivalent(__p1, __p2, &__ec);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool status_known(file_status __s) noexcept {
+  return __s.type() != file_type::none;
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool exists(file_status __s) noexcept {
+  return status_known(__s) && __s.type() != file_type::not_found;
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool exists(const path& __p) {
+  return exists(__status(__p));
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool exists(const path& __p,
+                                             error_code& __ec) noexcept {
+  auto __s = __status(__p, &__ec);
+  if (status_known(__s))
+    __ec.clear();
+  return exists(__s);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY uintmax_t file_size(const path& __p) {
+  return __file_size(__p);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY uintmax_t
+file_size(const path& __p, error_code& __ec) noexcept {
+  return __file_size(__p, &__ec);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY uintmax_t hard_link_count(const path& __p) {
+  return __hard_link_count(__p);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY uintmax_t
+hard_link_count(const path& __p, error_code& __ec) noexcept {
+  return __hard_link_count(__p, &__ec);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool is_block_file(file_status __s) noexcept {
+  return __s.type() == file_type::block;
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool is_block_file(const path& __p) {
+  return is_block_file(__status(__p));
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool is_block_file(const path& __p,
+                                                    error_code& __ec) noexcept {
+  return is_block_file(__status(__p, &__ec));
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool
+is_character_file(file_status __s) noexcept {
+  return __s.type() == file_type::character;
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool is_character_file(const path& __p) {
+  return is_character_file(__status(__p));
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool
+is_character_file(const path& __p, error_code& __ec) noexcept {
+  return is_character_file(__status(__p, &__ec));
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool is_directory(file_status __s) noexcept {
+  return __s.type() == file_type::directory;
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool is_directory(const path& __p) {
+  return is_directory(__status(__p));
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool is_directory(const path& __p,
+                                                   error_code& __ec) noexcept {
+  return is_directory(__status(__p, &__ec));
+}
+
+_LIBCPP_FUNC_VIS
+bool __fs_is_empty(const path& p, error_code* ec = nullptr);
+
+inline _LIBCPP_INLINE_VISIBILITY bool is_empty(const path& __p) {
+  return __fs_is_empty(__p);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool is_empty(const path& __p,
+                                               error_code& __ec) {
+  return __fs_is_empty(__p, &__ec);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool is_fifo(file_status __s) noexcept {
+  return __s.type() == file_type::fifo;
+}
+inline _LIBCPP_INLINE_VISIBILITY bool is_fifo(const path& __p) {
+  return is_fifo(__status(__p));
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool is_fifo(const path& __p,
+                                              error_code& __ec) noexcept {
+  return is_fifo(__status(__p, &__ec));
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool
+is_regular_file(file_status __s) noexcept {
+  return __s.type() == file_type::regular;
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool is_regular_file(const path& __p) {
+  return is_regular_file(__status(__p));
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool
+is_regular_file(const path& __p, error_code& __ec) noexcept {
+  return is_regular_file(__status(__p, &__ec));
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool is_symlink(file_status __s) noexcept {
+  return __s.type() == file_type::symlink;
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool is_symlink(const path& __p) {
+  return is_symlink(__symlink_status(__p));
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool is_symlink(const path& __p,
+                                                 error_code& __ec) noexcept {
+  return is_symlink(__symlink_status(__p, &__ec));
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool is_other(file_status __s) noexcept {
+  return exists(__s) && !is_regular_file(__s) && !is_directory(__s) &&
+         !is_symlink(__s);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool is_other(const path& __p) {
+  return is_other(__status(__p));
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool is_other(const path& __p,
+                                               error_code& __ec) noexcept {
+  return is_other(__status(__p, &__ec));
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool is_socket(file_status __s) noexcept {
+  return __s.type() == file_type::socket;
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool is_socket(const path& __p) {
+  return is_socket(__status(__p));
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool is_socket(const path& __p,
+                                                error_code& __ec) noexcept {
+  return is_socket(__status(__p, &__ec));
+}
+
+inline _LIBCPP_INLINE_VISIBILITY file_time_type
+last_write_time(const path& __p) {
+  return __last_write_time(__p);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY file_time_type
+last_write_time(const path& __p, error_code& __ec) noexcept {
+  return __last_write_time(__p, &__ec);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY void last_write_time(const path& __p,
+                                                      file_time_type __t) {
+  __last_write_time(__p, __t);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY void
+last_write_time(const path& __p, file_time_type __t,
+                error_code& __ec) noexcept {
+  __last_write_time(__p, __t, &__ec);
+}
+
+_LIBCPP_FUNC_VIS
+void __permissions(const path&, perms, perm_options, error_code* = nullptr);
+
+inline _LIBCPP_INLINE_VISIBILITY void
+permissions(const path& __p, perms __prms,
+            perm_options __opts = perm_options::replace) {
+  __permissions(__p, __prms, __opts);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY void permissions(const path& __p, perms __prms,
+                                                  error_code& __ec) noexcept {
+  __permissions(__p, __prms, perm_options::replace, &__ec);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY void permissions(const path& __p, perms __prms,
+                                                  perm_options __opts,
+                                                  error_code& __ec) {
+  __permissions(__p, __prms, __opts, &__ec);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY path proximate(const path& __p,
+                                                const path& __base,
+                                                error_code& __ec) {
+  path __tmp = __weakly_canonical(__p, &__ec);
+  if (__ec)
+    return {};
+  path __tmp_base = __weakly_canonical(__base, &__ec);
+  if (__ec)
+    return {};
+  return __tmp.lexically_proximate(__tmp_base);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY path proximate(const path& __p,
+                                                error_code& __ec) {
+  return proximate(__p, current_path(), __ec);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY path
+proximate(const path& __p, const path& __base = current_path()) {
+  return __weakly_canonical(__p).lexically_proximate(
+      __weakly_canonical(__base));
+}
+
+inline _LIBCPP_INLINE_VISIBILITY path read_symlink(const path& __p) {
+  return __read_symlink(__p);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY path read_symlink(const path& __p,
+                                                   error_code& __ec) {
+  return __read_symlink(__p, &__ec);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY path relative(const path& __p,
+                                               const path& __base,
+                                               error_code& __ec) {
+  path __tmp = __weakly_canonical(__p, &__ec);
+  if (__ec)
+    return path();
+  path __tmpbase = __weakly_canonical(__base, &__ec);
+  if (__ec)
+    return path();
+  return __tmp.lexically_relative(__tmpbase);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY path relative(const path& __p,
+                                               error_code& __ec) {
+  return relative(__p, current_path(), __ec);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY path
+relative(const path& __p, const path& __base = current_path()) {
+  return __weakly_canonical(__p).lexically_relative(__weakly_canonical(__base));
+}
+
+inline _LIBCPP_INLINE_VISIBILITY uintmax_t remove_all(const path& __p) {
+  return __remove_all(__p);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY uintmax_t remove_all(const path& __p,
+                                                      error_code& __ec) {
+  return __remove_all(__p, &__ec);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool remove(const path& __p) {
+  return __remove(__p);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool remove(const path& __p,
+                                             error_code& __ec) noexcept {
+  return __remove(__p, &__ec);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY void rename(const path& __from,
+                                             const path& __to) {
+  return __rename(__from, __to);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY void
+rename(const path& __from, const path& __to, error_code& __ec) noexcept {
+  return __rename(__from, __to, &__ec);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY void resize_file(const path& __p,
+                                                  uintmax_t __ns) {
+  return __resize_file(__p, __ns);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY void
+resize_file(const path& __p, uintmax_t __ns, error_code& __ec) noexcept {
+  return __resize_file(__p, __ns, &__ec);
+}
+
+_LIBCPP_FUNC_VIS
+space_info __space(const path&, error_code* __ec = nullptr);
+
+inline _LIBCPP_INLINE_VISIBILITY space_info space(const path& __p) {
+  return __space(__p);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY space_info space(const path& __p,
+                                                  error_code& __ec) noexcept {
+  return __space(__p, &__ec);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY file_status status(const path& __p) {
+  return __status(__p);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY file_status status(const path& __p,
+                                                    error_code& __ec) noexcept {
+  return __status(__p, &__ec);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY file_status symlink_status(const path& __p) {
+  return __symlink_status(__p);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY file_status
+symlink_status(const path& __p, error_code& __ec) noexcept {
+  return __symlink_status(__p, &__ec);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY path temp_directory_path() {
+  return __temp_directory_path();
+}
+
+inline _LIBCPP_INLINE_VISIBILITY path temp_directory_path(error_code& __ec) {
+  return __temp_directory_path(&__ec);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY path weakly_canonical(path const& __p) {
+  return __weakly_canonical(__p);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY path weakly_canonical(path const& __p,
+                                                       error_code& __ec) {
+  return __weakly_canonical(__p, &__ec);
+}
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_POP
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
+
+#endif // _LIBCPP_CXX03_LANG
+
+#endif // _LIBCPP___FILESYSTEM_OPERATIONS_H
diff --git a/libcxx/include/__filesystem/path.h b/libcxx/include/__filesystem/path.h
new file mode 100644
index 000000000000..a6d1ee997d91
--- /dev/null
+++ b/libcxx/include/__filesystem/path.h
@@ -0,0 +1,1018 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FILESYSTEM_PATH_H
+#define _LIBCPP___FILESYSTEM_PATH_H
+
+#include <__availability>
+#include <__config>
+#include <string>
+#include <type_traits>
+#include <__iterator/back_insert_iterator.h>
+#include <__iterator/iterator_traits.h>
+#include <cstddef>
+#include <string_view>
+
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+# include <locale>
+# include <iomanip> // for quoted
+#endif
+
+#ifndef _LIBCPP_CXX03_LANG
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_PUSH
+
+template <class _Tp>
+struct __can_convert_char {
+  static const bool value = false;
+};
+template <class _Tp>
+struct __can_convert_char<const _Tp> : public __can_convert_char<_Tp> {};
+template <>
+struct __can_convert_char<char> {
+  static const bool value = true;
+  using __char_type = char;
+};
+template <>
+struct __can_convert_char<wchar_t> {
+  static const bool value = true;
+  using __char_type = wchar_t;
+};
+#ifndef _LIBCPP_HAS_NO_CHAR8_T
+template <>
+struct __can_convert_char<char8_t> {
+  static const bool value = true;
+  using __char_type = char8_t;
+};
+#endif
+template <>
+struct __can_convert_char<char16_t> {
+  static const bool value = true;
+  using __char_type = char16_t;
+};
+template <>
+struct __can_convert_char<char32_t> {
+  static const bool value = true;
+  using __char_type = char32_t;
+};
+
+template <class _ECharT>
+typename enable_if<__can_convert_char<_ECharT>::value, bool>::type
+__is_separator(_ECharT __e) {
+#if defined(_LIBCPP_WIN32API)
+  return __e == _ECharT('/') || __e == _ECharT('\\');
+#else
+  return __e == _ECharT('/');
+#endif
+}
+
+#ifndef _LIBCPP_HAS_NO_CHAR8_T
+typedef u8string __u8_string;
+#else
+typedef string __u8_string;
+#endif
+
+struct _NullSentinel {};
+
+template <class _Tp>
+using _Void = void;
+
+template <class _Tp, class = void>
+struct __is_pathable_string : public false_type {};
+
+template <class _ECharT, class _Traits, class _Alloc>
+struct __is_pathable_string<
+    basic_string<_ECharT, _Traits, _Alloc>,
+    _Void<typename __can_convert_char<_ECharT>::__char_type> >
+    : public __can_convert_char<_ECharT> {
+  using _Str = basic_string<_ECharT, _Traits, _Alloc>;
+  using _Base = __can_convert_char<_ECharT>;
+  static _ECharT const* __range_begin(_Str const& __s) { return __s.data(); }
+  static _ECharT const* __range_end(_Str const& __s) {
+    return __s.data() + __s.length();
+  }
+  static _ECharT __first_or_null(_Str const& __s) {
+    return __s.empty() ? _ECharT{} : __s[0];
+  }
+};
+
+template <class _ECharT, class _Traits>
+struct __is_pathable_string<
+    basic_string_view<_ECharT, _Traits>,
+    _Void<typename __can_convert_char<_ECharT>::__char_type> >
+    : public __can_convert_char<_ECharT> {
+  using _Str = basic_string_view<_ECharT, _Traits>;
+  using _Base = __can_convert_char<_ECharT>;
+  static _ECharT const* __range_begin(_Str const& __s) { return __s.data(); }
+  static _ECharT const* __range_end(_Str const& __s) {
+    return __s.data() + __s.length();
+  }
+  static _ECharT __first_or_null(_Str const& __s) {
+    return __s.empty() ? _ECharT{} : __s[0];
+  }
+};
+
+template <class _Source, class _DS = typename decay<_Source>::type,
+          class _UnqualPtrType =
+              typename remove_const<typename remove_pointer<_DS>::type>::type,
+          bool _IsCharPtr = is_pointer<_DS>::value&&
+              __can_convert_char<_UnqualPtrType>::value>
+struct __is_pathable_char_array : false_type {};
+
+template <class _Source, class _ECharT, class _UPtr>
+struct __is_pathable_char_array<_Source, _ECharT*, _UPtr, true>
+    : __can_convert_char<typename remove_const<_ECharT>::type> {
+  using _Base = __can_convert_char<typename remove_const<_ECharT>::type>;
+
+  static _ECharT const* __range_begin(const _ECharT* __b) { return __b; }
+  static _ECharT const* __range_end(const _ECharT* __b) {
+    using _Iter = const _ECharT*;
+    const _ECharT __sentinel = _ECharT{};
+    _Iter __e = __b;
+    for (; *__e != __sentinel; ++__e)
+      ;
+    return __e;
+  }
+
+  static _ECharT __first_or_null(const _ECharT* __b) { return *__b; }
+};
+
+template <class _Iter, bool _IsIt = __is_cpp17_input_iterator<_Iter>::value,
+          class = void>
+struct __is_pathable_iter : false_type {};
+
+template <class _Iter>
+struct __is_pathable_iter<
+    _Iter, true,
+    _Void<typename __can_convert_char<
+        typename iterator_traits<_Iter>::value_type>::__char_type> >
+    : __can_convert_char<typename iterator_traits<_Iter>::value_type> {
+  using _ECharT = typename iterator_traits<_Iter>::value_type;
+  using _Base = __can_convert_char<_ECharT>;
+
+  static _Iter __range_begin(_Iter __b) { return __b; }
+  static _NullSentinel __range_end(_Iter) { return _NullSentinel{}; }
+
+  static _ECharT __first_or_null(_Iter __b) { return *__b; }
+};
+
+template <class _Tp, bool _IsStringT = __is_pathable_string<_Tp>::value,
+          bool _IsCharIterT = __is_pathable_char_array<_Tp>::value,
+          bool _IsIterT = !_IsCharIterT && __is_pathable_iter<_Tp>::value>
+struct __is_pathable : false_type {
+  static_assert(!_IsStringT && !_IsCharIterT && !_IsIterT, "Must all be false");
+};
+
+template <class _Tp>
+struct __is_pathable<_Tp, true, false, false> : __is_pathable_string<_Tp> {};
+
+template <class _Tp>
+struct __is_pathable<_Tp, false, true, false> : __is_pathable_char_array<_Tp> {
+};
+
+template <class _Tp>
+struct __is_pathable<_Tp, false, false, true> : __is_pathable_iter<_Tp> {};
+
+#if defined(_LIBCPP_WIN32API)
+typedef wstring __path_string;
+typedef wchar_t __path_value;
+#else
+typedef string __path_string;
+typedef char __path_value;
+#endif
+
+#if defined(_LIBCPP_WIN32API)
+_LIBCPP_FUNC_VIS
+size_t __wide_to_char(const wstring&, char*, size_t);
+_LIBCPP_FUNC_VIS
+size_t __char_to_wide(const string&, wchar_t*, size_t);
+#endif
+
+template <class _ECharT>
+struct _PathCVT;
+
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+template <class _ECharT>
+struct _PathCVT {
+  static_assert(__can_convert_char<_ECharT>::value,
+                "Char type not convertible");
+
+  typedef __narrow_to_utf8<sizeof(_ECharT) * __CHAR_BIT__> _Narrower;
+#if defined(_LIBCPP_WIN32API)
+  typedef __widen_from_utf8<sizeof(wchar_t) * __CHAR_BIT__> _Widener;
+#endif
+
+  static void __append_range(__path_string& __dest, _ECharT const* __b,
+                             _ECharT const* __e) {
+#if defined(_LIBCPP_WIN32API)
+    string __utf8;
+    _Narrower()(back_inserter(__utf8), __b, __e);
+    _Widener()(back_inserter(__dest), __utf8.data(), __utf8.data() + __utf8.size());
+#else
+    _Narrower()(back_inserter(__dest), __b, __e);
+#endif
+  }
+
+  template <class _Iter>
+  static void __append_range(__path_string& __dest, _Iter __b, _Iter __e) {
+    static_assert(!is_same<_Iter, _ECharT*>::value, "Call const overload");
+    if (__b == __e)
+      return;
+    basic_string<_ECharT> __tmp(__b, __e);
+#if defined(_LIBCPP_WIN32API)
+    string __utf8;
+    _Narrower()(back_inserter(__utf8), __tmp.data(),
+                __tmp.data() + __tmp.length());
+    _Widener()(back_inserter(__dest), __utf8.data(), __utf8.data() + __utf8.size());
+#else
+    _Narrower()(back_inserter(__dest), __tmp.data(),
+                __tmp.data() + __tmp.length());
+#endif
+  }
+
+  template <class _Iter>
+  static void __append_range(__path_string& __dest, _Iter __b, _NullSentinel) {
+    static_assert(!is_same<_Iter, _ECharT*>::value, "Call const overload");
+    const _ECharT __sentinel = _ECharT{};
+    if (*__b == __sentinel)
+      return;
+    basic_string<_ECharT> __tmp;
+    for (; *__b != __sentinel; ++__b)
+      __tmp.push_back(*__b);
+#if defined(_LIBCPP_WIN32API)
+    string __utf8;
+    _Narrower()(back_inserter(__utf8), __tmp.data(),
+                __tmp.data() + __tmp.length());
+    _Widener()(back_inserter(__dest), __utf8.data(), __utf8.data() + __utf8.size());
+#else
+    _Narrower()(back_inserter(__dest), __tmp.data(),
+                __tmp.data() + __tmp.length());
+#endif
+  }
+
+  template <class _Source>
+  static void __append_source(__path_string& __dest, _Source const& __s) {
+    using _Traits = __is_pathable<_Source>;
+    __append_range(__dest, _Traits::__range_begin(__s),
+                   _Traits::__range_end(__s));
+  }
+};
+#endif // !_LIBCPP_HAS_NO_LOCALIZATION
+
+template <>
+struct _PathCVT<__path_value> {
+
+  template <class _Iter>
+  static typename enable_if<__is_exactly_cpp17_input_iterator<_Iter>::value>::type
+  __append_range(__path_string& __dest, _Iter __b, _Iter __e) {
+    for (; __b != __e; ++__b)
+      __dest.push_back(*__b);
+  }
+
+  template <class _Iter>
+  static typename enable_if<__is_cpp17_forward_iterator<_Iter>::value>::type
+  __append_range(__path_string& __dest, _Iter __b, _Iter __e) {
+    __dest.append(__b, __e);
+  }
+
+  template <class _Iter>
+  static void __append_range(__path_string& __dest, _Iter __b, _NullSentinel) {
+    const char __sentinel = char{};
+    for (; *__b != __sentinel; ++__b)
+      __dest.push_back(*__b);
+  }
+
+  template <class _Source>
+  static void __append_source(__path_string& __dest, _Source const& __s) {
+    using _Traits = __is_pathable<_Source>;
+    __append_range(__dest, _Traits::__range_begin(__s),
+                   _Traits::__range_end(__s));
+  }
+};
+
+#if defined(_LIBCPP_WIN32API)
+template <>
+struct _PathCVT<char> {
+
+  static void
+  __append_string(__path_string& __dest, const basic_string<char> &__str) {
+      size_t __size = __char_to_wide(__str, nullptr, 0);
+      size_t __pos = __dest.size();
+      __dest.resize(__pos + __size);
+      __char_to_wide(__str, const_cast<__path_value*>(__dest.data()) + __pos, __size);
+  }
+
+  template <class _Iter>
+  static typename enable_if<__is_exactly_cpp17_input_iterator<_Iter>::value>::type
+  __append_range(__path_string& __dest, _Iter __b, _Iter __e) {
+    basic_string<char> __tmp(__b, __e);
+    __append_string(__dest, __tmp);
+  }
+
+  template <class _Iter>
+  static typename enable_if<__is_cpp17_forward_iterator<_Iter>::value>::type
+  __append_range(__path_string& __dest, _Iter __b, _Iter __e) {
+    basic_string<char> __tmp(__b, __e);
+    __append_string(__dest, __tmp);
+  }
+
+  template <class _Iter>
+  static void __append_range(__path_string& __dest, _Iter __b, _NullSentinel) {
+    const char __sentinel = char{};
+    basic_string<char> __tmp;
+    for (; *__b != __sentinel; ++__b)
+      __tmp.push_back(*__b);
+    __append_string(__dest, __tmp);
+  }
+
+  template <class _Source>
+  static void __append_source(__path_string& __dest, _Source const& __s) {
+    using _Traits = __is_pathable<_Source>;
+    __append_range(__dest, _Traits::__range_begin(__s),
+                   _Traits::__range_end(__s));
+  }
+};
+
+template <class _ECharT>
+struct _PathExport {
+  typedef __narrow_to_utf8<sizeof(wchar_t) * __CHAR_BIT__> _Narrower;
+  typedef __widen_from_utf8<sizeof(_ECharT) * __CHAR_BIT__> _Widener;
+
+  template <class _Str>
+  static void __append(_Str& __dest, const __path_string& __src) {
+    string __utf8;
+    _Narrower()(back_inserter(__utf8), __src.data(), __src.data() + __src.size());
+    _Widener()(back_inserter(__dest), __utf8.data(), __utf8.data() + __utf8.size());
+  }
+};
+
+template <>
+struct _PathExport<char> {
+  template <class _Str>
+  static void __append(_Str& __dest, const __path_string& __src) {
+    size_t __size = __wide_to_char(__src, nullptr, 0);
+    size_t __pos = __dest.size();
+    __dest.resize(__size);
+    __wide_to_char(__src, const_cast<char*>(__dest.data()) + __pos, __size);
+  }
+};
+
+template <>
+struct _PathExport<wchar_t> {
+  template <class _Str>
+  static void __append(_Str& __dest, const __path_string& __src) {
+    __dest.append(__src.begin(), __src.end());
+  }
+};
+
+template <>
+struct _PathExport<char16_t> {
+  template <class _Str>
+  static void __append(_Str& __dest, const __path_string& __src) {
+    __dest.append(__src.begin(), __src.end());
+  }
+};
+
+#ifndef _LIBCPP_HAS_NO_CHAR8_T
+template <>
+struct _PathExport<char8_t> {
+  typedef __narrow_to_utf8<sizeof(wchar_t) * __CHAR_BIT__> _Narrower;
+
+  template <class _Str>
+  static void __append(_Str& __dest, const __path_string& __src) {
+    _Narrower()(back_inserter(__dest), __src.data(), __src.data() + __src.size());
+  }
+};
+#endif /* !_LIBCPP_HAS_NO_CHAR8_T */
+#endif /* _LIBCPP_WIN32API */
+
+class _LIBCPP_TYPE_VIS path {
+  template <class _SourceOrIter, class _Tp = path&>
+  using _EnableIfPathable =
+      typename enable_if<__is_pathable<_SourceOrIter>::value, _Tp>::type;
+
+  template <class _Tp>
+  using _SourceChar = typename __is_pathable<_Tp>::__char_type;
+
+  template <class _Tp>
+  using _SourceCVT = _PathCVT<_SourceChar<_Tp> >;
+
+public:
+#if defined(_LIBCPP_WIN32API)
+  typedef wchar_t value_type;
+  static constexpr value_type preferred_separator = L'\\';
+#else
+  typedef char value_type;
+  static constexpr value_type preferred_separator = '/';
+#endif
+  typedef basic_string<value_type> string_type;
+  typedef basic_string_view<value_type> __string_view;
+
+  enum _LIBCPP_ENUM_VIS format : unsigned char {
+    auto_format,
+    native_format,
+    generic_format
+  };
+
+  // constructors and destructor
+  _LIBCPP_INLINE_VISIBILITY path() noexcept {}
+  _LIBCPP_INLINE_VISIBILITY path(const path& __p) : __pn_(__p.__pn_) {}
+  _LIBCPP_INLINE_VISIBILITY path(path&& __p) noexcept
+      : __pn_(_VSTD::move(__p.__pn_)) {}
+
+  _LIBCPP_INLINE_VISIBILITY
+  path(string_type&& __s, format = format::auto_format) noexcept
+      : __pn_(_VSTD::move(__s)) {}
+
+  template <class _Source, class = _EnableIfPathable<_Source, void> >
+  path(const _Source& __src, format = format::auto_format) {
+    _SourceCVT<_Source>::__append_source(__pn_, __src);
+  }
+
+  template <class _InputIt>
+  path(_InputIt __first, _InputIt __last, format = format::auto_format) {
+    typedef typename iterator_traits<_InputIt>::value_type _ItVal;
+    _PathCVT<_ItVal>::__append_range(__pn_, __first, __last);
+  }
+
+/*
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+  // TODO Implement locale conversions.
+  template <class _Source, class = _EnableIfPathable<_Source, void> >
+  path(const _Source& __src, const locale& __loc, format = format::auto_format);
+  template <class _InputIt>
+  path(_InputIt __first, _InputIt _last, const locale& __loc,
+       format = format::auto_format);
+#endif
+*/
+
+  _LIBCPP_INLINE_VISIBILITY
+  ~path() = default;
+
+  // assignments
+  _LIBCPP_INLINE_VISIBILITY
+  path& operator=(const path& __p) {
+    __pn_ = __p.__pn_;
+    return *this;
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  path& operator=(path&& __p) noexcept {
+    __pn_ = _VSTD::move(__p.__pn_);
+    return *this;
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  path& operator=(string_type&& __s) noexcept {
+    __pn_ = _VSTD::move(__s);
+    return *this;
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  path& assign(string_type&& __s) noexcept {
+    __pn_ = _VSTD::move(__s);
+    return *this;
+  }
+
+  template <class _Source>
+  _LIBCPP_INLINE_VISIBILITY _EnableIfPathable<_Source>
+  operator=(const _Source& __src) {
+    return this->assign(__src);
+  }
+
+  template <class _Source>
+  _EnableIfPathable<_Source> assign(const _Source& __src) {
+    __pn_.clear();
+    _SourceCVT<_Source>::__append_source(__pn_, __src);
+    return *this;
+  }
+
+  template <class _InputIt>
+  path& assign(_InputIt __first, _InputIt __last) {
+    typedef typename iterator_traits<_InputIt>::value_type _ItVal;
+    __pn_.clear();
+    _PathCVT<_ItVal>::__append_range(__pn_, __first, __last);
+    return *this;
+  }
+
+public:
+  // appends
+#if defined(_LIBCPP_WIN32API)
+  path& operator/=(const path& __p) {
+    auto __p_root_name = __p.__root_name();
+    auto __p_root_name_size = __p_root_name.size();
+    if (__p.is_absolute() ||
+        (!__p_root_name.empty() && __p_root_name != __string_view(root_name().__pn_))) {
+      __pn_ = __p.__pn_;
+      return *this;
+    }
+    if (__p.has_root_directory()) {
+      path __root_name_str = root_name();
+      __pn_ = __root_name_str.native();
+      __pn_ += __string_view(__p.__pn_).substr(__p_root_name_size);
+      return *this;
+    }
+    if (has_filename() || (!has_root_directory() && is_absolute()))
+      __pn_ += preferred_separator;
+    __pn_ += __string_view(__p.__pn_).substr(__p_root_name_size);
+    return *this;
+  }
+  template <class _Source>
+  _LIBCPP_INLINE_VISIBILITY _EnableIfPathable<_Source>
+  operator/=(const _Source& __src) {
+    return operator/=(path(__src));
+  }
+
+  template <class _Source>
+  _EnableIfPathable<_Source> append(const _Source& __src) {
+    return operator/=(path(__src));
+  }
+
+  template <class _InputIt>
+  path& append(_InputIt __first, _InputIt __last) {
+    return operator/=(path(__first, __last));
+  }
+#else
+  path& operator/=(const path& __p) {
+    if (__p.is_absolute()) {
+      __pn_ = __p.__pn_;
+      return *this;
+    }
+    if (has_filename())
+      __pn_ += preferred_separator;
+    __pn_ += __p.native();
+    return *this;
+  }
+
+  // FIXME: Use _LIBCPP_DIAGNOSE_WARNING to produce a diagnostic when __src
+  // is known at compile time to be "/' since the user almost certainly intended
+  // to append a separator instead of overwriting the path with "/"
+  template <class _Source>
+  _LIBCPP_INLINE_VISIBILITY _EnableIfPathable<_Source>
+  operator/=(const _Source& __src) {
+    return this->append(__src);
+  }
+
+  template <class _Source>
+  _EnableIfPathable<_Source> append(const _Source& __src) {
+    using _Traits = __is_pathable<_Source>;
+    using _CVT = _PathCVT<_SourceChar<_Source> >;
+    bool __source_is_absolute = __is_separator(_Traits::__first_or_null(__src));
+    if (__source_is_absolute)
+      __pn_.clear();
+    else if (has_filename())
+      __pn_ += preferred_separator;
+    _CVT::__append_source(__pn_, __src);
+    return *this;
+  }
+
+  template <class _InputIt>
+  path& append(_InputIt __first, _InputIt __last) {
+    typedef typename iterator_traits<_InputIt>::value_type _ItVal;
+    static_assert(__can_convert_char<_ItVal>::value, "Must convertible");
+    using _CVT = _PathCVT<_ItVal>;
+    if (__first != __last && __is_separator(*__first))
+      __pn_.clear();
+    else if (has_filename())
+      __pn_ += preferred_separator;
+    _CVT::__append_range(__pn_, __first, __last);
+    return *this;
+  }
+#endif
+
+  // concatenation
+  _LIBCPP_INLINE_VISIBILITY
+  path& operator+=(const path& __x) {
+    __pn_ += __x.__pn_;
+    return *this;
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  path& operator+=(const string_type& __x) {
+    __pn_ += __x;
+    return *this;
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  path& operator+=(__string_view __x) {
+    __pn_ += __x;
+    return *this;
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  path& operator+=(const value_type* __x) {
+    __pn_ += __x;
+    return *this;
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  path& operator+=(value_type __x) {
+    __pn_ += __x;
+    return *this;
+  }
+
+  template <class _ECharT>
+  typename enable_if<__can_convert_char<_ECharT>::value, path&>::type
+  operator+=(_ECharT __x) {
+    _PathCVT<_ECharT>::__append_source(__pn_,
+                                       basic_string_view<_ECharT>(&__x, 1));
+    return *this;
+  }
+
+  template <class _Source>
+  _EnableIfPathable<_Source> operator+=(const _Source& __x) {
+    return this->concat(__x);
+  }
+
+  template <class _Source>
+  _EnableIfPathable<_Source> concat(const _Source& __x) {
+    _SourceCVT<_Source>::__append_source(__pn_, __x);
+    return *this;
+  }
+
+  template <class _InputIt>
+  path& concat(_InputIt __first, _InputIt __last) {
+    typedef typename iterator_traits<_InputIt>::value_type _ItVal;
+    _PathCVT<_ItVal>::__append_range(__pn_, __first, __last);
+    return *this;
+  }
+
+  // modifiers
+  _LIBCPP_INLINE_VISIBILITY
+  void clear() noexcept { __pn_.clear(); }
+
+  path& make_preferred() {
+#if defined(_LIBCPP_WIN32API)
+    _VSTD::replace(__pn_.begin(), __pn_.end(), L'/', L'\\');
+#endif
+    return *this;
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  path& remove_filename() {
+    auto __fname = __filename();
+    if (!__fname.empty())
+      __pn_.erase(__fname.data() - __pn_.data());
+    return *this;
+  }
+
+  path& replace_filename(const path& __replacement) {
+    remove_filename();
+    return (*this /= __replacement);
+  }
+
+  path& replace_extension(const path& __replacement = path());
+
+  _LIBCPP_INLINE_VISIBILITY
+  void swap(path& __rhs) noexcept { __pn_.swap(__rhs.__pn_); }
+
+  // private helper to allow reserving memory in the path
+  _LIBCPP_INLINE_VISIBILITY
+  void __reserve(size_t __s) { __pn_.reserve(__s); }
+
+  // native format observers
+  _LIBCPP_INLINE_VISIBILITY
+  const string_type& native() const noexcept { return __pn_; }
+
+  _LIBCPP_INLINE_VISIBILITY
+  const value_type* c_str() const noexcept { return __pn_.c_str(); }
+
+  _LIBCPP_INLINE_VISIBILITY operator string_type() const { return __pn_; }
+
+#if defined(_LIBCPP_WIN32API)
+  _LIBCPP_INLINE_VISIBILITY _VSTD::wstring wstring() const { return __pn_; }
+
+  _VSTD::wstring generic_wstring() const {
+    _VSTD::wstring __s;
+    __s.resize(__pn_.size());
+    _VSTD::replace_copy(__pn_.begin(), __pn_.end(), __s.begin(), '\\', '/');
+    return __s;
+  }
+
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+  template <class _ECharT, class _Traits = char_traits<_ECharT>,
+            class _Allocator = allocator<_ECharT> >
+  basic_string<_ECharT, _Traits, _Allocator>
+  string(const _Allocator& __a = _Allocator()) const {
+    using _Str = basic_string<_ECharT, _Traits, _Allocator>;
+    _Str __s(__a);
+    __s.reserve(__pn_.size());
+    _PathExport<_ECharT>::__append(__s, __pn_);
+    return __s;
+  }
+
+  _LIBCPP_INLINE_VISIBILITY _VSTD::string string() const {
+    return string<char>();
+  }
+  _LIBCPP_INLINE_VISIBILITY __u8_string u8string() const {
+    using _CVT = __narrow_to_utf8<sizeof(wchar_t) * __CHAR_BIT__>;
+    __u8_string __s;
+    __s.reserve(__pn_.size());
+    _CVT()(back_inserter(__s), __pn_.data(), __pn_.data() + __pn_.size());
+    return __s;
+  }
+
+  _LIBCPP_INLINE_VISIBILITY _VSTD::u16string u16string() const {
+    return string<char16_t>();
+  }
+  _LIBCPP_INLINE_VISIBILITY _VSTD::u32string u32string() const {
+    return string<char32_t>();
+  }
+
+  // generic format observers
+  template <class _ECharT, class _Traits = char_traits<_ECharT>,
+            class _Allocator = allocator<_ECharT> >
+  basic_string<_ECharT, _Traits, _Allocator>
+  generic_string(const _Allocator& __a = _Allocator()) const {
+    using _Str = basic_string<_ECharT, _Traits, _Allocator>;
+    _Str __s = string<_ECharT, _Traits, _Allocator>(__a);
+    // Note: This (and generic_u8string below) is slightly suboptimal as
+    // it iterates twice over the string; once to convert it to the right
+    // character type, and once to replace path delimiters.
+    _VSTD::replace(__s.begin(), __s.end(),
+                   static_cast<_ECharT>('\\'), static_cast<_ECharT>('/'));
+    return __s;
+  }
+
+  _VSTD::string generic_string() const { return generic_string<char>(); }
+  _VSTD::u16string generic_u16string() const { return generic_string<char16_t>(); }
+  _VSTD::u32string generic_u32string() const { return generic_string<char32_t>(); }
+  __u8_string generic_u8string() const {
+    __u8_string __s = u8string();
+    _VSTD::replace(__s.begin(), __s.end(), '\\', '/');
+    return __s;
+  }
+#endif /* !_LIBCPP_HAS_NO_LOCALIZATION */
+#else /* _LIBCPP_WIN32API */
+
+  _LIBCPP_INLINE_VISIBILITY _VSTD::string string() const { return __pn_; }
+#ifndef _LIBCPP_HAS_NO_CHAR8_T
+  _LIBCPP_INLINE_VISIBILITY _VSTD::u8string u8string() const { return _VSTD::u8string(__pn_.begin(), __pn_.end()); }
+#else
+  _LIBCPP_INLINE_VISIBILITY _VSTD::string u8string() const { return __pn_; }
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+  template <class _ECharT, class _Traits = char_traits<_ECharT>,
+            class _Allocator = allocator<_ECharT> >
+  basic_string<_ECharT, _Traits, _Allocator>
+  string(const _Allocator& __a = _Allocator()) const {
+    using _CVT = __widen_from_utf8<sizeof(_ECharT) * __CHAR_BIT__>;
+    using _Str = basic_string<_ECharT, _Traits, _Allocator>;
+    _Str __s(__a);
+    __s.reserve(__pn_.size());
+    _CVT()(back_inserter(__s), __pn_.data(), __pn_.data() + __pn_.size());
+    return __s;
+  }
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+  _LIBCPP_INLINE_VISIBILITY _VSTD::wstring wstring() const {
+    return string<wchar_t>();
+  }
+#endif
+  _LIBCPP_INLINE_VISIBILITY _VSTD::u16string u16string() const {
+    return string<char16_t>();
+  }
+  _LIBCPP_INLINE_VISIBILITY _VSTD::u32string u32string() const {
+    return string<char32_t>();
+  }
+#endif /* !_LIBCPP_HAS_NO_LOCALIZATION */
+
+  // generic format observers
+  _VSTD::string generic_string() const { return __pn_; }
+#ifndef _LIBCPP_HAS_NO_CHAR8_T
+  _VSTD::u8string generic_u8string() const { return _VSTD::u8string(__pn_.begin(), __pn_.end()); }
+#else
+  _VSTD::string generic_u8string() const { return __pn_; }
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+  template <class _ECharT, class _Traits = char_traits<_ECharT>,
+            class _Allocator = allocator<_ECharT> >
+  basic_string<_ECharT, _Traits, _Allocator>
+  generic_string(const _Allocator& __a = _Allocator()) const {
+    return string<_ECharT, _Traits, _Allocator>(__a);
+  }
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+  _VSTD::wstring generic_wstring() const { return string<wchar_t>(); }
+#endif
+  _VSTD::u16string generic_u16string() const { return string<char16_t>(); }
+  _VSTD::u32string generic_u32string() const { return string<char32_t>(); }
+#endif /* !_LIBCPP_HAS_NO_LOCALIZATION */
+#endif /* !_LIBCPP_WIN32API */
+
+private:
+  int __compare(__string_view) const;
+  __string_view __root_name() const;
+  __string_view __root_directory() const;
+  __string_view __root_path_raw() const;
+  __string_view __relative_path() const;
+  __string_view __parent_path() const;
+  __string_view __filename() const;
+  __string_view __stem() const;
+  __string_view __extension() const;
+
+public:
+  // compare
+  _LIBCPP_INLINE_VISIBILITY int compare(const path& __p) const noexcept {
+    return __compare(__p.__pn_);
+  }
+  _LIBCPP_INLINE_VISIBILITY int compare(const string_type& __s) const {
+    return __compare(__s);
+  }
+  _LIBCPP_INLINE_VISIBILITY int compare(__string_view __s) const {
+    return __compare(__s);
+  }
+  _LIBCPP_INLINE_VISIBILITY int compare(const value_type* __s) const {
+    return __compare(__s);
+  }
+
+  // decomposition
+  _LIBCPP_INLINE_VISIBILITY path root_name() const {
+    return string_type(__root_name());
+  }
+  _LIBCPP_INLINE_VISIBILITY path root_directory() const {
+    return string_type(__root_directory());
+  }
+  _LIBCPP_INLINE_VISIBILITY path root_path() const {
+#if defined(_LIBCPP_WIN32API)
+    return string_type(__root_path_raw());
+#else
+    return root_name().append(string_type(__root_directory()));
+#endif
+  }
+  _LIBCPP_INLINE_VISIBILITY path relative_path() const {
+    return string_type(__relative_path());
+  }
+  _LIBCPP_INLINE_VISIBILITY path parent_path() const {
+    return string_type(__parent_path());
+  }
+  _LIBCPP_INLINE_VISIBILITY path filename() const {
+    return string_type(__filename());
+  }
+  _LIBCPP_INLINE_VISIBILITY path stem() const { return string_type(__stem()); }
+  _LIBCPP_INLINE_VISIBILITY path extension() const {
+    return string_type(__extension());
+  }
+
+  // query
+  _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_INLINE_VISIBILITY bool
+  empty() const noexcept {
+    return __pn_.empty();
+  }
+
+  _LIBCPP_INLINE_VISIBILITY bool has_root_name() const {
+    return !__root_name().empty();
+  }
+  _LIBCPP_INLINE_VISIBILITY bool has_root_directory() const {
+    return !__root_directory().empty();
+  }
+  _LIBCPP_INLINE_VISIBILITY bool has_root_path() const {
+    return !__root_path_raw().empty();
+  }
+  _LIBCPP_INLINE_VISIBILITY bool has_relative_path() const {
+    return !__relative_path().empty();
+  }
+  _LIBCPP_INLINE_VISIBILITY bool has_parent_path() const {
+    return !__parent_path().empty();
+  }
+  _LIBCPP_INLINE_VISIBILITY bool has_filename() const {
+    return !__filename().empty();
+  }
+  _LIBCPP_INLINE_VISIBILITY bool has_stem() const { return !__stem().empty(); }
+  _LIBCPP_INLINE_VISIBILITY bool has_extension() const {
+    return !__extension().empty();
+  }
+
+  _LIBCPP_INLINE_VISIBILITY bool is_absolute() const {
+#if defined(_LIBCPP_WIN32API)
+    __string_view __root_name_str = __root_name();
+    __string_view __root_dir = __root_directory();
+    if (__root_name_str.size() == 2 && __root_name_str[1] == ':') {
+      // A drive letter with no root directory is relative, e.g. x:example.
+      return !__root_dir.empty();
+    }
+    // If no root name, it's relative, e.g. \example is relative to the current drive
+    if (__root_name_str.empty())
+      return false;
+    if (__root_name_str.size() < 3)
+      return false;
+    // A server root name, like \\server, is always absolute
+    if (__root_name_str[0] != '/' && __root_name_str[0] != '\\')
+      return false;
+    if (__root_name_str[1] != '/' && __root_name_str[1] != '\\')
+      return false;
+    // Seems to be a server root name
+    return true;
+#else
+    return has_root_directory();
+#endif
+  }
+  _LIBCPP_INLINE_VISIBILITY bool is_relative() const { return !is_absolute(); }
+
+  // relative paths
+  path lexically_normal() const;
+  path lexically_relative(const path& __base) const;
+
+  _LIBCPP_INLINE_VISIBILITY path lexically_proximate(const path& __base) const {
+    path __result = this->lexically_relative(__base);
+    if (__result.native().empty())
+      return *this;
+    return __result;
+  }
+
+  // iterators
+  class _LIBCPP_TYPE_VIS iterator;
+  typedef iterator const_iterator;
+
+  iterator begin() const;
+  iterator end() const;
+
+#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+  template <class _CharT, class _Traits>
+  _LIBCPP_INLINE_VISIBILITY friend
+      typename enable_if<is_same<_CharT, value_type>::value &&
+                             is_same<_Traits, char_traits<value_type> >::value,
+                         basic_ostream<_CharT, _Traits>&>::type
+      operator<<(basic_ostream<_CharT, _Traits>& __os, const path& __p) {
+    __os << _VSTD::__quoted(__p.native());
+    return __os;
+  }
+
+  template <class _CharT, class _Traits>
+  _LIBCPP_INLINE_VISIBILITY friend
+      typename enable_if<!is_same<_CharT, value_type>::value ||
+                             !is_same<_Traits, char_traits<value_type> >::value,
+                         basic_ostream<_CharT, _Traits>&>::type
+      operator<<(basic_ostream<_CharT, _Traits>& __os, const path& __p) {
+    __os << _VSTD::__quoted(__p.string<_CharT, _Traits>());
+    return __os;
+  }
+
+  template <class _CharT, class _Traits>
+  _LIBCPP_INLINE_VISIBILITY friend basic_istream<_CharT, _Traits>&
+  operator>>(basic_istream<_CharT, _Traits>& __is, path& __p) {
+    basic_string<_CharT, _Traits> __tmp;
+    __is >> __quoted(__tmp);
+    __p = __tmp;
+    return __is;
+  }
+#endif // !_LIBCPP_HAS_NO_LOCALIZATION
+
+  friend _LIBCPP_INLINE_VISIBILITY bool operator==(const path& __lhs, const path& __rhs) noexcept {
+    return __lhs.__compare(__rhs.__pn_) == 0;
+  }
+  friend _LIBCPP_INLINE_VISIBILITY bool operator!=(const path& __lhs, const path& __rhs) noexcept {
+    return __lhs.__compare(__rhs.__pn_) != 0;
+  }
+  friend _LIBCPP_INLINE_VISIBILITY bool operator<(const path& __lhs, const path& __rhs) noexcept {
+    return __lhs.__compare(__rhs.__pn_) < 0;
+  }
+  friend _LIBCPP_INLINE_VISIBILITY bool operator<=(const path& __lhs, const path& __rhs) noexcept {
+    return __lhs.__compare(__rhs.__pn_) <= 0;
+  }
+  friend _LIBCPP_INLINE_VISIBILITY bool operator>(const path& __lhs, const path& __rhs) noexcept {
+    return __lhs.__compare(__rhs.__pn_) > 0;
+  }
+  friend _LIBCPP_INLINE_VISIBILITY bool operator>=(const path& __lhs, const path& __rhs) noexcept {
+    return __lhs.__compare(__rhs.__pn_) >= 0;
+  }
+
+  friend _LIBCPP_INLINE_VISIBILITY path operator/(const path& __lhs,
+                                                  const path& __rhs) {
+    path __result(__lhs);
+    __result /= __rhs;
+    return __result;
+  }
+private:
+  inline _LIBCPP_INLINE_VISIBILITY path&
+  __assign_view(__string_view const& __s) noexcept {
+    __pn_ = string_type(__s);
+    return *this;
+  }
+  string_type __pn_;
+};
+
+inline _LIBCPP_INLINE_VISIBILITY void swap(path& __lhs, path& __rhs) noexcept {
+  __lhs.swap(__rhs);
+}
+
+_LIBCPP_FUNC_VIS
+size_t hash_value(const path& __p) noexcept;
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_POP
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
+
+#endif // _LIBCPP_CXX03_LANG
+
+#endif // _LIBCPP___FILESYSTEM_PATH_H
diff --git a/libcxx/include/__filesystem/path_iterator.h b/libcxx/include/__filesystem/path_iterator.h
new file mode 100644
index 000000000000..62f8dc6fd357
--- /dev/null
+++ b/libcxx/include/__filesystem/path_iterator.h
@@ -0,0 +1,132 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FILESYSTEM_PATH_ITERATOR_H
+#define _LIBCPP___FILESYSTEM_PATH_ITERATOR_H
+
+#include <__availability>
+#include <__config>
+#include <__filesystem/path.h>
+#include <__iterator/iterator_traits.h>
+#include <__debug>
+#include <cstddef>
+#include <string>
+#include <string_view>
+
+#ifndef _LIBCPP_CXX03_LANG
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_PUSH
+
+class _LIBCPP_TYPE_VIS path::iterator {
+public:
+  enum _ParserState : unsigned char {
+    _Singular,
+    _BeforeBegin,
+    _InRootName,
+    _InRootDir,
+    _InFilenames,
+    _InTrailingSep,
+    _AtEnd
+  };
+
+public:
+  typedef bidirectional_iterator_tag iterator_category;
+
+  typedef path value_type;
+  typedef ptrdiff_t difference_type;
+  typedef const path* pointer;
+  typedef const path& reference;
+
+  typedef void
+      __stashing_iterator_tag; // See reverse_iterator and __is_stashing_iterator
+
+public:
+  _LIBCPP_INLINE_VISIBILITY
+  iterator()
+      : __stashed_elem_(), __path_ptr_(nullptr), __entry_(),
+        __state_(_Singular) {}
+
+  iterator(const iterator&) = default;
+  ~iterator() = default;
+
+  iterator& operator=(const iterator&) = default;
+
+  _LIBCPP_INLINE_VISIBILITY
+  reference operator*() const { return __stashed_elem_; }
+
+  _LIBCPP_INLINE_VISIBILITY
+  pointer operator->() const { return &__stashed_elem_; }
+
+  _LIBCPP_INLINE_VISIBILITY
+  iterator& operator++() {
+    _LIBCPP_ASSERT(__state_ != _Singular,
+                   "attempting to increment a singular iterator");
+    _LIBCPP_ASSERT(__state_ != _AtEnd,
+                   "attempting to increment the end iterator");
+    return __increment();
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  iterator operator++(int) {
+    iterator __it(*this);
+    this->operator++();
+    return __it;
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  iterator& operator--() {
+    _LIBCPP_ASSERT(__state_ != _Singular,
+                   "attempting to decrement a singular iterator");
+    _LIBCPP_ASSERT(__entry_.data() != __path_ptr_->native().data(),
+                   "attempting to decrement the begin iterator");
+    return __decrement();
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  iterator operator--(int) {
+    iterator __it(*this);
+    this->operator--();
+    return __it;
+  }
+
+private:
+  friend class path;
+
+  inline _LIBCPP_INLINE_VISIBILITY friend bool operator==(const iterator&,
+                                                          const iterator&);
+
+  iterator& __increment();
+  iterator& __decrement();
+
+  path __stashed_elem_;
+  const path* __path_ptr_;
+  path::__string_view __entry_;
+  _ParserState __state_;
+};
+
+inline _LIBCPP_INLINE_VISIBILITY bool operator==(const path::iterator& __lhs,
+                                                 const path::iterator& __rhs) {
+  return __lhs.__path_ptr_ == __rhs.__path_ptr_ &&
+         __lhs.__entry_.data() == __rhs.__entry_.data();
+}
+
+inline _LIBCPP_INLINE_VISIBILITY bool operator!=(const path::iterator& __lhs,
+                                                 const path::iterator& __rhs) {
+  return !(__lhs == __rhs);
+}
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_POP
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
+
+#endif // _LIBCPP_CXX03_LANG
+
+#endif // _LIBCPP___FILESYSTEM_PATH_ITERATOR_H
diff --git a/libcxx/include/__filesystem/perm_options.h b/libcxx/include/__filesystem/perm_options.h
new file mode 100644
index 000000000000..62cd8f575650
--- /dev/null
+++ b/libcxx/include/__filesystem/perm_options.h
@@ -0,0 +1,73 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FILESYSTEM_PERM_OPTIONS_H
+#define _LIBCPP___FILESYSTEM_PERM_OPTIONS_H
+
+#include <__availability>
+#include <__config>
+
+#ifndef _LIBCPP_CXX03_LANG
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_PUSH
+
+enum class _LIBCPP_ENUM_VIS perm_options : unsigned char {
+  replace = 1,
+  add = 2,
+  remove = 4,
+  nofollow = 8
+};
+
+_LIBCPP_INLINE_VISIBILITY
+inline constexpr perm_options operator&(perm_options _LHS, perm_options _RHS) {
+  return static_cast<perm_options>(static_cast<unsigned>(_LHS) &
+                                   static_cast<unsigned>(_RHS));
+}
+
+_LIBCPP_INLINE_VISIBILITY
+inline constexpr perm_options operator|(perm_options _LHS, perm_options _RHS) {
+  return static_cast<perm_options>(static_cast<unsigned>(_LHS) |
+                                   static_cast<unsigned>(_RHS));
+}
+
+_LIBCPP_INLINE_VISIBILITY
+inline constexpr perm_options operator^(perm_options _LHS, perm_options _RHS) {
+  return static_cast<perm_options>(static_cast<unsigned>(_LHS) ^
+                                   static_cast<unsigned>(_RHS));
+}
+
+_LIBCPP_INLINE_VISIBILITY
+inline constexpr perm_options operator~(perm_options _LHS) {
+  return static_cast<perm_options>(~static_cast<unsigned>(_LHS));
+}
+
+_LIBCPP_INLINE_VISIBILITY
+inline perm_options& operator&=(perm_options& _LHS, perm_options _RHS) {
+  return _LHS = _LHS & _RHS;
+}
+
+_LIBCPP_INLINE_VISIBILITY
+inline perm_options& operator|=(perm_options& _LHS, perm_options _RHS) {
+  return _LHS = _LHS | _RHS;
+}
+
+_LIBCPP_INLINE_VISIBILITY
+inline perm_options& operator^=(perm_options& _LHS, perm_options _RHS) {
+  return _LHS = _LHS ^ _RHS;
+}
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_POP
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
+
+#endif // _LIBCPP_CXX03_LANG
+
+#endif // _LIBCPP___FILESYSTEM_PERM_OPTIONS_H
diff --git a/libcxx/include/__filesystem/perms.h b/libcxx/include/__filesystem/perms.h
new file mode 100644
index 000000000000..832f8b07e55c
--- /dev/null
+++ b/libcxx/include/__filesystem/perms.h
@@ -0,0 +1,91 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FILESYSTEM_PERMS_H
+#define _LIBCPP___FILESYSTEM_PERMS_H
+
+#include <__availability>
+#include <__config>
+
+#ifndef _LIBCPP_CXX03_LANG
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_PUSH
+
+// On Windows, these permission bits map to one single readonly flag per
+// file, and the executable bit is always returned as set. When setting
+// permissions, as long as the write bit is set for either owner, group or
+// others, the readonly flag is cleared.
+enum class _LIBCPP_ENUM_VIS perms : unsigned {
+  none = 0,
+
+  owner_read = 0400,
+  owner_write = 0200,
+  owner_exec = 0100,
+  owner_all = 0700,
+
+  group_read = 040,
+  group_write = 020,
+  group_exec = 010,
+  group_all = 070,
+
+  others_read = 04,
+  others_write = 02,
+  others_exec = 01,
+  others_all = 07,
+
+  all = 0777,
+
+  set_uid = 04000,
+  set_gid = 02000,
+  sticky_bit = 01000,
+  mask = 07777,
+  unknown = 0xFFFF,
+};
+
+_LIBCPP_INLINE_VISIBILITY
+inline constexpr perms operator&(perms _LHS, perms _RHS) {
+  return static_cast<perms>(static_cast<unsigned>(_LHS) &
+                            static_cast<unsigned>(_RHS));
+}
+
+_LIBCPP_INLINE_VISIBILITY
+inline constexpr perms operator|(perms _LHS, perms _RHS) {
+  return static_cast<perms>(static_cast<unsigned>(_LHS) |
+                            static_cast<unsigned>(_RHS));
+}
+
+_LIBCPP_INLINE_VISIBILITY
+inline constexpr perms operator^(perms _LHS, perms _RHS) {
+  return static_cast<perms>(static_cast<unsigned>(_LHS) ^
+                            static_cast<unsigned>(_RHS));
+}
+
+_LIBCPP_INLINE_VISIBILITY
+inline constexpr perms operator~(perms _LHS) {
+  return static_cast<perms>(~static_cast<unsigned>(_LHS));
+}
+
+_LIBCPP_INLINE_VISIBILITY
+inline perms& operator&=(perms& _LHS, perms _RHS) { return _LHS = _LHS & _RHS; }
+
+_LIBCPP_INLINE_VISIBILITY
+inline perms& operator|=(perms& _LHS, perms _RHS) { return _LHS = _LHS | _RHS; }
+
+_LIBCPP_INLINE_VISIBILITY
+inline perms& operator^=(perms& _LHS, perms _RHS) { return _LHS = _LHS ^ _RHS; }
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_POP
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
+
+#endif // _LIBCPP_CXX03_LANG
+
+#endif // _LIBCPP___FILESYSTEM_PERMS_H
diff --git a/libcxx/include/__filesystem/recursive_directory_iterator.h b/libcxx/include/__filesystem/recursive_directory_iterator.h
new file mode 100644
index 000000000000..db7e793e8530
--- /dev/null
+++ b/libcxx/include/__filesystem/recursive_directory_iterator.h
@@ -0,0 +1,181 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FILESYSTEM_RECURSIVE_DIRECTORY_ITERATOR_H
+#define _LIBCPP___FILESYSTEM_RECURSIVE_DIRECTORY_ITERATOR_H
+
+#include <__availability>
+#include <__config>
+#include <__filesystem/directory_entry.h>
+#include <__filesystem/directory_options.h>
+#include <__filesystem/path.h>
+#include <__iterator/iterator_traits.h>
+#include <__memory/shared_ptr.h>
+#include <__ranges/enable_borrowed_range.h>
+#include <__ranges/enable_view.h>
+#include <cstddef>
+#include <system_error>
+
+#ifndef _LIBCPP_CXX03_LANG
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_PUSH
+
+class recursive_directory_iterator {
+public:
+  using value_type = directory_entry;
+  using difference_type = ptrdiff_t;
+  using pointer = directory_entry const*;
+  using reference = directory_entry const&;
+  using iterator_category = input_iterator_tag;
+
+public:
+  // constructors and destructor
+  _LIBCPP_INLINE_VISIBILITY
+  recursive_directory_iterator() noexcept : __rec_(false) {}
+
+  _LIBCPP_INLINE_VISIBILITY
+  explicit recursive_directory_iterator(
+      const path& __p, directory_options __xoptions = directory_options::none)
+      : recursive_directory_iterator(__p, __xoptions, nullptr) {}
+
+  _LIBCPP_INLINE_VISIBILITY
+  recursive_directory_iterator(const path& __p, directory_options __xoptions,
+                               error_code& __ec)
+      : recursive_directory_iterator(__p, __xoptions, &__ec) {}
+
+  _LIBCPP_INLINE_VISIBILITY
+  recursive_directory_iterator(const path& __p, error_code& __ec)
+      : recursive_directory_iterator(__p, directory_options::none, &__ec) {}
+
+  recursive_directory_iterator(const recursive_directory_iterator&) = default;
+  recursive_directory_iterator(recursive_directory_iterator&&) = default;
+
+  recursive_directory_iterator&
+  operator=(const recursive_directory_iterator&) = default;
+
+  _LIBCPP_INLINE_VISIBILITY
+  recursive_directory_iterator&
+  operator=(recursive_directory_iterator&& __o) noexcept {
+    // non-default implementation provided to support self-move assign.
+    if (this != &__o) {
+      __imp_ = _VSTD::move(__o.__imp_);
+      __rec_ = __o.__rec_;
+    }
+    return *this;
+  }
+
+  ~recursive_directory_iterator() = default;
+
+  _LIBCPP_INLINE_VISIBILITY
+  const directory_entry& operator*() const { return __dereference(); }
+
+  _LIBCPP_INLINE_VISIBILITY
+  const directory_entry* operator->() const { return &__dereference(); }
+
+  recursive_directory_iterator& operator++() { return __increment(); }
+
+  _LIBCPP_INLINE_VISIBILITY
+  __dir_element_proxy operator++(int) {
+    __dir_element_proxy __p(**this);
+    __increment();
+    return __p;
+  }
+
+  _LIBCPP_INLINE_VISIBILITY
+  recursive_directory_iterator& increment(error_code& __ec) {
+    return __increment(&__ec);
+  }
+
+  _LIBCPP_FUNC_VIS directory_options options() const;
+  _LIBCPP_FUNC_VIS int depth() const;
+
+  _LIBCPP_INLINE_VISIBILITY
+  void pop() { __pop(); }
+
+  _LIBCPP_INLINE_VISIBILITY
+  void pop(error_code& __ec) { __pop(&__ec); }
+
+  _LIBCPP_INLINE_VISIBILITY
+  bool recursion_pending() const { return __rec_; }
+
+  _LIBCPP_INLINE_VISIBILITY
+  void disable_recursion_pending() { __rec_ = false; }
+
+private:
+  _LIBCPP_FUNC_VIS
+  recursive_directory_iterator(const path& __p, directory_options __opt,
+                               error_code* __ec);
+
+  _LIBCPP_FUNC_VIS
+  const directory_entry& __dereference() const;
+
+  _LIBCPP_FUNC_VIS
+  bool __try_recursion(error_code* __ec);
+
+  _LIBCPP_FUNC_VIS
+  void __advance(error_code* __ec = nullptr);
+
+  _LIBCPP_FUNC_VIS
+  recursive_directory_iterator& __increment(error_code* __ec = nullptr);
+
+  _LIBCPP_FUNC_VIS
+  void __pop(error_code* __ec = nullptr);
+
+  inline _LIBCPP_INLINE_VISIBILITY friend bool
+  operator==(const recursive_directory_iterator&,
+             const recursive_directory_iterator&) noexcept;
+
+  struct _LIBCPP_HIDDEN __shared_imp;
+  shared_ptr<__shared_imp> __imp_;
+  bool __rec_;
+}; // class recursive_directory_iterator
+
+inline _LIBCPP_INLINE_VISIBILITY bool
+operator==(const recursive_directory_iterator& __lhs,
+           const recursive_directory_iterator& __rhs) noexcept {
+  return __lhs.__imp_ == __rhs.__imp_;
+}
+
+_LIBCPP_INLINE_VISIBILITY
+inline bool operator!=(const recursive_directory_iterator& __lhs,
+                       const recursive_directory_iterator& __rhs) noexcept {
+  return !(__lhs == __rhs);
+}
+// enable recursive_directory_iterator range-based for statements
+inline _LIBCPP_INLINE_VISIBILITY recursive_directory_iterator
+begin(recursive_directory_iterator __iter) noexcept {
+  return __iter;
+}
+
+inline _LIBCPP_INLINE_VISIBILITY recursive_directory_iterator
+end(recursive_directory_iterator) noexcept {
+  return recursive_directory_iterator();
+}
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_POP
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
+
+#if !defined(_LIBCPP_HAS_NO_RANGES)
+
+template <>
+_LIBCPP_AVAILABILITY_FILESYSTEM
+inline constexpr bool _VSTD::ranges::enable_borrowed_range<_VSTD_FS::recursive_directory_iterator> = true;
+
+template <>
+_LIBCPP_AVAILABILITY_FILESYSTEM
+inline constexpr bool _VSTD::ranges::enable_view<_VSTD_FS::recursive_directory_iterator> = true;
+
+#endif
+
+#endif // _LIBCPP_CXX03_LANG
+
+#endif // _LIBCPP___FILESYSTEM_RECURSIVE_DIRECTORY_ITERATOR_H
diff --git a/libcxx/include/__filesystem/space_info.h b/libcxx/include/__filesystem/space_info.h
new file mode 100644
index 000000000000..098f085678e4
--- /dev/null
+++ b/libcxx/include/__filesystem/space_info.h
@@ -0,0 +1,35 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FILESYSTEM_SPACE_INFO_H
+#define _LIBCPP___FILESYSTEM_SPACE_INFO_H
+
+#include <__availability>
+#include <__config>
+#include <cstdint>
+
+#ifndef _LIBCPP_CXX03_LANG
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_PUSH
+
+struct _LIBCPP_TYPE_VIS space_info {
+  uintmax_t capacity;
+  uintmax_t free;
+  uintmax_t available;
+};
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_POP
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
+
+#endif // _LIBCPP_CXX03_LANG
+
+#endif // _LIBCPP___FILESYSTEM_SPACE_INFO_H
diff --git a/libcxx/include/__filesystem/u8path.h b/libcxx/include/__filesystem/u8path.h
new file mode 100644
index 000000000000..dca3b0c5028b
--- /dev/null
+++ b/libcxx/include/__filesystem/u8path.h
@@ -0,0 +1,96 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FILESYSTEM_U8PATH_H
+#define _LIBCPP___FILESYSTEM_U8PATH_H
+
+#include <__availability>
+#include <__config>
+#include <__filesystem/path.h>
+#include <type_traits>
+
+#ifndef _LIBCPP_CXX03_LANG
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_PUSH
+
+template <class _InputIt>
+_LIBCPP_INLINE_VISIBILITY _LIBCPP_DEPRECATED_WITH_CHAR8_T
+    typename enable_if<__is_pathable<_InputIt>::value, path>::type
+    u8path(_InputIt __f, _InputIt __l) {
+  static_assert(
+#ifndef _LIBCPP_HAS_NO_CHAR8_T
+      is_same<typename __is_pathable<_InputIt>::__char_type, char8_t>::value ||
+#endif
+      is_same<typename __is_pathable<_InputIt>::__char_type, char>::value,
+      "u8path(Iter, Iter) requires Iter have a value_type of type 'char'"
+      " or 'char8_t'");
+#if defined(_LIBCPP_WIN32API)
+  string __tmp(__f, __l);
+  using _CVT = __widen_from_utf8<sizeof(wchar_t) * __CHAR_BIT__>;
+  _VSTD::wstring __w;
+  __w.reserve(__tmp.size());
+  _CVT()(back_inserter(__w), __tmp.data(), __tmp.data() + __tmp.size());
+  return path(__w);
+#else
+  return path(__f, __l);
+#endif /* !_LIBCPP_WIN32API */
+}
+
+#if defined(_LIBCPP_WIN32API)
+template <class _InputIt>
+_LIBCPP_INLINE_VISIBILITY _LIBCPP_DEPRECATED_WITH_CHAR8_T
+    typename enable_if<__is_pathable<_InputIt>::value, path>::type
+    u8path(_InputIt __f, _NullSentinel) {
+  static_assert(
+#ifndef _LIBCPP_HAS_NO_CHAR8_T
+      is_same<typename __is_pathable<_InputIt>::__char_type, char8_t>::value ||
+#endif
+      is_same<typename __is_pathable<_InputIt>::__char_type, char>::value,
+      "u8path(Iter, Iter) requires Iter have a value_type of type 'char'"
+      " or 'char8_t'");
+  string __tmp;
+  const char __sentinel = char{};
+  for (; *__f != __sentinel; ++__f)
+    __tmp.push_back(*__f);
+  using _CVT = __widen_from_utf8<sizeof(wchar_t) * __CHAR_BIT__>;
+  _VSTD::wstring __w;
+  __w.reserve(__tmp.size());
+  _CVT()(back_inserter(__w), __tmp.data(), __tmp.data() + __tmp.size());
+  return path(__w);
+}
+#endif /* _LIBCPP_WIN32API */
+
+template <class _Source>
+_LIBCPP_INLINE_VISIBILITY _LIBCPP_DEPRECATED_WITH_CHAR8_T
+    typename enable_if<__is_pathable<_Source>::value, path>::type
+    u8path(const _Source& __s) {
+  static_assert(
+#ifndef _LIBCPP_HAS_NO_CHAR8_T
+      is_same<typename __is_pathable<_Source>::__char_type, char8_t>::value ||
+#endif
+      is_same<typename __is_pathable<_Source>::__char_type, char>::value,
+      "u8path(Source const&) requires Source have a character type of type "
+      "'char' or 'char8_t'");
+#if defined(_LIBCPP_WIN32API)
+  using _Traits = __is_pathable<_Source>;
+  return u8path(_VSTD::__unwrap_iter(_Traits::__range_begin(__s)), _VSTD::__unwrap_iter(_Traits::__range_end(__s)));
+#else
+  return path(__s);
+#endif
+}
+
+_LIBCPP_AVAILABILITY_FILESYSTEM_POP
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
+
+#endif // _LIBCPP_CXX03_LANG
+
+#endif // _LIBCPP___FILESYSTEM_U8PATH_H
diff --git a/libcxx/include/__format/format_arg.h b/libcxx/include/__format/format_arg.h
index a9a8c1f0da03..59429c13d415 100644
--- a/libcxx/include/__format/format_arg.h
+++ b/libcxx/include/__format/format_arg.h
@@ -37,21 +37,20 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 #if !defined(_LIBCPP_HAS_NO_CONCEPTS)
 
 namespace __format {
-/** The type stored in @ref basic_format_arg. */
+/// The type stored in @ref basic_format_arg.
+///
+/// @note The 128-bit types are unconditionally in the list to avoid the values
+/// of the enums to depend on the availability of 128-bit integers.
 enum class _LIBCPP_ENUM_VIS __arg_t : uint8_t {
   __none,
   __boolean,
   __char_type,
   __int,
   __long_long,
-#ifndef _LIBCPP_HAS_NO_INT128
   __i128,
-#endif
   __unsigned,
   __unsigned_long_long,
-#ifndef _LIBCPP_HAS_NO_INT128
   __u128,
-#endif
   __float,
   __double,
   __long_double,
@@ -75,18 +74,22 @@ visit_format_arg(_Visitor&& __vis, basic_format_arg<_Context> __arg) {
     return _VSTD::invoke(_VSTD::forward<_Visitor>(__vis), __arg.__int);
   case __format::__arg_t::__long_long:
     return _VSTD::invoke(_VSTD::forward<_Visitor>(__vis), __arg.__long_long);
-#ifndef _LIBCPP_HAS_NO_INT128
   case __format::__arg_t::__i128:
+#ifndef _LIBCPP_HAS_NO_INT128
     return _VSTD::invoke(_VSTD::forward<_Visitor>(__vis), __arg.__i128);
+#else
+    _LIBCPP_UNREACHABLE();
 #endif
   case __format::__arg_t::__unsigned:
     return _VSTD::invoke(_VSTD::forward<_Visitor>(__vis), __arg.__unsigned);
   case __format::__arg_t::__unsigned_long_long:
     return _VSTD::invoke(_VSTD::forward<_Visitor>(__vis),
                          __arg.__unsigned_long_long);
-#ifndef _LIBCPP_HAS_NO_INT128
   case __format::__arg_t::__u128:
+#ifndef _LIBCPP_HAS_NO_INT128
     return _VSTD::invoke(_VSTD::forward<_Visitor>(__vis), __arg.__u128);
+#else
+   _LIBCPP_UNREACHABLE();
 #endif
   case __format::__arg_t::__float:
     return _VSTD::invoke(_VSTD::forward<_Visitor>(__vis), __arg.__float);
diff --git a/libcxx/include/__format/format_context.h b/libcxx/include/__format/format_context.h
index b4fe5cc7b12c..f8ec7c8eb001 100644
--- a/libcxx/include/__format/format_context.h
+++ b/libcxx/include/__format/format_context.h
@@ -14,10 +14,9 @@
 #include <__config>
 #include <__format/format_args.h>
 #include <__format/format_fwd.h>
+#include <__iterator/back_insert_iterator.h>
 #include <__iterator/concepts.h>
 #include <concepts>
-#include <iterator>
-#include <string>
 
 #ifndef _LIBCPP_HAS_NO_LOCALIZATION
 #include <locale>
diff --git a/libcxx/include/__format/formatter_string.h b/libcxx/include/__format/formatter_string.h
index 2be36a1ba947..75a81f5184a0 100644
--- a/libcxx/include/__format/formatter_string.h
+++ b/libcxx/include/__format/formatter_string.h
@@ -16,7 +16,6 @@
 #include <__format/format_string.h>
 #include <__format/formatter.h>
 #include <__format/parser_std_format_spec.h>
-#include <algorithm>
 #include <string_view>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__functional/bind.h b/libcxx/include/__functional/bind.h
index 0b74d91b7746..0eb95c824664 100644
--- a/libcxx/include/__functional/bind.h
+++ b/libcxx/include/__functional/bind.h
@@ -70,7 +70,7 @@ _LIBCPP_FUNC_VIS extern const __ph<10> _10;
 /* inline */ constexpr __ph<10> _10{};
 #endif // defined(_LIBCPP_CXX03_LANG) || defined(_LIBCPP_BUILDING_LIBRARY)
 
-}  // placeholders
+} // namespace placeholders
 
 template<int _Np>
 struct __is_placeholder<placeholders::__ph<_Np> >
diff --git a/libcxx/include/__functional/function.h b/libcxx/include/__functional/function.h
index 83fd7c12de2f..8336d85adf2e 100644
--- a/libcxx/include/__functional/function.h
+++ b/libcxx/include/__functional/function.h
@@ -946,7 +946,7 @@ public:
 
 #endif // _LIBCPP_HAS_EXTENSION_BLOCKS && !_LIBCPP_HAS_OBJC_ARC
 
-}  // __function
+} // namespace __function
 
 template<class _Rp, class ..._ArgTypes>
 class _LIBCPP_TEMPLATE_VIS function<_Rp(_ArgTypes...)>
@@ -1717,13 +1717,11 @@ public:
     // 20.7.16.2.3, function capacity:
     _LIBCPP_INLINE_VISIBILITY explicit operator bool() const {return __f_;}
 
-private:
-    // deleted overloads close possible hole in the type system
     template<class _R2>
-      bool operator==(const function<_R2()>&) const;// = delete;
+      bool operator==(const function<_R2()>&) const = delete;
     template<class _R2>
-      bool operator!=(const function<_R2()>&) const;// = delete;
-public:
+      bool operator!=(const function<_R2()>&) const = delete;
+
     // 20.7.16.2.4, function invocation:
     _Rp operator()() const;
 
@@ -1997,13 +1995,11 @@ public:
     // 20.7.16.2.3, function capacity:
     _LIBCPP_INLINE_VISIBILITY explicit operator bool() const {return __f_;}
 
-private:
-    // deleted overloads close possible hole in the type system
     template<class _R2, class _B0>
-      bool operator==(const function<_R2(_B0)>&) const;// = delete;
+      bool operator==(const function<_R2(_B0)>&) const = delete;
     template<class _R2, class _B0>
-      bool operator!=(const function<_R2(_B0)>&) const;// = delete;
-public:
+      bool operator!=(const function<_R2(_B0)>&) const = delete;
+
     // 20.7.16.2.4, function invocation:
     _Rp operator()(_A0) const;
 
@@ -2277,13 +2273,11 @@ public:
     // 20.7.16.2.3, function capacity:
     _LIBCPP_INLINE_VISIBILITY explicit operator bool() const {return __f_;}
 
-private:
-    // deleted overloads close possible hole in the type system
     template<class _R2, class _B0, class _B1>
-      bool operator==(const function<_R2(_B0, _B1)>&) const;// = delete;
+      bool operator==(const function<_R2(_B0, _B1)>&) const = delete;
     template<class _R2, class _B0, class _B1>
-      bool operator!=(const function<_R2(_B0, _B1)>&) const;// = delete;
-public:
+      bool operator!=(const function<_R2(_B0, _B1)>&) const = delete;
+
     // 20.7.16.2.4, function invocation:
     _Rp operator()(_A0, _A1) const;
 
@@ -2556,13 +2550,11 @@ public:
     // 20.7.16.2.3, function capacity:
     _LIBCPP_INLINE_VISIBILITY explicit operator bool() const {return __f_;}
 
-private:
-    // deleted overloads close possible hole in the type system
     template<class _R2, class _B0, class _B1, class _B2>
-      bool operator==(const function<_R2(_B0, _B1, _B2)>&) const;// = delete;
+      bool operator==(const function<_R2(_B0, _B1, _B2)>&) const = delete;
     template<class _R2, class _B0, class _B1, class _B2>
-      bool operator!=(const function<_R2(_B0, _B1, _B2)>&) const;// = delete;
-public:
+      bool operator!=(const function<_R2(_B0, _B1, _B2)>&) const = delete;
+
     // 20.7.16.2.4, function invocation:
     _Rp operator()(_A0, _A1, _A2) const;
 
diff --git a/libcxx/include/__iterator/advance.h b/libcxx/include/__iterator/advance.h
index a60052a08f0d..2236a57936cc 100644
--- a/libcxx/include/__iterator/advance.h
+++ b/libcxx/include/__iterator/advance.h
@@ -69,6 +69,7 @@ void advance(_InputIter& __i, _Distance __orig_n) {
 
 namespace ranges {
 // [range.iter.op.advance]
+// TODO(varconst): rename `__advance_fn` to `__fn`.
 struct __advance_fn final : private __function_like {
 private:
   template <class _Tp>
diff --git a/libcxx/include/__iterator/concepts.h b/libcxx/include/__iterator/concepts.h
index 531acdf0a5b2..d7a666743afb 100644
--- a/libcxx/include/__iterator/concepts.h
+++ b/libcxx/include/__iterator/concepts.h
@@ -28,8 +28,6 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 #if !defined(_LIBCPP_HAS_NO_RANGES)
 
-// clang-format off
-
 // [iterator.concept.readable]
 template<class _In>
 concept __indirectly_readable_impl =
@@ -259,8 +257,6 @@ concept indirectly_movable_storable =
 // Note: indirectly_swappable is located in iter_swap.h to prevent a dependency cycle
 // (both iter_swap and indirectly_swappable require indirectly_readable).
 
-// clang-format on
-
 #endif // !defined(_LIBCPP_HAS_NO_RANGES)
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__iterator/next.h b/libcxx/include/__iterator/next.h
index 0464708607d4..d332abfa8e0e 100644
--- a/libcxx/include/__iterator/next.h
+++ b/libcxx/include/__iterator/next.h
@@ -39,6 +39,7 @@ inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX14
 #if !defined(_LIBCPP_HAS_NO_RANGES)
 
 namespace ranges {
+// TODO(varconst): rename `__next_fn` to `__fn`.
 struct __next_fn final : private __function_like {
   _LIBCPP_HIDE_FROM_ABI
   constexpr explicit __next_fn(__tag __x) noexcept : __function_like(__x) {}
@@ -79,4 +80,4 @@ inline constexpr auto next = __next_fn(__function_like::__tag());
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___ITERATOR_PRIMITIVES_H
+#endif // _LIBCPP___ITERATOR_NEXT_H
diff --git a/libcxx/include/__iterator/prev.h b/libcxx/include/__iterator/prev.h
index cbe1e8759af3..57f2d04a1325 100644
--- a/libcxx/include/__iterator/prev.h
+++ b/libcxx/include/__iterator/prev.h
@@ -38,6 +38,7 @@ inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX14
 #if !defined(_LIBCPP_HAS_NO_RANGES)
 
 namespace ranges {
+// TODO(varconst): rename `__prev_fn` to `__fn`.
 struct __prev_fn final : private __function_like {
   _LIBCPP_HIDE_FROM_ABI
   constexpr explicit __prev_fn(__tag __x) noexcept : __function_like(__x) {}
diff --git a/libcxx/include/__locale b/libcxx/include/__locale
index 4296adbbd8e9..2582a90bb578 100644
--- a/libcxx/include/__locale
+++ b/libcxx/include/__locale
@@ -208,10 +208,11 @@ class _LIBCPP_TYPE_VIS locale::id
     static int32_t __next_id;
 public:
     _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR id() :__id_(0) {}
+    void operator=(const id&) = delete;
+    id(const id&) = delete;
+
 private:
     void __init();
-    void operator=(const id&); // = delete;
-    id(const id&); // = delete;
 public:  // only needed for tests
     long __get();
 
diff --git a/libcxx/include/__memory/allocator.h b/libcxx/include/__memory/allocator.h
index 283212fb703d..708bd82b02d2 100644
--- a/libcxx/include/__memory/allocator.h
+++ b/libcxx/include/__memory/allocator.h
@@ -89,7 +89,7 @@ public:
     typedef true_type   is_always_equal;
 
     _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
-    allocator() _NOEXCEPT _LIBCPP_DEFAULT
+    allocator() _NOEXCEPT = default;
 
     template <class _Up>
     _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
@@ -171,7 +171,7 @@ public:
     typedef true_type   is_always_equal;
 
     _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
-    allocator() _NOEXCEPT _LIBCPP_DEFAULT
+    allocator() _NOEXCEPT = default;
 
     template <class _Up>
     _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
diff --git a/libcxx/include/__memory/compressed_pair.h b/libcxx/include/__memory/compressed_pair.h
index fd1fcbe5bf39..fd6d7109f8eb 100644
--- a/libcxx/include/__memory/compressed_pair.h
+++ b/libcxx/include/__memory/compressed_pair.h
@@ -58,10 +58,8 @@ struct __compressed_pair_elem {
       : __value_(_VSTD::forward<_Args>(_VSTD::get<_Indexes>(__args))...) {}
 #endif
 
-
-  _LIBCPP_INLINE_VISIBILITY reference __get() _NOEXCEPT { return __value_; }
-  _LIBCPP_INLINE_VISIBILITY
-  const_reference __get() const _NOEXCEPT { return __value_; }
+  _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11 reference __get() _NOEXCEPT { return __value_; }
+  _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR const_reference __get() const _NOEXCEPT { return __value_; }
 
 private:
   _Tp __value_;
@@ -97,9 +95,8 @@ struct __compressed_pair_elem<_Tp, _Idx, true> : private _Tp {
       : __value_type(_VSTD::forward<_Args>(_VSTD::get<_Indexes>(__args))...) {}
 #endif
 
-  _LIBCPP_INLINE_VISIBILITY reference __get() _NOEXCEPT { return *this; }
-  _LIBCPP_INLINE_VISIBILITY
-  const_reference __get() const _NOEXCEPT { return *this; }
+  _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11 reference __get() _NOEXCEPT { return *this; }
+  _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR const_reference __get() const _NOEXCEPT { return *this; }
 };
 
 template <class _T1, class _T2>
@@ -143,23 +140,19 @@ public:
                typename __make_tuple_indices<sizeof...(_Args2)>::type()) {}
 #endif
 
-  _LIBCPP_INLINE_VISIBILITY
-  typename _Base1::reference first() _NOEXCEPT {
+  _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11 typename _Base1::reference first() _NOEXCEPT {
     return static_cast<_Base1&>(*this).__get();
   }
 
-  _LIBCPP_INLINE_VISIBILITY
-  typename _Base1::const_reference first() const _NOEXCEPT {
+  _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR typename _Base1::const_reference first() const _NOEXCEPT {
     return static_cast<_Base1 const&>(*this).__get();
   }
 
-  _LIBCPP_INLINE_VISIBILITY
-  typename _Base2::reference second() _NOEXCEPT {
+  _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11 typename _Base2::reference second() _NOEXCEPT {
     return static_cast<_Base2&>(*this).__get();
   }
 
-  _LIBCPP_INLINE_VISIBILITY
-  typename _Base2::const_reference second() const _NOEXCEPT {
+  _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR typename _Base2::const_reference second() const _NOEXCEPT {
     return static_cast<_Base2 const&>(*this).__get();
   }
 
@@ -172,11 +165,8 @@ public:
     return static_cast<_Base2*>(__pair);
   }
 
-  _LIBCPP_INLINE_VISIBILITY
-  void swap(__compressed_pair& __x)
-    _NOEXCEPT_(__is_nothrow_swappable<_T1>::value &&
-               __is_nothrow_swappable<_T2>::value)
-  {
+  _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11 void swap(__compressed_pair& __x)
+      _NOEXCEPT_(__is_nothrow_swappable<_T1>::value && __is_nothrow_swappable<_T2>::value) {
     using _VSTD::swap;
     swap(first(), __x.first());
     swap(second(), __x.second());
@@ -184,10 +174,9 @@ public:
 };
 
 template <class _T1, class _T2>
-inline _LIBCPP_INLINE_VISIBILITY
+inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11
 void swap(__compressed_pair<_T1, _T2>& __x, __compressed_pair<_T1, _T2>& __y)
-    _NOEXCEPT_(__is_nothrow_swappable<_T1>::value &&
-               __is_nothrow_swappable<_T2>::value) {
+    _NOEXCEPT_(__is_nothrow_swappable<_T1>::value && __is_nothrow_swappable<_T2>::value) {
   __x.swap(__y);
 }
 
diff --git a/libcxx/include/__memory/concepts.h b/libcxx/include/__memory/concepts.h
new file mode 100644
index 000000000000..4029b590fe8c
--- /dev/null
+++ b/libcxx/include/__memory/concepts.h
@@ -0,0 +1,66 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_CONCEPTS_H
+#define _LIBCPP___MEMORY_CONCEPTS_H
+
+#include <__config>
+#include <__iterator/concepts.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/readable_traits.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <concepts>
+#include <type_traits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if !defined(_LIBCPP_HAS_NO_RANGES)
+namespace ranges {
+
+// [special.mem.concepts]
+
+// This concept ensures that uninitialized algorithms can construct an object
+// at the address pointed-to by the iterator, which requires an lvalue.
+template <class _Ip>
+concept __nothrow_input_iterator =
+    input_iterator<_Ip> &&
+    is_lvalue_reference_v<iter_reference_t<_Ip>> &&
+    same_as<remove_cvref_t<iter_reference_t<_Ip>>, iter_value_t<_Ip>>;
+
+template <class _Sp, class _Ip>
+concept __nothrow_sentinel_for = sentinel_for<_Sp, _Ip>;
+
+template <class _Rp>
+concept __nothrow_input_range =
+    range<_Rp> &&
+    __nothrow_input_iterator<iterator_t<_Rp>> &&
+    __nothrow_sentinel_for<sentinel_t<_Rp>, iterator_t<_Rp>>;
+
+template <class _Ip>
+concept __nothrow_forward_iterator =
+    __nothrow_input_iterator<_Ip> &&
+    forward_iterator<_Ip> &&
+    __nothrow_sentinel_for<_Ip, _Ip>;
+
+template <class _Rp>
+concept __nothrow_forward_range =
+    __nothrow_input_range<_Rp> &&
+    __nothrow_forward_iterator<iterator_t<_Rp>>;
+
+} // namespace ranges
+#endif // !defined(_LIBCPP_HAS_NO_RANGES)
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MEMORY_CONCEPTS_H
diff --git a/libcxx/include/__memory/construct_at.h b/libcxx/include/__memory/construct_at.h
index 789677d7a613..3b58451c5009 100644
--- a/libcxx/include/__memory/construct_at.h
+++ b/libcxx/include/__memory/construct_at.h
@@ -41,44 +41,67 @@ constexpr _Tp* construct_at(_Tp* __location, _Args&& ...__args) {
 
 // destroy_at
 
-#if _LIBCPP_STD_VER > 14
+// The internal functions are available regardless of the language version (with the exception of the `__destroy_at`
+// taking an array).
 
 template <class _ForwardIterator>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX17
-void destroy(_ForwardIterator, _ForwardIterator);
+void __destroy(_ForwardIterator, _ForwardIterator);
 
-template <class _Tp, enable_if_t<!is_array_v<_Tp>, int> = 0>
+template <class _Tp, typename enable_if<!is_array<_Tp>::value, int>::type = 0>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX17
-void destroy_at(_Tp* __loc) {
+void __destroy_at(_Tp* __loc) {
     _LIBCPP_ASSERT(__loc, "null pointer given to destroy_at");
     __loc->~_Tp();
 }
 
 #if _LIBCPP_STD_VER > 17
+template <class _Tp, typename enable_if<is_array<_Tp>::value, int>::type = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX17
+void __destroy_at(_Tp* __loc) {
+    _LIBCPP_ASSERT(__loc, "null pointer given to destroy_at");
+    _VSTD::__destroy(_VSTD::begin(*__loc), _VSTD::end(*__loc));
+}
+#endif
+
+template <class _ForwardIterator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX17
+void __destroy(_ForwardIterator __first, _ForwardIterator __last) {
+    for (; __first != __last; ++__first)
+        _VSTD::__destroy_at(_VSTD::addressof(*__first));
+}
+
+#if _LIBCPP_STD_VER > 14
+
+template <class _Tp, enable_if_t<!is_array_v<_Tp>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX17
+void destroy_at(_Tp* __loc) {
+    _VSTD::__destroy_at(__loc);
+}
+
+#if _LIBCPP_STD_VER > 17
 template <class _Tp, enable_if_t<is_array_v<_Tp>, int> = 0>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX17
 void destroy_at(_Tp* __loc) {
-    _LIBCPP_ASSERT(__loc, "null pointer given to destroy_at");
-    _VSTD::destroy(_VSTD::begin(*__loc), _VSTD::end(*__loc));
+  _VSTD::__destroy_at(__loc);
 }
 #endif
 
 template <class _ForwardIterator>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX17
 void destroy(_ForwardIterator __first, _ForwardIterator __last) {
-    for (; __first != __last; ++__first)
-        _VSTD::destroy_at(_VSTD::addressof(*__first));
+    _VSTD::__destroy(_VSTD::move(__first), _VSTD::move(__last));
 }
 
 template <class _ForwardIterator, class _Size>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX17
 _ForwardIterator destroy_n(_ForwardIterator __first, _Size __n) {
     for (; __n > 0; (void)++__first, --__n)
-        _VSTD::destroy_at(_VSTD::addressof(*__first));
+        _VSTD::__destroy_at(_VSTD::addressof(*__first));
     return __first;
 }
 
-#endif
+#endif // _LIBCPP_STD_VER > 14
 
 _LIBCPP_END_NAMESPACE_STD
 
diff --git a/libcxx/include/__memory/ranges_uninitialized_algorithms.h b/libcxx/include/__memory/ranges_uninitialized_algorithms.h
new file mode 100644
index 000000000000..6ec803806c05
--- /dev/null
+++ b/libcxx/include/__memory/ranges_uninitialized_algorithms.h
@@ -0,0 +1,212 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_RANGES_UNINITIALIZED_ALGORITHMS_H
+#define _LIBCPP___MEMORY_RANGES_UNINITIALIZED_ALGORITHMS_H
+
+#include <__concepts/constructible.h>
+#include <__config>
+#include <__function_like.h>
+#include <__iterator/incrementable_traits.h>
+#include <__iterator/iterator_traits.h>
+#include <__iterator/readable_traits.h>
+#include <__memory/concepts.h>
+#include <__memory/uninitialized_algorithms.h>
+#include <__ranges/access.h>
+#include <__ranges/concepts.h>
+#include <__ranges/dangling.h>
+#include <__utility/move.h>
+#include <type_traits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if !defined(_LIBCPP_HAS_NO_RANGES)
+namespace ranges {
+
+// uninitialized_default_construct
+
+namespace __uninitialized_default_construct {
+
+struct __fn final : private __function_like {
+
+  constexpr explicit __fn(__tag __x) noexcept : __function_like(__x) {}
+
+  template <__nothrow_forward_iterator _ForwardIterator,
+            __nothrow_sentinel_for<_ForwardIterator> _Sentinel>
+    requires default_initializable<iter_value_t<_ForwardIterator>>
+  _ForwardIterator operator()(_ForwardIterator __first, _Sentinel __last) const {
+    using _ValueType = remove_reference_t<iter_reference_t<_ForwardIterator>>;
+    return _VSTD::__uninitialized_default_construct<_ValueType>(
+        _VSTD::move(__first), _VSTD::move(__last));
+  }
+
+  template <__nothrow_forward_range _ForwardRange>
+    requires default_initializable<range_value_t<_ForwardRange>>
+  borrowed_iterator_t<_ForwardRange> operator()(_ForwardRange&& __range) const {
+    return (*this)(ranges::begin(__range), ranges::end(__range));
+  }
+
+};
+
+} // namespace __uninitialized_default_construct
+
+inline namespace __cpo {
+inline constexpr auto uninitialized_default_construct =
+    __uninitialized_default_construct::__fn(__function_like::__tag());
+} // namespace __cpo
+
+// uninitialized_default_construct_n
+
+namespace __uninitialized_default_construct_n {
+
+struct __fn final : private __function_like {
+
+  constexpr explicit __fn(__tag __x) noexcept :
+      __function_like(__x) {}
+
+  template <__nothrow_forward_iterator _ForwardIterator>
+    requires default_initializable<iter_value_t<_ForwardIterator>>
+  _ForwardIterator operator()(_ForwardIterator __first,
+                              iter_difference_t<_ForwardIterator> __n) const {
+    using _ValueType = remove_reference_t<iter_reference_t<_ForwardIterator>>;
+    return _VSTD::__uninitialized_default_construct_n<_ValueType>(_VSTD::move(__first), __n);
+  }
+
+};
+
+} // namespace __uninitialized_default_construct_n
+
+inline namespace __cpo {
+inline constexpr auto uninitialized_default_construct_n =
+    __uninitialized_default_construct_n::__fn(__function_like::__tag());
+} // namespace __cpo
+
+// uninitialized_value_construct
+
+namespace __uninitialized_value_construct {
+
+struct __fn final : private __function_like {
+
+  constexpr explicit __fn(__tag __x) noexcept : __function_like(__x) {}
+
+  template <__nothrow_forward_iterator _ForwardIterator,
+            __nothrow_sentinel_for<_ForwardIterator> _Sentinel>
+    requires default_initializable<iter_value_t<_ForwardIterator>>
+  _ForwardIterator operator()(_ForwardIterator __first, _Sentinel __last) const {
+    using _ValueType = remove_reference_t<iter_reference_t<_ForwardIterator>>;
+    return _VSTD::__uninitialized_value_construct<_ValueType>(
+        _VSTD::move(__first), _VSTD::move(__last));
+  }
+
+  template <__nothrow_forward_range _ForwardRange>
+    requires default_initializable<range_value_t<_ForwardRange>>
+  borrowed_iterator_t<_ForwardRange> operator()(_ForwardRange&& __range) const {
+    return (*this)(ranges::begin(__range), ranges::end(__range));
+  }
+
+};
+
+} // namespace __uninitialized_value_construct
+
+inline namespace __cpo {
+inline constexpr auto uninitialized_value_construct =
+    __uninitialized_value_construct::__fn(__function_like::__tag());
+} // namespace __cpo
+
+// uninitialized_value_construct_n
+
+namespace __uninitialized_value_construct_n {
+
+struct __fn final : private __function_like {
+
+  constexpr explicit __fn(__tag __x) noexcept : __function_like(__x) {}
+
+  template <__nothrow_forward_iterator _ForwardIterator>
+    requires default_initializable<iter_value_t<_ForwardIterator>>
+  _ForwardIterator operator()(_ForwardIterator __first,
+                              iter_difference_t<_ForwardIterator> __n) const {
+    using _ValueType = remove_reference_t<iter_reference_t<_ForwardIterator>>;
+    return _VSTD::__uninitialized_value_construct_n<_ValueType>(_VSTD::move(__first), __n);
+  }
+
+};
+
+} // namespace __uninitialized_value_construct_n
+
+inline namespace __cpo {
+inline constexpr auto uninitialized_value_construct_n =
+    __uninitialized_value_construct_n::__fn(__function_like::__tag());
+} // namespace __cpo
+
+// uninitialized_fill
+
+namespace __uninitialized_fill {
+
+struct __fn final : private __function_like {
+
+  constexpr explicit __fn(__tag __x) noexcept : __function_like(__x) {}
+
+  template <__nothrow_forward_iterator _ForwardIterator,
+            __nothrow_sentinel_for<_ForwardIterator> _Sentinel,
+            class _Tp>
+    requires constructible_from<iter_value_t<_ForwardIterator>, const _Tp&>
+  _ForwardIterator operator()(_ForwardIterator __first, _Sentinel __last, const _Tp& __x) const {
+    using _ValueType = remove_reference_t<iter_reference_t<_ForwardIterator>>;
+    return _VSTD::__uninitialized_fill<_ValueType>(_VSTD::move(__first), _VSTD::move(__last), __x);
+  }
+
+  template <__nothrow_forward_range _ForwardRange, class _Tp>
+    requires constructible_from<range_value_t<_ForwardRange>, const _Tp&>
+  borrowed_iterator_t<_ForwardRange> operator()(_ForwardRange&& __range, const _Tp& __x) const {
+    return (*this)(ranges::begin(__range), ranges::end(__range), __x);
+  }
+
+};
+
+} // namespace __uninitialized_fil
+
+inline namespace __cpo {
+inline constexpr auto uninitialized_fill = __uninitialized_fill::__fn(__function_like::__tag());
+} // namespace __cpo
+
+// uninitialized_fill_n
+
+namespace __uninitialized_fill_n {
+
+struct __fn final : private __function_like {
+
+  constexpr explicit __fn(__tag __x) noexcept : __function_like(__x) {}
+
+  template <__nothrow_forward_iterator _ForwardIterator, class _Tp>
+    requires constructible_from<iter_value_t<_ForwardIterator>, const _Tp&>
+  _ForwardIterator operator()(_ForwardIterator __first,
+                              iter_difference_t<_ForwardIterator> __n,
+                              const _Tp& __x) const {
+    using _ValueType = remove_reference_t<iter_reference_t<_ForwardIterator>>;
+    return _VSTD::__uninitialized_fill_n<_ValueType>(_VSTD::move(__first), __n, __x);
+  }
+
+};
+
+} // namespace __uninitialized_fill_n
+
+inline namespace __cpo {
+inline constexpr auto uninitialized_fill_n = __uninitialized_fill_n::__fn(__function_like::__tag());
+} // namespace __cpo
+
+} // namespace ranges
+#endif // !defined(_LIBCPP_HAS_NO_RANGES)
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MEMORY_RANGES_UNINITIALIZED_ALGORITHMS_H
diff --git a/libcxx/include/__memory/uninitialized_algorithms.h b/libcxx/include/__memory/uninitialized_algorithms.h
index e83d62e0db08..69132633a48e 100644
--- a/libcxx/include/__memory/uninitialized_algorithms.h
+++ b/libcxx/include/__memory/uninitialized_algorithms.h
@@ -13,6 +13,7 @@
 #include <__config>
 #include <__memory/addressof.h>
 #include <__memory/construct_at.h>
+#include <__memory/voidify.h>
 #include <iterator>
 #include <utility>
 
@@ -70,130 +71,187 @@ uninitialized_copy_n(_InputIterator __f, _Size __n, _ForwardIterator __r)
     return __r;
 }
 
-template <class _ForwardIterator, class _Tp>
-void
-uninitialized_fill(_ForwardIterator __f, _ForwardIterator __l, const _Tp& __x)
+// uninitialized_fill
+
+template <class _ValueType, class _ForwardIterator, class _Sentinel, class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI
+_ForwardIterator __uninitialized_fill(_ForwardIterator __first, _Sentinel __last, const _Tp& __x)
 {
-    typedef typename iterator_traits<_ForwardIterator>::value_type value_type;
+    _ForwardIterator __idx = __first;
 #ifndef _LIBCPP_NO_EXCEPTIONS
-    _ForwardIterator __s = __f;
     try
     {
 #endif
-        for (; __f != __l; ++__f)
-            ::new ((void*)_VSTD::addressof(*__f)) value_type(__x);
+        for (; __idx != __last; ++__idx)
+            ::new (_VSTD::__voidify(*__idx)) _ValueType(__x);
 #ifndef _LIBCPP_NO_EXCEPTIONS
     }
     catch (...)
     {
-        for (; __s != __f; ++__s)
-            __s->~value_type();
+        _VSTD::__destroy(__first, __idx);
         throw;
     }
 #endif
+
+    return __idx;
 }
 
-template <class _ForwardIterator, class _Size, class _Tp>
-_ForwardIterator
-uninitialized_fill_n(_ForwardIterator __f, _Size __n, const _Tp& __x)
+template <class _ForwardIterator, class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI
+void uninitialized_fill(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __x)
 {
-    typedef typename iterator_traits<_ForwardIterator>::value_type value_type;
+    typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
+    (void)_VSTD::__uninitialized_fill<_ValueType>(__first, __last, __x);
+}
+
+// uninitialized_fill_n
+
+template <class _ValueType, class _ForwardIterator, class _Size, class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI
+_ForwardIterator __uninitialized_fill_n(_ForwardIterator __first, _Size __n, const _Tp& __x)
+{
+    _ForwardIterator __idx = __first;
 #ifndef _LIBCPP_NO_EXCEPTIONS
-    _ForwardIterator __s = __f;
     try
     {
 #endif
-        for (; __n > 0; ++__f, (void) --__n)
-            ::new ((void*)_VSTD::addressof(*__f)) value_type(__x);
+        for (; __n > 0; ++__idx, (void) --__n)
+            ::new (_VSTD::__voidify(*__idx)) _ValueType(__x);
 #ifndef _LIBCPP_NO_EXCEPTIONS
     }
     catch (...)
     {
-        for (; __s != __f; ++__s)
-            __s->~value_type();
+        _VSTD::__destroy(__first, __idx);
         throw;
     }
 #endif
-    return __f;
+
+    return __idx;
+}
+
+template <class _ForwardIterator, class _Size, class _Tp>
+inline _LIBCPP_HIDE_FROM_ABI
+_ForwardIterator uninitialized_fill_n(_ForwardIterator __first, _Size __n, const _Tp& __x)
+{
+    typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
+    return _VSTD::__uninitialized_fill_n<_ValueType>(__first, __n, __x);
 }
 
 #if _LIBCPP_STD_VER > 14
 
-template <class _ForwardIterator>
-inline _LIBCPP_INLINE_VISIBILITY
-void uninitialized_default_construct(_ForwardIterator __first, _ForwardIterator __last) {
-    using _Vt = typename iterator_traits<_ForwardIterator>::value_type;
+// uninitialized_default_construct
+
+template <class _ValueType, class _ForwardIterator, class _Sentinel>
+inline _LIBCPP_HIDE_FROM_ABI
+_ForwardIterator __uninitialized_default_construct(_ForwardIterator __first, _Sentinel __last) {
     auto __idx = __first;
 #ifndef _LIBCPP_NO_EXCEPTIONS
     try {
 #endif
     for (; __idx != __last; ++__idx)
-        ::new ((void*)_VSTD::addressof(*__idx)) _Vt;
+        ::new (_VSTD::__voidify(*__idx)) _ValueType;
 #ifndef _LIBCPP_NO_EXCEPTIONS
     } catch (...) {
-        _VSTD::destroy(__first, __idx);
+        _VSTD::__destroy(__first, __idx);
         throw;
     }
 #endif
+
+    return __idx;
 }
 
-template <class _ForwardIterator, class _Size>
-inline _LIBCPP_INLINE_VISIBILITY
-_ForwardIterator uninitialized_default_construct_n(_ForwardIterator __first, _Size __n) {
-    using _Vt = typename iterator_traits<_ForwardIterator>::value_type;
+template <class _ForwardIterator>
+inline _LIBCPP_HIDE_FROM_ABI
+void uninitialized_default_construct(_ForwardIterator __first, _ForwardIterator __last) {
+    using _ValueType = typename iterator_traits<_ForwardIterator>::value_type;
+    (void)_VSTD::__uninitialized_default_construct<_ValueType>(
+        _VSTD::move(__first), _VSTD::move(__last));
+}
+
+// uninitialized_default_construct_n
+
+template <class _ValueType, class _ForwardIterator, class _Size>
+inline _LIBCPP_HIDE_FROM_ABI
+_ForwardIterator __uninitialized_default_construct_n(_ForwardIterator __first, _Size __n) {
     auto __idx = __first;
 #ifndef _LIBCPP_NO_EXCEPTIONS
     try {
 #endif
     for (; __n > 0; ++__idx, (void) --__n)
-        ::new ((void*)_VSTD::addressof(*__idx)) _Vt;
-    return __idx;
+        ::new (_VSTD::__voidify(*__idx)) _ValueType;
 #ifndef _LIBCPP_NO_EXCEPTIONS
     } catch (...) {
-        _VSTD::destroy(__first, __idx);
+        _VSTD::__destroy(__first, __idx);
         throw;
     }
 #endif
+
+    return __idx;
 }
 
+template <class _ForwardIterator, class _Size>
+inline _LIBCPP_HIDE_FROM_ABI
+_ForwardIterator uninitialized_default_construct_n(_ForwardIterator __first, _Size __n) {
+    using _ValueType = typename iterator_traits<_ForwardIterator>::value_type;
+    return _VSTD::__uninitialized_default_construct_n<_ValueType>(_VSTD::move(__first), __n);
+}
 
-template <class _ForwardIterator>
-inline _LIBCPP_INLINE_VISIBILITY
-void uninitialized_value_construct(_ForwardIterator __first, _ForwardIterator __last) {
-    using _Vt = typename iterator_traits<_ForwardIterator>::value_type;
+// uninitialized_value_construct
+
+template <class _ValueType, class _ForwardIterator, class _Sentinel>
+inline _LIBCPP_HIDE_FROM_ABI
+_ForwardIterator __uninitialized_value_construct(_ForwardIterator __first, _Sentinel __last) {
     auto __idx = __first;
 #ifndef _LIBCPP_NO_EXCEPTIONS
     try {
 #endif
     for (; __idx != __last; ++__idx)
-        ::new ((void*)_VSTD::addressof(*__idx)) _Vt();
+        ::new (_VSTD::__voidify(*__idx)) _ValueType();
 #ifndef _LIBCPP_NO_EXCEPTIONS
     } catch (...) {
-        _VSTD::destroy(__first, __idx);
+        _VSTD::__destroy(__first, __idx);
         throw;
     }
 #endif
+
+    return __idx;
 }
 
-template <class _ForwardIterator, class _Size>
-inline _LIBCPP_INLINE_VISIBILITY
-_ForwardIterator uninitialized_value_construct_n(_ForwardIterator __first, _Size __n) {
-    using _Vt = typename iterator_traits<_ForwardIterator>::value_type;
+template <class _ForwardIterator>
+inline _LIBCPP_HIDE_FROM_ABI
+void uninitialized_value_construct(_ForwardIterator __first, _ForwardIterator __last) {
+    using _ValueType = typename iterator_traits<_ForwardIterator>::value_type;
+    (void)_VSTD::__uninitialized_value_construct<_ValueType>(
+        _VSTD::move(__first), _VSTD::move(__last));
+}
+
+// uninitialized_value_construct_n
+
+template <class _ValueType, class _ForwardIterator, class _Size>
+inline _LIBCPP_HIDE_FROM_ABI
+_ForwardIterator __uninitialized_value_construct_n(_ForwardIterator __first, _Size __n) {
     auto __idx = __first;
 #ifndef _LIBCPP_NO_EXCEPTIONS
     try {
 #endif
     for (; __n > 0; ++__idx, (void) --__n)
-        ::new ((void*)_VSTD::addressof(*__idx)) _Vt();
-    return __idx;
+        ::new (_VSTD::__voidify(*__idx)) _ValueType();
 #ifndef _LIBCPP_NO_EXCEPTIONS
     } catch (...) {
-        _VSTD::destroy(__first, __idx);
+        _VSTD::__destroy(__first, __idx);
         throw;
     }
 #endif
+
+    return __idx;
 }
 
+template <class _ForwardIterator, class _Size>
+inline _LIBCPP_HIDE_FROM_ABI
+_ForwardIterator uninitialized_value_construct_n(_ForwardIterator __first, _Size __n) {
+    using _ValueType = typename iterator_traits<_ForwardIterator>::value_type;
+    return __uninitialized_value_construct_n<_ValueType>(_VSTD::move(__first), __n);
+}
 
 template <class _InputIt, class _ForwardIt>
 inline _LIBCPP_INLINE_VISIBILITY
diff --git a/libcxx/include/__memory/voidify.h b/libcxx/include/__memory/voidify.h
new file mode 100644
index 000000000000..3a65c0e83fb7
--- /dev/null
+++ b/libcxx/include/__memory/voidify.h
@@ -0,0 +1,30 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_VOIDIFY_H
+#define _LIBCPP___MEMORY_VOIDIFY_H
+
+#include <__config>
+#include <__memory/addressof.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <typename _Tp>
+_LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX17 void* __voidify(_Tp& __from) {
+  // Cast away cv-qualifiers to allow modifying elements of a range through const iterators.
+  return const_cast<void*>(static_cast<const volatile void*>(_VSTD::addressof(__from)));
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MEMORY_VOIDIFY_H
diff --git a/libcxx/include/__mutex_base b/libcxx/include/__mutex_base
index da2967164a68..21e2075603b5 100644
--- a/libcxx/include/__mutex_base
+++ b/libcxx/include/__mutex_base
@@ -140,11 +140,9 @@ public:
             __m_->unlock();
     }
 
-private:
-    unique_lock(unique_lock const&); // = delete;
-    unique_lock& operator=(unique_lock const&); // = delete;
+    unique_lock(unique_lock const&) = delete;
+    unique_lock& operator=(unique_lock const&) = delete;
 
-public:
     _LIBCPP_INLINE_VISIBILITY
     unique_lock(unique_lock&& __u) _NOEXCEPT
         : __m_(__u.__m_), __owns_(__u.__owns_)
diff --git a/libcxx/include/__nullptr b/libcxx/include/__nullptr
index d02be215ef1d..c6645cd0150e 100644
--- a/libcxx/include/__nullptr
+++ b/libcxx/include/__nullptr
@@ -54,7 +54,7 @@ _LIBCPP_END_NAMESPACE_STD
 namespace std
 {
     typedef decltype(nullptr) nullptr_t;
-}
+} // namespace std
 
 #endif // _LIBCPP_HAS_NO_NULLPTR
 
diff --git a/libcxx/include/__random/clamp_to_integral.h b/libcxx/include/__random/clamp_to_integral.h
new file mode 100644
index 000000000000..dd5d2b0186e4
--- /dev/null
+++ b/libcxx/include/__random/clamp_to_integral.h
@@ -0,0 +1,60 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RANDOM_CLAMP_TO_INTEGRAL_H
+#define _LIBCPP___RANDOM_CLAMP_TO_INTEGRAL_H
+
+#include <__config>
+#include <cmath>
+#include <limits>
+#include <type_traits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _IntT, class _FloatT,
+    bool _FloatBigger = (numeric_limits<_FloatT>::digits > numeric_limits<_IntT>::digits),
+    int _Bits = (numeric_limits<_IntT>::digits - numeric_limits<_FloatT>::digits)>
+_LIBCPP_INLINE_VISIBILITY
+_LIBCPP_CONSTEXPR _IntT __max_representable_int_for_float() _NOEXCEPT {
+  static_assert(is_floating_point<_FloatT>::value, "must be a floating point type");
+  static_assert(is_integral<_IntT>::value, "must be an integral type");
+  static_assert(numeric_limits<_FloatT>::radix == 2, "FloatT has incorrect radix");
+  static_assert((_IsSame<_FloatT, float>::value || _IsSame<_FloatT, double>::value
+                 || _IsSame<_FloatT,long double>::value), "unsupported floating point type");
+  return _FloatBigger ? numeric_limits<_IntT>::max() :  (numeric_limits<_IntT>::max() >> _Bits << _Bits);
+}
+
+// Convert a floating point number to the specified integral type after
+// clamping to the integral type's representable range.
+//
+// The behavior is undefined if `__r` is NaN.
+template <class _IntT, class _RealT>
+_LIBCPP_INLINE_VISIBILITY
+_IntT __clamp_to_integral(_RealT __r) _NOEXCEPT {
+  using _Lim = numeric_limits<_IntT>;
+  const _IntT _MaxVal = __max_representable_int_for_float<_IntT, _RealT>();
+  if (__r >= ::nextafter(static_cast<_RealT>(_MaxVal), INFINITY)) {
+    return _Lim::max();
+  } else if (__r <= _Lim::lowest()) {
+    return _Lim::min();
+  }
+  return static_cast<_IntT>(__r);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___RANDOM_CLAMP_TO_INTEGRAL_H
diff --git a/libcxx/include/__random/poisson_distribution.h b/libcxx/include/__random/poisson_distribution.h
index fb213b0103ad..d157e8f230ef 100644
--- a/libcxx/include/__random/poisson_distribution.h
+++ b/libcxx/include/__random/poisson_distribution.h
@@ -10,6 +10,7 @@
 #define _LIBCPP___RANDOM_POISSON_DISTRIBUTION_H
 
 #include <__config>
+#include <__random/clamp_to_integral.h>
 #include <__random/exponential_distribution.h>
 #include <__random/normal_distribution.h>
 #include <__random/uniform_real_distribution.h>
diff --git a/libcxx/include/__random/random_device.h b/libcxx/include/__random/random_device.h
index f62f7a3d269b..835f726fdbcc 100644
--- a/libcxx/include/__random/random_device.h
+++ b/libcxx/include/__random/random_device.h
@@ -27,7 +27,7 @@ class _LIBCPP_TYPE_VIS random_device
 {
 #ifdef _LIBCPP_USING_DEV_RANDOM
     int __f_;
-#endif // defined(_LIBCPP_USING_DEV_RANDOM)
+#endif
 public:
     // types
     typedef unsigned result_type;
@@ -56,10 +56,8 @@ public:
     // property functions
     double entropy() const _NOEXCEPT;
 
-private:
-    // no copy functions
-    random_device(const random_device&); // = delete;
-    random_device& operator=(const random_device&); // = delete;
+    random_device(const random_device&) = delete;
+    void operator=(const random_device&) = delete;
 };
 
 #endif // !_LIBCPP_HAS_NO_RANDOM_DEVICE
diff --git a/libcxx/include/__random/seed_seq.h b/libcxx/include/__random/seed_seq.h
index 97bc88d0d4d1..bf27af6627a5 100644
--- a/libcxx/include/__random/seed_seq.h
+++ b/libcxx/include/__random/seed_seq.h
@@ -63,10 +63,8 @@ public:
         void param(_OutputIterator __dest) const
             {_VSTD::copy(__v_.begin(), __v_.end(), __dest);}
 
-private:
-    // no copy functions
-    seed_seq(const seed_seq&); // = delete;
-    void operator=(const seed_seq&); // = delete;
+    seed_seq(const seed_seq&) = delete;
+    void operator=(const seed_seq&) = delete;
 
     _LIBCPP_INLINE_VISIBILITY
     static result_type _Tp(result_type __x) {return __x ^ (__x >> 27);}
diff --git a/libcxx/include/__ranges/access.h b/libcxx/include/__ranges/access.h
index b0b89c0eeea3..91dc3055c86d 100644
--- a/libcxx/include/__ranges/access.h
+++ b/libcxx/include/__ranges/access.h
@@ -14,8 +14,7 @@
 #include <__iterator/readable_traits.h>
 #include <__ranges/enable_borrowed_range.h>
 #include <__utility/as_const.h>
-#include <__utility/decay_copy.h>
-#include <__utility/forward.h>
+#include <__utility/auto_cast.h>
 #include <concepts>
 #include <type_traits>
 
@@ -27,24 +26,21 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 #if !defined(_LIBCPP_HAS_NO_RANGES)
 
-// clang-format off
-
 namespace ranges {
   template <class _Tp>
   concept __can_borrow =
-      is_lvalue_reference_v<_Tp> || enable_borrowed_range<remove_cvref_t<_Tp> >;
-
-  template<class _Tp>
-  concept __is_complete = requires { sizeof(_Tp); };
+    is_lvalue_reference_v<_Tp> || enable_borrowed_range<remove_cvref_t<_Tp>>;
 } // namespace ranges
 
 // [range.access.begin]
-namespace ranges::__begin {
+
+namespace ranges {
+namespace __begin {
   template <class _Tp>
   concept __member_begin =
     __can_borrow<_Tp> &&
     requires(_Tp&& __t) {
-      { _VSTD::__decay_copy(__t.begin()) } -> input_or_output_iterator;
+      { _LIBCPP_AUTO_CAST(__t.begin()) } -> input_or_output_iterator;
     };
 
   void begin(auto&) = delete;
@@ -54,61 +50,61 @@ namespace ranges::__begin {
   concept __unqualified_begin =
     !__member_begin<_Tp> &&
     __can_borrow<_Tp> &&
-    __class_or_enum<remove_cvref_t<_Tp> > &&
+    __class_or_enum<remove_cvref_t<_Tp>> &&
     requires(_Tp && __t) {
-      { _VSTD::__decay_copy(begin(__t)) } -> input_or_output_iterator;
+      { _LIBCPP_AUTO_CAST(begin(__t)) } -> input_or_output_iterator;
     };
 
   struct __fn {
     template <class _Tp>
-    requires is_array_v<remove_cv_t<_Tp>>
-    [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp& __t) const noexcept {
-      constexpr bool __complete = __is_complete<iter_value_t<_Tp> >;
-      if constexpr (__complete) { // used to disable cryptic diagnostic
-        return __t + 0;
-      }
-      else {
-        static_assert(__complete, "`std::ranges::begin` is SFINAE-unfriendly on arrays of an incomplete type.");
-      }
+      requires is_array_v<remove_cv_t<_Tp>>
+    [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp& __t) const noexcept
+    {
+      return __t;
     }
 
     template <class _Tp>
-    requires __member_begin<_Tp>
+      requires __member_begin<_Tp>
     [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const
-    noexcept(noexcept(_VSTD::__decay_copy(__t.begin())))
+      noexcept(noexcept(_LIBCPP_AUTO_CAST(__t.begin())))
     {
-      return __t.begin();
+      return _LIBCPP_AUTO_CAST(__t.begin());
     }
 
     template <class _Tp>
-    requires __unqualified_begin<_Tp>
+      requires __unqualified_begin<_Tp>
     [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const
-    noexcept(noexcept(_VSTD::__decay_copy(begin(__t))))
+      noexcept(noexcept(_LIBCPP_AUTO_CAST(begin(__t))))
     {
-      return begin(__t);
+      return _LIBCPP_AUTO_CAST(begin(__t));
     }
 
     void operator()(auto&&) const = delete;
   };
-} // namespace ranges::__begin
+}
 
-namespace ranges {
-  inline namespace __cpo {
-    inline constexpr auto begin = __begin::__fn{};
-  } // namespace __cpo
+inline namespace __cpo {
+  inline constexpr auto begin = __begin::__fn{};
+} // namespace __cpo
+} // namespace ranges
+
+// [range.range]
 
+namespace ranges {
   template <class _Tp>
   using iterator_t = decltype(ranges::begin(declval<_Tp&>()));
 } // namespace ranges
 
 // [range.access.end]
-namespace ranges::__end {
+
+namespace ranges {
+namespace __end {
   template <class _Tp>
   concept __member_end =
     __can_borrow<_Tp> &&
     requires(_Tp&& __t) {
       typename iterator_t<_Tp>;
-      { _VSTD::__decay_copy(_VSTD::forward<_Tp>(__t).end()) } -> sentinel_for<iterator_t<_Tp> >;
+      { _LIBCPP_AUTO_CAST(__t.end()) } -> sentinel_for<iterator_t<_Tp>>;
     };
 
   void end(auto&) = delete;
@@ -118,98 +114,101 @@ namespace ranges::__end {
   concept __unqualified_end =
     !__member_end<_Tp> &&
     __can_borrow<_Tp> &&
-    __class_or_enum<remove_cvref_t<_Tp> > &&
+    __class_or_enum<remove_cvref_t<_Tp>> &&
     requires(_Tp && __t) {
       typename iterator_t<_Tp>;
-      { _VSTD::__decay_copy(end(_VSTD::forward<_Tp>(__t))) } -> sentinel_for<iterator_t<_Tp> >;
+      { _LIBCPP_AUTO_CAST(end(__t)) } -> sentinel_for<iterator_t<_Tp>>;
     };
 
   class __fn {
   public:
     template <class _Tp, size_t _Np>
-    [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp (&__t)[_Np]) const noexcept {
-      constexpr bool __complete = __is_complete<remove_cv_t<_Tp> >;
-      if constexpr (__complete) { // used to disable cryptic diagnostic
-        return __t + _Np;
-      }
-      else {
-        static_assert(__complete, "`std::ranges::end` is SFINAE-unfriendly on arrays of an incomplete type.");
-      }
+    [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp (&__t)[_Np]) const noexcept
+      requires (sizeof(*__t) != 0)  // Disallow incomplete element types.
+    {
+      return __t + _Np;
     }
 
     template <class _Tp>
-    requires __member_end<_Tp>
+      requires __member_end<_Tp>
     [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const
-    noexcept(noexcept(_VSTD::__decay_copy(__t.end())))
+      noexcept(noexcept(_LIBCPP_AUTO_CAST(__t.end())))
     {
-      return _VSTD::forward<_Tp>(__t).end();
+      return _LIBCPP_AUTO_CAST(__t.end());
     }
 
     template <class _Tp>
-    requires __unqualified_end<_Tp>
+      requires __unqualified_end<_Tp>
     [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const
-    noexcept(noexcept(_VSTD::__decay_copy(end(__t))))
+      noexcept(noexcept(_LIBCPP_AUTO_CAST(end(__t))))
     {
-      return end(__t);
+      return _LIBCPP_AUTO_CAST(end(__t));
     }
 
     void operator()(auto&&) const = delete;
   };
-} // namespace ranges::__end
+}
 
-namespace ranges::inline __cpo {
+inline namespace __cpo {
   inline constexpr auto end = __end::__fn{};
-} // namespace ranges::__cpo
+} // namespace __cpo
+} // namespace ranges
 
-namespace ranges::__cbegin {
+// [range.access.cbegin]
+
+namespace ranges {
+namespace __cbegin {
   struct __fn {
     template <class _Tp>
-    requires invocable<decltype(ranges::begin), _Tp const&>
+      requires invocable<decltype(ranges::begin), _Tp const&>
     [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp& __t) const
-    noexcept(noexcept(ranges::begin(_VSTD::as_const(__t))))
+      noexcept(noexcept(ranges::begin(_VSTD::as_const(__t))))
     {
       return ranges::begin(_VSTD::as_const(__t));
     }
 
     template <class _Tp>
-    requires is_rvalue_reference_v<_Tp> && invocable<decltype(ranges::begin), _Tp const&&>
+      requires is_rvalue_reference_v<_Tp> && invocable<decltype(ranges::begin), _Tp const&&>
     [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const
-    noexcept(noexcept(ranges::begin(static_cast<_Tp const&&>(__t))))
+      noexcept(noexcept(ranges::begin(static_cast<_Tp const&&>(__t))))
     {
       return ranges::begin(static_cast<_Tp const&&>(__t));
     }
   };
-} // namespace ranges::__cbegin
+}
 
-namespace ranges::inline __cpo {
+inline namespace __cpo {
   inline constexpr auto cbegin = __cbegin::__fn{};
-} // namespace ranges::__cpo
+} // namespace __cpo
+} // namespace ranges
+
+// [range.access.cend]
 
-namespace ranges::__cend {
+namespace ranges {
+namespace __cend {
   struct __fn {
     template <class _Tp>
-    requires invocable<decltype(ranges::end), _Tp const&>
+      requires invocable<decltype(ranges::end), _Tp const&>
     [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp& __t) const
-    noexcept(noexcept(ranges::end(_VSTD::as_const(__t))))
+      noexcept(noexcept(ranges::end(_VSTD::as_const(__t))))
     {
       return ranges::end(_VSTD::as_const(__t));
     }
 
     template <class _Tp>
-    requires is_rvalue_reference_v<_Tp> && invocable<decltype(ranges::end), _Tp const&&>
+      requires is_rvalue_reference_v<_Tp> && invocable<decltype(ranges::end), _Tp const&&>
     [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __t) const
-    noexcept(noexcept(ranges::end(static_cast<_Tp const&&>(__t))))
+      noexcept(noexcept(ranges::end(static_cast<_Tp const&&>(__t))))
     {
       return ranges::end(static_cast<_Tp const&&>(__t));
     }
   };
-} // namespace ranges::__cend
+}
 
-namespace ranges::inline __cpo {
+inline namespace __cpo {
   inline constexpr auto cend = __cend::__fn{};
-} // namespace ranges::__cpo
-
-// clang-format off
+} // namespace __cpo
+} // namespace ranges
 
 #endif // !defined(_LIBCPP_HAS_NO_RANGES)
 
diff --git a/libcxx/include/__ranges/all.h b/libcxx/include/__ranges/all.h
index affe13ee0862..ccc77258ba10 100644
--- a/libcxx/include/__ranges/all.h
+++ b/libcxx/include/__ranges/all.h
@@ -17,7 +17,7 @@
 #include <__ranges/range_adaptor.h>
 #include <__ranges/ref_view.h>
 #include <__ranges/subrange.h>
-#include <__utility/decay_copy.h>
+#include <__utility/auto_cast.h>
 #include <__utility/declval.h>
 #include <__utility/forward.h>
 #include <type_traits>
@@ -38,9 +38,9 @@ namespace __all {
       requires ranges::view<decay_t<_Tp>>
     [[nodiscard]] _LIBCPP_HIDE_FROM_ABI
     constexpr auto operator()(_Tp&& __t) const
-      noexcept(noexcept(_VSTD::__decay_copy(_VSTD::forward<_Tp>(__t))))
+      noexcept(noexcept(_LIBCPP_AUTO_CAST(_VSTD::forward<_Tp>(__t))))
     {
-      return _VSTD::forward<_Tp>(__t);
+      return _LIBCPP_AUTO_CAST(_VSTD::forward<_Tp>(__t));
     }
 
     template<class _Tp>
diff --git a/libcxx/include/__ranges/concepts.h b/libcxx/include/__ranges/concepts.h
index 6a8364006beb..bad23c8c4bfb 100644
--- a/libcxx/include/__ranges/concepts.h
+++ b/libcxx/include/__ranges/concepts.h
@@ -29,8 +29,6 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-// clang-format off
-
 #if !defined(_LIBCPP_HAS_NO_RANGES)
 
 namespace ranges {
@@ -126,8 +124,6 @@ namespace ranges {
 
 #endif // !defined(_LIBCPP_HAS_NO_RANGES)
 
-// clang-format on
-
 _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP___RANGES_CONCEPTS_H
diff --git a/libcxx/include/__ranges/counted.h b/libcxx/include/__ranges/counted.h
index d292bcbb1849..cb9784092420 100644
--- a/libcxx/include/__ranges/counted.h
+++ b/libcxx/include/__ranges/counted.h
@@ -9,6 +9,7 @@
 #ifndef _LIBCPP___RANGES_COUNTED_H
 #define _LIBCPP___RANGES_COUNTED_H
 
+#include <__concepts/convertible_to.h>
 #include <__config>
 #include <__iterator/concepts.h>
 #include <__iterator/counted_iterator.h>
@@ -16,10 +17,7 @@
 #include <__iterator/incrementable_traits.h>
 #include <__iterator/iterator_traits.h>
 #include <__memory/pointer_traits.h>
-#include <__ranges/concepts.h>
 #include <__ranges/subrange.h>
-#include <__utility/decay_copy.h>
-#include <__utility/declval.h>
 #include <__utility/forward.h>
 #include <__utility/move.h>
 #include <span>
@@ -36,50 +34,39 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 namespace ranges::views {
 
 namespace __counted {
-  template<class _From, class _To>
-  concept __explicitly_convertible = requires {
-    _To(_From{});
-  };
 
   struct __fn {
-    template<class _Iter, class _Diff>
-      requires contiguous_iterator<decay_t<_Iter>> &&
-               __explicitly_convertible<_Diff, iter_difference_t<_Iter>>
+    template<contiguous_iterator _It>
     _LIBCPP_HIDE_FROM_ABI
-    constexpr auto operator()(_Iter&& __it, _Diff __c) const
-      noexcept(noexcept(
-        span(_VSTD::to_address(__it), static_cast<iter_difference_t<_Iter>>(__c))
-      ))
-    {
-      return span(_VSTD::to_address(__it), static_cast<iter_difference_t<_Iter>>(__c));
-    }
-
-    template<class _Iter, class _Diff>
-      requires random_access_iterator<decay_t<_Iter>> &&
-               __explicitly_convertible<_Diff, iter_difference_t<_Iter>>
+    static constexpr auto __go(_It __it, iter_difference_t<_It> __count)
+      noexcept(noexcept(span(_VSTD::to_address(__it), static_cast<size_t>(__count))))
+      // Deliberately omit return-type SFINAE, because to_address is not SFINAE-friendly
+      { return          span(_VSTD::to_address(__it), static_cast<size_t>(__count)); }
+
+    template<random_access_iterator _It>
     _LIBCPP_HIDE_FROM_ABI
-    constexpr auto operator()(_Iter&& __it, _Diff __c) const
-      noexcept(
-        noexcept(__it + static_cast<iter_difference_t<_Iter>>(__c)) &&
-        noexcept(ranges::subrange(_VSTD::forward<_Iter>(__it), _VSTD::__decay_copy(__it)))
-      )
-    {
-      auto __last = __it + static_cast<iter_difference_t<_Iter>>(__c);
-      return ranges::subrange(_VSTD::forward<_Iter>(__it), _VSTD::move(__last));
-    }
-
-    template<class _Iter, class _Diff>
-      requires __explicitly_convertible<_Diff, iter_difference_t<_Iter>>
+    static constexpr auto __go(_It __it, iter_difference_t<_It> __count)
+      noexcept(noexcept(subrange(__it, __it + __count)))
+      -> decltype(      subrange(__it, __it + __count))
+      { return          subrange(__it, __it + __count); }
+
+    template<class _It>
     _LIBCPP_HIDE_FROM_ABI
-    constexpr auto operator()(_Iter&& __it, _Diff __c) const
-      noexcept(noexcept(
-        ranges::subrange(counted_iterator(_VSTD::forward<_Iter>(__it), __c), default_sentinel)
-      ))
-    {
-      return ranges::subrange(counted_iterator(_VSTD::forward<_Iter>(__it), __c), default_sentinel);
-    }
+    static constexpr auto __go(_It __it, iter_difference_t<_It> __count)
+      noexcept(noexcept(subrange(counted_iterator(_VSTD::move(__it), __count), default_sentinel)))
+      -> decltype(      subrange(counted_iterator(_VSTD::move(__it), __count), default_sentinel))
+      { return          subrange(counted_iterator(_VSTD::move(__it), __count), default_sentinel); }
+
+    template<class _It, convertible_to<iter_difference_t<_It>> _Diff>
+      requires input_or_output_iterator<decay_t<_It>>
+    [[nodiscard]] _LIBCPP_HIDE_FROM_ABI
+    constexpr auto operator()(_It&& __it, _Diff&& __count) const
+      noexcept(noexcept(__go(_VSTD::forward<_It>(__it), _VSTD::forward<_Diff>(__count))))
+      -> decltype(      __go(_VSTD::forward<_It>(__it), _VSTD::forward<_Diff>(__count)))
+      { return          __go(_VSTD::forward<_It>(__it), _VSTD::forward<_Diff>(__count)); }
   };
-}
+
+} // namespace __counted
 
 inline namespace __cpo {
   inline constexpr auto counted = __counted::__fn{};
diff --git a/libcxx/include/__ranges/data.h b/libcxx/include/__ranges/data.h
index 7eade494cceb..cc151c59f3d7 100644
--- a/libcxx/include/__ranges/data.h
+++ b/libcxx/include/__ranges/data.h
@@ -26,9 +26,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 #if !defined(_LIBCPP_HAS_NO_RANGES)
 
-// clang-format off
-namespace ranges {
 // [range.prim.data]
+
+namespace ranges {
 namespace __data {
   template <class _Tp>
   concept __ptr_to_object = is_pointer_v<_Tp> && is_object_v<remove_pointer_t<_Tp>>;
@@ -65,15 +65,13 @@ namespace __data {
       return _VSTD::to_address(ranges::begin(_VSTD::forward<_Tp>(__t)));
     }
   };
-} // end namespace __data
+}
 
 inline namespace __cpo {
   inline constexpr auto data = __data::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
-// clang-format off
-
 #endif // !defined(_LIBCPP_HAS_NO_RANGES)
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__ranges/empty.h b/libcxx/include/__ranges/empty.h
index fc6a938fd86e..e8a8aabf4aed 100644
--- a/libcxx/include/__ranges/empty.h
+++ b/libcxx/include/__ranges/empty.h
@@ -13,7 +13,6 @@
 #include <__iterator/concepts.h>
 #include <__ranges/access.h>
 #include <__ranges/size.h>
-#include <__utility/forward.h>
 #include <type_traits>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
@@ -24,19 +23,19 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 #if !defined(_LIBCPP_HAS_NO_RANGES)
 
-// clang-format off
-namespace ranges {
 // [range.prim.empty]
+
+namespace ranges {
 namespace __empty {
   template <class _Tp>
   concept __member_empty = requires(_Tp&& __t) {
-    bool(_VSTD::forward<_Tp>(__t).empty());
+    bool(__t.empty());
   };
 
   template<class _Tp>
   concept __can_invoke_size =
     !__member_empty<_Tp> &&
-    requires(_Tp&& __t) { ranges::size(_VSTD::forward<_Tp>(__t)); };
+    requires(_Tp&& __t) { ranges::size(__t); };
 
   template <class _Tp>
   concept __can_compare_begin_end =
@@ -51,13 +50,13 @@ namespace __empty {
     template <__member_empty _Tp>
     [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(_Tp&& __t) const
         noexcept(noexcept(bool(__t.empty()))) {
-      return __t.empty();
+      return bool(__t.empty());
     }
 
     template <__can_invoke_size _Tp>
     [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(_Tp&& __t) const
-        noexcept(noexcept(ranges::size(_VSTD::forward<_Tp>(__t)))) {
-      return ranges::size(_VSTD::forward<_Tp>(__t)) == 0;
+        noexcept(noexcept(ranges::size(__t))) {
+      return ranges::size(__t) == 0;
     }
 
     template<__can_compare_begin_end _Tp>
@@ -72,7 +71,6 @@ inline namespace __cpo {
   inline constexpr auto empty = __empty::__fn{};
 } // namespace __cpo
 } // namespace ranges
-// clang-format off
 
 #endif // !defined(_LIBCPP_HAS_NO_RANGES)
 
diff --git a/libcxx/include/__ranges/size.h b/libcxx/include/__ranges/size.h
index af0a8479f2ec..fc6641cf4887 100644
--- a/libcxx/include/__ranges/size.h
+++ b/libcxx/include/__ranges/size.h
@@ -13,8 +13,7 @@
 #include <__iterator/concepts.h>
 #include <__iterator/iterator_traits.h>
 #include <__ranges/access.h>
-#include <__utility/decay_copy.h>
-#include <__utility/forward.h>
+#include <__utility/auto_cast.h>
 #include <concepts>
 #include <type_traits>
 
@@ -26,12 +25,14 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 #if !defined(_LIBCPP_HAS_NO_RANGES)
 
-// clang-format off
 namespace ranges {
-template<class>
-inline constexpr bool disable_sized_range = false;
+  template<class>
+  inline constexpr bool disable_sized_range = false;
+}
 
 // [range.prim.size]
+
+namespace ranges {
 namespace __size {
   void size(auto&) = delete;
   void size(const auto&) = delete;
@@ -41,7 +42,7 @@ namespace __size {
 
   template <class _Tp>
   concept __member_size = __size_enabled<_Tp> && requires(_Tp&& __t) {
-    { _VSTD::__decay_copy(_VSTD::forward<_Tp>(__t).size()) } -> __integer_like;
+    { _LIBCPP_AUTO_CAST(__t.size()) } -> __integer_like;
   };
 
   template <class _Tp>
@@ -50,7 +51,7 @@ namespace __size {
     !__member_size<_Tp> &&
     __class_or_enum<remove_cvref_t<_Tp>> &&
     requires(_Tp&& __t) {
-      { _VSTD::__decay_copy(size(_VSTD::forward<_Tp>(__t))) } -> __integer_like;
+      { _LIBCPP_AUTO_CAST(size(__t)) } -> __integer_like;
     };
 
   template <class _Tp>
@@ -76,14 +77,14 @@ namespace __size {
 
     template <__member_size _Tp>
     [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __integer_like auto operator()(_Tp&& __t) const
-        noexcept(noexcept(_VSTD::forward<_Tp>(__t).size())) {
-      return _VSTD::forward<_Tp>(__t).size();
+        noexcept(noexcept(_LIBCPP_AUTO_CAST(__t.size()))) {
+      return _LIBCPP_AUTO_CAST(__t.size());
     }
 
     template <__unqualified_size _Tp>
     [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __integer_like auto operator()(_Tp&& __t) const
-        noexcept(noexcept(size(_VSTD::forward<_Tp>(__t)))) {
-      return size(_VSTD::forward<_Tp>(__t));
+        noexcept(noexcept(_LIBCPP_AUTO_CAST(size(__t)))) {
+      return _LIBCPP_AUTO_CAST(size(__t));
     }
 
     template<__difference _Tp>
@@ -92,18 +93,23 @@ namespace __size {
       return _VSTD::__to_unsigned_like(ranges::end(__t) - ranges::begin(__t));
     }
   };
-} // end namespace __size
+}
 
 inline namespace __cpo {
   inline constexpr auto size = __size::__fn{};
 } // namespace __cpo
+} // namespace ranges
+
+// [range.prim.ssize]
 
+namespace ranges {
 namespace __ssize {
   struct __fn {
     template<class _Tp>
       requires requires (_Tp&& __t) { ranges::size(__t); }
     [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr integral auto operator()(_Tp&& __t) const
-        noexcept(noexcept(ranges::size(__t))) {
+      noexcept(noexcept(ranges::size(__t)))
+    {
       using _Signed = make_signed_t<decltype(ranges::size(__t))>;
       if constexpr (sizeof(ptrdiff_t) > sizeof(_Signed))
         return static_cast<ptrdiff_t>(ranges::size(__t));
@@ -118,8 +124,6 @@ inline namespace __cpo {
 } // namespace __cpo
 } // namespace ranges
 
-// clang-format off
-
 #endif // !defined(_LIBCPP_HAS_NO_RANGES)
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__ranges/subrange.h b/libcxx/include/__ranges/subrange.h
index af4e27600625..8e984f2bf06c 100644
--- a/libcxx/include/__ranges/subrange.h
+++ b/libcxx/include/__ranges/subrange.h
@@ -91,14 +91,14 @@ namespace ranges {
     _LIBCPP_HIDE_FROM_ABI
     constexpr subrange(__convertible_to_non_slicing<_Iter> auto __iter, _Sent __sent)
       requires _MustProvideSizeAtConstruction
-      : __begin_(_VSTD::move(__iter)), __end_(std::move(__sent))
+      : __begin_(_VSTD::move(__iter)), __end_(_VSTD::move(__sent))
     { }
 
     _LIBCPP_HIDE_FROM_ABI
     constexpr subrange(__convertible_to_non_slicing<_Iter> auto __iter, _Sent __sent,
                        make_unsigned_t<iter_difference_t<_Iter>> __n)
       requires (_Kind == subrange_kind::sized)
-      : __begin_(_VSTD::move(__iter)), __end_(std::move(__sent)), __size_(__n)
+      : __begin_(_VSTD::move(__iter)), __end_(_VSTD::move(__sent)), __size_(__n)
     {
       if constexpr (sized_sentinel_for<_Sent, _Iter>)
         _LIBCPP_ASSERT((__end_ - __begin_) == static_cast<iter_difference_t<_Iter>>(__n),
diff --git a/libcxx/include/__ranges/transform_view.h b/libcxx/include/__ranges/transform_view.h
index 208a9a22694a..0f53fbaa7e68 100644
--- a/libcxx/include/__ranges/transform_view.h
+++ b/libcxx/include/__ranges/transform_view.h
@@ -46,11 +46,15 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 namespace ranges {
 
+template<class _Fn, class _View>
+concept __regular_invocable_with_range_ref =
+  regular_invocable<_Fn, range_reference_t<_View>>;
+
 template<class _View, class _Fn>
 concept __transform_view_constraints =
-           view<_View> && is_object_v<_Fn> &&
-           regular_invocable<_Fn&, range_reference_t<_View>> &&
-           __referenceable<invoke_result_t<_Fn&, range_reference_t<_View>>>;
+  view<_View> && is_object_v<_Fn> &&
+  regular_invocable<_Fn&, range_reference_t<_View>> &&
+  __referenceable<invoke_result_t<_Fn&, range_reference_t<_View>>>;
 
 template<input_range _View, copy_constructible _Fn>
   requires __transform_view_constraints<_View, _Fn>
@@ -82,7 +86,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI
   constexpr __iterator<true> begin() const
     requires range<const _View> &&
-             regular_invocable<const _Fn&, range_reference_t<const _View>>
+             __regular_invocable_with_range_ref<const _Fn&, const _View>
   {
     return __iterator<true>(*this, ranges::begin(__base_));
   }
@@ -100,14 +104,14 @@ public:
   _LIBCPP_HIDE_FROM_ABI
   constexpr __sentinel<true> end() const
     requires range<const _View> &&
-             regular_invocable<const _Fn&, range_reference_t<const _View>>
+             __regular_invocable_with_range_ref<const _Fn&, const _View>
   {
     return __sentinel<true>(ranges::end(__base_));
   }
   _LIBCPP_HIDE_FROM_ABI
   constexpr __iterator<true> end() const
     requires common_range<const _View> &&
-             regular_invocable<const _Fn&, range_reference_t<const _View>>
+             __regular_invocable_with_range_ref<const _Fn&, const _View>
   {
     return __iterator<true>(*this, ranges::end(__base_));
   }
diff --git a/libcxx/include/__string b/libcxx/include/__string
index 890fb21dd3f1..13ff7b35e47d 100644
--- a/libcxx/include/__string
+++ b/libcxx/include/__string
@@ -42,9 +42,6 @@ _LIBCPP_PUSH_MACROS
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-// The the extern template ABI lists are kept outside of <string> to improve the
-// readability of that header.
-
 // The extern template ABI lists are kept outside of <string> to improve the
 // readability of that header. We maintain 2 ABI lists:
 // - _LIBCPP_STRING_V1_EXTERN_TEMPLATE_LIST
@@ -293,31 +290,34 @@ char_traits<_CharT>::assign(char_type* __s, size_t __n, char_type __a)
 
 template <class _CharT>
 static inline _LIBCPP_CONSTEXPR_AFTER_CXX17
-_CharT* __move_constexpr(_CharT* __s1, const _CharT* __s2, size_t __n) _NOEXCEPT
+_CharT* __copy_constexpr(_CharT* __dest, const _CharT* __source, size_t __n) _NOEXCEPT
 {
-    if (__n == 0) return __s1;
-    if (__s1 < __s2) {
-      _VSTD::copy(__s2, __s2 + __n, __s1);
-    } else if (__s2 < __s1) {
-      _VSTD::copy_backward(__s2, __s2 + __n, __s1 + __n);
-    }
-    return __s1;
+  _LIBCPP_ASSERT(__libcpp_is_constant_evaluated(), "__copy_constexpr() should always be constant evaluated");
+  _VSTD::copy_n(__source, __n, __dest);
+  return __dest;
 }
 
 template <class _CharT>
 static inline _LIBCPP_CONSTEXPR_AFTER_CXX17
-_CharT* __copy_constexpr(_CharT* __s1, const _CharT* __s2, size_t __n) _NOEXCEPT
+_CharT* __move_constexpr(_CharT* __dest, const _CharT* __source, size_t __n) _NOEXCEPT
 {
-    _VSTD::copy_n(__s2, __n, __s1);
-    return __s1;
+  _LIBCPP_ASSERT(__libcpp_is_constant_evaluated(), "__move_constexpr() should always be constant evaluated");
+  if (__n == 0)
+    return __dest;
+  _CharT* __allocation = new _CharT[__n];
+  _VSTD::__copy_constexpr(__allocation, __source, __n);
+  _VSTD::__copy_constexpr(__dest, static_cast<const _CharT*>(__allocation), __n);
+  delete[] __allocation;
+  return __dest;
 }
 
 template <class _CharT>
 static inline _LIBCPP_CONSTEXPR_AFTER_CXX17
 _CharT* __assign_constexpr(_CharT* __s, size_t __n, _CharT __a) _NOEXCEPT
 {
-     _VSTD::fill_n(__s, __n, __a);
-     return __s;
+  _LIBCPP_ASSERT(__libcpp_is_constant_evaluated(), "__assign_constexpr() should always be constant evaluated");
+  _VSTD::fill_n(__s, __n, __a);
+  return __s;
 }
 
 // char_traits<char>
@@ -340,8 +340,21 @@ struct _LIBCPP_TEMPLATE_VIS char_traits<char>
 
     static _LIBCPP_CONSTEXPR_AFTER_CXX14
     int compare(const char_type* __s1, const char_type* __s2, size_t __n) _NOEXCEPT;
-    static inline size_t _LIBCPP_CONSTEXPR_AFTER_CXX14
-    length(const char_type* __s)  _NOEXCEPT {return __builtin_strlen(__s);}
+
+    static inline size_t _LIBCPP_CONSTEXPR_AFTER_CXX14 length(const char_type* __s)  _NOEXCEPT {
+      // GCC currently does not support __builtin_strlen during constant evaluation.
+      // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=70816
+#ifdef _LIBCPP_COMPILER_GCC
+      if (__libcpp_is_constant_evaluated()) {
+        size_t __i = 0;
+        for (; __s[__i] != char_type('\0'); ++__i)
+            ;
+        return __i;
+      }
+#endif
+      return __builtin_strlen(__s);
+    }
+
     static _LIBCPP_CONSTEXPR_AFTER_CXX14
     const char_type* find(const char_type* __s, size_t __n, const char_type& __a) _NOEXCEPT;
     static inline _LIBCPP_CONSTEXPR_AFTER_CXX17
diff --git a/libcxx/include/__threading_support b/libcxx/include/__threading_support
index 0094fca2fb38..68f381a62183 100644
--- a/libcxx/include/__threading_support
+++ b/libcxx/include/__threading_support
@@ -274,7 +274,8 @@ struct __libcpp_timed_backoff_policy {
 
 namespace __thread_detail {
 
-inline __libcpp_timespec_t __convert_to_timespec(const chrono::nanoseconds& __ns)
+_LIBCPP_HIDE_FROM_ABI inline
+__libcpp_timespec_t __convert_to_timespec(const chrono::nanoseconds& __ns)
 {
   using namespace chrono;
   seconds __s = duration_cast<seconds>(__ns);
@@ -296,10 +297,11 @@ inline __libcpp_timespec_t __convert_to_timespec(const chrono::nanoseconds& __ns
   return __ts;
 }
 
-}
+} // namespace __thread_detail
 
 #if defined(_LIBCPP_HAS_THREAD_API_PTHREAD)
 
+_LIBCPP_HIDE_FROM_ABI inline
 int __libcpp_recursive_mutex_init(__libcpp_recursive_mutex_t *__m)
 {
   pthread_mutexattr_t attr;
@@ -324,74 +326,88 @@ int __libcpp_recursive_mutex_init(__libcpp_recursive_mutex_t *__m)
   return 0;
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 int __libcpp_recursive_mutex_lock(__libcpp_recursive_mutex_t *__m)
 {
   return pthread_mutex_lock(__m);
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 bool __libcpp_recursive_mutex_trylock(__libcpp_recursive_mutex_t *__m)
 {
   return pthread_mutex_trylock(__m) == 0;
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 int __libcpp_recursive_mutex_unlock(__libcpp_recursive_mutex_t *__m)
 {
   return pthread_mutex_unlock(__m);
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 int __libcpp_recursive_mutex_destroy(__libcpp_recursive_mutex_t *__m)
 {
   return pthread_mutex_destroy(__m);
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 int __libcpp_mutex_lock(__libcpp_mutex_t *__m)
 {
   return pthread_mutex_lock(__m);
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 bool __libcpp_mutex_trylock(__libcpp_mutex_t *__m)
 {
   return pthread_mutex_trylock(__m) == 0;
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 int __libcpp_mutex_unlock(__libcpp_mutex_t *__m)
 {
   return pthread_mutex_unlock(__m);
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 int __libcpp_mutex_destroy(__libcpp_mutex_t *__m)
 {
   return pthread_mutex_destroy(__m);
 }
 
 // Condition Variable
+_LIBCPP_HIDE_FROM_ABI inline
 int __libcpp_condvar_signal(__libcpp_condvar_t *__cv)
 {
   return pthread_cond_signal(__cv);
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 int __libcpp_condvar_broadcast(__libcpp_condvar_t *__cv)
 {
   return pthread_cond_broadcast(__cv);
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 int __libcpp_condvar_wait(__libcpp_condvar_t *__cv, __libcpp_mutex_t *__m)
 {
   return pthread_cond_wait(__cv, __m);
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 int __libcpp_condvar_timedwait(__libcpp_condvar_t *__cv, __libcpp_mutex_t *__m,
                                __libcpp_timespec_t *__ts)
 {
   return pthread_cond_timedwait(__cv, __m, __ts);
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 int __libcpp_condvar_destroy(__libcpp_condvar_t *__cv)
 {
   return pthread_cond_destroy(__cv);
 }
 
 // Execute once
+_LIBCPP_HIDE_FROM_ABI inline
 int __libcpp_execute_once(__libcpp_exec_once_flag *flag,
                           void (*init_routine)()) {
   return pthread_once(flag, init_routine);
@@ -399,34 +415,40 @@ int __libcpp_execute_once(__libcpp_exec_once_flag *flag,
 
 // Thread id
 // Returns non-zero if the thread ids are equal, otherwise 0
+_LIBCPP_HIDE_FROM_ABI inline
 bool __libcpp_thread_id_equal(__libcpp_thread_id t1, __libcpp_thread_id t2)
 {
   return t1 == t2;
 }
 
 // Returns non-zero if t1 < t2, otherwise 0
+_LIBCPP_HIDE_FROM_ABI inline
 bool __libcpp_thread_id_less(__libcpp_thread_id t1, __libcpp_thread_id t2)
 {
   return t1 < t2;
 }
 
 // Thread
+_LIBCPP_HIDE_FROM_ABI inline
 bool __libcpp_thread_isnull(const __libcpp_thread_t *__t) {
   return __libcpp_thread_get_id(__t) == 0;
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 int __libcpp_thread_create(__libcpp_thread_t *__t, void *(*__func)(void *),
                            void *__arg)
 {
   return pthread_create(__t, nullptr, __func, __arg);
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 __libcpp_thread_id __libcpp_thread_get_current_id()
 {
   const __libcpp_thread_t thread = pthread_self();
   return __libcpp_thread_get_id(&thread);
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 __libcpp_thread_id __libcpp_thread_get_id(const __libcpp_thread_t *__t)
 {
 #if defined(__MVS__)
@@ -436,21 +458,25 @@ __libcpp_thread_id __libcpp_thread_get_id(const __libcpp_thread_t *__t)
 #endif
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 int __libcpp_thread_join(__libcpp_thread_t *__t)
 {
   return pthread_join(*__t, nullptr);
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 int __libcpp_thread_detach(__libcpp_thread_t *__t)
 {
   return pthread_detach(*__t);
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 void __libcpp_thread_yield()
 {
   sched_yield();
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 void __libcpp_thread_sleep_for(const chrono::nanoseconds& __ns)
 {
    __libcpp_timespec_t __ts = __thread_detail::__convert_to_timespec(__ns);
@@ -458,16 +484,19 @@ void __libcpp_thread_sleep_for(const chrono::nanoseconds& __ns)
 }
 
 // Thread local storage
+_LIBCPP_HIDE_FROM_ABI inline
 int __libcpp_tls_create(__libcpp_tls_key *__key, void (*__at_exit)(void *))
 {
   return pthread_key_create(__key, __at_exit);
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 void *__libcpp_tls_get(__libcpp_tls_key __key)
 {
   return pthread_getspecific(__key);
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 int __libcpp_tls_set(__libcpp_tls_key __key, void *__p)
 {
     return pthread_setspecific(__key, __p);
@@ -475,47 +504,56 @@ int __libcpp_tls_set(__libcpp_tls_key __key, void *__p)
 
 #elif defined(_LIBCPP_HAS_THREAD_API_C11)
 
+_LIBCPP_HIDE_FROM_ABI inline
 int __libcpp_recursive_mutex_init(__libcpp_recursive_mutex_t *__m)
 {
   return mtx_init(__m, mtx_plain | mtx_recursive) == thrd_success ? 0 : EINVAL;
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 int __libcpp_recursive_mutex_lock(__libcpp_recursive_mutex_t *__m)
 {
   return mtx_lock(__m) == thrd_success ? 0 : EINVAL;
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 bool __libcpp_recursive_mutex_trylock(__libcpp_recursive_mutex_t *__m)
 {
   return mtx_trylock(__m) == thrd_success;
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 int __libcpp_recursive_mutex_unlock(__libcpp_recursive_mutex_t *__m)
 {
   return mtx_unlock(__m) == thrd_success ? 0 : EINVAL;
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 int __libcpp_recursive_mutex_destroy(__libcpp_recursive_mutex_t *__m)
 {
   mtx_destroy(__m);
   return 0;
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 int __libcpp_mutex_lock(__libcpp_mutex_t *__m)
 {
   return mtx_lock(__m) == thrd_success ? 0 : EINVAL;
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 bool __libcpp_mutex_trylock(__libcpp_mutex_t *__m)
 {
   return mtx_trylock(__m) == thrd_success;
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 int __libcpp_mutex_unlock(__libcpp_mutex_t *__m)
 {
   return mtx_unlock(__m) == thrd_success ? 0 : EINVAL;
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 int __libcpp_mutex_destroy(__libcpp_mutex_t *__m)
 {
   mtx_destroy(__m);
@@ -523,21 +561,25 @@ int __libcpp_mutex_destroy(__libcpp_mutex_t *__m)
 }
 
 // Condition Variable
+_LIBCPP_HIDE_FROM_ABI inline
 int __libcpp_condvar_signal(__libcpp_condvar_t *__cv)
 {
   return cnd_signal(__cv) == thrd_success ? 0 : EINVAL;
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 int __libcpp_condvar_broadcast(__libcpp_condvar_t *__cv)
 {
   return cnd_broadcast(__cv) == thrd_success ? 0 : EINVAL;
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 int __libcpp_condvar_wait(__libcpp_condvar_t *__cv, __libcpp_mutex_t *__m)
 {
   return cnd_wait(__cv, __m) == thrd_success ? 0 : EINVAL;
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 int __libcpp_condvar_timedwait(__libcpp_condvar_t *__cv, __libcpp_mutex_t *__m,
                                timespec *__ts)
 {
@@ -545,6 +587,7 @@ int __libcpp_condvar_timedwait(__libcpp_condvar_t *__cv, __libcpp_mutex_t *__m,
   return __ec == thrd_timedout ? ETIMEDOUT : __ec;
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 int __libcpp_condvar_destroy(__libcpp_condvar_t *__cv)
 {
   cnd_destroy(__cv);
@@ -552,6 +595,7 @@ int __libcpp_condvar_destroy(__libcpp_condvar_t *__cv)
 }
 
 // Execute once
+_LIBCPP_HIDE_FROM_ABI inline
 int __libcpp_execute_once(__libcpp_exec_once_flag *flag,
                           void (*init_routine)(void)) {
   ::call_once(flag, init_routine);
@@ -560,22 +604,26 @@ int __libcpp_execute_once(__libcpp_exec_once_flag *flag,
 
 // Thread id
 // Returns non-zero if the thread ids are equal, otherwise 0
+_LIBCPP_HIDE_FROM_ABI inline
 bool __libcpp_thread_id_equal(__libcpp_thread_id t1, __libcpp_thread_id t2)
 {
   return thrd_equal(t1, t2) != 0;
 }
 
 // Returns non-zero if t1 < t2, otherwise 0
+_LIBCPP_HIDE_FROM_ABI inline
 bool __libcpp_thread_id_less(__libcpp_thread_id t1, __libcpp_thread_id t2)
 {
   return t1 < t2;
 }
 
 // Thread
+_LIBCPP_HIDE_FROM_ABI inline
 bool __libcpp_thread_isnull(const __libcpp_thread_t *__t) {
   return __libcpp_thread_get_id(__t) == 0;
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 int __libcpp_thread_create(__libcpp_thread_t *__t, void *(*__func)(void *),
                            void *__arg)
 {
@@ -583,31 +631,37 @@ int __libcpp_thread_create(__libcpp_thread_t *__t, void *(*__func)(void *),
   return __ec == thrd_nomem ? ENOMEM : __ec;
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 __libcpp_thread_id __libcpp_thread_get_current_id()
 {
   return thrd_current();
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 __libcpp_thread_id __libcpp_thread_get_id(const __libcpp_thread_t *__t)
 {
   return *__t;
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 int __libcpp_thread_join(__libcpp_thread_t *__t)
 {
   return thrd_join(*__t, nullptr) == thrd_success ? 0 : EINVAL;
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 int __libcpp_thread_detach(__libcpp_thread_t *__t)
 {
   return thrd_detach(*__t) == thrd_success ? 0 : EINVAL;
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 void __libcpp_thread_yield()
 {
   thrd_yield();
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 void __libcpp_thread_sleep_for(const chrono::nanoseconds& __ns)
 {
    __libcpp_timespec_t __ts = __thread_detail::__convert_to_timespec(__ns);
@@ -615,16 +669,19 @@ void __libcpp_thread_sleep_for(const chrono::nanoseconds& __ns)
 }
 
 // Thread local storage
+_LIBCPP_HIDE_FROM_ABI inline
 int __libcpp_tls_create(__libcpp_tls_key *__key, void (*__at_exit)(void *))
 {
   return tss_create(__key, __at_exit) == thrd_success ? 0 : EINVAL;
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 void *__libcpp_tls_get(__libcpp_tls_key __key)
 {
   return tss_get(__key);
 }
 
+_LIBCPP_HIDE_FROM_ABI inline
 int __libcpp_tls_set(__libcpp_tls_key __key, void *__p)
 {
   return tss_set(__key, __p) == thrd_success ? 0 : EINVAL;
@@ -643,7 +700,7 @@ namespace this_thread
 
 _LIBCPP_INLINE_VISIBILITY __thread_id get_id() _NOEXCEPT;
 
-}  // this_thread
+} // namespace this_thread
 
 template<> struct hash<__thread_id>;
 
@@ -713,7 +770,7 @@ get_id() _NOEXCEPT
     return __libcpp_thread_get_current_id();
 }
 
-}  // this_thread
+} // namespace this_thread
 
 #endif // !_LIBCPP_HAS_NO_THREADS
 
diff --git a/libcxx/include/__tuple b/libcxx/include/__tuple
index 11fbba260238..e8eb0b3aaf0c 100644
--- a/libcxx/include/__tuple
+++ b/libcxx/include/__tuple
@@ -387,7 +387,7 @@ template <bool ..._Preds>
 struct __all_dummy;
 
 template <bool ..._Pred>
-using __all = _IsSame<__all_dummy<_Pred...>, __all_dummy<((void)_Pred, true)...>>;
+struct __all : _IsSame<__all_dummy<_Pred...>, __all_dummy<((void)_Pred, true)...>> {};
 
 struct __tuple_sfinae_base {
   template <template <class, class...> class _Trait,
diff --git a/libcxx/include/__utility/decay_copy.h b/libcxx/include/__utility/auto_cast.h
index 5c9716a89279..5c368e077508 100644
--- a/libcxx/include/__utility/decay_copy.h
+++ b/libcxx/include/__utility/auto_cast.h
@@ -7,29 +7,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___UTILITY_DECAY_COPY_H
-#define _LIBCPP___UTILITY_DECAY_COPY_H
+#ifndef _LIBCPP___UTILITY_AUTO_CAST_H
+#define _LIBCPP___UTILITY_AUTO_CAST_H
 
 #include <__config>
-#include <__utility/forward.h>
 #include <type_traits>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #pragma GCC system_header
 #endif
 
-_LIBCPP_BEGIN_NAMESPACE_STD
+#define _LIBCPP_AUTO_CAST(expr) static_cast<typename decay<decltype((expr))>::type>(expr)
 
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR
-typename decay<_Tp>::type __decay_copy(_Tp&& __t)
-#if _LIBCPP_STD_VER > 17
-    noexcept(is_nothrow_convertible_v<_Tp, remove_reference_t<_Tp>>)
-#endif
-{
-  return _VSTD::forward<_Tp>(__t);
-}
-
-_LIBCPP_END_NAMESPACE_STD
-
-#endif // _LIBCPP___UTILITY_DECAY_COPY_H
+#endif // _LIBCPP___UTILITY_AUTO_CAST_H
diff --git a/libcxx/include/__utility/rel_ops.h b/libcxx/include/__utility/rel_ops.h
index c94b8fddafee..d59e96d72dc1 100644
--- a/libcxx/include/__utility/rel_ops.h
+++ b/libcxx/include/__utility/rel_ops.h
@@ -55,7 +55,7 @@ operator>=(const _Tp& __x, const _Tp& __y)
     return !(__x < __y);
 }
 
-}  // rel_ops
+} // namespace rel_ops
 
 _LIBCPP_END_NAMESPACE_STD
 
diff --git a/libcxx/include/__utility/transaction.h b/libcxx/include/__utility/transaction.h
new file mode 100644
index 000000000000..5bc3a500fdc5
--- /dev/null
+++ b/libcxx/include/__utility/transaction.h
@@ -0,0 +1,91 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___UTILITY_TRANSACTION_H
+#define _LIBCPP___UTILITY_TRANSACTION_H
+
+#include <__config>
+#include <__utility/exchange.h>
+#include <__utility/move.h>
+#include <type_traits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// __transaction is a helper class for writing code with the strong exception guarantee.
+//
+// When writing code that can throw an exception, one can store rollback instructions in a
+// transaction so that if an exception is thrown at any point during the lifetime of the
+// transaction, it will be rolled back automatically. When the transaction is done, one
+// must mark it as being complete so it isn't rolled back when the transaction is destroyed.
+//
+// Transactions are not default constructible, they can't be copied or assigned to, but
+// they can be moved around for convenience.
+//
+// __transaction can help greatly simplify code that would normally be cluttered by
+// `#if _LIBCPP_NO_EXCEPTIONS`. For example:
+//
+//    template <class Iterator, class Size, class OutputIterator>
+//    Iterator uninitialized_copy_n(Iterator iter, Size n, OutputIterator out) {
+//        typedef typename iterator_traits<Iterator>::value_type value_type;
+//        __transaction transaction([start=out, &out] {
+//            std::destroy(start, out);
+//        });
+//
+//        for (; n > 0; ++iter, ++out, --n) {
+//            ::new ((void*)std::addressof(*out)) value_type(*iter);
+//        }
+//        transaction.__complete();
+//        return out;
+//    }
+//
+template <class _Rollback>
+struct __transaction {
+    __transaction() = delete;
+
+    _LIBCPP_HIDE_FROM_ABI
+    _LIBCPP_CONSTEXPR_AFTER_CXX17 explicit __transaction(_Rollback __rollback)
+        : __rollback_(_VSTD::move(__rollback))
+        , __completed_(false)
+    { }
+
+    _LIBCPP_HIDE_FROM_ABI
+    _LIBCPP_CONSTEXPR_AFTER_CXX17 __transaction(__transaction&& __other)
+        _NOEXCEPT_(is_nothrow_move_constructible<_Rollback>::value)
+        : __rollback_(_VSTD::move(__other.__rollback_))
+        , __completed_(__other.__completed_)
+    {
+        __other.__completed_ = true;
+    }
+
+    __transaction(__transaction const&) = delete;
+    __transaction& operator=(__transaction const&) = delete;
+    __transaction& operator=(__transaction&&) = delete;
+
+    _LIBCPP_HIDE_FROM_ABI
+    _LIBCPP_CONSTEXPR_AFTER_CXX17 void __complete() _NOEXCEPT {
+        __completed_ = true;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI
+    _LIBCPP_CONSTEXPR_AFTER_CXX17 ~__transaction() {
+        if (!__completed_)
+            __rollback_();
+    }
+
+private:
+    _Rollback __rollback_;
+    bool __completed_;
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___UTILITY_TRANSACTION_H
diff --git a/libcxx/include/atomic b/libcxx/include/atomic
index cfd0e1d054a8..4b60d4d6802b 100644
--- a/libcxx/include/atomic
+++ b/libcxx/include/atomic
@@ -1443,7 +1443,7 @@ struct __cxx_atomic_impl : public _Base {
     static_assert(is_trivially_copyable<_Tp>::value,
       "std::atomic<T> requires that 'T' be a trivially copyable type");
 
-  _LIBCPP_INLINE_VISIBILITY __cxx_atomic_impl() _NOEXCEPT _LIBCPP_DEFAULT
+  _LIBCPP_INLINE_VISIBILITY __cxx_atomic_impl() _NOEXCEPT = default;
   _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR explicit __cxx_atomic_impl(_Tp value) _NOEXCEPT
     : _Base(value) {}
 };
@@ -1644,7 +1644,7 @@ struct __atomic_base  // false
     __atomic_base() noexcept(is_nothrow_default_constructible_v<_Tp>) : __a_(_Tp()) {}
 #else
     _LIBCPP_INLINE_VISIBILITY
-    __atomic_base() _NOEXCEPT _LIBCPP_DEFAULT
+    __atomic_base() _NOEXCEPT = default;
 #endif
 
     _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR
@@ -1673,7 +1673,7 @@ struct __atomic_base<_Tp, true>
     typedef __atomic_base<_Tp, false> __base;
 
     _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
-    __atomic_base() _NOEXCEPT _LIBCPP_DEFAULT
+    __atomic_base() _NOEXCEPT = default;
 
     _LIBCPP_INLINE_VISIBILITY
     _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __base(__d) {}
@@ -1762,7 +1762,7 @@ struct atomic
     atomic() = default;
 #else
     _LIBCPP_INLINE_VISIBILITY
-    atomic() _NOEXCEPT _LIBCPP_DEFAULT
+    atomic() _NOEXCEPT = default;
 #endif
 
     _LIBCPP_INLINE_VISIBILITY
@@ -1790,7 +1790,7 @@ struct atomic<_Tp*>
     typedef ptrdiff_t difference_type;
 
     _LIBCPP_INLINE_VISIBILITY
-    atomic() _NOEXCEPT _LIBCPP_DEFAULT
+    atomic() _NOEXCEPT = default;
 
     _LIBCPP_INLINE_VISIBILITY
     _LIBCPP_CONSTEXPR atomic(_Tp* __d) _NOEXCEPT : __base(__d) {}
@@ -2432,7 +2432,7 @@ typedef struct atomic_flag
     atomic_flag() _NOEXCEPT : __a_(false) {}
 #else
     _LIBCPP_INLINE_VISIBILITY
-    atomic_flag() _NOEXCEPT _LIBCPP_DEFAULT
+    atomic_flag() _NOEXCEPT = default;
 #endif
 
     _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR
@@ -2699,6 +2699,13 @@ typedef atomic<__libcpp_unsigned_lock_free> atomic_unsigned_lock_free;
 #define ATOMIC_FLAG_INIT {false}
 #define ATOMIC_VAR_INIT(__v) {__v}
 
+#if _LIBCPP_STD_VER > 17 && !defined(_LIBCPP_DISABLE_DEPRECATION_WARNINGS)
+# if defined(_LIBCPP_CLANG_VER) && _LIBCPP_CLANG_VER >= 1400
+#  pragma clang deprecated(ATOMIC_FLAG_INIT)
+#  pragma clang deprecated(ATOMIC_VAR_INIT)
+# endif
+#endif // _LIBCPP_STD_VER > 17 && !defined(_LIBCPP_DISABLE_DEPRECATION_WARNINGS)
+
 _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_ATOMIC
diff --git a/libcxx/include/bit b/libcxx/include/bit
index 0aab83e7a6eb..57a13768c493 100644
--- a/libcxx/include/bit
+++ b/libcxx/include/bit
@@ -308,7 +308,7 @@ bit_ceil(_Tp __t) noexcept
 {
     if (__t < 2) return 1;
     const unsigned __n = numeric_limits<_Tp>::digits - countl_zero((_Tp)(__t - 1u));
-    _LIBCPP_DEBUG_ASSERT(__libcpp_is_constant_evaluated() || __n != numeric_limits<_Tp>::digits, "Bad input to bit_ceil");
+    _LIBCPP_ASSERT(__n != numeric_limits<_Tp>::digits, "Bad input to bit_ceil");
 
     if constexpr (sizeof(_Tp) >= sizeof(unsigned))
         return _Tp{1} << __n;
diff --git a/libcxx/include/charconv b/libcxx/include/charconv
index 3c969dc79ab0..06634fe7bc39 100644
--- a/libcxx/include/charconv
+++ b/libcxx/include/charconv
@@ -105,7 +105,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 namespace __itoa {
 _LIBCPP_AVAILABILITY_TO_CHARS _LIBCPP_FUNC_VIS char* __u64toa(uint64_t __value, char* __buffer) _NOEXCEPT;
 _LIBCPP_AVAILABILITY_TO_CHARS _LIBCPP_FUNC_VIS char* __u32toa(uint32_t __value, char* __buffer) _NOEXCEPT;
-}
+} // namespace __itoa
 
 #ifndef _LIBCPP_CXX03_LANG
 
@@ -598,6 +598,38 @@ from_chars(const char* __first, const char* __last, _Tp& __value, int __base)
     return __from_chars_integral(__first, __last, __value, __base);
 }
 
+// Floating-point implementation starts here.
+// Unlike the other parts of charconv this is only available in C++17 and newer.
+#if _LIBCPP_STD_VER > 14
+
+_LIBCPP_AVAILABILITY_TO_CHARS_FLOATING_POINT _LIBCPP_FUNC_VIS
+to_chars_result to_chars(char* __first, char* __last, float __value);
+
+_LIBCPP_AVAILABILITY_TO_CHARS_FLOATING_POINT _LIBCPP_FUNC_VIS
+to_chars_result to_chars(char* __first, char* __last, double __value);
+
+_LIBCPP_AVAILABILITY_TO_CHARS_FLOATING_POINT _LIBCPP_FUNC_VIS
+to_chars_result to_chars(char* __first, char* __last, long double __value);
+
+_LIBCPP_AVAILABILITY_TO_CHARS_FLOATING_POINT _LIBCPP_FUNC_VIS
+to_chars_result to_chars(char* __first, char* __last, float __value, chars_format __fmt);
+
+_LIBCPP_AVAILABILITY_TO_CHARS_FLOATING_POINT _LIBCPP_FUNC_VIS
+to_chars_result to_chars(char* __first, char* __last, double __value, chars_format __fmt);
+
+_LIBCPP_AVAILABILITY_TO_CHARS_FLOATING_POINT _LIBCPP_FUNC_VIS
+to_chars_result to_chars(char* __first, char* __last, long double __value, chars_format __fmt);
+
+_LIBCPP_AVAILABILITY_TO_CHARS_FLOATING_POINT _LIBCPP_FUNC_VIS
+to_chars_result to_chars(char* __first, char* __last, float __value, chars_format __fmt, int __precision);
+
+_LIBCPP_AVAILABILITY_TO_CHARS_FLOATING_POINT _LIBCPP_FUNC_VIS
+to_chars_result to_chars(char* __first, char* __last, double __value, chars_format __fmt, int __precision);
+
+_LIBCPP_AVAILABILITY_TO_CHARS_FLOATING_POINT _LIBCPP_FUNC_VIS
+to_chars_result to_chars(char* __first, char* __last, long double __value, chars_format __fmt, int __precision);
+
+#  endif // _LIBCPP_STD_VER > 14
 #endif // _LIBCPP_CXX03_LANG
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/chrono b/libcxx/include/chrono
index d29734f6c358..90c9b0829a1f 100644
--- a/libcxx/include/chrono
+++ b/libcxx/include/chrono
@@ -738,7 +738,7 @@ struct __is_duration<volatile duration<_Rep, _Period> > : true_type  {};
 template <class _Rep, class _Period>
 struct __is_duration<const volatile duration<_Rep, _Period> > : true_type  {};
 
-} // chrono
+} // namespace chrono
 
 template <class _Rep1, class _Period1, class _Rep2, class _Period2>
 struct _LIBCPP_TEMPLATE_VIS common_type<chrono::duration<_Rep1, _Period1>,
@@ -1265,7 +1265,7 @@ public:
     _LIBCPP_INLINE_VISIBILITY static _LIBCPP_CONSTEXPR time_point max() _NOEXCEPT {return time_point(duration::max());}
 };
 
-} // chrono
+} // namespace chrono
 
 template <class _Clock, class _Duration1, class _Duration2>
 struct _LIBCPP_TEMPLATE_VIS common_type<chrono::time_point<_Clock, _Duration1>,
@@ -2701,7 +2701,7 @@ constexpr hours make24(const hours& __h, bool __is_pm) noexcept
 }
 
 #endif // _LIBCPP_STD_VER > 17
-} // chrono
+} // namespace chrono
 
 #if _LIBCPP_STD_VER > 11
 // Suffixes for duration literals [time.duration.literals]
@@ -2786,11 +2786,12 @@ inline namespace literals
         return chrono::year(static_cast<int>(__y));
     }
 #endif
-}}
+} // namespace chrono_literals
+} // namespace literals
 
 namespace chrono { // hoist the literals into namespace std::chrono
    using namespace literals::chrono_literals;
-}
+} // namespace chrono
 
 #endif
 
diff --git a/libcxx/include/cmath b/libcxx/include/cmath
index 3a7985f7d454..b5c332c81ad6 100644
--- a/libcxx/include/cmath
+++ b/libcxx/include/cmath
@@ -638,36 +638,6 @@ lerp(long double __a, long double __b, long double __t) _NOEXCEPT { return __ler
 
 #endif // _LIBCPP_STD_VER > 17
 
-template <class _IntT, class _FloatT,
-    bool _FloatBigger = (numeric_limits<_FloatT>::digits > numeric_limits<_IntT>::digits),
-    int _Bits = (numeric_limits<_IntT>::digits - numeric_limits<_FloatT>::digits)>
-_LIBCPP_INLINE_VISIBILITY
-_LIBCPP_CONSTEXPR _IntT __max_representable_int_for_float() _NOEXCEPT {
-  static_assert(is_floating_point<_FloatT>::value, "must be a floating point type");
-  static_assert(is_integral<_IntT>::value, "must be an integral type");
-  static_assert(numeric_limits<_FloatT>::radix == 2, "FloatT has incorrect radix");
-  static_assert((_IsSame<_FloatT, float>::value || _IsSame<_FloatT, double>::value
-                 || _IsSame<_FloatT,long double>::value), "unsupported floating point type");
-  return _FloatBigger ? numeric_limits<_IntT>::max() :  (numeric_limits<_IntT>::max() >> _Bits << _Bits);
-}
-
-// Convert a floating point number to the specified integral type after
-// clamping to the integral types representable range.
-//
-// The behavior is undefined if `__r` is NaN.
-template <class _IntT, class _RealT>
-_LIBCPP_INLINE_VISIBILITY
-_IntT __clamp_to_integral(_RealT __r) _NOEXCEPT {
-  using _Lim = numeric_limits<_IntT>;
-  const _IntT _MaxVal = __max_representable_int_for_float<_IntT, _RealT>();
-  if (__r >= ::nextafter(static_cast<_RealT>(_MaxVal), INFINITY)) {
-    return _Lim::max();
-  } else if (__r <= _Lim::lowest()) {
-    return _Lim::min();
-  }
-  return static_cast<_IntT>(__r);
-}
-
 _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
diff --git a/libcxx/include/compare b/libcxx/include/compare
index 5c4578da0b89..5d07ebaf2fbd 100644
--- a/libcxx/include/compare
+++ b/libcxx/include/compare
@@ -146,4 +146,8 @@ namespace std {
 #include <__compare/weak_order.h>
 #include <__config>
 
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#pragma GCC system_header
+#endif
+
 #endif // _LIBCPP_COMPARE
diff --git a/libcxx/include/complex b/libcxx/include/complex
index 6b74435c6101..2dc58c010495 100644
--- a/libcxx/include/complex
+++ b/libcxx/include/complex
@@ -1485,8 +1485,8 @@ inline namespace literals
     {
         return { 0.0f, static_cast<float>(__im) };
     }
-  }
-}
+  } // namespace complex_literals
+} // namespace literals
 #endif
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/exception b/libcxx/include/exception
index 5f5486149ac0..886daac4439a 100644
--- a/libcxx/include/exception
+++ b/libcxx/include/exception
@@ -328,6 +328,6 @@ rethrow_if_nested(const _Ep&,
 {
 }
 
-}  // std
+} // namespace std
 
 #endif // _LIBCPP_EXCEPTION
diff --git a/libcxx/include/execution b/libcxx/include/execution
index 32b05b85dac6..c1debcb72ff1 100644
--- a/libcxx/include/execution
+++ b/libcxx/include/execution
@@ -16,4 +16,8 @@
 #   include <__pstl_execution>
 #endif
 
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#pragma GCC system_header
+#endif
+
 #endif // _LIBCPP_EXECUTION
diff --git a/libcxx/include/experimental/__memory b/libcxx/include/experimental/__memory
index b38b664b339a..bd9bf95e5933 100644
--- a/libcxx/include/experimental/__memory
+++ b/libcxx/include/experimental/__memory
@@ -17,6 +17,10 @@
 #include <__functional_base>
 #include <type_traits>
 
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#pragma GCC system_header
+#endif
+
 _LIBCPP_BEGIN_NAMESPACE_LFTS
 
 template <
diff --git a/libcxx/include/experimental/iterator b/libcxx/include/experimental/iterator
index 10c903832d8e..9da08a0cff9d 100644
--- a/libcxx/include/experimental/iterator
+++ b/libcxx/include/experimental/iterator
@@ -52,15 +52,18 @@ namespace std {
 
 */
 
-#include <experimental/__config>
-
-#if _LIBCPP_STD_VER > 11
-
 #include <__memory/addressof.h>
 #include <__utility/move.h>
 #include <__utility/forward.h>
+#include <experimental/__config>
 #include <iterator>
 
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER > 11
+
 _LIBCPP_BEGIN_NAMESPACE_LFTS
 
 template <class _Delim, class _CharT = char, class _Traits = char_traits<_CharT>>
@@ -111,6 +114,6 @@ make_ostream_joiner(basic_ostream<_CharT, _Traits>& __os, _Delim && __d)
 
 _LIBCPP_END_NAMESPACE_LFTS
 
-#endif /* _LIBCPP_STD_VER > 11 */
+#endif // _LIBCPP_STD_VER > 11
 
 #endif // _LIBCPP_EXPERIMENTAL_ITERATOR
diff --git a/libcxx/include/ext/__hash b/libcxx/include/ext/__hash
index 268577f3c922..2f998ee40a93 100644
--- a/libcxx/include/ext/__hash
+++ b/libcxx/include/ext/__hash
@@ -129,6 +129,6 @@ template <> struct _LIBCPP_TEMPLATE_VIS hash<unsigned long>
         return __c;
     }
 };
-}
+} // namespace __gnu_cxx
 
 #endif // _LIBCPP_EXT_HASH
diff --git a/libcxx/include/ext/hash_map b/libcxx/include/ext/hash_map
index 6c757e2fba3e..d6ea26c2cf86 100644
--- a/libcxx/include/ext/hash_map
+++ b/libcxx/include/ext/hash_map
@@ -979,6 +979,6 @@ operator!=(const hash_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>& __x,
     return !(__x == __y);
 }
 
-} // __gnu_cxx
+} // namespace __gnu_cxx
 
 #endif // _LIBCPP_HASH_MAP
diff --git a/libcxx/include/ext/hash_set b/libcxx/include/ext/hash_set
index b61f5f1da448..7d19ccd006e2 100644
--- a/libcxx/include/ext/hash_set
+++ b/libcxx/include/ext/hash_set
@@ -205,6 +205,10 @@ template <class Value, class Hash, class Pred, class Alloc>
 #endif
 #endif
 
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#pragma GCC system_header
+#endif
+
 namespace __gnu_cxx {
 
 
@@ -654,6 +658,6 @@ operator!=(const hash_multiset<_Value, _Hash, _Pred, _Alloc>& __x,
     return !(__x == __y);
 }
 
-} // __gnu_cxx
+} // namespace __gnu_cxx
 
 #endif // _LIBCPP_HASH_SET
diff --git a/libcxx/include/filesystem b/libcxx/include/filesystem
index 39e8ca2e814b..9f5b42747b34 100644
--- a/libcxx/include/filesystem
+++ b/libcxx/include/filesystem
@@ -238,31 +238,26 @@ inline constexpr bool std::ranges::enable_view<std::filesystem::recursive_direct
 
 */
 
-#include <__availability>
 #include <__config>
-#include <__debug>
-#include <__ranges/enable_borrowed_range.h>
-#include <__ranges/enable_view.h>
-#include <__utility/forward.h>
-#include <chrono>
+#include <__filesystem/copy_options.h>
+#include <__filesystem/directory_entry.h>
+#include <__filesystem/directory_iterator.h>
+#include <__filesystem/directory_options.h>
+#include <__filesystem/file_status.h>
+#include <__filesystem/file_time_type.h>
+#include <__filesystem/file_type.h>
+#include <__filesystem/filesystem_error.h>
+#include <__filesystem/operations.h>
+#include <__filesystem/path_iterator.h>
+#include <__filesystem/path.h>
+#include <__filesystem/perm_options.h>
+#include <__filesystem/perms.h>
+#include <__filesystem/recursive_directory_iterator.h>
+#include <__filesystem/space_info.h>
+#include <__filesystem/u8path.h>
 #include <compare>
-#include <cstddef>
-#include <cstdlib>
-#include <iosfwd>
-#include <iterator>
-#include <memory>
-#include <stack>
-#include <string>
-#include <string_view>
-#include <system_error>
-#include <utility>
 #include <version>
 
-#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
-# include <locale>
-# include <iomanip> // for quoted
-#endif
-
 #if defined(_LIBCPP_HAS_NO_FILESYSTEM_LIBRARY)
 # error "The Filesystem library is not supported since libc++ has been configured with LIBCXX_ENABLE_FILESYSTEM disabled"
 #endif
@@ -271,2775 +266,4 @@ inline constexpr bool std::ranges::enable_view<std::filesystem::recursive_direct
 #pragma GCC system_header
 #endif
 
-_LIBCPP_PUSH_MACROS
-#include <__undef_macros>
-
-#ifndef _LIBCPP_CXX03_LANG
-
-_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
-
-_LIBCPP_AVAILABILITY_FILESYSTEM_PUSH
-
-typedef chrono::time_point<_FilesystemClock> file_time_type;
-
-struct _LIBCPP_TYPE_VIS space_info {
-  uintmax_t capacity;
-  uintmax_t free;
-  uintmax_t available;
-};
-
-// On Windows, the library never identifies files as block, character, fifo
-// or socket.
-enum class _LIBCPP_ENUM_VIS file_type : signed char {
-  none = 0,
-  not_found = -1,
-  regular = 1,
-  directory = 2,
-  symlink = 3,
-  block = 4,
-  character = 5,
-  fifo = 6,
-  socket = 7,
-  unknown = 8
-};
-
-// On Windows, these permission bits map to one single readonly flag per
-// file, and the executable bit is always returned as set. When setting
-// permissions, as long as the write bit is set for either owner, group or
-// others, the readonly flag is cleared.
-enum class _LIBCPP_ENUM_VIS perms : unsigned {
-  none = 0,
-
-  owner_read = 0400,
-  owner_write = 0200,
-  owner_exec = 0100,
-  owner_all = 0700,
-
-  group_read = 040,
-  group_write = 020,
-  group_exec = 010,
-  group_all = 070,
-
-  others_read = 04,
-  others_write = 02,
-  others_exec = 01,
-  others_all = 07,
-
-  all = 0777,
-
-  set_uid = 04000,
-  set_gid = 02000,
-  sticky_bit = 01000,
-  mask = 07777,
-  unknown = 0xFFFF,
-};
-
-_LIBCPP_INLINE_VISIBILITY
-inline constexpr perms operator&(perms _LHS, perms _RHS) {
-  return static_cast<perms>(static_cast<unsigned>(_LHS) &
-                            static_cast<unsigned>(_RHS));
-}
-
-_LIBCPP_INLINE_VISIBILITY
-inline constexpr perms operator|(perms _LHS, perms _RHS) {
-  return static_cast<perms>(static_cast<unsigned>(_LHS) |
-                            static_cast<unsigned>(_RHS));
-}
-
-_LIBCPP_INLINE_VISIBILITY
-inline constexpr perms operator^(perms _LHS, perms _RHS) {
-  return static_cast<perms>(static_cast<unsigned>(_LHS) ^
-                            static_cast<unsigned>(_RHS));
-}
-
-_LIBCPP_INLINE_VISIBILITY
-inline constexpr perms operator~(perms _LHS) {
-  return static_cast<perms>(~static_cast<unsigned>(_LHS));
-}
-
-_LIBCPP_INLINE_VISIBILITY
-inline perms& operator&=(perms& _LHS, perms _RHS) { return _LHS = _LHS & _RHS; }
-
-_LIBCPP_INLINE_VISIBILITY
-inline perms& operator|=(perms& _LHS, perms _RHS) { return _LHS = _LHS | _RHS; }
-
-_LIBCPP_INLINE_VISIBILITY
-inline perms& operator^=(perms& _LHS, perms _RHS) { return _LHS = _LHS ^ _RHS; }
-
-enum class _LIBCPP_ENUM_VIS perm_options : unsigned char {
-  replace = 1,
-  add = 2,
-  remove = 4,
-  nofollow = 8
-};
-
-_LIBCPP_INLINE_VISIBILITY
-inline constexpr perm_options operator&(perm_options _LHS, perm_options _RHS) {
-  return static_cast<perm_options>(static_cast<unsigned>(_LHS) &
-                                   static_cast<unsigned>(_RHS));
-}
-
-_LIBCPP_INLINE_VISIBILITY
-inline constexpr perm_options operator|(perm_options _LHS, perm_options _RHS) {
-  return static_cast<perm_options>(static_cast<unsigned>(_LHS) |
-                                   static_cast<unsigned>(_RHS));
-}
-
-_LIBCPP_INLINE_VISIBILITY
-inline constexpr perm_options operator^(perm_options _LHS, perm_options _RHS) {
-  return static_cast<perm_options>(static_cast<unsigned>(_LHS) ^
-                                   static_cast<unsigned>(_RHS));
-}
-
-_LIBCPP_INLINE_VISIBILITY
-inline constexpr perm_options operator~(perm_options _LHS) {
-  return static_cast<perm_options>(~static_cast<unsigned>(_LHS));
-}
-
-_LIBCPP_INLINE_VISIBILITY
-inline perm_options& operator&=(perm_options& _LHS, perm_options _RHS) {
-  return _LHS = _LHS & _RHS;
-}
-
-_LIBCPP_INLINE_VISIBILITY
-inline perm_options& operator|=(perm_options& _LHS, perm_options _RHS) {
-  return _LHS = _LHS | _RHS;
-}
-
-_LIBCPP_INLINE_VISIBILITY
-inline perm_options& operator^=(perm_options& _LHS, perm_options _RHS) {
-  return _LHS = _LHS ^ _RHS;
-}
-
-enum class _LIBCPP_ENUM_VIS copy_options : unsigned short {
-  none = 0,
-  skip_existing = 1,
-  overwrite_existing = 2,
-  update_existing = 4,
-  recursive = 8,
-  copy_symlinks = 16,
-  skip_symlinks = 32,
-  directories_only = 64,
-  create_symlinks = 128,
-  create_hard_links = 256,
-  __in_recursive_copy = 512,
-};
-
-_LIBCPP_INLINE_VISIBILITY
-inline constexpr copy_options operator&(copy_options _LHS, copy_options _RHS) {
-  return static_cast<copy_options>(static_cast<unsigned short>(_LHS) &
-                                   static_cast<unsigned short>(_RHS));
-}
-
-_LIBCPP_INLINE_VISIBILITY
-inline constexpr copy_options operator|(copy_options _LHS, copy_options _RHS) {
-  return static_cast<copy_options>(static_cast<unsigned short>(_LHS) |
-                                   static_cast<unsigned short>(_RHS));
-}
-
-_LIBCPP_INLINE_VISIBILITY
-inline constexpr copy_options operator^(copy_options _LHS, copy_options _RHS) {
-  return static_cast<copy_options>(static_cast<unsigned short>(_LHS) ^
-                                   static_cast<unsigned short>(_RHS));
-}
-
-_LIBCPP_INLINE_VISIBILITY
-inline constexpr copy_options operator~(copy_options _LHS) {
-  return static_cast<copy_options>(~static_cast<unsigned short>(_LHS));
-}
-
-_LIBCPP_INLINE_VISIBILITY
-inline copy_options& operator&=(copy_options& _LHS, copy_options _RHS) {
-  return _LHS = _LHS & _RHS;
-}
-
-_LIBCPP_INLINE_VISIBILITY
-inline copy_options& operator|=(copy_options& _LHS, copy_options _RHS) {
-  return _LHS = _LHS | _RHS;
-}
-
-_LIBCPP_INLINE_VISIBILITY
-inline copy_options& operator^=(copy_options& _LHS, copy_options _RHS) {
-  return _LHS = _LHS ^ _RHS;
-}
-
-enum class _LIBCPP_ENUM_VIS directory_options : unsigned char {
-  none = 0,
-  follow_directory_symlink = 1,
-  skip_permission_denied = 2
-};
-
-_LIBCPP_INLINE_VISIBILITY
-inline constexpr directory_options operator&(directory_options _LHS,
-                                             directory_options _RHS) {
-  return static_cast<directory_options>(static_cast<unsigned char>(_LHS) &
-                                        static_cast<unsigned char>(_RHS));
-}
-
-_LIBCPP_INLINE_VISIBILITY
-inline constexpr directory_options operator|(directory_options _LHS,
-                                             directory_options _RHS) {
-  return static_cast<directory_options>(static_cast<unsigned char>(_LHS) |
-                                        static_cast<unsigned char>(_RHS));
-}
-
-_LIBCPP_INLINE_VISIBILITY
-inline constexpr directory_options operator^(directory_options _LHS,
-                                             directory_options _RHS) {
-  return static_cast<directory_options>(static_cast<unsigned char>(_LHS) ^
-                                        static_cast<unsigned char>(_RHS));
-}
-
-_LIBCPP_INLINE_VISIBILITY
-inline constexpr directory_options operator~(directory_options _LHS) {
-  return static_cast<directory_options>(~static_cast<unsigned char>(_LHS));
-}
-
-_LIBCPP_INLINE_VISIBILITY
-inline directory_options& operator&=(directory_options& _LHS,
-                                     directory_options _RHS) {
-  return _LHS = _LHS & _RHS;
-}
-
-_LIBCPP_INLINE_VISIBILITY
-inline directory_options& operator|=(directory_options& _LHS,
-                                     directory_options _RHS) {
-  return _LHS = _LHS | _RHS;
-}
-
-_LIBCPP_INLINE_VISIBILITY
-inline directory_options& operator^=(directory_options& _LHS,
-                                     directory_options _RHS) {
-  return _LHS = _LHS ^ _RHS;
-}
-
-class _LIBCPP_TYPE_VIS file_status {
-public:
-  // constructors
-  _LIBCPP_INLINE_VISIBILITY
-  file_status() noexcept : file_status(file_type::none) {}
-  _LIBCPP_INLINE_VISIBILITY
-  explicit file_status(file_type __ft, perms __prms = perms::unknown) noexcept
-      : __ft_(__ft),
-        __prms_(__prms) {}
-
-  file_status(const file_status&) noexcept = default;
-  file_status(file_status&&) noexcept = default;
-
-  _LIBCPP_INLINE_VISIBILITY
-  ~file_status() {}
-
-  file_status& operator=(const file_status&) noexcept = default;
-  file_status& operator=(file_status&&) noexcept = default;
-
-  // observers
-  _LIBCPP_INLINE_VISIBILITY
-  file_type type() const noexcept { return __ft_; }
-
-  _LIBCPP_INLINE_VISIBILITY
-  perms permissions() const noexcept { return __prms_; }
-
-  // modifiers
-  _LIBCPP_INLINE_VISIBILITY
-  void type(file_type __ft) noexcept { __ft_ = __ft; }
-
-  _LIBCPP_INLINE_VISIBILITY
-  void permissions(perms __p) noexcept { __prms_ = __p; }
-
-private:
-  file_type __ft_;
-  perms __prms_;
-};
-
-class _LIBCPP_TYPE_VIS directory_entry;
-
-template <class _Tp>
-struct __can_convert_char {
-  static const bool value = false;
-};
-template <class _Tp>
-struct __can_convert_char<const _Tp> : public __can_convert_char<_Tp> {};
-template <>
-struct __can_convert_char<char> {
-  static const bool value = true;
-  using __char_type = char;
-};
-template <>
-struct __can_convert_char<wchar_t> {
-  static const bool value = true;
-  using __char_type = wchar_t;
-};
-#ifndef _LIBCPP_HAS_NO_CHAR8_T
-template <>
-struct __can_convert_char<char8_t> {
-  static const bool value = true;
-  using __char_type = char8_t;
-};
-#endif
-template <>
-struct __can_convert_char<char16_t> {
-  static const bool value = true;
-  using __char_type = char16_t;
-};
-template <>
-struct __can_convert_char<char32_t> {
-  static const bool value = true;
-  using __char_type = char32_t;
-};
-
-template <class _ECharT>
-typename enable_if<__can_convert_char<_ECharT>::value, bool>::type
-__is_separator(_ECharT __e) {
-#if defined(_LIBCPP_WIN32API)
-  return __e == _ECharT('/') || __e == _ECharT('\\');
-#else
-  return __e == _ECharT('/');
-#endif
-}
-
-#ifndef _LIBCPP_HAS_NO_CHAR8_T
-typedef u8string __u8_string;
-#else
-typedef string __u8_string;
-#endif
-
-struct _NullSentinel {};
-
-template <class _Tp>
-using _Void = void;
-
-template <class _Tp, class = void>
-struct __is_pathable_string : public false_type {};
-
-template <class _ECharT, class _Traits, class _Alloc>
-struct __is_pathable_string<
-    basic_string<_ECharT, _Traits, _Alloc>,
-    _Void<typename __can_convert_char<_ECharT>::__char_type> >
-    : public __can_convert_char<_ECharT> {
-  using _Str = basic_string<_ECharT, _Traits, _Alloc>;
-  using _Base = __can_convert_char<_ECharT>;
-  static _ECharT const* __range_begin(_Str const& __s) { return __s.data(); }
-  static _ECharT const* __range_end(_Str const& __s) {
-    return __s.data() + __s.length();
-  }
-  static _ECharT __first_or_null(_Str const& __s) {
-    return __s.empty() ? _ECharT{} : __s[0];
-  }
-};
-
-template <class _ECharT, class _Traits>
-struct __is_pathable_string<
-    basic_string_view<_ECharT, _Traits>,
-    _Void<typename __can_convert_char<_ECharT>::__char_type> >
-    : public __can_convert_char<_ECharT> {
-  using _Str = basic_string_view<_ECharT, _Traits>;
-  using _Base = __can_convert_char<_ECharT>;
-  static _ECharT const* __range_begin(_Str const& __s) { return __s.data(); }
-  static _ECharT const* __range_end(_Str const& __s) {
-    return __s.data() + __s.length();
-  }
-  static _ECharT __first_or_null(_Str const& __s) {
-    return __s.empty() ? _ECharT{} : __s[0];
-  }
-};
-
-template <class _Source, class _DS = typename decay<_Source>::type,
-          class _UnqualPtrType =
-              typename remove_const<typename remove_pointer<_DS>::type>::type,
-          bool _IsCharPtr = is_pointer<_DS>::value&&
-              __can_convert_char<_UnqualPtrType>::value>
-struct __is_pathable_char_array : false_type {};
-
-template <class _Source, class _ECharT, class _UPtr>
-struct __is_pathable_char_array<_Source, _ECharT*, _UPtr, true>
-    : __can_convert_char<typename remove_const<_ECharT>::type> {
-  using _Base = __can_convert_char<typename remove_const<_ECharT>::type>;
-
-  static _ECharT const* __range_begin(const _ECharT* __b) { return __b; }
-  static _ECharT const* __range_end(const _ECharT* __b) {
-    using _Iter = const _ECharT*;
-    const _ECharT __sentinel = _ECharT{};
-    _Iter __e = __b;
-    for (; *__e != __sentinel; ++__e)
-      ;
-    return __e;
-  }
-
-  static _ECharT __first_or_null(const _ECharT* __b) { return *__b; }
-};
-
-template <class _Iter, bool _IsIt = __is_cpp17_input_iterator<_Iter>::value,
-          class = void>
-struct __is_pathable_iter : false_type {};
-
-template <class _Iter>
-struct __is_pathable_iter<
-    _Iter, true,
-    _Void<typename __can_convert_char<
-        typename iterator_traits<_Iter>::value_type>::__char_type> >
-    : __can_convert_char<typename iterator_traits<_Iter>::value_type> {
-  using _ECharT = typename iterator_traits<_Iter>::value_type;
-  using _Base = __can_convert_char<_ECharT>;
-
-  static _Iter __range_begin(_Iter __b) { return __b; }
-  static _NullSentinel __range_end(_Iter) { return _NullSentinel{}; }
-
-  static _ECharT __first_or_null(_Iter __b) { return *__b; }
-};
-
-template <class _Tp, bool _IsStringT = __is_pathable_string<_Tp>::value,
-          bool _IsCharIterT = __is_pathable_char_array<_Tp>::value,
-          bool _IsIterT = !_IsCharIterT && __is_pathable_iter<_Tp>::value>
-struct __is_pathable : false_type {
-  static_assert(!_IsStringT && !_IsCharIterT && !_IsIterT, "Must all be false");
-};
-
-template <class _Tp>
-struct __is_pathable<_Tp, true, false, false> : __is_pathable_string<_Tp> {};
-
-template <class _Tp>
-struct __is_pathable<_Tp, false, true, false> : __is_pathable_char_array<_Tp> {
-};
-
-template <class _Tp>
-struct __is_pathable<_Tp, false, false, true> : __is_pathable_iter<_Tp> {};
-
-#if defined(_LIBCPP_WIN32API)
-typedef wstring __path_string;
-typedef wchar_t __path_value;
-#else
-typedef string __path_string;
-typedef char __path_value;
-#endif
-
-#if defined(_LIBCPP_WIN32API)
-_LIBCPP_FUNC_VIS
-size_t __wide_to_char(const wstring&, char*, size_t);
-_LIBCPP_FUNC_VIS
-size_t __char_to_wide(const string&, wchar_t*, size_t);
-#endif
-
-template <class _ECharT>
-struct _PathCVT;
-
-#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
-template <class _ECharT>
-struct _PathCVT {
-  static_assert(__can_convert_char<_ECharT>::value,
-                "Char type not convertible");
-
-  typedef __narrow_to_utf8<sizeof(_ECharT) * __CHAR_BIT__> _Narrower;
-#if defined(_LIBCPP_WIN32API)
-  typedef __widen_from_utf8<sizeof(wchar_t) * __CHAR_BIT__> _Widener;
-#endif
-
-  static void __append_range(__path_string& __dest, _ECharT const* __b,
-                             _ECharT const* __e) {
-#if defined(_LIBCPP_WIN32API)
-    string __utf8;
-    _Narrower()(back_inserter(__utf8), __b, __e);
-    _Widener()(back_inserter(__dest), __utf8.data(), __utf8.data() + __utf8.size());
-#else
-    _Narrower()(back_inserter(__dest), __b, __e);
-#endif
-  }
-
-  template <class _Iter>
-  static void __append_range(__path_string& __dest, _Iter __b, _Iter __e) {
-    static_assert(!is_same<_Iter, _ECharT*>::value, "Call const overload");
-    if (__b == __e)
-      return;
-    basic_string<_ECharT> __tmp(__b, __e);
-#if defined(_LIBCPP_WIN32API)
-    string __utf8;
-    _Narrower()(back_inserter(__utf8), __tmp.data(),
-                __tmp.data() + __tmp.length());
-    _Widener()(back_inserter(__dest), __utf8.data(), __utf8.data() + __utf8.size());
-#else
-    _Narrower()(back_inserter(__dest), __tmp.data(),
-                __tmp.data() + __tmp.length());
-#endif
-  }
-
-  template <class _Iter>
-  static void __append_range(__path_string& __dest, _Iter __b, _NullSentinel) {
-    static_assert(!is_same<_Iter, _ECharT*>::value, "Call const overload");
-    const _ECharT __sentinel = _ECharT{};
-    if (*__b == __sentinel)
-      return;
-    basic_string<_ECharT> __tmp;
-    for (; *__b != __sentinel; ++__b)
-      __tmp.push_back(*__b);
-#if defined(_LIBCPP_WIN32API)
-    string __utf8;
-    _Narrower()(back_inserter(__utf8), __tmp.data(),
-                __tmp.data() + __tmp.length());
-    _Widener()(back_inserter(__dest), __utf8.data(), __utf8.data() + __utf8.size());
-#else
-    _Narrower()(back_inserter(__dest), __tmp.data(),
-                __tmp.data() + __tmp.length());
-#endif
-  }
-
-  template <class _Source>
-  static void __append_source(__path_string& __dest, _Source const& __s) {
-    using _Traits = __is_pathable<_Source>;
-    __append_range(__dest, _Traits::__range_begin(__s),
-                   _Traits::__range_end(__s));
-  }
-};
-#endif // !_LIBCPP_HAS_NO_LOCALIZATION
-
-template <>
-struct _PathCVT<__path_value> {
-
-  template <class _Iter>
-  static typename enable_if<__is_exactly_cpp17_input_iterator<_Iter>::value>::type
-  __append_range(__path_string& __dest, _Iter __b, _Iter __e) {
-    for (; __b != __e; ++__b)
-      __dest.push_back(*__b);
-  }
-
-  template <class _Iter>
-  static typename enable_if<__is_cpp17_forward_iterator<_Iter>::value>::type
-  __append_range(__path_string& __dest, _Iter __b, _Iter __e) {
-    __dest.append(__b, __e);
-  }
-
-  template <class _Iter>
-  static void __append_range(__path_string& __dest, _Iter __b, _NullSentinel) {
-    const char __sentinel = char{};
-    for (; *__b != __sentinel; ++__b)
-      __dest.push_back(*__b);
-  }
-
-  template <class _Source>
-  static void __append_source(__path_string& __dest, _Source const& __s) {
-    using _Traits = __is_pathable<_Source>;
-    __append_range(__dest, _Traits::__range_begin(__s),
-                   _Traits::__range_end(__s));
-  }
-};
-
-#if defined(_LIBCPP_WIN32API)
-template <>
-struct _PathCVT<char> {
-
-  static void
-  __append_string(__path_string& __dest, const basic_string<char> &__str) {
-      size_t __size = __char_to_wide(__str, nullptr, 0);
-      size_t __pos = __dest.size();
-      __dest.resize(__pos + __size);
-      __char_to_wide(__str, const_cast<__path_value*>(__dest.data()) + __pos, __size);
-  }
-
-  template <class _Iter>
-  static typename enable_if<__is_exactly_cpp17_input_iterator<_Iter>::value>::type
-  __append_range(__path_string& __dest, _Iter __b, _Iter __e) {
-    basic_string<char> __tmp(__b, __e);
-    __append_string(__dest, __tmp);
-  }
-
-  template <class _Iter>
-  static typename enable_if<__is_cpp17_forward_iterator<_Iter>::value>::type
-  __append_range(__path_string& __dest, _Iter __b, _Iter __e) {
-    basic_string<char> __tmp(__b, __e);
-    __append_string(__dest, __tmp);
-  }
-
-  template <class _Iter>
-  static void __append_range(__path_string& __dest, _Iter __b, _NullSentinel) {
-    const char __sentinel = char{};
-    basic_string<char> __tmp;
-    for (; *__b != __sentinel; ++__b)
-      __tmp.push_back(*__b);
-    __append_string(__dest, __tmp);
-  }
-
-  template <class _Source>
-  static void __append_source(__path_string& __dest, _Source const& __s) {
-    using _Traits = __is_pathable<_Source>;
-    __append_range(__dest, _Traits::__range_begin(__s),
-                   _Traits::__range_end(__s));
-  }
-};
-
-template <class _ECharT>
-struct _PathExport {
-  typedef __narrow_to_utf8<sizeof(wchar_t) * __CHAR_BIT__> _Narrower;
-  typedef __widen_from_utf8<sizeof(_ECharT) * __CHAR_BIT__> _Widener;
-
-  template <class _Str>
-  static void __append(_Str& __dest, const __path_string& __src) {
-    string __utf8;
-    _Narrower()(back_inserter(__utf8), __src.data(), __src.data() + __src.size());
-    _Widener()(back_inserter(__dest), __utf8.data(), __utf8.data() + __utf8.size());
-  }
-};
-
-template <>
-struct _PathExport<char> {
-  template <class _Str>
-  static void __append(_Str& __dest, const __path_string& __src) {
-    size_t __size = __wide_to_char(__src, nullptr, 0);
-    size_t __pos = __dest.size();
-    __dest.resize(__size);
-    __wide_to_char(__src, const_cast<char*>(__dest.data()) + __pos, __size);
-  }
-};
-
-template <>
-struct _PathExport<wchar_t> {
-  template <class _Str>
-  static void __append(_Str& __dest, const __path_string& __src) {
-    __dest.append(__src.begin(), __src.end());
-  }
-};
-
-template <>
-struct _PathExport<char16_t> {
-  template <class _Str>
-  static void __append(_Str& __dest, const __path_string& __src) {
-    __dest.append(__src.begin(), __src.end());
-  }
-};
-
-#ifndef _LIBCPP_HAS_NO_CHAR8_T
-template <>
-struct _PathExport<char8_t> {
-  typedef __narrow_to_utf8<sizeof(wchar_t) * __CHAR_BIT__> _Narrower;
-
-  template <class _Str>
-  static void __append(_Str& __dest, const __path_string& __src) {
-    _Narrower()(back_inserter(__dest), __src.data(), __src.data() + __src.size());
-  }
-};
-#endif /* !_LIBCPP_HAS_NO_CHAR8_T */
-#endif /* _LIBCPP_WIN32API */
-
-class _LIBCPP_TYPE_VIS path {
-  template <class _SourceOrIter, class _Tp = path&>
-  using _EnableIfPathable =
-      typename enable_if<__is_pathable<_SourceOrIter>::value, _Tp>::type;
-
-  template <class _Tp>
-  using _SourceChar = typename __is_pathable<_Tp>::__char_type;
-
-  template <class _Tp>
-  using _SourceCVT = _PathCVT<_SourceChar<_Tp> >;
-
-public:
-#if defined(_LIBCPP_WIN32API)
-  typedef wchar_t value_type;
-  static constexpr value_type preferred_separator = L'\\';
-#else
-  typedef char value_type;
-  static constexpr value_type preferred_separator = '/';
-#endif
-  typedef basic_string<value_type> string_type;
-  typedef basic_string_view<value_type> __string_view;
-
-  enum _LIBCPP_ENUM_VIS format : unsigned char {
-    auto_format,
-    native_format,
-    generic_format
-  };
-
-  // constructors and destructor
-  _LIBCPP_INLINE_VISIBILITY path() noexcept {}
-  _LIBCPP_INLINE_VISIBILITY path(const path& __p) : __pn_(__p.__pn_) {}
-  _LIBCPP_INLINE_VISIBILITY path(path&& __p) noexcept
-      : __pn_(_VSTD::move(__p.__pn_)) {}
-
-  _LIBCPP_INLINE_VISIBILITY
-  path(string_type&& __s, format = format::auto_format) noexcept
-      : __pn_(_VSTD::move(__s)) {}
-
-  template <class _Source, class = _EnableIfPathable<_Source, void> >
-  path(const _Source& __src, format = format::auto_format) {
-    _SourceCVT<_Source>::__append_source(__pn_, __src);
-  }
-
-  template <class _InputIt>
-  path(_InputIt __first, _InputIt __last, format = format::auto_format) {
-    typedef typename iterator_traits<_InputIt>::value_type _ItVal;
-    _PathCVT<_ItVal>::__append_range(__pn_, __first, __last);
-  }
-
-/*
-#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
-  // TODO Implement locale conversions.
-  template <class _Source, class = _EnableIfPathable<_Source, void> >
-  path(const _Source& __src, const locale& __loc, format = format::auto_format);
-  template <class _InputIt>
-  path(_InputIt __first, _InputIt _last, const locale& __loc,
-       format = format::auto_format);
-#endif
-*/
-
-  _LIBCPP_INLINE_VISIBILITY
-  ~path() = default;
-
-  // assignments
-  _LIBCPP_INLINE_VISIBILITY
-  path& operator=(const path& __p) {
-    __pn_ = __p.__pn_;
-    return *this;
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  path& operator=(path&& __p) noexcept {
-    __pn_ = _VSTD::move(__p.__pn_);
-    return *this;
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  path& operator=(string_type&& __s) noexcept {
-    __pn_ = _VSTD::move(__s);
-    return *this;
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  path& assign(string_type&& __s) noexcept {
-    __pn_ = _VSTD::move(__s);
-    return *this;
-  }
-
-  template <class _Source>
-  _LIBCPP_INLINE_VISIBILITY _EnableIfPathable<_Source>
-  operator=(const _Source& __src) {
-    return this->assign(__src);
-  }
-
-  template <class _Source>
-  _EnableIfPathable<_Source> assign(const _Source& __src) {
-    __pn_.clear();
-    _SourceCVT<_Source>::__append_source(__pn_, __src);
-    return *this;
-  }
-
-  template <class _InputIt>
-  path& assign(_InputIt __first, _InputIt __last) {
-    typedef typename iterator_traits<_InputIt>::value_type _ItVal;
-    __pn_.clear();
-    _PathCVT<_ItVal>::__append_range(__pn_, __first, __last);
-    return *this;
-  }
-
-public:
-  // appends
-#if defined(_LIBCPP_WIN32API)
-  path& operator/=(const path& __p) {
-    auto __p_root_name = __p.__root_name();
-    auto __p_root_name_size = __p_root_name.size();
-    if (__p.is_absolute() ||
-        (!__p_root_name.empty() && __p_root_name != __string_view(root_name().__pn_))) {
-      __pn_ = __p.__pn_;
-      return *this;
-    }
-    if (__p.has_root_directory()) {
-      path __root_name_str = root_name();
-      __pn_ = __root_name_str.native();
-      __pn_ += __string_view(__p.__pn_).substr(__p_root_name_size);
-      return *this;
-    }
-    if (has_filename() || (!has_root_directory() && is_absolute()))
-      __pn_ += preferred_separator;
-    __pn_ += __string_view(__p.__pn_).substr(__p_root_name_size);
-    return *this;
-  }
-  template <class _Source>
-  _LIBCPP_INLINE_VISIBILITY _EnableIfPathable<_Source>
-  operator/=(const _Source& __src) {
-    return operator/=(path(__src));
-  }
-
-  template <class _Source>
-  _EnableIfPathable<_Source> append(const _Source& __src) {
-    return operator/=(path(__src));
-  }
-
-  template <class _InputIt>
-  path& append(_InputIt __first, _InputIt __last) {
-    return operator/=(path(__first, __last));
-  }
-#else
-  path& operator/=(const path& __p) {
-    if (__p.is_absolute()) {
-      __pn_ = __p.__pn_;
-      return *this;
-    }
-    if (has_filename())
-      __pn_ += preferred_separator;
-    __pn_ += __p.native();
-    return *this;
-  }
-
-  // FIXME: Use _LIBCPP_DIAGNOSE_WARNING to produce a diagnostic when __src
-  // is known at compile time to be "/' since the user almost certainly intended
-  // to append a separator instead of overwriting the path with "/"
-  template <class _Source>
-  _LIBCPP_INLINE_VISIBILITY _EnableIfPathable<_Source>
-  operator/=(const _Source& __src) {
-    return this->append(__src);
-  }
-
-  template <class _Source>
-  _EnableIfPathable<_Source> append(const _Source& __src) {
-    using _Traits = __is_pathable<_Source>;
-    using _CVT = _PathCVT<_SourceChar<_Source> >;
-    bool __source_is_absolute = __is_separator(_Traits::__first_or_null(__src));
-    if (__source_is_absolute)
-      __pn_.clear();
-    else if (has_filename())
-      __pn_ += preferred_separator;
-    _CVT::__append_source(__pn_, __src);
-    return *this;
-  }
-
-  template <class _InputIt>
-  path& append(_InputIt __first, _InputIt __last) {
-    typedef typename iterator_traits<_InputIt>::value_type _ItVal;
-    static_assert(__can_convert_char<_ItVal>::value, "Must convertible");
-    using _CVT = _PathCVT<_ItVal>;
-    if (__first != __last && __is_separator(*__first))
-      __pn_.clear();
-    else if (has_filename())
-      __pn_ += preferred_separator;
-    _CVT::__append_range(__pn_, __first, __last);
-    return *this;
-  }
-#endif
-
-  // concatenation
-  _LIBCPP_INLINE_VISIBILITY
-  path& operator+=(const path& __x) {
-    __pn_ += __x.__pn_;
-    return *this;
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  path& operator+=(const string_type& __x) {
-    __pn_ += __x;
-    return *this;
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  path& operator+=(__string_view __x) {
-    __pn_ += __x;
-    return *this;
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  path& operator+=(const value_type* __x) {
-    __pn_ += __x;
-    return *this;
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  path& operator+=(value_type __x) {
-    __pn_ += __x;
-    return *this;
-  }
-
-  template <class _ECharT>
-  typename enable_if<__can_convert_char<_ECharT>::value, path&>::type
-  operator+=(_ECharT __x) {
-    _PathCVT<_ECharT>::__append_source(__pn_,
-                                       basic_string_view<_ECharT>(&__x, 1));
-    return *this;
-  }
-
-  template <class _Source>
-  _EnableIfPathable<_Source> operator+=(const _Source& __x) {
-    return this->concat(__x);
-  }
-
-  template <class _Source>
-  _EnableIfPathable<_Source> concat(const _Source& __x) {
-    _SourceCVT<_Source>::__append_source(__pn_, __x);
-    return *this;
-  }
-
-  template <class _InputIt>
-  path& concat(_InputIt __first, _InputIt __last) {
-    typedef typename iterator_traits<_InputIt>::value_type _ItVal;
-    _PathCVT<_ItVal>::__append_range(__pn_, __first, __last);
-    return *this;
-  }
-
-  // modifiers
-  _LIBCPP_INLINE_VISIBILITY
-  void clear() noexcept { __pn_.clear(); }
-
-  path& make_preferred() {
-#if defined(_LIBCPP_WIN32API)
-    _VSTD::replace(__pn_.begin(), __pn_.end(), L'/', L'\\');
-#endif
-    return *this;
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  path& remove_filename() {
-    auto __fname = __filename();
-    if (!__fname.empty())
-      __pn_.erase(__fname.data() - __pn_.data());
-    return *this;
-  }
-
-  path& replace_filename(const path& __replacement) {
-    remove_filename();
-    return (*this /= __replacement);
-  }
-
-  path& replace_extension(const path& __replacement = path());
-
-  _LIBCPP_INLINE_VISIBILITY
-  void swap(path& __rhs) noexcept { __pn_.swap(__rhs.__pn_); }
-
-  // private helper to allow reserving memory in the path
-  _LIBCPP_INLINE_VISIBILITY
-  void __reserve(size_t __s) { __pn_.reserve(__s); }
-
-  // native format observers
-  _LIBCPP_INLINE_VISIBILITY
-  const string_type& native() const noexcept { return __pn_; }
-
-  _LIBCPP_INLINE_VISIBILITY
-  const value_type* c_str() const noexcept { return __pn_.c_str(); }
-
-  _LIBCPP_INLINE_VISIBILITY operator string_type() const { return __pn_; }
-
-#if defined(_LIBCPP_WIN32API)
-  _LIBCPP_INLINE_VISIBILITY _VSTD::wstring wstring() const { return __pn_; }
-
-  _VSTD::wstring generic_wstring() const {
-    _VSTD::wstring __s;
-    __s.resize(__pn_.size());
-    _VSTD::replace_copy(__pn_.begin(), __pn_.end(), __s.begin(), '\\', '/');
-    return __s;
-  }
-
-#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
-  template <class _ECharT, class _Traits = char_traits<_ECharT>,
-            class _Allocator = allocator<_ECharT> >
-  basic_string<_ECharT, _Traits, _Allocator>
-  string(const _Allocator& __a = _Allocator()) const {
-    using _Str = basic_string<_ECharT, _Traits, _Allocator>;
-    _Str __s(__a);
-    __s.reserve(__pn_.size());
-    _PathExport<_ECharT>::__append(__s, __pn_);
-    return __s;
-  }
-
-  _LIBCPP_INLINE_VISIBILITY _VSTD::string string() const {
-    return string<char>();
-  }
-  _LIBCPP_INLINE_VISIBILITY __u8_string u8string() const {
-    using _CVT = __narrow_to_utf8<sizeof(wchar_t) * __CHAR_BIT__>;
-    __u8_string __s;
-    __s.reserve(__pn_.size());
-    _CVT()(back_inserter(__s), __pn_.data(), __pn_.data() + __pn_.size());
-    return __s;
-  }
-
-  _LIBCPP_INLINE_VISIBILITY _VSTD::u16string u16string() const {
-    return string<char16_t>();
-  }
-  _LIBCPP_INLINE_VISIBILITY _VSTD::u32string u32string() const {
-    return string<char32_t>();
-  }
-
-  // generic format observers
-  template <class _ECharT, class _Traits = char_traits<_ECharT>,
-            class _Allocator = allocator<_ECharT> >
-  basic_string<_ECharT, _Traits, _Allocator>
-  generic_string(const _Allocator& __a = _Allocator()) const {
-    using _Str = basic_string<_ECharT, _Traits, _Allocator>;
-    _Str __s = string<_ECharT, _Traits, _Allocator>(__a);
-    // Note: This (and generic_u8string below) is slightly suboptimal as
-    // it iterates twice over the string; once to convert it to the right
-    // character type, and once to replace path delimiters.
-    _VSTD::replace(__s.begin(), __s.end(),
-                   static_cast<_ECharT>('\\'), static_cast<_ECharT>('/'));
-    return __s;
-  }
-
-  _VSTD::string generic_string() const { return generic_string<char>(); }
-  _VSTD::u16string generic_u16string() const { return generic_string<char16_t>(); }
-  _VSTD::u32string generic_u32string() const { return generic_string<char32_t>(); }
-  __u8_string generic_u8string() const {
-    __u8_string __s = u8string();
-    _VSTD::replace(__s.begin(), __s.end(), '\\', '/');
-    return __s;
-  }
-#endif /* !_LIBCPP_HAS_NO_LOCALIZATION */
-#else /* _LIBCPP_WIN32API */
-
-  _LIBCPP_INLINE_VISIBILITY _VSTD::string string() const { return __pn_; }
-#ifndef _LIBCPP_HAS_NO_CHAR8_T
-  _LIBCPP_INLINE_VISIBILITY _VSTD::u8string u8string() const { return _VSTD::u8string(__pn_.begin(), __pn_.end()); }
-#else
-  _LIBCPP_INLINE_VISIBILITY _VSTD::string u8string() const { return __pn_; }
-#endif
-
-#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
-  template <class _ECharT, class _Traits = char_traits<_ECharT>,
-            class _Allocator = allocator<_ECharT> >
-  basic_string<_ECharT, _Traits, _Allocator>
-  string(const _Allocator& __a = _Allocator()) const {
-    using _CVT = __widen_from_utf8<sizeof(_ECharT) * __CHAR_BIT__>;
-    using _Str = basic_string<_ECharT, _Traits, _Allocator>;
-    _Str __s(__a);
-    __s.reserve(__pn_.size());
-    _CVT()(back_inserter(__s), __pn_.data(), __pn_.data() + __pn_.size());
-    return __s;
-  }
-
-#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
-  _LIBCPP_INLINE_VISIBILITY _VSTD::wstring wstring() const {
-    return string<wchar_t>();
-  }
-#endif
-  _LIBCPP_INLINE_VISIBILITY _VSTD::u16string u16string() const {
-    return string<char16_t>();
-  }
-  _LIBCPP_INLINE_VISIBILITY _VSTD::u32string u32string() const {
-    return string<char32_t>();
-  }
-#endif /* !_LIBCPP_HAS_NO_LOCALIZATION */
-
-  // generic format observers
-  _VSTD::string generic_string() const { return __pn_; }
-#ifndef _LIBCPP_HAS_NO_CHAR8_T
-  _VSTD::u8string generic_u8string() const { return _VSTD::u8string(__pn_.begin(), __pn_.end()); }
-#else
-  _VSTD::string generic_u8string() const { return __pn_; }
-#endif
-
-#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
-  template <class _ECharT, class _Traits = char_traits<_ECharT>,
-            class _Allocator = allocator<_ECharT> >
-  basic_string<_ECharT, _Traits, _Allocator>
-  generic_string(const _Allocator& __a = _Allocator()) const {
-    return string<_ECharT, _Traits, _Allocator>(__a);
-  }
-
-#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
-  _VSTD::wstring generic_wstring() const { return string<wchar_t>(); }
-#endif
-  _VSTD::u16string generic_u16string() const { return string<char16_t>(); }
-  _VSTD::u32string generic_u32string() const { return string<char32_t>(); }
-#endif /* !_LIBCPP_HAS_NO_LOCALIZATION */
-#endif /* !_LIBCPP_WIN32API */
-
-private:
-  int __compare(__string_view) const;
-  __string_view __root_name() const;
-  __string_view __root_directory() const;
-  __string_view __root_path_raw() const;
-  __string_view __relative_path() const;
-  __string_view __parent_path() const;
-  __string_view __filename() const;
-  __string_view __stem() const;
-  __string_view __extension() const;
-
-public:
-  // compare
-  _LIBCPP_INLINE_VISIBILITY int compare(const path& __p) const noexcept {
-    return __compare(__p.__pn_);
-  }
-  _LIBCPP_INLINE_VISIBILITY int compare(const string_type& __s) const {
-    return __compare(__s);
-  }
-  _LIBCPP_INLINE_VISIBILITY int compare(__string_view __s) const {
-    return __compare(__s);
-  }
-  _LIBCPP_INLINE_VISIBILITY int compare(const value_type* __s) const {
-    return __compare(__s);
-  }
-
-  // decomposition
-  _LIBCPP_INLINE_VISIBILITY path root_name() const {
-    return string_type(__root_name());
-  }
-  _LIBCPP_INLINE_VISIBILITY path root_directory() const {
-    return string_type(__root_directory());
-  }
-  _LIBCPP_INLINE_VISIBILITY path root_path() const {
-#if defined(_LIBCPP_WIN32API)
-    return string_type(__root_path_raw());
-#else
-    return root_name().append(string_type(__root_directory()));
-#endif
-  }
-  _LIBCPP_INLINE_VISIBILITY path relative_path() const {
-    return string_type(__relative_path());
-  }
-  _LIBCPP_INLINE_VISIBILITY path parent_path() const {
-    return string_type(__parent_path());
-  }
-  _LIBCPP_INLINE_VISIBILITY path filename() const {
-    return string_type(__filename());
-  }
-  _LIBCPP_INLINE_VISIBILITY path stem() const { return string_type(__stem()); }
-  _LIBCPP_INLINE_VISIBILITY path extension() const {
-    return string_type(__extension());
-  }
-
-  // query
-  _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_INLINE_VISIBILITY bool
-  empty() const noexcept {
-    return __pn_.empty();
-  }
-
-  _LIBCPP_INLINE_VISIBILITY bool has_root_name() const {
-    return !__root_name().empty();
-  }
-  _LIBCPP_INLINE_VISIBILITY bool has_root_directory() const {
-    return !__root_directory().empty();
-  }
-  _LIBCPP_INLINE_VISIBILITY bool has_root_path() const {
-    return !__root_path_raw().empty();
-  }
-  _LIBCPP_INLINE_VISIBILITY bool has_relative_path() const {
-    return !__relative_path().empty();
-  }
-  _LIBCPP_INLINE_VISIBILITY bool has_parent_path() const {
-    return !__parent_path().empty();
-  }
-  _LIBCPP_INLINE_VISIBILITY bool has_filename() const {
-    return !__filename().empty();
-  }
-  _LIBCPP_INLINE_VISIBILITY bool has_stem() const { return !__stem().empty(); }
-  _LIBCPP_INLINE_VISIBILITY bool has_extension() const {
-    return !__extension().empty();
-  }
-
-  _LIBCPP_INLINE_VISIBILITY bool is_absolute() const {
-#if defined(_LIBCPP_WIN32API)
-    __string_view __root_name_str = __root_name();
-    __string_view __root_dir = __root_directory();
-    if (__root_name_str.size() == 2 && __root_name_str[1] == ':') {
-      // A drive letter with no root directory is relative, e.g. x:example.
-      return !__root_dir.empty();
-    }
-    // If no root name, it's relative, e.g. \example is relative to the current drive
-    if (__root_name_str.empty())
-      return false;
-    if (__root_name_str.size() < 3)
-      return false;
-    // A server root name, like \\server, is always absolute
-    if (__root_name_str[0] != '/' && __root_name_str[0] != '\\')
-      return false;
-    if (__root_name_str[1] != '/' && __root_name_str[1] != '\\')
-      return false;
-    // Seems to be a server root name
-    return true;
-#else
-    return has_root_directory();
-#endif
-  }
-  _LIBCPP_INLINE_VISIBILITY bool is_relative() const { return !is_absolute(); }
-
-  // relative paths
-  path lexically_normal() const;
-  path lexically_relative(const path& __base) const;
-
-  _LIBCPP_INLINE_VISIBILITY path lexically_proximate(const path& __base) const {
-    path __result = this->lexically_relative(__base);
-    if (__result.native().empty())
-      return *this;
-    return __result;
-  }
-
-  // iterators
-  class _LIBCPP_TYPE_VIS iterator;
-  typedef iterator const_iterator;
-
-  iterator begin() const;
-  iterator end() const;
-
-#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
-  template <class _CharT, class _Traits>
-  _LIBCPP_INLINE_VISIBILITY friend
-      typename enable_if<is_same<_CharT, value_type>::value &&
-                             is_same<_Traits, char_traits<value_type> >::value,
-                         basic_ostream<_CharT, _Traits>&>::type
-      operator<<(basic_ostream<_CharT, _Traits>& __os, const path& __p) {
-    __os << _VSTD::__quoted(__p.native());
-    return __os;
-  }
-
-  template <class _CharT, class _Traits>
-  _LIBCPP_INLINE_VISIBILITY friend
-      typename enable_if<!is_same<_CharT, value_type>::value ||
-                             !is_same<_Traits, char_traits<value_type> >::value,
-                         basic_ostream<_CharT, _Traits>&>::type
-      operator<<(basic_ostream<_CharT, _Traits>& __os, const path& __p) {
-    __os << _VSTD::__quoted(__p.string<_CharT, _Traits>());
-    return __os;
-  }
-
-  template <class _CharT, class _Traits>
-  _LIBCPP_INLINE_VISIBILITY friend basic_istream<_CharT, _Traits>&
-  operator>>(basic_istream<_CharT, _Traits>& __is, path& __p) {
-    basic_string<_CharT, _Traits> __tmp;
-    __is >> __quoted(__tmp);
-    __p = __tmp;
-    return __is;
-  }
-#endif // !_LIBCPP_HAS_NO_LOCALIZATION
-
-  friend _LIBCPP_INLINE_VISIBILITY bool operator==(const path& __lhs, const path& __rhs) noexcept {
-    return __lhs.__compare(__rhs.__pn_) == 0;
-  }
-  friend _LIBCPP_INLINE_VISIBILITY bool operator!=(const path& __lhs, const path& __rhs) noexcept {
-    return __lhs.__compare(__rhs.__pn_) != 0;
-  }
-  friend _LIBCPP_INLINE_VISIBILITY bool operator<(const path& __lhs, const path& __rhs) noexcept {
-    return __lhs.__compare(__rhs.__pn_) < 0;
-  }
-  friend _LIBCPP_INLINE_VISIBILITY bool operator<=(const path& __lhs, const path& __rhs) noexcept {
-    return __lhs.__compare(__rhs.__pn_) <= 0;
-  }
-  friend _LIBCPP_INLINE_VISIBILITY bool operator>(const path& __lhs, const path& __rhs) noexcept {
-    return __lhs.__compare(__rhs.__pn_) > 0;
-  }
-  friend _LIBCPP_INLINE_VISIBILITY bool operator>=(const path& __lhs, const path& __rhs) noexcept {
-    return __lhs.__compare(__rhs.__pn_) >= 0;
-  }
-
-  friend _LIBCPP_INLINE_VISIBILITY path operator/(const path& __lhs,
-                                                  const path& __rhs) {
-    path __result(__lhs);
-    __result /= __rhs;
-    return __result;
-  }
-private:
-  inline _LIBCPP_INLINE_VISIBILITY path&
-  __assign_view(__string_view const& __s) noexcept {
-    __pn_ = string_type(__s);
-    return *this;
-  }
-  string_type __pn_;
-};
-
-inline _LIBCPP_INLINE_VISIBILITY void swap(path& __lhs, path& __rhs) noexcept {
-  __lhs.swap(__rhs);
-}
-
-_LIBCPP_FUNC_VIS
-size_t hash_value(const path& __p) noexcept;
-
-template <class _InputIt>
-_LIBCPP_INLINE_VISIBILITY _LIBCPP_DEPRECATED_WITH_CHAR8_T
-    typename enable_if<__is_pathable<_InputIt>::value, path>::type
-    u8path(_InputIt __f, _InputIt __l) {
-  static_assert(
-#ifndef _LIBCPP_HAS_NO_CHAR8_T
-      is_same<typename __is_pathable<_InputIt>::__char_type, char8_t>::value ||
-#endif
-      is_same<typename __is_pathable<_InputIt>::__char_type, char>::value,
-      "u8path(Iter, Iter) requires Iter have a value_type of type 'char'"
-      " or 'char8_t'");
-#if defined(_LIBCPP_WIN32API)
-  string __tmp(__f, __l);
-  using _CVT = __widen_from_utf8<sizeof(wchar_t) * __CHAR_BIT__>;
-  _VSTD::wstring __w;
-  __w.reserve(__tmp.size());
-  _CVT()(back_inserter(__w), __tmp.data(), __tmp.data() + __tmp.size());
-  return path(__w);
-#else
-  return path(__f, __l);
-#endif /* !_LIBCPP_WIN32API */
-}
-
-#if defined(_LIBCPP_WIN32API)
-template <class _InputIt>
-_LIBCPP_INLINE_VISIBILITY _LIBCPP_DEPRECATED_WITH_CHAR8_T
-    typename enable_if<__is_pathable<_InputIt>::value, path>::type
-    u8path(_InputIt __f, _NullSentinel) {
-  static_assert(
-#ifndef _LIBCPP_HAS_NO_CHAR8_T
-      is_same<typename __is_pathable<_InputIt>::__char_type, char8_t>::value ||
-#endif
-      is_same<typename __is_pathable<_InputIt>::__char_type, char>::value,
-      "u8path(Iter, Iter) requires Iter have a value_type of type 'char'"
-      " or 'char8_t'");
-  string __tmp;
-  const char __sentinel = char{};
-  for (; *__f != __sentinel; ++__f)
-    __tmp.push_back(*__f);
-  using _CVT = __widen_from_utf8<sizeof(wchar_t) * __CHAR_BIT__>;
-  _VSTD::wstring __w;
-  __w.reserve(__tmp.size());
-  _CVT()(back_inserter(__w), __tmp.data(), __tmp.data() + __tmp.size());
-  return path(__w);
-}
-#endif /* _LIBCPP_WIN32API */
-
-template <class _Source>
-_LIBCPP_INLINE_VISIBILITY _LIBCPP_DEPRECATED_WITH_CHAR8_T
-    typename enable_if<__is_pathable<_Source>::value, path>::type
-    u8path(const _Source& __s) {
-  static_assert(
-#ifndef _LIBCPP_HAS_NO_CHAR8_T
-      is_same<typename __is_pathable<_Source>::__char_type, char8_t>::value ||
-#endif
-      is_same<typename __is_pathable<_Source>::__char_type, char>::value,
-      "u8path(Source const&) requires Source have a character type of type "
-      "'char' or 'char8_t'");
-#if defined(_LIBCPP_WIN32API)
-  using _Traits = __is_pathable<_Source>;
-  return u8path(_VSTD::__unwrap_iter(_Traits::__range_begin(__s)), _VSTD::__unwrap_iter(_Traits::__range_end(__s)));
-#else
-  return path(__s);
-#endif
-}
-
-class _LIBCPP_TYPE_VIS path::iterator {
-public:
-  enum _ParserState : unsigned char {
-    _Singular,
-    _BeforeBegin,
-    _InRootName,
-    _InRootDir,
-    _InFilenames,
-    _InTrailingSep,
-    _AtEnd
-  };
-
-public:
-  typedef bidirectional_iterator_tag iterator_category;
-
-  typedef path value_type;
-  typedef ptrdiff_t difference_type;
-  typedef const path* pointer;
-  typedef const path& reference;
-
-  typedef void
-      __stashing_iterator_tag; // See reverse_iterator and __is_stashing_iterator
-
-public:
-  _LIBCPP_INLINE_VISIBILITY
-  iterator()
-      : __stashed_elem_(), __path_ptr_(nullptr), __entry_(),
-        __state_(_Singular) {}
-
-  iterator(const iterator&) = default;
-  ~iterator() = default;
-
-  iterator& operator=(const iterator&) = default;
-
-  _LIBCPP_INLINE_VISIBILITY
-  reference operator*() const { return __stashed_elem_; }
-
-  _LIBCPP_INLINE_VISIBILITY
-  pointer operator->() const { return &__stashed_elem_; }
-
-  _LIBCPP_INLINE_VISIBILITY
-  iterator& operator++() {
-    _LIBCPP_ASSERT(__state_ != _Singular,
-                   "attempting to increment a singular iterator");
-    _LIBCPP_ASSERT(__state_ != _AtEnd,
-                   "attempting to increment the end iterator");
-    return __increment();
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  iterator operator++(int) {
-    iterator __it(*this);
-    this->operator++();
-    return __it;
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  iterator& operator--() {
-    _LIBCPP_ASSERT(__state_ != _Singular,
-                   "attempting to decrement a singular iterator");
-    _LIBCPP_ASSERT(__entry_.data() != __path_ptr_->native().data(),
-                   "attempting to decrement the begin iterator");
-    return __decrement();
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  iterator operator--(int) {
-    iterator __it(*this);
-    this->operator--();
-    return __it;
-  }
-
-private:
-  friend class path;
-
-  inline _LIBCPP_INLINE_VISIBILITY friend bool operator==(const iterator&,
-                                                          const iterator&);
-
-  iterator& __increment();
-  iterator& __decrement();
-
-  path __stashed_elem_;
-  const path* __path_ptr_;
-  path::__string_view __entry_;
-  _ParserState __state_;
-};
-
-inline _LIBCPP_INLINE_VISIBILITY bool operator==(const path::iterator& __lhs,
-                                                 const path::iterator& __rhs) {
-  return __lhs.__path_ptr_ == __rhs.__path_ptr_ &&
-         __lhs.__entry_.data() == __rhs.__entry_.data();
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool operator!=(const path::iterator& __lhs,
-                                                 const path::iterator& __rhs) {
-  return !(__lhs == __rhs);
-}
-
-// TODO(ldionne): We need to pop the pragma and push it again after
-//                filesystem_error to work around PR41078.
-_LIBCPP_AVAILABILITY_FILESYSTEM_POP
-
-class _LIBCPP_AVAILABILITY_FILESYSTEM _LIBCPP_EXCEPTION_ABI filesystem_error : public system_error {
-public:
-  _LIBCPP_INLINE_VISIBILITY
-  filesystem_error(const string& __what, error_code __ec)
-      : system_error(__ec, __what),
-        __storage_(make_shared<_Storage>(path(), path())) {
-    __create_what(0);
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  filesystem_error(const string& __what, const path& __p1, error_code __ec)
-      : system_error(__ec, __what),
-        __storage_(make_shared<_Storage>(__p1, path())) {
-    __create_what(1);
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  filesystem_error(const string& __what, const path& __p1, const path& __p2,
-                   error_code __ec)
-      : system_error(__ec, __what),
-        __storage_(make_shared<_Storage>(__p1, __p2)) {
-    __create_what(2);
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  const path& path1() const noexcept { return __storage_->__p1_; }
-
-  _LIBCPP_INLINE_VISIBILITY
-  const path& path2() const noexcept { return __storage_->__p2_; }
-
-  filesystem_error(const filesystem_error&) = default;
-  ~filesystem_error() override; // key function
-
-  _LIBCPP_INLINE_VISIBILITY
-  const char* what() const noexcept override {
-    return __storage_->__what_.c_str();
-  }
-
-  void __create_what(int __num_paths);
-
-private:
-  struct _LIBCPP_HIDDEN _Storage {
-    _LIBCPP_INLINE_VISIBILITY
-    _Storage(const path& __p1, const path& __p2) : __p1_(__p1), __p2_(__p2) {}
-
-    path __p1_;
-    path __p2_;
-    string __what_;
-  };
-  shared_ptr<_Storage> __storage_;
-};
-
-_LIBCPP_AVAILABILITY_FILESYSTEM_PUSH
-
-template <class... _Args>
-_LIBCPP_NORETURN inline _LIBCPP_INLINE_VISIBILITY
-#ifndef _LIBCPP_NO_EXCEPTIONS
-void __throw_filesystem_error(_Args&&... __args) {
-  throw filesystem_error(_VSTD::forward<_Args>(__args)...);
-}
-#else
-void __throw_filesystem_error(_Args&&...) {
-  _VSTD::abort();
-}
-#endif
-
-// operational functions
-
-_LIBCPP_FUNC_VIS
-path __absolute(const path&, error_code* __ec = nullptr);
-_LIBCPP_FUNC_VIS
-path __canonical(const path&, error_code* __ec = nullptr);
-_LIBCPP_FUNC_VIS
-void __copy(const path& __from, const path& __to, copy_options __opt,
-            error_code* __ec = nullptr);
-_LIBCPP_FUNC_VIS
-bool __copy_file(const path& __from, const path& __to, copy_options __opt,
-                 error_code* __ec = nullptr);
-_LIBCPP_FUNC_VIS
-void __copy_symlink(const path& __existing_symlink, const path& __new_symlink,
-                    error_code* __ec = nullptr);
-_LIBCPP_FUNC_VIS
-bool __create_directories(const path& p, error_code* ec = nullptr);
-_LIBCPP_FUNC_VIS
-bool __create_directory(const path& p, error_code* ec = nullptr);
-_LIBCPP_FUNC_VIS
-bool __create_directory(const path& p, const path& attributes,
-                        error_code* ec = nullptr);
-_LIBCPP_FUNC_VIS
-void __create_directory_symlink(const path& __to, const path& __new_symlink,
-                                error_code* __ec = nullptr);
-_LIBCPP_FUNC_VIS
-void __create_hard_link(const path& __to, const path& __new_hard_link,
-                        error_code* __ec = nullptr);
-_LIBCPP_FUNC_VIS
-void __create_symlink(const path& __to, const path& __new_symlink,
-                      error_code* __ec = nullptr);
-_LIBCPP_FUNC_VIS
-path __current_path(error_code* __ec = nullptr);
-_LIBCPP_FUNC_VIS
-void __current_path(const path&, error_code* __ec = nullptr);
-_LIBCPP_FUNC_VIS
-bool __equivalent(const path&, const path&, error_code* __ec = nullptr);
-_LIBCPP_FUNC_VIS
-uintmax_t __file_size(const path&, error_code* __ec = nullptr);
-_LIBCPP_FUNC_VIS
-uintmax_t __hard_link_count(const path&, error_code* __ec = nullptr);
-_LIBCPP_FUNC_VIS
-bool __fs_is_empty(const path& p, error_code* ec = nullptr);
-_LIBCPP_FUNC_VIS
-file_time_type __last_write_time(const path& p, error_code* ec = nullptr);
-_LIBCPP_FUNC_VIS
-void __last_write_time(const path& p, file_time_type new_time,
-                       error_code* ec = nullptr);
-_LIBCPP_FUNC_VIS
-void __permissions(const path&, perms, perm_options, error_code* = nullptr);
-_LIBCPP_FUNC_VIS
-path __read_symlink(const path& p, error_code* ec = nullptr);
-_LIBCPP_FUNC_VIS
-bool __remove(const path& p, error_code* ec = nullptr);
-_LIBCPP_FUNC_VIS
-uintmax_t __remove_all(const path& p, error_code* ec = nullptr);
-_LIBCPP_FUNC_VIS
-void __rename(const path& from, const path& to, error_code* ec = nullptr);
-_LIBCPP_FUNC_VIS
-void __resize_file(const path& p, uintmax_t size, error_code* ec = nullptr);
-_LIBCPP_FUNC_VIS
-space_info __space(const path&, error_code* __ec = nullptr);
-_LIBCPP_FUNC_VIS
-file_status __status(const path&, error_code* __ec = nullptr);
-_LIBCPP_FUNC_VIS
-file_status __symlink_status(const path&, error_code* __ec = nullptr);
-_LIBCPP_FUNC_VIS
-path __system_complete(const path&, error_code* __ec = nullptr);
-_LIBCPP_FUNC_VIS
-path __temp_directory_path(error_code* __ec = nullptr);
-_LIBCPP_FUNC_VIS
-path __weakly_canonical(path const& __p, error_code* __ec = nullptr);
-
-inline _LIBCPP_INLINE_VISIBILITY path current_path() {
-  return __current_path();
-}
-
-inline _LIBCPP_INLINE_VISIBILITY path current_path(error_code& __ec) {
-  return __current_path(&__ec);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY void current_path(const path& __p) {
-  __current_path(__p);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY void current_path(const path& __p,
-                                                   error_code& __ec) noexcept {
-  __current_path(__p, &__ec);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY path absolute(const path& __p) {
-  return __absolute(__p);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY path absolute(const path& __p,
-                                               error_code& __ec) {
-  return __absolute(__p, &__ec);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY path canonical(const path& __p) {
-  return __canonical(__p);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY path canonical(const path& __p,
-                                                error_code& __ec) {
-  return __canonical(__p, &__ec);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY void copy(const path& __from,
-                                           const path& __to) {
-  __copy(__from, __to, copy_options::none);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY void copy(const path& __from, const path& __to,
-                                           error_code& __ec) {
-  __copy(__from, __to, copy_options::none, &__ec);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY void copy(const path& __from, const path& __to,
-                                           copy_options __opt) {
-  __copy(__from, __to, __opt);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY void copy(const path& __from, const path& __to,
-                                           copy_options __opt,
-                                           error_code& __ec) {
-  __copy(__from, __to, __opt, &__ec);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool copy_file(const path& __from,
-                                                const path& __to) {
-  return __copy_file(__from, __to, copy_options::none);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool
-copy_file(const path& __from, const path& __to, error_code& __ec) {
-  return __copy_file(__from, __to, copy_options::none, &__ec);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool
-copy_file(const path& __from, const path& __to, copy_options __opt) {
-  return __copy_file(__from, __to, __opt);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool copy_file(const path& __from,
-                                                const path& __to,
-                                                copy_options __opt,
-                                                error_code& __ec) {
-  return __copy_file(__from, __to, __opt, &__ec);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY void copy_symlink(const path& __existing,
-                                                   const path& __new) {
-  __copy_symlink(__existing, __new);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY void
-copy_symlink(const path& __ext, const path& __new, error_code& __ec) noexcept {
-  __copy_symlink(__ext, __new, &__ec);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool create_directories(const path& __p) {
-  return __create_directories(__p);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool create_directories(const path& __p,
-                                                         error_code& __ec) {
-  return __create_directories(__p, &__ec);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool create_directory(const path& __p) {
-  return __create_directory(__p);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool
-create_directory(const path& __p, error_code& __ec) noexcept {
-  return __create_directory(__p, &__ec);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool create_directory(const path& __p,
-                                                       const path& __attrs) {
-  return __create_directory(__p, __attrs);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool
-create_directory(const path& __p, const path& __attrs,
-                 error_code& __ec) noexcept {
-  return __create_directory(__p, __attrs, &__ec);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY void
-create_directory_symlink(const path& __to, const path& __new) {
-  __create_directory_symlink(__to, __new);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY void
-create_directory_symlink(const path& __to, const path& __new,
-                         error_code& __ec) noexcept {
-  __create_directory_symlink(__to, __new, &__ec);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY void create_hard_link(const path& __to,
-                                                       const path& __new) {
-  __create_hard_link(__to, __new);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY void
-create_hard_link(const path& __to, const path& __new,
-                 error_code& __ec) noexcept {
-  __create_hard_link(__to, __new, &__ec);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY void create_symlink(const path& __to,
-                                                     const path& __new) {
-  __create_symlink(__to, __new);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY void
-create_symlink(const path& __to, const path& __new, error_code& __ec) noexcept {
-  return __create_symlink(__to, __new, &__ec);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool status_known(file_status __s) noexcept {
-  return __s.type() != file_type::none;
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool exists(file_status __s) noexcept {
-  return status_known(__s) && __s.type() != file_type::not_found;
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool exists(const path& __p) {
-  return exists(__status(__p));
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool exists(const path& __p,
-                                             error_code& __ec) noexcept {
-  auto __s = __status(__p, &__ec);
-  if (status_known(__s))
-    __ec.clear();
-  return exists(__s);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool equivalent(const path& __p1,
-                                                 const path& __p2) {
-  return __equivalent(__p1, __p2);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool
-equivalent(const path& __p1, const path& __p2, error_code& __ec) noexcept {
-  return __equivalent(__p1, __p2, &__ec);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY uintmax_t file_size(const path& __p) {
-  return __file_size(__p);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY uintmax_t
-file_size(const path& __p, error_code& __ec) noexcept {
-  return __file_size(__p, &__ec);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY uintmax_t hard_link_count(const path& __p) {
-  return __hard_link_count(__p);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY uintmax_t
-hard_link_count(const path& __p, error_code& __ec) noexcept {
-  return __hard_link_count(__p, &__ec);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool is_block_file(file_status __s) noexcept {
-  return __s.type() == file_type::block;
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool is_block_file(const path& __p) {
-  return is_block_file(__status(__p));
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool is_block_file(const path& __p,
-                                                    error_code& __ec) noexcept {
-  return is_block_file(__status(__p, &__ec));
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool
-is_character_file(file_status __s) noexcept {
-  return __s.type() == file_type::character;
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool is_character_file(const path& __p) {
-  return is_character_file(__status(__p));
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool
-is_character_file(const path& __p, error_code& __ec) noexcept {
-  return is_character_file(__status(__p, &__ec));
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool is_directory(file_status __s) noexcept {
-  return __s.type() == file_type::directory;
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool is_directory(const path& __p) {
-  return is_directory(__status(__p));
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool is_directory(const path& __p,
-                                                   error_code& __ec) noexcept {
-  return is_directory(__status(__p, &__ec));
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool is_empty(const path& __p) {
-  return __fs_is_empty(__p);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool is_empty(const path& __p,
-                                               error_code& __ec) {
-  return __fs_is_empty(__p, &__ec);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool is_fifo(file_status __s) noexcept {
-  return __s.type() == file_type::fifo;
-}
-inline _LIBCPP_INLINE_VISIBILITY bool is_fifo(const path& __p) {
-  return is_fifo(__status(__p));
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool is_fifo(const path& __p,
-                                              error_code& __ec) noexcept {
-  return is_fifo(__status(__p, &__ec));
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool
-is_regular_file(file_status __s) noexcept {
-  return __s.type() == file_type::regular;
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool is_regular_file(const path& __p) {
-  return is_regular_file(__status(__p));
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool
-is_regular_file(const path& __p, error_code& __ec) noexcept {
-  return is_regular_file(__status(__p, &__ec));
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool is_socket(file_status __s) noexcept {
-  return __s.type() == file_type::socket;
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool is_socket(const path& __p) {
-  return is_socket(__status(__p));
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool is_socket(const path& __p,
-                                                error_code& __ec) noexcept {
-  return is_socket(__status(__p, &__ec));
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool is_symlink(file_status __s) noexcept {
-  return __s.type() == file_type::symlink;
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool is_symlink(const path& __p) {
-  return is_symlink(__symlink_status(__p));
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool is_symlink(const path& __p,
-                                                 error_code& __ec) noexcept {
-  return is_symlink(__symlink_status(__p, &__ec));
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool is_other(file_status __s) noexcept {
-  return exists(__s) && !is_regular_file(__s) && !is_directory(__s) &&
-         !is_symlink(__s);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool is_other(const path& __p) {
-  return is_other(__status(__p));
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool is_other(const path& __p,
-                                               error_code& __ec) noexcept {
-  return is_other(__status(__p, &__ec));
-}
-
-inline _LIBCPP_INLINE_VISIBILITY file_time_type
-last_write_time(const path& __p) {
-  return __last_write_time(__p);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY file_time_type
-last_write_time(const path& __p, error_code& __ec) noexcept {
-  return __last_write_time(__p, &__ec);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY void last_write_time(const path& __p,
-                                                      file_time_type __t) {
-  __last_write_time(__p, __t);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY void
-last_write_time(const path& __p, file_time_type __t,
-                error_code& __ec) noexcept {
-  __last_write_time(__p, __t, &__ec);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY void
-permissions(const path& __p, perms __prms,
-            perm_options __opts = perm_options::replace) {
-  __permissions(__p, __prms, __opts);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY void permissions(const path& __p, perms __prms,
-                                                  error_code& __ec) noexcept {
-  __permissions(__p, __prms, perm_options::replace, &__ec);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY void permissions(const path& __p, perms __prms,
-                                                  perm_options __opts,
-                                                  error_code& __ec) {
-  __permissions(__p, __prms, __opts, &__ec);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY path proximate(const path& __p,
-                                                const path& __base,
-                                                error_code& __ec) {
-  path __tmp = __weakly_canonical(__p, &__ec);
-  if (__ec)
-    return {};
-  path __tmp_base = __weakly_canonical(__base, &__ec);
-  if (__ec)
-    return {};
-  return __tmp.lexically_proximate(__tmp_base);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY path proximate(const path& __p,
-                                                error_code& __ec) {
-  return proximate(__p, current_path(), __ec);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY path
-proximate(const path& __p, const path& __base = current_path()) {
-  return __weakly_canonical(__p).lexically_proximate(
-      __weakly_canonical(__base));
-}
-
-inline _LIBCPP_INLINE_VISIBILITY path read_symlink(const path& __p) {
-  return __read_symlink(__p);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY path read_symlink(const path& __p,
-                                                   error_code& __ec) {
-  return __read_symlink(__p, &__ec);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY path relative(const path& __p,
-                                               const path& __base,
-                                               error_code& __ec) {
-  path __tmp = __weakly_canonical(__p, &__ec);
-  if (__ec)
-    return path();
-  path __tmpbase = __weakly_canonical(__base, &__ec);
-  if (__ec)
-    return path();
-  return __tmp.lexically_relative(__tmpbase);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY path relative(const path& __p,
-                                               error_code& __ec) {
-  return relative(__p, current_path(), __ec);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY path
-relative(const path& __p, const path& __base = current_path()) {
-  return __weakly_canonical(__p).lexically_relative(__weakly_canonical(__base));
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool remove(const path& __p) {
-  return __remove(__p);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool remove(const path& __p,
-                                             error_code& __ec) noexcept {
-  return __remove(__p, &__ec);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY uintmax_t remove_all(const path& __p) {
-  return __remove_all(__p);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY uintmax_t remove_all(const path& __p,
-                                                      error_code& __ec) {
-  return __remove_all(__p, &__ec);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY void rename(const path& __from,
-                                             const path& __to) {
-  return __rename(__from, __to);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY void
-rename(const path& __from, const path& __to, error_code& __ec) noexcept {
-  return __rename(__from, __to, &__ec);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY void resize_file(const path& __p,
-                                                  uintmax_t __ns) {
-  return __resize_file(__p, __ns);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY void
-resize_file(const path& __p, uintmax_t __ns, error_code& __ec) noexcept {
-  return __resize_file(__p, __ns, &__ec);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY space_info space(const path& __p) {
-  return __space(__p);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY space_info space(const path& __p,
-                                                  error_code& __ec) noexcept {
-  return __space(__p, &__ec);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY file_status status(const path& __p) {
-  return __status(__p);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY file_status status(const path& __p,
-                                                    error_code& __ec) noexcept {
-  return __status(__p, &__ec);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY file_status symlink_status(const path& __p) {
-  return __symlink_status(__p);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY file_status
-symlink_status(const path& __p, error_code& __ec) noexcept {
-  return __symlink_status(__p, &__ec);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY path temp_directory_path() {
-  return __temp_directory_path();
-}
-
-inline _LIBCPP_INLINE_VISIBILITY path temp_directory_path(error_code& __ec) {
-  return __temp_directory_path(&__ec);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY path weakly_canonical(path const& __p) {
-  return __weakly_canonical(__p);
-}
-
-inline _LIBCPP_INLINE_VISIBILITY path weakly_canonical(path const& __p,
-                                                       error_code& __ec) {
-  return __weakly_canonical(__p, &__ec);
-}
-
-class directory_iterator;
-class recursive_directory_iterator;
-class _LIBCPP_HIDDEN __dir_stream;
-
-class directory_entry {
-  typedef _VSTD_FS::path _Path;
-
-public:
-  // constructors and destructors
-  directory_entry() noexcept = default;
-  directory_entry(directory_entry const&) = default;
-  directory_entry(directory_entry&&) noexcept = default;
-
-  _LIBCPP_INLINE_VISIBILITY
-  explicit directory_entry(_Path const& __p) : __p_(__p) {
-    error_code __ec;
-    __refresh(&__ec);
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  directory_entry(_Path const& __p, error_code& __ec) : __p_(__p) {
-    __refresh(&__ec);
-  }
-
-  ~directory_entry() {}
-
-  directory_entry& operator=(directory_entry const&) = default;
-  directory_entry& operator=(directory_entry&&) noexcept = default;
-
-  _LIBCPP_INLINE_VISIBILITY
-  void assign(_Path const& __p) {
-    __p_ = __p;
-    error_code __ec;
-    __refresh(&__ec);
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  void assign(_Path const& __p, error_code& __ec) {
-    __p_ = __p;
-    __refresh(&__ec);
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  void replace_filename(_Path const& __p) {
-    __p_.replace_filename(__p);
-    error_code __ec;
-    __refresh(&__ec);
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  void replace_filename(_Path const& __p, error_code& __ec) {
-    __p_ = __p_.parent_path() / __p;
-    __refresh(&__ec);
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  void refresh() { __refresh(); }
-
-  _LIBCPP_INLINE_VISIBILITY
-  void refresh(error_code& __ec) noexcept { __refresh(&__ec); }
-
-  _LIBCPP_INLINE_VISIBILITY
-  _Path const& path() const noexcept { return __p_; }
-
-  _LIBCPP_INLINE_VISIBILITY
-  operator const _Path&() const noexcept { return __p_; }
-
-  _LIBCPP_INLINE_VISIBILITY
-  bool exists() const { return _VSTD_FS::exists(file_status{__get_ft()}); }
-
-  _LIBCPP_INLINE_VISIBILITY
-  bool exists(error_code& __ec) const noexcept {
-    return _VSTD_FS::exists(file_status{__get_ft(&__ec)});
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  bool is_block_file() const { return __get_ft() == file_type::block; }
-
-  _LIBCPP_INLINE_VISIBILITY
-  bool is_block_file(error_code& __ec) const noexcept {
-    return __get_ft(&__ec) == file_type::block;
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  bool is_character_file() const { return __get_ft() == file_type::character; }
-
-  _LIBCPP_INLINE_VISIBILITY
-  bool is_character_file(error_code& __ec) const noexcept {
-    return __get_ft(&__ec) == file_type::character;
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  bool is_directory() const { return __get_ft() == file_type::directory; }
-
-  _LIBCPP_INLINE_VISIBILITY
-  bool is_directory(error_code& __ec) const noexcept {
-    return __get_ft(&__ec) == file_type::directory;
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  bool is_fifo() const { return __get_ft() == file_type::fifo; }
-
-  _LIBCPP_INLINE_VISIBILITY
-  bool is_fifo(error_code& __ec) const noexcept {
-    return __get_ft(&__ec) == file_type::fifo;
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  bool is_other() const { return _VSTD_FS::is_other(file_status{__get_ft()}); }
-
-  _LIBCPP_INLINE_VISIBILITY
-  bool is_other(error_code& __ec) const noexcept {
-    return _VSTD_FS::is_other(file_status{__get_ft(&__ec)});
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  bool is_regular_file() const { return __get_ft() == file_type::regular; }
-
-  _LIBCPP_INLINE_VISIBILITY
-  bool is_regular_file(error_code& __ec) const noexcept {
-    return __get_ft(&__ec) == file_type::regular;
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  bool is_socket() const { return __get_ft() == file_type::socket; }
-
-  _LIBCPP_INLINE_VISIBILITY
-  bool is_socket(error_code& __ec) const noexcept {
-    return __get_ft(&__ec) == file_type::socket;
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  bool is_symlink() const { return __get_sym_ft() == file_type::symlink; }
-
-  _LIBCPP_INLINE_VISIBILITY
-  bool is_symlink(error_code& __ec) const noexcept {
-    return __get_sym_ft(&__ec) == file_type::symlink;
-  }
-  _LIBCPP_INLINE_VISIBILITY
-  uintmax_t file_size() const { return __get_size(); }
-
-  _LIBCPP_INLINE_VISIBILITY
-  uintmax_t file_size(error_code& __ec) const noexcept {
-    return __get_size(&__ec);
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  uintmax_t hard_link_count() const { return __get_nlink(); }
-
-  _LIBCPP_INLINE_VISIBILITY
-  uintmax_t hard_link_count(error_code& __ec) const noexcept {
-    return __get_nlink(&__ec);
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  file_time_type last_write_time() const { return __get_write_time(); }
-
-  _LIBCPP_INLINE_VISIBILITY
-  file_time_type last_write_time(error_code& __ec) const noexcept {
-    return __get_write_time(&__ec);
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  file_status status() const { return __get_status(); }
-
-  _LIBCPP_INLINE_VISIBILITY
-  file_status status(error_code& __ec) const noexcept {
-    return __get_status(&__ec);
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  file_status symlink_status() const { return __get_symlink_status(); }
-
-  _LIBCPP_INLINE_VISIBILITY
-  file_status symlink_status(error_code& __ec) const noexcept {
-    return __get_symlink_status(&__ec);
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  bool operator<(directory_entry const& __rhs) const noexcept {
-    return __p_ < __rhs.__p_;
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  bool operator==(directory_entry const& __rhs) const noexcept {
-    return __p_ == __rhs.__p_;
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  bool operator!=(directory_entry const& __rhs) const noexcept {
-    return __p_ != __rhs.__p_;
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  bool operator<=(directory_entry const& __rhs) const noexcept {
-    return __p_ <= __rhs.__p_;
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  bool operator>(directory_entry const& __rhs) const noexcept {
-    return __p_ > __rhs.__p_;
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  bool operator>=(directory_entry const& __rhs) const noexcept {
-    return __p_ >= __rhs.__p_;
-  }
-
-private:
-  friend class directory_iterator;
-  friend class recursive_directory_iterator;
-  friend class __dir_stream;
-
-  enum _CacheType : unsigned char {
-    _Empty,
-    _IterSymlink,
-    _IterNonSymlink,
-    _RefreshSymlink,
-    _RefreshSymlinkUnresolved,
-    _RefreshNonSymlink
-  };
-
-  struct __cached_data {
-    uintmax_t __size_;
-    uintmax_t __nlink_;
-    file_time_type __write_time_;
-    perms __sym_perms_;
-    perms __non_sym_perms_;
-    file_type __type_;
-    _CacheType __cache_type_;
-
-    _LIBCPP_INLINE_VISIBILITY
-    __cached_data() noexcept { __reset(); }
-
-    _LIBCPP_INLINE_VISIBILITY
-    void __reset() {
-      __cache_type_ = _Empty;
-      __type_ = file_type::none;
-      __sym_perms_ = __non_sym_perms_ = perms::unknown;
-      __size_ = __nlink_ = uintmax_t(-1);
-      __write_time_ = file_time_type::min();
-    }
-  };
-
-  _LIBCPP_INLINE_VISIBILITY
-  static __cached_data __create_iter_result(file_type __ft) {
-    __cached_data __data;
-    __data.__type_ = __ft;
-    __data.__cache_type_ = [&]() {
-      switch (__ft) {
-      case file_type::none:
-        return _Empty;
-      case file_type::symlink:
-        return _IterSymlink;
-      default:
-        return _IterNonSymlink;
-      }
-    }();
-    return __data;
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  void __assign_iter_entry(_Path&& __p, __cached_data __dt) {
-    __p_ = _VSTD::move(__p);
-    __data_ = __dt;
-  }
-
-  _LIBCPP_FUNC_VIS
-  error_code __do_refresh() noexcept;
-
-  _LIBCPP_INLINE_VISIBILITY
-  static bool __is_dne_error(error_code const& __ec) {
-    if (!__ec)
-      return true;
-    switch (static_cast<errc>(__ec.value())) {
-    case errc::no_such_file_or_directory:
-    case errc::not_a_directory:
-      return true;
-    default:
-      return false;
-    }
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  void __handle_error(const char* __msg, error_code* __dest_ec,
-                      error_code const& __ec, bool __allow_dne = false) const {
-    if (__dest_ec) {
-      *__dest_ec = __ec;
-      return;
-    }
-    if (__ec && (!__allow_dne || !__is_dne_error(__ec)))
-      __throw_filesystem_error(__msg, __p_, __ec);
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  void __refresh(error_code* __ec = nullptr) {
-    __handle_error("in directory_entry::refresh", __ec, __do_refresh(),
-                   /*allow_dne*/ true);
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  file_type __get_sym_ft(error_code* __ec = nullptr) const {
-    switch (__data_.__cache_type_) {
-    case _Empty:
-      return __symlink_status(__p_, __ec).type();
-    case _IterSymlink:
-    case _RefreshSymlink:
-    case _RefreshSymlinkUnresolved:
-      if (__ec)
-        __ec->clear();
-      return file_type::symlink;
-    case _IterNonSymlink:
-    case _RefreshNonSymlink:
-      file_status __st(__data_.__type_);
-      if (__ec && !_VSTD_FS::exists(__st))
-        *__ec = make_error_code(errc::no_such_file_or_directory);
-      else if (__ec)
-        __ec->clear();
-      return __data_.__type_;
-    }
-    _LIBCPP_UNREACHABLE();
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  file_type __get_ft(error_code* __ec = nullptr) const {
-    switch (__data_.__cache_type_) {
-    case _Empty:
-    case _IterSymlink:
-    case _RefreshSymlinkUnresolved:
-      return __status(__p_, __ec).type();
-    case _IterNonSymlink:
-    case _RefreshNonSymlink:
-    case _RefreshSymlink: {
-      file_status __st(__data_.__type_);
-      if (__ec && !_VSTD_FS::exists(__st))
-        *__ec = make_error_code(errc::no_such_file_or_directory);
-      else if (__ec)
-        __ec->clear();
-      return __data_.__type_;
-    }
-    }
-    _LIBCPP_UNREACHABLE();
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  file_status __get_status(error_code* __ec = nullptr) const {
-    switch (__data_.__cache_type_) {
-    case _Empty:
-    case _IterNonSymlink:
-    case _IterSymlink:
-    case _RefreshSymlinkUnresolved:
-      return __status(__p_, __ec);
-    case _RefreshNonSymlink:
-    case _RefreshSymlink:
-      return file_status(__get_ft(__ec), __data_.__non_sym_perms_);
-    }
-    _LIBCPP_UNREACHABLE();
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  file_status __get_symlink_status(error_code* __ec = nullptr) const {
-    switch (__data_.__cache_type_) {
-    case _Empty:
-    case _IterNonSymlink:
-    case _IterSymlink:
-      return __symlink_status(__p_, __ec);
-    case _RefreshNonSymlink:
-      return file_status(__get_sym_ft(__ec), __data_.__non_sym_perms_);
-    case _RefreshSymlink:
-    case _RefreshSymlinkUnresolved:
-      return file_status(__get_sym_ft(__ec), __data_.__sym_perms_);
-    }
-    _LIBCPP_UNREACHABLE();
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  uintmax_t __get_size(error_code* __ec = nullptr) const {
-    switch (__data_.__cache_type_) {
-    case _Empty:
-    case _IterNonSymlink:
-    case _IterSymlink:
-    case _RefreshSymlinkUnresolved:
-      return _VSTD_FS::__file_size(__p_, __ec);
-    case _RefreshSymlink:
-    case _RefreshNonSymlink: {
-      error_code __m_ec;
-      file_status __st(__get_ft(&__m_ec));
-      __handle_error("in directory_entry::file_size", __ec, __m_ec);
-      if (_VSTD_FS::exists(__st) && !_VSTD_FS::is_regular_file(__st)) {
-        errc __err_kind = _VSTD_FS::is_directory(__st) ? errc::is_a_directory
-                                                       : errc::not_supported;
-        __handle_error("in directory_entry::file_size", __ec,
-                       make_error_code(__err_kind));
-      }
-      return __data_.__size_;
-    }
-    }
-    _LIBCPP_UNREACHABLE();
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  uintmax_t __get_nlink(error_code* __ec = nullptr) const {
-    switch (__data_.__cache_type_) {
-    case _Empty:
-    case _IterNonSymlink:
-    case _IterSymlink:
-    case _RefreshSymlinkUnresolved:
-      return _VSTD_FS::__hard_link_count(__p_, __ec);
-    case _RefreshSymlink:
-    case _RefreshNonSymlink: {
-      error_code __m_ec;
-      (void)__get_ft(&__m_ec);
-      __handle_error("in directory_entry::hard_link_count", __ec, __m_ec);
-      return __data_.__nlink_;
-    }
-    }
-    _LIBCPP_UNREACHABLE();
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  file_time_type __get_write_time(error_code* __ec = nullptr) const {
-    switch (__data_.__cache_type_) {
-    case _Empty:
-    case _IterNonSymlink:
-    case _IterSymlink:
-    case _RefreshSymlinkUnresolved:
-      return _VSTD_FS::__last_write_time(__p_, __ec);
-    case _RefreshSymlink:
-    case _RefreshNonSymlink: {
-      error_code __m_ec;
-      file_status __st(__get_ft(&__m_ec));
-      __handle_error("in directory_entry::last_write_time", __ec, __m_ec);
-      if (_VSTD_FS::exists(__st) &&
-          __data_.__write_time_ == file_time_type::min())
-        __handle_error("in directory_entry::last_write_time", __ec,
-                       make_error_code(errc::value_too_large));
-      return __data_.__write_time_;
-    }
-    }
-    _LIBCPP_UNREACHABLE();
-  }
-
-private:
-  _Path __p_;
-  __cached_data __data_;
-};
-
-class __dir_element_proxy {
-public:
-  inline _LIBCPP_INLINE_VISIBILITY directory_entry operator*() {
-    return _VSTD::move(__elem_);
-  }
-
-private:
-  friend class directory_iterator;
-  friend class recursive_directory_iterator;
-  explicit __dir_element_proxy(directory_entry const& __e) : __elem_(__e) {}
-  __dir_element_proxy(__dir_element_proxy&& __o)
-      : __elem_(_VSTD::move(__o.__elem_)) {}
-  directory_entry __elem_;
-};
-
-class directory_iterator {
-public:
-  typedef directory_entry value_type;
-  typedef ptrdiff_t difference_type;
-  typedef value_type const* pointer;
-  typedef value_type const& reference;
-  typedef input_iterator_tag iterator_category;
-
-public:
-  //ctor & dtor
-  directory_iterator() noexcept {}
-
-  explicit directory_iterator(const path& __p)
-      : directory_iterator(__p, nullptr) {}
-
-  directory_iterator(const path& __p, directory_options __opts)
-      : directory_iterator(__p, nullptr, __opts) {}
-
-  directory_iterator(const path& __p, error_code& __ec)
-      : directory_iterator(__p, &__ec) {}
-
-  directory_iterator(const path& __p, directory_options __opts,
-                     error_code& __ec)
-      : directory_iterator(__p, &__ec, __opts) {}
-
-  directory_iterator(const directory_iterator&) = default;
-  directory_iterator(directory_iterator&&) = default;
-  directory_iterator& operator=(const directory_iterator&) = default;
-
-  directory_iterator& operator=(directory_iterator&& __o) noexcept {
-    // non-default implementation provided to support self-move assign.
-    if (this != &__o) {
-      __imp_ = _VSTD::move(__o.__imp_);
-    }
-    return *this;
-  }
-
-  ~directory_iterator() = default;
-
-  const directory_entry& operator*() const {
-    _LIBCPP_ASSERT(__imp_, "The end iterator cannot be dereferenced");
-    return __dereference();
-  }
-
-  const directory_entry* operator->() const { return &**this; }
-
-  directory_iterator& operator++() { return __increment(); }
-
-  __dir_element_proxy operator++(int) {
-    __dir_element_proxy __p(**this);
-    __increment();
-    return __p;
-  }
-
-  directory_iterator& increment(error_code& __ec) { return __increment(&__ec); }
-
-private:
-  inline _LIBCPP_INLINE_VISIBILITY friend bool
-  operator==(const directory_iterator& __lhs,
-             const directory_iterator& __rhs) noexcept;
-
-  // construct the dir_stream
-  _LIBCPP_FUNC_VIS
-  directory_iterator(const path&, error_code*,
-                     directory_options = directory_options::none);
-
-  _LIBCPP_FUNC_VIS
-  directory_iterator& __increment(error_code* __ec = nullptr);
-
-  _LIBCPP_FUNC_VIS
-  const directory_entry& __dereference() const;
-
-private:
-  shared_ptr<__dir_stream> __imp_;
-};
-
-inline _LIBCPP_INLINE_VISIBILITY bool
-operator==(const directory_iterator& __lhs,
-           const directory_iterator& __rhs) noexcept {
-  return __lhs.__imp_ == __rhs.__imp_;
-}
-
-inline _LIBCPP_INLINE_VISIBILITY bool
-operator!=(const directory_iterator& __lhs,
-           const directory_iterator& __rhs) noexcept {
-  return !(__lhs == __rhs);
-}
-
-// enable directory_iterator range-based for statements
-inline _LIBCPP_INLINE_VISIBILITY directory_iterator
-begin(directory_iterator __iter) noexcept {
-  return __iter;
-}
-
-inline _LIBCPP_INLINE_VISIBILITY directory_iterator
-end(directory_iterator) noexcept {
-  return directory_iterator();
-}
-
-class recursive_directory_iterator {
-public:
-  using value_type = directory_entry;
-  using difference_type = ptrdiff_t;
-  using pointer = directory_entry const*;
-  using reference = directory_entry const&;
-  using iterator_category = input_iterator_tag;
-
-public:
-  // constructors and destructor
-  _LIBCPP_INLINE_VISIBILITY
-  recursive_directory_iterator() noexcept : __rec_(false) {}
-
-  _LIBCPP_INLINE_VISIBILITY
-  explicit recursive_directory_iterator(
-      const path& __p, directory_options __xoptions = directory_options::none)
-      : recursive_directory_iterator(__p, __xoptions, nullptr) {}
-
-  _LIBCPP_INLINE_VISIBILITY
-  recursive_directory_iterator(const path& __p, directory_options __xoptions,
-                               error_code& __ec)
-      : recursive_directory_iterator(__p, __xoptions, &__ec) {}
-
-  _LIBCPP_INLINE_VISIBILITY
-  recursive_directory_iterator(const path& __p, error_code& __ec)
-      : recursive_directory_iterator(__p, directory_options::none, &__ec) {}
-
-  recursive_directory_iterator(const recursive_directory_iterator&) = default;
-  recursive_directory_iterator(recursive_directory_iterator&&) = default;
-
-  recursive_directory_iterator&
-  operator=(const recursive_directory_iterator&) = default;
-
-  _LIBCPP_INLINE_VISIBILITY
-  recursive_directory_iterator&
-  operator=(recursive_directory_iterator&& __o) noexcept {
-    // non-default implementation provided to support self-move assign.
-    if (this != &__o) {
-      __imp_ = _VSTD::move(__o.__imp_);
-      __rec_ = __o.__rec_;
-    }
-    return *this;
-  }
-
-  ~recursive_directory_iterator() = default;
-
-  _LIBCPP_INLINE_VISIBILITY
-  const directory_entry& operator*() const { return __dereference(); }
-
-  _LIBCPP_INLINE_VISIBILITY
-  const directory_entry* operator->() const { return &__dereference(); }
-
-  recursive_directory_iterator& operator++() { return __increment(); }
-
-  _LIBCPP_INLINE_VISIBILITY
-  __dir_element_proxy operator++(int) {
-    __dir_element_proxy __p(**this);
-    __increment();
-    return __p;
-  }
-
-  _LIBCPP_INLINE_VISIBILITY
-  recursive_directory_iterator& increment(error_code& __ec) {
-    return __increment(&__ec);
-  }
-
-  _LIBCPP_FUNC_VIS directory_options options() const;
-  _LIBCPP_FUNC_VIS int depth() const;
-
-  _LIBCPP_INLINE_VISIBILITY
-  void pop() { __pop(); }
-
-  _LIBCPP_INLINE_VISIBILITY
-  void pop(error_code& __ec) { __pop(&__ec); }
-
-  _LIBCPP_INLINE_VISIBILITY
-  bool recursion_pending() const { return __rec_; }
-
-  _LIBCPP_INLINE_VISIBILITY
-  void disable_recursion_pending() { __rec_ = false; }
-
-private:
-  _LIBCPP_FUNC_VIS
-  recursive_directory_iterator(const path& __p, directory_options __opt,
-                               error_code* __ec);
-
-  _LIBCPP_FUNC_VIS
-  const directory_entry& __dereference() const;
-
-  _LIBCPP_FUNC_VIS
-  bool __try_recursion(error_code* __ec);
-
-  _LIBCPP_FUNC_VIS
-  void __advance(error_code* __ec = nullptr);
-
-  _LIBCPP_FUNC_VIS
-  recursive_directory_iterator& __increment(error_code* __ec = nullptr);
-
-  _LIBCPP_FUNC_VIS
-  void __pop(error_code* __ec = nullptr);
-
-  inline _LIBCPP_INLINE_VISIBILITY friend bool
-  operator==(const recursive_directory_iterator&,
-             const recursive_directory_iterator&) noexcept;
-
-  struct _LIBCPP_HIDDEN __shared_imp;
-  shared_ptr<__shared_imp> __imp_;
-  bool __rec_;
-}; // class recursive_directory_iterator
-
-inline _LIBCPP_INLINE_VISIBILITY bool
-operator==(const recursive_directory_iterator& __lhs,
-           const recursive_directory_iterator& __rhs) noexcept {
-  return __lhs.__imp_ == __rhs.__imp_;
-}
-
-_LIBCPP_INLINE_VISIBILITY
-inline bool operator!=(const recursive_directory_iterator& __lhs,
-                       const recursive_directory_iterator& __rhs) noexcept {
-  return !(__lhs == __rhs);
-}
-// enable recursive_directory_iterator range-based for statements
-inline _LIBCPP_INLINE_VISIBILITY recursive_directory_iterator
-begin(recursive_directory_iterator __iter) noexcept {
-  return __iter;
-}
-
-inline _LIBCPP_INLINE_VISIBILITY recursive_directory_iterator
-end(recursive_directory_iterator) noexcept {
-  return recursive_directory_iterator();
-}
-
-_LIBCPP_AVAILABILITY_FILESYSTEM_POP
-
-_LIBCPP_END_NAMESPACE_FILESYSTEM
-
-#if !defined(_LIBCPP_HAS_NO_RANGES)
-template <>
-_LIBCPP_AVAILABILITY_FILESYSTEM
-inline constexpr bool _VSTD::ranges::enable_borrowed_range<_VSTD_FS::directory_iterator> = true;
-template <>
-_LIBCPP_AVAILABILITY_FILESYSTEM
-inline constexpr bool _VSTD::ranges::enable_borrowed_range<_VSTD_FS::recursive_directory_iterator> = true;
-
-template <>
-_LIBCPP_AVAILABILITY_FILESYSTEM
-inline constexpr bool _VSTD::ranges::enable_view<_VSTD_FS::directory_iterator> = true;
-template <>
-_LIBCPP_AVAILABILITY_FILESYSTEM
-inline constexpr bool _VSTD::ranges::enable_view<_VSTD_FS::recursive_directory_iterator> = true;
-#endif
-
-#endif // !_LIBCPP_CXX03_LANG
-
-_LIBCPP_POP_MACROS
-
 #endif // _LIBCPP_FILESYSTEM
diff --git a/libcxx/include/future b/libcxx/include/future
index 99df8831a778..6b666a70f48e 100644
--- a/libcxx/include/future
+++ b/libcxx/include/future
@@ -366,7 +366,7 @@ template <class R, class Alloc> struct uses_allocator<packaged_task<R>, Alloc>;
 #include <__debug>
 #include <__memory/allocator_arg_t.h>
 #include <__memory/uses_allocator.h>
-#include <__utility/decay_copy.h>
+#include <__utility/auto_cast.h>
 #include <__utility/forward.h>
 #include <chrono>
 #include <exception>
@@ -2207,16 +2207,16 @@ async(launch __policy, _Fp&& __f, _Args&&... __args)
     {
 #endif
         if (__does_policy_contain(__policy, launch::async))
-        return _VSTD::__make_async_assoc_state<_Rp>(_BF(_VSTD::__decay_copy(_VSTD::forward<_Fp>(__f)),
-                                                     _VSTD::__decay_copy(_VSTD::forward<_Args>(__args))...));
+        return _VSTD::__make_async_assoc_state<_Rp>(_BF(_LIBCPP_AUTO_CAST(_VSTD::forward<_Fp>(__f)),
+                                                        _LIBCPP_AUTO_CAST(_VSTD::forward<_Args>(__args))...));
 #ifndef _LIBCPP_NO_EXCEPTIONS
     }
     catch ( ... ) { if (__policy == launch::async) throw ; }
 #endif
 
     if (__does_policy_contain(__policy, launch::deferred))
-        return _VSTD::__make_deferred_assoc_state<_Rp>(_BF(_VSTD::__decay_copy(_VSTD::forward<_Fp>(__f)),
-                                                        _VSTD::__decay_copy(_VSTD::forward<_Args>(__args))...));
+        return _VSTD::__make_deferred_assoc_state<_Rp>(_BF(_LIBCPP_AUTO_CAST(_VSTD::forward<_Fp>(__f)),
+                                                           _LIBCPP_AUTO_CAST(_VSTD::forward<_Args>(__args))...));
     return future<_Rp>{};
 }
 
diff --git a/libcxx/include/initializer_list b/libcxx/include/initializer_list
index d867ee6af017..fefaf8cf8b6f 100644
--- a/libcxx/include/initializer_list
+++ b/libcxx/include/initializer_list
@@ -112,6 +112,6 @@ end(initializer_list<_Ep> __il) _NOEXCEPT
 
 #endif // !defined(_LIBCPP_CXX03_LANG)
 
-}  // std
+} // namespace std
 
 #endif // _LIBCPP_INITIALIZER_LIST
diff --git a/libcxx/include/ios b/libcxx/include/ios
index b7d32946bff0..237d146dfb85 100644
--- a/libcxx/include/ios
+++ b/libcxx/include/ios
@@ -308,11 +308,9 @@ public:
     typedef void (*event_callback)(event, ios_base&, int __index);
     void register_callback(event_callback __fn, int __index);
 
-private:
-    ios_base(const ios_base&); // = delete;
-    ios_base& operator=(const ios_base&); // = delete;
+    ios_base(const ios_base&) = delete;
+    ios_base& operator=(const ios_base&) = delete;
 
-public:
     static bool sync_with_stdio(bool __sync = true);
 
     _LIBCPP_INLINE_VISIBILITY iostate rdstate() const;
diff --git a/libcxx/include/istream b/libcxx/include/istream
index 2f39e35f4646..de49d49e4564 100644
--- a/libcxx/include/istream
+++ b/libcxx/include/istream
@@ -291,15 +291,15 @@ class _LIBCPP_TEMPLATE_VIS basic_istream<_CharT, _Traits>::sentry
 {
     bool __ok_;
 
-    sentry(const sentry&); // = delete;
-    sentry& operator=(const sentry&); // = delete;
-
 public:
     explicit sentry(basic_istream<_CharT, _Traits>& __is, bool __noskipws = false);
 //    ~sentry() = default;
 
     _LIBCPP_INLINE_VISIBILITY
     explicit operator bool() const {return __ok_;}
+
+    sentry(const sentry&) = delete;
+    sentry& operator=(const sentry&) = delete;
 };
 
 template <class _CharT, class _Traits>
diff --git a/libcxx/include/memory b/libcxx/include/memory
index f12f3c70eadd..3ed1530f2a75 100644
--- a/libcxx/include/memory
+++ b/libcxx/include/memory
@@ -188,10 +188,22 @@ uninitialized_copy_n(InputIterator first, Size n, ForwardIterator result);
 template <class ForwardIterator, class T>
 void uninitialized_fill(ForwardIterator first, ForwardIterator last, const T& x);
 
+template <nothrow-forward-iterator ForwardIterator, nothrow-sentinel-for<ForwardIterator> Sentinel, class T>
+  requires constructible_from<iter_value_t<ForwardIterator>, const T&>
+ForwardIterator ranges::uninitialized_fill(ForwardIterator first, Sentinel last, const T& x); // since C++20
+
+template <nothrow-forward-range ForwardRange, class T>
+  requires constructible_from<range_value_t<ForwardRange>, const T&>
+borrowed_iterator_t<ForwardRange> ranges::uninitialized_fill(ForwardRange&& range, const T& x); // since C++20
+
 template <class ForwardIterator, class Size, class T>
 ForwardIterator
 uninitialized_fill_n(ForwardIterator first, Size n, const T& x);
 
+template <nothrow-forward-iterator ForwardIterator, class T>
+  requires constructible_from<iter_value_t<ForwardIterator>, const T&>
+ForwardIterator ranges::uninitialized_fill_n(ForwardIterator first, iter_difference_t<ForwardIterator> n); // since C++20
+
 template <class T, class ...Args>
 constexpr T* construct_at(T* location, Args&& ...args); // since C++20
 
@@ -213,15 +225,39 @@ template <class InputIterator, class Size, class ForwardIterator>
 template <class ForwardIterator>
  void uninitialized_value_construct(ForwardIterator first, ForwardIterator last);
 
+template <nothrow-forward-iterator ForwardIterator, nothrow-sentinel-for<ForwardIterator> Sentinel>
+  requires default_initializable<iter_value_t<ForwardIterator>>
+ ForwardIterator ranges::uninitialized_value_construct(ForwardIterator first, Sentinel last); // since C++20
+
+template <nothrow-forward-range ForwardRange>
+  requires default_initializable<range_value_t<ForwardRange>>
+ borrowed_iterator_t<ForwardRange> ranges::uninitialized_value_construct(ForwardRange&& r); // since C++20
+
 template <class ForwardIterator, class Size>
  ForwardIterator uninitialized_value_construct_n(ForwardIterator first, Size n);
 
+template <nothrow-forward-iterator ForwardIterator>
+  requires default_initializable<iter_value_t<ForwardIterator>>
+ ForwardIterator ranges::uninitialized_value_construct_n(ForwardIterator first, iter_difference_t<ForwardIterator> n); // since C++20
+
 template <class ForwardIterator>
  void uninitialized_default_construct(ForwardIterator first, ForwardIterator last);
 
+template <nothrow-forward-iterator ForwardIterator, nothrow-sentinel-for<ForwardIterator> Sentinel>
+  requires default_initializable<iter_value_t<ForwardIterator>>
+ ForwardIterator ranges::uninitialized_default_construct(ForwardIterator first, Sentinel last); // since C++20
+
+template <nothrow-forward-range ForwardRange>
+  requires default_initializable<range_value_t<ForwardRange>>
+ borrowed_iterator_t<ForwardRange> ranges::uninitialized_default_construct(ForwardRange&& r); // since C++20
+
 template <class ForwardIterator, class Size>
  ForwardIterator uninitialized_default_construct_n(ForwardIterator first, Size n);
 
+template <nothrow-forward-iterator ForwardIterator>
+  requires default_initializable<iter_value_t<ForwardIterator>>
+ ForwardIterator ranges::uninitialized_default_construct_n(ForwardIterator first, iter_difference_t<ForwardIterator> n); // since C++20
+
 template <class Y> struct auto_ptr_ref {};      // deprecated in C++11, removed in C++17
 
 template<class X>
@@ -669,8 +705,10 @@ void* align(size_t alignment, size_t size, void*& ptr, size_t& space);
 #include <__memory/allocator_arg_t.h>
 #include <__memory/allocator_traits.h>
 #include <__memory/compressed_pair.h>
+#include <__memory/concepts.h>
 #include <__memory/construct_at.h>
 #include <__memory/pointer_traits.h>
+#include <__memory/ranges_uninitialized_algorithms.h>
 #include <__memory/raw_storage_iterator.h>
 #include <__memory/shared_ptr.h>
 #include <__memory/temporary_buffer.h>
@@ -831,9 +869,9 @@ _LIBCPP_FUNC_VIS void* align(size_t __align, size_t __sz, void*& __ptr, size_t&
 
 // --- Helper for container swap --
 template <typename _Alloc>
-_LIBCPP_INLINE_VISIBILITY
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX11
 void __swap_allocator(_Alloc & __a1, _Alloc & __a2, true_type)
-#if _LIBCPP_STD_VER >= 14
+#if _LIBCPP_STD_VER > 11
     _NOEXCEPT
 #else
     _NOEXCEPT_(__is_nothrow_swappable<_Alloc>::value)
@@ -844,13 +882,13 @@ void __swap_allocator(_Alloc & __a1, _Alloc & __a2, true_type)
 }
 
 template <typename _Alloc>
-inline _LIBCPP_INLINE_VISIBILITY
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX11
 void __swap_allocator(_Alloc &, _Alloc &, false_type) _NOEXCEPT {}
 
 template <typename _Alloc>
-inline _LIBCPP_INLINE_VISIBILITY
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX11
 void __swap_allocator(_Alloc & __a1, _Alloc & __a2)
-#if _LIBCPP_STD_VER >= 14
+#if _LIBCPP_STD_VER > 11
     _NOEXCEPT
 #else
     _NOEXCEPT_(__is_nothrow_swappable<_Alloc>::value)
diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap
index a4a264bd9147..1dc6db406a74 100644
--- a/libcxx/include/module.modulemap
+++ b/libcxx/include/module.modulemap
@@ -353,9 +353,9 @@ module std [system] {
     export *
 
     module __charconv {
-      module chars_format      { private header "__charconv/chars_format.h"      }
+      module chars_format      { private header "__charconv/chars_format.h" }
       module from_chars_result { private header "__charconv/from_chars_result.h" }
-      module to_chars_result   { private header "__charconv/to_chars_result.h"   }
+      module to_chars_result   { private header "__charconv/to_chars_result.h" }
     }
 
   }
@@ -373,15 +373,15 @@ module std [system] {
 
     module __compare {
       module common_comparison_category { private header "__compare/common_comparison_category.h" }
-      module compare_three_way          { private header "__compare/compare_three_way.h"          }
-      module compare_three_way_result   { private header "__compare/compare_three_way_result.h"   }
-      module is_eq                      { private header "__compare/is_eq.h"                      }
-      module ordering                   { private header "__compare/ordering.h"                   }
-      module partial_order              { private header "__compare/partial_order.h"              }
-      module strong_order               { private header "__compare/strong_order.h"               }
-      module synth_three_way            { private header "__compare/synth_three_way.h"            }
-      module three_way_comparable       { private header "__compare/three_way_comparable.h"       }
-      module weak_order                 { private header "__compare/weak_order.h"                 }
+      module compare_three_way          { private header "__compare/compare_three_way.h" }
+      module compare_three_way_result   { private header "__compare/compare_three_way_result.h" }
+      module is_eq                      { private header "__compare/is_eq.h" }
+      module ordering                   { private header "__compare/ordering.h" }
+      module partial_order              { private header "__compare/partial_order.h" }
+      module strong_order               { private header "__compare/strong_order.h" }
+      module synth_three_way            { private header "__compare/synth_three_way.h" }
+      module three_way_comparable       { private header "__compare/three_way_comparable.h" }
+      module weak_order                 { private header "__compare/weak_order.h" }
     }
   }
   module complex {
@@ -393,28 +393,28 @@ module std [system] {
     export *
 
     module __concepts {
-      module arithmetic                 { private header "__concepts/arithmetic.h"                }
-      module assignable                 { private header "__concepts/assignable.h"                }
-      module boolean_testable           { private header "__concepts/boolean_testable.h"          }
-      module class_or_enum              { private header "__concepts/class_or_enum.h"             }
-      module common_reference_with      { private header "__concepts/common_reference_with.h"     }
-      module common_with                { private header "__concepts/common_with.h"               }
-      module constructible              { private header "__concepts/constructible.h"             }
-      module convertible_to             { private header "__concepts/convertible_to.h"            }
-      module copyable                   { private header "__concepts/copyable.h"                  }
-      module derived_from               { private header "__concepts/derived_from.h"              }
-      module destructible               { private header "__concepts/destructible.h"              }
-      module different_from             { private header "__concepts/different_from.h"            }
-      module equality_comparable        { private header "__concepts/equality_comparable.h"       }
-      module invocable                  { private header "__concepts/invocable.h"                 }
-      module movable                    { private header "__concepts/movable.h"                   }
-      module predicate                  { private header "__concepts/predicate.h"                 }
-      module regular                    { private header "__concepts/regular.h"                   }
-      module relation                   { private header "__concepts/relation.h"                  }
-      module same_as                    { private header "__concepts/same_as.h"                   }
-      module semiregular                { private header "__concepts/semiregular.h"               }
-      module swappable                  { private header "__concepts/swappable.h"                 }
-      module totally_ordered            { private header "__concepts/totally_ordered.h"           }
+      module arithmetic                 { private header "__concepts/arithmetic.h" }
+      module assignable                 { private header "__concepts/assignable.h" }
+      module boolean_testable           { private header "__concepts/boolean_testable.h" }
+      module class_or_enum              { private header "__concepts/class_or_enum.h" }
+      module common_reference_with      { private header "__concepts/common_reference_with.h" }
+      module common_with                { private header "__concepts/common_with.h" }
+      module constructible              { private header "__concepts/constructible.h" }
+      module convertible_to             { private header "__concepts/convertible_to.h" }
+      module copyable                   { private header "__concepts/copyable.h" }
+      module derived_from               { private header "__concepts/derived_from.h" }
+      module destructible               { private header "__concepts/destructible.h" }
+      module different_from             { private header "__concepts/different_from.h" }
+      module equality_comparable        { private header "__concepts/equality_comparable.h" }
+      module invocable                  { private header "__concepts/invocable.h" }
+      module movable                    { private header "__concepts/movable.h" }
+      module predicate                  { private header "__concepts/predicate.h" }
+      module regular                    { private header "__concepts/regular.h" }
+      module relation                   { private header "__concepts/relation.h" }
+      module same_as                    { private header "__concepts/same_as.h" }
+      module semiregular                { private header "__concepts/semiregular.h" }
+      module swappable                  { private header "__concepts/swappable.h" }
+      module totally_ordered            { private header "__concepts/totally_ordered.h" }
     }
   }
   module condition_variable {
@@ -428,10 +428,10 @@ module std [system] {
     export *
 
     module __coroutine {
-      module coroutine_handle             { private header "__coroutine/coroutine_handle.h"             }
-      module coroutine_traits             { private header "__coroutine/coroutine_traits.h"             }
-      module trivial_awaitables           { private header "__coroutine/trivial_awaitables.h"           }
-      module noop_coroutine_handle        { private header "__coroutine/noop_coroutine_handle.h"        }
+      module coroutine_handle           { private header "__coroutine/coroutine_handle.h" }
+      module coroutine_traits           { private header "__coroutine/coroutine_traits.h" }
+      module trivial_awaitables         { private header "__coroutine/trivial_awaitables.h" }
+      module noop_coroutine_handle      { private header "__coroutine/noop_coroutine_handle.h" }
     }
   }
   module deque {
@@ -450,30 +450,49 @@ module std [system] {
   module filesystem {
     header "filesystem"
     export *
+
+    module __filesystem {
+      module copy_options                 { private header "__filesystem/copy_options.h" }
+      module directory_entry              { private header "__filesystem/directory_entry.h" }
+      module directory_iterator           { private header "__filesystem/directory_iterator.h" }
+      module directory_options            { private header "__filesystem/directory_options.h" }
+      module file_status                  { private header "__filesystem/file_status.h" }
+      module file_time_type               { private header "__filesystem/file_time_type.h" }
+      module file_type                    { private header "__filesystem/file_type.h" }
+      module filesystem_error             { private header "__filesystem/filesystem_error.h" }
+      module operations                   { private header "__filesystem/operations.h" }
+      module path_iterator                { private header "__filesystem/path_iterator.h" }
+      module path                         { private header "__filesystem/path.h" }
+      module perm_options                 { private header "__filesystem/perm_options.h" }
+      module perms                        { private header "__filesystem/perms.h" }
+      module recursive_directory_iterator { private header "__filesystem/recursive_directory_iterator.h" }
+      module space_info                   { private header "__filesystem/space_info.h" }
+      module u8path                       { private header "__filesystem/u8path.h" }
+    }
   }
   module format {
     header "format"
     export *
 
     module __format {
-      module format_arg             { private header "__format/format_arg.h"           }
-      module format_args            { private header "__format/format_args.h"          }
+      module format_arg             { private header "__format/format_arg.h" }
+      module format_args            { private header "__format/format_args.h" }
       module format_context {
         private header "__format/format_context.h"
         export optional
         export locale
       }
-      module format_error           { private header "__format/format_error.h"           }
-      module format_fwd             { private header "__format/format_fwd.h"             }
-      module format_parse_context   { private header "__format/format_parse_context.h"   }
-      module format_string          { private header "__format/format_string.h"          }
-      module format_to_n_result     { private header "__format/format_to_n_result.h"     }
-      module formatter              { private header "__format/formatter.h"              }
-      module formatter_bool         { private header "__format/formatter_bool.h"         }
-      module formatter_char         { private header "__format/formatter_char.h"         }
-      module formatter_integer      { private header "__format/formatter_integer.h"      }
-      module formatter_integral     { private header "__format/formatter_integral.h"     }
-      module formatter_string       { private header "__format/formatter_string.h"       }
+      module format_error           { private header "__format/format_error.h" }
+      module format_fwd             { private header "__format/format_fwd.h" }
+      module format_parse_context   { private header "__format/format_parse_context.h" }
+      module format_string          { private header "__format/format_string.h" }
+      module format_to_n_result     { private header "__format/format_to_n_result.h" }
+      module formatter              { private header "__format/formatter.h" }
+      module formatter_bool         { private header "__format/formatter_bool.h" }
+      module formatter_char         { private header "__format/formatter_char.h" }
+      module formatter_integer      { private header "__format/formatter_integer.h" }
+      module formatter_integral     { private header "__format/formatter_integral.h" }
+      module formatter_string       { private header "__format/formatter_string.h" }
       module parser_std_format_spec { private header "__format/parser_std_format_spec.h" }
     }
   }
@@ -505,8 +524,8 @@ module std [system] {
       module identity                   { private header "__functional/identity.h" }
       module invoke                     { private header "__functional/invoke.h" }
       module is_transparent             { private header "__functional/is_transparent.h" }
-      module mem_fn                     { private header "__functional/mem_fn.h"  }
-      module mem_fun_ref                { private header "__functional/mem_fun_ref.h"  }
+      module mem_fn                     { private header "__functional/mem_fn.h" }
+      module mem_fun_ref                { private header "__functional/mem_fun_ref.h" }
       module not_fn                     { private header "__functional/not_fn.h" }
       module operations                 { private header "__functional/operations.h" }
       module perfect_forward            { private header "__functional/perfect_forward.h" }
@@ -630,21 +649,27 @@ module std [system] {
     export *
 
     module __memory {
-      module addressof                { private header "__memory/addressof.h"                }
-      module allocation_guard         { private header "__memory/allocation_guard.h"         }
-      module allocator                { private header "__memory/allocator.h"                }
-      module allocator_arg_t          { private header "__memory/allocator_arg_t.h"          }
-      module allocator_traits         { private header "__memory/allocator_traits.h"         }
-      module auto_ptr                 { private header "__memory/auto_ptr.h"                 }
-      module compressed_pair          { private header "__memory/compressed_pair.h"          }
-      module construct_at             { private header "__memory/construct_at.h"             }
-      module pointer_traits           { private header "__memory/pointer_traits.h"           }
-      module raw_storage_iterator     { private header "__memory/raw_storage_iterator.h"     }
-      module shared_ptr               { private header "__memory/shared_ptr.h"               }
-      module temporary_buffer         { private header "__memory/temporary_buffer.h"         }
-      module uninitialized_algorithms { private header "__memory/uninitialized_algorithms.h" }
-      module unique_ptr               { private header "__memory/unique_ptr.h"               }
-      module uses_allocator           { private header "__memory/uses_allocator.h"           }
+      module addressof                       { private header "__memory/addressof.h" }
+      module allocation_guard                { private header "__memory/allocation_guard.h" }
+      module allocator                       { private header "__memory/allocator.h" }
+      module allocator_arg_t                 { private header "__memory/allocator_arg_t.h" }
+      module allocator_traits                { private header "__memory/allocator_traits.h" }
+      module auto_ptr                        { private header "__memory/auto_ptr.h" }
+      module compressed_pair                 { private header "__memory/compressed_pair.h" }
+      module concepts                        { private header "__memory/concepts.h" }
+      module construct_at                    { private header "__memory/construct_at.h" }
+      module pointer_traits                  { private header "__memory/pointer_traits.h" }
+      module ranges_uninitialized_algorithms {
+        private header "__memory/ranges_uninitialized_algorithms.h"
+        export __function_like
+      }
+      module raw_storage_iterator            { private header "__memory/raw_storage_iterator.h" }
+      module shared_ptr                      { private header "__memory/shared_ptr.h" }
+      module temporary_buffer                { private header "__memory/temporary_buffer.h" }
+      module uninitialized_algorithms        { private header "__memory/uninitialized_algorithms.h" }
+      module unique_ptr                      { private header "__memory/unique_ptr.h" }
+      module uses_allocator                  { private header "__memory/uses_allocator.h" }
+      module voidify                         { private header "__memory/voidify.h" }
     }
   }
   module mutex {
@@ -699,41 +724,42 @@ module std [system] {
     export *
 
     module __random {
-      module bernoulli_distribution          { private header "__random/bernoulli_distribution.h"          }
-      module binomial_distribution           { private header "__random/binomial_distribution.h"           }
-      module cauchy_distribution             { private header "__random/cauchy_distribution.h"             }
-      module chi_squared_distribution        { private header "__random/chi_squared_distribution.h"        }
-      module default_random_engine           { private header "__random/default_random_engine.h"           }
-      module discard_block_engine            { private header "__random/discard_block_engine.h"            }
-      module discrete_distribution           { private header "__random/discrete_distribution.h"           }
-      module exponential_distribution        { private header "__random/exponential_distribution.h"        }
-      module extreme_value_distribution      { private header "__random/extreme_value_distribution.h"      }
-      module fisher_f_distribution           { private header "__random/fisher_f_distribution.h"           }
-      module gamma_distribution              { private header "__random/gamma_distribution.h"              }
-      module generate_canonical              { private header "__random/generate_canonical.h"              }
-      module geometric_distribution          { private header "__random/geometric_distribution.h"          }
-      module independent_bits_engine         { private header "__random/independent_bits_engine.h"         }
-      module is_seed_sequence                { private header "__random/is_seed_sequence.h"                }
-      module knuth_b                         { private header "__random/knuth_b.h"                         }
-      module linear_congruential_engine      { private header "__random/linear_congruential_engine.h"      }
-      module log2                            { private header "__random/log2.h"                            }
-      module lognormal_distribution          { private header "__random/lognormal_distribution.h"          }
-      module mersenne_twister_engine         { private header "__random/mersenne_twister_engine.h"         }
-      module negative_binomial_distribution  { private header "__random/negative_binomial_distribution.h"  }
-      module normal_distribution             { private header "__random/normal_distribution.h"             }
+      module bernoulli_distribution          { private header "__random/bernoulli_distribution.h" }
+      module binomial_distribution           { private header "__random/binomial_distribution.h" }
+      module cauchy_distribution             { private header "__random/cauchy_distribution.h" }
+      module chi_squared_distribution        { private header "__random/chi_squared_distribution.h" }
+      module clamp_to_integral               { private header "__random/clamp_to_integral.h" }
+      module default_random_engine           { private header "__random/default_random_engine.h" }
+      module discard_block_engine            { private header "__random/discard_block_engine.h" }
+      module discrete_distribution           { private header "__random/discrete_distribution.h" }
+      module exponential_distribution        { private header "__random/exponential_distribution.h" }
+      module extreme_value_distribution      { private header "__random/extreme_value_distribution.h" }
+      module fisher_f_distribution           { private header "__random/fisher_f_distribution.h" }
+      module gamma_distribution              { private header "__random/gamma_distribution.h" }
+      module generate_canonical              { private header "__random/generate_canonical.h" }
+      module geometric_distribution          { private header "__random/geometric_distribution.h" }
+      module independent_bits_engine         { private header "__random/independent_bits_engine.h" }
+      module is_seed_sequence                { private header "__random/is_seed_sequence.h" }
+      module knuth_b                         { private header "__random/knuth_b.h" }
+      module linear_congruential_engine      { private header "__random/linear_congruential_engine.h" }
+      module log2                            { private header "__random/log2.h" }
+      module lognormal_distribution          { private header "__random/lognormal_distribution.h" }
+      module mersenne_twister_engine         { private header "__random/mersenne_twister_engine.h" }
+      module negative_binomial_distribution  { private header "__random/negative_binomial_distribution.h" }
+      module normal_distribution             { private header "__random/normal_distribution.h" }
       module piecewise_constant_distribution { private header "__random/piecewise_constant_distribution.h" }
-      module piecewise_linear_distribution   { private header "__random/piecewise_linear_distribution.h"   }
-      module poisson_distribution            { private header "__random/poisson_distribution.h"            }
-      module random_device                   { private header "__random/random_device.h"                   }
-      module ranlux                          { private header "__random/ranlux.h"                          }
-      module seed_seq                        { private header "__random/seed_seq.h"                        }
-      module shuffle_order_engine            { private header "__random/shuffle_order_engine.h"            }
-      module student_t_distribution          { private header "__random/student_t_distribution.h"          }
-      module subtract_with_carry_engine      { private header "__random/subtract_with_carry_engine.h"      }
-      module uniform_int_distribution        { private header "__random/uniform_int_distribution.h"        }
-      module uniform_random_bit_generator    { private header "__random/uniform_random_bit_generator.h"    }
-      module uniform_real_distribution       { private header "__random/uniform_real_distribution.h"       }
-      module weibull_distribution            { private header "__random/weibull_distribution.h"            }
+      module piecewise_linear_distribution   { private header "__random/piecewise_linear_distribution.h" }
+      module poisson_distribution            { private header "__random/poisson_distribution.h" }
+      module random_device                   { private header "__random/random_device.h" }
+      module ranlux                          { private header "__random/ranlux.h" }
+      module seed_seq                        { private header "__random/seed_seq.h" }
+      module shuffle_order_engine            { private header "__random/shuffle_order_engine.h" }
+      module student_t_distribution          { private header "__random/student_t_distribution.h" }
+      module subtract_with_carry_engine      { private header "__random/subtract_with_carry_engine.h" }
+      module uniform_int_distribution        { private header "__random/uniform_int_distribution.h" }
+      module uniform_random_bit_generator    { private header "__random/uniform_random_bit_generator.h" }
+      module uniform_real_distribution       { private header "__random/uniform_real_distribution.h" }
+      module weibull_distribution            { private header "__random/weibull_distribution.h" }
     }
   }
   module ranges {
@@ -744,39 +770,42 @@ module std [system] {
     export *
 
     module __ranges {
-      module access                 { private header "__ranges/access.h"                }
+      module access                 { private header "__ranges/access.h" }
       module all                    {
         private header "__ranges/all.h"
         export functional.__functional.compose
         export functional.__functional.perfect_forward
       }
-      module common_view            { private header "__ranges/common_view.h"           }
-      module concepts               { private header "__ranges/concepts.h"              }
-      module copyable_box           { private header "__ranges/copyable_box.h"          }
-      module counted                { private header "__ranges/counted.h"               }
-      module dangling               { private header "__ranges/dangling.h"              }
-      module data                   { private header "__ranges/data.h"                  }
-      module drop_view              { private header "__ranges/drop_view.h"             }
-      module empty                  { private header "__ranges/empty.h"                 }
-      module empty_view             { private header "__ranges/empty_view.h"            }
+      module common_view            { private header "__ranges/common_view.h" }
+      module concepts               { private header "__ranges/concepts.h" }
+      module copyable_box           { private header "__ranges/copyable_box.h" }
+      module counted                {
+        private header "__ranges/counted.h"
+        export span
+      }
+      module dangling               { private header "__ranges/dangling.h" }
+      module data                   { private header "__ranges/data.h" }
+      module drop_view              { private header "__ranges/drop_view.h" }
+      module empty                  { private header "__ranges/empty.h" }
+      module empty_view             { private header "__ranges/empty_view.h" }
       module enable_borrowed_range  { private header "__ranges/enable_borrowed_range.h" }
-      module enable_view            { private header "__ranges/enable_view.h"           }
-      module iota_view              { private header "__ranges/iota_view.h"             }
-      module join_view              { private header "__ranges/join_view.h"             }
+      module enable_view            { private header "__ranges/enable_view.h" }
+      module iota_view              { private header "__ranges/iota_view.h" }
+      module join_view              { private header "__ranges/join_view.h" }
       module non_propagating_cache  { private header "__ranges/non_propagating_cache.h" }
-      module range_adaptor          { private header "__ranges/range_adaptor.h"         }
-      module ref_view               { private header "__ranges/ref_view.h"              }
-      module reverse_view           { private header "__ranges/reverse_view.h"          }
-      module size                   { private header "__ranges/size.h"                  }
-      module single_view            { private header "__ranges/single_view.h"           }
-      module subrange               { private header "__ranges/subrange.h"              }
-      module take_view              { private header "__ranges/take_view.h"             }
+      module range_adaptor          { private header "__ranges/range_adaptor.h" }
+      module ref_view               { private header "__ranges/ref_view.h" }
+      module reverse_view           { private header "__ranges/reverse_view.h" }
+      module size                   { private header "__ranges/size.h" }
+      module single_view            { private header "__ranges/single_view.h" }
+      module subrange               { private header "__ranges/subrange.h" }
+      module take_view              { private header "__ranges/take_view.h" }
       module transform_view         {
         private header "__ranges/transform_view.h"
         export functional.__functional.bind_back
         export functional.__functional.perfect_forward
       }
-      module view_interface         { private header "__ranges/view_interface.h"        }
+      module view_interface         { private header "__ranges/view_interface.h" }
     }
   }
   module ratio {
@@ -891,21 +920,22 @@ module std [system] {
     export *
 
     module __utility {
-      module as_const            { private header "__utility/as_const.h"            }
-      module cmp                 { private header "__utility/cmp.h"                 }
-      module decay_copy          { private header "__utility/decay_copy.h"          }
-      module declval             { private header "__utility/declval.h"             }
-      module exchange            { private header "__utility/exchange.h"            }
-      module forward             { private header "__utility/forward.h"             }
-      module in_place            { private header "__utility/in_place.h"            }
-      module integer_sequence    { private header "__utility/integer_sequence.h"    }
-      module move                { private header "__utility/move.h"                }
-      module pair                { private header "__utility/pair.h"                }
+      module as_const            { private header "__utility/as_const.h" }
+      module auto_cast           { private header "__utility/auto_cast.h" }
+      module cmp                 { private header "__utility/cmp.h" }
+      module declval             { private header "__utility/declval.h" }
+      module exchange            { private header "__utility/exchange.h" }
+      module forward             { private header "__utility/forward.h" }
+      module in_place            { private header "__utility/in_place.h" }
+      module integer_sequence    { private header "__utility/integer_sequence.h" }
+      module move                { private header "__utility/move.h" }
+      module pair                { private header "__utility/pair.h" }
       module piecewise_construct { private header "__utility/piecewise_construct.h" }
-      module priority_tag        { private header "__utility/priority_tag.h"        }
-      module rel_ops             { private header "__utility/rel_ops.h"             }
-      module swap                { private header "__utility/swap.h"                }
-      module to_underlying       { private header "__utility/to_underlying.h"       }
+      module priority_tag        { private header "__utility/priority_tag.h" }
+      module rel_ops             { private header "__utility/rel_ops.h" }
+      module swap                { private header "__utility/swap.h" }
+      module to_underlying       { private header "__utility/to_underlying.h" }
+      module transaction         { private header "__utility/transaction.h" }
     }
   }
   module valarray {
@@ -936,22 +966,22 @@ module std [system] {
   module __availability      { private header "__availability"      export * }
   module __bit_reference     { private header "__bit_reference"     export * }
   module __bits              { private header "__bits"              export * }
-  module __debug             { header "__debug"             export * }
+  module __debug             {         header "__debug"             export * }
   module __errc              { private header "__errc"              export * }
   module __function_like     { private header "__function_like.h"   export * }
-  module __hash_table        { header "__hash_table"        export * }
+  module __hash_table        {         header "__hash_table"        export * }
   module __locale            { private header "__locale"            export * }
   module __mbstate           { private header "__mbstate_t.h"       export * }
   module __mutex_base        { private header "__mutex_base"        export * }
   module __node_handle       { private header "__node_handle"       export * }
-  module __nullptr           { header "__nullptr"           export * }
+  module __nullptr           {         header "__nullptr"           export * }
   module __split_buffer      { private header "__split_buffer"      export * }
   module __std_stream        { private header "__std_stream"        export * }
   module __string            { private header "__string"            export * }
-  module __threading_support { header "__threading_support" export * }
-  module __tree              { header "__tree"              export * }
+  module __threading_support {         header "__threading_support" export * }
+  module __tree              {         header "__tree"              export * }
   module __tuple             { private header "__tuple"             export * }
-  module __undef_macros      { header "__undef_macros"      export * }
+  module __undef_macros      {         header "__undef_macros"      export * }
 
   module experimental {
     requires cplusplus11
diff --git a/libcxx/include/mutex b/libcxx/include/mutex
index 822abe1a3f17..b2a4d7993e0d 100644
--- a/libcxx/include/mutex
+++ b/libcxx/include/mutex
@@ -215,14 +215,12 @@ class _LIBCPP_TYPE_VIS recursive_mutex
     __libcpp_recursive_mutex_t __m_;
 
 public:
-     recursive_mutex();
-     ~recursive_mutex();
+    recursive_mutex();
+    ~recursive_mutex();
 
-private:
-    recursive_mutex(const recursive_mutex&); // = delete;
-    recursive_mutex& operator=(const recursive_mutex&); // = delete;
+    recursive_mutex(const recursive_mutex&) = delete;
+    recursive_mutex& operator=(const recursive_mutex&) = delete;
 
-public:
     void lock();
     bool try_lock() _NOEXCEPT;
     void unlock()  _NOEXCEPT;
@@ -242,9 +240,8 @@ public:
      timed_mutex();
      ~timed_mutex();
 
-private:
-    timed_mutex(const timed_mutex&); // = delete;
-    timed_mutex& operator=(const timed_mutex&); // = delete;
+    timed_mutex(const timed_mutex&) = delete;
+    timed_mutex& operator=(const timed_mutex&) = delete;
 
 public:
     void lock();
@@ -283,14 +280,12 @@ class _LIBCPP_TYPE_VIS recursive_timed_mutex
     size_t             __count_;
     __thread_id        __id_;
 public:
-     recursive_timed_mutex();
-     ~recursive_timed_mutex();
+    recursive_timed_mutex();
+    ~recursive_timed_mutex();
 
-private:
-    recursive_timed_mutex(const recursive_timed_mutex&); // = delete;
-    recursive_timed_mutex& operator=(const recursive_timed_mutex&); // = delete;
+    recursive_timed_mutex(const recursive_timed_mutex&) = delete;
+    recursive_timed_mutex& operator=(const recursive_timed_mutex&) = delete;
 
-public:
     void lock();
     bool try_lock() _NOEXCEPT;
     template <class _Rep, class _Period>
@@ -576,6 +571,8 @@ struct _LIBCPP_TEMPLATE_VIS once_flag
     _LIBCPP_INLINE_VISIBILITY
     _LIBCPP_CONSTEXPR
         once_flag() _NOEXCEPT : __state_(0) {}
+    once_flag(const once_flag&) = delete;
+    once_flag& operator=(const once_flag&) = delete;
 
 #if defined(_LIBCPP_ABI_MICROSOFT)
    typedef uintptr_t _State_type;
@@ -583,11 +580,7 @@ struct _LIBCPP_TEMPLATE_VIS once_flag
    typedef unsigned long _State_type;
 #endif
 
-
 private:
-    once_flag(const once_flag&); // = delete;
-    once_flag& operator=(const once_flag&); // = delete;
-
     _State_type __state_;
 
 #ifndef _LIBCPP_CXX03_LANG
diff --git a/libcxx/include/new b/libcxx/include/new
index 593af9d5c619..be0d972f42da 100644
--- a/libcxx/include/new
+++ b/libcxx/include/new
@@ -177,7 +177,7 @@ struct destroying_delete_t {
 inline constexpr destroying_delete_t destroying_delete{};
 #endif // _LIBCPP_STD_VER > 17
 
-}  // std
+} // namespace std
 
 #if defined(_LIBCPP_CXX03_LANG)
 #define _THROW_BAD_ALLOC throw(std::bad_alloc)
diff --git a/libcxx/include/optional b/libcxx/include/optional
index 837f867328fb..63753d9f9f03 100644
--- a/libcxx/include/optional
+++ b/libcxx/include/optional
@@ -132,6 +132,18 @@ namespace std {
     template <class U> constexpr T value_or(U &&) const &;
     template <class U> constexpr T value_or(U &&) &&;
 
+    // [optional.monadic], monadic operations
+    template<class F> constexpr auto and_then(F&& f) &;         // since C++23
+    template<class F> constexpr auto and_then(F&& f) &&;        // since C++23
+    template<class F> constexpr auto and_then(F&& f) const&;    // since C++23
+    template<class F> constexpr auto and_then(F&& f) const&&;   // since C++23
+    template<class F> constexpr auto transform(F&& f) &;        // since C++23
+    template<class F> constexpr auto transform(F&& f) &&;       // since C++23
+    template<class F> constexpr auto transform(F&& f) const&;   // since C++23
+    template<class F> constexpr auto transform(F&& f) const&&;  // since C++23
+    template<class F> constexpr optional or_else(F&& f) &&;     // since C++23
+    template<class F> constexpr optional or_else(F&& f) const&; // since C++23
+
     // 23.6.3.6, modifiers
     void reset() noexcept; // constexpr in C++20
 
@@ -147,6 +159,7 @@ template<class T>
 */
 
 #include <__availability>
+#include <__concepts/invocable.h>
 #include <__config>
 #include <__debug>
 #include <__functional_base>
@@ -175,7 +188,7 @@ public:
     virtual const char* what() const _NOEXCEPT;
 };
 
-}  // std
+} // namespace std
 
 #if _LIBCPP_STD_VER > 14
 
@@ -200,6 +213,8 @@ struct nullopt_t
 
 inline constexpr nullopt_t nullopt{nullopt_t::__secret_tag{}, nullopt_t::__secret_tag{}};
 
+struct __optional_construct_from_invoke_tag {};
+
 template <class _Tp, bool = is_trivially_destructible<_Tp>::value>
 struct __optional_destruct_base;
 
@@ -234,6 +249,13 @@ struct __optional_destruct_base<_Tp, false>
         :  __val_(_VSTD::forward<_Args>(__args)...),
            __engaged_(true) {}
 
+#if _LIBCPP_STD_VER > 20
+  template <class _Fp, class... _Args>
+  _LIBCPP_HIDE_FROM_ABI
+  constexpr __optional_destruct_base(__optional_construct_from_invoke_tag, _Fp&& __f, _Args&&... __args)
+      : __val_(_VSTD::invoke(_VSTD::forward<_Fp>(__f), _VSTD::forward<_Args>(__args)...)), __engaged_(true) {}
+#endif
+
     _LIBCPP_INLINE_VISIBILITY
     _LIBCPP_CONSTEXPR_AFTER_CXX17 void reset() noexcept
     {
@@ -269,6 +291,13 @@ struct __optional_destruct_base<_Tp, true>
         :  __val_(_VSTD::forward<_Args>(__args)...),
            __engaged_(true) {}
 
+#if _LIBCPP_STD_VER > 20
+  template <class _Fp, class... _Args>
+  _LIBCPP_HIDE_FROM_ABI
+  constexpr __optional_destruct_base(__optional_construct_from_invoke_tag, _Fp&& __f, _Args&&... __args)
+      : __val_(_VSTD::invoke(_VSTD::forward<_Fp>(__f), _VSTD::forward<_Args>(__args)...)), __engaged_(true) {}
+#endif
+
     _LIBCPP_INLINE_VISIBILITY
     _LIBCPP_CONSTEXPR_AFTER_CXX17 void reset() noexcept
     {
@@ -582,6 +611,12 @@ using __optional_sfinae_assign_base_t = __sfinae_assign_base<
     (is_move_constructible<_Tp>::value && is_move_assignable<_Tp>::value)
 >;
 
+template<class _Tp>
+class optional;
+template <class _Tp>
+struct __is_std_optional : false_type {};
+template <class _Tp> struct __is_std_optional<optional<_Tp>> : true_type {};
+
 template <class _Tp>
 class optional
     : private __optional_move_assign_base<_Tp>
@@ -684,6 +719,7 @@ private:
       _CheckOptionalLikeConstructor<_QualUp>,
       __check_tuple_constructor_fail
     >;
+
 public:
 
     _LIBCPP_INLINE_VISIBILITY constexpr optional() noexcept {}
@@ -759,6 +795,14 @@ public:
         this->__construct_from(_VSTD::move(__v));
     }
 
+#if _LIBCPP_STD_VER > 20
+  template<class _Fp, class... _Args>
+  _LIBCPP_HIDE_FROM_ABI
+  constexpr explicit optional(__optional_construct_from_invoke_tag, _Fp&& __f, _Args&&... __args)
+      : __base(__optional_construct_from_invoke_tag{}, _VSTD::forward<_Fp>(__f), _VSTD::forward<_Args>(__args)...) {
+  }
+#endif
+
     _LIBCPP_INLINE_VISIBILITY
     _LIBCPP_CONSTEXPR_AFTER_CXX17 optional& operator=(nullopt_t) noexcept
     {
@@ -993,6 +1037,132 @@ public:
                                   static_cast<value_type>(_VSTD::forward<_Up>(__v));
     }
 
+#if _LIBCPP_STD_VER > 20
+  template<class _Func>
+  _LIBCPP_HIDE_FROM_ABI
+  constexpr auto and_then(_Func&& __f) & {
+    using _Up = invoke_result_t<_Func, value_type&>;
+    static_assert(__is_std_optional<remove_cvref_t<_Up>>::value,
+                  "Result of f(value()) must be a specialization of std::optional");
+    if (*this)
+      return _VSTD::invoke(_VSTD::forward<_Func>(__f), value());
+    return remove_cvref_t<_Up>();
+  }
+
+  template<class _Func>
+  _LIBCPP_HIDE_FROM_ABI
+  constexpr auto and_then(_Func&& __f) const& {
+    using _Up = invoke_result_t<_Func, const value_type&>;
+    static_assert(__is_std_optional<remove_cvref_t<_Up>>::value,
+                  "Result of f(value()) must be a specialization of std::optional");
+    if (*this)
+      return _VSTD::invoke(_VSTD::forward<_Func>(__f), value());
+    return remove_cvref_t<_Up>();
+  }
+
+  template<class _Func>
+  _LIBCPP_HIDE_FROM_ABI
+  constexpr auto and_then(_Func&& __f) && {
+    using _Up = invoke_result_t<_Func, value_type&&>;
+    static_assert(__is_std_optional<remove_cvref_t<_Up>>::value,
+                  "Result of f(std::move(value())) must be a specialization of std::optional");
+    if (*this)
+      return _VSTD::invoke(_VSTD::forward<_Func>(__f), _VSTD::move(value()));
+    return remove_cvref_t<_Up>();
+  }
+
+  template<class _Func>
+  _LIBCPP_HIDE_FROM_ABI
+  constexpr auto and_then(_Func&& __f) const&& {
+    using _Up = invoke_result_t<_Func, const value_type&&>;
+    static_assert(__is_std_optional<remove_cvref_t<_Up>>::value,
+                  "Result of f(std::move(value())) must be a specialization of std::optional");
+    if (*this)
+      return _VSTD::invoke(_VSTD::forward<_Func>(__f), _VSTD::move(value()));
+    return remove_cvref_t<_Up>();
+  }
+
+  template<class _Func>
+  _LIBCPP_HIDE_FROM_ABI
+  constexpr auto transform(_Func&& __f) & {
+    using _Up = remove_cv_t<invoke_result_t<_Func, value_type&>>;
+    static_assert(!is_array_v<_Up>, "Result of f(value()) should not be an Array");
+    static_assert(!is_same_v<_Up, in_place_t>,
+                  "Result of f(value()) should not be std::in_place_t");
+    static_assert(!is_same_v<_Up, nullopt_t>,
+                  "Result of f(value()) should not be std::nullopt_t");
+    static_assert(is_object_v<_Up>, "Result of f(value()) should be an object type");
+    if (*this)
+      return optional<_Up>(__optional_construct_from_invoke_tag{}, _VSTD::forward<_Func>(__f), value());
+    return optional<_Up>();
+  }
+
+  template<class _Func>
+  _LIBCPP_HIDE_FROM_ABI
+  constexpr auto transform(_Func&& __f) const& {
+    using _Up = remove_cv_t<invoke_result_t<_Func, const value_type&>>;
+    static_assert(!is_array_v<_Up>, "Result of f(value()) should not be an Array");
+    static_assert(!is_same_v<_Up, in_place_t>,
+                  "Result of f(value()) should not be std::in_place_t");
+    static_assert(!is_same_v<_Up, nullopt_t>,
+                  "Result of f(value()) should not be std::nullopt_t");
+    static_assert(is_object_v<_Up>, "Result of f(value()) should be an object type");
+    if (*this)
+      return optional<_Up>(__optional_construct_from_invoke_tag{}, _VSTD::forward<_Func>(__f), value());
+    return optional<_Up>();
+  }
+
+  template<class _Func>
+  _LIBCPP_HIDE_FROM_ABI
+  constexpr auto transform(_Func&& __f) && {
+    using _Up = remove_cv_t<invoke_result_t<_Func, value_type&&>>;
+    static_assert(!is_array_v<_Up>, "Result of f(std::move(value())) should not be an Array");
+    static_assert(!is_same_v<_Up, in_place_t>,
+                  "Result of f(std::move(value())) should not be std::in_place_t");
+    static_assert(!is_same_v<_Up, nullopt_t>,
+                  "Result of f(std::move(value())) should not be std::nullopt_t");
+    static_assert(is_object_v<_Up>, "Result of f(std::move(value())) should be an object type");
+    if (*this)
+      return optional<_Up>(__optional_construct_from_invoke_tag{}, _VSTD::forward<_Func>(__f), _VSTD::move(value()));
+    return optional<_Up>();
+  }
+
+  template<class _Func>
+  _LIBCPP_HIDE_FROM_ABI
+  constexpr auto transform(_Func&& __f) const&& {
+    using _Up = remove_cvref_t<invoke_result_t<_Func, const value_type&&>>;
+    static_assert(!is_array_v<_Up>, "Result of f(std::move(value())) should not be an Array");
+    static_assert(!is_same_v<_Up, in_place_t>,
+                  "Result of f(std::move(value())) should not be std::in_place_t");
+    static_assert(!is_same_v<_Up, nullopt_t>,
+                  "Result of f(std::move(value())) should not be std::nullopt_t");
+    static_assert(is_object_v<_Up>, "Result of f(std::move(value())) should be an object type");
+    if (*this)
+      return optional<_Up>(__optional_construct_from_invoke_tag{}, _VSTD::forward<_Func>(__f), _VSTD::move(value()));
+    return optional<_Up>();
+  }
+
+  template<invocable _Func>
+  _LIBCPP_HIDE_FROM_ABI
+  constexpr optional or_else(_Func&& __f) const& requires is_copy_constructible_v<value_type> {
+    static_assert(is_same_v<remove_cvref_t<invoke_result_t<_Func>>, optional>,
+                  "Result of f() should be the same type as this optional");
+    if (*this)
+      return *this;
+    return _VSTD::forward<_Func>(__f)();
+  }
+
+  template<invocable _Func>
+  _LIBCPP_HIDE_FROM_ABI
+  constexpr optional or_else(_Func&& __f) && requires is_move_constructible_v<value_type> {
+    static_assert(is_same_v<remove_cvref_t<invoke_result_t<_Func>>, optional>,
+                  "Result of f() should be the same type as this optional");
+    if (*this)
+      return _VSTD::move(*this);
+    return _VSTD::forward<_Func>(__f)();
+  }
+#endif // _LIBCPP_STD_VER > 20
+
     using __base::reset;
 };
 
diff --git a/libcxx/include/ostream b/libcxx/include/ostream
index 98f36ea7acc1..d948d591c8f0 100644
--- a/libcxx/include/ostream
+++ b/libcxx/include/ostream
@@ -249,12 +249,11 @@ class _LIBCPP_TEMPLATE_VIS basic_ostream<_CharT, _Traits>::sentry
     bool __ok_;
     basic_ostream<_CharT, _Traits>& __os_;
 
-    sentry(const sentry&); // = delete;
-    sentry& operator=(const sentry&); // = delete;
-
 public:
     explicit sentry(basic_ostream<_CharT, _Traits>& __os);
     ~sentry();
+    sentry(const sentry&) = delete;
+    sentry& operator=(const sentry&) = delete;
 
     _LIBCPP_INLINE_VISIBILITY
     explicit operator bool() const {return __ok_;}
diff --git a/libcxx/include/random b/libcxx/include/random
index 9eb70bac00b9..c88bfce03b19 100644
--- a/libcxx/include/random
+++ b/libcxx/include/random
@@ -1682,6 +1682,7 @@ class piecewise_linear_distribution
 #include <__random/binomial_distribution.h>
 #include <__random/cauchy_distribution.h>
 #include <__random/chi_squared_distribution.h>
+#include <__random/clamp_to_integral.h>
 #include <__random/default_random_engine.h>
 #include <__random/discard_block_engine.h>
 #include <__random/discrete_distribution.h>
diff --git a/libcxx/include/regex b/libcxx/include/regex
index 815ff7d3862d..8203c81659c8 100644
--- a/libcxx/include/regex
+++ b/libcxx/include/regex
@@ -978,7 +978,7 @@ enum error_type
     __re_err_parse
 };
 
-}  // regex_constants
+} // namespace regex_constants
 
 class _LIBCPP_EXCEPTION_ABI regex_error
     : public runtime_error
diff --git a/libcxx/include/stdexcept b/libcxx/include/stdexcept
index ddbc6303b624..c1dc05669bb9 100644
--- a/libcxx/include/stdexcept
+++ b/libcxx/include/stdexcept
@@ -209,7 +209,7 @@ public:
 #endif
 };
 
-}  // std
+} // namespace std
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
diff --git a/libcxx/include/string b/libcxx/include/string
index 313a7b5c2376..9049d7acbb9b 100644
--- a/libcxx/include/string
+++ b/libcxx/include/string
@@ -826,9 +826,10 @@ public:
     basic_string(const _CharT* __s) : __r_(__default_init_tag(), __default_init_tag()) {
       _LIBCPP_ASSERT(__s != nullptr, "basic_string(const char*) detected nullptr");
       __init(__s, traits_type::length(__s));
-#   if _LIBCPP_DEBUG_LEVEL == 2
-      __get_db()->__insert_c(this);
-#   endif
+#if _LIBCPP_DEBUG_LEVEL == 2
+        if (!__libcpp_is_constant_evaluated())
+            __get_db()->__insert_c(this);
+#endif
     }
 
     template <class = __enable_if_t<__is_allocator<_Allocator>::value, nullptr_t> >
@@ -1694,6 +1695,13 @@ private:
       return *this;
     }
 
+    _LIBCPP_HIDE_FROM_ABI basic_string& __null_terminate_at(value_type* __p, size_type __newsz) {
+      __set_size(__newsz);
+      __invalidate_iterators_past(__newsz);
+      traits_type::assign(__p[__newsz], value_type());
+      return *this;
+    }
+
     _LIBCPP_INLINE_VISIBILITY void __invalidate_all_iterators();
     _LIBCPP_INLINE_VISIBILITY void __invalidate_iterators_past(size_type);
 
@@ -1778,7 +1786,8 @@ void
 basic_string<_CharT, _Traits, _Allocator>::__invalidate_all_iterators()
 {
 #if _LIBCPP_DEBUG_LEVEL == 2
-    __get_db()->__invalidate_all(this);
+    if (!__libcpp_is_constant_evaluated())
+        __get_db()->__invalidate_all(this);
 #endif
 }
 
@@ -1788,22 +1797,24 @@ void
 basic_string<_CharT, _Traits, _Allocator>::__invalidate_iterators_past(size_type __pos)
 {
 #if _LIBCPP_DEBUG_LEVEL == 2
-    __c_node* __c = __get_db()->__find_c_and_lock(this);
-    if (__c)
-    {
-        const_pointer __new_last = __get_pointer() + __pos;
-        for (__i_node** __p = __c->end_; __p != __c->beg_; )
+    if (!__libcpp_is_constant_evaluated()) {
+        __c_node* __c = __get_db()->__find_c_and_lock(this);
+        if (__c)
         {
-            --__p;
-            const_iterator* __i = static_cast<const_iterator*>((*__p)->__i_);
-            if (__i->base() > __new_last)
+            const_pointer __new_last = __get_pointer() + __pos;
+            for (__i_node** __p = __c->end_; __p != __c->beg_; )
             {
-                (*__p)->__c_ = nullptr;
-                if (--__c->end_ != __p)
-                    _VSTD::memmove(__p, __p+1, (__c->end_ - __p)*sizeof(__i_node*));
+                --__p;
+                const_iterator* __i = static_cast<const_iterator*>((*__p)->__i_);
+                if (__i->base() > __new_last)
+                {
+                    (*__p)->__c_ = nullptr;
+                    if (--__c->end_ != __p)
+                        _VSTD::memmove(__p, __p+1, (__c->end_ - __p)*sizeof(__i_node*));
+                }
             }
+            __get_db()->unlock();
         }
-        __get_db()->unlock();
     }
 #else
     (void)__pos;
@@ -1817,7 +1828,8 @@ basic_string<_CharT, _Traits, _Allocator>::basic_string()
      : __r_(__default_init_tag(), __default_init_tag())
 {
 #if _LIBCPP_DEBUG_LEVEL == 2
-    __get_db()->__insert_c(this);
+    if (!__libcpp_is_constant_evaluated())
+        __get_db()->__insert_c(this);
 #endif
     __zero();
 }
@@ -1833,7 +1845,8 @@ basic_string<_CharT, _Traits, _Allocator>::basic_string(const allocator_type& __
 : __r_(__default_init_tag(), __a)
 {
 #if _LIBCPP_DEBUG_LEVEL == 2
-    __get_db()->__insert_c(this);
+    if (!__libcpp_is_constant_evaluated())
+        __get_db()->__insert_c(this);
 #endif
     __zero();
 }
@@ -1895,7 +1908,8 @@ basic_string<_CharT, _Traits, _Allocator>::basic_string(const _CharT* __s, const
     _LIBCPP_ASSERT(__s != nullptr, "basic_string(const char*, allocator) detected nullptr");
     __init(__s, traits_type::length(__s));
 #if _LIBCPP_DEBUG_LEVEL == 2
-    __get_db()->__insert_c(this);
+    if (!__libcpp_is_constant_evaluated())
+        __get_db()->__insert_c(this);
 #endif
 }
 
@@ -1904,10 +1918,11 @@ inline
 basic_string<_CharT, _Traits, _Allocator>::basic_string(const _CharT* __s, size_type __n)
      : __r_(__default_init_tag(), __default_init_tag())
 {
-    _LIBCPP_ASSERT(__n == 0 || __s != nullptr, "basic_string(const char*, n) detected nullptr");
-    __init(__s, __n);
+      _LIBCPP_ASSERT(__n == 0 || __s != nullptr, "basic_string(const char*, n) detected nullptr");
+      __init(__s, __n);
 #if _LIBCPP_DEBUG_LEVEL == 2
-    __get_db()->__insert_c(this);
+    if (!__libcpp_is_constant_evaluated())
+        __get_db()->__insert_c(this);
 #endif
 }
 
@@ -1919,7 +1934,8 @@ basic_string<_CharT, _Traits, _Allocator>::basic_string(const _CharT* __s, size_
     _LIBCPP_ASSERT(__n == 0 || __s != nullptr, "basic_string(const char*, n, allocator) detected nullptr");
     __init(__s, __n);
 #if _LIBCPP_DEBUG_LEVEL == 2
-    __get_db()->__insert_c(this);
+    if (!__libcpp_is_constant_evaluated())
+        __get_db()->__insert_c(this);
 #endif
 }
 
@@ -1934,7 +1950,8 @@ basic_string<_CharT, _Traits, _Allocator>::basic_string(const basic_string& __st
                                   __str.__get_long_size());
 
 #if _LIBCPP_DEBUG_LEVEL == 2
-    __get_db()->__insert_c(this);
+    if (!__libcpp_is_constant_evaluated())
+        __get_db()->__insert_c(this);
 #endif
 }
 
@@ -1949,7 +1966,8 @@ basic_string<_CharT, _Traits, _Allocator>::basic_string(
         __init_copy_ctor_external(_VSTD::__to_address(__str.__get_long_pointer()),
                                   __str.__get_long_size());
 #if _LIBCPP_DEBUG_LEVEL == 2
-    __get_db()->__insert_c(this);
+    if (!__libcpp_is_constant_evaluated())
+        __get_db()->__insert_c(this);
 #endif
 }
 
@@ -1986,9 +2004,11 @@ basic_string<_CharT, _Traits, _Allocator>::basic_string(basic_string&& __str)
 {
     __str.__zero();
 #if _LIBCPP_DEBUG_LEVEL == 2
-    __get_db()->__insert_c(this);
-    if (__is_long())
-        __get_db()->swap(this, &__str);
+    if (!__libcpp_is_constant_evaluated()) {
+        __get_db()->__insert_c(this);
+        if (__is_long())
+            __get_db()->swap(this, &__str);
+    }
 #endif
 }
 
@@ -2005,9 +2025,11 @@ basic_string<_CharT, _Traits, _Allocator>::basic_string(basic_string&& __str, co
         __str.__zero();
     }
 #if _LIBCPP_DEBUG_LEVEL == 2
-    __get_db()->__insert_c(this);
-    if (__is_long())
-        __get_db()->swap(this, &__str);
+    if (!__libcpp_is_constant_evaluated()) {
+        __get_db()->__insert_c(this);
+        if (__is_long())
+            __get_db()->swap(this, &__str);
+    }
 #endif
 }
 
@@ -2044,7 +2066,8 @@ basic_string<_CharT, _Traits, _Allocator>::basic_string(size_type __n, _CharT __
 {
     __init(__n, __c);
 #if _LIBCPP_DEBUG_LEVEL == 2
-    __get_db()->__insert_c(this);
+    if (!__libcpp_is_constant_evaluated())
+        __get_db()->__insert_c(this);
 #endif
 }
 
@@ -2055,7 +2078,8 @@ basic_string<_CharT, _Traits, _Allocator>::basic_string(size_type __n, _CharT __
 {
     __init(__n, __c);
 #if _LIBCPP_DEBUG_LEVEL == 2
-    __get_db()->__insert_c(this);
+    if (!__libcpp_is_constant_evaluated())
+        __get_db()->__insert_c(this);
 #endif
 }
 
@@ -2070,7 +2094,8 @@ basic_string<_CharT, _Traits, _Allocator>::basic_string(const basic_string& __st
         this->__throw_out_of_range();
     __init(__str.data() + __pos, _VSTD::min(__n, __str_sz - __pos));
 #if _LIBCPP_DEBUG_LEVEL == 2
-    __get_db()->__insert_c(this);
+    if (!__libcpp_is_constant_evaluated())
+        __get_db()->__insert_c(this);
 #endif
 }
 
@@ -2085,7 +2110,8 @@ basic_string<_CharT, _Traits, _Allocator>::basic_string(const basic_string& __st
         this->__throw_out_of_range();
     __init(__str.data() + __pos, __str_sz - __pos);
 #if _LIBCPP_DEBUG_LEVEL == 2
-    __get_db()->__insert_c(this);
+    if (!__libcpp_is_constant_evaluated())
+        __get_db()->__insert_c(this);
 #endif
 }
 
@@ -2099,7 +2125,8 @@ basic_string<_CharT, _Traits, _Allocator>::basic_string(
     __self_view __sv = __sv0.substr(__pos, __n);
     __init(__sv.data(), __sv.size());
 #if _LIBCPP_DEBUG_LEVEL == 2
-    __get_db()->__insert_c(this);
+    if (!__libcpp_is_constant_evaluated())
+        __get_db()->__insert_c(this);
 #endif
 }
 
@@ -2111,7 +2138,8 @@ basic_string<_CharT, _Traits, _Allocator>::basic_string(const _Tp & __t)
     __self_view __sv = __t;
     __init(__sv.data(), __sv.size());
 #if _LIBCPP_DEBUG_LEVEL == 2
-    __get_db()->__insert_c(this);
+    if (!__libcpp_is_constant_evaluated())
+        __get_db()->__insert_c(this);
 #endif
 }
 
@@ -2123,7 +2151,8 @@ basic_string<_CharT, _Traits, _Allocator>::basic_string(const _Tp & __t, const _
     __self_view __sv = __t;
     __init(__sv.data(), __sv.size());
 #if _LIBCPP_DEBUG_LEVEL == 2
-    __get_db()->__insert_c(this);
+    if (!__libcpp_is_constant_evaluated())
+        __get_db()->__insert_c(this);
 #endif
 }
 
@@ -2205,7 +2234,8 @@ basic_string<_CharT, _Traits, _Allocator>::basic_string(_InputIterator __first,
 {
     __init(__first, __last);
 #if _LIBCPP_DEBUG_LEVEL == 2
-    __get_db()->__insert_c(this);
+    if (!__libcpp_is_constant_evaluated())
+        __get_db()->__insert_c(this);
 #endif
 }
 
@@ -2218,7 +2248,8 @@ basic_string<_CharT, _Traits, _Allocator>::basic_string(_InputIterator __first,
 {
     __init(__first, __last);
 #if _LIBCPP_DEBUG_LEVEL == 2
-    __get_db()->__insert_c(this);
+    if (!__libcpp_is_constant_evaluated())
+        __get_db()->__insert_c(this);
 #endif
 }
 
@@ -2232,7 +2263,8 @@ basic_string<_CharT, _Traits, _Allocator>::basic_string(
 {
     __init(__il.begin(), __il.end());
 #if _LIBCPP_DEBUG_LEVEL == 2
-    __get_db()->__insert_c(this);
+    if (!__libcpp_is_constant_evaluated())
+        __get_db()->__insert_c(this);
 #endif
 }
 
@@ -2245,7 +2277,8 @@ basic_string<_CharT, _Traits, _Allocator>::basic_string(
 {
     __init(__il.begin(), __il.end());
 #if _LIBCPP_DEBUG_LEVEL == 2
-    __get_db()->__insert_c(this);
+    if (!__libcpp_is_constant_evaluated())
+        __get_db()->__insert_c(this);
 #endif
 }
 
@@ -2255,7 +2288,8 @@ template <class _CharT, class _Traits, class _Allocator>
 basic_string<_CharT, _Traits, _Allocator>::~basic_string()
 {
 #if _LIBCPP_DEBUG_LEVEL == 2
-    __get_db()->__erase_c(this);
+    if (!__libcpp_is_constant_evaluated())
+        __get_db()->__erase_c(this);
 #endif
     if (__is_long())
         __alloc_traits::deallocate(__alloc(), __get_long_pointer(), __get_long_cap());
@@ -2351,14 +2385,12 @@ basic_string<_CharT, _Traits, _Allocator>::__assign_external(
   if (__cap >= __n) {
     value_type* __p = _VSTD::__to_address(__get_pointer());
     traits_type::move(__p, __s, __n);
-    traits_type::assign(__p[__n], value_type());
-    __set_size(__n);
-    __invalidate_iterators_past(__n);
+    return __null_terminate_at(__p, __n);
   } else {
     size_type __sz = size();
     __grow_by_and_replace(__cap, __n - __cap, __sz, 0, __sz, __n, __s);
+    return *this;
   }
-  return *this;
 }
 
 template <class _CharT, class _Traits, class _Allocator>
@@ -2383,10 +2415,7 @@ basic_string<_CharT, _Traits, _Allocator>::assign(size_type __n, value_type __c)
     }
     value_type* __p = _VSTD::__to_address(__get_pointer());
     traits_type::assign(__p, __n, __c);
-    traits_type::assign(__p[__n], value_type());
-    __set_size(__n);
-    __invalidate_iterators_past(__n);
-    return *this;
+    return __null_terminate_at(__p, __n);
 }
 
 template <class _CharT, class _Traits, class _Allocator>
@@ -2826,13 +2855,11 @@ __enable_if_t
 >
 basic_string<_CharT, _Traits, _Allocator>::insert(const_iterator __pos, _InputIterator __first, _InputIterator __last)
 {
-#if _LIBCPP_DEBUG_LEVEL == 2
-    _LIBCPP_ASSERT(__get_const_db()->__find_c_from_i(&__pos) == this,
-        "string::insert(iterator, range) called with an iterator not"
-        " referring to this string");
-#endif
-    const basic_string __temp(__first, __last, __alloc());
-    return insert(__pos, __temp.data(), __temp.data() + __temp.size());
+  _LIBCPP_DEBUG_ASSERT(__get_const_db()->__find_c_from_i(&__pos) == this,
+                       "string::insert(iterator, range) called with an iterator not"
+                       " referring to this string");
+  const basic_string __temp(__first, __last, __alloc());
+  return insert(__pos, __temp.data(), __temp.data() + __temp.size());
 }
 
 template <class _CharT, class _Traits, class _Allocator>
@@ -2844,11 +2871,10 @@ __enable_if_t
 >
 basic_string<_CharT, _Traits, _Allocator>::insert(const_iterator __pos, _ForwardIterator __first, _ForwardIterator __last)
 {
-#if _LIBCPP_DEBUG_LEVEL == 2
-    _LIBCPP_ASSERT(__get_const_db()->__find_c_from_i(&__pos) == this,
-        "string::insert(iterator, range) called with an iterator not"
-        " referring to this string");
-#endif
+  _LIBCPP_DEBUG_ASSERT(__get_const_db()->__find_c_from_i(&__pos) == this,
+                       "string::insert(iterator, range) called with an iterator not"
+                       " referring to this string");
+
     size_type __ip = static_cast<size_type>(__pos - begin());
     size_type __n = static_cast<size_type>(_VSTD::distance(__first, __last));
     if (__n)
@@ -2961,14 +2987,12 @@ inline
 typename basic_string<_CharT, _Traits, _Allocator>::iterator
 basic_string<_CharT, _Traits, _Allocator>::insert(const_iterator __pos, size_type __n, value_type __c)
 {
-#if _LIBCPP_DEBUG_LEVEL == 2
-    _LIBCPP_ASSERT(__get_const_db()->__find_c_from_i(&__pos) == this,
-        "string::insert(iterator, n, value) called with an iterator not"
-        " referring to this string");
-#endif
-    difference_type __p = __pos - begin();
-    insert(static_cast<size_type>(__p), __n, __c);
-    return begin() + __p;
+  _LIBCPP_DEBUG_ASSERT(__get_const_db()->__find_c_from_i(&__pos) == this,
+                       "string::insert(iterator, n, value) called with an iterator not"
+                       " referring to this string");
+  difference_type __p = __pos - begin();
+  insert(static_cast<size_type>(__p), __n, __c);
+  return begin() + __p;
 }
 
 // replace
@@ -2996,7 +3020,7 @@ basic_string<_CharT, _Traits, _Allocator>::replace(size_type __pos, size_type __
                 {
                     traits_type::move(__p + __pos, __s, __n2);
                     traits_type::move(__p + __pos + __n2, __p + __pos + __n1, __n_move);
-                    goto __finish;
+                    return __null_terminate_at(__p, __sz + (__n2 - __n1));
                 }
                 if (__p + __pos < __s && __s < __p + __sz)
                 {
@@ -3015,13 +3039,7 @@ basic_string<_CharT, _Traits, _Allocator>::replace(size_type __pos, size_type __
             }
         }
         traits_type::move(__p + __pos, __s, __n2);
-__finish:
-// __sz += __n2 - __n1; in this and the below function below can cause unsigned
-// integer overflow, but this is a safe operation, so we disable the check.
-        __sz += __n2 - __n1;
-        __set_size(__sz);
-        __invalidate_iterators_past(__sz);
-        traits_type::assign(__p[__sz], value_type());
+        return __null_terminate_at(__p, __sz + (__n2 - __n1));
     }
     else
         __grow_by_and_replace(__cap, __sz - __n1 + __n2 - __cap, __sz, __pos, __n1, __n2, __s);
@@ -3031,7 +3049,6 @@ __finish:
 template <class _CharT, class _Traits, class _Allocator>
 basic_string<_CharT, _Traits, _Allocator>&
 basic_string<_CharT, _Traits, _Allocator>::replace(size_type __pos, size_type __n1, size_type __n2, value_type __c)
-    _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK
 {
     size_type __sz = size();
     if (__pos > __sz)
@@ -3055,11 +3072,7 @@ basic_string<_CharT, _Traits, _Allocator>::replace(size_type __pos, size_type __
         __p = _VSTD::__to_address(__get_long_pointer());
     }
     traits_type::assign(__p + __pos, __n2, __c);
-    __sz += __n2 - __n1;
-    __set_size(__sz);
-    __invalidate_iterators_past(__sz);
-    traits_type::assign(__p[__sz], value_type());
-    return *this;
+    return __null_terminate_at(__p, __sz - (__n1 - __n2));
 }
 
 template <class _CharT, class _Traits, class _Allocator>
@@ -3170,10 +3183,7 @@ basic_string<_CharT, _Traits, _Allocator>::__erase_external_with_move(
         size_type __n_move = __sz - __pos - __n;
         if (__n_move != 0)
             traits_type::move(__p + __pos, __p + __pos + __n, __n_move);
-        __sz -= __n;
-        __set_size(__sz);
-        __invalidate_iterators_past(__sz);
-        traits_type::assign(__p[__sz], value_type());
+        __null_terminate_at(__p, __sz - __n);
     }
 }
 
@@ -3195,17 +3205,15 @@ inline
 typename basic_string<_CharT, _Traits, _Allocator>::iterator
 basic_string<_CharT, _Traits, _Allocator>::erase(const_iterator __pos)
 {
-#if _LIBCPP_DEBUG_LEVEL == 2
-    _LIBCPP_ASSERT(__get_const_db()->__find_c_from_i(&__pos) == this,
-        "string::erase(iterator) called with an iterator not"
-        " referring to this string");
-#endif
-    _LIBCPP_ASSERT(__pos != end(),
-        "string::erase(iterator) called with a non-dereferenceable iterator");
-    iterator __b = begin();
-    size_type __r = static_cast<size_type>(__pos - __b);
-    erase(__r, 1);
-    return __b + static_cast<difference_type>(__r);
+  _LIBCPP_DEBUG_ASSERT(__get_const_db()->__find_c_from_i(&__pos) == this,
+                       "string::erase(iterator) called with an iterator not"
+                       " referring to this string");
+
+  _LIBCPP_ASSERT(__pos != end(), "string::erase(iterator) called with a non-dereferenceable iterator");
+  iterator __b = begin();
+  size_type __r = static_cast<size_type>(__pos - __b);
+  erase(__r, 1);
+  return __b + static_cast<difference_type>(__r);
 }
 
 template <class _CharT, class _Traits, class _Allocator>
@@ -3213,16 +3221,15 @@ inline
 typename basic_string<_CharT, _Traits, _Allocator>::iterator
 basic_string<_CharT, _Traits, _Allocator>::erase(const_iterator __first, const_iterator __last)
 {
-#if _LIBCPP_DEBUG_LEVEL == 2
-    _LIBCPP_ASSERT(__get_const_db()->__find_c_from_i(&__first) == this,
-        "string::erase(iterator,  iterator) called with an iterator not"
-        " referring to this string");
-#endif
-    _LIBCPP_ASSERT(__first <= __last, "string::erase(first, last) called with invalid range");
-    iterator __b = begin();
-    size_type __r = static_cast<size_type>(__first - __b);
-    erase(__r, static_cast<size_type>(__last - __first));
-    return __b + static_cast<difference_type>(__r);
+  _LIBCPP_DEBUG_ASSERT(__get_const_db()->__find_c_from_i(&__first) == this,
+                       "string::erase(iterator,  iterator) called with an iterator not"
+                       " referring to this string");
+
+  _LIBCPP_ASSERT(__first <= __last, "string::erase(first, last) called with invalid range");
+  iterator __b = begin();
+  size_type __r = static_cast<size_type>(__first - __b);
+  erase(__r, static_cast<size_type>(__last - __first));
+  return __b + static_cast<difference_type>(__r);
 }
 
 template <class _CharT, class _Traits, class _Allocator>
@@ -3231,20 +3238,7 @@ void
 basic_string<_CharT, _Traits, _Allocator>::pop_back()
 {
     _LIBCPP_ASSERT(!empty(), "string::pop_back(): string is already empty");
-    size_type __sz;
-    if (__is_long())
-    {
-        __sz = __get_long_size() - 1;
-        __set_long_size(__sz);
-        traits_type::assign(*(__get_long_pointer() + __sz), value_type());
-    }
-    else
-    {
-        __sz = __get_short_size() - 1;
-        __set_short_size(__sz);
-        traits_type::assign(*(__get_short_pointer() + __sz), value_type());
-    }
-    __invalidate_iterators_past(__sz);
+    __erase_to_end(size() - 1);
 }
 
 template <class _CharT, class _Traits, class _Allocator>
@@ -3270,17 +3264,7 @@ inline
 void
 basic_string<_CharT, _Traits, _Allocator>::__erase_to_end(size_type __pos)
 {
-    if (__is_long())
-    {
-        traits_type::assign(*(__get_long_pointer() + __pos), value_type());
-        __set_long_size(__pos);
-    }
-    else
-    {
-        traits_type::assign(*(__get_short_pointer() + __pos), value_type());
-        __set_short_size(__pos);
-    }
-    __invalidate_iterators_past(__pos);
+  __null_terminate_at(_VSTD::__to_address(__get_pointer()), __pos);
 }
 
 template <class _CharT, class _Traits, class _Allocator>
@@ -3338,6 +3322,7 @@ basic_string<_CharT, _Traits, _Allocator>::reserve(size_type __requested_capacit
 }
 
 template <class _CharT, class _Traits, class _Allocator>
+inline
 void
 basic_string<_CharT, _Traits, _Allocator>::shrink_to_fit() _NOEXCEPT
 {
@@ -3348,6 +3333,7 @@ basic_string<_CharT, _Traits, _Allocator>::shrink_to_fit() _NOEXCEPT
 }
 
 template <class _CharT, class _Traits, class _Allocator>
+inline
 void
 basic_string<_CharT, _Traits, _Allocator>::__shrink_or_extend(size_type __target_capacity)
 {
@@ -3508,11 +3494,13 @@ basic_string<_CharT, _Traits, _Allocator>::swap(basic_string& __str)
 #endif
 {
 #if _LIBCPP_DEBUG_LEVEL == 2
-    if (!__is_long())
-        __get_db()->__invalidate_all(this);
-    if (!__str.__is_long())
-        __get_db()->__invalidate_all(&__str);
-    __get_db()->swap(this, &__str);
+    if (!__libcpp_is_constant_evaluated()) {
+        if (!__is_long())
+            __get_db()->__invalidate_all(this);
+        if (!__str.__is_long())
+            __get_db()->__invalidate_all(&__str);
+        __get_db()->swap(this, &__str);
+    }
 #endif
     _LIBCPP_ASSERT(
         __alloc_traits::propagate_on_container_swap::value ||
@@ -4571,8 +4559,8 @@ inline namespace literals
     {
         return basic_string<char32_t> (__str, __len);
     }
-  }
-}
+  } // namespace string_literals
+} // namespace literals
 #endif
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/string_view b/libcxx/include/string_view
index a5f85e88b502..3861ad67ca5d 100644
--- a/libcxx/include/string_view
+++ b/libcxx/include/string_view
@@ -356,7 +356,7 @@ public:
     size_type length()   const _NOEXCEPT { return __size; }
 
     _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY
-    size_type max_size() const _NOEXCEPT { return numeric_limits<size_type>::max(); }
+    size_type max_size() const _NOEXCEPT { return numeric_limits<size_type>::max() / sizeof(value_type); }
 
     _LIBCPP_NODISCARD_AFTER_CXX17 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR
     bool empty()         const _NOEXCEPT { return __size == 0; }
@@ -947,8 +947,8 @@ inline namespace literals
     {
         return basic_string_view<char32_t> (__str, __len);
     }
-  }
-}
+  } // namespace string_view_literals
+} // namespace literals
 #endif
 _LIBCPP_END_NAMESPACE_STD
 
diff --git a/libcxx/include/system_error b/libcxx/include/system_error
index 3aa869acff41..059fa0e2d511 100644
--- a/libcxx/include/system_error
+++ b/libcxx/include/system_error
@@ -206,13 +206,11 @@ public:
     error_category() _NOEXCEPT;
 #else
     _LIBCPP_INLINE_VISIBILITY
-    _LIBCPP_CONSTEXPR_AFTER_CXX11 error_category() _NOEXCEPT _LIBCPP_DEFAULT
+    _LIBCPP_CONSTEXPR_AFTER_CXX11 error_category() _NOEXCEPT = default;
 #endif
-private:
-    error_category(const error_category&);// = delete;
-    error_category& operator=(const error_category&);// = delete;
+    error_category(const error_category&) = delete;
+    error_category& operator=(const error_category&) = delete;
 
-public:
     virtual const char* name() const _NOEXCEPT = 0;
     virtual error_condition default_error_condition(int __ev) const _NOEXCEPT;
     virtual bool equivalent(int __code, const error_condition& __condition) const _NOEXCEPT;
diff --git a/libcxx/include/thread b/libcxx/include/thread
index a51a11c0d3c8..a4632f6fe524 100644
--- a/libcxx/include/thread
+++ b/libcxx/include/thread
@@ -88,7 +88,6 @@ void sleep_for(const chrono::duration<Rep, Period>& rel_time);
 #include <__mutex_base>
 #include <__thread/poll_with_backoff.h>
 #include <__threading_support>
-#include <__utility/decay_copy.h>
 #include <__utility/forward.h>
 #include <chrono>
 #include <cstddef>
@@ -303,8 +302,8 @@ thread::thread(_Fp&& __f, _Args&&... __args)
     typedef tuple<_TSPtr, typename decay<_Fp>::type, typename decay<_Args>::type...> _Gp;
     unique_ptr<_Gp> __p(
             new _Gp(_VSTD::move(__tsp),
-                    _VSTD::__decay_copy(_VSTD::forward<_Fp>(__f)),
-                    _VSTD::__decay_copy(_VSTD::forward<_Args>(__args))...));
+                    _VSTD::forward<_Fp>(__f),
+                    _VSTD::forward<_Args>(__args)...));
     int __ec = _VSTD::__libcpp_thread_create(&__t_, &__thread_proxy<_Gp>, __p.get());
     if (__ec == 0)
         __p.release();
@@ -403,7 +402,7 @@ sleep_until(const chrono::time_point<chrono::steady_clock, _Duration>& __t)
 inline _LIBCPP_INLINE_VISIBILITY
 void yield() _NOEXCEPT {__libcpp_thread_yield();}
 
-}  // this_thread
+} // namespace this_thread
 
 _LIBCPP_END_NAMESPACE_STD
 
diff --git a/libcxx/include/tuple b/libcxx/include/tuple
index 2e3d19627fab..8ee5c2eef51d 100644
--- a/libcxx/include/tuple
+++ b/libcxx/include/tuple
@@ -1200,7 +1200,7 @@ struct __find_exactly_one_checked<_T1> {
     static_assert(!is_same<_T1, _T1>::value, "type not in empty type list");
 };
 
-} // namespace __find_detail;
+} // namespace __find_detail
 
 template <typename _T1, typename... _Args>
 struct __find_exactly_one_t
@@ -1257,7 +1257,7 @@ struct __ignore_t
 
 namespace {
   constexpr __ignore_t<unsigned char> ignore = __ignore_t<unsigned char>();
-}
+} // namespace
 
 template <class... _Tp>
 inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11
diff --git a/libcxx/include/type_traits b/libcxx/include/type_traits
index bfb6fcb05134..155b775e4929 100644
--- a/libcxx/include/type_traits
+++ b/libcxx/include/type_traits
@@ -550,8 +550,6 @@ template <bool _Bp, class _If, class _Then> using conditional_t = typename condi
 
 // is_same
 
-#if __has_keyword(__is_same)
-
 template <class _Tp, class _Up>
 struct _LIBCPP_TEMPLATE_VIS is_same : _BoolConstant<__is_same(_Tp, _Up)> { };
 
@@ -560,36 +558,18 @@ template <class _Tp, class _Up>
 inline constexpr bool is_same_v = __is_same(_Tp, _Up);
 #endif
 
-#else
-
-template <class _Tp, class _Up> struct _LIBCPP_TEMPLATE_VIS is_same           : public false_type {};
-template <class _Tp>            struct _LIBCPP_TEMPLATE_VIS is_same<_Tp, _Tp> : public true_type {};
+// _IsSame<T,U> has the same effect as is_same<T,U> but instantiates fewer types:
+// is_same<A,B> and is_same<C,D> are guaranteed to be different types, but
+// _IsSame<A,B> and _IsSame<C,D> are the same type (namely, false_type).
+// Neither GCC nor Clang can mangle the __is_same builtin, so _IsSame
+// mustn't be directly used anywhere that contributes to name-mangling
+// (such as in a dependent return type).
 
-#if _LIBCPP_STD_VER > 14
 template <class _Tp, class _Up>
-inline constexpr bool is_same_v = is_same<_Tp, _Up>::value;
-#endif
-
-#endif // __is_same
+using _IsSame = _BoolConstant<__is_same(_Tp, _Up)>;
 
 template <class _Tp, class _Up>
-using _IsSame = _BoolConstant<
-#ifdef __clang__
-    __is_same(_Tp, _Up)
-#else
-    is_same<_Tp, _Up>::value
-#endif
->;
-
-template <class _Tp, class _Up>
-using _IsNotSame = _BoolConstant<
-#ifdef __clang__
-    !__is_same(_Tp, _Up)
-#else
-    !is_same<_Tp, _Up>::value
-#endif
->;
-
+using _IsNotSame = _BoolConstant<!__is_same(_Tp, _Up)>;
 
 template <class _Tp>
 using __test_for_primary_template = __enable_if_t<
@@ -600,9 +580,9 @@ using __is_primary_template = _IsValidExpansion<
     __test_for_primary_template, _Tp
   >;
 
-struct __two {char __lx[2];};
+// helper class
 
-// helper class:
+struct __two {char __lx[2];};
 
 // is_const
 
@@ -3953,7 +3933,7 @@ struct __nothrow_swappable_with {
 template <class _Tp, class _Up>
 struct __nothrow_swappable_with<_Tp, _Up, false> : false_type {};
 
-}  // __detail
+} // namespace __detail
 
 template <class _Tp>
 struct __is_swappable
diff --git a/libcxx/include/typeinfo b/libcxx/include/typeinfo
index d0f9db36a627..af8217548770 100644
--- a/libcxx/include/typeinfo
+++ b/libcxx/include/typeinfo
@@ -361,7 +361,7 @@ class _LIBCPP_EXCEPTION_ABI bad_typeid
   virtual const char* what() const _NOEXCEPT;
 };
 
-}  // std
+} // namespace std
 
 #endif // defined(_LIBCPP_ABI_VCRUNTIME)
 
diff --git a/libcxx/include/utility b/libcxx/include/utility
index 4fa90289a412..9dd7905516a8 100644
--- a/libcxx/include/utility
+++ b/libcxx/include/utility
@@ -218,6 +218,7 @@ template <class T>
 #include <__debug>
 #include <__tuple>
 #include <__utility/as_const.h>
+#include <__utility/auto_cast.h>
 #include <__utility/cmp.h>
 #include <__utility/declval.h>
 #include <__utility/exchange.h>
@@ -231,8 +232,13 @@ template <class T>
 #include <__utility/rel_ops.h>
 #include <__utility/swap.h>
 #include <__utility/to_underlying.h>
+#include <__utility/transaction.h>
 #include <compare>
 #include <initializer_list>
 #include <version>
 
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#pragma GCC system_header
+#endif
+
 #endif // _LIBCPP_UTILITY
diff --git a/libcxx/include/version b/libcxx/include/version
index 9322c3b8c05d..574dfe47b58f 100644
--- a/libcxx/include/version
+++ b/libcxx/include/version
@@ -109,6 +109,7 @@ __cpp_lib_map_try_emplace                               201411L <map>
 __cpp_lib_math_constants                                201907L <numbers>
 __cpp_lib_math_special_functions                        201603L <cmath>
 __cpp_lib_memory_resource                               201603L <memory_resource>
+__cpp_lib_monadic_optional                              202110L <optional>
 __cpp_lib_node_extract                                  201606L <map> <set> <unordered_map>
                                                                 <unordered_set>
 __cpp_lib_nonmember_container_access                    201411L <array> <deque> <forward_list>
@@ -347,6 +348,7 @@ __cpp_lib_void_t                                        201411L <type_traits>
 #if _LIBCPP_STD_VER > 20
 # define __cpp_lib_byteswap                             202110L
 # define __cpp_lib_is_scoped_enum                       202011L
+# define __cpp_lib_monadic_optional                     202110L
 // # define __cpp_lib_stacktrace                           202011L
 // # define __cpp_lib_stdatomic_h                          202011L
 # define __cpp_lib_string_contains                      202011L
diff --git a/libcxx/src/barrier.cpp b/libcxx/src/barrier.cpp
index 0f9dad987fad..54890b2d797e 100644
--- a/libcxx/src/barrier.cpp
+++ b/libcxx/src/barrier.cpp
@@ -15,14 +15,14 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if !defined(_LIBCPP_HAS_NO_TREE_BARRIER) && (_LIBCPP_STD_VER > 11)
+#if !defined(_LIBCPP_HAS_NO_TREE_BARRIER)
 
 class __barrier_algorithm_base {
 public:
     struct alignas(64) /* naturally-align the heap state */ __state_t
     {
         struct {
-            __atomic_base<__barrier_phase_t> __phase = ATOMIC_VAR_INIT(0);
+          __atomic_base<__barrier_phase_t> __phase{0};
         } __tickets[64];
     };
 
@@ -90,7 +90,7 @@ void __destroy_barrier_algorithm_base(__barrier_algorithm_base* __barrier)
     delete __barrier;
 }
 
-#endif //!defined(_LIBCPP_HAS_NO_TREE_BARRIER) && (_LIBCPP_STD_VER >= 11)
+#endif //!defined(_LIBCPP_HAS_NO_TREE_BARRIER)
 
 _LIBCPP_END_NAMESPACE_STD
 
diff --git a/libcxx/src/charconv.cpp b/libcxx/src/charconv.cpp
index 533e59b04d6f..60ec3eccc949 100644
--- a/libcxx/src/charconv.cpp
+++ b/libcxx/src/charconv.cpp
@@ -9,27 +9,14 @@
 #include "charconv"
 #include <string.h>
 
+#include "include/ryu/digit_table.h"
+#include "include/to_chars_floating_point.h"
+
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 namespace __itoa
 {
 
-static constexpr char cDigitsLut[200] = {
-    '0', '0', '0', '1', '0', '2', '0', '3', '0', '4', '0', '5', '0', '6', '0',
-    '7', '0', '8', '0', '9', '1', '0', '1', '1', '1', '2', '1', '3', '1', '4',
-    '1', '5', '1', '6', '1', '7', '1', '8', '1', '9', '2', '0', '2', '1', '2',
-    '2', '2', '3', '2', '4', '2', '5', '2', '6', '2', '7', '2', '8', '2', '9',
-    '3', '0', '3', '1', '3', '2', '3', '3', '3', '4', '3', '5', '3', '6', '3',
-    '7', '3', '8', '3', '9', '4', '0', '4', '1', '4', '2', '4', '3', '4', '4',
-    '4', '5', '4', '6', '4', '7', '4', '8', '4', '9', '5', '0', '5', '1', '5',
-    '2', '5', '3', '5', '4', '5', '5', '5', '6', '5', '7', '5', '8', '5', '9',
-    '6', '0', '6', '1', '6', '2', '6', '3', '6', '4', '6', '5', '6', '6', '6',
-    '7', '6', '8', '6', '9', '7', '0', '7', '1', '7', '2', '7', '3', '7', '4',
-    '7', '5', '7', '6', '7', '7', '7', '8', '7', '9', '8', '0', '8', '1', '8',
-    '2', '8', '3', '8', '4', '8', '5', '8', '6', '8', '7', '8', '8', '8', '9',
-    '9', '0', '9', '1', '9', '2', '9', '3', '9', '4', '9', '5', '9', '6', '9',
-    '7', '9', '8', '9', '9'};
-
 template <typename T>
 inline _LIBCPP_INLINE_VISIBILITY char*
 append1(char* buffer, T i) noexcept
@@ -42,7 +29,7 @@ template <typename T>
 inline _LIBCPP_INLINE_VISIBILITY char*
 append2(char* buffer, T i) noexcept
 {
-    memcpy(buffer, &cDigitsLut[(i)*2], 2);
+    memcpy(buffer, &__DIGIT_TABLE[(i)*2], 2);
     return buffer + 2;
 }
 
@@ -157,4 +144,53 @@ __u64toa(uint64_t value, char* buffer) noexcept
 
 }  // namespace __itoa
 
+// The original version of floating-point to_chars was written by Microsoft and
+// contributed with the following license.
+
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// This implementation is dedicated to the memory of Mary and Thavatchai.
+
+to_chars_result to_chars(char* __first, char* __last, float __value) {
+  return _Floating_to_chars<_Floating_to_chars_overload::_Plain>(__first, __last, __value, chars_format{}, 0);
+}
+
+to_chars_result to_chars(char* __first, char* __last, double __value) {
+  return _Floating_to_chars<_Floating_to_chars_overload::_Plain>(__first, __last, __value, chars_format{}, 0);
+}
+
+to_chars_result to_chars(char* __first, char* __last, long double __value) {
+  return _Floating_to_chars<_Floating_to_chars_overload::_Plain>(__first, __last, static_cast<double>(__value),
+                                                                 chars_format{}, 0);
+}
+
+to_chars_result to_chars(char* __first, char* __last, float __value, chars_format __fmt) {
+  return _Floating_to_chars<_Floating_to_chars_overload::_Format_only>(__first, __last, __value, __fmt, 0);
+}
+
+to_chars_result to_chars(char* __first, char* __last, double __value, chars_format __fmt) {
+  return _Floating_to_chars<_Floating_to_chars_overload::_Format_only>(__first, __last, __value, __fmt, 0);
+}
+
+to_chars_result to_chars(char* __first, char* __last, long double __value, chars_format __fmt) {
+  return _Floating_to_chars<_Floating_to_chars_overload::_Format_only>(__first, __last, static_cast<double>(__value),
+                                                                       __fmt, 0);
+}
+
+to_chars_result to_chars(char* __first, char* __last, float __value, chars_format __fmt, int __precision) {
+  return _Floating_to_chars<_Floating_to_chars_overload::_Format_precision>(__first, __last, __value, __fmt,
+                                                                            __precision);
+}
+
+to_chars_result to_chars(char* __first, char* __last, double __value, chars_format __fmt, int __precision) {
+  return _Floating_to_chars<_Floating_to_chars_overload::_Format_precision>(__first, __last, __value, __fmt,
+                                                                            __precision);
+}
+
+to_chars_result to_chars(char* __first, char* __last, long double __value, chars_format __fmt, int __precision) {
+  return _Floating_to_chars<_Floating_to_chars_overload::_Format_precision>(
+      __first, __last, static_cast<double>(__value), __fmt, __precision);
+}
+
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/src/chrono.cpp b/libcxx/src/chrono.cpp
index 8ff3faf9df6a..5aa7af75894b 100644
--- a/libcxx/src/chrono.cpp
+++ b/libcxx/src/chrono.cpp
@@ -80,9 +80,9 @@ public:
   GetSystemTimeAsFileTimePtr fp;
 };
 
-# 83 "chrono.cpp" 1 3
-GetSystemTimeInit GetSystemTimeAsFileTimeFunc _LIBCPP_INIT_PRIORITY_MAX;
-# 85 "chrono.cpp" 2
+// Pretend we're inside a system header so the compiler doesn't flag the use of the init_priority
+// attribute with a value that's reserved for the implementation (we're the implementation).
+#include "chrono_system_time_init.h"
 } // namespace
 
 #endif
diff --git a/libcxx/src/chrono_system_time_init.h b/libcxx/src/chrono_system_time_init.h
new file mode 100644
index 000000000000..3c5a0c33a56a
--- /dev/null
+++ b/libcxx/src/chrono_system_time_init.h
@@ -0,0 +1,2 @@
+#pragma GCC system_header
+GetSystemTimeInit GetSystemTimeAsFileTimeFunc _LIBCPP_INIT_PRIORITY_MAX;
\ No newline at end of file
diff --git a/libcxx/src/experimental/memory_resource.cpp b/libcxx/src/experimental/memory_resource.cpp
index be2fb47fad16..018d0159281a 100644
--- a/libcxx/src/experimental/memory_resource.cpp
+++ b/libcxx/src/experimental/memory_resource.cpp
@@ -76,9 +76,9 @@ union ResourceInitHelper {
   ~ResourceInitHelper() {}
 };
 
-# 79 "memory_resource.cpp" 1 3
-_LIBCPP_SAFE_STATIC ResourceInitHelper res_init _LIBCPP_INIT_PRIORITY_MAX;
-# 81 "memory_resource.cpp" 2
+// Pretend we're inside a system header so the compiler doesn't flag the use of the init_priority
+// attribute with a value that's reserved for the implementation (we're the implementation).
+#include "memory_resource_init_helper.h"
 
 } // end namespace
 
@@ -97,8 +97,7 @@ static memory_resource *
 __default_memory_resource(bool set = false, memory_resource * new_res = nullptr) noexcept
 {
 #ifndef _LIBCPP_HAS_NO_ATOMIC_HEADER
-    _LIBCPP_SAFE_STATIC static atomic<memory_resource*> __res =
-        ATOMIC_VAR_INIT(&res_init.resources.new_delete_res);
+    _LIBCPP_SAFE_STATIC static atomic<memory_resource*> __res{&res_init.resources.new_delete_res};
     if (set) {
         new_res = new_res ? new_res : new_delete_resource();
         // TODO: Can a weaker ordering be used?
diff --git a/libcxx/src/experimental/memory_resource_init_helper.h b/libcxx/src/experimental/memory_resource_init_helper.h
new file mode 100644
index 000000000000..2e1cae5ecc60
--- /dev/null
+++ b/libcxx/src/experimental/memory_resource_init_helper.h
@@ -0,0 +1,2 @@
+#pragma GCC system_header
+_LIBCPP_SAFE_STATIC ResourceInitHelper res_init _LIBCPP_INIT_PRIORITY_MAX;
\ No newline at end of file
diff --git a/libcxx/src/filesystem/directory_iterator.cpp b/libcxx/src/filesystem/directory_iterator.cpp
index 6219ceafd36e..90b255d9877f 100644
--- a/libcxx/src/filesystem/directory_iterator.cpp
+++ b/libcxx/src/filesystem/directory_iterator.cpp
@@ -6,8 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "filesystem"
 #include "__config"
+#include "filesystem"
+#include "stack"
 #if defined(_LIBCPP_WIN32API)
 #define WIN32_LEAN_AND_MEAN
 #define NOMINMAX
diff --git a/libcxx/src/filesystem/filesystem_common.h b/libcxx/src/filesystem/filesystem_common.h
index a2c340e61083..717894e537d2 100644
--- a/libcxx/src/filesystem/filesystem_common.h
+++ b/libcxx/src/filesystem/filesystem_common.h
@@ -17,6 +17,7 @@
 #include "cstdlib"
 #include "ctime"
 #include "filesystem"
+#include "system_error"
 
 #if !defined(_LIBCPP_WIN32API)
 # include <unistd.h>
diff --git a/libcxx/src/format.cpp b/libcxx/src/format.cpp
index 7ae2af59b7fa..2ebec28247e6 100644
--- a/libcxx/src/format.cpp
+++ b/libcxx/src/format.cpp
@@ -10,10 +10,6 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if _LIBCPP_STD_VER > 17
-
 format_error::~format_error() noexcept = default;
 
-#endif //_LIBCPP_STD_VER > 17
-
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/src/include/ryu/common.h b/libcxx/src/include/ryu/common.h
new file mode 100644
index 000000000000..52913120f107
--- /dev/null
+++ b/libcxx/src/include/ryu/common.h
@@ -0,0 +1,107 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// Copyright 2018 Ulf Adams
+// Copyright (c) Microsoft Corporation. All rights reserved.
+
+// Boost Software License - Version 1.0 - August 17th, 2003
+
+// Permission is hereby granted, free of charge, to any person or organization
+// obtaining a copy of the software and accompanying documentation covered by
+// this license (the "Software") to use, reproduce, display, distribute,
+// execute, and transmit the Software, and to prepare derivative works of the
+// Software, and to permit third-parties to whom the Software is furnished to
+// do so, all subject to the following:
+
+// The copyright notices in the Software and this entire statement, including
+// the above license grant, this restriction and the following disclaimer,
+// must be included in all copies of the Software, in whole or in part, and
+// all derivative works of the Software, unless such copies or derivative
+// works are solely in the form of machine-executable object code generated by
+// a source language processor.
+
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+// SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+// FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+// DEALINGS IN THE SOFTWARE.
+
+#ifndef _LIBCPP_SRC_INCLUDE_RYU_COMMON_H
+#define _LIBCPP_SRC_INCLUDE_RYU_COMMON_H
+
+// Avoid formatting to keep the changes with the original code minimal.
+// clang-format off
+
+#include "__config"
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline uint32_t __decimalLength9(const uint32_t __v) {
+  // Function precondition: __v is not a 10-digit number.
+  // (f2s: 9 digits are sufficient for round-tripping.)
+  // (d2fixed: We print 9-digit blocks.)
+  _LIBCPP_ASSERT(__v < 1000000000, "");
+  if (__v >= 100000000) { return 9; }
+  if (__v >= 10000000) { return 8; }
+  if (__v >= 1000000) { return 7; }
+  if (__v >= 100000) { return 6; }
+  if (__v >= 10000) { return 5; }
+  if (__v >= 1000) { return 4; }
+  if (__v >= 100) { return 3; }
+  if (__v >= 10) { return 2; }
+  return 1;
+}
+
+// Returns __e == 0 ? 1 : ceil(log_2(5^__e)).
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI  inline int32_t __pow5bits(const int32_t __e) {
+  // This approximation works up to the point that the multiplication overflows at __e = 3529.
+  // If the multiplication were done in 64 bits, it would fail at 5^4004 which is just greater
+  // than 2^9297.
+  _LIBCPP_ASSERT(__e >= 0, "");
+  _LIBCPP_ASSERT(__e <= 3528, "");
+  return static_cast<int32_t>(((static_cast<uint32_t>(__e) * 1217359) >> 19) + 1);
+}
+
+// Returns floor(log_10(2^__e)).
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI  inline uint32_t __log10Pow2(const int32_t __e) {
+  // The first value this approximation fails for is 2^1651 which is just greater than 10^297.
+  _LIBCPP_ASSERT(__e >= 0, "");
+  _LIBCPP_ASSERT(__e <= 1650, "");
+  return (static_cast<uint32_t>(__e) * 78913) >> 18;
+}
+
+// Returns floor(log_10(5^__e)).
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline uint32_t __log10Pow5(const int32_t __e) {
+  // The first value this approximation fails for is 5^2621 which is just greater than 10^1832.
+  _LIBCPP_ASSERT(__e >= 0, "");
+  _LIBCPP_ASSERT(__e <= 2620, "");
+  return (static_cast<uint32_t>(__e) * 732923) >> 20;
+}
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline uint32_t __float_to_bits(const float __f) {
+  uint32_t __bits = 0;
+  _VSTD::memcpy(&__bits, &__f, sizeof(float));
+  return __bits;
+}
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline uint64_t __double_to_bits(const double __d) {
+  uint64_t __bits = 0;
+  _VSTD::memcpy(&__bits, &__d, sizeof(double));
+  return __bits;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+// clang-format on
+
+#endif // _LIBCPP_SRC_INCLUDE_RYU_COMMON_H
diff --git a/libcxx/src/include/ryu/d2fixed.h b/libcxx/src/include/ryu/d2fixed.h
new file mode 100644
index 000000000000..e495c11b1d4f
--- /dev/null
+++ b/libcxx/src/include/ryu/d2fixed.h
@@ -0,0 +1,60 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// Copyright 2018 Ulf Adams
+// Copyright (c) Microsoft Corporation. All rights reserved.
+
+// Boost Software License - Version 1.0 - August 17th, 2003
+
+// Permission is hereby granted, free of charge, to any person or organization
+// obtaining a copy of the software and accompanying documentation covered by
+// this license (the "Software") to use, reproduce, display, distribute,
+// execute, and transmit the Software, and to prepare derivative works of the
+// Software, and to permit third-parties to whom the Software is furnished to
+// do so, all subject to the following:
+
+// The copyright notices in the Software and this entire statement, including
+// the above license grant, this restriction and the following disclaimer,
+// must be included in all copies of the Software, in whole or in part, and
+// all derivative works of the Software, unless such copies or derivative
+// works are solely in the form of machine-executable object code generated by
+// a source language processor.
+
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+// SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+// FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+// DEALINGS IN THE SOFTWARE.
+
+#ifndef _LIBCPP_SRC_INCLUDE_RYU_D2FIXED_H
+#define _LIBCPP_SRC_INCLUDE_RYU_D2FIXED_H
+
+// Avoid formatting to keep the changes with the original code minimal.
+// clang-format off
+
+#include "__config"
+#include "cstdint"
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+void __append_n_digits(const uint32_t __olength, uint32_t __digits, char* const __result);
+void __append_nine_digits(uint32_t __digits, char* const __result);
+
+[[nodiscard]] to_chars_result __d2fixed_buffered_n(char* _First, char* const _Last, const double __d, const uint32_t __precision);
+[[nodiscard]] to_chars_result __d2exp_buffered_n(char* _First, char* const _Last, const double __d, uint32_t __precision);
+
+_LIBCPP_END_NAMESPACE_STD
+
+// clang-format on
+
+#endif // _LIBCPP_SRC_INCLUDE_RYU_D2FIXED_H
diff --git a/libcxx/src/include/ryu/d2fixed_full_table.h b/libcxx/src/include/ryu/d2fixed_full_table.h
new file mode 100644
index 000000000000..7181cd8cf6e2
--- /dev/null
+++ b/libcxx/src/include/ryu/d2fixed_full_table.h
@@ -0,0 +1,4451 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// Copyright 2018 Ulf Adams
+// Copyright (c) Microsoft Corporation. All rights reserved.
+
+// Boost Software License - Version 1.0 - August 17th, 2003
+
+// Permission is hereby granted, free of charge, to any person or organization
+// obtaining a copy of the software and accompanying documentation covered by
+// this license (the "Software") to use, reproduce, display, distribute,
+// execute, and transmit the Software, and to prepare derivative works of the
+// Software, and to permit third-parties to whom the Software is furnished to
+// do so, all subject to the following:
+
+// The copyright notices in the Software and this entire statement, including
+// the above license grant, this restriction and the following disclaimer,
+// must be included in all copies of the Software, in whole or in part, and
+// all derivative works of the Software, unless such copies or derivative
+// works are solely in the form of machine-executable object code generated by
+// a source language processor.
+
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+// SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+// FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+// DEALINGS IN THE SOFTWARE.
+
+#ifndef _LIBCPP_SRC_INCLUDE_RYU_D2FIXED_FULL_TABLE_H
+#define _LIBCPP_SRC_INCLUDE_RYU_D2FIXED_FULL_TABLE_H
+
+// Avoid formatting to keep the changes with the original code minimal.
+// clang-format off
+
+#include "__config"
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+inline constexpr int __TABLE_SIZE = 64;
+
+inline constexpr uint16_t __POW10_OFFSET[__TABLE_SIZE] = {
+  0, 2, 5, 8, 12, 16, 21, 26, 32, 39,
+  46, 54, 62, 71, 80, 90, 100, 111, 122, 134,
+  146, 159, 173, 187, 202, 217, 233, 249, 266, 283,
+  301, 319, 338, 357, 377, 397, 418, 440, 462, 485,
+  508, 532, 556, 581, 606, 632, 658, 685, 712, 740,
+  769, 798, 828, 858, 889, 920, 952, 984, 1017, 1050,
+  1084, 1118, 1153, 1188
+};
+
+inline constexpr uint64_t __POW10_SPLIT[1224][3] = {
+  {                    1u,    72057594037927936u,                    0u },
+  {   699646928636035157u,             72057594u,                    0u },
+  {                    1u,                    0u,                  256u },
+  { 11902091922964236229u,        4722366482869u,                    0u },
+  {  6760415703743915872u,                 4722u,                    0u },
+  {                    1u,                    0u,             16777216u },
+  { 13369850649504950658u,   309485009821345068u,                    0u },
+  { 15151142278969419334u,            309485009u,                    0u },
+  {                    1u,                    0u,          75511627776u },
+  {  4635408826454083567u,  9437866644873197963u,                 1099u },
+  { 12367138975830625353u,       20282409603651u,                    0u },
+  {  7555853734021184432u,                20282u,                    0u },
+  {                    1u,                    0u,         250037927936u },
+  {  5171444645924616995u,   699646928636035156u,             72057594u },
+  { 16672297533003297786u,  1329227995784915872u,                    0u },
+  { 14479142226848862515u,           1329227995u,                    0u },
+  {                    1u,                    0u,         181645213696u },
+  { 12214193123817091081u, 11902091922964236228u,         114366482869u },
+  { 16592893013671929435u,  6760415703743915871u,                 4722u },
+  {  4549827147718617003u,       87112285931760u,                    0u },
+  {  5274510003815168971u,                87112u,                    0u },
+  {                    1u,                    0u,          44724781056u },
+  {  9794971998307800535u, 13369850649504950657u,         209821345068u },
+  { 14720142899209240169u, 15151142278969419333u,            309485009u },
+  {  4300745446091561535u,  5708990770823839524u,                    0u },
+  { 15197156861335443364u,           5708990770u,                    0u },
+  {                    1u,                    0u,         139251286016u },
+  { 13484604155038683037u,  4635408826454083566u,          67670423947u },
+  {  8356963862052375699u, 12367138975830625352u,          58409603651u },
+  {  5850852848337610021u,  7555853734021184431u,                20282u },
+  {  2712780827214982050u,      374144419156711u,                    0u },
+  {  7732076577307618052u,               374144u,                    0u },
+  {                    1u,                    0u,          84280344576u },
+  { 17296309485351745867u,  5171444645924616994u,         160903807060u },
+  { 16598859101615853088u, 16672297533003297785u,         219784915872u },
+  {  7469952526870444257u, 14479142226848862514u,           1329227995u },
+  { 13531654022114669524u,  6073184580144670117u,                    1u },
+  { 15757609704383306943u,          24519928653u,                    0u },
+  {  9590990814237149590u,                   24u,                    0u },
+  {                    1u,                    0u,         196662132736u },
+  { 15408590707489433890u, 12214193123817091080u,          95899502532u },
+  { 18332056844289122710u, 16592893013671929434u,         240246646623u },
+  { 11114572877353986193u,  4549827147718617002u,          72285931760u },
+  {  1703393793997526525u,  5274510003815168970u,                87112u },
+  {  5082852056285196265u,     1606938044258990u,                    0u },
+  {   816434266573722365u,              1606938u,                    0u },
+  {                    1u,                    0u,         129530986496u },
+  {  5736523019264798742u,  9794971998307800534u,          69797980545u },
+  { 10129314776268243339u, 14720142899209240168u,          36233143877u },
+  { 16511595775483995364u,  4300745446091561534u,          50823839524u },
+  { 12367293405401453325u, 15197156861335443363u,           5708990770u },
+  { 16934621733248854291u, 13078571300009428617u,                    5u },
+  { 10278280417769171336u,         105312291668u,                    0u },
+  {  5760764486226151240u,                  105u,                    0u },
+  {                    1u,                    0u,         238731001856u },
+  {  4128368337188369761u, 13484604155038683036u,          72453031918u },
+  { 10240941003671005056u,  8356963862052375698u,         175317175368u },
+  { 17933378316822368251u,  5850852848337610020u,         231147060143u },
+  {  8346249813075698616u,  2712780827214982049u,         128419156711u },
+  { 15906203609160902695u,  7732076577307618051u,               374144u },
+  { 14525607416135386328u,     6901746346790563u,                    0u },
+  {  6397156777364256320u,              6901746u,                    0u },
+  {                    1u,                    0u,          34937634816u },
+  { 16798760952716600048u, 17296309485351745866u,         249899825954u },
+  {  2419982808370854967u, 16598859101615853087u,          50404946937u },
+  {  2922947087773078956u,  7469952526870444256u,         165733552434u },
+  { 15419220167069510190u, 13531654022114669523u,          77854221733u },
+  {  3452124642157173416u, 15757609704383306942u,          24519928653u },
+  {  5979700067267186899u,  9590990814237149589u,                   24u },
+  {  4913998146922579597u,         452312848583u,                    0u },
+  {  5771037749337678924u,                  452u,                    0u },
+  {                    1u,                    0u,           8835301376u },
+  {  3464734175350698519u, 15408590707489433889u,          90993782792u },
+  {  9334527711335850125u, 18332056844289122709u,         170602522202u },
+  {  7269882896518450106u, 11114572877353986192u,         202092341162u },
+  {  1372511258182263196u,  1703393793997526524u,         174275541962u },
+  {  7571228438575951046u,  5082852056285196264u,          26044258990u },
+  {  2992506536646070406u,   816434266573722364u,              1606938u },
+  {   524517896824344606u,    29642774844752946u,                    0u },
+  { 15582941400898702773u,             29642774u,                    0u },
+  {                    1u,                    0u,         214310977536u },
+  {  3846112492507251066u,  5736523019264798741u,         104549111254u },
+  { 16681117750123089487u, 10129314776268243338u,          62895095400u },
+  { 14986314536556547267u, 16511595775483995363u,         163670432318u },
+  {  2573712825027107389u, 12367293405401453324u,         137918027683u },
+  {  7504855874008324928u, 16934621733248854290u,          84557186697u },
+  {  9572138030626879787u, 10278280417769171335u,         105312291668u },
+  {  8520676959353394843u,  5760764486226151239u,                  105u },
+  { 13448984662897903496u,        1942668892225u,                    0u },
+  { 12338883700918130648u,                 1942u,                    0u },
+  {                    1u,                    0u,         156223799296u },
+  {  2517285787892561600u,  4128368337188369760u,         146555162524u },
+  {  4338831817635138103u, 10240941003671005055u,          36972170386u },
+  {  1561495325934523196u, 17933378316822368250u,         161452451108u },
+  { 12262635050079398786u,  8346249813075698615u,           3862277025u },
+  { 11144065765517284188u, 15906203609160902694u,         163787434755u },
+  {  1212260522471875711u, 14525607416135386327u,         242346790563u },
+  {  9695352922247418869u,  6397156777364256319u,              6901746u },
+  {  7227025834627242948u,   127314748520905380u,                    0u },
+  {  9609008238705447829u,            127314748u,                    0u },
+  {                    1u,                    0u,          74910662656u },
+  {  3609144142396852269u, 16798760952716600047u,          31131187530u },
+  { 11568848377382068865u,  2419982808370854966u,         224158453279u },
+  { 10068303578029323957u,  2922947087773078955u,         211835877600u },
+  { 11645070846862630231u, 15419220167069510189u,         190187140051u },
+  { 12449386705878485055u,  3452124642157173415u,         149324160190u },
+  { 15025619323517318418u,  5979700067267186898u,         199266388373u },
+  { 14996237555047131272u,  4913998146922579596u,         196312848583u },
+  { 10211005638256058413u,  5771037749337678923u,                  452u },
+  {  1014743503555840530u,        8343699359066u,                    0u },
+  { 12900897707145290678u,                 8343u,                    0u },
+  {                    1u,                    0u,          33187823616u },
+  {  4718003016239473662u,  3464734175350698518u,         149506025761u },
+  { 14865830648693666725u,  9334527711335850124u,         144394101141u },
+  { 14754517212823091778u,  7269882896518450105u,         252074403984u },
+  { 11113946551474911901u,  1372511258182263195u,         232410437116u },
+  {  1963520352638130630u,  7571228438575951045u,         252162224104u },
+  { 13342587341404964200u,  2992506536646070405u,          50028434172u },
+  {  6240392545013573291u,   524517896824344605u,          22844752946u },
+  { 14377490861349714758u, 15582941400898702772u,             29642774u },
+  {  1717863312631397839u,   546812681195752981u,                    0u },
+  {  3611005143890591770u,            546812681u,                    0u },
+  {                    1u,                    0u,          21208498176u },
+  { 13168252824351245504u,  3846112492507251065u,         138904285205u },
+  {   735883891883379688u, 16681117750123089486u,         227812409738u },
+  { 10609203866866106404u, 14986314536556547266u,          12139521251u },
+  { 12358191111890306470u,  2573712825027107388u,          18406839052u },
+  { 15229916368406413528u,  7504855874008324927u,         135518906642u },
+  {  7241424335568075942u,  9572138030626879786u,          71461906823u },
+  {  6049715868779871913u,  8520676959353394842u,          65729070919u },
+  {  2000548404719336762u, 13448984662897903495u,         150668892225u },
+  {  1410974761895205301u, 12338883700918130647u,                 1942u },
+  { 16000132467694084868u,       35835915874844u,                    0u },
+  { 16894908866816792556u,                35835u,                    0u },
+  {                    1u,                    0u,          96136462336u },
+  {   589096329272056762u,  2517285787892561599u,         127235208544u },
+  {  7097729792403256904u,  4338831817635138102u,         250084648831u },
+  {  8553736750439287020u,  1561495325934523195u,         183664758778u },
+  {  2114152625261065696u, 12262635050079398785u,          38604121015u },
+  {  9817523680007641224u, 11144065765517284187u,         215065716774u },
+  { 13047215537500048015u,  1212260522471875710u,          63525586135u },
+  { 16755544192002345880u,  9695352922247418868u,         164391777855u },
+  {  6930119832670648356u,  7227025834627242947u,          60520905380u },
+  { 14560698131901886167u,  9609008238705447828u,            127314748u },
+  { 16408020927503338035u,  2348542582773833227u,                    0u },
+  { 14274703510609809116u,           2348542582u,                    0u },
+  {                    1u,                    0u,         239195652096u },
+  { 16428432973129962470u,  3609144142396852268u,          54627148527u },
+  {  3721112279790863774u, 11568848377382068864u,         171545803830u },
+  { 18032764903259620753u, 10068303578029323956u,          45631280555u },
+  { 18058455550468776079u, 11645070846862630230u,         167674882605u },
+  { 15692090139033993190u, 12449386705878485054u,         210814540455u },
+  {   389416944300619393u, 15025619323517318417u,         140812947666u },
+  { 12009691357260487293u, 14996237555047131271u,          75553539724u },
+  { 13494259174449809900u, 10211005638256058412u,          90055009355u },
+  { 18288583400616279877u,  1014743503555840529u,         151699359066u },
+  {     7216107869057472u, 12900897707145290677u,                 8343u },
+  { 17237061291959073878u,      153914086704665u,                    0u },
+  {  1599418782488783273u,               153914u,                    0u },
+  {                    1u,                    0u,          22255763456u },
+  {  9565464987240335777u,  4718003016239473661u,         140805878294u },
+  {   857713933775880687u, 14865830648693666724u,         185799843980u },
+  {  4621617820081363356u, 14754517212823091777u,         155602488249u },
+  {  9630162611715632528u, 11113946551474911900u,         197106442651u },
+  {  9283986497984645815u,  1963520352638130629u,         133723303109u },
+  {  8981807745082630996u, 13342587341404964199u,          29338292357u },
+  { 18350140531565934622u,  6240392545013573290u,         180779405341u },
+  {  4411619033127524143u, 14377490861349714757u,          21093125556u },
+  {  1852297584111266889u,  1717863312631397838u,           9195752981u },
+  { 11746243463811666096u,  3611005143890591769u,            546812681u },
+  {  6335244004343789147u, 10086913586276986678u,                    0u },
+  {  5109502367228239844u,          10086913586u,                    0u },
+  {  1603272682579847821u,                   10u,                    0u },
+  {                    1u,                    0u,         121713852416u },
+  {  6609546910952910052u, 13168252824351245503u,          78039892345u },
+  {  3911171343112928288u,   735883891883379687u,         194575126094u },
+  {  5254510615100863555u, 10609203866866106403u,          60669938882u },
+  {  3881927570803887650u, 12358191111890306469u,          63825615420u },
+  {  6379348759607163190u, 15229916368406413527u,          42392558399u },
+  { 14595733737222406466u,  7241424335568075941u,         154327955754u },
+  { 14670223432002373542u,  6049715868779871912u,         135108449946u },
+  {  4045087795619708513u,  2000548404719336761u,         215076489095u },
+  { 12598467307137142718u,  1410974761895205300u,          28867368919u },
+  {   734704388050777108u, 16000132467694084867u,         251915874844u },
+  {  5682201693687285822u, 16894908866816792555u,                35835u },
+  { 11048712694145438788u,      661055968790248u,                    0u },
+  { 17871025777010319485u,               661055u,                    0u },
+  {                    1u,                    0u,         191031934976u },
+  { 15268761435931663695u,   589096329272056761u,          54384768703u },
+  {  5016238054648555438u,  7097729792403256903u,          59463698998u },
+  { 14236047313993899750u,  8553736750439287019u,         129114608443u },
+  {  6957759675154690848u,  2114152625261065695u,          91532209025u },
+  { 18439367135478514473u,  9817523680007641223u,         126707290971u },
+  {  8539004472540641041u, 13047215537500048014u,         244908319870u },
+  {  1908462039431738399u, 16755544192002345879u,         195375682548u },
+  {   714690453250792146u,  6930119832670648355u,         148789337027u },
+  { 13782189447673929633u, 14560698131901886166u,          11889480596u },
+  {  3584742913798803164u, 16408020927503338034u,         118773833227u },
+  {  4347581515245125291u, 14274703510609809115u,           2348542582u },
+  { 16836742268156371392u,  6429475823218628948u,                    2u },
+  { 11764082328865615308u,          43322963970u,                    0u },
+  {  5957633711383291746u,                   43u,                    0u },
+  {                    1u,                    0u,          44890587136u },
+  {  9917186842884466953u, 16428432973129962469u,         128201721900u },
+  {  4751011869809829335u,  3721112279790863773u,         180977558144u },
+  { 11068497969931435029u, 18032764903259620752u,          86978950836u },
+  { 17118056985122509954u, 18058455550468776078u,          62850669910u },
+  { 14607066080907684459u, 15692090139033993189u,          17021110334u },
+  { 11768892370493391107u,   389416944300619392u,         135651046673u },
+  {  4043396447647747170u, 12009691357260487292u,          44731525255u },
+  {  1670341095362518057u, 13494259174449809899u,          17991426092u },
+  {  3190817644167043165u, 18288583400616279876u,         181000391185u },
+  { 10425820027224322486u,     7216107869057471u,          25934422965u },
+  { 13139964660506311565u, 17237061291959073877u,          58086704665u },
+  {  2297772885416059937u,  1599418782488783272u,               153914u },
+  {  7677687919964523763u,     2839213766779714u,                    0u },
+  { 14144589152747892828u,              2839213u,                    0u },
+  {                    1u,                    0u,         253518544896u },
+  { 17069730341503660290u,  9565464987240335776u,         164046496765u },
+  { 18167423787163077107u,   857713933775880686u,          65250538404u },
+  {  3765746945827805904u,  4621617820081363355u,         156522052161u },
+  { 10241734342430761691u,  9630162611715632527u,         197503285916u },
+  { 13345717282537140784u,  9283986497984645814u,         103486904773u },
+  {  9313926784816939953u,  8981807745082630995u,         170994763111u },
+  {   550974205049535019u, 18350140531565934621u,          69239154346u },
+  {  4494692285504086222u,  4411619033127524142u,         206100413253u },
+  {  1134308559863725587u,  1852297584111266888u,          25636765134u },
+  { 17587558045116130233u, 11746243463811666095u,          54343434265u },
+  {  9817142032346161594u,  6335244004343789146u,          50276986678u },
+  {  6071944935834172568u,  5109502367228239843u,          10086913586u },
+  { 11564168293299416955u,  1603272682579847820u,                   10u },
+  { 12458266507226064437u,         186070713419u,                    0u },
+  {  1304432355328256915u,                  186u,                    0u },
+  {                    1u,                    0u,         191358304256u },
+  { 15946798815542087355u,  6609546910952910051u,         231212025023u },
+  { 12082566083831286138u,  3911171343112928287u,          35284847591u },
+  { 11449623684706196411u,  5254510615100863554u,         165210439715u },
+  { 17518743620362604446u,  3881927570803887649u,         215345825189u },
+  {  9451061563087633805u,  6379348759607163189u,         165791236311u },
+  { 13191114787623314926u, 14595733737222406465u,         168795274405u },
+  {  8367349876734474799u, 14670223432002373541u,          57219284648u },
+  {  6544253801674393507u,  4045087795619708512u,         180682964281u },
+  { 16113906253336597498u, 12598467307137142717u,           3039828404u },
+  { 10294087136797312392u,   734704388050777107u,         235308032771u },
+  {  9127173070014462803u,  5682201693687285821u,         232598951915u },
+  { 16266900839595484952u, 11048712694145438787u,          63968790248u },
+  {  3299745387370952632u, 17871025777010319484u,               661055u },
+  { 12061115182604399189u,    12194330274671844u,                    0u },
+  {  5066801222582989646u,             12194330u,                    0u },
+  {                    1u,                    0u,         185827721216u },
+  {  7568423425299591513u, 15268761435931663694u,          71271930809u },
+  { 16561505984665207377u,  5016238054648555437u,         235771737671u },
+  {  4329114621856906245u, 14236047313993899749u,         223377180907u },
+  {  1477500474861899139u,  6957759675154690847u,         135999600095u },
+  { 16891579639263969684u, 18439367135478514472u,         142462900359u },
+  {  4684451357140027420u,  8539004472540641040u,         151103457934u },
+  { 14727186580409080709u,  1908462039431738398u,          35038743447u },
+  { 15864176859687308834u,   714690453250792145u,         214747133987u },
+  {  1755486942842684438u, 13782189447673929632u,          50194329302u },
+  { 17417077516652710041u,  3584742913798803163u,         219235682866u },
+  {  4290982361913532783u,  4347581515245125290u,          84912721627u },
+  { 11826659981004351409u, 16836742268156371391u,           2637732180u },
+  {   932930645678090820u, 11764082328865615307u,          43322963970u },
+  { 12707792781328052617u,  5957633711383291745u,                   43u },
+  { 16491596426880311906u,         799167628880u,                    0u },
+  {  3092207065214166010u,                  799u,                    0u },
+  {                    1u,                    0u,         229537611776u },
+  {  8142946531605512550u,  9917186842884466952u,         157257552869u },
+  {  5328402096432654515u,  4751011869809829334u,         144600024477u },
+  {  1932004361303814512u, 11068497969931435028u,         142927971728u },
+  {  2511477647985517771u, 17118056985122509953u,         229791850638u },
+  { 17451375493324716694u, 14607066080907684458u,         128637992933u },
+  {  9489266854478998489u, 11768892370493391106u,         124219192960u },
+  {  8803053132063235169u,  4043396447647747169u,         235090549372u },
+  { 16198682197142616773u,  1670341095362518056u,          68172974571u },
+  { 13696242485403414202u,  3190817644167043164u,         191565184836u },
+  { 16409082426079859931u, 10425820027224322485u,          85712318911u },
+  { 11653410736879597610u, 13139964660506311564u,         168124562517u },
+  { 13589514120653213261u,  2297772885416059936u,          66416208296u },
+  {  8032934885905905774u,  7677687919964523762u,         173766779714u },
+  {  2753021350129449273u, 14144589152747892827u,              2839213u },
+  { 16974897459201404133u,    52374249726338269u,                    0u },
+  { 13398576176159101589u,             52374249u,                    0u },
+  {                    1u,                    0u,         160925351936u },
+  { 10284586955251725351u, 17069730341503660289u,         238984858016u },
+  {  5294476488634150891u, 18167423787163077106u,         155204141550u },
+  { 15833244538135063323u,  3765746945827805903u,         143555205531u },
+  { 10348512742273116664u, 10241734342430761690u,         182723472783u },
+  { 13658504610142595663u, 13345717282537140783u,          83504908982u },
+  { 11956362239240850266u,  9313926784816939952u,          29029868371u },
+  { 13415901703662731781u,   550974205049535018u,          46243657757u },
+  {  5161774027546852762u,  4494692285504086221u,          72061490990u },
+  { 15274384838790587711u,  1134308559863725586u,         175953423432u },
+  { 14233354597679374929u, 17587558045116130232u,          90532188335u },
+  {  4274656492162486921u,  9817142032346161593u,         227329160794u },
+  { 12040276505541795046u,  6071944935834172567u,         140626894819u },
+  { 13238307206256765457u, 11564168293299416954u,          75675363980u },
+  { 12850161204172713271u, 12458266507226064436u,         186070713419u },
+  { 17531777095001445154u,  1304432355328256914u,                  186u },
+  {  5623628114515245990u,        3432398830065u,                    0u },
+  {  7357116143579573377u,                 3432u,                    0u },
+  {                    1u,                    0u,         227864477696u },
+  {  3555734177475596582u, 15946798815542087354u,          31654997219u },
+  { 14001876724756424382u, 12082566083831286137u,          66620685343u },
+  { 18159905057231476140u, 11449623684706196410u,          33949692994u },
+  {  5585207679308509467u, 17518743620362604445u,          53512343073u },
+  { 13948697622866724672u,  9451061563087633804u,          65715091765u },
+  {  9807691927739036432u, 13191114787623314925u,         165453594945u },
+  { 15818010096140820918u,  8367349876734474798u,          96354764709u },
+  {  5629845624785010943u,  6544253801674393506u,         189873536608u },
+  {  9517635131137734707u, 16113906253336597497u,          19558043581u },
+  {   619338244618780585u, 10294087136797312391u,          61494785043u },
+  { 11632367007491958899u,  9127173070014462802u,          67881830461u },
+  { 12083314261009739916u, 16266900839595484951u,         124178879555u },
+  { 16880538609458881650u,  3299745387370952631u,         228653834364u },
+  { 17404223674486504228u, 12061115182604399188u,          26274671844u },
+  {  7089067015287185433u,  5066801222582989645u,             12194330u },
+  {  2592264228029443648u,   224945689727159819u,                    0u },
+  { 13413731084370224440u,            224945689u,                    0u },
+  {                    1u,                    0u,          78410285056u },
+  {  9323915941641553425u,  7568423425299591512u,         173897801038u },
+  { 12155831029092699564u, 16561505984665207376u,         229234681773u },
+  { 17397171276588232676u,  4329114621856906244u,          31080095461u },
+  { 11874560617553253769u,  1477500474861899138u,          40915694367u },
+  { 13444839516837727954u, 16891579639263969683u,          16253944616u },
+  { 16994416043584590671u,  4684451357140027419u,          30798362384u },
+  { 15879694502877015730u, 14727186580409080708u,         209859998750u },
+  {  4234647645735263359u, 15864176859687308833u,         160095165137u },
+  {  7978589901512919496u,  1755486942842684437u,         219944181664u },
+  {  6114237175390859894u, 17417077516652710040u,         170232614619u },
+  {  8658612872088282708u,  4290982361913532782u,         191641124522u },
+  { 10253813330683324853u, 11826659981004351408u,         203050574271u },
+  { 13289465061747830991u,   932930645678090819u,          97688890827u },
+  {  4123165538545565412u, 12707792781328052616u,          80894011233u },
+  {  7846417485927038481u, 16491596426880311905u,          31167628880u },
+  { 10562273346358018864u,  3092207065214166009u,                  799u },
+  {  2691512658346619120u,       14742040721959u,                    0u },
+  {   751187558544605998u,                14742u,                    0u },
+  {                    1u,                    0u,           8441430016u },
+  {  3757709791947931308u,  8142946531605512549u,         214288853256u },
+  {  3452755398462519465u,  5328402096432654514u,          20104734166u },
+  {  3105818720159874523u,  1932004361303814511u,         129136147476u },
+  { 16859138458894499364u,  2511477647985517770u,         106946040961u },
+  { 12271894740606233755u, 17451375493324716693u,           2514414186u },
+  {  5429638071845793701u,  9489266854478998488u,          97477214466u },
+  {   145278150038876889u,  8803053132063235168u,          40878132321u },
+  {  9050266019724932450u, 16198682197142616772u,          92742474792u },
+  { 11907016253451490866u, 13696242485403414201u,         181889538140u },
+  {  2472757296513770735u, 16409082426079859930u,         140631732661u },
+  { 10558733798178239360u, 11653410736879597609u,          32736689036u },
+  { 15917322570831255850u, 13589514120653213260u,         242435466272u },
+  { 12254334656791355238u,  8032934885905905773u,          91149241586u },
+  {  7869542424662730262u,  2753021350129449272u,         221920211035u },
+  {  1378558986933000253u, 16974897459201404132u,         233726338269u },
+  { 13521405041909411105u, 13398576176159101588u,             52374249u },
+  {  3206744593298092012u,   966134380754314586u,                    0u },
+  { 13914648122214918505u,            966134380u,                    0u },
+  {                    1u,                    0u,           1557528576u },
+  {  1235541077112082496u, 10284586955251725350u,         242287014145u },
+  { 12014985518315533846u,  5294476488634150890u,         207858321906u },
+  {  1561535086344155741u, 15833244538135063322u,         218560993999u },
+  { 12761747276316224577u, 10348512742273116663u,          47740429018u },
+  {  9745594781103966137u, 13658504610142595662u,         176648155695u },
+  { 17514238702394846785u, 11956362239240850265u,          42727277488u },
+  {  2428898913707151713u, 13415901703662731780u,         205279820330u },
+  {    71666709959904945u,  5161774027546852761u,          18828026061u },
+  {  4049380135452919193u, 15274384838790587710u,         184771591698u },
+  { 18422240861777453733u, 14233354597679374928u,         185231729592u },
+  {  2914504416394425696u,  4274656492162486920u,         151652704697u },
+  { 12721377795748989418u, 12040276505541795045u,         122717650071u },
+  {  2626074459217717422u, 13238307206256765456u,          52696608634u },
+  {  4261529925046307655u, 12850161204172713270u,         146950399540u },
+  { 11536038685430305586u, 17531777095001445153u,         241304857490u },
+  { 12555757789435162768u,  5623628114515245989u,         104398830065u },
+  { 11905178684546080059u,  7357116143579573376u,                 3432u },
+  { 14032797718924543051u,       63316582777114u,                    0u },
+  { 10750340288005853484u,                63316u,                    0u },
+  {                    1u,                    0u,         186192756736u },
+  {  9660290106216358253u,  3555734177475596581u,         121759043258u },
+  { 14820142034615351103u, 14001876724756424381u,         186984450425u },
+  { 12674041783707777619u, 18159905057231476139u,         157302774714u },
+  { 15386686816442679994u,  5585207679308509466u,         140756160413u },
+  {  5679510383719146248u, 13948697622866724671u,         237531676044u },
+  {  1391101719248678506u,  9807691927739036431u,          46857496045u },
+  {  3364596672173710517u, 15818010096140820917u,         162305194542u },
+  { 11276509210104319732u,  5629845624785010942u,         249515952034u },
+  {  5316312656902630164u,  9517635131137734706u,         135033574393u },
+  { 17470981304473644647u,   619338244618780584u,          82630591879u },
+  {  7373293636384920591u, 11632367007491958898u,          23655037778u },
+  {  7616810902585191937u, 12083314261009739915u,         183915095831u },
+  { 12740295655921903924u, 16880538609458881649u,          84943484855u },
+  { 18366635945916526940u, 17404223674486504227u,          77384299092u },
+  {  4472171448243407067u,  7089067015287185432u,          11140526925u },
+  {   229592460858185629u,  2592264228029443647u,          25727159819u },
+  { 12749672866417114996u, 13413731084370224439u,            224945689u },
+  {  9452256722867098693u,  4149515568880992958u,                    0u },
+  { 16251451636418604634u,           4149515568u,                    0u },
+  {                    1u,                    0u,          88505450496u },
+  {  4515791283442995454u,  9323915941641553424u,          80658968920u },
+  { 13306155670047701346u, 12155831029092699563u,           4943102544u },
+  {  4456930152933417601u, 17397171276588232675u,         130643721220u },
+  {  9089157128546489637u, 11874560617553253768u,         147728846210u },
+  { 12437332180345515840u, 13444839516837727953u,          27921269139u },
+  {  3433060408790452524u, 16994416043584590670u,         132860839963u },
+  {  8275594526021936172u, 15879694502877015729u,          33229560708u },
+  {  3846512444641107689u,  4234647645735263358u,          21432520225u },
+  {  6210962618469046250u,  7978589901512919495u,         152331453461u },
+  {  7272858906616296575u,  6114237175390859893u,         110469384344u },
+  {  3710743300451225347u,  8658612872088282707u,         176555860334u },
+  {  6424677242672030600u, 10253813330683324852u,          67720423344u },
+  { 11485842256170301862u, 13289465061747830990u,         136223517251u },
+  {  7355797963557024308u,  4123165538545565411u,          97425355144u },
+  {  6358188982569427273u,  7846417485927038480u,         249572581985u },
+  { 12475094728768767402u, 10562273346358018863u,          39145907193u },
+  { 17288154837907896183u,  2691512658346619119u,         150040721959u },
+  {  2983850577727105262u,   751187558544605997u,                14742u },
+  { 13918604635001185935u,      271942652322184u,                    0u },
+  { 12033220395769876327u,               271942u,                    0u },
+  {                    1u,                    0u,         101203705856u },
+  {  5782377197813462997u,  3757709791947931307u,         178187174245u },
+  { 17732139848231399226u,  3452755398462519464u,         111168366770u },
+  {  3628839527415562921u,  3105818720159874522u,         202913935727u },
+  {  3188692267613601004u, 16859138458894499363u,         149665260746u },
+  {  5168130193478377352u, 12271894740606233754u,         216294341269u },
+  { 12556227529405091290u,  5429638071845793700u,          96007875544u },
+  { 15087090312791441192u,   145278150038876888u,         196490615904u },
+  { 10281804758610642494u,  9050266019724932449u,         185645480644u },
+  { 14238177586158586580u, 11907016253451490865u,         218134048441u },
+  {  7107927498217678128u,  2472757296513770734u,          41572390106u },
+  {  3845814658485364450u, 10558733798178239359u,          76862879785u },
+  {   714293333681725946u, 15917322570831255849u,         109664308812u },
+  { 16766172658649116982u, 12254334656791355237u,          56426608749u },
+  {   812461421432632215u,  7869542424662730261u,         228074731832u },
+  { 15218024718633799196u,  1378558986933000252u,         148732996836u },
+  {  8110797782612805146u, 13521405041909411104u,          90173837972u },
+  { 15941193964933529227u,  3206744593298092011u,         108754314586u },
+  { 14144280602323277933u, 13914648122214918504u,            966134380u },
+  { 15072402647813125245u, 17822033662586700072u,                    0u },
+  { 10822706091283369889u,          17822033662u,                    0u },
+  { 15163844593710966731u,                   17u,                    0u },
+  {                    1u,                    0u,          38066978816u },
+  {  2408529687792073670u,  1235541077112082495u,         234651333670u },
+  {  3980682212356510808u, 12014985518315533845u,          26084650986u },
+  {  4202670442792148519u,  1561535086344155740u,         247691815706u },
+  {  9419583343154651922u, 12761747276316224576u,          78528309751u },
+  { 16359166491570434575u,  9745594781103966136u,          89949448782u },
+  { 12567727056384237385u, 17514238702394846784u,           4131670873u },
+  {  2068388267923286639u,  2428898913707151712u,         153003885060u },
+  {  5689135844565021196u,    71666709959904944u,          62219517337u },
+  {  3104061965171139313u,  4049380135452919192u,          80998671678u },
+  {  7955173880156328016u, 18422240861777453732u,         136157995600u },
+  {  1445179403240833754u,  2914504416394425695u,         229689627272u },
+  { 12538201164459126715u, 12721377795748989417u,          16142359781u },
+  {  7580606719088482667u,  2626074459217717421u,          54231018000u },
+  {  8168318283218819755u,  4261529925046307654u,          33625369910u },
+  {  5249615277755961676u, 11536038685430305585u,         165680648993u },
+  {  6312997372068219831u, 12555757789435162767u,         128645381029u },
+  {  9183815417025176703u, 11905178684546080058u,          26760719488u },
+  { 10683849953373876937u, 14032797718924543050u,          84582777114u },
+  { 17175012155615667568u, 10750340288005853483u,                63316u },
+  { 18003508288378896912u,     1167984798111281u,                    0u },
+  { 14722554560950996951u,              1167984u,                    0u },
+  {                    1u,                    0u,          37523685376u },
+  { 15059324482416394930u,  9660290106216358252u,         189803401509u },
+  {  4134778595813308312u, 14820142034615351102u,         171687061181u },
+  { 16321118342639660948u, 12674041783707777618u,          26834113963u },
+  {  1523550293123468805u, 15386686816442679993u,          63307886874u },
+  {  8016371634569878509u,  5679510383719146247u,          15075411775u },
+  {  9884220139611134110u,  1391101719248678505u,         181182395151u },
+  {  7218073002727840414u,  3364596672173710516u,         254611300789u },
+  { 16062235669481359233u, 11276509210104319731u,          50288197886u },
+  { 15558048660560338002u,  5316312656902630163u,         168947103794u },
+  {  8394398745765058609u, 17470981304473644646u,         114399707048u },
+  {  5693296366442904274u,  7373293636384920590u,         139412908146u },
+  { 11783494675061161358u,  7616810902585191936u,         113690652811u },
+  { 13377293110865447894u, 12740295655921903923u,          35995657329u },
+  { 12840734051093062130u, 18366635945916526939u,          24242436899u },
+  {  7009868331566697505u,  4472171448243407066u,          63012446232u },
+  {  5019690705031194477u,   229592460858185628u,          55691161151u },
+  {  8608277240439804984u, 12749672866417114995u,         190512407863u },
+  { 12172482590657749222u,  9452256722867098692u,          48880992958u },
+  { 16613484892678771990u, 16251451636418604633u,           4149515568u },
+  {  5721488662757049244u,  2758075434182769113u,                    4u },
+  {   386931106438877039u,          76545051729u,                    0u },
+  { 10054429752182825659u,                   76u,                    0u },
+  {                    1u,                    0u,          16244801536u },
+  {  8634592106137071313u,  4515791283442995453u,         171721328144u },
+  { 12626356501369830731u, 13306155670047701345u,         227241610667u },
+  {  4803333258178976933u,  4456930152933417600u,         136492724195u },
+  { 13613083223558209297u,  9089157128546489636u,         209674229128u },
+  { 16106967997237446989u, 12437332180345515839u,          78186106577u },
+  { 14832921244380020170u,  3433060408790452523u,         177448620878u },
+  { 13774024637717231397u,  8275594526021936171u,         126208519857u },
+  {  9673012968505228885u,  3846512444641107688u,         199336696958u },
+  {  5391832334264815667u,  6210962618469046249u,         117394262471u },
+  { 16514436292632703088u,  7272858906616296574u,          83201159797u },
+  { 12025036352783454153u,  3710743300451225346u,         180348282451u },
+  {  7059867105311401050u,  6424677242672030599u,         206622648756u },
+  { 12769210631552594670u, 11485842256170301861u,         227398758606u },
+  {  8328873878884556145u,  7355797963557024307u,          16344678115u },
+  {  1016565892414238685u,  6358188982569427272u,          47676276240u },
+  {  9662978461927250281u, 12475094728768767401u,         239937192751u },
+  { 13729967277551868112u, 17288154837907896182u,          45161754863u },
+  {  6371593776693359475u,  2983850577727105261u,         136754529069u },
+  { 17617208110845643245u, 13918604635001185934u,          70652322184u },
+  { 14960960225633086797u, 12033220395769876326u,               271942u },
+  { 12090634301321662558u,     5016456510113118u,                    0u },
+  {  9409926148478635503u,              5016456u,                    0u },
+  {                    1u,                    0u,         171313463296u },
+  {  4307062684900157136u,  5782377197813462996u,         168961261227u },
+  { 15300759383869911853u, 17732139848231399225u,         218196719784u },
+  { 16007534237643445447u,  3628839527415562920u,          35172859354u },
+  {  7138502295759677634u,  3188692267613601003u,         154280164899u },
+  {  8218537071653683708u,  5168130193478377351u,         164680674458u },
+  {  2254219416760329296u, 12556227529405091289u,         216817872804u },
+  {  3057410459568460683u, 15087090312791441191u,          97557377752u },
+  {  8217810929938874370u, 10281804758610642493u,          49771853153u },
+  { 11741126472498340929u, 14238177586158586579u,         238385321521u },
+  {  1175325363726654805u,  7107927498217678127u,         127208482030u },
+  {  9428843070696730900u,  3845814658485364449u,          41038721919u },
+  { 12662500978715131896u,   714293333681725945u,         101908896041u },
+  {  6443045597035184564u, 16766172658649116981u,          21044043621u },
+  {  1921385512639171183u,   812461421432632214u,          60824970773u },
+  { 10469475094355551399u, 15218024718633799195u,          32439687228u },
+  { 14679174489076953574u,  8110797782612805145u,         235864173856u },
+  { 11853074234719825644u, 15941193964933529226u,         104766762987u },
+  {  8270896886596139124u, 14144280602323277932u,          40817076584u },
+  { 16532667046659118126u, 15072402647813125244u,         254586700072u },
+  {   148341279888833483u, 10822706091283369888u,          17822033662u },
+  { 10364629296397276041u, 15163844593710966730u,                   17u },
+  { 14265682585545771671u,         328758493846u,                    0u },
+  { 13991741872911347878u,                  328u,                    0u },
+  {                    1u,                    0u,          63130566656u },
+  { 14029045786848724433u,  2408529687792073669u,          21215793215u },
+  {  4005878521026842341u,  3980682212356510807u,          92227827221u },
+  {  3428326338640386488u,  4202670442792148518u,          64510636636u },
+  {  1010001558294829380u,  9419583343154651921u,         184886832192u },
+  {  2012063724327403418u, 16359166491570434574u,          64681297848u },
+  { 10997154538851372612u, 12567727056384237384u,          96112127552u },
+  {  1917749645489607898u,  2068388267923286638u,         176308408672u },
+  {  9763872523711218805u,  5689135844565021195u,         152168271536u },
+  { 15875699078454059311u,  3104061965171139312u,         164431250840u },
+  { 10966529452671276106u,  7955173880156328015u,          95078343332u },
+  { 18073244132105736913u,  1445179403240833753u,         233679697247u },
+  {  4435241176994913173u, 12538201164459126714u,         173410945513u },
+  {  5464400086219074323u,  7580606719088482666u,          70442805421u },
+  {  2445909179323258812u,  8168318283218819754u,          49284582214u },
+  {   873962058644121211u,  5249615277755961675u,         143342228273u },
+  { 16675872194112650857u,  6312997372068219830u,          58497855631u },
+  { 10680102689274800355u,  9183815417025176702u,          74579172666u },
+  {  2370498083108897524u, 10683849953373876936u,          43931059274u },
+  { 15354400521451334666u, 17175012155615667567u,          49975972139u },
+  {   259991949657381021u, 18003508288378896911u,         112798111281u },
+  { 10335286558772966917u, 14722554560950996950u,              1167984u },
+  { 16337526653906757263u,    21545516652742137u,                    0u },
+  { 12040967163702784894u,             21545516u,                    0u },
+  {                    1u,                    0u,         108816367616u },
+  {  3373309160242342187u, 15059324482416394929u,          62224146796u },
+  { 13639841054510584221u,  4134778595813308311u,          82884769598u },
+  { 15898855427739708031u, 16321118342639660947u,         185082591826u },
+  {  4544387940067005419u,  1523550293123468804u,           7434568377u },
+  {  5281598644835398575u,  8016371634569878508u,         105535824647u },
+  { 13675642405083408835u,  9884220139611134109u,         180391292521u },
+  {  3973392623768015721u,  7218073002727840413u,         243870735540u },
+  {  4491285101509114191u, 16062235669481359232u,          19843403507u },
+  { 15002304272810270500u, 15558048660560338001u,         102455061267u },
+  { 17325098540619893468u,  8394398745765058608u,          14308634214u },
+  {  1137212864974584822u,  5693296366442904273u,            638784526u },
+  {  2619406097224859078u, 11783494675061161357u,          51725184512u },
+  {  8281347529729293732u, 13377293110865447893u,          91696097587u },
+  { 11344719666795450104u, 12840734051093062129u,         218380005723u },
+  { 17283870506679425783u,  7009868331566697504u,         156272117978u },
+  { 11054210518010603775u,  5019690705031194476u,         115466655644u },
+  {  6399455551799092885u,  8608277240439804983u,          68659871603u },
+  { 12930529916573967170u, 12172482590657749221u,          89900618820u },
+  { 14550097052337552404u, 16613484892678771989u,         217310162521u },
+  { 12487632712206414748u,  5721488662757049243u,          81020975577u },
+  {  5791017277843595715u,   386931106438877038u,          76545051729u },
+  { 10227264183449036113u, 10054429752182825658u,                   76u },
+  {  2006055278511721441u,        1412006979354u,                    0u },
+  {   128746359043876333u,                 1412u,                    0u },
+  {                    1u,                    0u,         253468082176u },
+  {  7408146306870995754u,  8634592106137071312u,          97684476157u },
+  {  8299024588195267962u, 12626356501369830730u,         128260389217u },
+  {  1497052939192040881u,  4803333258178976932u,          36737966720u },
+  { 16771714264265803747u, 13613083223558209296u,          63873160484u },
+  {   142988846654429432u, 16106967997237446988u,          43804094271u },
+  { 11839838367716104145u, 14832921244380020169u,          43746691371u },
+  {  6019646776647679765u, 13774024637717231396u,         232524375083u },
+  {  4611972391702034948u,  9673012968505228884u,         233292291816u },
+  { 16447182322205429545u,  5391832334264815666u,         126895249385u },
+  {  2113477168726764245u, 16514436292632703087u,           2651878526u },
+  {  3536261187802311516u, 12025036352783454152u,         135382716162u },
+  { 18444381860986709854u,  7059867105311401049u,         165692220295u },
+  {  4734315730275909838u, 12769210631552594669u,          51451509157u },
+  {  9974936316849658174u,  8328873878884556144u,          72055108147u },
+  { 11864423681540657642u,  1016565892414238684u,         169523831112u },
+  {  8207245621417902667u,  9662978461927250280u,         118744303017u },
+  {  7992526918695295028u, 13729967277551868111u,         237345404790u },
+  {  8679354522130259987u,  6371593776693359474u,         142955030765u },
+  {  6065763799692166461u, 17617208110845643244u,         102811035278u },
+  { 18143341109049024976u, 14960960225633086796u,          94655434598u },
+  { 15242492331283350570u, 12090634301321662557u,         136510113118u },
+  {  9986352353182266963u,  9409926148478635502u,              5016456u },
+  { 17340463289911536077u,    92537289398950870u,                    0u },
+  {  7359344614214233035u,             92537289u,                    0u },
+  {                    1u,                    0u,         212233486336u },
+  {   419091135888749535u,  4307062684900157135u,          57829455828u },
+  {  1073142712661309790u, 15300759383869911852u,         168867770169u },
+  { 11076438902195672286u, 16007534237643445446u,         235386978984u },
+  {  1820390940081322073u,  7138502295759677633u,         135445527787u },
+  { 18417808973944523597u,  8218537071653683707u,         217122201479u },
+  { 10251294197731582957u,  2254219416760329295u,          39165742553u },
+  {  1502394029870156428u,  3057410459568460682u,          61445488423u },
+  {   321014853559106075u,  8217810929938874369u,         211636487741u },
+  {  2390953058510591778u, 11741126472498340928u,          47063714515u },
+  { 10685224265907994087u,  1175325363726654804u,         225511138607u },
+  {  5967405799190505023u,  9428843070696730899u,         249686435553u },
+  { 11210723659228214761u, 12662500978715131895u,          53349278201u },
+  { 12327123641078462773u,  6443045597035184563u,         150104158517u },
+  {  1709976940107894237u,  1921385512639171182u,          27567551382u },
+  { 16607686590938553511u, 10469475094355551398u,          25795759643u },
+  { 18332088094272679457u, 14679174489076953573u,         138642556441u },
+  {  2946170632136780882u, 11853074234719825643u,         108448366218u },
+  {  4824449494694383419u,  8270896886596139123u,         124896237676u },
+  { 17008332258693407134u, 16532667046659118125u,         160008041596u },
+  {  1773419466622750661u,   148341279888833482u,         202561867680u },
+  {  3892343466023784379u, 10364629296397276040u,         150773344202u },
+  { 12001571085575422796u, 14265682585545771670u,          72758493846u },
+  { 12933506765500977582u, 13991741872911347877u,                  328u },
+  { 11884830007749143734u,        6064523798049u,                    0u },
+  {  9662368568096205337u,                 6064u,                    0u },
+  {                    1u,                    0u,         197760516096u },
+  { 16801499925276664442u, 14029045786848724432u,          87217159109u },
+  { 10492407990787637084u,  4005878521026842340u,          38185849943u },
+  {  7673849751013230269u,  3428326338640386487u,          17054752294u },
+  {  6046724489853072367u,  1010001558294829379u,          14109074193u },
+  {  3723941391207507903u,  2012063724327403417u,          72596156942u },
+  { 16844122108860347659u, 10997154538851372611u,         110103961416u },
+  { 10622020182694668027u,  1917749645489607897u,          11529300590u },
+  {  8741198820686854862u,  9763872523711218804u,         240860623371u },
+  {  6855480461211306807u, 15875699078454059310u,          79594496752u },
+  { 10005708458011566304u, 10966529452671276105u,         217979752527u },
+  {  8932093106442919061u, 18073244132105736912u,         186240434905u },
+  {  9062763476260756743u,  4435241176994913172u,         106296225722u },
+  { 13664977682032775521u,  5464400086219074322u,         170132593002u },
+  {  1078499125430623453u,  2445909179323258811u,          75047377578u },
+  {  6554586738078431161u,   873962058644121210u,         182904000843u },
+  { 12177313698643242883u, 16675872194112650856u,         126578969526u },
+  { 16615072271904633953u, 10680102689274800354u,         200128504958u },
+  { 16375404983106569285u,  2370498083108897523u,         111832363720u },
+  { 13552251831473522729u, 15354400521451334665u,          15014094191u },
+  {  8330500218412111874u,   259991949657381020u,         214560277007u },
+  {  7044338079053294004u, 10335286558772966916u,         249885659094u },
+  {  2688849443046530184u, 16337526653906757262u,          44652742137u },
+  {   855940991879596845u, 12040967163702784893u,             21545516u },
+  {  7344363609485825662u,   397444631628981487u,                    0u },
+  { 11602660525134634992u,            397444631u,                    0u },
+  {                    1u,                    0u,         177182867456u },
+  { 16945343208344873835u,  3373309160242342186u,         151739417265u },
+  {  9617992661337889145u, 13639841054510584220u,         147861878679u },
+  { 18280344933262742088u, 15898855427739708030u,           4246351763u },
+  {  5179975582362777795u,  4544387940067005418u,         236286316036u },
+  {  1798918997870037130u,  5281598644835398574u,         157741358060u },
+  {  6327667344756325883u, 13675642405083408834u,         157215398045u },
+  { 18380327574124007701u,  3973392623768015720u,         128243473053u },
+  { 18015447557304295289u,  4491285101509114190u,          81813276544u },
+  { 10315590748073249878u, 15002304272810270499u,          48939195473u },
+  {  7697916092577993382u, 17325098540619893467u,         209061648432u },
+  {  3124132817942110723u,  1137212864974584821u,         141141998289u },
+  {  7448238998520507049u,  2619406097224859077u,         213448932749u },
+  { 13892823322374205297u,  8281347529729293731u,         241614998485u },
+  { 11042137840046332564u, 11344719666795450103u,          32936960497u },
+  { 10513966307445593804u, 17283870506679425782u,         108599249952u },
+  {  9388437460943526958u, 11054210518010603774u,          55346915180u },
+  { 10967228614677896228u,  6399455551799092884u,         229700965431u },
+  {  2310996671540235542u, 12930529916573967169u,          21788762341u },
+  {  4989110555003898587u, 14550097052337552403u,         155676955925u },
+  { 16271452421983657679u, 12487632712206414747u,         110313931675u },
+  {  9523160181437090473u,  5791017277843595714u,         186554421102u },
+  { 13137707423765072250u, 10227264183449036112u,          26108748474u },
+  { 16846859744221860705u,  2006055278511721440u,         132006979354u },
+  {  7767140033449795569u,   128746359043876332u,                 1412u },
+  { 17169456915721160017u,       26046931378436u,                    0u },
+  { 17180899661833327819u,                26046u,                    0u },
+  {                    1u,                    0u,         208401596416u },
+  { 17572520700934791416u,  7408146306870995753u,          74449891024u },
+  { 17968798858233825417u,  8299024588195267961u,         164081155402u },
+  { 15338423313945305609u,  1497052939192040880u,          16909196452u },
+  { 17895321323836726301u, 16771714264265803746u,          76007751440u },
+  {   814069333008965773u,   142988846654429431u,         201641838924u },
+  {  7200328959852723947u, 11839838367716104144u,          36326325705u },
+  {   759884557248133773u,  6019646776647679764u,          84250015524u },
+  { 13410165861863974851u,  4611972391702034947u,          50891603540u },
+  {  6278452420856351570u, 16447182322205429544u,         111114571826u },
+  {  9072115382556676442u,  2113477168726764244u,         200191701103u },
+  {  2755882551854926563u,  3536261187802311515u,          89999871944u },
+  {  8496072611504649269u, 18444381860986709853u,         237256647769u },
+  {  4122009033579215815u,  4734315730275909837u,         112540742381u },
+  { 10222217724450527221u,  9974936316849658173u,         220643171696u },
+  {  2064539481554006325u, 11864423681540657641u,         104444915676u },
+  {  7935605886598063693u,  8207245621417902666u,         207433275752u },
+  {  7805147585347548429u,  7992526918695295027u,         114470508751u },
+  {  5709020905457661273u,  8679354522130259986u,         236328825714u },
+  { 16257370307404906674u,  6065763799692166460u,          76983552492u },
+  { 14971258192939373646u, 18143341109049024975u,          93826297164u },
+  {  1133404845901376390u, 15242492331283350569u,         238541361245u },
+  {  9460827548162822047u,  9986352353182266962u,         214940028398u },
+  {  1273897659779791346u, 17340463289911536076u,         201398950870u },
+  {  7833262224435092783u,  7359344614214233034u,             92537289u },
+  {  3033420566713364587u,  1707011694817242694u,                    0u },
+  { 15075466825360349103u,           1707011694u,                    0u },
+  {                    1u,                    0u,         207022718976u },
+  {  2484134775182816690u,   419091135888749534u,          44058175183u },
+  { 18400539815335991277u,  1073142712661309789u,         198600454956u },
+  {   485494064952118286u, 11076438902195672285u,         193098683590u },
+  { 17577048805241314891u,  1820390940081322072u,         251998431425u },
+  {  2863946907557583807u, 18417808973944523596u,          79555723771u },
+  { 13045307417786230800u, 10251294197731582956u,         138081444943u },
+  { 12032088871615097766u,  1502394029870156427u,           1017402250u },
+  {  8848763446997690580u,   321014853559106074u,          64129613825u },
+  { 10031289150307672684u,  2390953058510591777u,          84579247168u },
+  { 11592215575498656563u, 10685224265907994086u,          19323493716u },
+  { 15894436747956898388u,  5967405799190505022u,         247607734547u },
+  {  2091546719588500923u, 11210723659228214760u,         179668254711u },
+  {  5863809244813756109u, 12327123641078462772u,         110092698035u },
+  { 11303008753675411245u,  1709976940107894236u,         166900304494u },
+  { 13238426537506910532u, 16607686590938553510u,         229993784486u },
+  { 17258458071023005565u, 18332088094272679456u,         235159712229u },
+  {  8385733444777075179u,  2946170632136780881u,         115261533931u },
+  {  9530757096163247300u,  4824449494694383418u,          45922023539u },
+  { 14423000845391072217u, 17008332258693407133u,         202096137261u },
+  { 10953140011159884311u,  1773419466622750660u,         136211004362u },
+  { 12228340237948264127u,  3892343466023784378u,         150650606472u },
+  { 11279134946966259189u, 12001571085575422795u,         165701126806u },
+  { 14640097792684582651u, 12933506765500977581u,          33644277925u },
+  {  6232313315128656728u, 11884830007749143733u,         176523798049u },
+  { 16136121832933322088u,  9662368568096205336u,                 6064u },
+  { 15074767079673358271u,      111870718431542u,                    0u },
+  { 13252722804829281908u,               111870u,                    0u },
+  {                    1u,                    0u,         208910811136u },
+  {  7740175894281560509u, 16801499925276664441u,         228568794576u },
+  { 15670495392425593226u, 10492407990787637083u,         183416000228u },
+  { 15152257626756992778u,  7673849751013230268u,          67327793591u },
+  {  4090073428152440422u,  6046724489853072366u,         153201875267u },
+  { 14450327772834205584u,  3723941391207507902u,          67913121689u },
+  {  4466091895542494216u, 16844122108860347658u,         217575820867u },
+  { 10454115378553795377u, 10622020182694668026u,         116473861337u },
+  {  2267817233475657788u,  8741198820686854861u,          46371636340u },
+  {  5500455702636497521u,  6855480461211306806u,          73542410542u },
+  { 15178768299492252549u, 10005708458011566303u,         208484209737u },
+  {  7062359872332045590u,  8932093106442919060u,         148491293392u },
+  { 12297347290027942576u,  9062763476260756742u,          18740779924u },
+  {  8030124596941085588u, 13664977682032775520u,         187058465554u },
+  {  6526656990996654843u,  1078499125430623452u,         122355324859u },
+  {  6254287345256979850u,  6554586738078431160u,         104660133498u },
+  {  6642007136244870032u, 12177313698643242882u,         226900704872u },
+  {  2027592955437164718u, 16615072271904633952u,         243887712482u },
+  {   942718349157325567u, 16375404983106569284u,           9734669043u },
+  { 14617066671884002278u, 13552251831473522728u,         156451597321u },
+  {  6831631114396133348u,  8330500218412111873u,           4381874332u },
+  { 14603040013386939258u,  7044338079053294003u,         142145762820u },
+  {  9906106765319401103u,  2688849443046530183u,         125046400654u },
+  {  1396179595609933063u,   855940991879596844u,         239398138749u },
+  { 11524884268464976417u,  7344363609485825661u,          23628981487u },
+  {   382929570730827274u, 11602660525134634991u,            397444631u },
+  {  6109721884461301381u,  7331559403129590068u,                    0u },
+  {  2390514825000339691u,           7331559403u,                    0u },
+  {  6116191454763441755u,                    7u,                    0u },
+  {                    1u,                    0u,          42918608896u },
+  { 11598868771099176310u, 16945343208344873834u,         156521392426u },
+  { 14449966445520085105u,  9617992661337889144u,         126990979484u },
+  { 11675595287405614726u, 18280344933262742087u,         234280807038u },
+  { 15860796398550489897u,  5179975582362777794u,         174097519594u },
+  { 16180408435245829662u,  1798918997870037129u,         194343023534u },
+  { 13756992797154950706u,  6327667344756325882u,         104996399554u },
+  {  8830551328786758466u, 18380327574124007700u,          78976619368u },
+  { 16699955256560951264u, 18015447557304295288u,          35559209294u },
+  { 10038983627153402074u, 10315590748073249877u,         219417304867u },
+  { 15085100736692127346u,  7697916092577993381u,         245169359579u },
+  { 10007783780289711125u,  3124132817942110722u,         197403769845u },
+  { 17596907048353602192u,  7448238998520507048u,         163753131461u },
+  { 13530650344896573509u, 13892823322374205296u,         247598595491u },
+  {  6337724853398437005u, 11042137840046332563u,         246569963255u },
+  { 12768885008904063297u, 10513966307445593803u,         254508948214u },
+  {  2759773619512884114u,  9388437460943526957u,         148594534654u },
+  {  8434364600126655292u, 10967228614677896227u,          65125279380u },
+  {  3843827521199949338u,  2310996671540235541u,          19270460225u },
+  {  4661660852957808994u,  4989110555003898586u,         155882077203u },
+  { 15298044134177324417u, 16271452421983657678u,         194516251547u },
+  {  7747773274913338217u,  9523160181437090472u,          80712196546u },
+  { 10348785912020632966u, 13137707423765072249u,         224913270096u },
+  {  4175372293197190170u, 16846859744221860704u,         236421057504u },
+  { 11326064156813083145u,  7767140033449795568u,           4930758124u },
+  {  8100407170505981763u, 17169456915721160016u,         190931378436u },
+  {  1706556116319916846u, 17180899661833327818u,                26046u },
+  { 15028897280749641942u,      480481077043500u,                    0u },
+  {  1421201742071739121u,               480481u,                    0u },
+  {                    1u,                    0u,          41952608256u },
+  {  8480737406125178272u, 17572520700934791415u,         121974090537u },
+  { 10947205650755620361u, 17968798858233825416u,         176831497593u },
+  {   868577942165647781u, 15338423313945305608u,         226970107312u },
+  { 16017710019091388479u, 17895321323836726300u,         247044130786u },
+  {  6610879150827623375u,   814069333008965772u,         208390330615u },
+  { 12110095866223762092u,  7200328959852723946u,          20041193424u },
+  {  7756802952949470775u,   759884557248133772u,           3726966548u },
+  {  2941800790804618759u, 13410165861863974850u,          40340355587u },
+  { 11703600274199927522u,  6278452420856351569u,         212491800360u },
+  {   806737539257940346u,  9072115382556676441u,          91149396692u },
+  { 14579028397110132023u,  2755882551854926562u,          93460573019u },
+  { 14247808875344366934u,  8496072611504649268u,         205223454557u },
+  {  9713379923695279513u,  4122009033579215814u,          61554147533u },
+  {  2246428675703313877u, 10222217724450527220u,         233111918909u },
+  {  3549783776592680620u,  2064539481554006324u,          74430190057u },
+  { 12645029747929213033u,  7935605886598063692u,          51423117898u },
+  { 16279009267476580506u,  7805147585347548428u,          18309486643u },
+  {   343358782242907186u,  5709020905457661272u,          60881313810u },
+  { 10077054739085890321u, 16257370307404906673u,         207811593532u },
+  { 10526715404712173586u, 14971258192939373645u,          41061441999u },
+  { 11438715865125144243u,  1133404845901376389u,          82512872489u },
+  {  5040916178827294801u,  9460827548162822046u,         204069058130u },
+  { 16643761637275849508u,  1273897659779791345u,         202424641996u },
+  {  4852542977279030386u,  7833262224435092782u,          70164442058u },
+  {  7883373066544387129u,  3033420566713364586u,         110817242694u },
+  { 16699064314768500978u, 15075466825360349102u,           1707011694u },
+  {  6805863634444817214u, 13042063791413317777u,                    1u },
+  {  2266540253968903500u,          31488807865u,                    0u },
+  {  9016913589137908810u,                   31u,                    0u },
+  {                    1u,                    0u,         222134665216u },
+  { 11654451024602552034u,  2484134775182816689u,          93997495262u },
+  {  5299013208454526793u, 18400539815335991276u,         221026318685u },
+  { 14918550373926182540u,   485494064952118285u,          88952853725u },
+  {  6225552657491071054u, 17577048805241314890u,          76155254872u },
+  { 10344713496596235785u,  2863946907557583806u,         236707187532u },
+  { 12972405634433280209u, 13045307417786230799u,         139652260844u },
+  { 12911885282402784945u, 12032088871615097765u,          26479692427u },
+  {  6934311832970995868u,  8848763446997690579u,          33543797274u },
+  {  9975729197003430461u, 10031289150307672683u,         230628415265u },
+  {  1982857556803548935u, 11592215575498656562u,          62861639142u },
+  {  2095735223386298223u, 15894436747956898387u,         232113382974u },
+  {  7110931538347639365u,  2091546719588500922u,          52317877736u },
+  { 15822183724630969535u,  5863809244813756108u,         220612737332u },
+  { 16931982690156327501u, 11303008753675411244u,         166717656540u },
+  {  6740069226761666110u, 13238426537506910531u,          32935582886u },
+  {  3138792961008474902u, 17258458071023005564u,          81454591520u },
+  { 12154594426971851390u,  8385733444777075178u,          58516663377u },
+  { 15780127219221910902u,  9530757096163247299u,         157781872442u },
+  { 16421541930960194381u, 14423000845391072216u,         196593770909u },
+  {  7485894627196740576u, 10953140011159884310u,         186662899652u },
+  {  8897269432694476707u, 12228340237948264126u,          75611443130u },
+  { 17189823634941678805u, 11279134946966259188u,         173793641291u },
+  {  9585582064286255216u, 14640097792684582650u,         181337854381u },
+  { 12835472279575022097u,  6232313315128656727u,          24874740917u },
+  {  6776016669542754608u, 16136121832933322087u,          54817204760u },
+  { 18340015775620871027u, 15074767079673358270u,         254718431542u },
+  {  5254188752292365830u, 13252722804829281907u,               111870u },
+  {  6798802596750151183u,     2063650512248692u,                    0u },
+  {  9449320530215272000u,              2063650u,                    0u },
+  {                    1u,                    0u,         121419595776u },
+  { 17110720482574968811u,  7740175894281560508u,          91849499257u },
+  { 16172441693558688213u, 15670495392425593225u,         188821405531u },
+  {  6234654946353717320u, 15152257626756992777u,         238221723324u },
+  { 11180283100679445438u,  4090073428152440421u,         190783353838u },
+  { 14852260031176961272u, 14450327772834205583u,          10242107326u },
+  {  4481533167346438750u,  4466091895542494215u,         250566718730u },
+  {  4269718344362365664u, 10454115378553795376u,         205122938618u },
+  { 11520029752381101466u,  2267817233475657787u,          54298180301u },
+  { 16778682550309368417u,  5500455702636497520u,         223822842678u },
+  {  9687587467301363608u, 15178768299492252548u,         148382851295u },
+  { 10093971076828497318u,  7062359872332045589u,           6666640532u },
+  {  1913763026490934696u, 12297347290027942575u,          96435313926u },
+  { 12701450127613557000u,  8030124596941085587u,         220353810784u },
+  {  8974572160711134644u,  6526656990996654842u,         184339045596u },
+  {  9890000077336694124u,  6254287345256979849u,         130360063928u },
+  {  4292326716201059148u,  6642007136244870031u,          96109916034u },
+  { 14644519175104337420u,  2027592955437164717u,          68051104864u },
+  {  5051178622270136798u,   942718349157325566u,          40792392772u },
+  {   675983118348065839u, 14617066671884002277u,           1370343464u },
+  {  4431647660065117244u,  6831631114396133347u,         179791632385u },
+  {  8316115180008411962u, 14603040013386939257u,         135537011123u },
+  {  9621158095544965602u,  9906106765319401102u,          44075687047u },
+  { 15283478958951102072u,  1396179595609933062u,         125624765228u },
+  { 13981553073094447813u, 11524884268464976416u,         239020758653u },
+  {  4558368743929911607u,   382929570730827273u,          52331208687u },
+  { 15217004469858477791u,  6109721884461301380u,         235129590068u },
+  { 11589190369996515737u,  2390514825000339690u,           7331559403u },
+  {  3670624237398152929u,  6116191454763441754u,                    7u },
+  { 13471713758418039777u,         135243399970u,                    0u },
+  {  4489936967610296411u,                  135u,                    0u },
+  {                    1u,                    0u,         106628775936u },
+  {  9052049303222747950u, 11598868771099176309u,         120783334250u },
+  {  1011330006193020538u, 14449966445520085104u,          71632935288u },
+  { 17412075644359478612u, 11675595287405614725u,         194859815495u },
+  {  6358678384745980468u, 15860796398550489896u,         137877141698u },
+  { 15262353928842850919u, 16180408435245829661u,         250745768073u },
+  { 11145257686438581736u, 13756992797154950705u,          20478705146u },
+  {  1600562031807691890u,  8830551328786758465u,         120905306388u },
+  {  6775147337046626724u, 16699955256560951263u,          85544214392u },
+  { 15772127322106297822u, 10038983627153402073u,         165817764949u },
+  {  4141472200527441474u, 15085100736692127345u,           2542523045u },
+  { 18246007807879281267u, 10007783780289711124u,         168953930242u },
+  {   960746958654787123u, 17596907048353602191u,         112733498024u },
+  { 11355981212264408477u, 13530650344896573508u,         147343568752u },
+  {  1573078209576251481u,  6337724853398437004u,         203692202643u },
+  {  6245294478780491367u, 12768885008904063296u,          45149607627u },
+  {  7523292955659721510u,  2759773619512884113u,          35457227821u },
+  { 14454736751015226505u,  8434364600126655291u,          21208374307u },
+  {  7219786377781411316u,  3843827521199949337u,         218252709141u },
+  { 10597123082209392431u,  4661660852957808993u,         206829308634u },
+  {  6922353544343010714u, 15298044134177324416u,         168420007630u },
+  { 14317523356293377430u,  7747773274913338216u,         121561008808u },
+  {  4057766168681892717u, 10348785912020632965u,          96226347385u },
+  { 15214083611901244045u,  4175372293197190169u,         240613987168u },
+  {  8390569016883950721u, 11326064156813083144u,          80439123952u },
+  { 10680472538208175055u,  8100407170505981762u,         202092512592u },
+  { 12173567833130544927u,  1706556116319916845u,          44814718154u },
+  {  1386341248286610026u, 15028897280749641941u,         225077043500u },
+  { 12487300952797237352u,  1421201742071739120u,               480481u },
+  {  2614759871804869720u,     8863311460481781u,                    0u },
+  {  8494389567327729477u,              8863311u,                    0u },
+  {                    1u,                    0u,         247459741696u },
+  {  6260469580539185878u,  8480737406125178271u,         136593449207u },
+  { 17818573101084525841u, 10947205650755620360u,           8047085704u },
+  {  2201029069927307150u,   868577942165647780u,          28868321800u },
+  { 10397997613804897039u, 16017710019091388478u,         140358376476u },
+  { 14269915965770103741u,  6610879150827623374u,         234656489612u },
+  { 16776139909196366727u, 12110095866223762091u,         140420497130u },
+  {  6246513436385199720u,  7756802952949470774u,         194159475340u },
+  {  2926026498821554288u,  2941800790804618758u,          81634453442u },
+  { 15725499391028340982u, 11703600274199927521u,          89043733329u },
+  {  8576577277771450827u,   806737539257940345u,         226790330713u },
+  { 15523351176022259335u, 14579028397110132022u,          52772375266u },
+  {  4775158829429176134u, 14247808875344366933u,         198526563380u },
+  { 10141817222123532462u,  9713379923695279512u,         244121779142u },
+  { 12847658900242624586u,  2246428675703313876u,          52192434164u },
+  { 13708197964460514655u,  3549783776592680619u,          76685488436u },
+  {  1951540006613246932u, 12645029747929213032u,          12882486860u },
+  {  9979297327280092199u, 16279009267476580505u,          88018613516u },
+  { 15381307706282553684u,   343358782242907185u,         177546278232u },
+  { 10037428657543061177u, 10077054739085890320u,          77570654385u },
+  {  2584877324547208668u, 10526715404712173585u,         133620094029u },
+  {  1126624732730703576u, 11438715865125144242u,         158273268613u },
+  {  1501064139624981020u,  5040916178827294800u,         241902260126u },
+  {  5219661484955306109u, 16643761637275849507u,          46263056881u },
+  {  5336997298570282212u,  4852542977279030385u,         106427358510u },
+  { 12191131175733833362u,  7883373066544387128u,         174905258090u },
+  {  3707068178994436536u, 16699064314768500977u,         145368946606u },
+  {  5045484691732942022u,  6805863634444817213u,         185122869393u },
+  { 14847900542908711232u,  2266540253968903499u,          31488807865u },
+  {  9097257915916965135u,  9016913589137908809u,                   31u },
+  {  2472027983230314217u,         580865979874u,                    0u },
+  { 15974509111133272205u,                  580u,                    0u },
+  {                    1u,                    0u,         177631789056u },
+  { 12099486841948187399u, 11654451024602552033u,         236287260081u },
+  {  5319910566029976328u,  5299013208454526792u,          13808736236u },
+  { 11549214421017285864u, 14918550373926182539u,          74337487885u },
+  {  1998791413186046700u,  6225552657491071053u,         190560788042u },
+  { 17075171930090011210u, 10344713496596235784u,          15703235518u },
+  { 15158296003813501474u, 12972405634433280208u,         165699954703u },
+  {  1360083178079384115u, 12911885282402784944u,         211375909797u },
+  {  6167980558592741158u,  6934311832970995867u,         107540785363u },
+  {  3630180428124865653u,  9975729197003430460u,          50107490923u },
+  {  2276550099763657677u,  1982857556803548934u,          83113610034u },
+  {   407006713016100655u,  2095735223386298222u,         186385484371u },
+  { 14242579061653496002u,  7110931538347639364u,         204857722298u },
+  { 17944493332678643704u, 15822183724630969534u,          44917884620u },
+  {   987185901870869452u, 16931982690156327500u,          67365379884u },
+  {  5578665155415167745u,  6740069226761666109u,         124170154307u },
+  {  4849210377429577536u,  3138792961008474901u,         234658901884u },
+  { 10811995403388891862u, 12154594426971851389u,         195855442410u },
+  {  7051931074990177294u, 15780127219221910901u,         216890213571u },
+  {  2030832259446664275u, 16421541930960194380u,          22405811160u },
+  {  6069512651054767896u,  7485894627196740575u,         190482321942u },
+  { 10608701253763958799u,  8897269432694476706u,         244931862206u },
+  { 15700053443426906717u, 17189823634941678804u,         250519635444u },
+  { 17759719234725541222u,  9585582064286255215u,          87695812346u },
+  { 15187321568916405210u, 12835472279575022096u,         103367328599u },
+  { 11040156458113129594u,  6776016669542754607u,         190994214247u },
+  {  2800727824598008497u, 18340015775620871026u,         115284830142u },
+  {  2997236166375604479u,  5254188752292365829u,         116368563827u },
+  {  6260091886451512841u,  6798802596750151182u,          34512248692u },
+  { 17573059315228347474u,  9449320530215271999u,              2063650u },
+  {  7519453664590169251u,    38067632857031246u,                    0u },
+  { 15809436065653866529u,             38067632u,                    0u },
+  {                    1u,                    0u,         188927574016u },
+  {   228921437623588922u, 17110720482574968810u,         137876709820u },
+  {  2195862230003073884u, 16172441693558688212u,           9337981321u },
+  {   960207412233973688u,  6234654946353717319u,         101606084361u },
+  {  2464387149230492479u, 11180283100679445437u,         143805142629u },
+  {  3631866936444955213u, 14852260031176961271u,           7242944399u },
+  {  1578304441149380227u,  4481533167346438749u,          48231461895u },
+  { 18190538519673445181u,  4269718344362365663u,          59624502064u },
+  {  1271000736479934749u, 11520029752381101465u,         112909574203u },
+  { 18292963032817745634u, 16778682550309368416u,         132525165168u },
+  { 17168014021925537455u,  9687587467301363607u,          21547195268u },
+  { 18046757712870378949u, 10093971076828497317u,         175103745301u },
+  { 14857998893911743220u,  1913763026490934695u,         147688546991u },
+  { 11933607369968684575u, 12701450127613556999u,         250486512531u },
+  {  3483798509902859162u,  8974572160711134643u,         137536137978u },
+  {  7378828438829845831u,  9890000077336694123u,         143232687497u },
+  { 15791137430347699565u,  4292326716201059147u,         173793880975u },
+  { 17044141236829932641u, 14644519175104337419u,         254273824941u },
+  {  9075651910862456484u,  5051178622270136797u,         229036645118u },
+  { 17811207355884564095u,   675983118348065838u,         227240240101u },
+  {  4438638126207305937u,  4431647660065117243u,         121450817507u },
+  { 12507972635512950185u,  8316115180008411961u,         142521564025u },
+  { 14658269128098109408u,  9621158095544965601u,           6828519054u },
+  {  3642436268910286111u, 15283478958951102071u,          32757941510u },
+  {  3783099432964819561u, 13981553073094447812u,           9247109664u },
+  {  9497579866027539638u,  4558368743929911606u,         132824915465u },
+  {  3395179445046271361u, 15217004469858477790u,         234628251268u },
+  {  5938502732309497276u, 11589190369996515736u,          90198984938u },
+  {  5793671185917606255u,  3670624237398152928u,          34730303066u },
+  {   889272970253526588u, 13471713758418039776u,         135243399970u },
+  {  8594177504370135501u,  4489936967610296410u,                  135u },
+  {  7374354721120724712u,        2494800386918u,                    0u },
+  { 14764532643665507567u,                 2494u,                    0u },
+  {                    1u,                    0u,         117490712576u },
+  {  5392404173658087695u,  9052049303222747949u,         112054824309u },
+  {  4976586473237854316u,  1011330006193020537u,         133943910512u },
+  {  6308932742419013569u, 17412075644359478611u,          40344704645u },
+  {  4831846642430703059u,  6358678384745980467u,          29827373864u },
+  { 18139507855949846901u, 15262353928842850918u,          49604185629u },
+  {  4865833876326628410u, 11145257686438581735u,          65086766641u },
+  { 14296661839130179261u,  1600562031807691889u,         223367281473u },
+  {  9254773150378118248u,  6775147337046626723u,         217855008735u },
+  { 12174712433727875143u, 15772127322106297821u,         113224509657u },
+  {   705653145340915199u,  4141472200527441473u,          20989118065u },
+  { 17763928858962481812u, 18246007807879281266u,         143052082196u },
+  {  3982836567612046296u,   960746958654787122u,          68615608975u },
+  { 12730849277561967739u, 11355981212264408476u,         140085276740u },
+  { 17314488764367235908u,  1573078209576251480u,          64338558092u },
+  { 15951418930590301119u,  6245294478780491366u,         145407838528u },
+  {  7193356087283467261u,  7523292955659721509u,          59783592849u },
+  { 17592945625696089446u, 14454736751015226504u,          25391385403u },
+  {  3554461664875361428u,  7219786377781411315u,          97574471193u },
+  {  2213779057785318208u, 10597123082209392430u,         128375261537u },
+  {  3880940796082421148u,  6922353544343010713u,         104776154496u },
+  {  4528237545358141043u, 14317523356293377429u,         133219971944u },
+  { 11681196539088147363u,  4057766168681892716u,          25824757125u },
+  {  9835005502912643017u, 15214083611901244044u,           8454853657u },
+  {  4964088126040986696u,  8390569016883950720u,          66578989576u },
+  {  3355564873147047622u, 10680472538208175054u,          45659930434u },
+  {  1853093467828272927u, 12173567833130544926u,         213075153709u },
+  { 14755341584803008677u,  1386341248286610025u,         240676937941u },
+  {  4701571132542556621u, 12487300952797237351u,         245141746416u },
+  {  6128849686644853851u,  2614759871804869719u,          79460481781u },
+  { 12026867901170202094u,  8494389567327729476u,              8863311u },
+  { 17909760324981426303u,   163499238157084246u,                    0u },
+  {  2897692901883393664u,            163499238u,                    0u },
+  {                    1u,                    0u,         159339380736u },
+  { 12323704802554838154u,  6260469580539185877u,           8965946783u },
+  {  7135886931147821732u, 17818573101084525840u,         164119318024u },
+  { 15341283120292884947u,  2201029069927307149u,          62563676580u },
+  {  3092789040392634166u, 10397997613804897038u,         206773573694u },
+  {  8811761390822097865u, 14269915965770103740u,         171909436366u },
+  { 16870860798610218169u, 16776139909196366726u,          54338624171u },
+  { 17452041453591904833u,  6246513436385199719u,           6158620214u },
+  { 10314783684009874908u,  2926026498821554287u,         225852481030u },
+  {  4932636630789274903u, 15725499391028340981u,         121464937185u },
+  { 18143884346082124480u,  8576577277771450826u,          54841522553u },
+  {  2823209155405527322u, 15523351176022259334u,          85258861878u },
+  { 16195396106620226251u,  4775158829429176133u,         152549789013u },
+  {  1150544491807648944u, 10141817222123532461u,         212696472984u },
+  {  7767455475523884824u, 12847658900242624585u,         171743122900u },
+  { 15204378045683991808u, 13708197964460514654u,         104105793195u },
+  { 17239732561718805622u,  1951540006613246931u,         153540978792u },
+  { 12886430624522800062u,  9979297327280092198u,          49833822361u },
+  { 18162250541178258136u, 15381307706282553683u,          16544130097u },
+  { 17028935366700158084u, 10037428657543061176u,          17140126480u },
+  { 16075467823964198637u,  2584877324547208667u,         178061074449u },
+  {  9803858825574498304u,  1126624732730703575u,          80081372850u },
+  { 17464070808143041817u,  1501064139624981019u,          35282958416u },
+  { 17682703471239266776u,  5219661484955306108u,         113289319203u },
+  { 18147688354161351336u,  5336997298570282211u,          56660882545u },
+  {  6663423873348080051u, 12191131175733833361u,         241200960568u },
+  {  9417270363716235133u,  3707068178994436535u,          61273516273u },
+  {  9295013721571344179u,  5045484691732942021u,          75804906301u },
+  {  6199479138350037783u, 14847900542908711231u,          73493163339u },
+  {   887603005365085688u,  9097257915916965134u,         226134008905u },
+  {   333989628642975696u,  2472027983230314216u,          68865979874u },
+  {  4620735991403939439u, 15974509111133272204u,                  580u },
+  { 12418523063962801201u,       10715086071862u,                    0u },
+  {  1587745622680169419u,                10715u,                    0u },
+  {                    1u,                    0u,         225655914496u },
+  { 10968905082284365638u, 12099486841948187398u,          72288392929u },
+  { 14076907092801977812u,  5319910566029976327u,         139626084168u },
+  {  3438322122816124202u, 11549214421017285863u,          77108354699u },
+  { 14645413324829073676u,  1998791413186046699u,           8925646925u },
+  { 12271281439492289999u, 17075171930090011209u,         208821732872u },
+  {  6233751789862708246u, 15158296003813501473u,         176073730256u },
+  {  1962644459455827991u,  1360083178079384114u,         155334366896u },
+  {  8726934184642952500u,  6167980558592741157u,          60196792475u },
+  {  4531087719737475147u,  3630180428124865652u,           6123412028u },
+  {   481513520412720775u,  2276550099763657676u,         110022063878u },
+  {   992149349835802669u,   407006713016100654u,          68772091758u },
+  { 11165474436676191361u, 14242579061653496001u,         190972772932u },
+  { 10240785855143707184u, 17944493332678643703u,          76053515454u },
+  { 10059329918238932466u,   987185901870869451u,          61302420044u },
+  { 14791716450947031886u,  5578665155415167744u,          21262876221u },
+  { 15378882314737417403u,  4849210377429577535u,         125586119445u },
+  { 14726970229242271128u, 10811995403388891861u,         117382285949u },
+  {  5090110549507128156u,  7051931074990177293u,          76110091637u },
+  { 17185220781106503841u,  2030832259446664274u,         223329028940u },
+  {  9858517691519529306u,  6069512651054767895u,         162575098847u },
+  {  5595905546638020703u, 10608701253763958798u,         212851101602u },
+  { 15555173226968030256u, 15700053443426906716u,         111962756308u },
+  { 10745236628845355771u, 17759719234725541221u,          16823306351u },
+  {  9973314042399760760u, 15187321568916405209u,          47598488080u },
+  {  4374506813558796576u, 11040156458113129593u,         114151827759u },
+  { 15960826480426749933u,  2800727824598008496u,           5162480498u },
+  {  9636454862798615738u,  2997236166375604478u,          14339360261u },
+  { 17973331528911319269u,  6260091886451512840u,          63952637454u },
+  {  7366495200039369602u, 17573059315228347473u,          78407630399u },
+  { 10505831326526933399u,  7519453664590169250u,         176857031246u },
+  {  2803218632575724145u, 15809436065653866528u,             38067632u },
+  {  8425731874431741636u,   702223880805592151u,                    0u },
+  { 14860552245711912111u,            702223880u,                    0u },
+  {                    1u,                    0u,         234012409856u },
+  {  6993664200669526994u,   228921437623588921u,         212119037930u },
+  {  4065363582031999356u,  2195862230003073883u,          71052052948u },
+  {  6899780515342669867u,   960207412233973687u,         189133594695u },
+  { 17713500890201844939u,  2464387149230492478u,         247196883901u },
+  {  6445781125105107086u,  3631866936444955212u,          93085560055u },
+  { 13563044070717478571u,  1578304441149380226u,         223986111069u },
+  { 13167612994149348885u, 18190538519673445180u,         153068901087u },
+  {  5505463469596727288u,  1271000736479934748u,          96991663513u },
+  { 12125446212518819372u, 18292963032817745633u,         151930679904u },
+  { 12537707724735421794u, 17168014021925537454u,         165978316695u },
+  { 15173675086703777069u, 18046757712870378948u,         167805453733u },
+  { 13535510174093048476u, 14857998893911743219u,           7646922151u },
+  { 10698912997087096629u, 11933607369968684574u,         179188857095u },
+  { 16952559548431933861u,  3483798509902859161u,         107400007091u },
+  { 13528255827744249993u,  7378828438829845830u,          75856039275u },
+  { 14122167436324771955u, 15791137430347699564u,          11923964747u },
+  { 13071007137740038297u, 17044141236829932640u,         221491992075u },
+  { 13011887609328904025u,  9075651910862456483u,          46965547485u },
+  {  3116434332871336590u, 17811207355884564094u,          59240619054u },
+  {  9050993820536772770u,  4438638126207305936u,          57678058555u },
+  { 11993719123438634238u, 12507972635512950184u,         225794626361u },
+  {  1414857165879849301u, 14658269128098109407u,         119197456865u },
+  { 13819438220812375094u,  3642436268910286110u,         196205082231u },
+  {  6073063033888264440u,  3783099432964819560u,          54514864836u },
+  {  6828883869150720294u,  9497579866027539637u,         222184053046u },
+  {  4548265621068768345u,  3395179445046271360u,         152321926878u },
+  { 10422524923581371874u,  5938502732309497275u,         224314075544u },
+  {  1858996082510682634u,  5793671185917606254u,         224048207584u },
+  {   890276727450878316u,   889272970253526587u,          90465891296u },
+  {  3886008133802710905u,  8594177504370135500u,         102399764570u },
+  {   612074409233016757u,  7374354721120724711u,         190800386918u },
+  {  3927020336901729264u, 14764532643665507566u,                 2494u },
+  {  5298603480094474942u,       46020944252475u,                    0u },
+  { 17418383752590430025u,                46020u,                    0u },
+  {                    1u,                    0u,          45292322816u },
+  {  8973799690601597929u,  5392404173658087694u,         121269781293u },
+  {  1343055462055792431u,  4976586473237854315u,          83342007929u },
+  { 17425118728683169659u,  6308932742419013568u,          51261934931u },
+  { 18389781726026675967u,  4831846642430703058u,         102983344691u },
+  {   272526939565961561u, 18139507855949846900u,         231263777382u },
+  { 11293026845930963228u,  4865833876326628409u,         113775023591u },
+  { 13997416438903902597u, 14296661839130179260u,         163501702257u },
+  {  6186605805999441184u,  9254773150378118247u,         221659992483u },
+  {  4401776373281836138u, 12174712433727875142u,          65038253533u },
+  { 16338917089754547008u,   705653145340915198u,         114962984513u },
+  { 13337700757935003056u, 17763928858962481811u,          50215910002u },
+  { 14612496890816348693u,  3982836567612046295u,         156690140722u },
+  {  3219935399907691719u, 12730849277561967738u,          88938620316u },
+  { 10887238730052330387u, 17314488764367235907u,         102864728152u },
+  {   360256418697768294u, 15951418930590301118u,          37389952614u },
+  {   321440824631118565u,  7193356087283467260u,         136953715493u },
+  { 10069228080701402580u, 17592945625696089445u,         243192687752u },
+  {  9428069607611622975u,  3554461664875361427u,          46120009203u },
+  { 14736799017468812344u,  2213779057785318207u,         153210386222u },
+  { 10875332567307979280u,  3880940796082421147u,         149245476249u },
+  {  4611492910339012807u,  4528237545358141042u,         108633238933u },
+  { 10743508637597314786u, 11681196539088147362u,         140533156716u },
+  {  9356196315668016028u,  9835005502912643016u,         128269103756u },
+  { 15755598617722189347u,  4964088126040986695u,         206181905536u },
+  {  1275276394173375542u,  3355564873147047621u,          30100456398u },
+  { 12644999363867216251u,  1853093467828272926u,         105799888670u },
+  {  4553830511509832021u, 14755341584803008676u,         103254872681u },
+  {  8869400642218174412u,  4701571132542556620u,          87332245607u },
+  { 16570849151159054040u,  6128849686644853850u,          68651977815u },
+  { 16127119334101797673u, 12026867901170202093u,          86970890052u },
+  {  9686867250420930550u, 17909760324981426302u,         230157084246u },
+  { 10678226869774428035u,  2897692901883393663u,            163499238u },
+  {  7767227962910162068u,  3016028602530220424u,                    0u },
+  {  9780840471948993674u,           3016028602u,                    0u },
+  {                    1u,                    0u,         213668069376u },
+  {  6288709332106746357u, 12323704802554838153u,          16386837205u },
+  {  9066785620141948673u,  7135886931147821731u,         141831652624u },
+  {  8442375916704414909u, 15341283120292884946u,          14167660429u },
+  { 11604629218100425803u,  3092789040392634165u,         188477686542u },
+  {  3877248044010875762u,  8811761390822097864u,         134914571196u },
+  { 16435137704395217283u, 16870860798610218168u,         103946077062u },
+  { 14994442577577813271u, 17452041453591904832u,         111559165543u },
+  {  4410105917142436089u, 10314783684009874907u,         245267398767u },
+  {  4632574728444936970u,  4932636630789274902u,         202983581941u },
+  {  9117147535650050359u, 18143884346082124479u,         134153046474u },
+  {   588939301256904809u,  2823209155405527321u,          69877954182u },
+  {   324393982565305683u, 16195396106620226250u,         173062371141u },
+  {  9380909186923521175u,  1150544491807648943u,          73421074605u },
+  {  4463385697777230217u,  7767455475523884823u,          94824230985u },
+  { 16378985502426333808u, 15204378045683991807u,         211934567774u },
+  { 18210894922387834354u, 17239732561718805621u,          38698574803u },
+  {  1555748035329493205u, 12886430624522800061u,          83984577574u },
+  {  4277055533891898507u, 18162250541178258135u,         184923140435u },
+  { 11574429772510874408u, 17028935366700158083u,         219871452856u },
+  { 17391099253493808815u, 16075467823964198636u,         215531468251u },
+  {  5791212393959129882u,  9803858825574498303u,          27946729175u },
+  { 11254268231455680880u, 17464070808143041816u,         124958581275u },
+  { 16355477587312235322u, 17682703471239266775u,         227983788156u },
+  {  2411485149249320633u, 18147688354161351335u,         145361224931u },
+  { 12763114642070638360u,  6663423873348080050u,         183510511249u },
+  {  1147543073987366419u,  9417270363716235132u,         197503883703u },
+  {  8410777835225272692u,  9295013721571344178u,          63336074437u },
+  {  8134725822306818018u,  6199479138350037782u,          14048117055u },
+  {  8899607004752328377u,   887603005365085687u,         232018105614u },
+  {   690976506652396830u,   333989628642975695u,         140250490600u },
+  { 12281570945595192074u,  4620735991403939438u,          54673209484u },
+  { 12592957291365552899u, 12418523063962801200u,         219086071862u },
+  { 13595807339013970272u,  1587745622680169418u,                10715u },
+  {  9698096389749839992u,      197658450495420u,                    0u },
+  {  8310173728816391804u,               197658u,                    0u },
+};
+
+inline constexpr int __TABLE_SIZE_2 = 69;
+inline constexpr int __ADDITIONAL_BITS_2 = 120;
+
+inline constexpr uint16_t __POW10_OFFSET_2[__TABLE_SIZE_2] = {
+     0,    2,    6,   12,   20,   29,   40,   52,   66,   80,
+    95,  112,  130,  150,  170,  192,  215,  240,  265,  292,
+   320,  350,  381,  413,  446,  480,  516,  552,  590,  629,
+   670,  712,  755,  799,  845,  892,  940,  989, 1040, 1092,
+  1145, 1199, 1254, 1311, 1369, 1428, 1488, 1550, 1613, 1678,
+  1743, 1810, 1878, 1947, 2017, 2088, 2161, 2235, 2311, 2387,
+  2465, 2544, 2625, 2706, 2789, 2873, 2959, 3046, 3133
+};
+
+inline constexpr uint8_t __MIN_BLOCK_2[__TABLE_SIZE_2] = {
+     0,    0,    0,    0,    0,    0,    1,    1,    2,    3,
+     3,    4,    4,    5,    5,    6,    6,    7,    7,    8,
+     8,    9,    9,   10,   11,   11,   12,   12,   13,   13,
+    14,   14,   15,   15,   16,   16,   17,   17,   18,   19,
+    19,   20,   20,   21,   21,   22,   22,   23,   23,   24,
+    24,   25,   26,   26,   27,   27,   28,   28,   29,   29,
+    30,   30,   31,   31,   32,   32,   33,   34,    0
+};
+
+inline constexpr uint64_t __POW10_SPLIT_2[3133][3] = {
+  {                    0u,                    0u,              3906250u },
+  {                    0u,                    0u,         202000000000u },
+  {                    0u, 11153727427136454656u,                   59u },
+  {                    0u,  7205759403792793600u,          59604644775u },
+  {                    0u,                    0u,         167390625000u },
+  {                    0u,                    0u,         232000000000u },
+  {                    0u,    16777216000000000u,                    0u },
+  {                    0u, 12945425605062557696u,               909494u },
+  {                    0u,  4388757836872548352u,         182701772928u },
+  {                    0u,  1152921504606846976u,         128237915039u },
+  {                    0u,                    0u,         159062500000u },
+  {                    0u,                    0u,         160000000000u },
+  {                    0u,         256000000000u,                    0u },
+  {                    0u, 16192327041775828992u,                   13u },
+  {                    0u, 15024075324038053888u,          13877787807u },
+  {                    0u,  5449091666327633920u,         159814456755u },
+  {                    0u,  2494994193563254784u,         179295395851u },
+  {                    0u,  4611686018427387904u,          11135253906u },
+  {                    0u,                    0u,         146250000000u },
+  {                    0u,                    0u,         128000000000u },
+  {                    0u,              3906250u,                    0u },
+  {                    0u,     3906250000000000u,                    0u },
+  {                    0u,  4368439412768899072u,               211758u },
+  {                    0u,  1563676642168012800u,          46236813575u },
+  {                    0u, 11532349341402398720u,           7084767080u },
+  {                    0u,  9048364970084925440u,         104625169910u },
+  {                    0u, 16609275425742389248u,         246490512847u },
+  {                    0u,                    0u,         207900390625u },
+  {                    0u,                    0u,         225000000000u },
+  { 11153727427136454656u,                   59u,                    0u },
+  {  7205759403792793600u,          59604644775u,                    0u },
+  {                    0u,  4264412554261970152u,                    3u },
+  {                    0u, 14485570586272534528u,           3231174267u },
+  {                    0u, 17827675094632103936u,         123785264354u },
+  {                    0u,  7347197909193981952u,         226966440203u },
+  {                    0u, 13677404030777688064u,          11398292396u },
+  {                    0u,  3810326759732150272u,         172741453558u },
+  {                    0u,  9943947977234055168u,         246206558227u },
+  {                    0u,                    0u,          19539062500u },
+  {                    0u,                    0u,         228000000000u },
+  { 12945425605062557696u,               909494u,                    0u },
+  {  4388757836872548352u,      909494701772928u,                    0u },
+  {  1152921504606846976u, 14878706826214591391u,                49303u },
+  {                    0u,  4387341015746028192u,         151806576313u },
+  {                    0u,   651726680428265472u,         185237838233u },
+  {                    0u,  2570638187944738816u,         153035330174u },
+  {                    0u,  7419175577111756800u,         126139354575u },
+  {                    0u, 17299322326264840192u,         207402194313u },
+  {                    0u,  7990511638862102528u,         137937798142u },
+  {                    0u, 16717361816799281152u,         254433166503u },
+  {                    0u,                    0u,         167906250000u },
+  {                    0u,                    0u,          16000000000u },
+  { 16192327041775828992u,                   13u,                    0u },
+  { 15024075324038053888u,          13877787807u,                    0u },
+  {  5449091666327633920u, 13877787807814456755u,                    0u },
+  {  2494994193563254784u,  9707857417284919307u,            752316384u },
+  {  4611686018427387904u,  1844515466944871826u,         224526264005u },
+  {                    0u, 15167599819856275072u,         197099991383u },
+  {                    0u, 14830185305589481472u,          87822237233u },
+  {                    0u,  6163721531743535104u,          49803945956u },
+  {                    0u, 14122847407012052992u,         228334136013u },
+  {                    0u,   335491783960035328u,         205765601092u },
+  {                    0u,   941252322120433664u,          68018187046u },
+  {                    0u, 11529215046068469760u,          38051025390u },
+  {                    0u,                    0u,         238625000000u },
+  {                    0u,                    0u,          64000000000u },
+  {  4368439412768899072u,               211758u,                    0u },
+  {  1563676642168012800u,      211758236813575u,                    0u },
+  { 11532349341402398720u,  8061591463141767016u,                11479u },
+  {  9048364970084925440u, 16628725344207857142u,         215437019748u },
+  { 16609275425742389248u,  3555541870038531535u,         100901445007u },
+  {                    0u, 18316647450161853665u,         143192746310u },
+  {                    0u, 16709574568378075648u,          70992947447u },
+  {                    0u,  7696022835795591168u,         247905827852u },
+  {                    0u, 16664449640376041472u,          12417202233u },
+  {                    0u,  3109186955116544000u,          57903381625u },
+  {                    0u, 10515518101817131008u,         121168549362u },
+  {                    0u,  9961962375743537152u,         242570047378u },
+  {                    0u,  9223372036854775808u,         146540039062u },
+  {                    0u,                    0u,         150500000000u },
+  { 14485570586272534528u,           3231174267u,                    0u },
+  { 17827675094632103936u,  3231174267785264354u,                    0u },
+  {  7347197909193981952u,   748977172262750475u,            175162308u },
+  { 13677404030777688064u, 15965033457315095468u,         196040602133u },
+  {  3810326759732150272u, 16809402149066729206u,          21865466197u },
+  {  9943947977234055168u,  7563769067065700371u,          85911239516u },
+  {                    0u, 13550322810840051428u,          92410032742u },
+  {                    0u,  8663209637545764864u,         102734564471u },
+  {                    0u,  8969247575312957440u,         119469633535u },
+  {                    0u,  6193172891660451840u,         255486223885u },
+  {                    0u,  3427954273864908800u,          13335732575u },
+  {                    0u, 10058367555266936832u,          95185829773u },
+  {                    0u, 13907115649320091648u,         141545265197u },
+  {                    0u,                    0u,          45753906250u },
+  {                    0u,                    0u,          74000000000u },
+  { 14878706826214591391u,                49303u,                    0u },
+  {  4387341015746028192u,       49303806576313u,                    0u },
+  {   651726680428265472u, 14106411361315920281u,                 2672u },
+  {  2570638187944738816u,  3609034283485221502u,         112764710092u },
+  {  7419175577111756800u,  9896072247338192335u,         204195646140u },
+  { 17299322326264840192u,  8889095178479228297u,         188536467151u },
+  {  7990511638862102528u,  3631796911038383102u,         207481878815u },
+  { 16717361816799281152u,   898318840772166823u,          31196880105u },
+  {                    0u, 17293677953982795024u,         233048697961u },
+  {                    0u,  7353628266884669440u,         105937492160u },
+  {                    0u,  2404693032470315008u,         192398640987u },
+  {                    0u,  9191155893041889280u,          91130358670u },
+  {                    0u,  6353946855033798656u,         142498253559u },
+  {                    0u,  3767824038248841216u,         247344448149u },
+  {                    0u,  7205759403792793600u,         149204254150u },
+  {                    0u,                    0u,         198390625000u },
+  {                    0u,                    0u,         232000000000u },
+  {  9707857417284919307u,            752316384u,                    0u },
+  {  1844515466944871826u,   752316384526264005u,                    0u },
+  { 15167599819856275072u, 17063068157692817751u,             40783152u },
+  { 14830185305589481472u,  5385330256507239985u,          48924990778u },
+  {  6163721531743535104u,  3373050282752075748u,          58291939338u },
+  { 14122847407012052992u,  4116064001262906061u,          10182853422u },
+  {   335491783960035328u, 11306582046748043076u,          46223132276u },
+  {   941252322120433664u, 17035410946089626406u,         116612931040u },
+  { 11529215046068469760u, 15618595715183448558u,         224923491477u },
+  {                    0u,  5141740092277295680u,         149846685770u },
+  {                    0u, 16973644291514990592u,          74278734288u },
+  {                    0u, 14625255268443750400u,         208920143100u },
+  {                    0u, 14021170507320131584u,         252792836676u },
+  {                    0u,  4451355232865091584u,          68760089176u },
+  {                    0u, 12891553933348044800u,          88241308450u },
+  {                    0u,  1152921504606846976u,          34698852539u },
+  {                    0u,                    0u,         187062500000u },
+  {                    0u,                    0u,         160000000000u },
+  {  8061591463141767016u,                11479u,                    0u },
+  { 16628725344207857142u,       11479437019748u,                    0u },
+  {  3555541870038531535u,  5562205901560339855u,                  622u },
+  { 18316647450161853665u,  2106077949367544134u,         110301527786u },
+  { 16709574568378075648u,  7496855998374373623u,         234114170714u },
+  {  7696022835795591168u,   229183437194837004u,          90406405378u },
+  { 16664449640376041472u,   465169186276472889u,           2012424059u },
+  {  3109186955116544000u,  2152980561625316473u,         123025216872u },
+  { 10515518101817131008u,  2059790725449340402u,         104116713310u },
+  {  9961962375743537152u, 17891190926410198930u,          94111661478u },
+  {  9223372036854775808u,  9930696175609809814u,         166969883403u },
+  {                    0u,  7276914261609005312u,          11538344118u },
+  {                    0u, 10539762974036983808u,         182394482312u },
+  {                    0u, 12851089458992250880u,         136571361695u },
+  {                    0u,  9449311677678878720u,         159696658955u },
+  {                    0u,  8699564697382289408u,          11512248212u },
+  {                    0u,  4224376450473525248u,         148471604347u },
+  {                    0u,  4611686018427387904u,         123229003906u },
+  {                    0u,                    0u,         130250000000u },
+  {                    0u,                    0u,         128000000000u },
+  {   748977172262750475u,            175162308u,                    0u },
+  { 15965033457315095468u,   175162308040602133u,                    0u },
+  { 16809402149066729206u, 13756840147955779925u,              9495567u },
+  {  7563769067065700371u, 13788447602092505948u,          15745759798u },
+  { 13550322810840051428u,  4972540435632173670u,          54747473242u },
+  {  8663209637545764864u,  2844874687533091959u,          90269561957u },
+  {  8969247575312957440u, 15377573779532804095u,         101154220965u },
+  {  6193172891660451840u, 17824715805091194381u,         165833619944u },
+  {  3427954273864908800u, 18277569135638159711u,         232966279779u },
+  { 10058367555266936832u,  4254645803379752845u,          99990829008u },
+  { 13907115649320091648u,  2933643244178200621u,         208230644811u },
+  {                    0u, 17188148801879487562u,          75159033118u },
+  {                    0u, 11069762501163246592u,          30931771413u },
+  {                    0u, 11676570643941818368u,          21600093027u },
+  {                    0u, 17840016768744030208u,          99632988162u },
+  {                    0u, 16463817321652158464u,           2967109246u },
+  {                    0u,  6954191143357644800u,         126892505325u },
+  {                    0u,  5080060379673919488u,         237376987457u },
+  {                    0u,                    0u,          65275390625u },
+  {                    0u,                    0u,         161000000000u },
+  { 14106411361315920281u,                 2672u,                    0u },
+  {  3609034283485221502u,        2672764710092u,                    0u },
+  {  9896072247338192335u, 16433563478020213436u,                  144u },
+  {  8889095178479228297u,  4194750497955655375u,         144890865261u },
+  {  3631796911038383102u,  2691539602252904735u,         109227397880u },
+  {   898318840772166823u,  3775467271962795241u,         248145908654u },
+  { 17293677953982795024u, 16980212613224918121u,         174204668490u },
+  {  7353628266884669440u,  4172857038337333440u,          74920499170u },
+  {  2404693032470315008u,  5936867627376461659u,         226226211033u },
+  {  9191155893041889280u, 17856837443266866062u,         217321838238u },
+  {  6353946855033798656u,  8956297047799810807u,         158968021097u },
+  {  3767824038248841216u, 15356974049716912789u,         105485521835u },
+  {  7205759403792793600u,  6923608913322982854u,         171832503231u },
+  {                    0u,  4855902993563955944u,         191375329591u },
+  {                    0u, 13835893222288330752u,          55263239028u },
+  {                    0u,  9114973913760137216u,         116750045274u },
+  {                    0u, 17937099003422310400u,          90494123725u },
+  {                    0u,  7007960010734960640u,         205972372085u },
+  {                    0u,  7683422439270776832u,         117379902273u },
+  {                    0u,   720575940379279360u,          65416519165u },
+  {                    0u,                    0u,         253039062500u },
+  {                    0u,                    0u,         228000000000u },
+  { 17063068157692817751u,             40783152u,                    0u },
+  {  5385330256507239985u,    40783152924990778u,                    0u },
+  {  3373050282752075748u,  2768933352715741194u,              2210859u },
+  {  4116064001262906061u, 15201941611824153390u,          43150104177u },
+  { 11306582046748043076u,  1418128541727000180u,         113824098906u },
+  { 17035410946089626406u,  5353350204565757408u,          90076876902u },
+  { 15618595715183448558u,  1721001680354286741u,         102290205696u },
+  {  5141740092277295680u,   637631411660453962u,             93295688u },
+  { 16973644291514990592u,  1630012588870568400u,          72034566068u },
+  { 14625255268443750400u,  9253063571656828156u,         180088363159u },
+  { 14021170507320131584u,  6029146854993203780u,         151501609581u },
+  {  4451355232865091584u, 16987401965352759896u,         109326840705u },
+  { 12891553933348044800u, 14499131620542087970u,         129920888905u },
+  {  1152921504606846976u,  1978417255298660539u,          73785999500u },
+  {                    0u,  5790079354402454176u,         140107250214u },
+  {                    0u, 13748918935842078720u,          38313880830u },
+  {                    0u, 18047438014740692992u,         254745330388u },
+  {                    0u,  3116889656839372800u,         212978353575u },
+  {                    0u, 15995952446606147584u,         167168966926u },
+  {                    0u, 12530140063251562496u,          14867142319u },
+  {                    0u, 16717361816799281152u,         175679260253u },
+  {                    0u,                    0u,          93906250000u },
+  {                    0u,                    0u,          16000000000u },
+  {  5562205901560339855u,                  622u,                    0u },
+  {  2106077949367544134u,         622301527786u,                    0u },
+  {  7496855998374373623u, 13558973353698967386u,                   33u },
+  {   229183437194837004u,  6228991722850501890u,          33735033418u },
+  {   465169186276472889u, 16886831391703377787u,          74337674317u },
+  {  2152980561625316473u,  1181713637872883048u,          77915436964u },
+  {  2059790725449340402u, 12393932434925221726u,         164064060824u },
+  { 17891190926410198930u, 10684799845419711910u,         152671876423u },
+  {  9930696175609809814u,  4590318792215640843u,          71579224160u },
+  {  7276914261609005312u,  6383712187366189238u,          96248841680u },
+  { 10539762974036983808u,  1904270214927675016u,         208346061731u },
+  { 12851089458992250880u,  3711506775113308575u,         163103230695u },
+  {  9449311677678878720u,  8091219444738793995u,         231201201185u },
+  {  8699564697382289408u,    39436684991068052u,          33438625885u },
+  {  4224376450473525248u, 18025182908196512891u,          93002137866u },
+  {  4611686018427387904u,  7853924592034603138u,          10977147123u },
+  {                    0u,  4815749283615688320u,         243425762105u },
+  {                    0u, 14242399906544287744u,          57261062291u },
+  {                    0u,    76242322576113664u,         147772082046u },
+  {                    0u, 10858088421377703936u,         126004133104u },
+  {                    0u, 14293835879041466368u,         240588618152u },
+  {                    0u, 12182236992037191680u,         168774870395u },
+  {                    0u, 11529215046068469760u,         123660400390u },
+  {                    0u,                    0u,           6625000000u },
+  {                    0u,                    0u,          64000000000u },
+  { 13756840147955779925u,              9495567u,                    0u },
+  { 13788447602092505948u,     9495567745759798u,                    0u },
+  {  4972540435632173670u, 14000097438505379162u,               514755u },
+  {  2844874687533091959u, 16451062686452429925u,         195758946802u },
+  { 15377573779532804095u,  4009347599785716645u,         242891813895u },
+  { 17824715805091194381u, 16544162347546196456u,           7217347168u },
+  { 18277569135638159711u, 17674258299745817187u,          96896860837u },
+  {  4254645803379752845u,  5215238411201214416u,         165958123462u },
+  {  2933643244178200621u, 14253990228345322571u,         198282718640u },
+  { 17188148801879487562u, 11214836553940194590u,         176772710358u },
+  { 11069762501163246592u, 14620711348380590101u,         214607957507u },
+  { 11676570643941818368u,  6638710787931587427u,           3792590350u },
+  { 17840016768744030208u, 17320000343692853250u,          14359885232u },
+  { 16463817321652158464u,    75147386268843646u,         176938919100u },
+  {  6954191143357644800u, 17938801582125480173u,         188004073747u },
+  {  5080060379673919488u,  6573358613626446145u,          19972464382u },
+  {                    0u,  8688505427903736481u,         254356342484u },
+  {                    0u,   539870168696556032u,         212471004823u },
+  {                    0u,  9002861336394465280u,         151029266420u },
+  {                    0u, 17989846818158018560u,         244488046090u },
+  {                    0u,  2700938287723315200u,          10975231550u },
+  {                    0u, 17800090499088908288u,          62146418157u },
+  {                    0u,  8809040871136690176u,         237964944839u },
+  {                    0u,  9223372036854775808u,         199477539062u },
+  {                    0u,                    0u,         246500000000u },
+  { 16433563478020213436u,                  144u,                    0u },
+  {  4194750497955655375u,         144890865261u,                    0u },
+  {  2691539602252904735u, 15763656745260536568u,                    7u },
+  {  3775467271962795241u,  8787336846248645550u,           7854549544u },
+  { 16980212613224918121u, 17584084447880694346u,          40476362484u },
+  {  4172857038337333440u, 18041672551129683938u,         244953235127u },
+  {  5936867627376461659u, 14025886302294509785u,         183978041028u },
+  { 17856837443266866062u, 18430498103283160734u,         196760344819u },
+  {  8956297047799810807u,  3292348826238025833u,         243999119304u },
+  { 15356974049716912789u,  9211721212658275243u,         200178478587u },
+  {  6923608913322982854u, 10233245872666307519u,         251499368407u },
+  {  4855902993563955944u,  6200995035623311671u,         215554745370u },
+  { 13835893222288330752u,  8480542380570450804u,          26336156614u },
+  {  9114973913760137216u, 11870363864499900506u,         198459731123u },
+  { 17937099003422310400u,  9301051379839581901u,         179643493714u },
+  {  7007960010734960640u, 11456694803569638005u,          82504211005u },
+  {  7683422439270776832u, 14327208890643983169u,          61621068669u },
+  {   720575940379279360u,  4510081789599866365u,         125776679550u },
+  {                    0u, 13255356976020303332u,         126244492023u },
+  {                    0u,  9658806854127314944u,         247718574341u },
+  {                    0u, 13708435528809971712u,           5523604968u },
+  {                    0u,  1580190652103131136u,         232743135779u },
+  {                    0u, 16557336970347413504u,          35085662306u },
+  {                    0u, 12751520132434493440u,          98897575035u },
+  {                    0u,  9295429630892703744u,         123691261291u },
+  {                    0u,                    0u,         107503906250u },
+  {                    0u,                    0u,         202000000000u },
+  {  2768933352715741194u,              2210859u,                    0u },
+  { 15201941611824153390u,     2210859150104177u,                    0u },
+  {  1418128541727000180u, 16872870088062921306u,               119850u },
+  {  5353350204565757408u,  5112979788807802982u,          42914680120u },
+  {  1721001680354286741u, 13742728082020150272u,          56277175189u },
+  {   637631411660453962u,  2217110934613627976u,         149744994782u },
+  {  1630012588870568400u, 11021433940188610484u,         222120189824u },
+  {  9253063571656828156u,  1713669895470733463u,         128597473131u },
+  {  6029146854993203780u,  3313382510572018285u,         107092898231u },
+  { 16987401965352759896u, 14976595232784069505u,         183179618825u },
+  { 14499131620542087970u,  7213172372862496841u,           9811882854u },
+  {  1978417255298660539u, 15836474542502248588u,         102391026857u },
+  {  5790079354402454176u,  3221099285878340134u,         169858497005u },
+  { 13748918935842078720u,  3265814602578095358u,         237174616142u },
+  { 18047438014740692992u,  6502528252282225364u,          78177040164u },
+  {  3116889656839372800u, 16392476834556790183u,          36352502762u },
+  { 15995952446606147584u, 15167629413417091342u,         234888637949u },
+  { 12530140063251562496u,  1366763272626280111u,         253822238838u },
+  { 16717361816799281152u,  8720523635169216093u,         118074092385u },
+  {                    0u,  9649171375767398672u,          97472740533u },
+  {                    0u,  7647980704001073152u,         181523082628u },
+  {                    0u, 13286434495608651776u,         132414597864u },
+  {                    0u,  4358271637167013888u,         232720259057u },
+  {                    0u, 15954987941890097152u,         241236262378u },
+  {                    0u,  7911135695429697536u,         234864921629u },
+  {                    0u,  7205759403792793600u,          29428863525u },
+  {                    0u,                    0u,          37390625000u },
+  {                    0u,                    0u,         232000000000u },
+  { 13558973353698967386u,                   33u,                    0u },
+  {  6228991722850501890u,          33735033418u,                    0u },
+  { 16886831391703377787u, 15288289344628122701u,                    1u },
+  {  1181713637872883048u,   952589339068938148u,           1828779826u },
+  { 12393932434925221726u, 10058155040190817688u,          50051639971u },
+  { 10684799845419711910u,  5322725640026584391u,         163545253677u },
+  {  4590318792215640843u,  2269982385930389600u,          45288545535u },
+  {  6383712187366189238u, 13216683679976310224u,         255123055991u },
+  {  1904270214927675016u, 17417440642083494819u,         119716477857u },
+  {  3711506775113308575u,  3029180749090900711u,         161944201349u },
+  {  8091219444738793995u,  8315443826261908513u,         133164212217u },
+  {    39436684991068052u,  1488962797247197277u,         249450781113u },
+  { 18025182908196512891u, 18009099634999034122u,         185080716834u },
+  {  7853924592034603138u,  8092455412807497971u,          34976275247u },
+  {  4815749283615688320u, 17808458047236758329u,          47438692886u },
+  { 14242399906544287744u,  3164591817527425171u,          22965398445u },
+  {    76242322576113664u,  3314036340472350590u,         173171552866u },
+  { 10858088421377703936u,    33234902404332784u,          98179654270u },
+  { 14293835879041466368u, 12349284717857274280u,         126001801667u },
+  { 12182236992037191680u, 18209607903013119355u,         195669456065u },
+  { 11529215046068469760u,  7891549145984268038u,         193987144822u },
+  {                    0u,  7703609897518594624u,         118427801736u },
+  {                    0u,  6336912652634587136u,         136417613529u },
+  {                    0u,  4461621834659397632u,         217343524723u },
+  {                    0u,  5484660635557953536u,         115241865004u },
+  {                    0u, 15142619273265938432u,          44297324048u },
+  {                    0u, 12170977992968765440u,          16820883035u },
+  {                    0u,  1152921504606846976u,          91659790039u },
+  {                    0u,                    0u,         215062500000u },
+  {                    0u,                    0u,         160000000000u },
+  { 14000097438505379162u,               514755u,                    0u },
+  { 16451062686452429925u,      514755758946802u,                    0u },
+  {  4009347599785716645u, 17812314011563521031u,                27904u },
+  { 16544162347546196456u,  7684138864490314336u,            965607477u },
+  { 17674258299745817187u,  9740522787420029605u,          53416558002u },
+  {  5215238411201214416u,  6701109407732989894u,         178528034798u },
+  { 14253990228345322571u, 16534886227502443952u,         238363267868u },
+  { 11214836553940194590u,  8908667306968317910u,          28896357978u },
+  { 14620711348380590101u,  7531472173477105155u,          90482939822u },
+  {  6638710787931587427u, 11527371604834801166u,         174408281924u },
+  { 17320000343692853250u, 15688593496691078576u,          68624900066u },
+  {    75147386268843646u, 11394944804253312188u,         226850480357u },
+  { 17938801582125480173u, 11182279880854372627u,         229617721195u },
+  {  6573358613626446145u,   150579373068361470u,         107606192607u },
+  {  8688505427903736481u,  3147220002440857300u,         223008162924u },
+  {   539870168696556032u,  3630514817795505815u,         108170611138u },
+  {  9002861336394465280u, 11708796588334233588u,         194196810602u },
+  { 17989846818158018560u, 16844495466426369546u,         106634735134u },
+  {  2700938287723315200u, 17636655472325475902u,          30913141928u },
+  { 17800090499088908288u, 17038926655686645229u,         168956085008u },
+  {  8809040871136690176u, 15602838456783529415u,          16923682064u },
+  {  9223372036854775808u, 10869815869248876790u,          16845831567u },
+  {                    0u, 18407124180939800832u,         143589253898u },
+  {                    0u,  5705018517251293184u,          10997852201u },
+  {                    0u,  9660452258743058432u,          41309269673u },
+  {                    0u,  5646292272224927744u,         169523694166u },
+  {                    0u,  7410409304047484928u,          86306086117u },
+  {                    0u,  5953758707383795712u,         229401719093u },
+  {                    0u,  4611686018427387904u,          53322753906u },
+  {                    0u,                    0u,         114250000000u },
+  {                    0u,                    0u,         128000000000u },
+  { 15763656745260536568u,                    7u,                    0u },
+  {  8787336846248645550u,           7854549544u,                    0u },
+  { 17584084447880694346u,  7854549544476362484u,                    0u },
+  { 18041672551129683938u,    15035424419724983u,            425795984u },
+  { 14025886302294509785u, 18280822466032836292u,         144000815071u },
+  { 18430498103283160734u, 11524250747302615283u,         223991005371u },
+  {  3292348826238025833u, 15212285943691810760u,         187624730884u },
+  {  9211721212658275243u,  7951804027551297019u,           4824659673u },
+  { 10233245872666307519u,  1706416229965221847u,         217431068160u },
+  {  6200995035623311671u,  3406023111930700826u,             92505009u },
+  {  8480542380570450804u, 16132696204133391302u,         177184640882u },
+  { 11870363864499900506u, 11593846688794356915u,         114874555213u },
+  {  9301051379839581901u,  6875759884161133906u,          77628503688u },
+  { 11456694803569638005u,  3593593325323835965u,         136372735690u },
+  { 14327208890643983169u,  9542049733257388925u,         202194809084u },
+  {  4510081789599866365u,  9926551925937787518u,         252517275552u },
+  { 13255356976020303332u,  3128491553219547895u,         160538119458u },
+  {  9658806854127314944u, 17158408656931354885u,          34169595866u },
+  { 13708435528809971712u,  2065169543154992616u,         218930159197u },
+  {  1580190652103131136u,  4832622393556232739u,          93111953065u },
+  { 16557336970347413504u, 16505930714733656162u,         169261976984u },
+  { 12751520132434493440u, 18270988073492888699u,         152894788296u },
+  {  9295429630892703744u,  2525111411519708523u,         200990472248u },
+  {                    0u, 16728989342518570442u,          56136886563u },
+  {                    0u,  7974052022039438336u,          35906880329u },
+  {                    0u,  5356554962386550784u,          73432274226u },
+  {                    0u,  6693869495028547584u,          50290379426u },
+  {                    0u,  8157517147199766528u,         162362875392u },
+  {                    0u, 12065776720423157760u,            442219890u },
+  {                    0u, 11997589407315001344u,         114654087066u },
+  {                    0u,                    0u,         154650390625u },
+  {                    0u,                    0u,          97000000000u },
+  { 16872870088062921306u,               119850u,                    0u },
+  {  5112979788807802982u,      119850914680120u,                    0u },
+  { 13742728082020150272u,  2418433229320326037u,                 6497u },
+  {  2217110934613627976u,  1143911773589293534u,          97131103528u },
+  { 11021433940188610484u,  9276183703610924928u,          40062011581u },
+  {  1713669895470733463u,  3532180128827684715u,         189502862926u },
+  {  3313382510572018285u,  8563997501322031543u,          78191479868u },
+  { 14976595232784069505u, 14843890409658460681u,          60464255234u },
+  {  7213172372862496841u,  9489417861634552678u,           2804688911u },
+  { 15836474542502248588u,  1113198223322322089u,          15514422373u },
+  {  3221099285878340134u, 11190777557146597869u,         101060346596u },
+  {  3265814602578095358u, 17764553645932638286u,         228606653266u },
+  {  6502528252282225364u, 14900777150991234852u,          82963018382u },
+  { 16392476834556790183u, 17364899863357893610u,         142807772747u },
+  { 15167629413417091342u, 15537570181590167037u,          75941353107u },
+  {  1366763272626280111u,  5558052627121307766u,         147842293367u },
+  {  8720523635169216093u, 12095241565795232609u,         119301302636u },
+  {  9649171375767398672u,  2187936505958366389u,         108655684359u },
+  {  7647980704001073152u, 12009203621325860228u,           7118608275u },
+  { 13286434495608651776u, 14814842834750302952u,         147651020232u },
+  {  4358271637167013888u,  5965296499605198833u,         200803114239u },
+  { 15954987941890097152u,  4051026394962148842u,         255323379371u },
+  {  7911135695429697536u, 16799526299141688349u,         171219606580u },
+  {  7205759403792793600u,  9460214166646215205u,          52910704145u },
+  {                    0u, 10750736995029068008u,          17512839237u },
+  {                    0u,  5377963045376430080u,          69582798620u },
+  {                    0u, 15996910350253424640u,          28291539960u },
+  {                    0u, 13651157529655246848u,         248867194247u },
+  {                    0u,  9771305410219737088u,         135740030732u },
+  {                    0u, 12709439623416250368u,          12529703527u },
+  {                    0u,  9943947977234055168u,         103688980102u },
+  {                    0u,                    0u,         134539062500u },
+  {                    0u,                    0u,         228000000000u },
+  {   952589339068938148u,           1828779826u,                    0u },
+  { 10058155040190817688u,  1828779826051639971u,                    0u },
+  {  5322725640026584391u,   371564423966525229u,             99138353u },
+  {  2269982385930389600u, 14464859121514339583u,          49020142547u },
+  { 13216683679976310224u,  3913119023023056247u,         211784141584u },
+  { 17417440642083494819u,  5493396321716566945u,          16212130607u },
+  {  3029180749090900711u,  5837454566818211973u,          47297797611u },
+  {  8315443826261908513u,  2886670683193253881u,         235316449046u },
+  {  1488962797247197277u,  5504823105587173817u,          22156486731u },
+  { 18009099634999034122u,  9431834277334851106u,          75298417058u },
+  {  8092455412807497971u, 12921661346456247087u,         162511300760u },
+  { 17808458047236758329u,  3643076516404724246u,         152700484665u },
+  {  3164591817527425171u, 12559396953196866477u,          57197491573u },
+  {  3314036340472350590u,  1626880974916825698u,         117680846273u },
+  {    33234902404332784u,  6806994170946429566u,         193088193394u },
+  { 12349284717857274280u,  7596631230206896579u,         114369007893u },
+  { 18209607903013119355u,  3100480253729502401u,          21411814204u },
+  {  7891549145984268038u,  6310570748781063286u,          60168077371u },
+  {  7703609897518594624u, 14251867077375744136u,          59342096725u },
+  {  6336912652634587136u,  6701165793751570137u,          85772595262u },
+  {  4461621834659397632u, 10856833140463959923u,          62363270925u },
+  {  5484660635557953536u, 15867563727561248556u,          13588550103u },
+  { 15142619273265938432u,  5048961008671491600u,         215860182353u },
+  { 12170977992968765440u, 13278183119599849051u,          81273704724u },
+  {  1152921504606846976u,  4547591784941053655u,          20719811749u },
+  {                    0u, 11815437715887182496u,         165246525444u },
+  {                    0u,   398495392178782208u,           4640516162u },
+  {                    0u,  9154841240825495552u,          66021602478u },
+  {                    0u,  1902683298245640192u,         174496284938u },
+  {                    0u,  5081900962138816512u,          10103144668u },
+  {                    0u,  3234710432358858752u,         220275490403u },
+  {                    0u, 16717361816799281152u,          99175354003u },
+  {                    0u,                    0u,         147906250000u },
+  {                    0u,                    0u,          16000000000u },
+  { 17812314011563521031u,                27904u,                    0u },
+  {  7684138864490314336u,       27904965607477u,                    0u },
+  {  9740522787420029605u, 13488568028574514610u,                 1512u },
+  {  6701109407732989894u,   275784718433886190u,         232731216738u },
+  { 16534886227502443952u, 10020568880357102364u,          98014950319u },
+  {  8908667306968317910u,  8876397213146246746u,         175543216127u },
+  {  7531472173477105155u,  2155905919114811310u,         255481190457u },
+  { 11527371604834801166u,  1087100407155601220u,          57116871894u },
+  { 15688593496691078576u,  2903498381705011170u,         214058931831u },
+  { 11394944804253312188u, 12223476257006657765u,         119157398962u },
+  { 11182279880854372627u, 12148657163736735595u,         178662635975u },
+  {   150579373068361470u,  8951241323311673823u,         199658580024u },
+  {  3147220002440857300u,  8463862715901576300u,          56485247764u },
+  {  3630514817795505815u,  3873401978748963266u,          20458826917u },
+  { 11708796588334233588u,   248364795947002730u,         165209977542u },
+  { 16844495466426369546u, 10454378025404001822u,         198013463882u },
+  { 17636655472325475902u,  6574176865628265640u,          74566732968u },
+  { 17038926655686645229u,    16703315293848336u,         168356386842u },
+  { 15602838456783529415u,  9896033222450013456u,          26000905488u },
+  { 10869815869248876790u, 17311376269334085007u,          16536465035u },
+  { 18407124180939800832u, 18378511316495639306u,         139938451587u },
+  {  5705018517251293184u, 15120796393727584297u,         131996301094u },
+  {  9660452258743058432u, 18253447805740347049u,          38819700014u },
+  {  5646292272224927744u,  5842497225601731158u,          46989521388u },
+  {  7410409304047484928u,  4369968404176723173u,         236316722409u },
+  {  5953758707383795712u, 16142207253674488117u,         233236896461u },
+  {  4611686018427387904u, 12124259227391928178u,         205875070808u },
+  {                    0u, 13019483264566077056u,          88657257409u },
+  {                    0u,    74901376448135168u,         193705787602u },
+  {                    0u, 13897060093813325824u,         210004060411u },
+  {                    0u,  4495486210810052608u,         251753361137u },
+  {                    0u, 14885496280087265280u,         241243700795u },
+  {                    0u,  4976477588244398080u,          59806944370u },
+  {                    0u, 11529215046068469760u,         114269775390u },
+  {                    0u,                    0u,          30625000000u },
+  {                    0u,                    0u,          64000000000u },
+  {    15035424419724983u,            425795984u,                    0u },
+  { 18280822466032836292u,   425795984000815071u,                    0u },
+  { 11524250747302615283u, 10043594327130472635u,             23082446u },
+  { 15212285943691810760u,  8336034337032909060u,         206544464339u },
+  {  7951804027551297019u, 16717215784895280857u,         211451897326u },
+  {  1706416229965221847u, 10968831263951212032u,         238906242083u },
+  {  3406023111930700826u,  5536629379734406065u,          35594621534u },
+  { 16132696204133391302u,  1618806894932332402u,          94300141280u },
+  { 11593846688794356915u, 11363331325254998861u,         224087755697u },
+  {  6875759884161133906u,  8775167772751754888u,         177616007425u },
+  {  3593593325323835965u,  2898202945316114122u,           1475702798u },
+  {  9542049733257388925u,  8868842714495185148u,          14157111896u },
+  {  9926551925937787518u, 17052094667531999136u,          88480780926u },
+  {  3128491553219547895u,  3658615537031138594u,         126924395904u },
+  { 17158408656931354885u, 12486952437987190746u,         128198333945u },
+  {  2065169543154992616u,   912079238520577629u,         249676919048u },
+  {  4832622393556232739u, 10960072898031888041u,           8049443914u },
+  { 16505930714733656162u,  6129550094334741912u,          74594146742u },
+  { 18270988073492888699u,  7965724516573729480u,         182332283576u },
+  {  2525111411519708523u,  5801761178810791992u,         184431822791u },
+  { 16728989342518570442u, 13197466483098446115u,         199314514103u },
+  {  7974052022039438336u, 11326268638393107273u,         183715436091u },
+  {  5356554962386550784u,  3597339351794947378u,          59613998253u },
+  {  6693869495028547584u,   353880726151383714u,         173195012157u },
+  {  8157517147199766528u, 11154818162602073600u,          61019183912u },
+  { 12065776720423157760u,  5141043976157511026u,          40604703904u },
+  { 11997589407315001344u,  7188225141808859034u,         160278696552u },
+  {                    0u, 13894168943295705185u,         104389674465u },
+  {                    0u, 12176538069834828288u,         225753204407u },
+  {                    0u,  7994239409235165184u,         183660091451u },
+  {                    0u, 13707777025480065024u,          59433368586u },
+  {                    0u, 10120227247676719104u,          10743100081u },
+  {                    0u,  7358494763030413312u,         177548618618u },
+  {                    0u,  7656119366529843200u,         122398904800u },
+  {                    0u,  9223372036854775808u,         224415039062u },
+  {                    0u,                    0u,          86500000000u },
+  {  2418433229320326037u,                 6497u,                    0u },
+  {  1143911773589293534u,        6497131103528u,                    0u },
+  {  9276183703610924928u,  3877189582299842749u,                  352u },
+  {  3532180128827684715u,  7625565791857948238u,          96210182868u },
+  {  8563997501322031543u, 16568435163612007484u,         212413382749u },
+  { 14843890409658460681u, 17592071940521808130u,          93898176669u },
+  {  9489417861634552678u, 15158637878035490831u,         157953668130u },
+  {  1113198223322322089u, 17789243229146401893u,          34821751405u },
+  { 11190777557146597869u, 14677686051252896484u,         109964356807u },
+  { 17764553645932638286u,  3531237481269211986u,         199795678955u },
+  { 14900777150991234852u,  8074435404989280910u,         235191428767u },
+  { 17364899863357893610u,  7086549341467684427u,         159437716020u },
+  { 15537570181590167037u, 10556134770918626963u,          52384162609u },
+  {  5558052627121307766u, 10772666134712966775u,          49572249212u },
+  { 12095241565795232609u,  6195173298198112620u,         124583987401u },
+  {  2187936505958366389u,  8144773843324250887u,         201335841017u },
+  { 12009203621325860228u, 14144284817150924691u,         249441529074u },
+  { 14814842834750302952u,  6464447844648863176u,         242766763216u },
+  {  5965296499605198833u, 15760468443293179135u,         208350438419u },
+  {  4051026394962148842u,  5172191224908322475u,          19854376706u },
+  { 16799526299141688349u,  2357554307308969012u,           2280385048u },
+  {  9460214166646215205u,  1602046917604361745u,          24127803275u },
+  { 10750736995029068008u,  7830970218109515845u,         139086847137u },
+  {  5377963045376430080u,  2899479134887821084u,         161424517746u },
+  { 15996910350253424640u, 15792042302392017912u,         114157181078u },
+  { 13651157529655246848u, 11286099112296056199u,         150856088328u },
+  {  9771305410219737088u, 15161477829153947404u,           8611820658u },
+  { 12709439623416250368u,   423831848142641767u,         114821905360u },
+  {  9943947977234055168u,  9707413321046312582u,         208022975970u },
+  {                    0u, 10969483299803835620u,         226526239930u },
+  {                    0u,  4326479556120930304u,         186594656881u },
+  {                    0u, 12876227232041795584u,         113234538926u },
+  {                    0u, 16967986827791171584u,         174698021676u },
+  {                    0u,  1288146316538413056u,          44919836409u },
+  {                    0u, 13715290452691779584u,         249069830551u },
+  {                    0u,  4683743612465315840u,         151743507385u },
+  {                    0u,                    0u,         185253906250u },
+  {                    0u,                    0u,          74000000000u },
+  {   371564423966525229u,             99138353u,                    0u },
+  { 14464859121514339583u,    99138353020142547u,                    0u },
+  {  3913119023023056247u, 16344805304534272784u,              5374300u },
+  {  5493396321716566945u,    26429987091348271u,          92886053671u },
+  {  5837454566818211973u,  8691371289609838059u,          39001432772u },
+  {  2886670683193253881u, 12980168378493046550u,         196471160181u },
+  {  5504823105587173817u, 14010125458129496139u,         117703656337u },
+  {  9431834277334851106u, 17061829677031795106u,         145759490422u },
+  { 12921661346456247087u,  2227928323072698520u,         118924923640u },
+  {  3643076516404724246u,  7394752319272287289u,         248120776236u },
+  { 12559396953196866477u,  8805771303577744757u,          44400870326u },
+  {  1626880974916825698u, 16371027194302248385u,         182477361818u },
+  {  6806994170946429566u,  9114324123731231602u,         154887475162u },
+  {  7596631230206896579u, 14468189808746991893u,         218494088500u },
+  {  3100480253729502401u,  2376054557800684348u,          52784322141u },
+  {  6310570748781063286u, 12462238943546048571u,          93128806175u },
+  { 14251867077375744136u, 15334855370842605909u,          31675579326u },
+  {  6701165793751570137u,  7211347914013798462u,         190831304175u },
+  { 10856833140463959923u, 13763642332572548877u,         239390927953u },
+  { 15867563727561248556u, 16868268377740071383u,          81746128545u },
+  {  5048961008671491600u,  1120013377627684177u,         161914430661u },
+  { 13278183119599849051u, 15898107650717274388u,         197060716046u },
+  {  4547591784941053655u, 12281923376333274277u,          14861838142u },
+  { 11815437715887182496u,  6383530489286615044u,          62665804400u },
+  {   398495392178782208u,  4253822060257126466u,         112346051881u },
+  {  9154841240825495552u, 17614372438391501998u,          41230600155u },
+  {  1902683298245640192u,  4309951310554333450u,         219954877043u },
+  {  5081900962138816512u, 13106185988973773020u,         115233642928u },
+  {  3234710432358858752u,  2070134359761960547u,         176710487766u },
+  { 16717361816799281152u,  9399359914137865875u,         214112222208u },
+  {                    0u, 17415053284723541264u,            509540321u },
+  {                    0u,  4840502610448261120u,         225944071930u },
+  {                    0u,  5690599259712258048u,         250262404172u },
+  {                    0u,   114769594245185536u,          76308488004u },
+  {                    0u,  3150620882578178048u,          68006221672u },
+  {                    0u,  5136918324969472000u,         104170795500u },
+  {                    0u,  7205759403792793600u,         236278472900u },
+  {                    0u,                    0u,         196390625000u },
+  {                    0u,                    0u,         232000000000u },
+  { 13488568028574514610u,                 1512u,                    0u },
+  {   275784718433886190u,        1512731216738u,                    0u },
+  { 10020568880357102364u,    98202693831717807u,                   82u },
+  {  8876397213146246746u, 12909287260170414079u,          82005323578u },
+  {  2155905919114811310u, 11728631949380786233u,          58699813864u },
+  {  1087100407155601220u, 18263701925522197718u,         232635810411u },
+  {  2903498381705011170u,  4868886449713321591u,         107990077265u },
+  { 12223476257006657765u,  5870139507184082354u,          81263942863u },
+  { 12148657163736735595u,  5978562500822661575u,         207318220900u },
+  {  8951241323311673823u, 10821136839630268472u,         100324098522u },
+  {  8463862715901576300u,  9490907630136752916u,         218586615003u },
+  {  3873401978748963266u, 10564005678001613989u,         219514503133u },
+  {   248364795947002730u,  5754050547468481222u,         221572675895u },
+  { 10454378025404001822u,  3833909949855542602u,          55311927705u },
+  {  6574176865628265640u, 15446538552665967784u,         153207836674u },
+  {    16703315293848336u, 14924837848804399130u,           2837358532u },
+  {  9896033222450013456u, 18140170340418344208u,         196809077080u },
+  { 17311376269334085007u, 11380424819825208971u,          88983380604u },
+  { 18378511316495639306u, 12416915664152252547u,         124616934065u },
+  { 15120796393727584297u, 17195282241626289958u,         177673122346u },
+  { 18253447805740347049u,  2649541045825281326u,          42932158118u },
+  {  5842497225601731158u, 16577429864268509676u,         166143631907u },
+  {  4369968404176723173u, 12051257060168107241u,          35898664273u },
+  { 16142207253674488117u,  5363884561143470797u,          81653299954u },
+  { 12124259227391928178u, 13054029903083620184u,         242290776764u },
+  { 13019483264566077056u,   566314952158634945u,         188707660379u },
+  {    74901376448135168u,  1329472079642345682u,          91030699995u },
+  { 13897060093813325824u, 15686237486658857211u,         219072070825u },
+  {  4495486210810052608u,  1069073549290598129u,         169850352638u },
+  { 14885496280087265280u,  4323599065125928507u,         254057954593u },
+  {  4976477588244398080u, 17861823329752681586u,          33234382774u },
+  { 11529215046068469760u, 17220149985412802078u,         182968291382u },
+  {                    0u,  4344934572159429184u,          54933506201u },
+  {                    0u,  2252927464837120000u,         153235539375u },
+  {                    0u, 10910018171964489728u,         175122131442u },
+  {                    0u,  3597328585515335680u,         242591433270u },
+  {                    0u,  6972808074239148032u,          54195011573u },
+  {                    0u,  2227030015734710272u,         245377996683u },
+  {                    0u,  1152921504606846976u,         139120727539u },
+  {                    0u,                    0u,         243062500000u },
+  {                    0u,                    0u,         160000000000u },
+  { 10043594327130472635u,             23082446u,                    0u },
+  {  8336034337032909060u,    23082446544464339u,                    0u },
+  { 16717215784895280857u, 17238287503805244910u,              1251301u },
+  { 10968831263951212032u,  1434575446038410275u,         229934489438u },
+  {  5536629379734406065u, 14009569747841241694u,          94077768490u },
+  {  1618806894932332402u, 14938795732275951328u,          42759460297u },
+  { 11363331325254998861u,  6687653542888983473u,         201809833739u },
+  {  8775167772751754888u,    28238723295162625u,          11362538425u },
+  {  2898202945316114122u,  4745270274832691214u,         185001530824u },
+  {  8868842714495185148u,   926478968112308824u,         200257241617u },
+  { 17052094667531999136u,  9213681606604198526u,          17050224525u },
+  {  3658615537031138594u, 13346223820579313024u,         141499474680u },
+  { 12486952437987190746u,   691642518601291257u,         248723500243u },
+  {   912079238520577629u,  1153720150033789192u,         211037494016u },
+  { 10960072898031888041u, 12089015034721780810u,             62543294u },
+  {  6129550094334741912u,  3555868702841788854u,         190655346818u },
+  {  7965724516573729480u, 11708406782758214328u,         130192764028u },
+  {  5801761178810791992u,  9417497762905343943u,         124634714003u },
+  { 13197466483098446115u, 12838336066957615287u,         147510523576u },
+  { 11326268638393107273u, 13737708142128207419u,         184695967592u },
+  {  3597339351794947378u, 11683434809834695853u,         104744722650u },
+  {   353880726151383714u,  2689114340106315837u,         218633360270u },
+  { 11154818162602073600u,  8859225263374261032u,         142145777180u },
+  {  5141043976157511026u, 15761671984578600096u,          28480259563u },
+  {  7188225141808859034u,  7087267079878005352u,         235854441950u },
+  { 13894168943295705185u,  4601291730423121377u,         222384201518u },
+  { 12176538069834828288u,  9559411037059581623u,          46249436524u },
+  {  7994239409235165184u, 12969820289641388091u,         108518216710u },
+  { 13707777025480065024u, 13628239920285957130u,           6703095366u },
+  { 10120227247676719104u,  8049893933765800625u,          70738788366u },
+  {  7358494763030413312u, 10391755948840250234u,          14436385624u },
+  {  7656119366529843200u, 14454650777462444512u,          88563338218u },
+  {  9223372036854775808u, 14244638523341127254u,         234783588188u },
+  {                    0u, 12246016810439753984u,          92772203401u },
+  {                    0u,  9382741764551081984u,         137663857901u },
+  {                    0u,  4608696190291148800u,         237508639450u },
+  {                    0u,  1696483666416369664u,         218249837921u },
+  {                    0u, 15416683541605384192u,          97091966563u },
+  {                    0u,  7683140964294066176u,          99835740089u },
+  {                    0u,  4611686018427387904u,         185416503906u },
+  {                    0u,                    0u,          98250000000u },
+  {                    0u,                    0u,         128000000000u },
+  {  3877189582299842749u,                  352u,                    0u },
+  {  7625565791857948238u,         352210182868u,                    0u },
+  { 16568435163612007484u,  1722045467931902045u,                   19u },
+  { 17592071940521808130u, 16095324008152856733u,          19093352271u },
+  { 15158637878035490831u, 15216188060094280738u,          79872529262u },
+  { 17789243229146401893u, 10793385929903030893u,         110824871207u },
+  { 14677686051252896484u, 12613277226875940039u,          39585110623u },
+  {  3531237481269211986u, 10644539625155600107u,          95683767128u },
+  {  8074435404989280910u,  6181262895644173983u,          88577041649u },
+  {  7086549341467684427u,   148914399627082292u,         241335086933u },
+  { 10556134770918626963u, 14379289774887985969u,          85008072665u },
+  { 10772666134712966775u, 11743339675582627452u,         217779502860u },
+  {  6195173298198112620u,  7841621929809463497u,          12636607719u },
+  {  8144773843324250887u, 11168944680251236601u,         231425095176u },
+  { 14144284817150924691u,  6178560202529287410u,           8605469704u },
+  {  6464447844648863176u, 13295243308201596112u,           8334940419u },
+  { 15760468443293179135u, 17040673746172470291u,           3720736583u },
+  {  5172191224908322475u, 14957442487039409922u,          71923776774u },
+  {  2357554307308969012u, 17778155426506992152u,           6810844581u },
+  {  1602046917604361745u, 14945404984219733899u,         165963755736u },
+  {  7830970218109515845u, 11590754866058681505u,         216810192027u },
+  {  2899479134887821084u,  6020790784469412466u,         155628336080u },
+  { 15792042302392017912u,  7934351824569522326u,         208326387722u },
+  { 11286099112296056199u,  5038361112172116744u,          10430122074u },
+  { 15161477829153947404u,  3305187319649924210u,          90273130103u },
+  {   423831848142641767u, 11470175511099161552u,         119179174563u },
+  {  9707413321046312582u,  7308362160352048610u,         163621799460u },
+  { 10969483299803835620u, 10666410671225576634u,          36396187106u },
+  {  4326479556120930304u,  2181639019945820785u,         226578227281u },
+  { 12876227232041795584u,  4615749499734847918u,          81118266888u },
+  { 16967986827791171584u, 14076159200958497580u,           8250220281u },
+  {  1288146316538413056u,  5470405257862074105u,         249763070119u },
+  { 13715290452691779584u,  4565741478181339543u,         167296551263u },
+  {  4683743612465315840u,  8901832997861862329u,          95247509341u },
+  {                    0u, 14190141170191714122u,          93482569333u },
+  {                    0u,  4240772322245764096u,         117769249094u },
+  {                    0u,  4422842195340951552u,          70229892728u },
+  {                    0u, 15448426386733137920u,         120239762755u },
+  {                    0u,  9203504548935630848u,          67837460872u },
+  {                    0u,  5936377627571912704u,         136498922981u },
+  {                    0u,   468374361246531584u,         229321811676u },
+  {                    0u,                    0u,         220025390625u },
+  {                    0u,                    0u,          33000000000u },
+  { 16344805304534272784u,              5374300u,                    0u },
+  {    26429987091348271u,     5374300886053671u,                    0u },
+  {  8691371289609838059u,  8020875056524075716u,               291341u },
+  { 12980168378493046550u,  1400288714762747253u,          13434812508u },
+  { 14010125458129496139u,  6136037711314764689u,          92075909803u },
+  { 17061829677031795106u, 15735488086392394102u,         171332635270u },
+  {  2227928323072698520u,  7735094782793634552u,         134853022518u },
+  {  7394752319272287289u,  7273689191766726188u,          54419320328u },
+  {  8805771303577744757u,  3410634565056431030u,           8394307481u },
+  { 16371027194302248385u,  4600927904885215898u,         153184890870u },
+  {  9114324123731231602u,  9154871331680374746u,         246249416801u },
+  { 14468189808746991893u,  6117978272461042996u,          97496286569u },
+  {  2376054557800684348u, 13116904339287496285u,         105331656266u },
+  { 12462238943546048571u,   867037205615660831u,          74711068809u },
+  { 15334855370842605909u,  1802487145191504830u,         137047002181u },
+  {  7211347914013798462u, 17242009718457409007u,          69097713023u },
+  { 13763642332572548877u, 13620802355488468049u,         127934691219u },
+  { 16868268377740071383u,  4442227880594435745u,         147738385175u },
+  {  1120013377627684177u, 17354849212854314181u,          23240813655u },
+  { 15898107650717274388u, 18202319179831567886u,          87940808260u },
+  { 12281923376333274277u, 17568634016348874558u,          68986749699u },
+  {  6383530489286615044u,  7496925598312450672u,           3952397558u },
+  {  4253822060257126466u,   601870379496813865u,         246406409151u },
+  { 17614372438391501998u, 11995106565680728027u,         191032627458u },
+  {  4309951310554333450u, 16331071694764184179u,           2650256029u },
+  { 13106185988973773020u,  9665962217000524208u,         157885309170u },
+  {  2070134359761960547u, 13682661374415474390u,         242523992861u },
+  {  9399359914137865875u,  6940361789924260864u,          29741738559u },
+  { 17415053284723541264u,  9658039831644010465u,          63376237766u },
+  {  4840502610448261120u,  6843715893910236922u,         198523563388u },
+  {  5690599259712258048u,    47089792870595660u,         124370998582u },
+  {   114769594245185536u, 14510386192097156932u,          54002552742u },
+  {  3150620882578178048u, 12059931208360040296u,         166786609611u },
+  {  5136918324969472000u, 14877013468459184620u,         203653770180u },
+  {  7205759403792793600u,  2397668560671695044u,         196806484516u },
+  {                    0u,  2195572305559232232u,          36129977873u },
+  {                    0u,  3261686279425953792u,          17119022213u },
+  {                    0u,  9333850662059900928u,         133176816367u },
+  {                    0u,  5036522340217782272u,         239505989058u },
+  {                    0u,  2800120215143186432u,         194273030423u },
+  {                    0u,   441634238459019264u,          23151794821u },
+  {                    0u,   720575940379279360u,         133023941040u },
+  {                    0u,                    0u,         176039062500u },
+  {                    0u,                    0u,         228000000000u },
+  {    98202693831717807u,                   82u,                    0u },
+  { 12909287260170414079u,          82005323578u,                    0u },
+  { 11728631949380786233u,  8218347283861607400u,                    4u },
+  { 18263701925522197718u, 17896200385973633643u,           4445517498u },
+  {  4868886449713321591u, 16333242102094352209u,         186970154966u },
+  {  5870139507184082354u,  9981905728606788815u,         214885426828u },
+  {  5978562500822661575u, 15219470018924839012u,         140541120193u },
+  { 10821136839630268472u, 17152070168529617370u,         193825049122u },
+  {  9490907630136752916u, 17841343440958328027u,          34929815586u },
+  { 10564005678001613989u, 17291078023923990493u,          34967181165u },
+  {  5754050547468481222u, 16744804581790759223u,         109937351217u },
+  {  3833909949855542602u,  5001622214111594905u,          49907737675u },
+  { 15446538552665967784u,  9676746897435398146u,          75271138483u },
+  { 14924837848804399130u,  8109025833995118532u,         179524577500u },
+  { 18140170340418344208u,  5495826424046694744u,         220439591171u },
+  { 11380424819825208971u,  7890288164365705852u,           3297929347u },
+  { 12416915664152252547u,  8616438349039895217u,         131427733378u },
+  { 17195282241626289958u, 15787154801788760618u,         130467098058u },
+  {  2649541045825281326u, 12418659311480782502u,         202855823376u },
+  { 16577429864268509676u,  4486988874116669987u,          16673216870u },
+  { 12051257060168107241u,  4828971301551875409u,         102243240154u },
+  {  5363884561143470797u, 14769106422014442226u,         218261779058u },
+  { 13054029903083620184u,  7763933466423188156u,         114800634863u },
+  {   566314952158634945u, 10449097116253839963u,         239420883676u },
+  {  1329472079642345682u, 12870692502472900571u,         220566446689u },
+  { 15686237486658857211u, 11597479481311003817u,          97697721638u },
+  {  1069073549290598129u,  8294994869530047486u,          38628700622u },
+  {  4323599065125928507u, 16879315829924478241u,         206449672572u },
+  { 17861823329752681586u, 11873324837601439670u,         124915029544u },
+  { 17220149985412802078u,  3277599055636107318u,          40643654229u },
+  {  4344934572159429184u, 15363467897354242201u,          85177679000u },
+  {  2252927464837120000u, 10351182204479784367u,         152832855263u },
+  { 10910018171964489728u, 12811517584931924466u,         223561138711u },
+  {  3597328585515335680u, 16988930699558748726u,          23694513759u },
+  {  6972808074239148032u, 11683499918824718325u,          95920971778u },
+  {  2227030015734710272u, 13119300691281647499u,           2633363799u },
+  {  1152921504606846976u, 10125549106595354099u,          87711198715u },
+  {                    0u, 17505352699870800544u,         251548907116u },
+  {                    0u,  6756039242241163264u,         108948967071u },
+  {                    0u,  3537338758766526464u,         159366245621u },
+  {                    0u,  6522626374119718912u,         245191759518u },
+  {                    0u,  4733294203482669056u,         158353592284u },
+  {                    0u, 16997710893603094528u,         220256592392u },
+  {                    0u, 16717361816799281152u,           8921447753u },
+  {                    0u,                    0u,          73906250000u },
+  {                    0u,                    0u,          16000000000u },
+  { 17238287503805244910u,              1251301u,                    0u },
+  {  1434575446038410275u,     1251301934489438u,                    0u },
+  { 14009569747841241694u,  3943737498063000362u,                67833u },
+  { 14938795732275951328u,  2870731037991212489u,         249213790438u },
+  {  6687653542888983473u,  7389433400402095883u,         230155622641u },
+  {    28238723295162625u,  5675049236146197433u,         241400581987u },
+  {  4745270274832691214u,  9953779846262904264u,          99307645035u },
+  {   926478968112308824u, 12691978937179636241u,         107539595486u },
+  {  9213681606604198526u, 15523327331528198029u,         222688033556u },
+  { 13346223820579313024u, 15722603279568118520u,          20841521260u },
+  {   691642518601291257u, 11838632364171816147u,         108852324031u },
+  {  1153720150033789192u,  7832751832367143680u,         191641773546u },
+  { 12089015034721780810u, 12167724027162940862u,         234424614327u },
+  {  3555868702841788854u,  4108211144748152962u,         183659613641u },
+  { 11708406782758214328u,  7530983398136343676u,         201222706572u },
+  {  9417497762905343943u,  1117587133956542355u,         140408255428u },
+  { 12838336066957615287u, 17134748625149490872u,         196060584519u },
+  { 13737708142128207419u,  4039918359454207848u,          71928876584u },
+  { 11683434809834695853u,  1830218764589441242u,          40219004413u },
+  {  2689114340106315837u,   637895981480825742u,         253099216358u },
+  {  8859225263374261032u,  8246879226348334620u,         230034580410u },
+  { 15761671984578600096u, 12389239568142583275u,         186447064218u },
+  {  7087267079878005352u, 14041257178803154398u,         154671622022u },
+  {  4601291730423121377u, 16312515716494630702u,         134761178076u },
+  {  9559411037059581623u, 17088522799596987756u,         220884303248u },
+  { 12969820289641388091u,  3588932524637852678u,         144926370677u },
+  { 13628239920285957130u,   107218049069817414u,         117194556422u },
+  {  8049893933765800625u,  1596707240462008334u,           6005812302u },
+  { 10391755948840250234u, 17461913142391587672u,          78086557672u },
+  { 14454650777462444512u,  4366474266651610090u,         232946612208u },
+  { 14244638523341127254u,  5539304013194805084u,         240236707044u },
+  { 12246016810439753984u,  4762470619211987849u,         228300286272u },
+  {  9382741764551081984u, 10835638458986644717u,          64258174049u },
+  {  4608696190291148800u, 16141642290510052058u,          97587401137u },
+  {  1696483666416369664u, 17390568670756355425u,         177875040181u },
+  { 15416683541605384192u, 12536768491333867107u,         181942744616u },
+  {  7683140964294066176u, 13145148522871947193u,          40679619581u },
+  {  4611686018427387904u,  5665349945233068642u,         253712599929u },
+  {                    0u, 17074607537751066240u,         121307119235u },
+  {                    0u,  6241525660962062336u,         131925616329u },
+  {                    0u,  1142860629783085056u,         201338353784u },
+  {                    0u, 16287527416870469632u,         120061954598u },
+  {                    0u,  9028002014738513920u,          38882948630u },
+  {                    0u, 16217462258161156096u,          22489408969u },
+  {                    0u, 11529215046068469760u,         201879150390u },
+  {                    0u,                    0u,          54625000000u },
+  {                    0u,                    0u,          64000000000u },
+  {  1722045467931902045u,                   19u,                    0u },
+  { 16095324008152856733u,          19093352271u,                    0u },
+  { 15216188060094280738u,   646608198162977646u,                    1u },
+  { 10793385929903030893u, 12170458846894708007u,           1035052700u },
+  { 12613277226875940039u,  1797330480103086687u,         156659761896u },
+  { 10644539625155600107u, 10332188564497263448u,         232097433480u },
+  {  6181262895644173983u,  7524259485079594225u,         136560109064u },
+  {   148914399627082292u,    62681109059153749u,           8407890924u },
+  { 14379289774887985969u, 13480636451804037081u,         236003397949u },
+  { 11743339675582627452u,  6948168233012789004u,          61730786766u },
+  {  7841621929809463497u, 12015502974041806055u,         206376660954u },
+  { 11168944680251236601u,  7343801660689004040u,         218651361721u },
+  {  6178560202529287410u, 13670580858640731144u,         185398108285u },
+  { 13295243308201596112u,  5605073897566574851u,         125741083673u },
+  { 17040673746172470291u, 15387788940505247559u,          25303851664u },
+  { 14957442487039409922u, 17565181499678113030u,         144834173709u },
+  { 17778155426506992152u,  1893743623847493029u,          13952210397u },
+  { 14945404984219733899u, 10243498996716269784u,         221102660047u },
+  { 11590754866058681505u,  5619675836950314139u,         207555301193u },
+  {  6020790784469412466u, 10224869737511515088u,          73304643237u },
+  {  7934351824569522326u,  2574495974386198538u,         165554291299u },
+  {  5038361112172116744u,  7825756347302873178u,          99139563706u },
+  {  3305187319649924210u, 12071550103794656887u,         186424235101u },
+  { 11470175511099161552u,  7195875213867606691u,          93654400042u },
+  {  7308362160352048610u, 18271364438406891044u,          42390089176u },
+  { 10666410671225576634u, 16966521933952564706u,         216990492650u },
+  {  2181639019945820785u,   289920862029570129u,         234919756997u },
+  {  4615749499734847918u,  7804199568098625032u,         197015716641u },
+  { 14076159200958497580u,  5758118571242446585u,          33423066506u },
+  {  5470405257862074105u,  4030788293606375591u,         138312148233u },
+  {  4565741478181339543u,  4387716460037196127u,           9218509471u },
+  {  8901832997861862329u, 16807506478881285981u,         159237858585u },
+  { 14190141170191714122u, 17033060604413529717u,          25911136751u },
+  {  4240772322245764096u, 10498418508292170054u,         239923364065u },
+  {  4422842195340951552u, 13237752038744465016u,         225569120407u },
+  { 15448426386733137920u, 17737618428304633155u,         151717619975u },
+  {  9203504548935630848u, 13546183833248825736u,           7961558221u },
+  {  5936377627571912704u,   826778452978976229u,         205734340097u },
+  {   468374361246531584u, 13728076626990147292u,           1044819749u },
+  {                    0u,  2794860281883592225u,          37744200525u },
+  {                    0u,  8680705720425908736u,          77151509679u },
+  {                    0u,   731520517439488000u,         175470582000u },
+  {                    0u, 13120812320768917504u,         240039655806u },
+  {                    0u,  2722954908557901824u,         126711280661u },
+  {                    0u,  6860847004205973504u,          21147611681u },
+  {                    0u,  6503197861922996224u,          33371927261u },
+  {                    0u,  9223372036854775808u,         221352539062u },
+  {                    0u,                    0u,         182500000000u },
+  {  8020875056524075716u,               291341u,                    0u },
+  {  1400288714762747253u,      291341434812508u,                    0u },
+  {  6136037711314764689u, 12005656413127238315u,                15793u },
+  { 15735488086392394102u,  4821130826186787462u,         177650827938u },
+  {  7735094782793634552u, 14377899467066168118u,         162261354025u },
+  {  7273689191766726188u, 16575613239625444872u,          41779427491u },
+  {  3410634565056431030u,  4317827099179284377u,         163898565794u },
+  {  4600927904885215898u,  1242354770412171254u,         162234069876u },
+  {  9154871331680374746u,   994838588328896609u,         116067348187u },
+  {  6117978272461042996u, 17283309862013060457u,         219053930307u },
+  { 13116904339287496285u,   124242522249856586u,          67936930105u },
+  {   867037205615660831u, 11564608014666985609u,          57006735200u },
+  {  1802487145191504830u, 12401028575581654085u,          96626918656u },
+  { 17242009718457409007u,  2490725392961465727u,            672261106u },
+  { 13620802355488468049u,  1949482237120640915u,         242135022494u },
+  {  4442227880594435745u, 15410502396166200087u,         158105681643u },
+  { 17354849212854314181u, 15694919529799920727u,         235835405008u },
+  { 18202319179831567886u, 10324869370171768388u,         208850823292u },
+  { 17568634016348874558u,  1631866459122189059u,         124559712290u },
+  {  7496925598312450672u,   172020494461226230u,          34088463658u },
+  {   601870379496813865u, 12734610307908856767u,          42009325249u },
+  { 11995106565680728027u,  1467513250829340930u,         193690344608u },
+  { 16331071694764184179u, 13558759428494307997u,         160079554052u },
+  {  9665962217000524208u,  7915355143999496434u,           4735021821u },
+  { 13682661374415474390u,  2876370200608797469u,         253429092262u },
+  {  6940361789924260864u,   343685370404989503u,         166155928341u },
+  {  9658039831644010465u,  4837266557407634630u,          21018631221u },
+  {  6843715893910236922u,  9622591415747161468u,          53262228745u },
+  {    47089792870595660u, 16503783814424220982u,           9521641725u },
+  { 14510386192097156932u,  5377083431343591334u,         253894671913u },
+  { 12059931208360040296u, 16508482371299291595u,          41291492276u },
+  { 14877013468459184620u, 10515883558812249028u,         180894926622u },
+  {  2397668560671695044u,    63492062913405476u,          30570067190u },
+  {  2195572305559232232u, 11571919759617799697u,         246003441911u },
+  {  3261686279425953792u,  2956602334970088581u,         247627315027u },
+  {  9333850662059900928u, 13604736747717849839u,          83160277733u },
+  {  5036522340217782272u, 16573540719338151362u,         229737514256u },
+  {  2800120215143186432u, 12620703004601168151u,          16898453442u },
+  {   441634238459019264u, 14649407809089591941u,         194684169680u },
+  {   720575940379279360u, 11290375247898624432u,         208794145988u },
+  {                    0u, 11020319450292874212u,         196612052468u },
+  {                    0u,  8754634933362354176u,         244597412714u },
+  {                    0u, 12976319450332528640u,         106474589710u },
+  {                    0u, 17447331119627239424u,          14703447686u },
+  {                    0u,  3665184902673858560u,         134945821715u },
+  {                    0u, 12949678516038795264u,          19198690071u },
+  {                    0u,    72057594037927936u,          23702003479u },
+  {                    0u,                    0u,          23003906250u },
+  {                    0u,                    0u,         202000000000u },
+  {  8218347283861607400u,                    4u,                    0u },
+  { 17896200385973633643u,           4445517498u,                    0u },
+  { 16333242102094352209u,  4445517498970154966u,                    0u },
+  {  9981905728606788815u,  9413159735776077452u,            240991986u },
+  { 15219470018924839012u, 14279163482889998017u,         242510288411u },
+  { 17152070168529617370u,  8693044629541194274u,          27774075003u },
+  { 17841343440958328027u, 11863110253260222498u,         123471250893u },
+  { 17291078023923990493u,  8319293368489531245u,         205643100495u },
+  { 16744804581790759223u,  3376307525676489265u,          79450989797u },
+  {  5001622214111594905u, 13205662254759912523u,         229183029997u },
+  {  9676746897435398146u,  5276250334231686323u,         237715880385u },
+  {  8109025833995118532u, 13790198520922745052u,         193286026103u },
+  {  5495826424046694744u, 14195535250150996227u,         119747568159u },
+  {  7890288164365705852u, 16425228796427004035u,          31769541507u },
+  {  8616438349039895217u,  4295900841296269186u,         131890413437u },
+  { 15787154801788760618u,  4533952595483946442u,         125232881251u },
+  { 12418659311480782502u, 12885038019373447184u,          99245786062u },
+  {  4486988874116669987u, 12140736240487831910u,         206698499310u },
+  {  4828971301551875409u,  6927124077155322074u,         238658150630u },
+  { 14769106422014442226u, 12477788342407819890u,         230375520148u },
+  {  7763933466423188156u,  7980854329409711087u,         148676422261u },
+  { 10449097116253839963u,  2062671021810827996u,         117432642980u },
+  { 12870692502472900571u,  2739521363598172769u,         164111817620u },
+  { 11597479481311003817u, 12897585686593465638u,         148148509750u },
+  {  8294994869530047486u,  1127632646629044686u,          54699179521u },
+  { 16879315829924478241u,  4833775019274666364u,           1061129088u },
+  { 11873324837601439670u, 15867662672939849256u,         128262039468u },
+  {  3277599055636107318u,  2092350330982953557u,         172860187717u },
+  { 15363467897354242201u, 13330062299842493592u,          69113426538u },
+  { 10351182204479784367u,  4479193352178519263u,         106722624125u },
+  { 12811517584931924466u,  3149393938889064983u,         125242817558u },
+  { 16988930699558748726u,  9736379904070620767u,          22170728987u },
+  { 11683499918824718325u,  3816238703055069186u,          27527810212u },
+  { 13119300691281647499u, 11598915938798661975u,         164206878714u },
+  { 10125549106595354099u, 17821633264606555643u,         250628778492u },
+  { 17505352699870800544u,  2514623558764574316u,         252966112675u },
+  {  6756039242241163264u,  4976730480406253215u,         163136318016u },
+  {  3537338758766526464u, 17276563697191611637u,          64269789099u },
+  {  6522626374119718912u, 12524734095940998814u,         171936564394u },
+  {  4733294203482669056u, 15331551308930355164u,         170678967195u },
+  { 16997710893603094528u, 15417115581125943816u,         155831125061u },
+  { 16717361816799281152u,  6010750237807115593u,          69835763510u },
+  {                    0u,  5624630987553628432u,          54325843423u },
+  {                    0u, 14881848243837640704u,         223304911856u },
+  {                    0u, 15281613886881529856u,         240806746609u },
+  {                    0u, 14057902358273196032u,         241828417948u },
+  {                    0u, 16075318494433902592u,         156762080413u },
+  {                    0u, 13891916000577716224u,         157871444761u },
+  {                    0u,  7205759403792793600u,          25753082275u },
+  {                    0u,                    0u,         163390625000u },
+  {                    0u,                    0u,         232000000000u },
+  {  3943737498063000362u,                67833u,                    0u },
+  {  2870731037991212489u,       67833213790438u,                    0u },
+  {  7389433400402095883u,  4535831408134330609u,                 3677u },
+  {  5675049236146197433u,  6204770794376564579u,          93245887913u },
+  {  9953779846262904264u, 13869812122751887467u,         169336361298u },
+  { 12691978937179636241u, 14253229412394467550u,          82751884021u },
+  { 15523327331528198029u, 12776557610216045332u,         245772669114u },
+  { 15722603279568118520u, 16493640728678654060u,         186692618575u },
+  { 11838632364171816147u,  9434398296825833151u,          79894122055u },
+  {  7832751832367143680u,  8773374058285327850u,          71511439756u },
+  { 12167724027162940862u, 12932015276748029367u,         140475605560u },
+  {  4108211144748152962u, 16293958583527755209u,          56701045952u },
+  {  7530983398136343676u, 13511893936143127948u,         192883297264u },
+  {  1117587133956542355u, 18409936402005226436u,         240732481237u },
+  { 17134748625149490872u,  2189663026458466887u,         213998004652u },
+  {  4039918359454207848u,  9497725274248154664u,         172118701870u },
+  {  1830218764589441242u, 14766925481127792125u,          46514872718u },
+  {   637895981480825742u,  6982373971809635814u,         142800516634u },
+  {  8246879226348334620u,  8616702383006884794u,          26378515251u },
+  { 12389239568142583275u,  3059473300040871066u,          51467112372u },
+  { 14041257178803154398u, 17123843157031495558u,         180165854379u },
+  { 16312515716494630702u, 11210627174210626524u,         171928285397u },
+  { 17088522799596987756u, 15868067138625928592u,         213607729316u },
+  {  3588932524637852678u,  4467869511636937589u,         164860209643u },
+  {   107218049069817414u, 10052108125844341766u,         235242203691u },
+  {  1596707240462008334u,  7470588003218451534u,          43544925873u },
+  { 17461913142391587672u,  2613527085490786280u,         177404981387u },
+  {  4366474266651610090u,  3632919450036549616u,         139141679587u },
+  {  5539304013194805084u,   179367907231218916u,         227196940958u },
+  {  4762470619211987849u, 13553068184555874624u,         158009723553u },
+  { 10835638458986644717u,  8798774862365584481u,         161734713298u },
+  { 16141642290510052058u,   910911255817064881u,         210476982541u },
+  { 17390568670756355425u,  2304331144765093813u,          13049380598u },
+  { 12536768491333867107u, 12248937023083640360u,         246124918041u },
+  { 13145148522871947193u, 10206039550662130685u,          25664016206u },
+  {  5665349945233068642u, 12267881323837852537u,          78553270512u },
+  { 17074607537751066240u,  2858642007937891971u,         240665043179u },
+  {  6241525660962062336u, 14171330289750320841u,         235154967293u },
+  {  1142860629783085056u,  6601103619749017720u,         253768229354u },
+  { 16287527416870469632u,  4919573414486739494u,         234357846544u },
+  {  9028002014738513920u,  3401998285294974486u,          16266690609u },
+  { 16217462258161156096u, 10799436256515532233u,          49184422696u },
+  { 11529215046068469760u, 10083786644665753398u,          40585438612u },
+  {                    0u,  6481194517685688896u,         148546643169u },
+  {                    0u, 15104161756860547072u,         225351346258u },
+  {                    0u,  9556039274244079616u,          82818798249u },
+  {                    0u,  1376343134954323968u,         169518033927u },
+  {                    0u, 15682488278596976640u,           7074611710u },
+  {                    0u,  1506454075355430912u,         254850149393u },
+  {                    0u,  1152921504606846976u,          17081665039u },
+  {                    0u,                    0u,          15062500000u },
+  {                    0u,                    0u,         160000000000u },
+  { 12170458846894708007u,           1035052700u,                    0u },
+  {  1797330480103086687u,  1035052700659761896u,                    0u },
+  { 10332188564497263448u,  6172559441576707976u,             56110319u },
+  {  7524259485079594225u, 15083329738554729992u,         239334615117u },
+  {    62681109059153749u, 10013126833549229036u,          77817668943u },
+  { 13480636451804037081u,  5817156823499936061u,          79542812693u },
+  {  6948168233012789004u,  5282692560913632718u,          21315348703u },
+  { 12015502974041806055u, 10252307034225766362u,         223286375337u },
+  {  7343801660689004040u, 17981881283247669689u,         169555778677u },
+  { 13670580858640731144u, 11689290159733383293u,         117974799737u },
+  {  5605073897566574851u,  5530668968487988249u,         121633677689u },
+  { 15387788940505247559u, 10083765740821947024u,         121299818165u },
+  { 17565181499678113030u,  2798423656816843533u,         181546642036u },
+  {  1893743623847493029u,  7614494481582904797u,         116151702850u },
+  { 10243498996716269784u, 17811318500083423695u,          66412782572u },
+  {  5619675836950314139u, 11641467412200329033u,         236965553510u },
+  { 10224869737511515088u, 17733593025296340645u,         102631085212u },
+  {  2574495974386198538u,  3689424000190644835u,         156961340004u },
+  {  7825756347302873178u, 14966634145516728506u,         100200004075u },
+  { 12071550103794656887u, 14171681941562070109u,         235811342862u },
+  {  7195875213867606691u,  8130575762882608170u,          14768248417u },
+  { 18271364438406891044u,  5234550794400656856u,          97440759395u },
+  { 16966521933952564706u,  3020576149360486378u,          99283765567u },
+  {   289920862029570129u,  3038675756589057221u,          63163745761u },
+  {  7804199568098625032u, 15470260187120878369u,         225164726942u },
+  {  5758118571242446585u,  3497929414841828746u,         158838644485u },
+  {  4030788293606375591u,  9935840636861015305u,           5189623133u },
+  {  4387716460037196127u,  3647355485153741471u,          93538623000u },
+  { 16807506478881285981u,   766100215038272793u,          24197723537u },
+  { 17033060604413529717u, 16128087474216800751u,         145041530375u },
+  { 10498418508292170054u, 16216631732633731297u,           7874305373u },
+  { 13237752038744465016u, 13760220872779997335u,          93879105367u },
+  { 17737618428304633155u,  3826276262374222087u,          87745943068u },
+  { 13546183833248825736u, 14938032745839181005u,          28207422851u },
+  {   826778452978976229u, 14479259995009508865u,         131809792377u },
+  { 13728076626990147292u,  2372033248156102437u,         121784922257u },
+  {  2794860281883592225u,   792005346826701645u,         145128588180u },
+  {  8680705720425908736u, 16278924527931792559u,         148042934695u },
+  {   731520517439488000u, 17442516423538940144u,         167882482266u },
+  { 13120812320768917504u,    13844184233048446u,          90945560710u },
+  {  2722954908557901824u, 13486193870480782357u,         134000750494u },
+  {  6860847004205973504u, 11931315179184648737u,         158731088034u },
+  {  6503197861922996224u, 16492562205587485405u,         162646797891u },
+  {  9223372036854775808u, 12128987217680380854u,          67894063588u },
+  {                    0u, 10568123814189138176u,         228657513714u },
+  {                    0u, 17007583519117541376u,         242572899139u },
+  {                    0u,   143791533903052800u,          67921982950u },
+  {                    0u, 12398714235792654336u,         230007794954u },
+  {                    0u,  9659957317919047680u,          10672135645u },
+  {                    0u,  9412523221204336640u,         221523667335u },
+  {                    0u,  4611686018427387904u,         135510253906u },
+  {                    0u,                    0u,          82250000000u },
+  {                    0u,                    0u,         128000000000u },
+  { 12005656413127238315u,                15793u,                    0u },
+  {  4821130826186787462u,       15793650827938u,                    0u },
+  { 14377899467066168118u,  3237900842885170729u,                  856u },
+  { 16575613239625444872u,  7515893506498066595u,          88175526956u },
+  {  4317827099179284377u,  7300206309181072546u,          44407437403u },
+  {  1242354770412171254u,     5999737279837044u,          91395744977u },
+  {   994838588328896609u,  7556839307242450651u,         209000325246u },
+  { 17283309862013060457u, 12946035041643640643u,         126409657079u },
+  {   124242522249856586u, 15885877642352740665u,         247701805965u },
+  { 11564608014666985609u, 10770818348246089568u,         141861175152u },
+  { 12401028575581654085u, 11635415503599551744u,         112583887232u },
+  {  2490725392961465727u,  6248053924100826098u,         128630757138u },
+  {  1949482237120640915u, 16894170802729859998u,          18338707681u },
+  { 15410502396166200087u,  6143589029651889899u,         225915834834u },
+  { 15694919529799920727u, 11812087701837886160u,         210333044628u },
+  { 10324869370171768388u,  7306705080150829180u,         148640334557u },
+  {  1631866459122189059u,  1485332570280714274u,         221396097276u },
+  {   172020494461226230u, 18042602303295630634u,         252080520039u },
+  { 12734610307908856767u, 13397029889257074369u,         103978091430u },
+  {  1467513250829340930u,  9948104869613411488u,         166726254445u },
+  { 13558759428494307997u, 10836066241170646532u,         109539287845u },
+  {  7915355143999496434u, 18330574781234459389u,          37587424327u },
+  {  2876370200608797469u,   666297360208433062u,          71993702450u },
+  {   343685370404989503u,  5035352224889324309u,          50036120052u },
+  {  4837266557407634630u,  1341745796439923765u,         244272966991u },
+  {  9622591415747161468u,  6846932182653803785u,          79072736185u },
+  { 16503783814424220982u,  6727685027257825533u,         185371172937u },
+  {  5377083431343591334u,  2168538874806877737u,          73364708536u },
+  { 16508482371299291595u, 17694936100676971444u,         184117556727u },
+  { 10515883558812249028u,  2163944241059563294u,         247959244408u },
+  {    63492062913405476u,  6727780864524301558u,         120117307652u },
+  { 11571919759617799697u,  8599551977795002615u,           4364713731u },
+  {  2956602334970088581u, 15428264807806859091u,           3466182646u },
+  { 13604736747717849839u,  2126771385339683557u,         246836367911u },
+  { 16573540719338151362u, 15094316562082972944u,          39115292507u },
+  { 12620703004601168151u,  8111300598225956802u,          91818264540u },
+  { 14649407809089591941u,  9481215200564260304u,         220439714486u },
+  { 11290375247898624432u, 16836674128623424708u,         182513977705u },
+  { 11020319450292874212u,  7087243115299722740u,         105912717933u },
+  {  8754634933362354176u,  2343560867338408810u,         109384200219u },
+  { 12976319450332528640u,  3431385749090422286u,          27127044689u },
+  { 17447331119627239424u,  3504545517469224582u,          81186015794u },
+  {  3665184902673858560u,  3333759805712094227u,          50189981793u },
+  { 12949678516038795264u,  3595183476205994775u,          97180723481u },
+  {    72057594037927936u, 14191566632569921303u,          25194895286u },
+  {                    0u, 12917427671358095562u,         182769326368u },
+  {                    0u,  3883793922738316288u,          32700255157u },
+  {                    0u,  7857281689266421760u,         181210540890u },
+  {                    0u, 15987081651486195712u,          90425944093u },
+  {                    0u, 16827562156399525888u,          29866661432u },
+  {                    0u,  7012737938513461248u,          56912223972u },
+  {                    0u,  7385903388887613440u,         228380161285u },
+  {                    0u,                    0u,           5400390625u },
+  {                    0u,                    0u,         225000000000u },
+  {  9413159735776077452u,            240991986u,                    0u },
+  { 14279163482889998017u,   240991986510288411u,                    0u },
+  {  8693044629541194274u, 14135788013842776187u,             13064201u },
+  { 11863110253260222498u, 13284322918167594445u,           9766302603u },
+  {  8319293368489531245u,  7264587765474046287u,         139720144588u },
+  {  3376307525676489265u, 16176482219778368741u,         204393814091u },
+  { 13205662254759912523u,  5401983818872095469u,          75876928858u },
+  {  5276250334231686323u, 11208857446851049921u,          90292842129u },
+  { 13790198520922745052u, 13794690008281035639u,         145607633379u },
+  { 14195535250150996227u, 14519782740993303071u,         227747811643u },
+  { 16425228796427004035u, 10885858587044789123u,          59787118999u },
+  {  4295900841296269186u,  8710500938899914621u,         151590123576u },
+  {  4533952595483946442u,  1284182587483102819u,          56472197202u },
+  { 12885038019373447184u, 10346074482131502030u,          82069615677u },
+  { 12140736240487831910u,  9429804686255246574u,          61560861821u },
+  {  6927124077155322074u,  6412022633845121254u,         125511190736u },
+  { 12477788342407819890u,  8892351297529018260u,         208347596443u },
+  {  7980854329409711087u, 14098160105983060597u,         155482055329u },
+  {  2062671021810827996u, 13793833029739474340u,         161764262790u },
+  {  2739521363598172769u, 16367653765996977044u,         134747765186u },
+  { 12897585686593465638u, 10684788343333772342u,         194887292288u },
+  {  1127632646629044686u, 13272681218705145345u,         128579223536u },
+  {  4833775019274666364u, 11093568615497829248u,         240719513490u },
+  { 15867662672939849256u, 12488220765137758124u,         146601383559u },
+  {  2092350330982953557u,  3727114642519696453u,         135676987804u },
+  { 13330062299842493592u, 11549865375695057514u,         156202047289u },
+  {  4479193352178519263u, 11292809154908783229u,          57626119456u },
+  {  3149393938889064983u, 17723904861837310998u,          32612184410u },
+  {  9736379904070620767u, 14877674388187150875u,          90960814807u },
+  {  3816238703055069186u, 12178961950105734308u,         215806520344u },
+  { 11598915938798661975u,  4540604068069253114u,          24660222850u },
+  { 17821633264606555643u, 13832478722153359868u,         130246146639u },
+  {  2514623558764574316u,  1308046668730371491u,          79749860174u },
+  {  4976730480406253215u, 18400531023544756800u,          78070909351u },
+  { 17276563697191611637u,  9789823458621466539u,         167997494785u },
+  { 12524734095940998814u,  1924870562610267306u,           1530707393u },
+  { 15331551308930355164u,  5290016144582400923u,         193104347442u },
+  { 15417115581125943816u, 15162883663174059077u,          50286772349u },
+  {  6010750237807115593u,  8078086116520046390u,         125821981570u },
+  {  5624630987553628432u, 15731407332173190623u,         130437913925u },
+  { 14881848243837640704u,  5346389182763011056u,          69852801300u },
+  { 15281613886881529856u,  6368422217216252401u,          20289828338u },
+  { 14057902358273196032u,  2961453088119116188u,         242345232860u },
+  { 16075318494433902592u, 10932141691610170525u,         220160540693u },
+  { 13891916000577716224u, 11034016191361782553u,          21592632588u },
+  {  7205759403792793600u,  5455325785621453219u,          12598155216u },
+  {                    0u,  7735615202566149352u,         208295733803u },
+  {                    0u,  7502396497775759360u,          43419348540u },
+  {                    0u,  1601286435751591936u,          60406705729u },
+  {                    0u, 11449383158571597824u,          65086805911u },
+  {                    0u, 13043944595690356736u,         151620672304u },
+  {                    0u,  7773494431818186752u,          48707113653u },
+  {                    0u,  9943947977234055168u,         181421401977u },
+  {                    0u,                    0u,         121539062500u },
+  {                    0u,                    0u,         228000000000u },
+  {  4535831408134330609u,                 3677u,                    0u },
+  {  6204770794376564579u,        3677245887913u,                    0u },
+  { 13869812122751887467u,  6343817245135589714u,                  199u },
+  { 14253229412394467550u, 17549323075660516085u,         199343899021u },
+  { 12776557610216045332u,  3948641822109421754u,         141951350710u },
+  { 16493640728678654060u,  1750739713693534543u,         182214056302u },
+  {  9434398296825833151u,   962163898128633415u,         110094907790u },
+  {  8773374058285327850u,  7967320249386531212u,         142052159009u },
+  { 12932015276748029367u,  3018466665533383224u,          33431909296u },
+  { 16293958583527755209u, 15076865731854945472u,         176163631405u },
+  { 13511893936143127948u,   691187172844604400u,          45817318529u },
+  { 18409936402005226436u, 13274492813370992341u,         129037469331u },
+  {  2189663026458466887u,  6364168818499152300u,         147719611697u },
+  {  9497725274248154664u, 17599380787401914158u,          49345002282u },
+  { 14766925481127792125u,  3782323149461692814u,          42954064344u },
+  {  6982373971809635814u, 14470163442442237466u,         216205040148u },
+  {  8616702383006884794u,   476109872130437939u,          20784429132u },
+  {  3059473300040871066u, 16330548844673355700u,          76025809967u },
+  { 17123843157031495558u, 14089158961463739563u,          47885280826u },
+  { 11210627174210626524u, 13385510793074798805u,          58763774837u },
+  { 15868067138625928592u,  1549401308746959012u,         117725629994u },
+  {  4467869511636937589u,  4607384943843027435u,          42083993213u },
+  { 10052108125844341766u,  5157353797716093483u,         125249766838u },
+  {  7470588003218451534u, 10846828782671550129u,         182279580709u },
+  {  2613527085490786280u,  9915857350819131531u,          37588007766u },
+  {  3632919450036549616u,  1673544973504317923u,          86537539704u },
+  {   179367907231218916u, 14780986291622785694u,         120090723054u },
+  { 13553068184555874624u,  8168111319515466401u,         238801278872u },
+  {  8798774862365584481u, 16345760387859734482u,         152442794201u },
+  {   910911255817064881u,  3177475373321281805u,         217886105446u },
+  {  2304331144765093813u,  2558676822419554038u,         102172251285u },
+  { 12248937023083640360u,  8813474062662382873u,         149138706148u },
+  { 10206039550662130685u,  5426294560236228430u,         228477779386u },
+  { 12267881323837852537u,  9919177474128333040u,         186294160017u },
+  {  2858642007937891971u,  6197383943089627371u,         145537719688u },
+  { 14171330289750320841u, 13673239314867423997u,         136335960856u },
+  {  6601103619749017720u,  9309584098968723946u,          24741227788u },
+  {  4919573414486739494u,  4647101757759615504u,          12504673565u },
+  {  3401998285294974486u,  1405809295505096753u,          29251919891u },
+  { 10799436256515532233u, 11332704079573859112u,          19076209074u },
+  { 10083786644665753398u,  2960072434514044308u,         178614347119u },
+  {  6481194517685688896u,  3887266602785432801u,         111160465848u },
+  { 15104161756860547072u, 14545546084687849554u,         184210729144u },
+  {  9556039274244079616u,  4617763804182385321u,         184788515633u },
+  {  1376343134954323968u,  7857823815580249095u,          49250329477u },
+  { 15682488278596976640u, 10939326736548364798u,         133425973482u },
+  {  1506454075355430912u, 12262012446566951953u,         234593022090u },
+  {  1152921504606846976u, 12555024338687723023u,         138664725026u },
+  {                    0u,  3332969632922829472u,          34680609233u },
+  {                    0u, 15535060143360327680u,         209180680645u },
+  {                    0u, 15794322927987458048u,         197842157297u },
+  {                    0u, 10571474314433921024u,         241856211961u },
+  {                    0u, 16679514427547975680u,         249573080770u },
+  {                    0u, 16925653299565166592u,         194904198288u },
+  {                    0u, 16717361816799281152u,         144917541503u },
+  {                    0u,                    0u,         127906250000u },
+  {                    0u,                    0u,          16000000000u },
+  {  6172559441576707976u,             56110319u,                    0u },
+  { 15083329738554729992u,    56110319334615117u,                    0u },
+  { 10013126833549229036u,  9335385384027907407u,              3041746u },
+  {  5817156823499936061u, 13237828406194798613u,         210506072255u },
+  {  5282692560913632718u, 15667486867836528863u,         191717624115u },
+  { 10252307034225766362u, 17982325043592934313u,          51849336164u },
+  { 17981881283247669689u, 17159117626917379189u,         100974823793u },
+  { 11689290159733383293u,  8336208968408929657u,         113930197630u },
+  {  5530668968487988249u, 12767090573379150201u,         126451906793u },
+  { 10083765740821947024u, 14736070002412246709u,         233692105366u },
+  {  2798423656816843533u,  9697296975344560756u,         150798843955u },
+  {  7614494481582904797u,  7291706381199103298u,          51525691522u },
+  { 17811318500083423695u, 18098546597780825068u,         130395284194u },
+  { 11641467412200329033u,   132913902678533478u,         226981124177u },
+  { 17733593025296340645u,  1879347741692007580u,          81007205277u },
+  {  3689424000190644835u,  4056624629214083684u,         157101879645u },
+  { 14966634145516728506u, 14713227692042795499u,          93219910061u },
+  { 14171681941562070109u,  7366415124022528526u,         173797605671u },
+  {  8130575762882608170u,   825770353378039393u,          39399334164u },
+  {  5234550794400656856u, 10244023944395357795u,          20044765100u },
+  {  3020576149360486378u, 14302658294713551167u,         172555329650u },
+  {  3038675756589057221u, 14246653166206862817u,         114775348659u },
+  { 15470260187120878369u, 12404486258134291102u,         179772312615u },
+  {  3497929414841828746u,  8887442218637942533u,          39672448547u },
+  {  9935840636861015305u,  1186724038081863005u,          35481789208u },
+  {  3647355485153741471u,   211331772484951576u,          24064332439u },
+  {   766100215038272793u,  6311919513247413649u,         151011456318u },
+  { 16128087474216800751u,  8131780018703965703u,          62342169842u },
+  { 16216631732633731297u,  2262544347226725725u,         242440824678u },
+  { 13760220872779997335u, 15318188749880522583u,         102122652774u },
+  {  3826276262374222087u,  1073117094162650652u,         102830400676u },
+  { 14938032745839181005u,  4447950380665871747u,         164058173794u },
+  { 14479259995009508865u,  5373227185066463609u,          98241123873u },
+  {  2372033248156102437u,  6739731406934274193u,          33291283229u },
+  {   792005346826701645u, 12328812617001239444u,          29365361571u },
+  { 16278924527931792559u,  3246111484407310759u,         163668346271u },
+  { 17442516423538940144u,  3250825415176839770u,         159175972056u },
+  {    13844184233048446u, 16146270540000862342u,         216176227598u },
+  { 13486193870480782357u, 15686773375425916830u,          14875291079u },
+  { 11931315179184648737u, 11920791905793880226u,         199850381688u },
+  { 16492562205587485405u,  1853290561644080707u,         120646227424u },
+  { 12128987217680380854u, 12157689141506159076u,         224100467082u },
+  { 10568123814189138176u, 18100318838862562546u,         138659069648u },
+  { 17007583519117541376u,  7171257882533475139u,         208981220250u },
+  {   143791533903052800u, 14477550873015039462u,         154388754668u },
+  { 12398714235792654336u,  8109481182495403274u,         236784829605u },
+  {  9659957317919047680u, 14565395719337663965u,         165439615855u },
+  {  9412523221204336640u,  1860318978161305991u,         111789591684u },
+  {  4611686018427387904u, 16268646275151585618u,         132100848093u },
+  {                    0u, 13759019338835519104u,         221881925081u },
+  {                    0u, 17003783176010661888u,         217745877932u },
+  {                    0u, 18357489540307877888u,         172921776932u },
+  {                    0u,   905481790074912768u,          36995161502u },
+  {                    0u,  3638882110636294144u,         158049086266u },
+  {                    0u,  9011702854368362496u,          58197264194u },
+  {                    0u, 11529215046068469760u,          66488525390u },
+  {                    0u,                    0u,          78625000000u },
+  {                    0u,                    0u,          64000000000u },
+  {  3237900842885170729u,                  856u,                    0u },
+  {  7515893506498066595u,         856175526956u,                    0u },
+  {  7300206309181072546u,  7625299565768063067u,                   46u },
+  {     5999737279837044u, 13889021769065194705u,          46413368317u },
+  {  7556839307242450651u, 14498170692313014398u,         253752925378u },
+  { 12946035041643640643u,  1541631360972245751u,         194785947408u },
+  { 15885877642352740665u,  9903958882920799117u,          16083572003u },
+  { 10770818348246089568u, 15744148547788062576u,          35536894686u },
+  { 11635415503599551744u, 17936061801321712000u,         222853492002u },
+  {  6248053924100826098u,  9986394078324430610u,          34972315858u },
+  { 16894170802729859998u, 13849561248103430369u,         210541363507u },
+  {  6143589029651889899u, 12142378807953854930u,          51750786219u },
+  { 11812087701837886160u,  2513847703931031444u,         171658239674u },
+  {  7306705080150829180u,  1752183758129038045u,         186136275957u },
+  {  1485332570280714274u, 15824833342220556540u,         245094986071u },
+  { 18042602303295630634u,  8168747198299470695u,          87857865934u },
+  { 13397029889257074369u, 17414799840149357478u,         206442828672u },
+  {  9948104869613411488u,    83147520704167789u,         128944058191u },
+  { 10836066241170646532u,  2383542703041471269u,          79004507436u },
+  { 18330574781234459389u, 15540952725549257799u,          44129212108u },
+  {   666297360208433062u,  6949835416232048690u,         204842476735u },
+  {  5035352224889324309u, 15398868937585367540u,         191376751332u },
+  {  1341745796439923765u, 14710915985268256079u,         228834774357u },
+  {  6846932182653803785u,  9665704836873335737u,          85797480353u },
+  {  6727685027257825533u,  2528789298740305993u,         161523978909u },
+  {  2168538874806877737u, 10562914675687726264u,         157137085942u },
+  { 17694936100676971444u, 17671658300096837111u,         246572616751u },
+  {  2163944241059563294u,   356471401631698552u,          47957982516u },
+  {  6727780864524301558u,  7450677157218003204u,          52019324353u },
+  {  8599551977795002615u,   317174560787152643u,         193403902018u },
+  { 15428264807806859091u,  7251937674440720374u,          66017194067u },
+  {  2126771385339683557u,  1252631516699038247u,          83393128329u },
+  { 15094316562082972944u, 10818009768860843867u,         137067905290u },
+  {  8111300598225956802u, 12330114194950162396u,          10586445484u },
+  {  9481215200564260304u, 15826681638261168822u,         172668416829u },
+  { 16836674128623424708u, 14240150078499211625u,          61857966130u },
+  {  7087243115299722740u, 10725372116242125421u,          50771960082u },
+  {  2343560867338408810u,  8434925524647833627u,          18581423587u },
+  {  3431385749090422286u, 17133902668520348241u,         227457258228u },
+  {  3504545517469224582u, 15093996047981365810u,         244928830724u },
+  {  3333759805712094227u,  6187974166976813153u,           4818247165u },
+  {  3595183476205994775u, 13946144707720259865u,         253335450751u },
+  { 14191566632569921303u,  9138079832881862582u,         127756022019u },
+  { 12917427671358095562u,  6600697628576225568u,           3495376300u },
+  {  3883793922738316288u,  8137099536646556597u,         172357824535u },
+  {  7857281689266421760u, 14169855543453903706u,          23441113049u },
+  { 15987081651486195712u,  3706403268650100765u,         217768149408u },
+  { 16827562156399525888u, 14736932266877982264u,         160200924523u },
+  {  7012737938513461248u, 18004795125138956004u,         107798890698u },
+  {  7385903388887613440u,  9068489270661002501u,         202976041899u },
+  {                    0u,  7758835715193269217u,         171491603788u },
+  {                    0u, 16943947811135261184u,          76420607326u },
+  {                    0u,  6745843108403216384u,          94918533251u },
+  {                    0u, 12338229654069444608u,         131365692887u },
+  {                    0u, 14358176069683511296u,         215668856769u },
+  {                    0u,  7083775185760813056u,         193778358284u },
+  {                    0u,  5350276357316149248u,          12384012222u },
+  {                    0u,  9223372036854775808u,         190290039062u },
+  {                    0u,                    0u,          22500000000u },
+  { 14135788013842776187u,             13064201u,                    0u },
+  { 13284322918167594445u,    13064201766302603u,                    0u },
+  {  7264587765474046287u, 14699116688460625612u,               708211u },
+  { 16176482219778368741u,  6684126021499623499u,         115796840712u },
+  {  5401983818872095469u, 12614606079692508506u,           8362347197u },
+  { 11208857446851049921u, 15358270276683001489u,         189683839165u },
+  { 13794690008281035639u, 18077126190953408995u,         189832573499u },
+  { 14519782740993303071u,  7864121581925945659u,          59979962974u },
+  { 10885858587044789123u,  3518026639210514839u,          94426314885u },
+  {  8710500938899914621u,  4698310163811252280u,         133190712606u },
+  {  1284182587483102819u,  6101155398200416338u,          30254695904u },
+  { 10346074482131502030u, 16049178580360033341u,         224330744296u },
+  {  9429804686255246574u,  3167464649127375997u,         232870027714u },
+  {  6412022633845121254u, 12778923935480989904u,         194171708602u },
+  {  8892351297529018260u, 11875553912612980379u,         186692746854u },
+  { 14098160105983060597u, 10628760849351697057u,         102643775067u },
+  { 13793833029739474340u,  3408944711673234310u,          91576186280u },
+  { 16367653765996977044u,  2102091496050506178u,         168184799263u },
+  { 10684788343333772342u,  6254611118630245760u,          31113954608u },
+  { 13272681218705145345u,  2647941151989776368u,          48339063148u },
+  { 11093568615497829248u,  8855437735410157458u,         108143545177u },
+  { 12488220765137758124u, 10184270603132180103u,          89480054241u },
+  {  3727114642519696453u, 12079083162535627164u,         225552090415u },
+  { 11549865375695057514u,  5952952868716156729u,          47654808410u },
+  { 11292809154908783229u, 11958907037815852320u,          90322710221u },
+  { 17723904861837310998u, 10101562137321697626u,         205648293649u },
+  { 14877674388187150875u, 13633527411279258327u,          17547606780u },
+  { 12178961950105734308u, 16555627393501768728u,         252739075001u },
+  {  4540604068069253114u,  6359650463500280706u,         185897482359u },
+  { 13832478722153359868u,  8093923611102181967u,         119344757342u },
+  {  1308046668730371491u,  2848827352928635726u,          94438772478u },
+  { 18400531023544756800u,  4686723431961561511u,         254154435240u },
+  {  9789823458621466539u,  6245554925867652609u,         168254067786u },
+  {  1924870562610267306u, 17527406820792516033u,          74338572210u },
+  {  5290016144582400923u, 12119966834653692210u,         178950162627u },
+  { 15162883663174059077u, 11606502845877928061u,         195657024718u },
+  {  8078086116520046390u,   424311496652297090u,         206629189780u },
+  { 15731407332173190623u,  5977664048034127173u,         148023001972u },
+  {  5346389182763011056u,  6702712461535947028u,         116324049817u },
+  {  6368422217216252401u, 11384349854055020018u,         153363354770u },
+  {  2961453088119116188u,  3782955013294836188u,         146617146842u },
+  { 10932141691610170525u,  3531805968821207061u,         218205074402u },
+  { 11034016191361782553u,  3867566898657193228u,         226191459585u },
+  {  5455325785621453219u, 12688734637425072080u,           1209661221u },
+  {  7735615202566149352u, 18435982764454619691u,          37687857682u },
+  {  7502396497775759360u,  4728836163964677692u,          18999416628u },
+  {  1601286435751591936u,  2120012917348838977u,          52256350722u },
+  { 11449383158571597824u,  9856965465824679831u,           2114926130u },
+  { 13043944595690356736u, 11217197671061248816u,          50534347168u },
+  {  7773494431818186752u,  3840562972677739189u,         160608085504u },
+  {  9943947977234055168u, 17104366978925258617u,            208197335u },
+  {                    0u, 16177877219841993444u,         215927229591u },
+  {                    0u,  7338522384267208704u,         151877004481u },
+  {                    0u, 10935240458612244480u,         193397822095u },
+  {                    0u,  1732868046462124032u,         143592800573u },
+  {                    0u,   557965042578882560u,          61093938965u },
+  {                    0u, 10454684322475540480u,          21030247345u },
+  {                    0u, 13907115649320091648u,         177566749572u },
+  {                    0u,                    0u,         132753906250u },
+  {                    0u,                    0u,          74000000000u },
+  {  6343817245135589714u,                  199u,                    0u },
+  { 17549323075660516085u,         199343899021u,                    0u },
+  {  3948641822109421754u, 14876458284855834550u,                   10u },
+  {  1750739713693534543u, 10450704926982265198u,          10806454419u },
+  {   962163898128633415u,  5385653213018257806u,         147566533849u },
+  {  7967320249386531212u, 12735569669880147489u,         217291956845u },
+  {  3018466665533383224u,  3619762560577729456u,         109690396615u },
+  { 15076865731854945472u, 11123448126624084269u,         199196227721u },
+  {   691187172844604400u,  4072715118852885633u,         137603003331u },
+  { 13274492813370992341u, 18239087231420827283u,         195220782328u },
+  {  6364168818499152300u,   423431461216085297u,         248988742900u },
+  { 17599380787401914158u,  9360976716520160042u,         244022954265u },
+  {  3782323149461692814u, 11655927117263208920u,          25507459564u },
+  { 14470163442442237466u,  2646622721938364948u,         236631869075u },
+  {   476109872130437939u,  4496462484548171852u,         147143473705u },
+  { 16330548844673355700u, 13140258519803350063u,          41243753719u },
+  { 14089158961463739563u, 13089764333320627770u,         247712334841u },
+  { 13385510793074798805u,  6926286827289840501u,         249709597546u },
+  {  1549401308746959012u,  4985580225290866218u,         106375474761u },
+  {  4607384943843027435u, 10478790837359789693u,          73270268845u },
+  {  5157353797716093483u, 10041191967455692214u,         173568056389u },
+  { 10846828782671550129u,  5035461258013813797u,          69544334107u },
+  {  9915857350819131531u, 14208759661559249750u,          27272972901u },
+  {  1673544973504317923u, 12347272163241758840u,         101770258404u },
+  { 14780986291622785694u,  3372534174410277614u,         228669346965u },
+  {  8168111319515466401u, 17226704187274712984u,         149182825443u },
+  { 16345760387859734482u,  4250480179449852121u,         227933861505u },
+  {  3177475373321281805u,  4303723537755414374u,         129230418992u },
+  {  2558676822419554038u,  8680503847344854165u,          48233305320u },
+  {  8813474062662382873u,  8817608623911079652u,         232470571056u },
+  {  5426294560236228430u,  5692030448698539450u,          48478003521u },
+  {  9919177474128333040u, 16908836314686769809u,          65308565588u },
+  {  6197383943089627371u,  6073762347067727240u,          84916629853u },
+  { 13673239314867423997u, 10931066692585106200u,          93329259316u },
+  {  9309584098968723946u, 14466591364061539596u,          52592574312u },
+  {  4647101757759615504u,  4958077340960173341u,         104784235489u },
+  {  1405809295505096753u,  4076890037156765715u,         225268777911u },
+  { 11332704079573859112u, 14083973146609179058u,         183221008651u },
+  {  2960072434514044308u,  2565183738039805295u,          11763493714u },
+  {  3887266602785432801u,  1482420938751351224u,          82139058889u },
+  { 14545546084687849554u,  2151089495335413944u,         201080362200u },
+  {  4617763804182385321u,  3738604531753220913u,         216116610795u },
+  {  7857823815580249095u, 14195686514836005765u,         235202670157u },
+  { 10939326736548364798u, 17808833916231796970u,          77769549707u },
+  { 12262012446566951953u,  1302384553035657354u,         139965418821u },
+  { 12555024338687723023u,  1672033517974833698u,          69070602408u },
+  {  3332969632922829472u, 11673925532927662545u,         168090641118u },
+  { 15535060143360327680u,  3905334232240480709u,         222632844771u },
+  { 15794322927987458048u, 17411087320267472625u,         227211708592u },
+  { 10571474314433921024u, 16573305231063706617u,         176943856934u },
+  { 16679514427547975680u, 15481103236037148354u,          38898440676u },
+  { 16925653299565166592u,   907440704754420880u,         228839232288u },
+  { 16717361816799281152u,  3224970785139077759u,          32049192459u },
+  {                    0u, 10560826509734608144u,          11174826016u },
+  {                    0u,  4700940027512659968u,          32572503552u },
+  {                    0u,  9733694683502084096u,            254838469u },
+  {                    0u,  1995535635724632064u,         197527664646u },
+  {                    0u, 10629833226245373952u,           6108178203u },
+  {                    0u, 15729384648544878592u,          27576244413u },
+  {                    0u,  7205759403792793600u,         189852691650u },
+  {                    0u,                    0u,         194390625000u },
+  {                    0u,                    0u,         232000000000u },
+  {  9335385384027907407u,              3041746u,                    0u },
+  { 13237828406194798613u,     3041746506072255u,                    0u },
+  { 15667486867836528863u,  7535526066623007027u,               164893u },
+  { 17982325043592934313u, 11302146918409311588u,          29408501686u },
+  { 17159117626917379189u,  2480833299122194801u,         182612690612u },
+  {  8336208968408929657u, 11513226205589330558u,         180134486242u },
+  { 12767090573379150201u,  4073957068281936105u,         226624133243u },
+  { 14736070002412246709u,  3729887061093812886u,         123220849655u },
+  {  9697296975344560756u, 13616911779739451443u,         247202197582u },
+  {  7291706381199103298u, 13039053282195777666u,          78738174266u },
+  { 18098546597780825068u, 14490756113210417890u,          58706848494u },
+  {   132913902678533478u, 17432486112977557585u,         238785545462u },
+  {  1879347741692007580u, 14308820825344039837u,         246945016965u },
+  {  4056624629214083684u,  4190949538817536349u,         133775682731u },
+  { 14713227692042795499u, 13616552502810964397u,         171227191829u },
+  {  7366415124022528526u,  4898145803694965031u,          21738154790u },
+  {   825770353378039393u,  1399036321001644308u,          38265529016u },
+  { 10244023944395357795u, 17170331128243738540u,         184075841910u },
+  { 14302658294713551167u, 10641321388205367410u,         118930805515u },
+  { 14246653166206862817u,  6648873641312572851u,          11576867188u },
+  { 12404486258134291102u,  5988456964560374823u,         116360436162u },
+  {  8887442218637942533u,  9972593758348346915u,         194324634902u },
+  {  1186724038081863005u, 16709668921872818968u,          22540615390u },
+  {   211331772484951576u,  6094829131503407767u,         222905832967u },
+  {  6311919513247413649u,  4892016478899926334u,           7330401349u },
+  {  8131780018703965703u, 13150857244079031538u,          69265196744u },
+  {  2262544347226725725u, 12983943395318785894u,         200712909399u },
+  { 15318188749880522583u, 15341644584614757478u,          87703860981u },
+  {  1073117094162650652u,  7507635124856644772u,         245831672219u },
+  {  4447950380665871747u, 11619655367084544354u,         155406989715u },
+  {  5373227185066463609u, 11553116952478783009u,         147629902779u },
+  {  6739731406934274193u, 17392150014233193245u,         187626295724u },
+  { 12328812617001239444u,  8877887560294980515u,         172942830341u },
+  {  3246111484407310759u, 18404180619915609503u,           5481271248u },
+  {  3250825415176839770u, 10079413095288181976u,         208997692630u },
+  { 16146270540000862342u, 14102802966539105550u,         214546406078u },
+  { 15686773375425916830u, 13333966026135891399u,         190764514480u },
+  { 11920791905793880226u, 12344968670173516152u,         176722835746u },
+  {  1853290561644080707u, 10577007819804726752u,          34669222092u },
+  { 12157689141506159076u, 15337041354031088010u,         204573380742u },
+  { 18100318838862562546u, 14333607285614673616u,         134831422677u },
+  {  7171257882533475139u, 17171597563219696538u,         213777026407u },
+  { 14477550873015039462u,  2849642930482147564u,         103930874169u },
+  {  8109481182495403274u, 14791248423979435173u,          57154479452u },
+  { 14565395719337663965u, 13882371364576310127u,          92801835183u },
+  {  1860318978161305991u, 11735995808941329540u,         175752564859u },
+  { 16268646275151585618u, 11376996674339273181u,         123636209607u },
+  { 13759019338835519104u,  9849638057168043481u,         199616748225u },
+  { 17003783176010661888u, 18241520229279361964u,         193533949948u },
+  { 18357489540307877888u,  1865852368526961444u,         252988874793u },
+  {   905481790074912768u, 10601487369276448158u,          41101148059u },
+  {  3638882110636294144u, 15999931310312762170u,         155574707781u },
+  {  9011702854368362496u,  5773775867713013570u,          69867358014u },
+  { 11529215046068469760u, 17726239863982547534u,          62312997016u },
+  {                    0u,  9711316695888316992u,         152960941388u },
+  {                    0u, 17872002620723724288u,          76526451532u },
+  {                    0u,  7429694208660733952u,          76968843203u },
+  {                    0u,  1782821038871019520u,         195402764530u },
+  {                    0u,  3225250234313474048u,         242096646922u },
+  {                    0u, 10009250171830927360u,          10174841165u },
+  {                    0u,  1152921504606846976u,          77542602539u },
+  {                    0u,                    0u,          43062500000u },
+  {                    0u,                    0u,         160000000000u },
+  {  7625299565768063067u,                   46u,                    0u },
+  { 13889021769065194705u,          46413368317u,                    0u },
+  { 14498170692313014398u,  9519880170333822146u,                    2u },
+  {  1541631360972245751u,  2285186318012886800u,           2516073738u },
+  {  9903958882920799117u,  9706420951402272035u,          10123880198u },
+  { 15744148547788062576u,  2369632031840402142u,           6526186134u },
+  { 17936061801321712000u, 15599123897979399458u,         150128458009u },
+  {  9986394078324430610u, 17579576584023912658u,          25845630200u },
+  { 13849561248103430369u,  3480927339588501811u,         248952990756u },
+  { 12142378807953854930u,  3547346616671294635u,          36188701449u },
+  {  2513847703931031444u,  7705317123868384954u,           9192302045u },
+  {  1752183758129038045u,  4969425237478353909u,         221417706078u },
+  { 15824833342220556540u, 17043246700132217175u,          94269393081u },
+  {  8168747198299470695u, 17053788362783499470u,         185923916254u },
+  { 17414799840149357478u, 11102988228454224768u,         222924487719u },
+  {    83147520704167789u, 16944305387801685839u,          39601894197u },
+  {  2383542703041471269u, 11725142977459199276u,          53918552635u },
+  { 15540952725549257799u,  8175984171998533324u,          59635621274u },
+  {  6949835416232048690u,  1372352885142856895u,         154443220990u },
+  { 15398868937585367540u, 17975093466502888164u,         254074395398u },
+  { 14710915985268256079u,  6467823391459085653u,           6974431769u },
+  {  9665704836873335737u, 11319386883146885025u,          25350621408u },
+  {  2528789298740305993u,  9141999262922068637u,         224613625192u },
+  { 10562914675687726264u,  1587330393383478774u,         104495588773u },
+  { 17671658300096837111u,   884187548095712303u,         165086049353u },
+  {   356471401631698552u,   488841225726377268u,          73047931903u },
+  {  7450677157218003204u, 17462624199405856193u,         255026500135u },
+  {   317174560787152643u, 13183677579115583554u,          39946650754u },
+  {  7251937674440720374u, 11645015818917277779u,         130714688593u },
+  {  1252631516699038247u,  8760523002035971977u,          81631277572u },
+  { 10818009768860843867u, 10068817678491468042u,           4474908903u },
+  { 12330114194950162396u,  1273658177787418284u,         231545831700u },
+  { 15826681638261168822u,  3100019384328057661u,          20069045148u },
+  { 14240150078499211625u, 10363063568089458738u,         156168052387u },
+  { 10725372116242125421u, 13030756371481403666u,         163561782801u },
+  {  8434925524647833627u,  6538878900684195299u,          17706398718u },
+  { 17133902668520348241u,  8984884716779098868u,         254354473335u },
+  { 15093996047981365810u,  8728727397070363908u,         119487071576u },
+  {  6187974166976813153u,  6398650562917867005u,          88473185260u },
+  { 13946144707720259865u,  1190873176164938879u,         236346871542u },
+  {  9138079832881862582u,  4383628525805121795u,         246064557364u },
+  {  6600697628576225568u, 10189374699734119852u,          52237636978u },
+  {  8137099536646556597u,  5276291920541626391u,         114552367109u },
+  { 14169855543453903706u,  2692252373800386521u,           5286028358u },
+  {  3706403268650100765u, 11578684995169173920u,          70145947293u },
+  { 14736932266877982264u,  5799408022254132587u,         157627681771u },
+  { 18004795125138956004u, 15548569837712345290u,         235314386538u },
+  {  9068489270661002501u, 15763030464322902955u,         106842889659u },
+  {  7758835715193269217u, 13257749746581255500u,         187854515593u },
+  { 16943947811135261184u, 16152470009188707678u,         137718704053u },
+  {  6745843108403216384u, 13806790848493904003u,         181875627153u },
+  { 12338229654069444608u, 11981226523265951191u,         145748467631u },
+  { 14358176069683511296u,  5133628726077003713u,         175649503591u },
+  {  7083775185760813056u, 16183955741910833164u,         103278294570u },
+  {  5350276357316149248u, 13640425554331371454u,          42877333998u },
+  {  9223372036854775808u, 18108120906868035862u,         238739448950u },
+  {                    0u,  6324011669895037184u,         118981643201u },
+  {                    0u, 10444437689515769856u,         193342825359u },
+  {                    0u, 12324712543665782784u,         143566194101u },
+  {                    0u, 13928941951563857920u,         181668124005u },
+  {                    0u,  3975288688270639104u,         101755089456u },
+  {                    0u, 11141905478114607104u,          48215500831u },
+  {                    0u,  4611686018427387904u,          31604003906u },
+  {                    0u,                    0u,          66250000000u },
+  {                    0u,                    0u,         128000000000u },
+  { 14699116688460625612u,               708211u,                    0u },
+  {  6684126021499623499u,      708211796840712u,                    0u },
+  { 12614606079692508506u,  4398362855256705725u,                38392u },
+  { 15358270276683001489u,  2812083125569302717u,         248238435728u },
+  { 18077126190953408995u, 12868509142973100603u,         144152443331u },
+  {  7864121581925945659u,  8726243776748165726u,         195697603278u },
+  {  3518026639210514839u,   358304413426858117u,         206473050623u },
+  {  4698310163811252280u,  3180720351566429470u,         255019423721u },
+  {  6101155398200416338u, 14053818240400098784u,         233172427195u },
+  { 16049178580360033341u,  7340140541492429288u,         187761859013u },
+  {  3167464649127375997u,  1323571167904965058u,         197397909816u },
+  { 12778923935480989904u, 14463851737583396026u,          56071750936u },
+  { 11875553912612980379u, 15122784818916048486u,          24784086973u },
+  { 10628760849351697057u, 13557974621377508955u,         189819807807u },
+  {  3408944711673234310u, 17525172074563876264u,          63734979276u },
+  {  2102091496050506178u, 15148880683074215967u,         204950041481u },
+  {  6254611118630245760u,  6744828147558597936u,         137821222467u },
+  {  2647941151989776368u,  9799290779647971692u,          67365637866u },
+  {  8855437735410157458u, 11170890203898678105u,         234531220617u },
+  { 10184270603132180103u,  7068779781287527905u,         137605575171u },
+  { 12079083162535627164u, 14474741922505540911u,           3383199319u },
+  {  5952952868716156729u, 17107062680405191514u,          87784677331u },
+  { 11958907037815852320u,  2712598571300237005u,         211927375726u },
+  { 10101562137321697626u,  3767556054903418641u,         110147050263u },
+  { 13633527411279258327u, 18158239681706277628u,          23204239622u },
+  { 16555627393501768728u, 10531652712128330681u,           6984360145u },
+  {  6359650463500280706u,  9548395326934120567u,         209570922037u },
+  {  8093923611102181967u, 15875647850297719390u,          53517619547u },
+  {  2848827352928635726u,  8215825295203192574u,          91860620594u },
+  {  4686723431961561511u, 12747310908260543144u,          50445380781u },
+  {  6245554925867652609u,    77706528053613642u,         173691033109u },
+  { 17527406820792516033u,  6024737704056756146u,          21004212479u },
+  { 12119966834653692210u,  6819452388570089667u,         255326601685u },
+  { 11606502845877928061u, 13695926775373186254u,         213369683254u },
+  {   424311496652297090u,  3746531715392682132u,          54742457678u },
+  {  5977664048034127173u,  4717376233154528116u,          78203099891u },
+  {  6702712461535947028u,   385190957950313369u,         243255729478u },
+  { 11384349854055020018u, 12388374310648616082u,          70020881243u },
+  {  3782955013294836188u,  1078067332084407770u,          91671575117u },
+  {  3531805968821207061u,  3257295319358714850u,          77058442147u },
+  {  3867566898657193228u,  1545453099660723457u,         163176578333u },
+  { 12688734637425072080u,  7495477664653506341u,          29083779180u },
+  { 18435982764454619691u,  7225503732673614354u,         108406330658u },
+  {  4728836163964677692u,  3935478326103643956u,          34391695342u },
+  {  2120012917348838977u, 10082240682742686210u,         238213342707u },
+  {  9856965465824679831u, 10838712705567897138u,         243546559362u },
+  { 11217197671061248816u,  2142546572501643680u,         130587567793u },
+  {  3840562972677739189u,  7893042119150331392u,         177116147682u },
+  { 17104366978925258617u, 12084811642251302615u,         226427882670u },
+  { 16177877219841993444u, 15317234482572954775u,         174655118951u },
+  {  7338522384267208704u,  2283226355108359361u,         103830348945u },
+  { 10935240458612244480u, 13359725152575722127u,         145123773948u },
+  {  1732868046462124032u, 13126551011491594557u,         252724232151u },
+  {   557965042578882560u,  3598021288691861269u,         215711591756u },
+  { 10454684322475540480u, 16462621795896662961u,          76195049124u },
+  { 13907115649320091648u, 14682112756964627332u,         164892440515u },
+  {                    0u,  7174112100896070218u,         195795918927u },
+  {                    0u,  5023109019590616064u,          79388909396u },
+  {                    0u, 10765223023086141440u,          84272303285u },
+  {                    0u,  8228137177297453056u,         181583583909u },
+  {                    0u,  2891199497780592640u,         165446048210u },
+  {                    0u, 15294857653247803392u,         210156732238u },
+  {                    0u, 14303432416528695296u,          78829135894u },
+  {                    0u,                    0u,          22775390625u },
+  {                    0u,                    0u,         161000000000u },
+  { 14876458284855834550u,                   10u,                    0u },
+  { 10450704926982265198u,          10806454419u,                    0u },
+  {  5385653213018257806u, 10806454419566533849u,                    0u },
+  { 12735569669880147489u, 17118225092618494573u,            585819067u },
+  {  3619762560577729456u, 13385738875341807559u,         187927980841u },
+  { 11123448126624084269u,  8272682717439277193u,          41725642358u },
+  {  4072715118852885633u, 13402436483369350083u,         118448463028u },
+  { 18239087231420827283u, 10946328903241612536u,         180726547537u },
+  {   423431461216085297u, 16265808923426731252u,          81593401678u },
+  {  9360976716520160042u, 11080374459871185177u,          78881771268u },
+  { 11655927117263208920u,  1240761893433831916u,           4600668303u },
+  {  2646622721938364948u,   367264070493390483u,         143067261837u },
+  {  4496462484548171852u,  2863675693461092905u,         141019909425u },
+  { 13140258519803350063u,  7511929581752138999u,          49155240170u },
+  { 13089764333320627770u, 11154557789993845753u,         234407222518u },
+  {  6926286827289840501u,  8325416539745948522u,         246604689789u },
+  {  4985580225290866218u, 17745129874679852617u,         125451321734u },
+  { 10478790837359789693u,  1074820986392253357u,         134961965418u },
+  { 10041191967455692214u,  7820952682162838597u,         106058266162u },
+  {  5035461258013813797u,  8215518006273528603u,          50423974694u },
+  { 14208759661559249750u,  9680426791089900133u,          38445364123u },
+  { 12347272163241758840u, 16128495723604797412u,         155524776987u },
+  {  3372534174410277614u,  2264789053583348885u,          27874327505u },
+  { 17226704187274712984u, 11175458488686298083u,         209122774460u },
+  {  4250480179449852121u, 11026777810412287617u,         188605822818u },
+  {  4303723537755414374u, 16199890034895598640u,          98597762822u },
+  {  8680503847344854165u,  9094320719494763752u,           6878197798u },
+  {  8817608623911079652u,  1250835564687222832u,          38493004114u },
+  {  5692030448698539450u, 15362466642459337025u,          82067807931u },
+  { 16908836314686769809u,  7831109835595423828u,         187832800985u },
+  {  6073762347067727240u, 15426237284335022429u,         217424525314u },
+  { 10931066692585106200u, 15636308361455434548u,           2836257998u },
+  { 14466591364061539596u, 13967173875944980328u,         206847645974u },
+  {  4958077340960173341u, 18245979923595824097u,          22757162012u },
+  {  4076890037156765715u, 11335054479675278263u,          28989116553u },
+  { 14083973146609179058u, 11165339882630461707u,         137614474534u },
+  {  2565183738039805295u, 15944437408299395922u,          38605274287u },
+  {  1482420938751351224u, 15806416348777321161u,         175864349683u },
+  {  2151089495335413944u,  4201030477408556248u,         243856867547u },
+  {  3738604531753220913u,  9485474942554588907u,         219227738318u },
+  { 14195686514836005765u, 18238757647663230541u,         206514208626u },
+  { 17808833916231796970u,  4642199687824746379u,         114988725033u },
+  {  1302384553035657354u,  6134575894869364037u,          41251654149u },
+  {  1672033517974833698u, 11524208547121316008u,           5332556025u },
+  { 11673925532927662545u,  2734683241527878366u,         249624728597u },
+  {  3905334232240480709u, 10629223456178675171u,          21148247475u },
+  { 17411087320267472625u,  2788042336985254064u,         179576211358u },
+  { 16573305231063706617u, 17285498758066142502u,         158151140077u },
+  { 15481103236037148354u,  5525538192421886436u,         237937048765u },
+  {   907440704754420880u, 11414325503043801888u,         189299540025u },
+  {  3224970785139077759u,  7246608114685173259u,          57618771825u },
+  { 10560826509734608144u,  1007884269852184608u,         113392839413u },
+  {  4700940027512659968u, 13823717876510029312u,         245054637515u },
+  {  9733694683502084096u, 12487410768239429317u,         203749385247u },
+  {  1995535635724632064u,  3361062421598631942u,          31676943894u },
+  { 10629833226245373952u, 17853337379088328475u,          22182203558u },
+  { 15729384648544878592u, 11551561037491869885u,         166967831358u },
+  {  7205759403792793600u, 11480877996635204802u,          62626211378u },
+  {                    0u,  5527488381934471912u,          50622379643u },
+  {                    0u, 11143438404407726080u,         123299645745u },
+  {                    0u,  6472279730688098304u,          49604087006u },
+  {                    0u,  4561816853579563008u,         222350862987u },
+  {                    0u,  2888714464062865408u,         139247296587u },
+  {                    0u, 16258276129784201216u,          75156597524u },
+  {                    0u,   720575940379279360u,          20881362915u },
+  {                    0u,                    0u,         227039062500u },
+  {                    0u,                    0u,         228000000000u },
+  {  7535526066623007027u,               164893u,                    0u },
+  { 11302146918409311588u,      164893408501686u,                    0u },
+  {  2480833299122194801u, 16409970870640346804u,                 8938u },
+  { 11513226205589330558u,  7721907286269370594u,         234889586303u },
+  {  4073957068281936105u, 14300743897882155131u,         127418605432u },
+  {  3729887061093812886u,  2068482633821123575u,         120775244880u },
+  { 13616911779739451443u,  4922882895416406094u,          80112132668u },
+  { 13039053282195777666u,  9317632875623428410u,          60266870016u },
+  { 14490756113210417890u,  5693844901999766254u,            505109890u },
+  { 17432486112977557585u, 11569484900262102262u,         130308663950u },
+  { 14308820825344039837u,  3138170119352085637u,         142627183033u },
+  {  4190949538817536349u,   950584692575235243u,         185170120543u },
+  { 13616552502810964397u,  8136430299747162645u,          95051531299u },
+  {  4898145803694965031u,  6698711700804594470u,          35441076770u },
+  {  1399036321001644308u, 17401191571004302008u,          34363137888u },
+  { 17170331128243738540u,  4721732028538188150u,          96943320485u },
+  { 10641321388205367410u,  2984214103553086219u,         165255965606u },
+  {  6648873641312572851u, 13128675202005662068u,         166161774570u },
+  {  5988456964560374823u, 14638512997670672834u,         234711706908u },
+  {  9972593758348346915u, 12942085665769692438u,          28793555379u },
+  { 16709668921872818968u, 14131134357119205086u,         179701591869u },
+  {  6094829131503407767u,  8921946894736102919u,          61766050328u },
+  {  4892016478899926334u,  5601522560505809989u,          24483659710u },
+  { 13150857244079031538u,  8602606493507716808u,         190303659146u },
+  { 12983943395318785894u,  8576789731078566487u,         138466348232u },
+  { 15341644584614757478u, 17881118138842658549u,         200464948702u },
+  {  7507635124856644772u, 11624372674432704923u,         222969337356u },
+  { 11619655367084544354u,  6826284072848095635u,          12630158505u },
+  { 11553116952478783009u,  1646466632033733563u,         169370053601u },
+  { 17392150014233193245u, 17871081657060299180u,         225089255134u },
+  {  8877887560294980515u, 15910893124677544709u,         222968793277u },
+  { 18404180619915609503u, 11031217459450580944u,         189862531244u },
+  { 10079413095288181976u, 13554987390037243094u,         172598003496u },
+  { 14102802966539105550u, 15026714590903687870u,          40734817338u },
+  { 13333966026135891399u,  4406379654994689200u,          58814599830u },
+  { 12344968670173516152u, 13596329092861950242u,         150238870319u },
+  { 10577007819804726752u,   284812388227373260u,          47737058477u },
+  { 15337041354031088010u,  9285079159392309382u,         173015439710u },
+  { 14333607285614673616u, 15046108141952711893u,          94503345149u },
+  { 17171597563219696538u, 13795366909944958311u,         253815651156u },
+  {  2849642930482147564u, 12909920641180059961u,          84747848338u },
+  { 14791248423979435173u,  5333762939889788252u,         146699848200u },
+  { 13882371364576310127u,  6411331390005944495u,           8289143868u },
+  { 11735995808941329540u,  1447104583224217723u,          60347558971u },
+  { 11376996674339273181u, 11940049226167932871u,          59078447696u },
+  {  9849638057168043481u,  9772290783590472385u,          80647271365u },
+  { 18241520229279361964u, 16351989577831528444u,         197529756944u },
+  {  1865852368526961444u,  4376738725895725097u,          16886443131u },
+  { 10601487369276448158u, 13851276297739812763u,         123237263481u },
+  { 15999931310312762170u, 12641996203470333509u,         121750879192u },
+  {  5773775867713013570u,  7707081716407945022u,         216685323987u },
+  { 17726239863982547534u,   417638323657040024u,         211417801737u },
+  {  9711316695888316992u, 16438047707692449100u,           9022640218u },
+  { 17872002620723724288u, 14850108107043306316u,          90891108351u },
+  {  7429694208660733952u, 10423290807904720835u,         255805025973u },
+  {  1782821038871019520u, 16951162310302339314u,         181565047726u },
+  {  3225250234313474048u,  2752437506572397322u,         174918924350u },
+  { 10009250171830927360u,  3925815842962784589u,          62149209936u },
+  {  1152921504606846976u,  5274166674003605291u,          80212818903u },
+  {                    0u,  5538963350863452832u,         215285913148u },
+  {                    0u, 16900671634439028736u,          60300267804u },
+  {                    0u,  2326997710751662080u,          28916187245u },
+  {                    0u, 12327726161625874432u,         109126146798u },
+  {                    0u,  5756455743825903616u,         238668287374u },
+  {                    0u,  3018537650245074944u,         142312058091u },
+  {                    0u, 16717361816799281152u,         235163635253u },
+  {                    0u,                    0u,          53906250000u },
+  {                    0u,                    0u,          16000000000u },
+  {  2285186318012886800u,           2516073738u,                    0u },
+  {  9706420951402272035u,  2516073738123880198u,                    0u },
+  {  2369632031840402142u, 11997425759292732054u,            136396630u },
+  { 15599123897979399458u, 11491152661270395161u,          86650381753u },
+  { 17579576584023912658u, 18181063258234881272u,         185622936633u },
+  {  3480927339588501811u,  2466921813123869732u,          57985597414u },
+  {  3547346616671294635u,  8430880678232179465u,         230133732099u },
+  {  7705317123868384954u,  6738034873677997533u,           3457038957u },
+  {  4969425237478353909u,  7678250951042929246u,         109365269602u },
+  { 17043246700132217175u,  1853560606315563193u,          98416238818u },
+  { 17053788362783499470u, 14942676593409905118u,         226100481721u },
+  { 11102988228454224768u,  4909892170837638183u,         185810044121u },
+  { 16944305387801685839u, 16871149368312132405u,         217266165787u },
+  { 11725142977459199276u, 16096130589333770811u,          27914586839u },
+  {  8175984171998533324u, 12512479187631824282u,         215872572987u },
+  {  1372352885142856895u, 16980304980540557310u,          59678302855u },
+  { 17975093466502888164u,  8640919162749295366u,         135920504177u },
+  {  6467823391459085653u,  7862382415464063513u,         113468425166u },
+  { 11319386883146885025u, 14534157903009925344u,         206426220604u },
+  {  9141999262922068637u, 12627464554215107944u,          60787898278u },
+  {  1587330393383478774u,  2456849734836299173u,         166684536225u },
+  {   884187548095712303u, 18428252197697827913u,         161133186090u },
+  {   488841225726377268u,  7244734215936736255u,          42998997553u },
+  { 17462624199405856193u, 14756175050504770087u,          49392737828u },
+  { 13183677579115583554u,  6764116534566945922u,          36799933852u },
+  { 11645015818917277779u,  1588822142405565521u,         156366683492u },
+  {  8760523002035971977u, 17053265624843842052u,         100086130220u },
+  { 10068817678491468042u, 16996891591759999207u,          44924459381u },
+  {  1273658177787418284u,  8565556232370585876u,         117921403339u },
+  {  3100019384328057661u, 14464960359145886620u,         203464339733u },
+  { 10363063568089458738u,  5813189542048784035u,          21784147072u },
+  { 13030756371481403666u,  9739241026882027025u,         128315133636u },
+  {  6538878900684195299u, 18175068535675302910u,         196527965313u },
+  {  8984884716779098868u, 10562697212061761911u,         129985272439u },
+  {  8728727397070363908u,  4264834835660801368u,         119572604963u },
+  {  6398650562917867005u, 13019066443690126316u,          35231197159u },
+  {  1190873176164938879u,  1828040177823321846u,         231705765006u },
+  {  4383628525805121795u, 11240369830376975668u,         142099098256u },
+  { 10189374699734119852u,  8886938465302549874u,         144609341669u },
+  {  5276291920541626391u,  9985240313589688325u,         229481761899u },
+  {  2692252373800386521u,   722909126956573766u,         107541300962u },
+  { 11578684995169173920u,  5493363474638452381u,         226039188982u },
+  {  5799408022254132587u, 12410535279213120491u,         246297795830u },
+  { 15548569837712345290u, 10543108918366869098u,         246672776465u },
+  { 15763030464322902955u, 12953909016524823995u,          17571543079u },
+  { 13257749746581255500u, 16505942145872588169u,          39702232814u },
+  { 16152470009188707678u, 12428594380392015797u,         238894788916u },
+  { 13806790848493904003u,  7528259605829768337u,          52673755451u },
+  { 11981226523265951191u, 18147447600042811311u,          59408107770u },
+  {  5133628726077003713u, 12021069431116183911u,         250983775105u },
+  { 16183955741910833164u, 11819985069665662506u,         129651663479u },
+  { 13640425554331371454u, 10401877114068152814u,         119640762674u },
+  { 18108120906868035862u,  4611631138117837942u,          50563886888u },
+  {  6324011669895037184u, 17200813398607252417u,          40249997024u },
+  { 10444437689515769856u, 14100466137553658767u,         224932457962u },
+  { 12324712543665782784u, 17887776768825509301u,         234764387800u },
+  { 13928941951563857920u, 12632656857970087269u,         216969698321u },
+  {  3975288688270639104u,  8923681664054686256u,          17684817700u },
+  { 11141905478114607104u,  6213926103737837599u,          36483753752u },
+  {  4611686018427387904u,  1233118281776157762u,          24336857609u },
+  {                    0u,    30716279628678784u,           9066847476u },
+  {                    0u, 15775734650898546688u,         244001665132u },
+  {                    0u,   976806005729918976u,         108855204289u },
+  {                    0u, 12460098853279891456u,         193052952759u },
+  {                    0u,  5635665595421687808u,         183675463312u },
+  {                    0u,  1805943450575568896u,         144305510044u },
+  {                    0u, 11529215046068469760u,         156097900390u },
+  {                    0u,                    0u,         102625000000u },
+  {                    0u,                    0u,          64000000000u },
+  {  4398362855256705725u,                38392u,                    0u },
+  {  2812083125569302717u,       38392238435728u,                    0u },
+  { 12868509142973100603u,  4564018338575530435u,                 2081u },
+  {  8726243776748165726u, 16553437246451512014u,          33247415929u },
+  {   358304413426858117u,  4339777136957372927u,         121897363631u },
+  {  3180720351566429470u, 18439463366554654697u,         175235259789u },
+  { 14053818240400098784u,  1370067356680643003u,         141999605312u },
+  {  7340140541492429288u,  4210124040914115013u,          64074271500u },
+  {  1323571167904965058u, 10692225626142609720u,          12228231281u },
+  { 14463851737583396026u, 11592856673895384344u,         113579626712u },
+  { 15122784818916048486u, 10284479231227406269u,         216628450019u },
+  { 13557974621377508955u,  4961071383534266431u,         227557522736u },
+  { 17525172074563876264u, 10960611551445686988u,          48268940218u },
+  { 15148880683074215967u, 14616396723115619209u,         186594175942u },
+  {  6744828147558597936u,  1025604265437492803u,         198792356454u },
+  {  9799290779647971692u, 11711588454892179178u,         102055598118u },
+  { 11170890203898678105u,  5580373263251565705u,          38634886482u },
+  {  7068779781287527905u, 14109334653033148931u,          82302512640u },
+  { 14474741922505540911u,  2899414033769399895u,            764868564u },
+  { 17107062680405191514u, 13233457234892808147u,         212157177549u },
+  {  2712598571300237005u,  3287946691509034862u,         205717387154u },
+  {  3767556054903418641u,  5488480288717445911u,         146178239947u },
+  { 18158239681706277628u, 11687233053874362630u,         203297531112u },
+  { 10531652712128330681u,  6783772100089274577u,         232633566173u },
+  {  9548395326934120567u,  7898291058728402485u,         221367749022u },
+  { 15875647850297719390u,  4423684977486598491u,         158428167216u },
+  {  8215825295203192574u,  2750833684599526706u,          48239808443u },
+  { 12747310908260543144u, 15669689830489025709u,         187149122992u },
+  {    77706528053613642u, 15117307274214954517u,         176849455587u },
+  {  6024737704056756146u,  8148639818575698175u,         227819510869u },
+  {  6819452388570089667u, 13006484426078994901u,          85441738649u },
+  { 13695926775373186254u, 10287496057845513526u,         153705082933u },
+  {  3746531715392682132u, 14159876032966532430u,          53557686278u },
+  {  4717376233154528116u, 15742212196465548019u,           6767608417u },
+  {   385190957950313369u,  2892220461917134150u,          97853387033u },
+  { 12388374310648616082u,  7487151560715393883u,          25156787585u },
+  {  1078067332084407770u,  7245756744165177933u,         129405879299u },
+  {  3257295319358714850u,  3067122860671533987u,           3392793260u },
+  {  1545453099660723457u,  8135043905834122525u,         172166269063u },
+  {  7495477664653506341u, 14730019368921022572u,         135441001613u },
+  {  7225503732673614354u,   495969939682055458u,         141798515950u },
+  {  3935478326103643956u,  5617761407265775598u,         238026886584u },
+  { 10082240682742686210u,  2087044847072781811u,         184304539456u },
+  { 10838712705567897138u, 15929674232061203330u,          64113138927u },
+  {  2142546572501643680u,  8658086469608285873u,         239863549370u },
+  {  7893042119150331392u, 18369871790780313570u,         186469355807u },
+  { 12084811642251302615u,  3545648451947416750u,          31995832745u },
+  { 15317234482572954775u, 13347376792767929959u,         169192209987u },
+  {  2283226355108359361u, 14482164459838203025u,          67723562745u },
+  { 13359725152575722127u,  8899577765623565820u,         249785079708u },
+  { 13126551011491594557u,  7095320096604405719u,         156482447077u },
+  {  3598021288691861269u,  2968593824439315788u,         229384638073u },
+  { 16462621795896662961u, 12621408323612585636u,         121160927793u },
+  { 14682112756964627332u,  3954422936414648259u,          49684207916u },
+  {  7174112100896070218u, 17143730087577690191u,          44214369696u },
+  {  5023109019590616064u,  5033045529399041876u,         160929363470u },
+  { 10765223023086141440u, 15857648521994521781u,          14272841944u },
+  {  8228137177297453056u, 16655573486499109541u,         216859644848u },
+  {  2891199497780592640u, 16652154439190075858u,         176902900447u },
+  { 15294857653247803392u, 18016950600164130638u,         223902715100u },
+  { 14303432416528695296u,  2086292996072613910u,         220976700849u },
+  {                    0u, 17324462585194799521u,         177113098169u },
+  {                    0u, 11079151463184927232u,         185939160998u },
+  {                    0u,  5239846817488961536u,         166600602004u },
+  {                    0u,  2778806963520143360u,         148284052665u },
+  {                    0u,  6240890740138835968u,         185150639427u },
+  {                    0u, 17250651344549707776u,          67338319364u },
+  {                    0u,  4197354852709302272u,           4935159683u },
+  {                    0u,  9223372036854775808u,         131227539062u },
+  {                    0u,                    0u,         118500000000u },
+  { 17118225092618494573u,            585819067u,                    0u },
+  { 13385738875341807559u,   585819067927980841u,                    0u },
+  {  8272682717439277193u,  5654803392547571318u,             31757315u },
+  { 13402436483369350083u,  2931628102185393332u,           3306547506u },
+  { 10946328903241612536u, 15964697617980212305u,          50158923877u },
+  { 16265808923426731252u,   450380868305846606u,         101865447992u },
+  { 11080374459871185177u, 14631133530814566148u,          56024415195u },
+  {  1240761893433831916u,    31969822783742095u,         219793155338u },
+  {   367264070493390483u, 10437269029385743245u,          10001733087u },
+  {  2863675693461092905u, 15196146496377392433u,         223565805487u },
+  {  7511929581752138999u,  4409099735137480938u,         175823784752u },
+  { 11154557789993845753u, 10644987914903248118u,          48239017775u },
+  {  8325416539745948522u,  3154431617534062973u,          47577065951u },
+  { 17745129874679852617u, 11702056331247960454u,         223171002080u },
+  {  1074820986392253357u, 15575315065965259114u,         224634369744u },
+  {  7820952682162838597u, 10759747609480050226u,         208844339521u },
+  {  8215518006273528603u, 12538236653960743718u,          65583287086u },
+  {  9680426791089900133u, 17857942663978005403u,          46679699170u },
+  { 16128495723604797412u, 11443004154750813211u,         226968081011u },
+  {  2264789053583348885u,  4004313188770806737u,         115620326498u },
+  { 11175458488686298083u, 17134872954824183228u,          98217074252u },
+  { 11026777810412287617u,  2659553912986171234u,          76928883324u },
+  { 16199890034895598640u,  9501854300969137926u,         124144174706u },
+  {  9094320719494763752u, 14528169966301018150u,         114515096553u },
+  {  1250835564687222832u, 18172091996515901778u,         233787573671u },
+  { 15362466642459337025u,  1133541705604751035u,         167985111081u },
+  {  7831109835595423828u, 18280349987988641497u,          41061449418u },
+  { 15426237284335022429u,  9936015874712336386u,         202990979758u },
+  { 15636308361455434548u, 15876720399740689614u,         174538632499u },
+  { 13967173875944980328u,  8618117825152456982u,          51860678737u },
+  { 18245979923595824097u,  8085525680745921564u,          81467189103u },
+  { 11335054479675278263u,  8072355444669730953u,         111438317225u },
+  { 11165339882630461707u,  9395030504766848294u,         169437603265u },
+  { 15944437408299395922u,  3537903114058185903u,         193509305624u },
+  { 15806416348777321161u,  2126094743961928691u,          24191790112u },
+  {  4201030477408556248u,   289185362555601115u,          32115255827u },
+  {  9485474942554588907u, 16909937501450129614u,          19015676769u },
+  { 18238757647663230541u, 14449642060360499058u,          97916689548u },
+  {  4642199687824746379u, 12433818908498244393u,         140783316665u },
+  {  6134575894869364037u, 11884444034578008581u,         185674038673u },
+  { 11524208547121316008u,   988625838444140793u,         145644257002u },
+  {  2734683241527878366u,  1675370907158909973u,         234053593514u },
+  { 10629223456178675171u, 15920186275316934067u,         170090822038u },
+  {  2788042336985254064u,  5600921198503757726u,         150863035027u },
+  { 17285498758066142502u, 10457357161776341741u,         147303626546u },
+  {  5525538192421886436u, 12225356765775740093u,          50566894467u },
+  { 11414325503043801888u,  4486633318598164537u,         131662737918u },
+  {  7246608114685173259u, 10302486602879381361u,         254243220879u },
+  {  1007884269852184608u, 15536428611301239541u,         143558498917u },
+  { 13823717876510029312u, 12026126645955462603u,         101842231482u },
+  { 12487410768239429317u, 14877968141142123551u,         186651937631u },
+  {  3361062421598631942u,   734560801645383190u,          95806536269u },
+  { 17853337379088328475u, 15648943144911081638u,          77039820620u },
+  { 11551561037491869885u, 13664182862003235646u,          76848330907u },
+  { 11480877996635204802u,  3895127525902132786u,         155740736837u },
+  {  5527488381934471912u,  5249187334214137467u,          69211155286u },
+  { 11143438404407726080u, 10642260063359027505u,          86284559015u },
+  {  6472279730688098304u,   783598951897779422u,         167576918074u },
+  {  4561816853579563008u,  5538576558607624843u,          58042478984u },
+  {  2888714464062865408u, 15974581187564609611u,         136300246836u },
+  { 16258276129784201216u,  7474269406918257428u,          52865983781u },
+  {   720575940379279360u,  8045286838779138019u,          37405180956u },
+  {                    0u,  8184246376556341732u,          28436135873u },
+  {                    0u,  1493267152679331840u,         193443668885u },
+  {                    0u, 10179074811222818816u,         149080950174u },
+  {                    0u,  3892499202005008384u,         158551808751u },
+  {                    0u, 10341173215925108736u,         239211012804u },
+  {                    0u,  6230307872002015232u,         196560596123u },
+  {                    0u,  9295429630892703744u,         155337745666u },
+  {                    0u,                    0u,           2503906250u },
+  {                    0u,                    0u,         202000000000u },
+  { 16409970870640346804u,                 8938u,                    0u },
+  {  7721907286269370594u,        8938889586303u,                    0u },
+  { 14300743897882155131u, 10665454627995623288u,                  484u },
+  {  2068482633821123575u, 16803537892767562832u,         228578175453u },
+  {  4922882895416406094u,  8099123106849104444u,         221910921614u },
+  {  9317632875623428410u,  7077413686679401728u,         142439054343u },
+  {  5693844901999766254u, 13536636358372449666u,           7383667364u },
+  { 11569484900262102262u,  7280632235418610318u,         164733822527u },
+  {  3138170119352085637u,  6187823673116858809u,          63394683864u },
+  {   950584692575235243u,  8624343686231740255u,         216335442593u },
+  {  8136430299747162645u,   806211610822132771u,         161467526608u },
+  {  6698711700804594470u, 18388078233202190882u,         208043704818u },
+  { 17401191571004302008u,  7628864426595573600u,         242996819718u },
+  {  4721732028538188150u,  4530799784343874981u,           6413561569u },
+  {  2984214103553086219u,  8561580552078486438u,         225245615148u },
+  { 13128675202005662068u, 13349114951221999594u,          44464124211u },
+  { 14638512997670672834u, 10029144738508991772u,          51723656971u },
+  { 12942085665769692438u, 12601907197916268979u,          11543681025u },
+  { 14131134357119205086u,  1329580921391066941u,           1683150758u },
+  {  8921946894736102919u,  3198179786356761112u,         166072076726u },
+  {  5601522560505809989u, 11406753413634654142u,         182173373673u },
+  {  8602606493507716808u, 11131812960525182090u,         233618361341u },
+  {  8576789731078566487u, 14299636753645227208u,         253603456789u },
+  { 17881118138842658549u, 12964114684643663326u,          21775184861u },
+  { 11624372674432704923u,  5019257593846306316u,         221702786065u },
+  {  6826284072848095635u,  6929086798159998121u,          17272094499u },
+  {  1646466632033733563u, 18359765766933703649u,          35375626547u },
+  { 17871081657060299180u,  9993076234752063198u,          51995284896u },
+  { 15910893124677544709u,  3257189215046584509u,         160541725748u },
+  { 11031217459450580944u,  2905234736672690348u,          52176572581u },
+  { 13554987390037243094u, 12064985302079670056u,         165157493090u },
+  { 15026714590903687870u, 14315096064942799930u,          98654044163u },
+  {  4406379654994689200u, 11943971043551974038u,           3776022912u },
+  { 13596329092861950242u, 12472773152119929647u,         128647483967u },
+  {   284812388227373260u,  7791259796982183085u,          63676150387u },
+  {  9285079159392309382u, 16866829442051086686u,         115422365039u },
+  { 15046108141952711893u,  3702498393844653053u,         111914352656u },
+  { 13795366909944958311u,  2057239613841701716u,          16200712840u },
+  { 12909920641180059961u, 17201969976738286226u,         136111523182u },
+  {  5333762939889788252u, 18271566505443461640u,         110932520660u },
+  {  6411331390005944495u, 18368509115417119804u,         212990503604u },
+  {  1447104583224217723u,  7613923684154518587u,         180995758874u },
+  { 11940049226167932871u, 17984805084714865232u,          26412751629u },
+  {  9772290783590472385u,  4220802739051410373u,          13974958237u },
+  { 16351989577831528444u, 17812459042810815760u,         157228810174u },
+  {  4376738725895725097u, 10629526089664605307u,         190965615339u },
+  { 13851276297739812763u, 17437443267816548473u,         235576227763u },
+  { 12641996203470333509u, 12506371893701049304u,         179945285693u },
+  {  7707081716407945022u, 15737221540003030739u,          61677971778u },
+  {   417638323657040024u,  2358380859011605513u,          66853116489u },
+  { 16438047707692449100u, 10042972713837039706u,          73127848082u },
+  { 14850108107043306316u, 13424397272769642495u,         146544430641u },
+  { 10423290807904720835u,  6867102315755663029u,          49727738034u },
+  { 16951162310302339314u,  8690748404825506734u,         178372266362u },
+  {  2752437506572397322u,   956229930815387710u,         122471126415u },
+  {  3925815842962784589u,  7734449506297687888u,         143051837328u },
+  {  5274166674003605291u, 16332184961683848151u,         144419285347u },
+  {  5538963350863452832u, 15580777817612768828u,          99885369520u },
+  { 16900671634439028736u, 17404245271944696092u,         176844635657u },
+  {  2326997710751662080u, 13201420160494469229u,           9943486026u },
+  { 12327726161625874432u, 16511717657124068078u,          74715650420u },
+  {  5756455743825903616u, 14131292492116594062u,         116895102007u },
+  {  3018537650245074944u, 18429136031865875691u,          55766058900u },
+  { 16717361816799281152u,  2563978348305862197u,         148999045466u },
+  {                    0u, 14239974392147482896u,          90138993544u },
+  {                    0u, 11164201396098998272u,         136771950558u },
+  {                    0u,  7116971104932986880u,         222605212570u },
+  {                    0u, 12437629862867369984u,         154385811776u },
+  {                    0u, 16501893821638901760u,          64674245265u },
+  {                    0u, 10649324268870959104u,         145894569456u },
+  {                    0u,  7205759403792793600u,         240577301025u },
+  {                    0u,                    0u,          33390625000u },
+  {                    0u,                    0u,         232000000000u },
+  { 11997425759292732054u,            136396630u,                    0u },
+  { 11491152661270395161u,   136396630650381753u,                    0u },
+  { 18181063258234881272u,  3016823727048309817u,              7394076u },
+  {  2466921813123869732u, 17405973192644624358u,          28163542341u },
+  {  8430880678232179465u,  8937219978302591747u,          69943579697u },
+  {  6738034873677997533u, 15178463196824222317u,          49484487665u },
+  {  7678250951042929246u, 11979404627460330594u,         241822826138u },
+  {  1853560606315563193u,  2006448052689740002u,         154649404826u },
+  { 14942676593409905118u, 16330465320863239865u,         154108769766u },
+  {  4909892170837638183u, 17136208883957646553u,         230885276298u },
+  { 16871149368312132405u,   140455118208931867u,         138928955745u },
+  { 16096130589333770811u,  3964972929179372247u,          97007614087u },
+  { 12512479187631824282u,  3378050330022776379u,         135214941613u },
+  { 16980304980540557310u,  6065353437512901255u,         173183124475u },
+  {  8640919162749295366u, 12768753059854699889u,         251328803468u },
+  {  7862382415464063513u,  6848720690951013326u,         140692195490u },
+  { 14534157903009925344u, 10953228058585475132u,         162371269892u },
+  { 12627464554215107944u, 15539127852083296166u,           4593775682u },
+  {  2456849734836299173u, 14534853647735598497u,          66842377808u },
+  { 18428252197697827913u,  1506909603576368170u,          80787935995u },
+  {  7244734215936736255u,  5475702579938239025u,         251081689733u },
+  { 14756175050504770087u, 12039747373985783332u,         133296838431u },
+  {  6764116534566945922u, 17572399137760898460u,          31652676012u },
+  {  1588822142405565521u,   869552790852091236u,         172952601666u },
+  { 17053265624843842052u,  4549585778048181804u,          66047138551u },
+  { 16996891591759999207u,  4121918231767210357u,         247246633539u },
+  {  8565556232370585876u,  1558397953312543179u,          67223449635u },
+  { 14464960359145886620u,  6067524298738069781u,          35084480922u },
+  {  5813189542048784035u,  5811095224555517056u,         154328921151u },
+  {  9739241026882027025u,  6440894514158997188u,          63315020103u },
+  { 18175068535675302910u,  4612748874388784257u,          71349161591u },
+  { 10562697212061761911u,  9908101430749813367u,         119250057617u },
+  {  4264834835660801368u, 15150017990912190499u,         145537119254u },
+  { 13019066443690126316u, 17470426264690059239u,          22821284120u },
+  {  1828040177823321846u,  9615161096851907726u,          24947073705u },
+  { 11240369830376975668u,  9227932132124142224u,         169521238927u },
+  {  8886938465302549874u,  4794113194321211621u,         143500247203u },
+  {  9985240313589688325u,   391512698859146347u,         163259889397u },
+  {   722909126956573766u, 17209658878068655842u,         245021223945u },
+  {  5493363474638452381u,  3077364726606876150u,           9932937477u },
+  { 12410535279213120491u,  1952989567673965814u,           5166824276u },
+  { 10543108918366869098u, 11172860676923186449u,          84105871776u },
+  { 12953909016524823995u, 17338078544784947239u,         160605681990u },
+  { 16505942145872588169u,  4593380466519703278u,          70939899121u },
+  { 12428594380392015797u,   786884753602720052u,         241249007654u },
+  {  7528259605829768337u, 17848875822468020539u,          38042657107u },
+  { 18147447600042811311u,  2899664567187130618u,          83967589497u },
+  { 12021069431116183911u,  2973178834961857409u,         121157191131u },
+  { 11819985069665662506u, 11117453141176836727u,         219161176347u },
+  { 10401877114068152814u,  7535238370146462002u,          27602678342u },
+  {  4611631138117837942u, 10246175467290865448u,          70408486090u },
+  { 17200813398607252417u,  1203128834127050464u,         202555446285u },
+  { 14100466137553658767u, 14518048959078919658u,          13065221744u },
+  { 17887776768825509301u,  1553474987376920024u,         112787025011u },
+  { 12632656857970087269u, 14956572380830948369u,         115084214047u },
+  {  8923681664054686256u,  7594162606042048292u,          31810797413u },
+  {  6213926103737837599u, 14461296147288811288u,         101411680379u },
+  {  1233118281776157762u, 18305427728131488265u,         123783948434u },
+  {    30716279628678784u, 10253208939347909876u,         146992339225u },
+  { 15775734650898546688u,  6446028915490812012u,          25555827570u },
+  {   976806005729918976u, 12986063676957432257u,         114349439927u },
+  { 12460098853279891456u,  9769714697972762807u,         183703975922u },
+  {  5635665595421687808u,    97429465146664592u,         242529617295u },
+  {  1805943450575568896u, 16395571728207795868u,         143005281661u },
+  { 11529215046068469760u,  6331668478323650406u,         125888805724u },
+  {                    0u, 18129911846294207040u,          92343240435u },
+  {                    0u,  9890094564876124160u,         243982824490u },
+  {                    0u, 12290856656987750400u,          42536143100u },
+  {                    0u,  8498454992640802816u,         252666288674u },
+  {                    0u,  5341660584200896512u,          34460702168u },
+  {                    0u,  9288674231451648000u,         216289572000u },
+  {                    0u,  1152921504606846976u,         160503540039u },
+  {                    0u,                    0u,          71062500000u },
+  {                    0u,                    0u,         160000000000u },
+  {  4564018338575530435u,                 2081u,                    0u },
+  { 16553437246451512014u,        2081247415929u,                    0u },
+  {  4339777136957372927u, 15212079674427582639u,                  112u },
+  { 18439463366554654697u, 10179808126814248333u,         112824648491u },
+  {  1370067356680643003u,  6066766544199222848u,          43551848504u },
+  {  4210124040914115013u,  6625308131806923532u,          56328880073u },
+  { 10692225626142609720u,  9122786786400665713u,         201359158673u },
+  { 11592856673895384344u, 11932880778639151320u,         145494547262u },
+  { 10284479231227406269u,  3884040911779255011u,          62646882763u },
+  {  4961071383534266431u, 13441817515637357872u,         203210554279u },
+  { 10960611551445686988u, 11628577856022352826u,         167728682387u },
+  { 14616396723115619209u, 13296656925520243654u,         147630386468u },
+  {  1025604265437492803u,  5020720704545399398u,          36720813216u },
+  { 11711588454892179178u, 14121973606499014694u,         160272173814u },
+  {  5580373263251565705u,  3642481034345420114u,         246765553723u },
+  { 14109334653033148931u,  9845536238569696768u,          59197459292u },
+  {  2899414033769399895u, 17655403572195686356u,          92533727588u },
+  { 13233457234892808147u,  8377495365136654029u,         100957101345u },
+  {  3287946691509034862u, 13713682649609025426u,          33454144933u },
+  {  5488480288717445911u,  1367709905452854731u,         165743420226u },
+  { 11687233053874362630u,  9981467701727208680u,          66074143702u },
+  {  6783772100089274577u,  6277920117543306205u,         214541096448u },
+  {  7898291058728402485u,  9344111460418701726u,            340326731u },
+  {  4423684977486598491u,  4918507011364617264u,          75506545297u },
+  {  2750833684599526706u,  6554777203830755259u,         145266632799u },
+  { 15669689830489025709u,  4198262173120265648u,          95355335184u },
+  { 15117307274214954517u,  8080325935698446819u,          16227588248u },
+  {  8148639818575698175u, 12797633874200091733u,         152438035346u },
+  { 13006484426078994901u,  8376502502208665497u,         146693761122u },
+  { 10287496057845513526u,  9891973386793349173u,          98454091110u },
+  { 14159876032966532430u, 14877430279003795462u,         102536244951u },
+  { 15742212196465548019u,  8759933935842067041u,         215806507111u },
+  {  2892220461917134150u,  3753418510388703513u,         103474876970u },
+  {  7487151560715393883u,  2961383332545305985u,          42203473225u },
+  {  7245756744165177933u,  2497674184068629507u,          73160536912u },
+  {  3067122860671533987u, 15244544070742305452u,          80135399188u },
+  {  8135043905834122525u,    45953573565810823u,          20826408390u },
+  { 14730019368921022572u,  3960077421351906445u,         198002491148u },
+  {   495969939682055458u,  3173330011013883118u,          12214676227u },
+  {  5617761407265775598u, 11026266219545759160u,           3172026564u },
+  {  2087044847072781811u,  8886757764964685632u,         196597735089u },
+  { 15929674232061203330u, 13952322129918090479u,         177481752103u },
+  {  8658086469608285873u,  4127250666614902202u,          39756356898u },
+  { 18369871790780313570u, 17649958504065306911u,          34223738706u },
+  {  3545648451947416750u, 13269305359002216873u,          82956806167u },
+  { 13347376792767929959u, 16236593433831947843u,          23719330484u },
+  { 14482164459838203025u, 13580930396682424057u,         180880187493u },
+  {  8899577765623565820u,   421976357197961116u,         101736223712u },
+  {  7095320096604405719u,  2962130818798626533u,         224022875384u },
+  {  2968593824439315788u,  8234383947306356345u,         248160577433u },
+  { 12621408323612585636u,  4380469931801381425u,         153446386848u },
+  {  3954422936414648259u, 15279887469027055916u,         160237465750u },
+  { 17143730087577690191u,  8534542821713755552u,         150828324359u },
+  {  5033045529399041876u,  7814613482565088782u,           7462658493u },
+  { 15857648521994521781u, 13771954404705323224u,         189423631045u },
+  { 16655573486499109541u,  4568173274762548144u,         197746579144u },
+  { 16652154439190075858u,  8105292616250821343u,         200247641169u },
+  { 18016950600164130638u,  2923678426777275612u,          81439388793u },
+  {  2086292996072613910u,  1808633176918384049u,         121158492925u },
+  { 17324462585194799521u, 18118642609460438969u,         253098046200u },
+  { 11079151463184927232u, 18138164175864360870u,         248982213583u },
+  {  5239846817488961536u,  4031433690465792404u,         207983271850u },
+  {  2778806963520143360u,  5012226396942308537u,         170218544458u },
+  {  6240890740138835968u,  7889712298793536835u,          74271713337u },
+  { 17250651344549707776u, 13500762396543628804u,          57427702160u },
+  {  4197354852709302272u,   501020624068841347u,         144731877796u },
+  {  9223372036854775808u,  8370653768288261750u,         164027160382u },
+  {                    0u,   647579990023635200u,          62453774050u },
+  {                    0u, 11106569307181154304u,         226035105381u },
+  {                    0u, 10797461613892861952u,         101602088328u },
+  {                    0u, 17627230675448889344u,         136585331566u },
+  {                    0u, 12197735707942322176u,         110955574089u },
+  {                    0u, 12871287735024877568u,          73661240577u },
+  {                    0u,  4611686018427387904u,           1697753906u },
+  {                    0u,                    0u,          50250000000u },
+  {                    0u,                    0u,         128000000000u },
+  {  5654803392547571318u,             31757315u,                    0u },
+  {  2931628102185393332u,    31757315306547506u,                    0u },
+  { 15964697617980212305u,  9451803574512021605u,              1721567u },
+  {   450380868305846606u,  8662766454758138424u,         223512383298u },
+  { 14631133530814566148u,  9207992007314947035u,          66469609510u },
+  {    31969822783742095u, 17118602861291201802u,          38499166246u },
+  { 10437269029385743245u, 11186560605745599967u,          38928001320u },
+  { 15196146496377392433u, 10505549821532796847u,          40606424665u },
+  {  4409099735137480938u, 18133667530488679216u,          89569506996u },
+  { 10644987914903248118u, 10778135771244330799u,         180983028086u },
+  {  3154431617534062973u, 17087985777033767391u,         118584283910u },
+  { 11702056331247960454u,  2639185991757283040u,           6926341565u },
+  { 15575315065965259114u,  5401720287293896400u,         189143070559u },
+  { 10759747609480050226u,  9816495392633895233u,          95292827843u },
+  { 12538236653960743718u, 10042051500090034990u,         195532153281u },
+  { 17857942663978005403u, 11629689537856384738u,         193544380702u },
+  { 11443004154750813211u,  2099086731766010483u,          30630446733u },
+  {  4004313188770806737u, 13665537898516458594u,         141113791719u },
+  { 17134872954824183228u, 16375672064669490764u,         231740810293u },
+  {  2659553912986171234u,  7770550512184564348u,          53887726961u },
+  {  9501854300969137926u,  6197048880720627314u,         113421242387u },
+  { 14528169966301018150u, 17963594118523106281u,          19335942692u },
+  { 18172091996515901778u,  8255454642407818663u,          36973808388u },
+  {  1133541705604751035u, 16744201957549498409u,           4447529092u },
+  { 18280349987988641497u, 17442505417202859722u,         132907705006u },
+  {  9936015874712336386u,  6383975767786687150u,         174945560113u },
+  { 15876720399740689614u, 15245442964998335795u,          49346076019u },
+  {  8618117825152456982u,  2910016124519524433u,         115826457119u },
+  {  8085525680745921564u,  3847913871169988463u,          31157752290u },
+  {  8072355444669730953u, 17210451512590059177u,         226208595828u },
+  {  9395030504766848294u, 17899408909991454145u,         116932980445u },
+  {  3537903114058185903u,  5920601932753251608u,         221970328901u },
+  {  2126094743961928691u, 16521781895108979744u,          69320956473u },
+  {   289185362555601115u,  3697493405554698771u,          57895647591u },
+  { 16909937501450129614u,  2816108280295732065u,         103200441519u },
+  { 14449642060360499058u, 14251078772056398988u,         175152661535u },
+  { 12433818908498244393u,  4543066550096031417u,          31772552528u },
+  { 11884444034578008581u,  3099369389734296977u,          80246280131u },
+  {   988625838444140793u,  5243484113636490986u,         195168017151u },
+  {  1675370907158909973u,  6823370511605197226u,         255284249843u },
+  { 15920186275316934067u, 11396290277624641942u,         243369895656u },
+  {  5600921198503757726u, 15934361408437566099u,         232617794133u },
+  { 10457357161776341741u, 14939272230935131954u,          85863803462u },
+  { 12225356765775740093u,  7500666177940329347u,          70809859570u },
+  {  4486633318598164537u,  4806714453065462270u,         242406611928u },
+  { 10302486602879381361u, 11557851247268441487u,         216260572512u },
+  { 15536428611301239541u, 10655523157206817381u,          96626552371u },
+  { 12026126645955462603u, 14769600176490881210u,          51577637067u },
+  { 14877968141142123551u, 16688495540925795167u,         203800661629u },
+  {   734560801645383190u,   909793965395524173u,         125904685156u },
+  { 15648943144911081638u, 12724590949761703756u,         100049320029u },
+  { 13664182862003235646u, 10810739657314826395u,          93689801457u },
+  {  3895127525902132786u,  2431218615388671301u,         241586051371u },
+  {  5249187334214137467u,  4235001167959059286u,          43131796625u },
+  { 10642260063359027505u,  6253317787396334247u,         145229579873u },
+  {   783598951897779422u,  9534525563070371898u,          97338993036u },
+  {  5538576558607624843u,  8392783992374030728u,         140516867666u },
+  { 15974581187564609611u, 16356257019231647540u,          82454973731u },
+  {  7474269406918257428u, 12896334001521091877u,          35886674469u },
+  {  8045286838779138019u,  1427636373320877084u,          37699111667u },
+  {  8184246376556341732u, 16116755731295043521u,         243077392322u },
+  {  1493267152679331840u, 15945633911163986837u,         194873691078u },
+  { 10179074811222818816u,  7510154241072743838u,         198864414546u },
+  {  3892499202005008384u,  3571560509790395119u,          82407126277u },
+  { 10341173215925108736u,  3576991649007035076u,           5193614683u },
+  {  6230307872002015232u, 15509961892750732443u,          91193909105u },
+  {  9295429630892703744u, 17789791359353349378u,         113840796718u },
+  {                    0u, 18331227331079738314u,          46964386521u },
+  {                    0u, 15386712883100476416u,         217993737824u },
+  {                    0u, 14082462055028752384u,          96834115376u },
+  {                    0u, 12919043128765186048u,          48763411797u },
+  {                    0u,  6125373368465096704u,          85700342731u },
+  {                    0u, 12335992698065387520u,         203332057155u },
+  {                    0u,  2774217370460225536u,          67668735504u },
+  {                    0u,                    0u,          16150390625u },
+  {                    0u,                    0u,          97000000000u },
+  { 10665454627995623288u,                  484u,                    0u },
+  { 16803537892767562832u,         484578175453u,                    0u },
+  {  8099123106849104444u,  4962829537462579598u,                   26u },
+  {  7077413686679401728u,  5711259460785241095u,          26269035528u },
+  { 13536636358372449666u, 13845894607204897444u,           8309607995u },
+  {  7280632235418610318u, 12116633056637003327u,          59750587450u },
+  {  6187823673116858809u,  2965791047992089560u,          58656843994u },
+  {  8624343686231740255u, 16021997451315962529u,         218160775854u },
+  {   806211610822132771u,  3942052271663803856u,         174868554222u },
+  { 18388078233202190882u, 15669876414782439922u,         238213699081u },
+  {  7628864426595573600u, 10594415915406145286u,           9849465702u },
+  {  4530799784343874981u, 10789820553031921377u,         102574324437u },
+  {  8561580552078486438u,  3989990218583987244u,         213584917344u },
+  { 13349114951221999594u,  2937341169808224563u,          96216297803u },
+  { 10029144738508991772u, 16267436558584536843u,          75159233583u },
+  { 12601907197916268979u, 16221580362814625793u,          47881859502u },
+  {  1329580921391066941u,  9695437602320209830u,         174879373633u },
+  {  3198179786356761112u, 10729753156793715126u,          65525590725u },
+  { 11406753413634654142u,  2609241432056861929u,         197581661084u },
+  { 11131812960525182090u,  8462663743997037565u,         156141447261u },
+  { 14299636753645227208u, 14993422143908194069u,          93458761920u },
+  { 12964114684643663326u,  1307443894537745373u,         192812795043u },
+  {  5019257593846306316u, 10017257439419829265u,         163070876675u },
+  {  6929086798159998121u, 16754772009970777891u,           3543036613u },
+  { 18359765766933703649u, 11722573031602862387u,         197908278010u },
+  {  9993076234752063198u,  7363764277467092384u,         250635481957u },
+  {  3257189215046584509u,  6733958494847390772u,         101399190461u },
+  {  2905234736672690348u,  8799796600227451045u,         189365048621u },
+  { 12064985302079670056u, 10512023194742249826u,          45477037929u },
+  { 14315096064942799930u,  4572542132337197059u,         105569857919u },
+  { 11943971043551974038u, 12600500455757416832u,         127247878005u },
+  { 12472773152119929647u,  7873789864743195199u,         117683074498u },
+  {  7791259796982183085u, 15724851676325671539u,         194426839003u },
+  { 16866829442051086686u,  8748017220462413167u,         219852445917u },
+  {  3702498393844653053u, 14172589522760466448u,         221474230963u },
+  {  2057239613841701716u,  9520545591489413768u,         179768297617u },
+  { 17201969976738286226u, 12488551088392570222u,         145516109810u },
+  { 18271566505443461640u,  1135798823651241684u,         242677005711u },
+  { 18368509115417119804u, 11168725610120161972u,         143061571777u },
+  {  7613923684154518587u,  9580104948718508826u,         193605457828u },
+  { 17984805084714865232u, 16638722716909738765u,         164519338529u },
+  {  4220802739051410373u, 15732724012348272797u,          33901986965u },
+  { 17812459042810815760u, 12269722190021214142u,         149852872677u },
+  { 10629526089664605307u, 13110655916311972587u,         229665142972u },
+  { 17437443267816548473u,  6618112997062866867u,         188710730081u },
+  { 12506371893701049304u,  8457936459015989309u,          97358768624u },
+  { 15737221540003030739u,  3329167139937134914u,         240458505654u },
+  {  2358380859011605513u,  5245511557216705097u,         182180474512u },
+  { 10042972713837039706u,  5655931353280440466u,         144284359751u },
+  { 13424397272769642495u,   604622132328697393u,          71306608653u },
+  {  6867102315755663029u,  8673282619234652338u,          13032776631u },
+  {  8690748404825506734u, 16929477433058445690u,         183470179592u },
+  {   956229930815387710u, 11036952409253549455u,           8917748810u },
+  {  7734449506297687888u, 18199392190170386320u,          74598314388u },
+  { 16332184961683848151u,  9683116091880335715u,         148986591027u },
+  { 15580777817612768828u,  2993913337608915120u,          51524922775u },
+  { 17404245271944696092u,  4490779842162392585u,         151162300367u },
+  { 13201420160494469229u,   946849923353644618u,         207243445663u },
+  { 16511717657124068078u,  3613491058474899828u,         159051328837u },
+  { 14131292492116594062u, 14624054199004410935u,          69195887742u },
+  { 18429136031865875691u, 12088470271991908244u,         126792771566u },
+  {  2563978348305862197u, 10071980927725011290u,         238655317286u },
+  { 14239974392147482896u,  2833441711428854664u,          38546003180u },
+  { 11164201396098998272u, 17655572411864340446u,         236153601182u },
+  {  7116971104932986880u,  4997642792058747802u,         158957110498u },
+  { 12437629862867369984u, 11489200787635734848u,         226270922758u },
+  { 16501893821638901760u, 12983586226429536913u,           6622830822u },
+  { 10649324268870959104u, 12311150768725063152u,         230703841619u },
+  {  7205759403792793600u,  8530052476845967905u,          83667388820u },
+  {                    0u,  6282736361499820264u,         148462415071u },
+  {                    0u, 11337164765929082880u,         223340587820u },
+  {                    0u,  8343856200414134272u,          44614588933u },
+  {                    0u, 17889330377156198400u,           5452321350u },
+  {                    0u, 17730714064155312128u,          70969782542u },
+  {                    0u,  7449235258647511040u,          14961183935u },
+  {                    0u,  9943947977234055168u,         191403823852u },
+  {                    0u,                    0u,         236539062500u },
+  {                    0u,                    0u,         228000000000u },
+  {  3016823727048309817u,              7394076u,                    0u },
+  { 17405973192644624358u,     7394076163542341u,                    0u },
+  {  8937219978302591747u, 12396245121240683569u,               400833u },
+  { 15178463196824222317u, 10248996648596888561u,         193672001794u },
+  { 11979404627460330594u, 11257495103713935002u,           2555599221u },
+  {  2006448052689740002u,  7555396579247433114u,         117610270032u },
+  { 16330465320863239865u,  4805022328730367462u,          80409578869u },
+  { 17136208883957646553u,  7056637817080232586u,         117260480782u },
+  {   140455118208931867u, 10811411483818434913u,          14382541102u },
+  {  3964972929179372247u, 16962406704495245447u,          46586087790u },
+  {  3378050330022776379u, 18074517319117194669u,         110919533909u },
+  {  6065353437512901255u,  3702019776117654523u,          85979821547u },
+  { 12768753059854699889u,  3551977551381082764u,         235200686894u },
+  {  6848720690951013326u, 16442608985936005282u,          46192553088u },
+  { 10953228058585475132u,  3580046275479139588u,         128891355619u },
+  { 15539127852083296166u,  8737412692712715330u,         227194074697u },
+  { 14534853647735598497u,  3082033243045084752u,          73473656091u },
+  {  1506909603576368170u, 16401023756841128699u,          27167077356u },
+  {  5475702579938239025u,  7520296082779572869u,         236889101279u },
+  { 12039747373985783332u,  9854104766152464159u,         223407676067u },
+  { 17572399137760898460u, 14169188802648310188u,         163534192089u },
+  {   869552790852091236u,  2018609909210367042u,         217768113264u },
+  {  4549585778048181804u,  8270271948267674359u,         112109429062u },
+  {  4121918231767210357u, 12320338602894572099u,          70448332340u },
+  {  1558397953312543179u, 17538536685990080547u,          52667886893u },
+  {  6067524298738069781u, 15833914616956760474u,          45950765978u },
+  {  5811095224555517056u,  6137696141415969855u,         154858358231u },
+  {  6440894514158997188u,  9757490468419438919u,         215332725174u },
+  {  4612748874388784257u,  3566639201356598903u,         182528954618u },
+  {  9908101430749813367u,  9760900035773954449u,         250193347898u },
+  { 15150017990912190499u,  3873778773990716438u,          58529139451u },
+  { 17470426264690059239u,  2295668377270167832u,         251209997968u },
+  {  9615161096851907726u,  1791721710912807593u,         144124448432u },
+  {  9227932132124142224u, 10571009006922683279u,         176097129428u },
+  {  4794113194321211621u,  9840791932778184867u,         212573055546u },
+  {   391512698859146347u, 11525464956561274613u,          58533470399u },
+  { 17209658878068655842u,  4435781488897895433u,         191624796707u },
+  {  3077364726606876150u,  6395563367070996741u,          35240464196u },
+  {  1952989567673965814u, 15538690795135662932u,          68346704184u },
+  { 11172860676923186449u, 16294558813563371936u,          56842354115u },
+  { 17338078544784947239u,  4942096228426070342u,         195883329803u },
+  {  4593380466519703278u,  6910116424372647153u,          11267911573u },
+  {   786884753602720052u, 17923400669760829478u,         149374598161u },
+  { 17848875822468020539u,  4134686917293039955u,          17971629497u },
+  {  2899664567187130618u, 16857102463116098681u,         185224141826u },
+  {  2973178834961857409u, 11364321508775167451u,           2913825355u },
+  { 11117453141176836727u,  7966947780972783899u,          75616061103u },
+  {  7535238370146462002u, 11261055695926686278u,         175431889104u },
+  { 10246175467290865448u,  9227040437353594058u,         208610463052u },
+  {  1203128834127050464u,  7185344074282882061u,          76500198864u },
+  { 14518048959078919658u, 14197856148610578032u,         208389518282u },
+  {  1553474987376920024u,   885688687260429427u,         202769667324u },
+  { 14956572380830948369u, 17407816160380305183u,         252048013279u },
+  {  7594162606042048292u, 17812728703806357349u,         223943679604u },
+  { 14461296147288811288u, 17120198191964319867u,         116965629957u },
+  { 18305427728131488265u, 12091952048375408786u,           5928087803u },
+  { 10253208939347909876u,   405056939269888281u,         251655506034u },
+  {  6446028915490812012u, 12485440679452408690u,         114021958180u },
+  { 12986063676957432257u,  8394369900823444407u,          36676837095u },
+  {  9769714697972762807u,  2877421667354294258u,         231455059704u },
+  {    97429465146664592u,  2676980714750756239u,         248155985341u },
+  { 16395571728207795868u,  6119309228579057021u,         189145119415u },
+  {  6331668478323650406u, 18203256146533333852u,         183331728417u },
+  { 18129911846294207040u,   351919978865493747u,          33986800493u },
+  {  9890094564876124160u,  5190010931882390570u,         109019077620u },
+  { 12290856656987750400u,  6982466386088036604u,         244281351056u },
+  {  8498454992640802816u,  4707293888784996898u,         144378520261u },
+  {  5341660584200896512u,   690306801165964760u,         197255182913u },
+  {  9288674231451648000u, 12456770961278956704u,          65037421606u },
+  {  1152921504606846976u, 16946092489294063943u,          38675282906u },
+  {                    0u, 11098404173866185376u,         218918649514u },
+  {                    0u, 15152070965853306880u,         170601645695u },
+  {                    0u, 17370091362040414208u,         127821395412u },
+  {                    0u, 10141938552171134976u,         212941634539u },
+  {                    0u, 10586988556645826560u,         235549795590u },
+  {                    0u, 12169852093061922816u,           6573921799u },
+  {                    0u, 16717361816799281152u,           7659729003u },
+  {                    0u,                    0u,         107906250000u },
+  {                    0u,                    0u,          16000000000u },
+  { 15212079674427582639u,                  112u,                    0u },
+  { 10179808126814248333u,         112824648491u,                    0u },
+  {  6066766544199222848u,  2144184049294538808u,                    6u },
+  {  6625308131806923532u,  4108002197393276873u,           6116236450u },
+  {  9122786786400665713u,  6446230217393892753u,         162222695245u },
+  { 11932880778639151320u,  5571068025259989822u,          77349450840u },
+  {  3884040911779255011u, 14804812668872528331u,          88302008202u },
+  { 13441817515637357872u, 17369928488562523047u,         138802570502u },
+  { 11628577856022352826u,  2967474173531035027u,           6941625710u },
+  { 13296656925520243654u,  5291425437992807716u,         110160867097u },
+  {  5020720704545399398u, 14219547193739388064u,          25286848747u },
+  { 14121973606499014694u, 17720313647158217462u,         235770843197u },
+  {  3642481034345420114u, 12334850628290578491u,          61960620127u },
+  {  9845536238569696768u,  7818499847417334620u,          95668673592u },
+  { 17655403572195686356u,   136007040922198372u,          56423841726u },
+  {  8377495365136654029u,  8523477092112604449u,         190007372956u },
+  { 13713682649609025426u,   367934822655966629u,         156462058619u },
+  {  1367709905452854731u, 12964987687054730050u,         123019945786u },
+  {  9981467701727208680u, 15267036012420885462u,          58702833390u },
+  {  6277920117543306205u, 11142900264750765568u,         238827627680u },
+  {  9344111460418701726u, 13680181547777718603u,         160604057833u },
+  {  4918507011364617264u, 13001922925761426065u,         233741604127u },
+  {  6554777203830755259u,  2397730045956515935u,          31704835654u },
+  {  4198262173120265648u,  4482395522588406288u,          70129981206u },
+  {  8080325935698446819u,  3255525722490493080u,          22242991148u },
+  { 12797633874200091733u,   836222287193822098u,          44176482403u },
+  {  8376502502208665497u,   420898743993182306u,          99045331701u },
+  {  9891973386793349173u, 11652649973356574054u,         245022816966u },
+  { 14877430279003795462u, 15058402726661910231u,         198631691420u },
+  {  8759933935842067041u,  9600134495208339559u,         156816317647u },
+  {  3753418510388703513u, 14626343323989004842u,         207520424333u },
+  {  2961383332545305985u,  6813981265331086665u,         141792895660u },
+  {  2497674184068629507u, 10281745288790487888u,         172369386664u },
+  { 15244544070742305452u, 17569829347075761940u,         168557374528u },
+  {    45953573565810823u,  7654580675237889478u,          64952462357u },
+  {  3960077421351906445u, 16194838649686212364u,          21414955649u },
+  {  3173330011013883118u,  6495102772252453635u,         129877923962u },
+  { 11026266219545759160u, 14935159852819761348u,         122352100226u },
+  {  8886757764964685632u, 17381879863441579697u,         130809636637u },
+  { 13952322129918090479u,  9062335510435372583u,          29942273595u },
+  {  4127250666614902202u,  7569219009130126626u,          59491270192u },
+  { 17649958504065306911u, 12652124168176193362u,          48410328184u },
+  { 13269305359002216873u,  8940200224697247767u,         120685873025u },
+  { 16236593433831947843u,  5600570701927432884u,         129484649225u },
+  { 13580930396682424057u,  2018432801986093157u,           9303607546u },
+  {   421976357197961116u,  8235849749361824736u,         250109419461u },
+  {  2962130818798626533u,  9705097287982370040u,         197446466309u },
+  {  8234383947306356345u,  3517483139049842585u,           5526114378u },
+  {  4380469931801381425u,   958281614186777760u,          74190683143u },
+  { 15279887469027055916u,  7336473432636108950u,           7051948550u },
+  {  8534542821713755552u, 12955383920176764423u,           6397711021u },
+  {  7814613482565088782u, 10735469126281273789u,         173702312769u },
+  { 13771954404705323224u,  8637888232514730693u,          65581970947u },
+  {  4568173274762548144u,  6806336737533581000u,           3468260859u },
+  {  8105292616250821343u, 16142569672872330321u,         251368972253u },
+  {  2923678426777275612u,  8141285259947963513u,         221875090455u },
+  {  1808633176918384049u,  5220241098754220797u,          23441339958u },
+  { 18118642609460438969u,   154438799943119608u,          54282989837u },
+  { 18138164175864360870u,  2226876628677628879u,          13008372144u },
+  {  4031433690465792404u, 17219557081221357482u,         176120719223u },
+  {  5012226396942308537u, 15401507148161015114u,         119933474059u },
+  {  7889712298793536835u,  8842629766613985337u,          11834917375u },
+  { 13500762396543628804u,  3180100571546071440u,         255479359920u },
+  {   501020624068841347u,  7740848704392475044u,         176172393597u },
+  {  8370653768288261750u,  2014314126623495998u,         125419632249u },
+  {   647579990023635200u, 11209566016506885858u,         121109196187u },
+  { 11106569307181154304u,  7117166613733441125u,         155607671791u },
+  { 10797461613892861952u,  4197646860931880328u,         239385822375u },
+  { 17627230675448889344u,  5487263271238026094u,         167227554892u },
+  { 12197735707942322176u, 18148076225293562697u,          76297465137u },
+  { 12871287735024877568u,  9127276943027950849u,          49983809183u },
+  {  4611686018427387904u,  9691696125379324722u,         159494790674u },
+  {                    0u, 13102362262487705216u,          18525387899u },
+  {                    0u,  8929385439893192704u,         123710280481u },
+  {                    0u, 11891353410743566336u,          33484062954u },
+  {                    0u,  1587423090877399040u,         234644631560u },
+  {                    0u,  3489137423026225152u,           8086054378u },
+  {                    0u, 13046928120492326912u,         234189146518u },
+  {                    0u, 11529215046068469760u,         150707275390u },
+  {                    0u,                    0u,         126625000000u },
+  {                    0u,                    0u,          64000000000u },
+  {  9451803574512021605u,              1721567u,                    0u },
+  {  8662766454758138424u,     1721567512383298u,                    0u },
+  {  9207992007314947035u,  6674960280855494694u,                93326u },
+  { 17118602861291201802u, 16378845781483497510u,         142361850321u },
+  { 11186560605745599967u, 17606907750956804392u,         209887899008u },
+  { 10505549821532796847u, 13225609159240506969u,         128954472381u },
+  { 18133667530488679216u,  2668084873338435252u,         189716961709u },
+  { 10778135771244330799u, 14802814305275861366u,         173144637170u },
+  { 17087985777033767391u,  8005510553372365574u,         242802462171u },
+  {  2639185991757283040u, 12748500143273514429u,         219433979596u },
+  {  5401720287293896400u, 10393733905569036127u,         204691097577u },
+  {  9816495392633895233u,   603389089974790339u,         233563445444u },
+  { 10042051500090034990u,  2033494532597735873u,         196032709788u },
+  { 11629689537856384738u,  9204796763694620958u,         156110235959u },
+  {  2099086731766010483u,  7826260310402107021u,          55498993032u },
+  { 13665537898516458594u, 10122690201685169383u,         136424262421u },
+  { 16375672064669490764u,  7438455564568110133u,          21548752135u },
+  {  7770550512184564348u,  2805412574380520817u,           7403239484u },
+  {  6197048880720627314u,  7250965427231182867u,          60152081720u },
+  { 17963594118523106281u,  8136242944826085924u,          56393075623u },
+  {  8255454642407818663u, 15357191647956011780u,         167441066613u },
+  { 16744201957549498409u,  7369614426695395460u,         117832515027u },
+  { 17442505417202859722u, 10886957545142526638u,         211399507598u },
+  {  6383975767786687150u,  2030047207417538097u,         142590183151u },
+  { 15245442964998335795u, 11557093828502314355u,         239110049079u },
+  {  2910016124519524433u, 15201062539664128543u,          55626511311u },
+  {  3847913871169988463u,  8846936323343880674u,         207824051251u },
+  { 17210451512590059177u,  1485291750116245364u,          51479593379u },
+  { 17899408909991454145u,  2076024439668322013u,         163080517827u },
+  {  5920601932753251608u,  7029497773682748741u,         195112541510u },
+  { 16521781895108979744u, 16333533921668749881u,          70381069837u },
+  {  3697493405554698771u,  2065057316131928423u,          13885442648u },
+  {  2816108280295732065u,  7800502648925570223u,          88111946981u },
+  { 14251078772056398988u, 17011619967093802015u,         229422866095u },
+  {  4543066550096031417u,  5368819344429198672u,         175922201766u },
+  {  3099369389734296977u, 15598879366754275267u,         166291044279u },
+  {  5243484113636490986u, 16393893486035835647u,         183845616944u },
+  {  6823370511605197226u, 12042046205096920307u,          48888714746u },
+  { 11396290277624641942u, 15437070428008474344u,         250652800632u },
+  { 15934361408437566099u, 13704569163204647509u,         120836845264u },
+  { 14939272230935131954u, 18192483750856993350u,         208742926182u },
+  {  7500666177940329347u,  5152535865317963250u,         102986216520u },
+  {  4806714453065462270u, 17512614083933854680u,          72279319528u },
+  { 11557851247268441487u, 14481918350603613536u,         232949360711u },
+  { 10655523157206817381u, 16124419709964004915u,          71785066366u },
+  { 14769600176490881210u, 18088011566435813579u,         126874106543u },
+  { 16688495540925795167u, 15008862380698848893u,         175980553071u },
+  {   909793965395524173u, 18160498644611827812u,         111813632059u },
+  { 12724590949761703756u,  3604680497457231965u,          59984482604u },
+  { 10810739657314826395u,  5957615565551495921u,          44195410121u },
+  {  2431218615388671301u, 17528455034961565995u,         201322962986u },
+  {  4235001167959059286u,  8503772325120113809u,          42950219451u },
+  {  6253317787396334247u,  8501492578048509537u,         187460990421u },
+  {  9534525563070371898u,  2296237701094386060u,         213460866836u },
+  {  8392783992374030728u,  3753593040591076946u,          20124479295u },
+  { 16356257019231647540u,  8518075399775653155u,          63203482686u },
+  { 12896334001521091877u, 12757855675959554597u,          62461765792u },
+  {  1427636373320877084u,   121631169379748595u,         160691604742u },
+  { 16116755731295043521u, 16679062494579173314u,           6006593638u },
+  { 15945633911163986837u, 10739912744743898054u,         102904173789u },
+  {  7510154241072743838u,  9367340677776287570u,         221582211836u },
+  {  3571560509790395119u, 12227321512794715397u,         252507804555u },
+  {  3576991649007035076u,  7241061891859170651u,         139662844427u },
+  { 15509961892750732443u, 13148571323079237489u,          11392538751u },
+  { 17789791359353349378u, 12509763434355012654u,         127712785479u },
+  { 18331227331079738314u, 11812768946960181977u,          71678155634u },
+  { 15386712883100476416u, 14170358803552564832u,         114640371487u },
+  { 14082462055028752384u, 18179989524780635952u,          31768176689u },
+  { 12919043128765186048u, 17091718978514754901u,          49985539206u },
+  {  6125373368465096704u,  7394768384359232459u,         134926543942u },
+  { 12335992698065387520u,  6778628272692852803u,          70400871197u },
+  {  2774217370460225536u, 18193335045875234320u,          29367470174u },
+  {                    0u,  1378519212560967521u,          94986262669u },
+  {                    0u,  4677732610631043584u,         141074729676u },
+  {                    0u, 17296098591070486528u,         204253580392u },
+  {                    0u,  7343735382392963072u,         104937623383u },
+  {                    0u, 14525996728454217728u,          87398104692u },
+  {                    0u,  9691359370008330240u,         116787455860u },
+  {                    0u,  3044433348102455296u,         116525369644u },
+  {                    0u,  9223372036854775808u,          44165039062u },
+  {                    0u,                    0u,         214500000000u },
+  {  4962829537462579598u,                   26u,                    0u },
+  {  5711259460785241095u,          26269035528u,                    0u },
+  { 13845894607204897444u,  7822291454600056379u,                    1u },
+  { 12116633056637003327u,  8201586317771250746u,           1424047269u },
+  {  2965791047992089560u,  3278889188817135834u,         165444608885u },
+  { 16021997451315962529u,  1710725240251040430u,         117177748939u },
+  {  3942052271663803856u,  1850175733663425006u,         203092738601u },
+  { 15669876414782439922u,  9147599666163914249u,          41100298227u },
+  { 10594415915406145286u, 10221885933644344166u,         243495892371u },
+  { 10789820553031921377u, 14901479793736678101u,         147554129546u },
+  {  3989990218583987244u,  5181831442059703136u,         138807810838u },
+  {  2937341169808224563u,  6396246577759793483u,          22280907645u },
+  { 16267436558584536843u, 14167229556464870447u,         125346741221u },
+  { 16221580362814625793u,  2969982933326311854u,         229768007053u },
+  {  9695437602320209830u,  7892677766222018881u,         141161003097u },
+  { 10729753156793715126u,   798698968922663621u,          89427862919u },
+  {  2609241432056861929u, 15926812109043458972u,         135043297557u },
+  {  8462663743997037565u,  8663842590352697437u,          21863394214u },
+  { 14993422143908194069u, 17093523026636671168u,         166469667847u },
+  {  1307443894537745373u,   839764004742743203u,           7926641740u },
+  { 10017257439419829265u, 16894643909298232323u,          76045523697u },
+  { 16754772009970777891u,  9066702926218949317u,         241915860481u },
+  { 11722573031602862387u,  9119392417260546810u,           1491506950u },
+  {  7363764277467092384u,  9723021096578315109u,           6494363253u },
+  {  6733958494847390772u, 14787464248751217597u,         117527086029u },
+  {  8799796600227451045u,  3733434565920249133u,         205801630043u },
+  { 10512023194742249826u,  6643788868836820841u,          91202389893u },
+  {  4572542132337197059u,  4729646697422664063u,         133360160516u },
+  { 12600500455757416832u,  4090144564201555829u,           4256394661u },
+  {  7873789864743195199u,  2109480737093400002u,         165221727181u },
+  { 15724851676325671539u, 16577155033369419739u,         205114355179u },
+  {  8748017220462413167u,   745377248603805917u,         235898649375u },
+  { 14172589522760466448u, 11305561465807999667u,          31040406981u },
+  {  9520545591489413768u,  2211245518782892177u,         197612875715u },
+  { 12488551088392570222u, 14170095199249735666u,         195119871859u },
+  {  1135798823651241684u, 17849973668116118927u,         115768162399u },
+  { 11168725610120161972u,  9020960204585720001u,          95967649011u },
+  {  9580104948718508826u, 10807134002871850916u,         243489027232u },
+  { 16638722716909738765u,  3925122626254791201u,         160585855908u },
+  { 15732724012348272797u, 17208463291312718997u,         164212781323u },
+  { 12269722190021214142u,  5145077219589447653u,          11932872664u },
+  { 13110655916311972587u, 17602397765035489468u,         216278915194u },
+  {  6618112997062866867u, 16422643262490753377u,         122954227894u },
+  {  8457936459015989309u,  2902509461400906224u,         182890273275u },
+  {  3329167139937134914u,  3422418805967265206u,         251157345353u },
+  {  5245511557216705097u,  4228874576277237392u,          73185529695u },
+  {  5655931353280440466u,  2553488530807495751u,          95229247750u },
+  {   604622132328697393u, 11546099176912486413u,           6138424890u },
+  {  8673282619234652338u, 10460791037534167991u,          58625915290u },
+  { 16929477433058445690u,  8127117908566000904u,         154567080618u },
+  { 11036952409253549455u, 11541304458088287306u,         170440571944u },
+  { 18199392190170386320u,  6249718665174839700u,          40625655368u },
+  {  9683116091880335715u, 13102508413386290995u,          72338797927u },
+  {  2993913337608915120u,  6274675218640661911u,         103710288404u },
+  {  4490779842162392585u,  3404497118599817167u,          20340150825u },
+  {   946849923353644618u, 11258566093988562335u,          41184558158u },
+  {  3613491058474899828u, 16762592482501635397u,          78610328090u },
+  { 14624054199004410935u,  5550125446725071998u,          26908701959u },
+  { 12088470271991908244u,  6370033225258510318u,           7300872903u },
+  { 10071980927725011290u,  1503521728674735398u,         199345320193u },
+  {  2833441711428854664u,  4250415082606384364u,           1081506076u },
+  { 17655572411864340446u,  6020091901030562974u,          28230415463u },
+  {  4997642792058747802u, 16288222967151527138u,         103326349835u },
+  { 11489200787635734848u,  6377016228656203782u,          11882986336u },
+  { 12983586226429536913u,  8378856515587563750u,          96345698742u },
+  { 12311150768725063152u, 15812881490200838483u,         182454218721u },
+  {  8530052476845967905u,  4548570371183413652u,         225857218023u },
+  {  6282736361499820264u, 16731431495283420383u,         231246578493u },
+  { 11337164765929082880u, 14737727629551135532u,          61907012718u },
+  {  8343856200414134272u, 12413722258104293893u,         110798933815u },
+  { 17889330377156198400u,   800899742400762438u,          55672949232u },
+  { 17730714064155312128u,   603197008376033550u,         240043416862u },
+  {  7449235258647511040u,  6380777281587743935u,          30032699375u },
+  {  9943947977234055168u, 10001440249018225388u,         239345902629u },
+  {                    0u,  5505914461980436708u,          37542179162u },
+  {                    0u,  1105464290051876864u,          90298476221u },
+  {                    0u,  4500443576769970176u,         189059927339u },
+  {                    0u,  2843045143185981440u,          43243969535u },
+  {                    0u,   660949699682893824u,         255154121786u },
+  {                    0u,   276549164618219520u,          58035830155u },
+  {                    0u,  4683743612465315840u,         139014991760u },
+  {                    0u,                    0u,         144253906250u },
+  {                    0u,                    0u,          74000000000u },
+  { 12396245121240683569u,               400833u,                    0u },
+  { 10248996648596888561u,      400833672001794u,                    0u },
+  { 11257495103713935002u,  4370024159708535157u,                21729u },
+  {  7555396579247433114u,  7166684413908503888u,         225236899484u },
+  {  4805022328730367462u, 10217286283215687029u,         156388506740u },
+  {  7056637817080232586u,  4767369911989629198u,         116553880199u },
+  { 10811411483818434913u, 14407999214182082862u,         135258439640u },
+  { 16962406704495245447u,  8472271297615317358u,         216781059202u },
+  { 18074517319117194669u,  6236024012584764757u,         130459282747u },
+  {  3702019776117654523u,  1951826556984620523u,          59338055539u },
+  {  3551977551381082764u, 12357130551551830830u,         115105808729u },
+  { 16442608985936005282u,  8927758011099278464u,          89669881389u },
+  {  3580046275479139588u, 10199854049407140323u,          45483974731u },
+  {  8737412692712715330u, 17895455027038549577u,          75552935195u },
+  {  3082033243045084752u, 16539200343720527131u,          27970114560u },
+  { 16401023756841128699u,  3536976106235802604u,            896591847u },
+  {  7520296082779572869u, 16980391644793590751u,         231191739858u },
+  {  9854104766152464159u, 10090294316609084067u,         210920508875u },
+  { 14169188802648310188u, 17603457857266236889u,         203546995950u },
+  {  2018609909210367042u, 11164962743035868272u,         238954285362u },
+  {  8270271948267674359u,  1585686890718568774u,          50605253843u },
+  { 12320338602894572099u, 10882524700472655412u,         211085960258u },
+  { 17538536685990080547u,  2194808754940947757u,          66589942846u },
+  { 15833914616956760474u,   274100791137209242u,          62118980821u },
+  {  6137696141415969855u, 12203404582981010903u,         213014859033u },
+  {  9757490468419438919u,   541940706340938166u,          25661547888u },
+  {  3566639201356598903u, 10305434016011833594u,         112029378664u },
+  {  9760900035773954449u,  7900783531944543546u,         104558658697u },
+  {  3873778773990716438u,  8920818625012419323u,         137428302333u },
+  {  2295668377270167832u, 12532363335400447632u,         253483598546u },
+  {  1791721710912807593u, 13483507182924762800u,         210679380777u },
+  { 10571009006922683279u,   415911049779278804u,          41730942389u },
+  {  9840791932778184867u,  3441628281170127418u,         181022546583u },
+  { 11525464956561274613u, 17830811568183566527u,         151186571042u },
+  {  4435781488897895433u, 17897295813176613411u,          34966610231u },
+  {  6395563367070996741u,  2086148701331574596u,          55970214350u },
+  { 15538690795135662932u, 13015567826878853432u,         206113090347u },
+  { 16294558813563371936u, 12944531121587846595u,          43705575345u },
+  {  4942096228426070342u,  3534180912913737995u,         177701724438u },
+  {  6910116424372647153u,  3447584022400118677u,          22191588331u },
+  { 17923400669760829478u,  6375676813770849297u,         235186893904u },
+  {  4134686917293039955u, 11580694081479200185u,          80345626132u },
+  { 16857102463116098681u,  1872134358882196482u,          20627790684u },
+  { 11364321508775167451u, 17602652840520938059u,          92101488606u },
+  {  7966947780972783899u, 10331040597716338351u,         222954241722u },
+  { 11261055695926686278u,    73785407041056976u,         186560046833u },
+  {  9227040437353594058u, 17166209109167902028u,         241003999914u },
+  {  7185344074282882061u,  8762475644006589904u,         170930582060u },
+  { 14197856148610578032u,  8839001228645872586u,          44475014756u },
+  {   885688687260429427u, 13558262784529110268u,         100479163216u },
+  { 17407816160380305183u,  5640853896420358111u,          80734994898u },
+  { 17812728703806357349u,  8459930353450835572u,         210305791302u },
+  { 17120198191964319867u,  7643830211500171269u,          70458613743u },
+  { 12091952048375408786u,  1308629115231236347u,         239414372866u },
+  {   405056939269888281u,  8957268500971669618u,           2070940926u },
+  { 12485440679452408690u,  7645679094277669412u,         254485574498u },
+  {  8394369900823444407u,  3821107497040617191u,          98414473094u },
+  {  2877421667354294258u,  8847137191985934072u,         134207142652u },
+  {  2676980714750756239u,  3531126524756088253u,         252479604268u },
+  {  6119309228579057021u,  8726915034124352183u,          44191422752u },
+  { 18203256146533333852u, 17611136727168068641u,          32473087011u },
+  {   351919978865493747u, 18017743272784259949u,          35954701634u },
+  {  5190010931882390570u, 18113575006829616116u,          66976743819u },
+  {  6982466386088036604u, 12805550441678740368u,         139981938868u },
+  {  4707293888784996898u,  8061966093393027781u,         180694190280u },
+  {   690306801165964760u, 11954593141554100801u,         200437040057u },
+  { 12456770961278956704u, 14068656112359197734u,         185648059792u },
+  { 16946092489294063943u,   895878255770467290u,         144762663376u },
+  { 11098404173866185376u, 10319906489512197802u,         208048565657u },
+  { 15152070965853306880u, 14551142616794302079u,         153559443251u },
+  { 17370091362040414208u, 15933181735739307476u,          51788819021u },
+  { 10141938552171134976u, 11524527334398983147u,          77863739512u },
+  { 10586988556645826560u, 11828012606225556742u,         120624745878u },
+  { 12169852093061922816u,  3556238869349799431u,         150641197848u },
+  { 16717361816799281152u,  7403090230513381483u,          24192784095u },
+  {                    0u, 10172292854665622800u,         223401322325u },
+  {                    0u, 11240746576366182400u,          85551441100u },
+  {                    0u, 17021927826892259328u,         204609362092u },
+  {                    0u,  9046328496309141504u,         172922760556u },
+  {                    0u,  8038996803112140800u,         108490402450u },
+  {                    0u, 17098478935265509376u,         146435794889u },
+  {                    0u,  7205759403792793600u,         201926910400u },
+  {                    0u,                    0u,         192390625000u },
+  {                    0u,                    0u,         232000000000u },
+  {  2144184049294538808u,                    6u,                    0u },
+  {  4108002197393276873u,           6116236450u,                    0u },
+  {  6446230217393892753u,  6116236450222695245u,                    0u },
+  {  5571068025259989822u,  6240972538554414168u,            331561842u },
+  { 14804812668872528331u,  4356262642990299018u,         114338323799u },
+  { 17369928488562523047u,  1335108558830511366u,          87236153471u },
+  {  2967474173531035027u, 18435704923261947246u,         127072376379u },
+  {  5291425437992807716u,  8395401931972636441u,          59999401566u },
+  { 14219547193739388064u, 12482665946362458347u,          94455115650u },
+  { 17720313647158217462u, 16101242875289374781u,         130676686676u },
+  { 12334850628290578491u,  4708983440241068127u,          84872850125u },
+  {  7818499847417334620u, 14856666972541426744u,         205255274503u },
+  {   136007040922198372u,  6938795288315789246u,           7805381530u },
+  {  8523477092112604449u,  5556307628265073820u,         154376152846u },
+  {   367934822655966629u,  1441404248927865979u,          14301208040u },
+  { 12964987687054730050u, 16710378912353838906u,         232078138680u },
+  { 15267036012420885462u, 18289940136919312110u,          56905871455u },
+  { 11142900264750765568u, 10217414145292657824u,          95991499641u },
+  { 13680181547777718603u, 12461165826430955753u,         121553887130u },
+  { 13001922925761426065u,   662762458988270879u,         154675521153u },
+  {  2397730045956515935u, 16488546856395302470u,         129035928424u },
+  {  4482395522588406288u,  2612816787977180950u,         104893845916u },
+  {  3255525722490493080u, 16446616379327454252u,         156141641081u },
+  {   836222287193822098u,  7842178508581740643u,         121891572860u },
+  {   420898743993182306u, 14779029861369369333u,         124425125348u },
+  { 11652649973356574054u,  2697664446153849542u,         228801172814u },
+  { 15058402726661910231u, 12135106444393649308u,          78146240682u },
+  {  9600134495208339559u,  9550285041205189839u,         170657845438u },
+  { 14626343323989004842u,  8790318168586740109u,         190517721989u },
+  {  6813981265331086665u, 14038474217155846828u,         133476524102u },
+  { 10281745288790487888u,  4263144264274812072u,          70761027212u },
+  { 17569829347075761940u, 11940456333341715520u,         140231105513u },
+  {  7654580675237889478u, 15751110736831573013u,         233647293434u },
+  { 16194838649686212364u, 18384528705472318081u,         250853869423u },
+  {  6495102772252453635u,  2393654818032310394u,         111996627298u },
+  { 14935159852819761348u, 12812209822018626434u,          98129760287u },
+  { 17381879863441579697u,  3110778569433458461u,          31694551286u },
+  {  9062335510435372583u,  2860264756226872891u,         246168635644u },
+  {  7569219009130126626u,  2384146980060315184u,         252155055263u },
+  { 12652124168176193362u, 14117430062880324728u,         159129244866u },
+  {  8940200224697247767u,  3769610173216737153u,         194765307417u },
+  {  5600570701927432884u, 17731974340232672009u,          25204350976u },
+  {  2018432801986093157u,  1971479303384713466u,            961252255u },
+  {  8235849749361824736u,  3449462959779012549u,         159106874107u },
+  {  9705097287982370040u, 13743454852043766533u,         251186995761u },
+  {  3517483139049842585u,  7417711187131879498u,          49745034180u },
+  {   958281614186777760u,  3650992383501007879u,         196402114929u },
+  {  7336473432636108950u, 12838770342493958662u,         113197920693u },
+  { 12955383920176764423u, 16025068246546338477u,         181695991134u },
+  { 10735469126281273789u,  6579965938260177729u,          94868720690u },
+  {  8637888232514730693u,  4742939430174291459u,          50356700668u },
+  {  6806336737533581000u, 13062256857527449083u,         252257115261u },
+  { 16142569672872330321u,  2301174570202439645u,         125708106363u },
+  {  8141285259947963513u,  7638687886069412887u,         123124746923u },
+  {  5220241098754220797u,   936322449610274358u,         171414094100u },
+  {   154438799943119608u, 12926010544311283981u,          20050758141u },
+  {  2226876628677628879u, 12647854908989899184u,         253700720435u },
+  { 17219557081221357482u,  8862093163358513015u,          51685641588u },
+  { 15401507148161015114u,   444784343917630731u,         116480415033u },
+  {  8842629766613985337u, 11033952249213387263u,          57024111807u },
+  {  3180100571546071440u, 18168634046363183536u,         191598151749u },
+  {  7740848704392475044u,  3837904761417065597u,          69984923625u },
+  {  2014314126623495998u,   111459007020906105u,         233208053234u },
+  { 11209566016506885858u, 16191761957496794523u,         242006042204u },
+  {  7117166613733441125u,  9856250800340378607u,          92877757174u },
+  {  4197646860931880328u,  9491800102275105959u,         246534308426u },
+  {  5487263271238026094u, 10777328578953608268u,          74514551514u },
+  { 18148076225293562697u, 17424440628313779505u,         218584240152u },
+  {  9127276943027950849u,  3285814872419755679u,          24944580819u },
+  {  9691696125379324722u,  2824823424107240978u,         211178124381u },
+  { 13102362262487705216u, 12271707680713669755u,          93153133984u },
+  {  8929385439893192704u,  6951481875178001185u,         160665250606u },
+  { 11891353410743566336u, 10202522487003824362u,          46376840587u },
+  {  1587423090877399040u,  4834668463880990728u,         139553079852u },
+  {  3489137423026225152u, 10871520987687904746u,          44262087902u },
+  { 13046928120492326912u, 12057698794225322390u,         222589346333u },
+  { 11529215046068469760u,  7263351819222681214u,          29653649161u },
+  {                    0u,  1778055686910650944u,           9393747091u },
+  {                    0u, 17108187120491986944u,         147096388591u },
+  {                    0u,  3067636961549221888u,         239927436682u },
+  {                    0u, 16702141595163557888u,         138166296932u },
+  {                    0u,  2432053749942845440u,         100905424910u },
+  {                    0u, 17791470327927144448u,          14131841897u },
+  {                    0u,  1152921504606846976u,         105964477539u },
+  {                    0u,                    0u,          99062500000u },
+  {                    0u,                    0u,         160000000000u },
+  {  6674960280855494694u,                93326u,                    0u },
+  { 16378845781483497510u,       93326361850321u,                    0u },
+  { 17606907750956804392u,  4283581425266273664u,                 5059u },
+  { 13225609159240506969u,  6725911039793895357u,         195232213414u },
+  {  2668084873338435252u,  1188689198788975021u,         166364612368u },
+  { 14802814305275861366u, 10825527435847761650u,          16064438970u },
+  {  8005510553372365574u,  3917696829526085083u,         186586853018u },
+  { 12748500143273514429u, 12646861173976387276u,         154212378770u },
+  { 10393733905569036127u, 18398576063183996905u,         146685587717u },
+  {   603389089974790339u, 16919251228485834948u,           5997388806u },
+  {  2033494532597735873u, 17296019588687185052u,           6917194446u },
+  {  9204796763694620958u, 12365301604512770359u,         206937619100u },
+  {  7826260310402107021u,  2814271599679204744u,         156670324343u },
+  { 10122690201685169383u,  2154994415780170517u,         119152561969u },
+  {  7438455564568110133u,  6717373824370072839u,          49116822481u },
+  {  2805412574380520817u, 12709155755801344060u,         209364149564u },
+  {  7250965427231182867u,   826847911966403896u,          60688964714u },
+  {  8136242944826085924u,  2277322703890025383u,         106044823515u },
+  { 15357191647956011780u,  2774508958389496437u,         219123453911u },
+  {  7369614426695395460u,   245697774950120915u,         215150406432u },
+  { 10886957545142526638u,  1268929063431863950u,          32013319303u },
+  {  2030047207417538097u,  6735665673159411439u,         135068788782u },
+  { 11557093828502314355u, 14734771742997073207u,          46365141167u },
+  { 15201062539664128543u, 13683287077957612495u,         175798773576u },
+  {  8846936323343880674u, 15370263741354826803u,          72741772478u },
+  {  1485291750116245364u,    48035913070297507u,         190833223667u },
+  {  2076024439668322013u,  1206547475966802115u,         243002604032u },
+  {  7029497773682748741u, 13512340386605768006u,             65407069u },
+  { 16333533921668749881u,  2325760467700278797u,          93732505440u },
+  {  2065057316131928423u, 10848110652847753816u,          96126079727u },
+  {  7800502648925570223u, 15846378960784301285u,         239588077256u },
+  { 17011619967093802015u, 14121839924449844911u,         200859033924u },
+  {  5368819344429198672u,  5147613424753296550u,          68765546476u },
+  { 15598879366754275267u, 16817040482828810167u,         236279052682u },
+  { 16393893486035835647u,  5773528746119363888u,         138911653591u },
+  { 12042046205096920307u,  8716201595536184826u,         215312983620u },
+  { 15437070428008474344u,  5259122109038474872u,          68472506235u },
+  { 13704569163204647509u, 14744540084230155984u,         123285097580u },
+  { 18192483750856993350u, 10719345477982635878u,         108799303119u },
+  {  5152535865317963250u, 13698037261310555208u,         207581096882u },
+  { 17512614083933854680u, 16141171632951976936u,         178742572087u },
+  { 14481918350603613536u, 10060790174955808839u,          55875014667u },
+  { 16124419709964004915u,  4250043307981877118u,          11545396528u },
+  { 18088011566435813579u,  7075646198054337199u,          48230395309u },
+  { 15008862380698848893u, 18141738384245531503u,         173383571548u },
+  { 18160498644611827812u,  8174370508376809531u,          92983465608u },
+  {  3604680497457231965u,  3581964982731575596u,         136443133513u },
+  {  5957615565551495921u, 14798509948722114761u,          73194178710u },
+  { 17528455034961565995u, 14713923334885122090u,         150802228831u },
+  {  8503772325120113809u,  5042978054260414139u,          95797643382u },
+  {  8501492578048509537u,  2052996319372883413u,         118273380388u },
+  {  2296237701094386060u,  8825683007899981588u,          36111293153u },
+  {  3753593040591076946u,  9992196755378745151u,         225478441234u },
+  {  8518075399775653155u,  9301073417573669950u,          18541678071u },
+  { 12757855675959554597u,  5331614769144850592u,         247504212200u },
+  {   121631169379748595u, 14354009428310052102u,         232289027415u },
+  { 16679062494579173314u,  5581221063029119078u,          87778132410u },
+  { 10739912744743898054u,  1529260335339476189u,         186302558600u },
+  {  9367340677776287570u, 16483061525949201148u,         136082901368u },
+  { 12227321512794715397u, 14431217812333089675u,         120893548555u },
+  {  7241061891859170651u,  3452349151135392267u,          11782317885u },
+  { 13148571323079237489u,  9075317899834447999u,          61187152222u },
+  { 12509763434355012654u,  2764331337978901575u,          94491973969u },
+  { 11812768946960181977u,  1942890683708857202u,          81149854702u },
+  { 14170358803552564832u,   165089169728028447u,         238105324315u },
+  { 18179989524780635952u, 15193620741871233073u,          27008949501u },
+  { 17091718978514754901u, 14995000835194145926u,         253823647830u },
+  {  7394768384359232459u,  1788823614552255558u,          86812880624u },
+  {  6778628272692852803u,  8384901184618498845u,         240096972322u },
+  { 18193335045875234320u,   405511217862281310u,          34454546404u },
+  {  1378519212560967521u,  3111530463755196557u,         228021982807u },
+  {  4677732610631043584u,  7893558450035460812u,          87168676404u },
+  { 17296098591070486528u,   156573858237402216u,          52427910661u },
+  {  7343735382392963072u, 15915324019419451223u,           5008487885u },
+  { 14525996728454217728u, 16293363012778802804u,         205862771443u },
+  {  9691359370008330240u, 14342105318291351412u,         243883264978u },
+  {  3044433348102455296u,  3788398842525387052u,         210777487087u },
+  {  9223372036854775808u, 14118764407048307670u,         239205369512u },
+  {                    0u,  2705021334614720768u,         168765379752u },
+  {                    0u,  7017988973805568000u,         168146639500u },
+  {                    0u, 10956732053634154496u,         140380445944u },
+  {                    0u, 14657517938546835456u,         248593965634u },
+  {                    0u, 11268868284797157376u,          66794585639u },
+  {                    0u, 14600669991935148032u,          39610886573u },
+  {                    0u,  4611686018427387904u,         173791503906u },
+  {                    0u,                    0u,          34250000000u },
+  {                    0u,                    0u,         128000000000u },
+  {  8201586317771250746u,           1424047269u,                    0u },
+  {  3278889188817135834u,  1424047269444608885u,                    0u },
+  {  1710725240251040430u,  3001188830946823627u,             77197757u },
+  {  1850175733663425006u,  9732296932705387049u,         189162694772u },
+  {  9147599666163914249u, 16337535782679529459u,         116527588873u },
+  { 10221885933644344166u,  7969742269895046547u,           9885659589u },
+  { 14901479793736678101u,  2923592083903829642u,         197432040594u },
+  {  5181831442059703136u,  8144196241160608534u,         146158488244u },
+  {  6396246577759793483u, 16431078457793424253u,         180441497762u },
+  { 14167229556464870447u,   202362949592775653u,         162890730548u },
+  {  2969982933326311854u,  8835125248522947981u,          52010970117u },
+  {  7892677766222018881u,  7959873808777345113u,           5478953099u },
+  {   798698968922663621u, 14929747122315126151u,         139431505623u },
+  { 15926812109043458972u,  4310328817360515349u,         215809343213u },
+  {  8663842590352697437u,  7294899422760201126u,         237233663393u },
+  { 17093523026636671168u,  2047461597291187207u,         161395457290u },
+  {   839764004742743203u, 10942374468813517900u,          10110993115u },
+  { 16894643909298232323u, 10364795403063433969u,         219593187308u },
+  {  9066702926218949317u, 12330859528790939137u,         236561876684u },
+  {  9119392417260546810u,  8973160144879916806u,         204668457234u },
+  {  9723021096578315109u,  2895354388547509877u,          18486435986u },
+  { 14787464248751217597u, 16766844772497556429u,         146156957475u },
+  {  3733434565920249133u,  7442407174620948827u,          35908932476u },
+  {  6643788868836820841u,  6683013428676659077u,         124403453701u },
+  {  4729646697422664063u, 16713703375071907588u,           5362286883u },
+  {  4090144564201555829u,  8791044883080637861u,          35906051675u },
+  {  2109480737093400002u,   602844107089214413u,          91476563498u },
+  { 16577155033369419739u,  9754832281172880875u,          42032680244u },
+  {   745377248603805917u, 10587846778003503903u,          52528810517u },
+  { 11305561465807999667u, 17206244172922947013u,          21573968323u },
+  {  2211245518782892177u, 11620628420699303875u,         195932752365u },
+  { 14170095199249735666u, 17864732368219338611u,         237629955528u },
+  { 17849973668116118927u,  4146383014621345887u,         200968449082u },
+  {  9020960204585720001u, 11445705075042688243u,          58224775873u },
+  { 10807134002871850916u,  7369147888966546592u,         193620472915u },
+  {  3925122626254791201u,  9762476865090597796u,          83399482307u },
+  { 17208463291312718997u,  5507001428194242827u,         195529224931u },
+  {  5145077219589447653u, 11371471148365328344u,         227298535145u },
+  { 17602397765035489468u,  3148788104946538618u,         233616448686u },
+  { 16422643262490753377u,  3762722308424507574u,         174170696145u },
+  {  2902509461400906224u,  1156171244825745915u,         209203977585u },
+  {  3422418805967265206u, 14208921674868257865u,         113062676168u },
+  {  4228874576277237392u,  7903080886897905503u,         200770267187u },
+  {  2553488530807495751u,  6367240794154270982u,          51428426873u },
+  { 11546099176912486413u,  1623672396662369850u,         121345168815u },
+  { 10460791037534167991u, 18323231215381674394u,         175088019456u },
+  {  8127117908566000904u,  9842279843006544554u,            993304354u },
+  { 11541304458088287306u,  7376839231308610600u,          34533551059u },
+  {  6249718665174839700u,   609751749293657672u,         211399899256u },
+  { 13102508413386290995u, 10386457966860989799u,         120033054708u },
+  {  6274675218640661911u, 11160336020836149780u,         244563051014u },
+  {  3404497118599817167u, 17947559933847409193u,           6605003027u },
+  { 11258566093988562335u, 10229787001712704590u,          19972939173u },
+  { 16762592482501635397u, 10441677090043619866u,         165554557864u },
+  {  5550125446725071998u,  4996681336392922375u,         168566044449u },
+  {  6370033225258510318u,   124497102381021895u,          33270870638u },
+  {  1503521728674735398u,  8180812057779384577u,         110006749001u },
+  {  4250415082606384364u,  5294232873532946716u,          73443482710u },
+  {  6020091901030562974u,  2885620189169448039u,          86287000939u },
+  { 16288222967151527138u, 16662526875008170507u,         107156429783u },
+  {  6377016228656203782u, 15663095032402672480u,         215903277391u },
+  {  8378856515587563750u,  1824281504410546614u,          79849098083u },
+  { 15812881490200838483u,  9506565509584809953u,          99098894498u },
+  {  4548570371183413652u, 16941136942345070055u,         162515351948u },
+  { 16731431495283420383u, 15924115693705937725u,         140918380873u },
+  { 14737727629551135532u,  9247807690406628462u,          73863248041u },
+  { 12413722258104293893u,  7993916633864834871u,         169501324659u },
+  {   800899742400762438u,  1018504409177639408u,         115433351089u },
+  {   603197008376033550u, 12097800686634130718u,         177055213234u },
+  {  6380777281587743935u,  6221488888422637551u,         178655823089u },
+  { 10001440249018225388u,  8229322865256080421u,         241337267588u },
+  {  5505914461980436708u,  7927745108183101786u,         132446112486u },
+  {  1105464290051876864u,  8488683721235326653u,         230429763923u },
+  {  4500443576769970176u, 11165516518170922283u,          83460172466u },
+  {  2843045143185981440u,  5463648141113596927u,         178605283863u },
+  {   660949699682893824u,  3958440403860778042u,          23296184959u },
+  {   276549164618219520u,  5091534813990256011u,         127214587484u },
+  {  4683743612465315840u,  6100166970623291280u,          92276012655u },
+  {                    0u,  1913011027739012426u,         111330690714u },
+  {                    0u, 11310957650604221440u,         154103704535u },
+  {                    0u, 16303817257009020928u,         215613168242u },
+  {                    0u,  9090406322154766336u,         114883831704u },
+  {                    0u,  3003279315069566976u,         152492791914u },
+  {                    0u, 16582887146675765248u,         106162808097u },
+  {                    0u,  9691746398101307392u,          33898960113u },
+  {                    0u,                    0u,         241525390625u },
+  {                    0u,                    0u,          33000000000u },
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+// clang-format on
+
+#endif // _LIBCPP_SRC_INCLUDE_RYU_D2FIXED_FULL_TABLE_H
diff --git a/libcxx/src/include/ryu/d2s.h b/libcxx/src/include/ryu/d2s.h
new file mode 100644
index 000000000000..15d50fd1ee54
--- /dev/null
+++ b/libcxx/src/include/ryu/d2s.h
@@ -0,0 +1,62 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// Copyright 2018 Ulf Adams
+// Copyright (c) Microsoft Corporation. All rights reserved.
+
+// Boost Software License - Version 1.0 - August 17th, 2003
+
+// Permission is hereby granted, free of charge, to any person or organization
+// obtaining a copy of the software and accompanying documentation covered by
+// this license (the "Software") to use, reproduce, display, distribute,
+// execute, and transmit the Software, and to prepare derivative works of the
+// Software, and to permit third-parties to whom the Software is furnished to
+// do so, all subject to the following:
+
+// The copyright notices in the Software and this entire statement, including
+// the above license grant, this restriction and the following disclaimer,
+// must be included in all copies of the Software, in whole or in part, and
+// all derivative works of the Software, unless such copies or derivative
+// works are solely in the form of machine-executable object code generated by
+// a source language processor.
+
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+// SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+// FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+// DEALINGS IN THE SOFTWARE.
+
+#ifndef _LIBCPP_SRC_INCLUDE_RYU_DS2_H
+#define _LIBCPP_SRC_INCLUDE_RYU_DS2_H
+
+// Avoid formatting to keep the changes with the original code minimal.
+// clang-format off
+
+#include "__config"
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+inline constexpr int __DOUBLE_MANTISSA_BITS = 52;
+inline constexpr int __DOUBLE_EXPONENT_BITS = 11;
+inline constexpr int __DOUBLE_BIAS = 1023;
+
+inline constexpr int __DOUBLE_POW5_INV_BITCOUNT = 122;
+inline constexpr int __DOUBLE_POW5_BITCOUNT = 121;
+
+[[nodiscard]] to_chars_result __d2s_buffered_n(char* const _First, char* const _Last, const double __f, const chars_format _Fmt);
+
+_LIBCPP_END_NAMESPACE_STD
+
+// clang-format on
+
+#endif // _LIBCPP_SRC_INCLUDE_RYU_DS2_H
diff --git a/libcxx/src/include/ryu/d2s_full_table.h b/libcxx/src/include/ryu/d2s_full_table.h
new file mode 100644
index 000000000000..106fef0015c1
--- /dev/null
+++ b/libcxx/src/include/ryu/d2s_full_table.h
@@ -0,0 +1,368 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// Copyright 2018 Ulf Adams
+// Copyright (c) Microsoft Corporation. All rights reserved.
+
+// Boost Software License - Version 1.0 - August 17th, 2003
+
+// Permission is hereby granted, free of charge, to any person or organization
+// obtaining a copy of the software and accompanying documentation covered by
+// this license (the "Software") to use, reproduce, display, distribute,
+// execute, and transmit the Software, and to prepare derivative works of the
+// Software, and to permit third-parties to whom the Software is furnished to
+// do so, all subject to the following:
+
+// The copyright notices in the Software and this entire statement, including
+// the above license grant, this restriction and the following disclaimer,
+// must be included in all copies of the Software, in whole or in part, and
+// all derivative works of the Software, unless such copies or derivative
+// works are solely in the form of machine-executable object code generated by
+// a source language processor.
+
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+// SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+// FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+// DEALINGS IN THE SOFTWARE.
+
+#ifndef _LIBCPP_SRC_INCLUDE_RYU_D2S_FULL_TABLE_H
+#define _LIBCPP_SRC_INCLUDE_RYU_D2S_FULL_TABLE_H
+
+// Avoid formatting to keep the changes with the original code minimal.
+// clang-format off
+
+#include "__config"
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+inline constexpr uint64_t __DOUBLE_POW5_INV_SPLIT[292][2] = {
+  {                    1u, 288230376151711744u }, {  3689348814741910324u, 230584300921369395u },
+  {  2951479051793528259u, 184467440737095516u }, { 17118578500402463900u, 147573952589676412u },
+  { 12632330341676300947u, 236118324143482260u }, { 10105864273341040758u, 188894659314785808u },
+  { 15463389048156653253u, 151115727451828646u }, { 17362724847566824558u, 241785163922925834u },
+  { 17579528692795369969u, 193428131138340667u }, {  6684925324752475329u, 154742504910672534u },
+  { 18074578149087781173u, 247588007857076054u }, { 18149011334012135262u, 198070406285660843u },
+  {  3451162622983977240u, 158456325028528675u }, {  5521860196774363583u, 253530120045645880u },
+  {  4417488157419490867u, 202824096036516704u }, {  7223339340677503017u, 162259276829213363u },
+  {  7867994130342094503u, 259614842926741381u }, {  2605046489531765280u, 207691874341393105u },
+  {  2084037191625412224u, 166153499473114484u }, { 10713157136084480204u, 265845599156983174u },
+  { 12259874523609494487u, 212676479325586539u }, { 13497248433629505913u, 170141183460469231u },
+  { 14216899864323388813u, 272225893536750770u }, { 11373519891458711051u, 217780714829400616u },
+  {  5409467098425058518u, 174224571863520493u }, {  4965798542738183305u, 278759314981632789u },
+  {  7661987648932456967u, 223007451985306231u }, {  2440241304404055250u, 178405961588244985u },
+  {  3904386087046488400u, 285449538541191976u }, { 17880904128604832013u, 228359630832953580u },
+  { 14304723302883865611u, 182687704666362864u }, { 15133127457049002812u, 146150163733090291u },
+  { 16834306301794583852u, 233840261972944466u }, {  9778096226693756759u, 187072209578355573u },
+  { 15201174610838826053u, 149657767662684458u }, {  2185786488890659746u, 239452428260295134u },
+  {  5437978005854438120u, 191561942608236107u }, { 15418428848909281466u, 153249554086588885u },
+  {  6222742084545298729u, 245199286538542217u }, { 16046240111861969953u, 196159429230833773u },
+  {  1768945645263844993u, 156927543384667019u }, { 10209010661905972635u, 251084069415467230u },
+  {  8167208529524778108u, 200867255532373784u }, { 10223115638361732810u, 160693804425899027u },
+  {  1599589762411131202u, 257110087081438444u }, {  4969020624670815285u, 205688069665150755u },
+  {  3975216499736652228u, 164550455732120604u }, { 13739044029062464211u, 263280729171392966u },
+  {  7301886408508061046u, 210624583337114373u }, { 13220206756290269483u, 168499666669691498u },
+  { 17462981995322520850u, 269599466671506397u }, {  6591687966774196033u, 215679573337205118u },
+  { 12652048002903177473u, 172543658669764094u }, {  9175230360419352987u, 276069853871622551u },
+  {  3650835473593572067u, 220855883097298041u }, { 17678063637842498946u, 176684706477838432u },
+  { 13527506561580357021u, 282695530364541492u }, {  3443307619780464970u, 226156424291633194u },
+  {  6443994910566282300u, 180925139433306555u }, {  5155195928453025840u, 144740111546645244u },
+  { 15627011115008661990u, 231584178474632390u }, { 12501608892006929592u, 185267342779705912u },
+  {  2622589484121723027u, 148213874223764730u }, {  4196143174594756843u, 237142198758023568u },
+  { 10735612169159626121u, 189713759006418854u }, { 12277838550069611220u, 151771007205135083u },
+  { 15955192865369467629u, 242833611528216133u }, {  1696107848069843133u, 194266889222572907u },
+  { 12424932722681605476u, 155413511378058325u }, {  1433148282581017146u, 248661618204893321u },
+  { 15903913885032455010u, 198929294563914656u }, {  9033782293284053685u, 159143435651131725u },
+  { 14454051669254485895u, 254629497041810760u }, { 11563241335403588716u, 203703597633448608u },
+  { 16629290697806691620u, 162962878106758886u }, {   781423413297334329u, 260740604970814219u },
+  {  4314487545379777786u, 208592483976651375u }, {  3451590036303822229u, 166873987181321100u },
+  {  5522544058086115566u, 266998379490113760u }, {  4418035246468892453u, 213598703592091008u },
+  { 10913125826658934609u, 170878962873672806u }, { 10082303693170474728u, 273406340597876490u },
+  {  8065842954536379782u, 218725072478301192u }, { 17520720807854834795u, 174980057982640953u },
+  {  5897060404116273733u, 279968092772225526u }, {  1028299508551108663u, 223974474217780421u },
+  { 15580034865808528224u, 179179579374224336u }, { 17549358155809824511u, 286687326998758938u },
+  {  2971440080422128639u, 229349861599007151u }, { 17134547323305344204u, 183479889279205720u },
+  { 13707637858644275364u, 146783911423364576u }, { 14553522944347019935u, 234854258277383322u },
+  {  4264120725993795302u, 187883406621906658u }, { 10789994210278856888u, 150306725297525326u },
+  {  9885293106962350374u, 240490760476040522u }, {   529536856086059653u, 192392608380832418u },
+  {  7802327114352668369u, 153914086704665934u }, {  1415676938738538420u, 246262538727465495u },
+  {  1132541550990830736u, 197010030981972396u }, { 15663428499760305882u, 157608024785577916u },
+  { 17682787970132668764u, 252172839656924666u }, { 10456881561364224688u, 201738271725539733u },
+  { 15744202878575200397u, 161390617380431786u }, { 17812026976236499989u, 258224987808690858u },
+  {  3181575136763469022u, 206579990246952687u }, { 13613306553636506187u, 165263992197562149u },
+  { 10713244041592678929u, 264422387516099439u }, { 12259944048016053467u, 211537910012879551u },
+  {  6118606423670932450u, 169230328010303641u }, {  2411072648389671274u, 270768524816485826u },
+  { 16686253377679378312u, 216614819853188660u }, { 13349002702143502650u, 173291855882550928u },
+  { 17669055508687693916u, 277266969412081485u }, { 14135244406950155133u, 221813575529665188u },
+  {   240149081334393137u, 177450860423732151u }, { 11452284974360759988u, 283921376677971441u },
+  {  5472479164746697667u, 227137101342377153u }, { 11756680961281178780u, 181709681073901722u },
+  {  2026647139541122378u, 145367744859121378u }, { 18000030682233437097u, 232588391774594204u },
+  { 18089373360528660001u, 186070713419675363u }, {  3403452244197197031u, 148856570735740291u },
+  { 16513570034941246220u, 238170513177184465u }, { 13210856027952996976u, 190536410541747572u },
+  {  3189987192878576934u, 152429128433398058u }, {  1414630693863812771u, 243886605493436893u },
+  {  8510402184574870864u, 195109284394749514u }, { 10497670562401807014u, 156087427515799611u },
+  {  9417575270359070576u, 249739884025279378u }, { 14912757845771077107u, 199791907220223502u },
+  {  4551508647133041040u, 159833525776178802u }, { 10971762650154775986u, 255733641241886083u },
+  { 16156107749607641435u, 204586912993508866u }, {  9235537384944202825u, 163669530394807093u },
+  { 11087511001168814197u, 261871248631691349u }, { 12559357615676961681u, 209496998905353079u },
+  { 13736834907283479668u, 167597599124282463u }, { 18289587036911657145u, 268156158598851941u },
+  { 10942320814787415393u, 214524926879081553u }, { 16132554281313752961u, 171619941503265242u },
+  { 11054691591134363444u, 274591906405224388u }, { 16222450902391311402u, 219673525124179510u },
+  { 12977960721913049122u, 175738820099343608u }, { 17075388340318968271u, 281182112158949773u },
+  {  2592264228029443648u, 224945689727159819u }, {  5763160197165465241u, 179956551781727855u },
+  {  9221056315464744386u, 287930482850764568u }, { 14755542681855616155u, 230344386280611654u },
+  { 15493782960226403247u, 184275509024489323u }, {  1326979923955391628u, 147420407219591459u },
+  {  9501865507812447252u, 235872651551346334u }, { 11290841220991868125u, 188698121241077067u },
+  {  1653975347309673853u, 150958496992861654u }, { 10025058185179298811u, 241533595188578646u },
+  {  4330697733401528726u, 193226876150862917u }, { 14532604630946953951u, 154581500920690333u },
+  {  1116074521063664381u, 247330401473104534u }, {  4582208431592841828u, 197864321178483627u },
+  { 14733813189500004432u, 158291456942786901u }, { 16195403473716186445u, 253266331108459042u },
+  {  5577625149489128510u, 202613064886767234u }, {  8151448934333213131u, 162090451909413787u },
+  { 16731667109675051333u, 259344723055062059u }, { 17074682502481951390u, 207475778444049647u },
+  {  6281048372501740465u, 165980622755239718u }, {  6360328581260874421u, 265568996408383549u },
+  {  8777611679750609860u, 212455197126706839u }, { 10711438158542398211u, 169964157701365471u },
+  {  9759603424184016492u, 271942652322184754u }, { 11497031554089123517u, 217554121857747803u },
+  { 16576322872755119460u, 174043297486198242u }, { 11764721337440549842u, 278469275977917188u },
+  { 16790474699436260520u, 222775420782333750u }, { 13432379759549008416u, 178220336625867000u },
+  {  3045063541568861850u, 285152538601387201u }, { 17193446092222730773u, 228122030881109760u },
+  { 13754756873778184618u, 182497624704887808u }, { 18382503128506368341u, 145998099763910246u },
+  {  3586563302416817083u, 233596959622256395u }, {  2869250641933453667u, 186877567697805116u },
+  { 17052795772514404226u, 149502054158244092u }, { 12527077977055405469u, 239203286653190548u },
+  { 17400360011128145022u, 191362629322552438u }, {  2852241564676785048u, 153090103458041951u },
+  { 15631632947708587046u, 244944165532867121u }, {  8815957543424959314u, 195955332426293697u },
+  { 18120812478965698421u, 156764265941034957u }, { 14235904707377476180u, 250822825505655932u },
+  {  4010026136418160298u, 200658260404524746u }, { 17965416168102169531u, 160526608323619796u },
+  {  2919224165770098987u, 256842573317791675u }, {  2335379332616079190u, 205474058654233340u },
+  {  1868303466092863352u, 164379246923386672u }, {  6678634360490491686u, 263006795077418675u },
+  {  5342907488392393349u, 210405436061934940u }, {  4274325990713914679u, 168324348849547952u },
+  { 10528270399884173809u, 269318958159276723u }, { 15801313949391159694u, 215455166527421378u },
+  {  1573004715287196786u, 172364133221937103u }, { 17274202803427156150u, 275782613155099364u },
+  { 17508711057483635243u, 220626090524079491u }, { 10317620031244997871u, 176500872419263593u },
+  { 12818843235250086271u, 282401395870821749u }, { 13944423402941979340u, 225921116696657399u },
+  { 14844887537095493795u, 180736893357325919u }, { 15565258844418305359u, 144589514685860735u },
+  {  6457670077359736959u, 231343223497377177u }, { 16234182506113520537u, 185074578797901741u },
+  {  9297997190148906106u, 148059663038321393u }, { 11187446689496339446u, 236895460861314229u },
+  { 12639306166338981880u, 189516368689051383u }, { 17490142562555006151u, 151613094951241106u },
+  {  2158786396894637579u, 242580951921985771u }, { 16484424376483351356u, 194064761537588616u },
+  {  9498190686444770762u, 155251809230070893u }, { 11507756283569722895u, 248402894768113429u },
+  { 12895553841597688639u, 198722315814490743u }, { 17695140702761971558u, 158977852651592594u },
+  { 17244178680193423523u, 254364564242548151u }, { 10105994129412828495u, 203491651394038521u },
+  {  4395446488788352473u, 162793321115230817u }, { 10722063196803274280u, 260469313784369307u },
+  {  1198952927958798777u, 208375451027495446u }, { 15716557601334680315u, 166700360821996356u },
+  { 17767794532651667857u, 266720577315194170u }, { 14214235626121334286u, 213376461852155336u },
+  {  7682039686155157106u, 170701169481724269u }, {  1223217053622520399u, 273121871170758831u },
+  { 15735968901865657612u, 218497496936607064u }, { 16278123936234436413u, 174797997549285651u },
+  {   219556594781725998u, 279676796078857043u }, {  7554342905309201445u, 223741436863085634u },
+  {  9732823138989271479u, 178993149490468507u }, {   815121763415193074u, 286389039184749612u },
+  { 11720143854957885429u, 229111231347799689u }, { 13065463898708218666u, 183288985078239751u },
+  {  6763022304224664610u, 146631188062591801u }, {  3442138057275642729u, 234609900900146882u },
+  { 13821756890046245153u, 187687920720117505u }, { 11057405512036996122u, 150150336576094004u },
+  {  6623802375033462826u, 240240538521750407u }, { 16367088344252501231u, 192192430817400325u },
+  { 13093670675402000985u, 153753944653920260u }, {  2503129006933649959u, 246006311446272417u },
+  { 13070549649772650937u, 196805049157017933u }, { 17835137349301941396u, 157444039325614346u },
+  {  2710778055689733971u, 251910462920982955u }, {  2168622444551787177u, 201528370336786364u },
+  {  5424246770383340065u, 161222696269429091u }, {  1300097203129523457u, 257956314031086546u },
+  { 15797473021471260058u, 206365051224869236u }, {  8948629602435097724u, 165092040979895389u },
+  {  3249760919670425388u, 264147265567832623u }, {  9978506365220160957u, 211317812454266098u },
+  { 15361502721659949412u, 169054249963412878u }, {  2442311466204457120u, 270486799941460606u },
+  { 16711244431931206989u, 216389439953168484u }, { 17058344360286875914u, 173111551962534787u },
+  { 12535955717491360170u, 276978483140055660u }, { 10028764573993088136u, 221582786512044528u },
+  { 15401709288678291155u, 177266229209635622u }, {  9885339602917624555u, 283625966735416996u },
+  {  4218922867592189321u, 226900773388333597u }, { 14443184738299482427u, 181520618710666877u },
+  {  4175850161155765295u, 145216494968533502u }, { 10370709072591134795u, 232346391949653603u },
+  { 15675264887556728482u, 185877113559722882u }, {  5161514280561562140u, 148701690847778306u },
+  {   879725219414678777u, 237922705356445290u }, {   703780175531743021u, 190338164285156232u },
+  { 11631070584651125387u, 152270531428124985u }, {   162968861732249003u, 243632850284999977u },
+  { 11198421533611530172u, 194906280227999981u }, {  5269388412147313814u, 155925024182399985u },
+  {  8431021459435702103u, 249480038691839976u }, {  3055468352806651359u, 199584030953471981u },
+  { 17201769941212962380u, 159667224762777584u }, { 16454785461715008838u, 255467559620444135u },
+  { 13163828369372007071u, 204374047696355308u }, { 17909760324981426303u, 163499238157084246u },
+  {  2830174816776909822u, 261598781051334795u }, {  2264139853421527858u, 209279024841067836u },
+  { 16568707141704863579u, 167423219872854268u }, {  4373838538276319787u, 267877151796566830u },
+  {  3499070830621055830u, 214301721437253464u }, {  6488605479238754987u, 171441377149802771u },
+  {  3003071137298187333u, 274306203439684434u }, {  6091805724580460189u, 219444962751747547u },
+  { 15941491023890099121u, 175555970201398037u }, { 10748990379256517301u, 280889552322236860u },
+  {  8599192303405213841u, 224711641857789488u }, { 14258051472207991719u, 179769313486231590u }
+};
+
+inline constexpr uint64_t __DOUBLE_POW5_SPLIT[326][2] = {
+  {                    0u,  72057594037927936u }, {                    0u,  90071992547409920u },
+  {                    0u, 112589990684262400u }, {                    0u, 140737488355328000u },
+  {                    0u,  87960930222080000u }, {                    0u, 109951162777600000u },
+  {                    0u, 137438953472000000u }, {                    0u,  85899345920000000u },
+  {                    0u, 107374182400000000u }, {                    0u, 134217728000000000u },
+  {                    0u,  83886080000000000u }, {                    0u, 104857600000000000u },
+  {                    0u, 131072000000000000u }, {                    0u,  81920000000000000u },
+  {                    0u, 102400000000000000u }, {                    0u, 128000000000000000u },
+  {                    0u,  80000000000000000u }, {                    0u, 100000000000000000u },
+  {                    0u, 125000000000000000u }, {                    0u,  78125000000000000u },
+  {                    0u,  97656250000000000u }, {                    0u, 122070312500000000u },
+  {                    0u,  76293945312500000u }, {                    0u,  95367431640625000u },
+  {                    0u, 119209289550781250u }, {  4611686018427387904u,  74505805969238281u },
+  { 10376293541461622784u,  93132257461547851u }, {  8358680908399640576u, 116415321826934814u },
+  {   612489549322387456u,  72759576141834259u }, { 14600669991935148032u,  90949470177292823u },
+  { 13639151471491547136u, 113686837721616029u }, {  3213881284082270208u, 142108547152020037u },
+  {  4314518811765112832u,  88817841970012523u }, {   781462496279003136u, 111022302462515654u },
+  { 10200200157203529728u, 138777878078144567u }, { 13292654125893287936u,  86736173798840354u },
+  {  7392445620511834112u, 108420217248550443u }, {  4628871007212404736u, 135525271560688054u },
+  { 16728102434789916672u,  84703294725430033u }, {  7075069988205232128u, 105879118406787542u },
+  { 18067209522111315968u, 132348898008484427u }, {  8986162942105878528u,  82718061255302767u },
+  {  6621017659204960256u, 103397576569128459u }, {  3664586055578812416u, 129246970711410574u },
+  { 16125424340018921472u,  80779356694631608u }, {  1710036351314100224u, 100974195868289511u },
+  { 15972603494424788992u, 126217744835361888u }, {  9982877184015493120u,  78886090522101180u },
+  { 12478596480019366400u,  98607613152626475u }, { 10986559581596820096u, 123259516440783094u },
+  {  2254913720070624656u,  77037197775489434u }, { 12042014186943056628u,  96296497219361792u },
+  { 15052517733678820785u, 120370621524202240u }, {  9407823583549262990u,  75231638452626400u },
+  { 11759779479436578738u,  94039548065783000u }, { 14699724349295723422u, 117549435082228750u },
+  {  4575641699882439235u,  73468396926392969u }, { 10331238143280436948u,  91835496157991211u },
+  {  8302361660673158281u, 114794370197489014u }, {  1154580038986672043u, 143492962746861268u },
+  {  9944984561221445835u,  89683101716788292u }, { 12431230701526807293u, 112103877145985365u },
+  {  1703980321626345405u, 140129846432481707u }, { 17205888765512323542u,  87581154020301066u },
+  { 12283988920035628619u, 109476442525376333u }, {  1519928094762372062u, 136845553156720417u },
+  { 12479170105294952299u,  85528470722950260u }, { 15598962631618690374u, 106910588403687825u },
+  {  5663645234241199255u, 133638235504609782u }, { 17374836326682913246u,  83523897190381113u },
+  {  7883487353071477846u, 104404871487976392u }, {  9854359191339347308u, 130506089359970490u },
+  { 10770660513014479971u,  81566305849981556u }, { 13463325641268099964u, 101957882312476945u },
+  {  2994098996302961243u, 127447352890596182u }, { 15706369927971514489u,  79654595556622613u },
+  {  5797904354682229399u,  99568244445778267u }, {  2635694424925398845u, 124460305557222834u },
+  {  6258995034005762182u,  77787690973264271u }, {  3212057774079814824u,  97234613716580339u },
+  { 17850130272881932242u, 121543267145725423u }, { 18073860448192289507u,  75964541966078389u },
+  {  8757267504958198172u,  94955677457597987u }, {  6334898362770359811u, 118694596821997484u },
+  { 13182683513586250689u,  74184123013748427u }, { 11866668373555425458u,  92730153767185534u },
+  {  5609963430089506015u, 115912692208981918u }, { 17341285199088104971u,  72445432630613698u },
+  { 12453234462005355406u,  90556790788267123u }, { 10954857059079306353u, 113195988485333904u },
+  { 13693571323849132942u, 141494985606667380u }, { 17781854114260483896u,  88434366004167112u },
+  {  3780573569116053255u, 110542957505208891u }, {   114030942967678664u, 138178696881511114u },
+  {  4682955357782187069u,  86361685550944446u }, { 15077066234082509644u, 107952106938680557u },
+  {  5011274737320973344u, 134940133673350697u }, { 14661261756894078100u,  84337583545844185u },
+  {  4491519140835433913u, 105421979432305232u }, {  5614398926044292391u, 131777474290381540u },
+  { 12732371365632458552u,  82360921431488462u }, {  6692092170185797382u, 102951151789360578u },
+  { 17588487249587022536u, 128688939736700722u }, { 15604490549419276989u,  80430587335437951u },
+  { 14893927168346708332u, 100538234169297439u }, { 14005722942005997511u, 125672792711621799u },
+  { 15671105866394830300u,  78545495444763624u }, {  1142138259283986260u,  98181869305954531u },
+  { 15262730879387146537u, 122727336632443163u }, {  7233363790403272633u,  76704585395276977u },
+  { 13653390756431478696u,  95880731744096221u }, {  3231680390257184658u, 119850914680120277u },
+  {  4325643253124434363u,  74906821675075173u }, { 10018740084832930858u,  93633527093843966u },
+  {  3300053069186387764u, 117041908867304958u }, { 15897591223523656064u,  73151193042065598u },
+  { 10648616992549794273u,  91438991302581998u }, {  4087399203832467033u, 114298739128227498u },
+  { 14332621041645359599u, 142873423910284372u }, { 18181260187883125557u,  89295889943927732u },
+  {  4279831161144355331u, 111619862429909666u }, { 14573160988285219972u, 139524828037387082u },
+  { 13719911636105650386u,  87203017523366926u }, {  7926517508277287175u, 109003771904208658u },
+  {   684774848491833161u, 136254714880260823u }, {  7345513307948477581u,  85159196800163014u },
+  { 18405263671790372785u, 106448996000203767u }, { 18394893571310578077u, 133061245000254709u },
+  { 13802651491282805250u,  83163278125159193u }, {  3418256308821342851u, 103954097656448992u },
+  {  4272820386026678563u, 129942622070561240u }, {  2670512741266674102u,  81214138794100775u },
+  { 17173198981865506339u, 101517673492625968u }, {  3019754653622331308u, 126897091865782461u },
+  {  4193189667727651020u,  79310682416114038u }, { 14464859121514339583u,  99138353020142547u },
+  { 13469387883465536574u, 123922941275178184u }, {  8418367427165960359u,  77451838296986365u },
+  { 15134645302384838353u,  96814797871232956u }, {   471562554271496325u, 121018497339041196u },
+  {  9518098633274461011u,  75636560836900747u }, {  7285937273165688360u,  94545701046125934u },
+  { 18330793628311886258u, 118182126307657417u }, {  4539216990053847055u,  73863828942285886u },
+  { 14897393274422084627u,  92329786177857357u }, {  4786683537745442072u, 115412232722321697u },
+  { 14520892257159371055u,  72132645451451060u }, { 18151115321449213818u,  90165806814313825u },
+  {  8853836096529353561u, 112707258517892282u }, {  1843923083806916143u, 140884073147365353u },
+  { 12681666973447792349u,  88052545717103345u }, {  2017025661527576725u, 110065682146379182u },
+  { 11744654113764246714u, 137582102682973977u }, {   422879793461572340u,  85988814176858736u },
+  {   528599741826965425u, 107486017721073420u }, {   660749677283706782u, 134357522151341775u },
+  {  7330497575943398595u,  83973451344588609u }, { 13774807988356636147u, 104966814180735761u },
+  {  3383451930163631472u, 131208517725919702u }, { 15949715511634433382u,  82005323578699813u },
+  {  6102086334260878016u, 102506654473374767u }, {  3015921899398709616u, 128133318091718459u },
+  { 18025852251620051174u,  80083323807324036u }, {  4085571240815512351u, 100104154759155046u },
+  { 14330336087874166247u, 125130193448943807u }, { 15873989082562435760u,  78206370905589879u },
+  { 15230800334775656796u,  97757963631987349u }, {  5203442363187407284u, 122197454539984187u },
+  {   946308467778435600u,  76373409087490117u }, {  5794571603150432404u,  95466761359362646u },
+  { 16466586540792816313u, 119333451699203307u }, {  7985773578781816244u,  74583407312002067u },
+  {  5370530955049882401u,  93229259140002584u }, {  6713163693812353001u, 116536573925003230u },
+  { 18030785363914884337u,  72835358703127018u }, { 13315109668038829614u,  91044198378908773u },
+  {  2808829029766373305u, 113805247973635967u }, { 17346094342490130344u, 142256559967044958u },
+  {  6229622945628943561u,  88910349979403099u }, {  3175342663608791547u, 111137937474253874u },
+  { 13192550366365765242u, 138922421842817342u }, {  3633657960551215372u,  86826513651760839u },
+  { 18377130505971182927u, 108533142064701048u }, {  4524669058754427043u, 135666427580876311u },
+  {  9745447189362598758u,  84791517238047694u }, {  2958436949848472639u, 105989396547559618u },
+  { 12921418224165366607u, 132486745684449522u }, { 12687572408530742033u,  82804216052780951u },
+  { 11247779492236039638u, 103505270065976189u }, {   224666310012885835u, 129381587582470237u },
+  {  2446259452971747599u,  80863492239043898u }, { 12281196353069460307u, 101079365298804872u },
+  { 15351495441336825384u, 126349206623506090u }, { 14206370669262903769u,  78968254139691306u },
+  {  8534591299723853903u,  98710317674614133u }, { 15279925143082205283u, 123387897093267666u },
+  { 14161639232853766206u,  77117435683292291u }, { 13090363022639819853u,  96396794604115364u },
+  { 16362953778299774816u, 120495993255144205u }, { 12532689120651053212u,  75309995784465128u },
+  { 15665861400813816515u,  94137494730581410u }, { 10358954714162494836u, 117671868413226763u },
+  {  4168503687137865320u,  73544917758266727u }, {   598943590494943747u,  91931147197833409u },
+  {  5360365506546067587u, 114913933997291761u }, { 11312142901609972388u, 143642417496614701u },
+  {  9375932322719926695u,  89776510935384188u }, { 11719915403399908368u, 112220638669230235u },
+  { 10038208235822497557u, 140275798336537794u }, { 10885566165816448877u,  87672373960336121u },
+  { 18218643725697949000u, 109590467450420151u }, { 18161618638695048346u, 136988084313025189u },
+  { 13656854658398099168u,  85617552695640743u }, { 12459382304570236056u, 107021940869550929u },
+  {  1739169825430631358u, 133777426086938662u }, { 14922039196176308311u,  83610891304336663u },
+  { 14040862976792997485u, 104513614130420829u }, {  3716020665709083144u, 130642017663026037u },
+  {  4628355925281870917u,  81651261039391273u }, { 10397130925029726550u, 102064076299239091u },
+  {  8384727637859770284u, 127580095374048864u }, {  5240454773662356427u,  79737559608780540u },
+  {  6550568467077945534u,  99671949510975675u }, {  3576524565420044014u, 124589936888719594u },
+  {  6847013871814915412u,  77868710555449746u }, { 17782139376623420074u,  97335888194312182u },
+  { 13004302183924499284u, 121669860242890228u }, { 17351060901807587860u,  76043662651806392u },
+  {  3242082053549933210u,  95054578314757991u }, { 17887660622219580224u, 118818222893447488u },
+  { 11179787888887237640u,  74261389308404680u }, { 13974734861109047050u,  92826736635505850u },
+  {  8245046539531533005u, 116033420794382313u }, { 16682369133275677888u,  72520887996488945u },
+  {  7017903361312433648u,  90651109995611182u }, { 17995751238495317868u, 113313887494513977u },
+  {  8659630992836983623u, 141642359368142472u }, {  5412269370523114764u,  88526474605089045u },
+  { 11377022731581281359u, 110658093256361306u }, {  4997906377621825891u, 138322616570451633u },
+  { 14652906532082110942u,  86451635356532270u }, {  9092761128247862869u, 108064544195665338u },
+  {  2142579373455052779u, 135080680244581673u }, { 12868327154477877747u,  84425425152863545u },
+  {  2250350887815183471u, 105531781441079432u }, {  2812938609768979339u, 131914726801349290u },
+  {  6369772649532999991u,  82446704250843306u }, { 17185587848771025797u, 103058380313554132u },
+  {  3035240737254230630u, 128822975391942666u }, {  6508711479211282048u,  80514359619964166u },
+  { 17359261385868878368u, 100642949524955207u }, { 17087390713908710056u, 125803686906194009u },
+  {  3762090168551861929u,  78627304316371256u }, {  4702612710689827411u,  98284130395464070u },
+  { 15101637925217060072u, 122855162994330087u }, { 16356052730901744401u,  76784476871456304u },
+  {  1998321839917628885u,  95980596089320381u }, {  7109588318324424010u, 119975745111650476u },
+  { 13666864735807540814u,  74984840694781547u }, { 12471894901332038114u,  93731050868476934u },
+  {  6366496589810271835u, 117163813585596168u }, {  3979060368631419896u,  73227383490997605u },
+  {  9585511479216662775u,  91534229363747006u }, {  2758517312166052660u, 114417786704683758u },
+  { 12671518677062341634u, 143022233380854697u }, {  1002170145522881665u,  89388895863034186u },
+  { 10476084718758377889u, 111736119828792732u }, { 13095105898447972362u, 139670149785990915u },
+  {  5878598177316288774u,  87293843616244322u }, { 16571619758500136775u, 109117304520305402u },
+  { 11491152661270395161u, 136396630650381753u }, {   264441385652915120u,  85247894156488596u },
+  {   330551732066143900u, 106559867695610745u }, {  5024875683510067779u, 133199834619513431u },
+  { 10058076329834874218u,  83249896637195894u }, {  3349223375438816964u, 104062370796494868u },
+  {  4186529219298521205u, 130077963495618585u }, { 14145795808130045513u,  81298727184761615u },
+  { 13070558741735168987u, 101623408980952019u }, { 11726512408741573330u, 127029261226190024u },
+  {  7329070255463483331u,  79393288266368765u }, { 13773023837756742068u,  99241610332960956u },
+  { 17216279797195927585u, 124052012916201195u }, {  8454331864033760789u,  77532508072625747u },
+  {  5956228811614813082u,  96915635090782184u }, {  7445286014518516353u, 121144543863477730u },
+  {  9264989777501460624u,  75715339914673581u }, { 16192923240304213684u,  94644174893341976u },
+  {  1794409976670715490u, 118305218616677471u }, {  8039035263060279037u,  73940761635423419u },
+  {  5437108060397960892u,  92425952044279274u }, { 16019757112352226923u, 115532440055349092u },
+  {   788976158365366019u,  72207775034593183u }, { 14821278253238871236u,  90259718793241478u },
+  {  9303225779693813237u, 112824648491551848u }, { 11629032224617266546u, 141030810614439810u },
+  { 11879831158813179495u,  88144256634024881u }, {  1014730893234310657u, 110180320792531102u },
+  { 10491785653397664129u, 137725400990663877u }, {  8863209042587234033u,  86078375619164923u },
+  {  6467325284806654637u, 107597969523956154u }, { 17307528642863094104u, 134497461904945192u },
+  { 10817205401789433815u,  84060913690590745u }, { 18133192770664180173u, 105076142113238431u },
+  { 18054804944902837312u, 131345177641548039u }, { 18201782118205355176u,  82090736025967524u },
+  {  4305483574047142354u, 102613420032459406u }, { 14605226504413703751u, 128266775040574257u },
+  {  2210737537617482988u,  80166734400358911u }, { 16598479977304017447u, 100208418000448638u },
+  { 11524727934775246001u, 125260522500560798u }, {  2591268940807140847u,  78287826562850499u },
+  { 17074144231291089770u,  97859783203563123u }, { 16730994270686474309u, 122324729004453904u },
+  { 10456871419179046443u,  76452955627783690u }, {  3847717237119032246u,  95566194534729613u },
+  {  9421332564826178211u, 119457743168412016u }, {  5888332853016361382u,  74661089480257510u },
+  { 16583788103125227536u,  93326361850321887u }, { 16118049110479146516u, 116657952312902359u },
+  { 16991309721690548428u,  72911220195563974u }, { 12015765115258409727u,  91139025244454968u },
+  { 15019706394073012159u, 113923781555568710u }, {  9551260955736489391u, 142404726944460888u },
+  {  5969538097335305869u,  89002954340288055u }, {  2850236603241744433u, 111253692925360069u }
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+// clang-format on
+
+#endif // _LIBCPP_SRC_INCLUDE_RYU_D2S_FULL_TABLE_H
diff --git a/libcxx/src/include/ryu/d2s_intrinsics.h b/libcxx/src/include/ryu/d2s_intrinsics.h
new file mode 100644
index 000000000000..9f6632e1b2b4
--- /dev/null
+++ b/libcxx/src/include/ryu/d2s_intrinsics.h
@@ -0,0 +1,257 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// Copyright 2018 Ulf Adams
+// Copyright (c) Microsoft Corporation. All rights reserved.
+
+// Boost Software License - Version 1.0 - August 17th, 2003
+
+// Permission is hereby granted, free of charge, to any person or organization
+// obtaining a copy of the software and accompanying documentation covered by
+// this license (the "Software") to use, reproduce, display, distribute,
+// execute, and transmit the Software, and to prepare derivative works of the
+// Software, and to permit third-parties to whom the Software is furnished to
+// do so, all subject to the following:
+
+// The copyright notices in the Software and this entire statement, including
+// the above license grant, this restriction and the following disclaimer,
+// must be included in all copies of the Software, in whole or in part, and
+// all derivative works of the Software, unless such copies or derivative
+// works are solely in the form of machine-executable object code generated by
+// a source language processor.
+
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+// SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+// FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+// DEALINGS IN THE SOFTWARE.
+
+#ifndef _LIBCPP_SRC_INCLUDE_RYU_DS2_INTRINSICS_H
+#define _LIBCPP_SRC_INCLUDE_RYU_DS2_INTRINSICS_H
+
+// Avoid formatting to keep the changes with the original code minimal.
+// clang-format off
+
+#include "__config"
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if defined(_M_X64) && defined(_MSC_VER)
+#define _LIBCPP_INTRINSIC128 1
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline uint64_t __ryu_umul128(const uint64_t __a, const uint64_t __b, uint64_t* const __productHi) {
+  return _umul128(__a, __b, __productHi);
+}
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline uint64_t __ryu_shiftright128(const uint64_t __lo, const uint64_t __hi, const uint32_t __dist) {
+  // For the __shiftright128 intrinsic, the shift value is always
+  // modulo 64.
+  // In the current implementation of the double-precision version
+  // of Ryu, the shift value is always < 64.
+  // (The shift value is in the range [49, 58].)
+  // Check this here in case a future change requires larger shift
+  // values. In this case this function needs to be adjusted.
+  _LIBCPP_ASSERT(__dist < 64, "");
+  return __shiftright128(__lo, __hi, static_cast<unsigned char>(__dist));
+}
+
+// ^^^ intrinsics available ^^^ / vvv __int128 available vvv
+#elif defined(__SIZEOF_INT128__) && ( \
+    (defined(__clang__) && !defined(_MSC_VER)) || \
+    (defined(__GNUC__) && !defined(__clang__) && !defined(__CUDACC__)))
+#define _LIBCPP_INTRINSIC128 1
+  // We have __uint128 support in clang or gcc
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline uint64_t __ryu_umul128(const uint64_t __a, const uint64_t __b, uint64_t* const __productHi) {
+  auto __temp = __a * (unsigned __int128)__b;
+  *__productHi = __temp >> 64;
+  return static_cast<uint64_t>(__temp);
+}
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline uint64_t __ryu_shiftright128(const uint64_t __lo, const uint64_t __hi, const uint32_t __dist) {
+  // In the current implementation of the double-precision version
+  // of Ryu, the shift value is always < 64.
+  // (The shift value is in the range [49, 58].)
+  // Check this here in case a future change requires larger shift
+  // values. In this case this function needs to be adjusted.
+  _LIBCPP_ASSERT(__dist < 64, "");
+  auto __temp = __lo | ((unsigned __int128)__hi << 64);
+  // For x64 128-bit shfits using the `shrd` instruction and two 64-bit
+  // registers, the shift value is modulo 64.  Thus the `& 63` is free.
+  return static_cast<uint64_t>(__temp >> (__dist & 63));
+}
+#else // ^^^ __int128 available ^^^ / vvv intrinsics unavailable vvv
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline _LIBCPP_ALWAYS_INLINE uint64_t __ryu_umul128(const uint64_t __a, const uint64_t __b, uint64_t* const __productHi) {
+  // TRANSITION, VSO-634761
+  // The casts here help MSVC to avoid calls to the __allmul library function.
+  const uint32_t __aLo = static_cast<uint32_t>(__a);
+  const uint32_t __aHi = static_cast<uint32_t>(__a >> 32);
+  const uint32_t __bLo = static_cast<uint32_t>(__b);
+  const uint32_t __bHi = static_cast<uint32_t>(__b >> 32);
+
+  const uint64_t __b00 = static_cast<uint64_t>(__aLo) * __bLo;
+  const uint64_t __b01 = static_cast<uint64_t>(__aLo) * __bHi;
+  const uint64_t __b10 = static_cast<uint64_t>(__aHi) * __bLo;
+  const uint64_t __b11 = static_cast<uint64_t>(__aHi) * __bHi;
+
+  const uint32_t __b00Lo = static_cast<uint32_t>(__b00);
+  const uint32_t __b00Hi = static_cast<uint32_t>(__b00 >> 32);
+
+  const uint64_t __mid1 = __b10 + __b00Hi;
+  const uint32_t __mid1Lo = static_cast<uint32_t>(__mid1);
+  const uint32_t __mid1Hi = static_cast<uint32_t>(__mid1 >> 32);
+
+  const uint64_t __mid2 = __b01 + __mid1Lo;
+  const uint32_t __mid2Lo = static_cast<uint32_t>(__mid2);
+  const uint32_t __mid2Hi = static_cast<uint32_t>(__mid2 >> 32);
+
+  const uint64_t __pHi = __b11 + __mid1Hi + __mid2Hi;
+  const uint64_t __pLo = (static_cast<uint64_t>(__mid2Lo) << 32) | __b00Lo;
+
+  *__productHi = __pHi;
+  return __pLo;
+}
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline uint64_t __ryu_shiftright128(const uint64_t __lo, const uint64_t __hi, const uint32_t __dist) {
+  // We don't need to handle the case __dist >= 64 here (see above).
+  _LIBCPP_ASSERT(__dist < 64, "");
+#ifdef _LIBCPP_64_BIT
+  _LIBCPP_ASSERT(__dist > 0, "");
+  return (__hi << (64 - __dist)) | (__lo >> __dist);
+#else // ^^^ 64-bit ^^^ / vvv 32-bit vvv
+  // Avoid a 64-bit shift by taking advantage of the range of shift values.
+  _LIBCPP_ASSERT(__dist >= 32, "");
+  return (__hi << (64 - __dist)) | (static_cast<uint32_t>(__lo >> 32) >> (__dist - 32));
+#endif // ^^^ 32-bit ^^^
+}
+
+#endif // ^^^ intrinsics unavailable ^^^
+
+#ifndef _LIBCPP_64_BIT
+
+// Returns the high 64 bits of the 128-bit product of __a and __b.
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline uint64_t __umulh(const uint64_t __a, const uint64_t __b) {
+  // Reuse the __ryu_umul128 implementation.
+  // Optimizers will likely eliminate the instructions used to compute the
+  // low part of the product.
+  uint64_t __hi;
+  (void) __ryu_umul128(__a, __b, &__hi);
+  return __hi;
+}
+
+// On 32-bit platforms, compilers typically generate calls to library
+// functions for 64-bit divisions, even if the divisor is a constant.
+//
+// TRANSITION, LLVM-37932
+//
+// The functions here perform division-by-constant using multiplications
+// in the same way as 64-bit compilers would do.
+//
+// NB:
+// The multipliers and shift values are the ones generated by clang x64
+// for expressions like x/5, x/10, etc.
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline uint64_t __div5(const uint64_t __x) {
+  return __umulh(__x, 0xCCCCCCCCCCCCCCCDu) >> 2;
+}
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline uint64_t __div10(const uint64_t __x) {
+  return __umulh(__x, 0xCCCCCCCCCCCCCCCDu) >> 3;
+}
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline uint64_t __div100(const uint64_t __x) {
+  return __umulh(__x >> 2, 0x28F5C28F5C28F5C3u) >> 2;
+}
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline uint64_t __div1e8(const uint64_t __x) {
+  return __umulh(__x, 0xABCC77118461CEFDu) >> 26;
+}
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline uint64_t __div1e9(const uint64_t __x) {
+  return __umulh(__x >> 9, 0x44B82FA09B5A53u) >> 11;
+}
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline uint32_t __mod1e9(const uint64_t __x) {
+  // Avoid 64-bit math as much as possible.
+  // Returning static_cast<uint32_t>(__x - 1000000000 * __div1e9(__x)) would
+  // perform 32x64-bit multiplication and 64-bit subtraction.
+  // __x and 1000000000 * __div1e9(__x) are guaranteed to differ by
+  // less than 10^9, so their highest 32 bits must be identical,
+  // so we can truncate both sides to uint32_t before subtracting.
+  // We can also simplify static_cast<uint32_t>(1000000000 * __div1e9(__x)).
+  // We can truncate before multiplying instead of after, as multiplying
+  // the highest 32 bits of __div1e9(__x) can't affect the lowest 32 bits.
+  return static_cast<uint32_t>(__x) - 1000000000 * static_cast<uint32_t>(__div1e9(__x));
+}
+
+#else // ^^^ 32-bit ^^^ / vvv 64-bit vvv
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline uint64_t __div5(const uint64_t __x) {
+  return __x / 5;
+}
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline uint64_t __div10(const uint64_t __x) {
+  return __x / 10;
+}
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline uint64_t __div100(const uint64_t __x) {
+  return __x / 100;
+}
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline uint64_t __div1e8(const uint64_t __x) {
+  return __x / 100000000;
+}
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline uint64_t __div1e9(const uint64_t __x) {
+  return __x / 1000000000;
+}
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline uint32_t __mod1e9(const uint64_t __x) {
+  return static_cast<uint32_t>(__x - 1000000000 * __div1e9(__x));
+}
+
+#endif // ^^^ 64-bit ^^^
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline uint32_t __pow5Factor(uint64_t __value) {
+  uint32_t __count = 0;
+  for (;;) {
+    _LIBCPP_ASSERT(__value != 0, "");
+    const uint64_t __q = __div5(__value);
+    const uint32_t __r = static_cast<uint32_t>(__value) - 5 * static_cast<uint32_t>(__q);
+    if (__r != 0) {
+      break;
+    }
+    __value = __q;
+    ++__count;
+  }
+  return __count;
+}
+
+// Returns true if __value is divisible by 5^__p.
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline bool __multipleOfPowerOf5(const uint64_t __value, const uint32_t __p) {
+  // I tried a case distinction on __p, but there was no performance difference.
+  return __pow5Factor(__value) >= __p;
+}
+
+// Returns true if __value is divisible by 2^__p.
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline bool __multipleOfPowerOf2(const uint64_t __value, const uint32_t __p) {
+  _LIBCPP_ASSERT(__value != 0, "");
+  _LIBCPP_ASSERT(__p < 64, "");
+  // __builtin_ctzll doesn't appear to be faster here.
+  return (__value & ((1ull << __p) - 1)) == 0;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+// clang-format on
+
+#endif // _LIBCPP_SRC_INCLUDE_RYU_DS2_INTRINSICS_H
diff --git a/libcxx/src/include/ryu/digit_table.h b/libcxx/src/include/ryu/digit_table.h
new file mode 100644
index 000000000000..3962f6693140
--- /dev/null
+++ b/libcxx/src/include/ryu/digit_table.h
@@ -0,0 +1,68 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// Copyright 2018 Ulf Adams
+// Copyright (c) Microsoft Corporation. All rights reserved.
+
+// Boost Software License - Version 1.0 - August 17th, 2003
+
+// Permission is hereby granted, free of charge, to any person or organization
+// obtaining a copy of the software and accompanying documentation covered by
+// this license (the "Software") to use, reproduce, display, distribute,
+// execute, and transmit the Software, and to prepare derivative works of the
+// Software, and to permit third-parties to whom the Software is furnished to
+// do so, all subject to the following:
+
+// The copyright notices in the Software and this entire statement, including
+// the above license grant, this restriction and the following disclaimer,
+// must be included in all copies of the Software, in whole or in part, and
+// all derivative works of the Software, unless such copies or derivative
+// works are solely in the form of machine-executable object code generated by
+// a source language processor.
+
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+// SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+// FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+// DEALINGS IN THE SOFTWARE.
+
+#ifndef _LIBCPP_SRC_INCLUDE_RYU_DIGIT_TABLE_H
+#define _LIBCPP_SRC_INCLUDE_RYU_DIGIT_TABLE_H
+
+// Avoid formatting to keep the changes with the original code minimal.
+// clang-format off
+
+#include "__config"
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// A table of all two-digit numbers. This is used to speed up decimal digit
+// generation by copying pairs of digits into the final output.
+inline constexpr char __DIGIT_TABLE[200] = {
+  '0','0','0','1','0','2','0','3','0','4','0','5','0','6','0','7','0','8','0','9',
+  '1','0','1','1','1','2','1','3','1','4','1','5','1','6','1','7','1','8','1','9',
+  '2','0','2','1','2','2','2','3','2','4','2','5','2','6','2','7','2','8','2','9',
+  '3','0','3','1','3','2','3','3','3','4','3','5','3','6','3','7','3','8','3','9',
+  '4','0','4','1','4','2','4','3','4','4','4','5','4','6','4','7','4','8','4','9',
+  '5','0','5','1','5','2','5','3','5','4','5','5','5','6','5','7','5','8','5','9',
+  '6','0','6','1','6','2','6','3','6','4','6','5','6','6','6','7','6','8','6','9',
+  '7','0','7','1','7','2','7','3','7','4','7','5','7','6','7','7','7','8','7','9',
+  '8','0','8','1','8','2','8','3','8','4','8','5','8','6','8','7','8','8','8','9',
+  '9','0','9','1','9','2','9','3','9','4','9','5','9','6','9','7','9','8','9','9'
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+// clang-format on
+
+#endif // _LIBCPP_SRC_INCLUDE_RYU_DIGIT_TABLE_H
diff --git a/libcxx/src/include/ryu/f2s.h b/libcxx/src/include/ryu/f2s.h
new file mode 100644
index 000000000000..80fdcd458b72
--- /dev/null
+++ b/libcxx/src/include/ryu/f2s.h
@@ -0,0 +1,55 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// Copyright 2018 Ulf Adams
+// Copyright (c) Microsoft Corporation. All rights reserved.
+
+// Boost Software License - Version 1.0 - August 17th, 2003
+
+// Permission is hereby granted, free of charge, to any person or organization
+// obtaining a copy of the software and accompanying documentation covered by
+// this license (the "Software") to use, reproduce, display, distribute,
+// execute, and transmit the Software, and to prepare derivative works of the
+// Software, and to permit third-parties to whom the Software is furnished to
+// do so, all subject to the following:
+
+// The copyright notices in the Software and this entire statement, including
+// the above license grant, this restriction and the following disclaimer,
+// must be included in all copies of the Software, in whole or in part, and
+// all derivative works of the Software, unless such copies or derivative
+// works are solely in the form of machine-executable object code generated by
+// a source language processor.
+
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+// SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+// FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+// DEALINGS IN THE SOFTWARE.
+
+#ifndef _LIBCPP_SRC_INCLUDE_RYU_FS2_H
+#define _LIBCPP_SRC_INCLUDE_RYU_FS2_H
+
+// Avoid formatting to keep the changes with the original code minimal.
+// clang-format off
+
+#include "__config"
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+[[nodiscard]] to_chars_result __f2s_buffered_n(char* const _First, char* const _Last, const float __f, const chars_format _Fmt);
+
+_LIBCPP_END_NAMESPACE_STD
+
+// clang-format on
+
+#endif // _LIBCPP_SRC_INCLUDE_RYU_FS2_H
diff --git a/libcxx/src/include/ryu/ryu.h b/libcxx/src/include/ryu/ryu.h
new file mode 100644
index 000000000000..9c4025fc61a5
--- /dev/null
+++ b/libcxx/src/include/ryu/ryu.h
@@ -0,0 +1,148 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+
+// Copyright 2018 Ulf Adams
+// Copyright (c) Microsoft Corporation. All rights reserved.
+
+// Boost Software License - Version 1.0 - August 17th, 2003
+
+// Permission is hereby granted, free of charge, to any person or organization
+// obtaining a copy of the software and accompanying documentation covered by
+// this license (the "Software") to use, reproduce, display, distribute,
+// execute, and transmit the Software, and to prepare derivative works of the
+// Software, and to permit third-parties to whom the Software is furnished to
+// do so, all subject to the following:
+
+// The copyright notices in the Software and this entire statement, including
+// the above license grant, this restriction and the following disclaimer,
+// must be included in all copies of the Software, in whole or in part, and
+// all derivative works of the Software, unless such copies or derivative
+// works are solely in the form of machine-executable object code generated by
+// a source language processor.
+
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+// SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+// FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+// DEALINGS IN THE SOFTWARE.
+
+#ifndef _LIBCPP_SRC_INCLUDE_RYU_RYU_H
+#define _LIBCPP_SRC_INCLUDE_RYU_RYU_H
+
+// Avoid formatting to keep the changes with the original code minimal.
+// clang-format off
+
+#include "__charconv/chars_format.h"
+#include "__charconv/to_chars_result.h"
+#include "__config"
+#include "__debug"
+#include "__errc"
+#include "cstdint"
+#include "cstring"
+#include "type_traits"
+#include "include/ryu/f2s.h"
+#include "include/ryu/d2s.h"
+#include "include/ryu/d2fixed.h"
+
+#if defined(_M_X64) && defined(_LIBCPP_COMPILER_MSVC)
+#include <intrin0.h> // for _umul128() and __shiftright128()
+#endif // defined(_M_X64) && defined(_LIBCPP_COMPILER_MSVC)
+
+#if defined(_WIN64) || defined(_M_AMD64) || defined(__x86_64__) ||  defined(__aarch64__)
+#define _LIBCPP_64_BIT
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// https://github.com/ulfjack/ryu/tree/59661c3/ryu
+
+#if !defined(_LIBCPP_COMPILER_MSVC)
+_LIBCPP_HIDE_FROM_ABI inline unsigned char _BitScanForward64(unsigned long* __index, unsigned long long __mask) {
+  if (__mask == 0) {
+    return false;
+  }
+  *__index = __builtin_ctzll(__mask);
+  return true;
+}
+
+_LIBCPP_HIDE_FROM_ABI inline unsigned char _BitScanForward(unsigned long* __index, unsigned int __mask) {
+  if (__mask == 0) {
+    return false;
+  }
+  *__index = __builtin_ctz(__mask);
+  return true;
+}
+#endif  // _LIBCPP_COMPILER_MSVC
+
+template <class _Floating>
+[[nodiscard]] to_chars_result _Floating_to_chars_ryu(
+    char* const _First, char* const _Last, const _Floating _Value, const chars_format _Fmt) noexcept {
+    if constexpr (_IsSame<_Floating, float>::value) {
+        return __f2s_buffered_n(_First, _Last, _Value, _Fmt);
+    } else {
+        return __d2s_buffered_n(_First, _Last, _Value, _Fmt);
+    }
+}
+
+template <class _Floating>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI to_chars_result _Floating_to_chars_scientific_precision(
+    char* const _First, char* const _Last, const _Floating _Value, int _Precision) noexcept {
+
+    // C11 7.21.6.1 "The fprintf function"/5:
+    // "A negative precision argument is taken as if the precision were omitted."
+    // /8: "e,E [...] if the precision is missing, it is taken as 6"
+
+    if (_Precision < 0) {
+        _Precision = 6;
+    } else if (_Precision < 1'000'000'000) { // Match ' to fix compilation with GCC in C++11 mode
+        // _Precision is ok.
+    } else {
+        // Avoid integer overflow.
+        // (This defensive check is slightly nonconformant; it can be carefully improved in the future.)
+        return {_Last, errc::value_too_large};
+    }
+
+    return __d2exp_buffered_n(_First, _Last, _Value, static_cast<uint32_t>(_Precision));
+}
+
+template <class _Floating>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI to_chars_result _Floating_to_chars_fixed_precision(
+    char* const _First, char* const _Last, const _Floating _Value, int _Precision) noexcept {
+
+    // C11 7.21.6.1 "The fprintf function"/5:
+    // "A negative precision argument is taken as if the precision were omitted."
+    // /8: "f,F [...] If the precision is missing, it is taken as 6"
+
+    if (_Precision < 0) {
+        _Precision = 6;
+    } else if (_Precision < 1'000'000'000) { // Match ' to fix compilation with GCC in C++11 mode
+        // _Precision is ok.
+    } else {
+        // Avoid integer overflow.
+        // (This defensive check is slightly nonconformant; it can be carefully improved in the future.)
+        return {_Last, errc::value_too_large};
+    }
+
+    return __d2fixed_buffered_n(_First, _Last, _Value, static_cast<uint32_t>(_Precision));
+}
+
+#undef _LIBCPP_64_BIT
+#undef _LIBCPP_INTRINSIC128
+
+_LIBCPP_END_NAMESPACE_STD
+
+// clang-format on
+
+#endif // _LIBCPP_SRC_INCLUDE_RYU_RYU_H
diff --git a/libcxx/src/include/to_chars_floating_point.h b/libcxx/src/include/to_chars_floating_point.h
new file mode 100644
index 000000000000..081d671fc59e
--- /dev/null
+++ b/libcxx/src/include/to_chars_floating_point.h
@@ -0,0 +1,1076 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// This implementation is dedicated to the memory of Mary and Thavatchai.
+
+#ifndef _LIBCPP_SRC_INCLUDE_TO_CHARS_FLOATING_POINT_H
+#define _LIBCPP_SRC_INCLUDE_TO_CHARS_FLOATING_POINT_H
+
+// Avoid formatting to keep the changes with the original code minimal.
+// clang-format off
+
+#include "__algorithm/find.h"
+#include "__algorithm/find_if.h"
+#include "__algorithm/lower_bound.h"
+#include "__algorithm/min.h"
+#include "__config"
+#include "__iterator/access.h"
+#include "__iterator/size.h"
+#include "bit"
+#include "cfloat"
+#include "climits"
+#include "include/ryu/ryu.h"
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace __itoa {
+inline constexpr char _Charconv_digits[] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e',
+    'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'};
+static_assert(_VSTD::size(_Charconv_digits) == 36);
+} // __itoa
+
+// vvvvvvvvvv DERIVED FROM corecrt_internal_fltintrn.h vvvvvvvvvv
+
+template <class _FloatingType>
+struct _Floating_type_traits;
+
+template <>
+struct _Floating_type_traits<float> {
+    static constexpr int32_t _Mantissa_bits = FLT_MANT_DIG;
+    static constexpr int32_t _Exponent_bits = sizeof(float) * CHAR_BIT - FLT_MANT_DIG;
+
+    static constexpr int32_t _Maximum_binary_exponent = FLT_MAX_EXP - 1;
+    static constexpr int32_t _Minimum_binary_exponent = FLT_MIN_EXP - 1;
+
+    static constexpr int32_t _Exponent_bias = 127;
+
+    static constexpr int32_t _Sign_shift     = _Exponent_bits + _Mantissa_bits - 1;
+    static constexpr int32_t _Exponent_shift = _Mantissa_bits - 1;
+
+    using _Uint_type = uint32_t;
+
+    static constexpr uint32_t _Exponent_mask             = (1u << _Exponent_bits) - 1;
+    static constexpr uint32_t _Normal_mantissa_mask      = (1u << _Mantissa_bits) - 1;
+    static constexpr uint32_t _Denormal_mantissa_mask    = (1u << (_Mantissa_bits - 1)) - 1;
+    static constexpr uint32_t _Special_nan_mantissa_mask = 1u << (_Mantissa_bits - 2);
+    static constexpr uint32_t _Shifted_sign_mask         = 1u << _Sign_shift;
+    static constexpr uint32_t _Shifted_exponent_mask     = _Exponent_mask << _Exponent_shift;
+};
+
+template <>
+struct _Floating_type_traits<double> {
+    static constexpr int32_t _Mantissa_bits = DBL_MANT_DIG;
+    static constexpr int32_t _Exponent_bits = sizeof(double) * CHAR_BIT - DBL_MANT_DIG;
+
+    static constexpr int32_t _Maximum_binary_exponent = DBL_MAX_EXP - 1;
+    static constexpr int32_t _Minimum_binary_exponent = DBL_MIN_EXP - 1;
+
+    static constexpr int32_t _Exponent_bias = 1023;
+
+    static constexpr int32_t _Sign_shift     = _Exponent_bits + _Mantissa_bits - 1;
+    static constexpr int32_t _Exponent_shift = _Mantissa_bits - 1;
+
+    using _Uint_type = uint64_t;
+
+    static constexpr uint64_t _Exponent_mask             = (1ULL << _Exponent_bits) - 1;
+    static constexpr uint64_t _Normal_mantissa_mask      = (1ULL << _Mantissa_bits) - 1;
+    static constexpr uint64_t _Denormal_mantissa_mask    = (1ULL << (_Mantissa_bits - 1)) - 1;
+    static constexpr uint64_t _Special_nan_mantissa_mask = 1ULL << (_Mantissa_bits - 2);
+    static constexpr uint64_t _Shifted_sign_mask         = 1ULL << _Sign_shift;
+    static constexpr uint64_t _Shifted_exponent_mask     = _Exponent_mask << _Exponent_shift;
+};
+
+// ^^^^^^^^^^ DERIVED FROM corecrt_internal_fltintrn.h ^^^^^^^^^^
+
+// FUNCTION to_chars (FLOATING-POINT TO STRING)
+template <class _Floating>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI
+to_chars_result _Floating_to_chars_hex_precision(
+    char* _First, char* const _Last, const _Floating _Value, int _Precision) noexcept {
+
+    // * Determine the effective _Precision.
+    // * Later, we'll decrement _Precision when printing each hexit after the decimal point.
+
+    // The hexits after the decimal point correspond to the explicitly stored fraction bits.
+    // float explicitly stores 23 fraction bits. 23 / 4 == 5.75, which is 6 hexits.
+    // double explicitly stores 52 fraction bits. 52 / 4 == 13, which is 13 hexits.
+    constexpr int _Full_precision         = _IsSame<_Floating, float>::value ? 6 : 13;
+    constexpr int _Adjusted_explicit_bits = _Full_precision * 4;
+
+    if (_Precision < 0) {
+        // C11 7.21.6.1 "The fprintf function"/5: "A negative precision argument is taken as if the precision were
+        // omitted." /8: "if the precision is missing and FLT_RADIX is a power of 2, then the precision is sufficient
+        // for an exact representation of the value"
+        _Precision = _Full_precision;
+    }
+
+    // * Extract the _Ieee_mantissa and _Ieee_exponent.
+    using _Traits    = _Floating_type_traits<_Floating>;
+    using _Uint_type = typename _Traits::_Uint_type;
+
+    const _Uint_type _Uint_value    = _VSTD::bit_cast<_Uint_type>(_Value);
+    const _Uint_type _Ieee_mantissa = _Uint_value & _Traits::_Denormal_mantissa_mask;
+    const int32_t _Ieee_exponent    = static_cast<int32_t>(_Uint_value >> _Traits::_Exponent_shift);
+
+    // * Prepare the _Adjusted_mantissa. This is aligned to hexit boundaries,
+    // * with the implicit bit restored (0 for zero values and subnormal values, 1 for normal values).
+    // * Also calculate the _Unbiased_exponent. This unifies the processing of zero, subnormal, and normal values.
+    _Uint_type _Adjusted_mantissa;
+
+    if constexpr (_IsSame<_Floating, float>::value) {
+        _Adjusted_mantissa = _Ieee_mantissa << 1; // align to hexit boundary (23 isn't divisible by 4)
+    } else {
+        _Adjusted_mantissa = _Ieee_mantissa; // already aligned (52 is divisible by 4)
+    }
+
+    int32_t _Unbiased_exponent;
+
+    if (_Ieee_exponent == 0) { // zero or subnormal
+        // implicit bit is 0
+
+        if (_Ieee_mantissa == 0) { // zero
+            // C11 7.21.6.1 "The fprintf function"/8: "If the value is zero, the exponent is zero."
+            _Unbiased_exponent = 0;
+        } else { // subnormal
+            _Unbiased_exponent = 1 - _Traits::_Exponent_bias;
+        }
+    } else { // normal
+        _Adjusted_mantissa |= _Uint_type{1} << _Adjusted_explicit_bits; // implicit bit is 1
+        _Unbiased_exponent = _Ieee_exponent - _Traits::_Exponent_bias;
+    }
+
+    // _Unbiased_exponent is within [-126, 127] for float, [-1022, 1023] for double.
+
+    // * Decompose _Unbiased_exponent into _Sign_character and _Absolute_exponent.
+    char _Sign_character;
+    uint32_t _Absolute_exponent;
+
+    if (_Unbiased_exponent < 0) {
+        _Sign_character    = '-';
+        _Absolute_exponent = static_cast<uint32_t>(-_Unbiased_exponent);
+    } else {
+        _Sign_character    = '+';
+        _Absolute_exponent = static_cast<uint32_t>(_Unbiased_exponent);
+    }
+
+    // _Absolute_exponent is within [0, 127] for float, [0, 1023] for double.
+
+    // * Perform a single bounds check.
+    {
+        int32_t _Exponent_length;
+
+        if (_Absolute_exponent < 10) {
+            _Exponent_length = 1;
+        } else if (_Absolute_exponent < 100) {
+            _Exponent_length = 2;
+        } else if constexpr (_IsSame<_Floating, float>::value) {
+            _Exponent_length = 3;
+        } else if (_Absolute_exponent < 1000) {
+            _Exponent_length = 3;
+        } else {
+            _Exponent_length = 4;
+        }
+
+        // _Precision might be enormous; avoid integer overflow by testing it separately.
+        ptrdiff_t _Buffer_size = _Last - _First;
+
+        if (_Buffer_size < _Precision) {
+            return {_Last, errc::value_too_large};
+        }
+
+        _Buffer_size -= _Precision;
+
+        const int32_t _Length_excluding_precision = 1 // leading hexit
+                                                    + static_cast<int32_t>(_Precision > 0) // possible decimal point
+                                                    // excluding `+ _Precision`, hexits after decimal point
+                                                    + 2 // "p+" or "p-"
+                                                    + _Exponent_length; // exponent
+
+        if (_Buffer_size < _Length_excluding_precision) {
+            return {_Last, errc::value_too_large};
+        }
+    }
+
+    // * Perform rounding when we've been asked to omit hexits.
+    if (_Precision < _Full_precision) {
+        // _Precision is within [0, 5] for float, [0, 12] for double.
+
+        // _Dropped_bits is within [4, 24] for float, [4, 52] for double.
+        const int _Dropped_bits = (_Full_precision - _Precision) * 4;
+
+        // Perform rounding by adding an appropriately-shifted bit.
+
+        // This can propagate carries all the way into the leading hexit. Examples:
+        // "0.ff9" rounded to a precision of 2 is "1.00".
+        // "1.ff9" rounded to a precision of 2 is "2.00".
+
+        // Note that the leading hexit participates in the rounding decision. Examples:
+        // "0.8" rounded to a precision of 0 is "0".
+        // "1.8" rounded to a precision of 0 is "2".
+
+        // Reference implementation with suboptimal codegen:
+        // bool _Should_round_up(bool _Lsb_bit, bool _Round_bit, bool _Has_tail_bits) {
+        //    // If there are no insignificant set bits, the value is exactly-representable and should not be rounded.
+        //    //
+        //    // If there are insignificant set bits, we need to round according to round_to_nearest.
+        //    // We need to handle two cases: we round up if either [1] the value is slightly greater
+        //    // than the midpoint between two exactly-representable values or [2] the value is exactly the midpoint
+        //    // between two exactly-representable values and the greater of the two is even (this is "round-to-even").
+        //    return _Round_bit && (_Has_tail_bits || _Lsb_bit);
+        //}
+        // const bool _Lsb_bit       = (_Adjusted_mantissa & (_Uint_type{1} << _Dropped_bits)) != 0;
+        // const bool _Round_bit     = (_Adjusted_mantissa & (_Uint_type{1} << (_Dropped_bits - 1))) != 0;
+        // const bool _Has_tail_bits = (_Adjusted_mantissa & ((_Uint_type{1} << (_Dropped_bits - 1)) - 1)) != 0;
+        // const bool _Should_round = _Should_round_up(_Lsb_bit, _Round_bit, _Has_tail_bits);
+        // _Adjusted_mantissa += _Uint_type{_Should_round} << _Dropped_bits;
+
+        // Example for optimized implementation: Let _Dropped_bits be 8.
+        //          Bit index: ...[8]76543210
+        // _Adjusted_mantissa: ...[L]RTTTTTTT (not depicting known details, like hexit alignment)
+        // By focusing on the bit at index _Dropped_bits, we can avoid unnecessary branching and shifting.
+
+        // Bit index: ...[8]76543210
+        //  _Lsb_bit: ...[L]RTTTTTTT
+        const _Uint_type _Lsb_bit = _Adjusted_mantissa;
+
+        //  Bit index: ...9[8]76543210
+        // _Round_bit: ...L[R]TTTTTTT0
+        const _Uint_type _Round_bit = _Adjusted_mantissa << 1;
+
+        // We can detect (without branching) whether any of the trailing bits are set.
+        // Due to _Should_round below, this computation will be used if and only if R is 1, so we can assume that here.
+        //      Bit index: ...9[8]76543210
+        //     _Round_bit: ...L[1]TTTTTTT0
+        // _Has_tail_bits: ....[H]........
+
+        // If all of the trailing bits T are 0, then `_Round_bit - 1` will produce 0 for H (due to R being 1).
+        // If any of the trailing bits T are 1, then `_Round_bit - 1` will produce 1 for H (due to R being 1).
+        const _Uint_type _Has_tail_bits = _Round_bit - 1;
+
+        // Finally, we can use _Should_round_up() logic with bitwise-AND and bitwise-OR,
+        // selecting just the bit at index _Dropped_bits. This is the appropriately-shifted bit that we want.
+        const _Uint_type _Should_round = _Round_bit & (_Has_tail_bits | _Lsb_bit) & (_Uint_type{1} << _Dropped_bits);
+
+        // This rounding technique is dedicated to the memory of Peppermint. =^..^=
+        _Adjusted_mantissa += _Should_round;
+    }
+
+    // * Print the leading hexit, then mask it away.
+    {
+        const uint32_t _Nibble = static_cast<uint32_t>(_Adjusted_mantissa >> _Adjusted_explicit_bits);
+        _LIBCPP_ASSERT(_Nibble < 3, "");
+        const char _Leading_hexit = static_cast<char>('0' + _Nibble);
+
+        *_First++ = _Leading_hexit;
+
+        constexpr _Uint_type _Mask = (_Uint_type{1} << _Adjusted_explicit_bits) - 1;
+        _Adjusted_mantissa &= _Mask;
+    }
+
+    // * Print the decimal point and trailing hexits.
+
+    // C11 7.21.6.1 "The fprintf function"/8:
+    // "if the precision is zero and the # flag is not specified, no decimal-point character appears."
+    if (_Precision > 0) {
+        *_First++ = '.';
+
+        int32_t _Number_of_bits_remaining = _Adjusted_explicit_bits; // 24 for float, 52 for double
+
+        for (;;) {
+            _LIBCPP_ASSERT(_Number_of_bits_remaining >= 4, "");
+            _LIBCPP_ASSERT(_Number_of_bits_remaining % 4 == 0, "");
+            _Number_of_bits_remaining -= 4;
+
+            const uint32_t _Nibble = static_cast<uint32_t>(_Adjusted_mantissa >> _Number_of_bits_remaining);
+            _LIBCPP_ASSERT(_Nibble < 16, "");
+            const char _Hexit = __itoa::_Charconv_digits[_Nibble];
+
+            *_First++ = _Hexit;
+
+            // _Precision is the number of hexits that still need to be printed.
+            --_Precision;
+            if (_Precision == 0) {
+                break; // We're completely done with this phase.
+            }
+            // Otherwise, we need to keep printing hexits.
+
+            if (_Number_of_bits_remaining == 0) {
+                // We've finished printing _Adjusted_mantissa, so all remaining hexits are '0'.
+                _VSTD::memset(_First, '0', static_cast<size_t>(_Precision));
+                _First += _Precision;
+                break;
+            }
+
+            // Mask away the hexit that we just printed, then keep looping.
+            // (We skip this when breaking out of the loop above, because _Adjusted_mantissa isn't used later.)
+            const _Uint_type _Mask = (_Uint_type{1} << _Number_of_bits_remaining) - 1;
+            _Adjusted_mantissa &= _Mask;
+        }
+    }
+
+    // * Print the exponent.
+
+    // C11 7.21.6.1 "The fprintf function"/8: "The exponent always contains at least one digit, and only as many more
+    // digits as necessary to represent the decimal exponent of 2."
+
+    // Performance note: We should take advantage of the known ranges of possible exponents.
+
+    *_First++ = 'p';
+    *_First++ = _Sign_character;
+
+    // We've already printed '-' if necessary, so uint32_t _Absolute_exponent avoids testing that again.
+    return _VSTD::to_chars(_First, _Last, _Absolute_exponent);
+}
+
+template <class _Floating>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI
+to_chars_result _Floating_to_chars_hex_shortest(
+    char* _First, char* const _Last, const _Floating _Value) noexcept {
+
+    // This prints "1.728p+0" instead of "2.e5p-1".
+    // This prints "0.000002p-126" instead of "1p-149" for float.
+    // This prints "0.0000000000001p-1022" instead of "1p-1074" for double.
+    // This prioritizes being consistent with printf's de facto behavior (and hex-precision's behavior)
+    // over minimizing the number of characters printed.
+
+    using _Traits    = _Floating_type_traits<_Floating>;
+    using _Uint_type = typename _Traits::_Uint_type;
+
+    const _Uint_type _Uint_value = _VSTD::bit_cast<_Uint_type>(_Value);
+
+    if (_Uint_value == 0) { // zero detected; write "0p+0" and return
+        // C11 7.21.6.1 "The fprintf function"/8: "If the value is zero, the exponent is zero."
+        // Special-casing zero is necessary because of the exponent.
+        const char* const _Str = "0p+0";
+        const size_t _Len      = 4;
+
+        if (_Last - _First < static_cast<ptrdiff_t>(_Len)) {
+            return {_Last, errc::value_too_large};
+        }
+
+        _VSTD::memcpy(_First, _Str, _Len);
+
+        return {_First + _Len, errc{}};
+    }
+
+    const _Uint_type _Ieee_mantissa = _Uint_value & _Traits::_Denormal_mantissa_mask;
+    const int32_t _Ieee_exponent    = static_cast<int32_t>(_Uint_value >> _Traits::_Exponent_shift);
+
+    char _Leading_hexit; // implicit bit
+    int32_t _Unbiased_exponent;
+
+    if (_Ieee_exponent == 0) { // subnormal
+        _Leading_hexit     = '0';
+        _Unbiased_exponent = 1 - _Traits::_Exponent_bias;
+    } else { // normal
+        _Leading_hexit     = '1';
+        _Unbiased_exponent = _Ieee_exponent - _Traits::_Exponent_bias;
+    }
+
+    // Performance note: Consider avoiding per-character bounds checking when there's plenty of space.
+
+    if (_First == _Last) {
+        return {_Last, errc::value_too_large};
+    }
+
+    *_First++ = _Leading_hexit;
+
+    if (_Ieee_mantissa == 0) {
+        // The fraction bits are all 0. Trim them away, including the decimal point.
+    } else {
+        if (_First == _Last) {
+            return {_Last, errc::value_too_large};
+        }
+
+        *_First++ = '.';
+
+        // The hexits after the decimal point correspond to the explicitly stored fraction bits.
+        // float explicitly stores 23 fraction bits. 23 / 4 == 5.75, so we'll print at most 6 hexits.
+        // double explicitly stores 52 fraction bits. 52 / 4 == 13, so we'll print at most 13 hexits.
+        _Uint_type _Adjusted_mantissa;
+        int32_t _Number_of_bits_remaining;
+
+        if constexpr (_IsSame<_Floating, float>::value) {
+            _Adjusted_mantissa        = _Ieee_mantissa << 1; // align to hexit boundary (23 isn't divisible by 4)
+            _Number_of_bits_remaining = 24; // 23 fraction bits + 1 alignment bit
+        } else {
+            _Adjusted_mantissa        = _Ieee_mantissa; // already aligned (52 is divisible by 4)
+            _Number_of_bits_remaining = 52; // 52 fraction bits
+        }
+
+        // do-while: The condition _Adjusted_mantissa != 0 is initially true - we have nonzero fraction bits and we've
+        // printed the decimal point. Each iteration, we print a hexit, mask it away, and keep looping if we still have
+        // nonzero fraction bits. If there would be trailing '0' hexits, this trims them. If there wouldn't be trailing
+        // '0' hexits, the same condition works (as we print the final hexit and mask it away); we don't need to test
+        // _Number_of_bits_remaining.
+        do {
+            _LIBCPP_ASSERT(_Number_of_bits_remaining >= 4, "");
+            _LIBCPP_ASSERT(_Number_of_bits_remaining % 4 == 0, "");
+            _Number_of_bits_remaining -= 4;
+
+            const uint32_t _Nibble = static_cast<uint32_t>(_Adjusted_mantissa >> _Number_of_bits_remaining);
+            _LIBCPP_ASSERT(_Nibble < 16, "");
+            const char _Hexit = __itoa::_Charconv_digits[_Nibble];
+
+            if (_First == _Last) {
+                return {_Last, errc::value_too_large};
+            }
+
+            *_First++ = _Hexit;
+
+            const _Uint_type _Mask = (_Uint_type{1} << _Number_of_bits_remaining) - 1;
+            _Adjusted_mantissa &= _Mask;
+
+        } while (_Adjusted_mantissa != 0);
+    }
+
+    // C11 7.21.6.1 "The fprintf function"/8: "The exponent always contains at least one digit, and only as many more
+    // digits as necessary to represent the decimal exponent of 2."
+
+    // Performance note: We should take advantage of the known ranges of possible exponents.
+
+    // float: _Unbiased_exponent is within [-126, 127].
+    // double: _Unbiased_exponent is within [-1022, 1023].
+
+    if (_Last - _First < 2) {
+        return {_Last, errc::value_too_large};
+    }
+
+    *_First++ = 'p';
+
+    if (_Unbiased_exponent < 0) {
+        *_First++          = '-';
+        _Unbiased_exponent = -_Unbiased_exponent;
+    } else {
+        *_First++ = '+';
+    }
+
+    // We've already printed '-' if necessary, so static_cast<uint32_t> avoids testing that again.
+    return _VSTD::to_chars(_First, _Last, static_cast<uint32_t>(_Unbiased_exponent));
+}
+
+// For general precision, we can use lookup tables to avoid performing trial formatting.
+
+// For a simple example, imagine counting the number of digits D in an integer, and needing to know
+// whether D is less than 3, equal to 3/4/5/6, or greater than 6. We could use a lookup table:
+// D | Largest integer with D digits
+// 2 |      99
+// 3 |     999
+// 4 |   9'999
+// 5 |  99'999
+// 6 | 999'999
+// 7 | table end
+// Looking up an integer in this table with lower_bound() will work:
+// * Too-small integers, like 7, 70, and 99, will cause lower_bound() to return the D == 2 row. (If all we care
+//   about is whether D is less than 3, then it's okay to smash the D == 1 and D == 2 cases together.)
+// * Integers in [100, 999] will cause lower_bound() to return the D == 3 row, and so forth.
+// * Too-large integers, like 1'000'000 and above, will cause lower_bound() to return the end of the table. If we
+//   compute D from that index, this will be considered D == 7, which will activate any "greater than 6" logic.
+
+// Floating-point is slightly more complicated.
+
+// The ordinary lookup tables are for X within [-5, 38] for float, and [-5, 308] for double.
+// (-5 absorbs too-negative exponents, outside the P > X >= -4 criterion. 38 and 308 are the maximum exponents.)
+// Due to the P > X condition, we can use a subset of the table for X within [-5, P - 1], suitably clamped.
+
+// When P is small, rounding can affect X. For example:
+// For P == 1, the largest double with X == 0 is: 9.4999999999999982236431605997495353221893310546875
+// For P == 2, the largest double with X == 0 is: 9.949999999999999289457264239899814128875732421875
+// For P == 3, the largest double with X == 0 is: 9.9949999999999992184029906638897955417633056640625
+
+// Exponent adjustment is a concern for P within [1, 7] for float, and [1, 15] for double (determined via
+// brute force). While larger values of P still perform rounding, they can't trigger exponent adjustment.
+// This is because only values with repeated '9' digits can undergo exponent adjustment during rounding,
+// and floating-point granularity limits the number of consecutive '9' digits that can appear.
+
+// So, we need special lookup tables for small values of P.
+// These tables have varying lengths due to the P > X >= -4 criterion. For example:
+// For P == 1, need table entries for X: -5, -4, -3, -2, -1, 0
+// For P == 2, need table entries for X: -5, -4, -3, -2, -1, 0, 1
+// For P == 3, need table entries for X: -5, -4, -3, -2, -1, 0, 1, 2
+// For P == 4, need table entries for X: -5, -4, -3, -2, -1, 0, 1, 2, 3
+
+// We can concatenate these tables for compact storage, using triangular numbers to access them.
+// The table for P begins at index (P - 1) * (P + 10) / 2 with length P + 5.
+
+// For both the ordinary and special lookup tables, after an index I is returned by lower_bound(), X is I - 5.
+
+// We need to special-case the floating-point value 0.0, which is considered to have X == 0.
+// Otherwise, the lookup tables would consider it to have a highly negative X.
+
+// Finally, because we're working with positive floating-point values,
+// representation comparisons behave identically to floating-point comparisons.
+
+// The following code generated the lookup tables for the scientific exponent X. Don't remove this code.
+#if 0
+// cl /EHsc /nologo /W4 /MT /O2 /std:c++17 generate_tables.cpp && generate_tables
+
+#include <algorithm>
+#include <assert.h>
+#include <charconv>
+#include <cmath>
+#include <limits>
+#include <map>
+#include <stdint.h>
+#include <stdio.h>
+#include <system_error>
+#include <type_traits>
+#include <vector>
+using namespace std;
+
+template <typename UInt, typename Pred>
+[[nodiscard]] UInt uint_partition_point(UInt first, const UInt last, Pred pred) {
+    // Find the beginning of the false partition in [first, last).
+    // [first, last) is partitioned when all of the true values occur before all of the false values.
+
+    static_assert(is_unsigned_v<UInt>);
+    assert(first <= last);
+
+    for (UInt n = last - first; n > 0;) {
+        const UInt n2  = n / 2;
+        const UInt mid = first + n2;
+
+        if (pred(mid)) {
+            first = mid + 1;
+            n     = n - n2 - 1;
+        } else {
+            n = n2;
+        }
+    }
+
+    return first;
+}
+
+template <typename Floating>
+[[nodiscard]] int scientific_exponent_X(const int P, const Floating flt) {
+    char buf[400]; // more than enough
+
+    // C11 7.21.6.1 "The fprintf function"/8 performs trial formatting with scientific precision P - 1.
+    const auto to_result = to_chars(buf, end(buf), flt, chars_format::scientific, P - 1);
+    assert(to_result.ec == errc{});
+
+    const char* exp_ptr = find(buf, to_result.ptr, 'e');
+    assert(exp_ptr != to_result.ptr);
+
+    ++exp_ptr; // advance past 'e'
+
+    if (*exp_ptr == '+') {
+        ++exp_ptr; // advance past '+' which from_chars() won't parse
+    }
+
+    int X;
+    const auto from_result = from_chars(exp_ptr, to_result.ptr, X);
+    assert(from_result.ec == errc{});
+    return X;
+}
+
+template <typename UInt>
+void print_table(const vector<UInt>& v, const char* const name) {
+    constexpr const char* UIntName = _IsSame<UInt, uint32_t>::value ? "uint32_t" : "uint64_t";
+
+    printf("static constexpr %s %s[%zu] = {\n", UIntName, name, v.size());
+
+    for (const auto& val : v) {
+        if constexpr (_IsSame<UInt, uint32_t>::value) {
+            printf("0x%08Xu,\n", val);
+        } else {
+            printf("0x%016llXu,\n", val);
+        }
+    }
+
+    printf("};\n");
+}
+
+enum class Mode { Tables, Tests };
+
+template <typename Floating>
+void generate_tables(const Mode mode) {
+    using Limits = numeric_limits<Floating>;
+    using UInt   = conditional_t<_IsSame<Floating, float>::value, uint32_t, uint64_t>;
+
+    map<int, map<int, UInt>> P_X_LargestValWithX;
+
+    constexpr int MaxP = Limits::max_exponent10 + 1; // MaxP performs no rounding during trial formatting
+
+    for (int P = 1; P <= MaxP; ++P) {
+        for (int X = -5; X < P; ++X) {
+            constexpr Floating first = static_cast<Floating>(9e-5); // well below 9.5e-5, otherwise arbitrary
+            constexpr Floating last  = Limits::infinity(); // one bit above Limits::max()
+
+            const UInt val_beyond_X = uint_partition_point(reinterpret_cast<const UInt&>(first),
+                reinterpret_cast<const UInt&>(last),
+                [P, X](const UInt u) { return scientific_exponent_X(P, reinterpret_cast<const Floating&>(u)) <= X; });
+
+            P_X_LargestValWithX[P][X] = val_beyond_X - 1;
+        }
+    }
+
+    constexpr const char* FloatingName = _IsSame<Floating, float>::value ? "float" : "double";
+
+    constexpr int MaxSpecialP = _IsSame<Floating, float>::value ? 7 : 15; // MaxSpecialP is affected by exponent adjustment
+
+    if (mode == Mode::Tables) {
+        printf("template <>\n");
+        printf("struct _General_precision_tables<%s> {\n", FloatingName);
+
+        printf("static constexpr int _Max_special_P = %d;\n", MaxSpecialP);
+
+        vector<UInt> special;
+
+        for (int P = 1; P <= MaxSpecialP; ++P) {
+            for (int X = -5; X < P; ++X) {
+                const UInt val = P_X_LargestValWithX[P][X];
+                special.push_back(val);
+            }
+        }
+
+        print_table(special, "_Special_X_table");
+
+        for (int P = MaxSpecialP + 1; P < MaxP; ++P) {
+            for (int X = -5; X < P; ++X) {
+                const UInt val = P_X_LargestValWithX[P][X];
+                assert(val == P_X_LargestValWithX[MaxP][X]);
+            }
+        }
+
+        printf("static constexpr int _Max_P = %d;\n", MaxP);
+
+        vector<UInt> ordinary;
+
+        for (int X = -5; X < MaxP; ++X) {
+            const UInt val = P_X_LargestValWithX[MaxP][X];
+            ordinary.push_back(val);
+        }
+
+        print_table(ordinary, "_Ordinary_X_table");
+
+        printf("};\n");
+    } else {
+        printf("==========\n");
+        printf("Test cases for %s:\n", FloatingName);
+
+        constexpr int Hexits         = _IsSame<Floating, float>::value ? 6 : 13;
+        constexpr const char* Suffix = _IsSame<Floating, float>::value ? "f" : "";
+
+        for (int P = 1; P <= MaxP; ++P) {
+            for (int X = -5; X < P; ++X) {
+                if (P <= MaxSpecialP || P == 25 || P == MaxP || X == P - 1) {
+                    const UInt val1   = P_X_LargestValWithX[P][X];
+                    const Floating f1 = reinterpret_cast<const Floating&>(val1);
+                    const UInt val2   = val1 + 1;
+                    const Floating f2 = reinterpret_cast<const Floating&>(val2);
+
+                    printf("{%.*a%s, chars_format::general, %d, \"%.*g\"},\n", Hexits, f1, Suffix, P, P, f1);
+                    if (isfinite(f2)) {
+                        printf("{%.*a%s, chars_format::general, %d, \"%.*g\"},\n", Hexits, f2, Suffix, P, P, f2);
+                    }
+                }
+            }
+        }
+    }
+}
+
+int main() {
+    printf("template <class _Floating>\n");
+    printf("struct _General_precision_tables;\n");
+    generate_tables<float>(Mode::Tables);
+    generate_tables<double>(Mode::Tables);
+    generate_tables<float>(Mode::Tests);
+    generate_tables<double>(Mode::Tests);
+}
+#endif // 0
+
+template <class _Floating>
+struct _General_precision_tables;
+
+template <>
+struct _General_precision_tables<float> {
+    static constexpr int _Max_special_P = 7;
+
+    static constexpr uint32_t _Special_X_table[63] = {0x38C73ABCu, 0x3A79096Bu, 0x3C1BA5E3u, 0x3DC28F5Cu, 0x3F733333u,
+        0x4117FFFFu, 0x38D0AAA7u, 0x3A826AA8u, 0x3C230553u, 0x3DCBC6A7u, 0x3F7EB851u, 0x411F3333u, 0x42C6FFFFu,
+        0x38D19C3Fu, 0x3A8301A7u, 0x3C23C211u, 0x3DCCB295u, 0x3F7FDF3Bu, 0x411FEB85u, 0x42C7E666u, 0x4479DFFFu,
+        0x38D1B468u, 0x3A8310C1u, 0x3C23D4F1u, 0x3DCCCA2Du, 0x3F7FFCB9u, 0x411FFDF3u, 0x42C7FD70u, 0x4479FCCCu,
+        0x461C3DFFu, 0x38D1B6D2u, 0x3A831243u, 0x3C23D6D4u, 0x3DCCCC89u, 0x3F7FFFACu, 0x411FFFCBu, 0x42C7FFBEu,
+        0x4479FFAEu, 0x461C3FCCu, 0x47C34FBFu, 0x38D1B710u, 0x3A83126Au, 0x3C23D704u, 0x3DCCCCC6u, 0x3F7FFFF7u,
+        0x411FFFFAu, 0x42C7FFF9u, 0x4479FFF7u, 0x461C3FFAu, 0x47C34FF9u, 0x497423F7u, 0x38D1B716u, 0x3A83126Eu,
+        0x3C23D709u, 0x3DCCCCCCu, 0x3F7FFFFFu, 0x411FFFFFu, 0x42C7FFFFu, 0x4479FFFFu, 0x461C3FFFu, 0x47C34FFFu,
+        0x497423FFu, 0x4B18967Fu};
+
+    static constexpr int _Max_P = 39;
+
+    static constexpr uint32_t _Ordinary_X_table[44] = {0x38D1B717u, 0x3A83126Eu, 0x3C23D70Au, 0x3DCCCCCCu, 0x3F7FFFFFu,
+        0x411FFFFFu, 0x42C7FFFFu, 0x4479FFFFu, 0x461C3FFFu, 0x47C34FFFu, 0x497423FFu, 0x4B18967Fu, 0x4CBEBC1Fu,
+        0x4E6E6B27u, 0x501502F8u, 0x51BA43B7u, 0x5368D4A5u, 0x551184E7u, 0x56B5E620u, 0x58635FA9u, 0x5A0E1BC9u,
+        0x5BB1A2BCu, 0x5D5E0B6Bu, 0x5F0AC723u, 0x60AD78EBu, 0x6258D726u, 0x64078678u, 0x65A96816u, 0x6753C21Bu,
+        0x69045951u, 0x6AA56FA5u, 0x6C4ECB8Fu, 0x6E013F39u, 0x6FA18F07u, 0x7149F2C9u, 0x72FC6F7Cu, 0x749DC5ADu,
+        0x76453719u, 0x77F684DFu, 0x799A130Bu, 0x7B4097CEu, 0x7CF0BDC2u, 0x7E967699u, 0x7F7FFFFFu};
+};
+
+template <>
+struct _General_precision_tables<double> {
+    static constexpr int _Max_special_P = 15;
+
+    static constexpr uint64_t _Special_X_table[195] = {0x3F18E757928E0C9Du, 0x3F4F212D77318FC5u, 0x3F8374BC6A7EF9DBu,
+        0x3FB851EB851EB851u, 0x3FEE666666666666u, 0x4022FFFFFFFFFFFFu, 0x3F1A1554FBDAD751u, 0x3F504D551D68C692u,
+        0x3F8460AA64C2F837u, 0x3FB978D4FDF3B645u, 0x3FEFD70A3D70A3D7u, 0x4023E66666666666u, 0x4058DFFFFFFFFFFFu,
+        0x3F1A3387ECC8EB96u, 0x3F506034F3FD933Eu, 0x3F84784230FCF80Du, 0x3FB99652BD3C3611u, 0x3FEFFBE76C8B4395u,
+        0x4023FD70A3D70A3Du, 0x4058FCCCCCCCCCCCu, 0x408F3BFFFFFFFFFFu, 0x3F1A368D04E0BA6Au, 0x3F506218230C7482u,
+        0x3F847A9E2BCF91A3u, 0x3FB99945B6C3760Bu, 0x3FEFFF972474538Eu, 0x4023FFBE76C8B439u, 0x4058FFAE147AE147u,
+        0x408F3F9999999999u, 0x40C387BFFFFFFFFFu, 0x3F1A36DA54164F19u, 0x3F506248748DF16Fu, 0x3F847ADA91B16DCBu,
+        0x3FB99991361DC93Eu, 0x3FEFFFF583A53B8Eu, 0x4023FFF972474538u, 0x4058FFF7CED91687u, 0x408F3FF5C28F5C28u,
+        0x40C387F999999999u, 0x40F869F7FFFFFFFFu, 0x3F1A36E20F35445Du, 0x3F50624D49814ABAu, 0x3F847AE09BE19D69u,
+        0x3FB99998C2DA04C3u, 0x3FEFFFFEF39085F4u, 0x4023FFFF583A53B8u, 0x4058FFFF2E48E8A7u, 0x408F3FFEF9DB22D0u,
+        0x40C387FF5C28F5C2u, 0x40F869FF33333333u, 0x412E847EFFFFFFFFu, 0x3F1A36E2D51EC34Bu, 0x3F50624DC5333A0Eu,
+        0x3F847AE136800892u, 0x3FB9999984200AB7u, 0x3FEFFFFFE5280D65u, 0x4023FFFFEF39085Fu, 0x4058FFFFEB074A77u,
+        0x408F3FFFE5C91D14u, 0x40C387FFEF9DB22Du, 0x40F869FFEB851EB8u, 0x412E847FE6666666u, 0x416312CFEFFFFFFFu,
+        0x3F1A36E2E8E94FFCu, 0x3F50624DD191D1FDu, 0x3F847AE145F6467Du, 0x3FB999999773D81Cu, 0x3FEFFFFFFD50CE23u,
+        0x4023FFFFFE5280D6u, 0x4058FFFFFDE7210Bu, 0x408F3FFFFD60E94Eu, 0x40C387FFFE5C91D1u, 0x40F869FFFDF3B645u,
+        0x412E847FFD70A3D7u, 0x416312CFFE666666u, 0x4197D783FDFFFFFFu, 0x3F1A36E2EAE3F7A7u, 0x3F50624DD2CE7AC8u,
+        0x3F847AE14782197Bu, 0x3FB9999999629FD9u, 0x3FEFFFFFFFBB47D0u, 0x4023FFFFFFD50CE2u, 0x4058FFFFFFCA501Au,
+        0x408F3FFFFFBCE421u, 0x40C387FFFFD60E94u, 0x40F869FFFFCB923Au, 0x412E847FFFBE76C8u, 0x416312CFFFD70A3Du,
+        0x4197D783FFCCCCCCu, 0x41CDCD64FFBFFFFFu, 0x3F1A36E2EB16A205u, 0x3F50624DD2EE2543u, 0x3F847AE147A9AE94u,
+        0x3FB9999999941A39u, 0x3FEFFFFFFFF920C8u, 0x4023FFFFFFFBB47Du, 0x4058FFFFFFFAA19Cu, 0x408F3FFFFFF94A03u,
+        0x40C387FFFFFBCE42u, 0x40F869FFFFFAC1D2u, 0x412E847FFFF97247u, 0x416312CFFFFBE76Cu, 0x4197D783FFFAE147u,
+        0x41CDCD64FFF99999u, 0x4202A05F1FFBFFFFu, 0x3F1A36E2EB1BB30Fu, 0x3F50624DD2F14FE9u, 0x3F847AE147ADA3E3u,
+        0x3FB9999999990CDCu, 0x3FEFFFFFFFFF5014u, 0x4023FFFFFFFF920Cu, 0x4058FFFFFFFF768Fu, 0x408F3FFFFFFF5433u,
+        0x40C387FFFFFF94A0u, 0x40F869FFFFFF79C8u, 0x412E847FFFFF583Au, 0x416312CFFFFF9724u, 0x4197D783FFFF7CEDu,
+        0x41CDCD64FFFF5C28u, 0x4202A05F1FFF9999u, 0x42374876E7FF7FFFu, 0x3F1A36E2EB1C34C3u, 0x3F50624DD2F1A0FAu,
+        0x3F847AE147AE0938u, 0x3FB9999999998B86u, 0x3FEFFFFFFFFFEE68u, 0x4023FFFFFFFFF501u, 0x4058FFFFFFFFF241u,
+        0x408F3FFFFFFFEED1u, 0x40C387FFFFFFF543u, 0x40F869FFFFFFF294u, 0x412E847FFFFFEF39u, 0x416312CFFFFFF583u,
+        0x4197D783FFFFF2E4u, 0x41CDCD64FFFFEF9Du, 0x4202A05F1FFFF5C2u, 0x42374876E7FFF333u, 0x426D1A94A1FFEFFFu,
+        0x3F1A36E2EB1C41BBu, 0x3F50624DD2F1A915u, 0x3F847AE147AE135Au, 0x3FB9999999999831u, 0x3FEFFFFFFFFFFE3Du,
+        0x4023FFFFFFFFFEE6u, 0x4058FFFFFFFFFEA0u, 0x408F3FFFFFFFFE48u, 0x40C387FFFFFFFEEDu, 0x40F869FFFFFFFEA8u,
+        0x412E847FFFFFFE52u, 0x416312CFFFFFFEF3u, 0x4197D783FFFFFEB0u, 0x41CDCD64FFFFFE5Cu, 0x4202A05F1FFFFEF9u,
+        0x42374876E7FFFEB8u, 0x426D1A94A1FFFE66u, 0x42A2309CE53FFEFFu, 0x3F1A36E2EB1C4307u, 0x3F50624DD2F1A9E4u,
+        0x3F847AE147AE145Eu, 0x3FB9999999999975u, 0x3FEFFFFFFFFFFFD2u, 0x4023FFFFFFFFFFE3u, 0x4058FFFFFFFFFFDCu,
+        0x408F3FFFFFFFFFD4u, 0x40C387FFFFFFFFE4u, 0x40F869FFFFFFFFDDu, 0x412E847FFFFFFFD5u, 0x416312CFFFFFFFE5u,
+        0x4197D783FFFFFFDEu, 0x41CDCD64FFFFFFD6u, 0x4202A05F1FFFFFE5u, 0x42374876E7FFFFDFu, 0x426D1A94A1FFFFD7u,
+        0x42A2309CE53FFFE6u, 0x42D6BCC41E8FFFDFu, 0x3F1A36E2EB1C4328u, 0x3F50624DD2F1A9F9u, 0x3F847AE147AE1477u,
+        0x3FB9999999999995u, 0x3FEFFFFFFFFFFFFBu, 0x4023FFFFFFFFFFFDu, 0x4058FFFFFFFFFFFCu, 0x408F3FFFFFFFFFFBu,
+        0x40C387FFFFFFFFFDu, 0x40F869FFFFFFFFFCu, 0x412E847FFFFFFFFBu, 0x416312CFFFFFFFFDu, 0x4197D783FFFFFFFCu,
+        0x41CDCD64FFFFFFFBu, 0x4202A05F1FFFFFFDu, 0x42374876E7FFFFFCu, 0x426D1A94A1FFFFFBu, 0x42A2309CE53FFFFDu,
+        0x42D6BCC41E8FFFFCu, 0x430C6BF52633FFFBu};
+
+    static constexpr int _Max_P = 309;
+
+    static constexpr uint64_t _Ordinary_X_table[314] = {0x3F1A36E2EB1C432Cu, 0x3F50624DD2F1A9FBu, 0x3F847AE147AE147Au,
+        0x3FB9999999999999u, 0x3FEFFFFFFFFFFFFFu, 0x4023FFFFFFFFFFFFu, 0x4058FFFFFFFFFFFFu, 0x408F3FFFFFFFFFFFu,
+        0x40C387FFFFFFFFFFu, 0x40F869FFFFFFFFFFu, 0x412E847FFFFFFFFFu, 0x416312CFFFFFFFFFu, 0x4197D783FFFFFFFFu,
+        0x41CDCD64FFFFFFFFu, 0x4202A05F1FFFFFFFu, 0x42374876E7FFFFFFu, 0x426D1A94A1FFFFFFu, 0x42A2309CE53FFFFFu,
+        0x42D6BCC41E8FFFFFu, 0x430C6BF52633FFFFu, 0x4341C37937E07FFFu, 0x4376345785D89FFFu, 0x43ABC16D674EC7FFu,
+        0x43E158E460913CFFu, 0x4415AF1D78B58C3Fu, 0x444B1AE4D6E2EF4Fu, 0x4480F0CF064DD591u, 0x44B52D02C7E14AF6u,
+        0x44EA784379D99DB4u, 0x45208B2A2C280290u, 0x4554ADF4B7320334u, 0x4589D971E4FE8401u, 0x45C027E72F1F1281u,
+        0x45F431E0FAE6D721u, 0x46293E5939A08CE9u, 0x465F8DEF8808B024u, 0x4693B8B5B5056E16u, 0x46C8A6E32246C99Cu,
+        0x46FED09BEAD87C03u, 0x4733426172C74D82u, 0x476812F9CF7920E2u, 0x479E17B84357691Bu, 0x47D2CED32A16A1B1u,
+        0x48078287F49C4A1Du, 0x483D6329F1C35CA4u, 0x48725DFA371A19E6u, 0x48A6F578C4E0A060u, 0x48DCB2D6F618C878u,
+        0x4911EFC659CF7D4Bu, 0x49466BB7F0435C9Eu, 0x497C06A5EC5433C6u, 0x49B18427B3B4A05Bu, 0x49E5E531A0A1C872u,
+        0x4A1B5E7E08CA3A8Fu, 0x4A511B0EC57E6499u, 0x4A8561D276DDFDC0u, 0x4ABABA4714957D30u, 0x4AF0B46C6CDD6E3Eu,
+        0x4B24E1878814C9CDu, 0x4B5A19E96A19FC40u, 0x4B905031E2503DA8u, 0x4BC4643E5AE44D12u, 0x4BF97D4DF19D6057u,
+        0x4C2FDCA16E04B86Du, 0x4C63E9E4E4C2F344u, 0x4C98E45E1DF3B015u, 0x4CCF1D75A5709C1Au, 0x4D03726987666190u,
+        0x4D384F03E93FF9F4u, 0x4D6E62C4E38FF872u, 0x4DA2FDBB0E39FB47u, 0x4DD7BD29D1C87A19u, 0x4E0DAC74463A989Fu,
+        0x4E428BC8ABE49F63u, 0x4E772EBAD6DDC73Cu, 0x4EACFA698C95390Bu, 0x4EE21C81F7DD43A7u, 0x4F16A3A275D49491u,
+        0x4F4C4C8B1349B9B5u, 0x4F81AFD6EC0E1411u, 0x4FB61BCCA7119915u, 0x4FEBA2BFD0D5FF5Bu, 0x502145B7E285BF98u,
+        0x50559725DB272F7Fu, 0x508AFCEF51F0FB5Eu, 0x50C0DE1593369D1Bu, 0x50F5159AF8044462u, 0x512A5B01B605557Au,
+        0x516078E111C3556Cu, 0x5194971956342AC7u, 0x51C9BCDFABC13579u, 0x5200160BCB58C16Cu, 0x52341B8EBE2EF1C7u,
+        0x526922726DBAAE39u, 0x529F6B0F092959C7u, 0x52D3A2E965B9D81Cu, 0x53088BA3BF284E23u, 0x533EAE8CAEF261ACu,
+        0x53732D17ED577D0Bu, 0x53A7F85DE8AD5C4Eu, 0x53DDF67562D8B362u, 0x5412BA095DC7701Du, 0x5447688BB5394C25u,
+        0x547D42AEA2879F2Eu, 0x54B249AD2594C37Cu, 0x54E6DC186EF9F45Cu, 0x551C931E8AB87173u, 0x5551DBF316B346E7u,
+        0x558652EFDC6018A1u, 0x55BBE7ABD3781ECAu, 0x55F170CB642B133Eu, 0x5625CCFE3D35D80Eu, 0x565B403DCC834E11u,
+        0x569108269FD210CBu, 0x56C54A3047C694FDu, 0x56FA9CBC59B83A3Du, 0x5730A1F5B8132466u, 0x5764CA732617ED7Fu,
+        0x5799FD0FEF9DE8DFu, 0x57D03E29F5C2B18Bu, 0x58044DB473335DEEu, 0x583961219000356Au, 0x586FB969F40042C5u,
+        0x58A3D3E2388029BBu, 0x58D8C8DAC6A0342Au, 0x590EFB1178484134u, 0x59435CEAEB2D28C0u, 0x59783425A5F872F1u,
+        0x59AE412F0F768FADu, 0x59E2E8BD69AA19CCu, 0x5A17A2ECC414A03Fu, 0x5A4D8BA7F519C84Fu, 0x5A827748F9301D31u,
+        0x5AB7151B377C247Eu, 0x5AECDA62055B2D9Du, 0x5B22087D4358FC82u, 0x5B568A9C942F3BA3u, 0x5B8C2D43B93B0A8Bu,
+        0x5BC19C4A53C4E697u, 0x5BF6035CE8B6203Du, 0x5C2B843422E3A84Cu, 0x5C6132A095CE492Fu, 0x5C957F48BB41DB7Bu,
+        0x5CCADF1AEA12525Au, 0x5D00CB70D24B7378u, 0x5D34FE4D06DE5056u, 0x5D6A3DE04895E46Cu, 0x5DA066AC2D5DAEC3u,
+        0x5DD4805738B51A74u, 0x5E09A06D06E26112u, 0x5E400444244D7CABu, 0x5E7405552D60DBD6u, 0x5EA906AA78B912CBu,
+        0x5EDF485516E7577Eu, 0x5F138D352E5096AFu, 0x5F48708279E4BC5Au, 0x5F7E8CA3185DEB71u, 0x5FB317E5EF3AB327u,
+        0x5FE7DDDF6B095FF0u, 0x601DD55745CBB7ECu, 0x6052A5568B9F52F4u, 0x60874EAC2E8727B1u, 0x60BD22573A28F19Du,
+        0x60F2357684599702u, 0x6126C2D4256FFCC2u, 0x615C73892ECBFBF3u, 0x6191C835BD3F7D78u, 0x61C63A432C8F5CD6u,
+        0x61FBC8D3F7B3340Bu, 0x62315D847AD00087u, 0x6265B4E5998400A9u, 0x629B221EFFE500D3u, 0x62D0F5535FEF2084u,
+        0x630532A837EAE8A5u, 0x633A7F5245E5A2CEu, 0x63708F936BAF85C1u, 0x63A4B378469B6731u, 0x63D9E056584240FDu,
+        0x64102C35F729689Eu, 0x6444374374F3C2C6u, 0x647945145230B377u, 0x64AF965966BCE055u, 0x64E3BDF7E0360C35u,
+        0x6518AD75D8438F43u, 0x654ED8D34E547313u, 0x6583478410F4C7ECu, 0x65B819651531F9E7u, 0x65EE1FBE5A7E7861u,
+        0x6622D3D6F88F0B3Cu, 0x665788CCB6B2CE0Cu, 0x668D6AFFE45F818Fu, 0x66C262DFEEBBB0F9u, 0x66F6FB97EA6A9D37u,
+        0x672CBA7DE5054485u, 0x6761F48EAF234AD3u, 0x679671B25AEC1D88u, 0x67CC0E1EF1A724EAu, 0x680188D357087712u,
+        0x6835EB082CCA94D7u, 0x686B65CA37FD3A0Du, 0x68A11F9E62FE4448u, 0x68D56785FBBDD55Au, 0x690AC1677AAD4AB0u,
+        0x6940B8E0ACAC4EAEu, 0x6974E718D7D7625Au, 0x69AA20DF0DCD3AF0u, 0x69E0548B68A044D6u, 0x6A1469AE42C8560Cu,
+        0x6A498419D37A6B8Fu, 0x6A7FE52048590672u, 0x6AB3EF342D37A407u, 0x6AE8EB0138858D09u, 0x6B1F25C186A6F04Cu,
+        0x6B537798F428562Fu, 0x6B88557F31326BBBu, 0x6BBE6ADEFD7F06AAu, 0x6BF302CB5E6F642Au, 0x6C27C37E360B3D35u,
+        0x6C5DB45DC38E0C82u, 0x6C9290BA9A38C7D1u, 0x6CC734E940C6F9C5u, 0x6CFD022390F8B837u, 0x6D3221563A9B7322u,
+        0x6D66A9ABC9424FEBu, 0x6D9C5416BB92E3E6u, 0x6DD1B48E353BCE6Fu, 0x6E0621B1C28AC20Bu, 0x6E3BAA1E332D728Eu,
+        0x6E714A52DFFC6799u, 0x6EA59CE797FB817Fu, 0x6EDB04217DFA61DFu, 0x6F10E294EEBC7D2Bu, 0x6F451B3A2A6B9C76u,
+        0x6F7A6208B5068394u, 0x6FB07D457124123Cu, 0x6FE49C96CD6D16CBu, 0x7019C3BC80C85C7Eu, 0x70501A55D07D39CFu,
+        0x708420EB449C8842u, 0x70B9292615C3AA53u, 0x70EF736F9B3494E8u, 0x7123A825C100DD11u, 0x7158922F31411455u,
+        0x718EB6BAFD91596Bu, 0x71C33234DE7AD7E2u, 0x71F7FEC216198DDBu, 0x722DFE729B9FF152u, 0x7262BF07A143F6D3u,
+        0x72976EC98994F488u, 0x72CD4A7BEBFA31AAu, 0x73024E8D737C5F0Au, 0x7336E230D05B76CDu, 0x736C9ABD04725480u,
+        0x73A1E0B622C774D0u, 0x73D658E3AB795204u, 0x740BEF1C9657A685u, 0x74417571DDF6C813u, 0x7475D2CE55747A18u,
+        0x74AB4781EAD1989Eu, 0x74E10CB132C2FF63u, 0x75154FDD7F73BF3Bu, 0x754AA3D4DF50AF0Au, 0x7580A6650B926D66u,
+        0x75B4CFFE4E7708C0u, 0x75EA03FDE214CAF0u, 0x7620427EAD4CFED6u, 0x7654531E58A03E8Bu, 0x768967E5EEC84E2Eu,
+        0x76BFC1DF6A7A61BAu, 0x76F3D92BA28C7D14u, 0x7728CF768B2F9C59u, 0x775F03542DFB8370u, 0x779362149CBD3226u,
+        0x77C83A99C3EC7EAFu, 0x77FE494034E79E5Bu, 0x7832EDC82110C2F9u, 0x7867A93A2954F3B7u, 0x789D9388B3AA30A5u,
+        0x78D27C35704A5E67u, 0x79071B42CC5CF601u, 0x793CE2137F743381u, 0x79720D4C2FA8A030u, 0x79A6909F3B92C83Du,
+        0x79DC34C70A777A4Cu, 0x7A11A0FC668AAC6Fu, 0x7A46093B802D578Bu, 0x7A7B8B8A6038AD6Eu, 0x7AB137367C236C65u,
+        0x7AE585041B2C477Eu, 0x7B1AE64521F7595Eu, 0x7B50CFEB353A97DAu, 0x7B8503E602893DD1u, 0x7BBA44DF832B8D45u,
+        0x7BF06B0BB1FB384Bu, 0x7C2485CE9E7A065Eu, 0x7C59A742461887F6u, 0x7C9008896BCF54F9u, 0x7CC40AABC6C32A38u,
+        0x7CF90D56B873F4C6u, 0x7D2F50AC6690F1F8u, 0x7D63926BC01A973Bu, 0x7D987706B0213D09u, 0x7DCE94C85C298C4Cu,
+        0x7E031CFD3999F7AFu, 0x7E37E43C8800759Bu, 0x7E6DDD4BAA009302u, 0x7EA2AA4F4A405BE1u, 0x7ED754E31CD072D9u,
+        0x7F0D2A1BE4048F90u, 0x7F423A516E82D9BAu, 0x7F76C8E5CA239028u, 0x7FAC7B1F3CAC7433u, 0x7FE1CCF385EBC89Fu,
+        0x7FEFFFFFFFFFFFFFu};
+};
+
+template <class _Floating>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI
+to_chars_result _Floating_to_chars_general_precision(
+    char* _First, char* const _Last, const _Floating _Value, int _Precision) noexcept {
+
+    using _Traits    = _Floating_type_traits<_Floating>;
+    using _Uint_type = typename _Traits::_Uint_type;
+
+    const _Uint_type _Uint_value = _VSTD::bit_cast<_Uint_type>(_Value);
+
+    if (_Uint_value == 0) { // zero detected; write "0" and return; _Precision is irrelevant due to zero-trimming
+        if (_First == _Last) {
+            return {_Last, errc::value_too_large};
+        }
+
+        *_First++ = '0';
+
+        return {_First, errc{}};
+    }
+
+    // C11 7.21.6.1 "The fprintf function"/5:
+    // "A negative precision argument is taken as if the precision were omitted."
+    // /8: "g,G [...] Let P equal the precision if nonzero, 6 if the precision is omitted,
+    // or 1 if the precision is zero."
+
+    // Performance note: It's possible to rewrite this for branchless codegen,
+    // but profiling will be necessary to determine whether that's faster.
+    if (_Precision < 0) {
+        _Precision = 6;
+    } else if (_Precision == 0) {
+        _Precision = 1;
+    } else if (_Precision < 1'000'000) {
+        // _Precision is ok.
+    } else {
+        // Avoid integer overflow.
+        // Due to general notation's zero-trimming behavior, we can simply clamp _Precision.
+        // This is further clamped below.
+        _Precision = 1'000'000;
+    }
+
+    // _Precision is now the Standard's P.
+
+    // /8: "Then, if a conversion with style E would have an exponent of X:
+    // - if P > X >= -4, the conversion is with style f (or F) and precision P - (X + 1).
+    // - otherwise, the conversion is with style e (or E) and precision P - 1."
+
+    // /8: "Finally, [...] any trailing zeros are removed from the fractional portion of the result
+    // and the decimal-point character is removed if there is no fractional portion remaining."
+
+    using _Tables = _General_precision_tables<_Floating>;
+
+    const _Uint_type* _Table_begin;
+    const _Uint_type* _Table_end;
+
+    if (_Precision <= _Tables::_Max_special_P) {
+        _Table_begin = _Tables::_Special_X_table + (_Precision - 1) * (_Precision + 10) / 2;
+        _Table_end   = _Table_begin + _Precision + 5;
+    } else {
+        _Table_begin = _Tables::_Ordinary_X_table;
+        _Table_end   = _Table_begin + _VSTD::min(_Precision, _Tables::_Max_P) + 5;
+    }
+
+    // Profiling indicates that linear search is faster than binary search for small tables.
+    // Performance note: lambda captures may have a small performance cost.
+    const _Uint_type* const _Table_lower_bound = [=] {
+        if constexpr (!_IsSame<_Floating, float>::value) {
+            if (_Precision > 155) { // threshold determined via profiling
+                return _VSTD::lower_bound(_Table_begin, _Table_end, _Uint_value, less{});
+            }
+        }
+
+        return _VSTD::find_if(_Table_begin, _Table_end, [=](const _Uint_type _Elem) { return _Uint_value <= _Elem; });
+    }();
+
+    const ptrdiff_t _Table_index     = _Table_lower_bound - _Table_begin;
+    const int _Scientific_exponent_X = static_cast<int>(_Table_index - 5);
+    const bool _Use_fixed_notation   = _Precision > _Scientific_exponent_X && _Scientific_exponent_X >= -4;
+
+    // Performance note: it might (or might not) be faster to modify Ryu Printf to perform zero-trimming.
+    // Such modifications would involve a fairly complicated state machine (notably, both '0' and '9' digits would
+    // need to be buffered, due to rounding), and that would have performance costs due to increased branching.
+    // Here, we're using a simpler approach: writing into a local buffer, manually zero-trimming, and then copying into
+    // the output range. The necessary buffer size is reasonably small, the zero-trimming logic is simple and fast,
+    // and the final copying is also fast.
+
+    constexpr int _Max_output_length =
+        _IsSame<_Floating, float>::value ? 117 : 773; // cases: 0x1.fffffep-126f and 0x1.fffffffffffffp-1022
+    constexpr int _Max_fixed_precision =
+        _IsSame<_Floating, float>::value ? 37 : 66; // cases: 0x1.fffffep-14f and 0x1.fffffffffffffp-14
+    constexpr int _Max_scientific_precision =
+        _IsSame<_Floating, float>::value ? 111 : 766; // cases: 0x1.fffffep-126f and 0x1.fffffffffffffp-1022
+
+    // Note that _Max_output_length is determined by scientific notation and is more than enough for fixed notation.
+    // 0x1.fffffep+127f is 39 digits, plus 1 for '.', plus _Max_fixed_precision for '0' digits, equals 77.
+    // 0x1.fffffffffffffp+1023 is 309 digits, plus 1 for '.', plus _Max_fixed_precision for '0' digits, equals 376.
+
+    char _Buffer[_Max_output_length];
+    const char* const _Significand_first = _Buffer; // e.g. "1.234"
+    const char* _Significand_last        = nullptr;
+    const char* _Exponent_first          = nullptr; // e.g. "e-05"
+    const char* _Exponent_last           = nullptr;
+    int _Effective_precision; // number of digits printed after the decimal point, before trimming
+
+    // Write into the local buffer.
+    // Clamping _Effective_precision allows _Buffer to be as small as possible, and increases efficiency.
+    if (_Use_fixed_notation) {
+        _Effective_precision = _VSTD::min(_Precision - (_Scientific_exponent_X + 1), _Max_fixed_precision);
+        const to_chars_result _Buf_result =
+            _Floating_to_chars_fixed_precision(_Buffer, _VSTD::end(_Buffer), _Value, _Effective_precision);
+        _LIBCPP_ASSERT(_Buf_result.ec == errc{}, "");
+        _Significand_last = _Buf_result.ptr;
+    } else {
+        _Effective_precision = _VSTD::min(_Precision - 1, _Max_scientific_precision);
+        const to_chars_result _Buf_result =
+            _Floating_to_chars_scientific_precision(_Buffer, _VSTD::end(_Buffer), _Value, _Effective_precision);
+        _LIBCPP_ASSERT(_Buf_result.ec == errc{}, "");
+        _Significand_last = _VSTD::find(_Buffer, _Buf_result.ptr, 'e');
+        _Exponent_first   = _Significand_last;
+        _Exponent_last    = _Buf_result.ptr;
+    }
+
+    // If we printed a decimal point followed by digits, perform zero-trimming.
+    if (_Effective_precision > 0) {
+        while (_Significand_last[-1] == '0') { // will stop at '.' or a nonzero digit
+            --_Significand_last;
+        }
+
+        if (_Significand_last[-1] == '.') {
+            --_Significand_last;
+        }
+    }
+
+    // Copy the significand to the output range.
+    const ptrdiff_t _Significand_distance = _Significand_last - _Significand_first;
+    if (_Last - _First < _Significand_distance) {
+        return {_Last, errc::value_too_large};
+    }
+    _VSTD::memcpy(_First, _Significand_first, static_cast<size_t>(_Significand_distance));
+    _First += _Significand_distance;
+
+    // Copy the exponent to the output range.
+    if (!_Use_fixed_notation) {
+        const ptrdiff_t _Exponent_distance = _Exponent_last - _Exponent_first;
+        if (_Last - _First < _Exponent_distance) {
+            return {_Last, errc::value_too_large};
+        }
+        _VSTD::memcpy(_First, _Exponent_first, static_cast<size_t>(_Exponent_distance));
+        _First += _Exponent_distance;
+    }
+
+    return {_First, errc{}};
+}
+
+enum class _Floating_to_chars_overload { _Plain, _Format_only, _Format_precision };
+
+template <_Floating_to_chars_overload _Overload, class _Floating>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI
+to_chars_result _Floating_to_chars(
+    char* _First, char* const _Last, _Floating _Value, const chars_format _Fmt, const int _Precision) noexcept {
+
+    if constexpr (_Overload == _Floating_to_chars_overload::_Plain) {
+        _LIBCPP_ASSERT(_Fmt == chars_format{}, ""); // plain overload must pass chars_format{} internally
+    } else {
+        _LIBCPP_ASSERT(_Fmt == chars_format::general || _Fmt == chars_format::scientific || _Fmt == chars_format::fixed
+                         || _Fmt == chars_format::hex,
+            "invalid format in to_chars()");
+    }
+
+    using _Traits    = _Floating_type_traits<_Floating>;
+    using _Uint_type = typename _Traits::_Uint_type;
+
+    _Uint_type _Uint_value = _VSTD::bit_cast<_Uint_type>(_Value);
+
+    const bool _Was_negative = (_Uint_value & _Traits::_Shifted_sign_mask) != 0;
+
+    if (_Was_negative) { // sign bit detected; write minus sign and clear sign bit
+        if (_First == _Last) {
+            return {_Last, errc::value_too_large};
+        }
+
+        *_First++ = '-';
+
+        _Uint_value &= ~_Traits::_Shifted_sign_mask;
+        _Value = _VSTD::bit_cast<_Floating>(_Uint_value);
+    }
+
+    if ((_Uint_value & _Traits::_Shifted_exponent_mask) == _Traits::_Shifted_exponent_mask) {
+        // inf/nan detected; write appropriate string and return
+        const char* _Str;
+        size_t _Len;
+
+        const _Uint_type _Mantissa = _Uint_value & _Traits::_Denormal_mantissa_mask;
+
+        if (_Mantissa == 0) {
+            _Str = "inf";
+            _Len = 3;
+        } else if (_Was_negative && _Mantissa == _Traits::_Special_nan_mantissa_mask) {
+            // When a NaN value has the sign bit set, the quiet bit set, and all other mantissa bits cleared,
+            // the UCRT interprets it to mean "indeterminate", and indicates this by printing "-nan(ind)".
+            _Str = "nan(ind)";
+            _Len = 8;
+        } else if ((_Mantissa & _Traits::_Special_nan_mantissa_mask) != 0) {
+            _Str = "nan";
+            _Len = 3;
+        } else {
+            _Str = "nan(snan)";
+            _Len = 9;
+        }
+
+        if (_Last - _First < static_cast<ptrdiff_t>(_Len)) {
+            return {_Last, errc::value_too_large};
+        }
+
+        _VSTD::memcpy(_First, _Str, _Len);
+
+        return {_First + _Len, errc{}};
+    }
+
+    if constexpr (_Overload == _Floating_to_chars_overload::_Plain) {
+        return _Floating_to_chars_ryu(_First, _Last, _Value, chars_format{});
+    } else if constexpr (_Overload == _Floating_to_chars_overload::_Format_only) {
+        if (_Fmt == chars_format::hex) {
+            return _Floating_to_chars_hex_shortest(_First, _Last, _Value);
+        }
+
+        return _Floating_to_chars_ryu(_First, _Last, _Value, _Fmt);
+    } else if constexpr (_Overload == _Floating_to_chars_overload::_Format_precision) {
+        switch (_Fmt) {
+        case chars_format::scientific:
+            return _Floating_to_chars_scientific_precision(_First, _Last, _Value, _Precision);
+        case chars_format::fixed:
+            return _Floating_to_chars_fixed_precision(_First, _Last, _Value, _Precision);
+        case chars_format::general:
+            return _Floating_to_chars_general_precision(_First, _Last, _Value, _Precision);
+        case chars_format::hex:
+        default: // avoid MSVC warning C4715: not all control paths return a value
+            return _Floating_to_chars_hex_precision(_First, _Last, _Value, _Precision);
+        }
+    }
+}
+
+// clang-format on
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_SRC_INCLUDE_TO_CHARS_FLOATING_POINT_H
diff --git a/libcxx/src/ios.cpp b/libcxx/src/ios.cpp
index a9bd1dc32397..56fbcca417aa 100644
--- a/libcxx/src/ios.cpp
+++ b/libcxx/src/ios.cpp
@@ -137,7 +137,7 @@ ios_base::getloc() const
 
 // xalloc
 #if defined(_LIBCPP_HAS_C_ATOMIC_IMP) && !defined(_LIBCPP_HAS_NO_THREADS)
-atomic<int> ios_base::__xindex_ = ATOMIC_VAR_INIT(0);
+atomic<int> ios_base::__xindex_{0};
 #else
 int ios_base::__xindex_ = 0;
 #endif
diff --git a/libcxx/src/iostream.cpp b/libcxx/src/iostream.cpp
index 6070449621a0..d87bf7af84ba 100644
--- a/libcxx/src/iostream.cpp
+++ b/libcxx/src/iostream.cpp
@@ -85,12 +85,9 @@ __asm__("?wclog@" _LIBCPP_ABI_NAMESPACE_STR "@std@@3V?$basic_ostream@_WU?$char_t
 ;
 #endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS
 
-// Hacky way to make the compiler believe that we're inside a system header so
-// it doesn't flag the use of the init_priority attribute with a value that's
-// reserved for the implementation (we're the implementation).
-# 80 "iostream.cpp" 1 3
-_LIBCPP_HIDDEN ios_base::Init __start_std_streams _LIBCPP_INIT_PRIORITY_MAX;
-# 82 "iostream.cpp" 2
+// Pretend we're inside a system header so the compiler doesn't flag the use of the init_priority
+// attribute with a value that's reserved for the implementation (we're the implementation).
+#include "iostream_init.h"
 
 // On Windows the TLS storage for locales needs to be initialized before we create
 // the standard streams, otherwise it may not be alive during program termination
diff --git a/libcxx/src/iostream_init.h b/libcxx/src/iostream_init.h
new file mode 100644
index 000000000000..b0a60f42a67c
--- /dev/null
+++ b/libcxx/src/iostream_init.h
@@ -0,0 +1,2 @@
+#pragma GCC system_header
+_LIBCPP_HIDDEN ios_base::Init __start_std_streams _LIBCPP_INIT_PRIORITY_MAX;
\ No newline at end of file
diff --git a/libcxx/src/random.cpp b/libcxx/src/random.cpp
index 86740dd49032..286a45785154 100644
--- a/libcxx/src/random.cpp
+++ b/libcxx/src/random.cpp
@@ -9,8 +9,8 @@
 #include <__config>
 
 #if defined(_LIBCPP_USING_WIN32_RANDOM)
-// Must be defined before including stdlib.h to enable rand_s().
-#define _CRT_RAND_S
+    // Must be defined before including stdlib.h to enable rand_s().
+#   define _CRT_RAND_S
 #endif // defined(_LIBCPP_USING_WIN32_RANDOM)
 
 #include "limits"
@@ -18,7 +18,7 @@
 #include "system_error"
 
 #if defined(__sun__)
-#define rename solaris_headers_are_broken
+#   define rename solaris_headers_are_broken
 #endif // defined(__sun__)
 
 #include <errno.h>
@@ -26,16 +26,16 @@
 #include <stdlib.h>
 
 #if defined(_LIBCPP_USING_GETENTROPY)
-#include <sys/random.h>
+#   include <sys/random.h>
 #elif defined(_LIBCPP_USING_DEV_RANDOM)
-#include <fcntl.h>
-#include <unistd.h>
-#if __has_include(<sys/ioctl.h>) && __has_include(<linux/random.h>)
-#include <sys/ioctl.h>
-#include <linux/random.h>
-#endif
+#   include <fcntl.h>
+#   include <unistd.h>
+#   if __has_include(<sys/ioctl.h>) && __has_include(<linux/random.h>)
+#       include <sys/ioctl.h>
+#       include <linux/random.h>
+#   endif
 #elif defined(_LIBCPP_USING_NACL_RANDOM)
-#include <nacl/nacl_random.h>
+#   include <nacl/nacl_random.h>
 #endif
 
 
diff --git a/libcxx/src/ryu/README.txt b/libcxx/src/ryu/README.txt
new file mode 100644
index 000000000000..e3d17469d476
--- /dev/null
+++ b/libcxx/src/ryu/README.txt
@@ -0,0 +1,11 @@
+The code in this directory is based on Ulf Adams's work. The upstream for the
+code is:
+
+https://github.com/ulfjack/ryu/tree/59661c3/ryu
+
+The code has been adapted by Stephan T. Lavavej of Microsoft for usage in
+std::to_chars. This code has been contributed by Microsoft for inclusion in
+libc++.
+
+The code in this directory has a different coding style than other parts to
+minimize the number of changes by both upstream sources.
diff --git a/libcxx/src/ryu/d2fixed.cpp b/libcxx/src/ryu/d2fixed.cpp
new file mode 100644
index 000000000000..699f9158f4c3
--- /dev/null
+++ b/libcxx/src/ryu/d2fixed.cpp
@@ -0,0 +1,669 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// Copyright 2018 Ulf Adams
+// Copyright (c) Microsoft Corporation. All rights reserved.
+
+// Boost Software License - Version 1.0 - August 17th, 2003
+
+// Permission is hereby granted, free of charge, to any person or organization
+// obtaining a copy of the software and accompanying documentation covered by
+// this license (the "Software") to use, reproduce, display, distribute,
+// execute, and transmit the Software, and to prepare derivative works of the
+// Software, and to permit third-parties to whom the Software is furnished to
+// do so, all subject to the following:
+
+// The copyright notices in the Software and this entire statement, including
+// the above license grant, this restriction and the following disclaimer,
+// must be included in all copies of the Software, in whole or in part, and
+// all derivative works of the Software, unless such copies or derivative
+// works are solely in the form of machine-executable object code generated by
+// a source language processor.
+
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+// SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+// FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+// DEALINGS IN THE SOFTWARE.
+
+// Avoid formatting to keep the changes with the original code minimal.
+// clang-format off
+
+#include "__config"
+#include "charconv"
+#include "cstring"
+#include "system_error"
+
+#include "include/ryu/common.h"
+#include "include/ryu/d2fixed.h"
+#include "include/ryu/d2fixed_full_table.h"
+#include "include/ryu/d2s.h"
+#include "include/ryu/d2s_intrinsics.h"
+#include "include/ryu/digit_table.h"
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+inline constexpr int __POW10_ADDITIONAL_BITS = 120;
+
+#ifdef _LIBCPP_INTRINSIC128
+// Returns the low 64 bits of the high 128 bits of the 256-bit product of a and b.
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline uint64_t __umul256_hi128_lo64(
+  const uint64_t __aHi, const uint64_t __aLo, const uint64_t __bHi, const uint64_t __bLo) {
+  uint64_t __b00Hi;
+  const uint64_t __b00Lo = __ryu_umul128(__aLo, __bLo, &__b00Hi);
+  uint64_t __b01Hi;
+  const uint64_t __b01Lo = __ryu_umul128(__aLo, __bHi, &__b01Hi);
+  uint64_t __b10Hi;
+  const uint64_t __b10Lo = __ryu_umul128(__aHi, __bLo, &__b10Hi);
+  uint64_t __b11Hi;
+  const uint64_t __b11Lo = __ryu_umul128(__aHi, __bHi, &__b11Hi);
+  (void) __b00Lo; // unused
+  (void) __b11Hi; // unused
+  const uint64_t __temp1Lo = __b10Lo + __b00Hi;
+  const uint64_t __temp1Hi = __b10Hi + (__temp1Lo < __b10Lo);
+  const uint64_t __temp2Lo = __b01Lo + __temp1Lo;
+  const uint64_t __temp2Hi = __b01Hi + (__temp2Lo < __b01Lo);
+  return __b11Lo + __temp1Hi + __temp2Hi;
+}
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline uint32_t __uint128_mod1e9(const uint64_t __vHi, const uint64_t __vLo) {
+  // After multiplying, we're going to shift right by 29, then truncate to uint32_t.
+  // This means that we need only 29 + 32 = 61 bits, so we can truncate to uint64_t before shifting.
+  const uint64_t __multiplied = __umul256_hi128_lo64(__vHi, __vLo, 0x89705F4136B4A597u, 0x31680A88F8953031u);
+
+  // For uint32_t truncation, see the __mod1e9() comment in d2s_intrinsics.h.
+  const uint32_t __shifted = static_cast<uint32_t>(__multiplied >> 29);
+
+  return static_cast<uint32_t>(__vLo) - 1000000000 * __shifted;
+}
+#endif // ^^^ intrinsics available ^^^
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline uint32_t __mulShift_mod1e9(const uint64_t __m, const uint64_t* const __mul, const int32_t __j) {
+  uint64_t __high0;                                               // 64
+  const uint64_t __low0 = __ryu_umul128(__m, __mul[0], &__high0); // 0
+  uint64_t __high1;                                               // 128
+  const uint64_t __low1 = __ryu_umul128(__m, __mul[1], &__high1); // 64
+  uint64_t __high2;                                               // 192
+  const uint64_t __low2 = __ryu_umul128(__m, __mul[2], &__high2); // 128
+  const uint64_t __s0low = __low0;                  // 0
+  (void) __s0low; // unused
+  const uint64_t __s0high = __low1 + __high0;       // 64
+  const uint32_t __c1 = __s0high < __low1;
+  const uint64_t __s1low = __low2 + __high1 + __c1; // 128
+  const uint32_t __c2 = __s1low < __low2; // __high1 + __c1 can't overflow, so compare against __low2
+  const uint64_t __s1high = __high2 + __c2;         // 192
+  _LIBCPP_ASSERT(__j >= 128, "");
+  _LIBCPP_ASSERT(__j <= 180, "");
+#ifdef _LIBCPP_INTRINSIC128
+  const uint32_t __dist = static_cast<uint32_t>(__j - 128); // __dist: [0, 52]
+  const uint64_t __shiftedhigh = __s1high >> __dist;
+  const uint64_t __shiftedlow = __ryu_shiftright128(__s1low, __s1high, __dist);
+  return __uint128_mod1e9(__shiftedhigh, __shiftedlow);
+#else // ^^^ intrinsics available ^^^ / vvv intrinsics unavailable vvv
+  if (__j < 160) { // __j: [128, 160)
+    const uint64_t __r0 = __mod1e9(__s1high);
+    const uint64_t __r1 = __mod1e9((__r0 << 32) | (__s1low >> 32));
+    const uint64_t __r2 = ((__r1 << 32) | (__s1low & 0xffffffff));
+    return __mod1e9(__r2 >> (__j - 128));
+  } else { // __j: [160, 192)
+    const uint64_t __r0 = __mod1e9(__s1high);
+    const uint64_t __r1 = ((__r0 << 32) | (__s1low >> 32));
+    return __mod1e9(__r1 >> (__j - 160));
+  }
+#endif // ^^^ intrinsics unavailable ^^^
+}
+
+void __append_n_digits(const uint32_t __olength, uint32_t __digits, char* const __result) {
+  uint32_t __i = 0;
+  while (__digits >= 10000) {
+#ifdef __clang__ // TRANSITION, LLVM-38217
+    const uint32_t __c = __digits - 10000 * (__digits / 10000);
+#else
+    const uint32_t __c = __digits % 10000;
+#endif
+    __digits /= 10000;
+    const uint32_t __c0 = (__c % 100) << 1;
+    const uint32_t __c1 = (__c / 100) << 1;
+    _VSTD::memcpy(__result + __olength - __i - 2, __DIGIT_TABLE + __c0, 2);
+    _VSTD::memcpy(__result + __olength - __i - 4, __DIGIT_TABLE + __c1, 2);
+    __i += 4;
+  }
+  if (__digits >= 100) {
+    const uint32_t __c = (__digits % 100) << 1;
+    __digits /= 100;
+    _VSTD::memcpy(__result + __olength - __i - 2, __DIGIT_TABLE + __c, 2);
+    __i += 2;
+  }
+  if (__digits >= 10) {
+    const uint32_t __c = __digits << 1;
+    _VSTD::memcpy(__result + __olength - __i - 2, __DIGIT_TABLE + __c, 2);
+  } else {
+    __result[0] = static_cast<char>('0' + __digits);
+  }
+}
+
+_LIBCPP_HIDE_FROM_ABI inline void __append_d_digits(const uint32_t __olength, uint32_t __digits, char* const __result) {
+  uint32_t __i = 0;
+  while (__digits >= 10000) {
+#ifdef __clang__ // TRANSITION, LLVM-38217
+    const uint32_t __c = __digits - 10000 * (__digits / 10000);
+#else
+    const uint32_t __c = __digits % 10000;
+#endif
+    __digits /= 10000;
+    const uint32_t __c0 = (__c % 100) << 1;
+    const uint32_t __c1 = (__c / 100) << 1;
+    _VSTD::memcpy(__result + __olength + 1 - __i - 2, __DIGIT_TABLE + __c0, 2);
+    _VSTD::memcpy(__result + __olength + 1 - __i - 4, __DIGIT_TABLE + __c1, 2);
+    __i += 4;
+  }
+  if (__digits >= 100) {
+    const uint32_t __c = (__digits % 100) << 1;
+    __digits /= 100;
+    _VSTD::memcpy(__result + __olength + 1 - __i - 2, __DIGIT_TABLE + __c, 2);
+    __i += 2;
+  }
+  if (__digits >= 10) {
+    const uint32_t __c = __digits << 1;
+    __result[2] = __DIGIT_TABLE[__c + 1];
+    __result[1] = '.';
+    __result[0] = __DIGIT_TABLE[__c];
+  } else {
+    __result[1] = '.';
+    __result[0] = static_cast<char>('0' + __digits);
+  }
+}
+
+_LIBCPP_HIDE_FROM_ABI inline void __append_c_digits(const uint32_t __count, uint32_t __digits, char* const __result) {
+  uint32_t __i = 0;
+  for (; __i < __count - 1; __i += 2) {
+    const uint32_t __c = (__digits % 100) << 1;
+    __digits /= 100;
+    _VSTD::memcpy(__result + __count - __i - 2, __DIGIT_TABLE + __c, 2);
+  }
+  if (__i < __count) {
+    const char __c = static_cast<char>('0' + (__digits % 10));
+    __result[__count - __i - 1] = __c;
+  }
+}
+
+void __append_nine_digits(uint32_t __digits, char* const __result) {
+  if (__digits == 0) {
+    _VSTD::memset(__result, '0', 9);
+    return;
+  }
+
+  for (uint32_t __i = 0; __i < 5; __i += 4) {
+#ifdef __clang__ // TRANSITION, LLVM-38217
+    const uint32_t __c = __digits - 10000 * (__digits / 10000);
+#else
+    const uint32_t __c = __digits % 10000;
+#endif
+    __digits /= 10000;
+    const uint32_t __c0 = (__c % 100) << 1;
+    const uint32_t __c1 = (__c / 100) << 1;
+    _VSTD::memcpy(__result + 7 - __i, __DIGIT_TABLE + __c0, 2);
+    _VSTD::memcpy(__result + 5 - __i, __DIGIT_TABLE + __c1, 2);
+  }
+  __result[0] = static_cast<char>('0' + __digits);
+}
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline uint32_t __indexForExponent(const uint32_t __e) {
+  return (__e + 15) / 16;
+}
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline uint32_t __pow10BitsForIndex(const uint32_t __idx) {
+  return 16 * __idx + __POW10_ADDITIONAL_BITS;
+}
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline uint32_t __lengthForIndex(const uint32_t __idx) {
+  // +1 for ceil, +16 for mantissa, +8 to round up when dividing by 9
+  return (__log10Pow2(16 * static_cast<int32_t>(__idx)) + 1 + 16 + 8) / 9;
+}
+
+[[nodiscard]] to_chars_result __d2fixed_buffered_n(char* _First, char* const _Last, const double __d,
+  const uint32_t __precision) {
+  char* const _Original_first = _First;
+
+  const uint64_t __bits = __double_to_bits(__d);
+
+  // Case distinction; exit early for the easy cases.
+  if (__bits == 0) {
+    const int32_t _Total_zero_length = 1 // leading zero
+      + static_cast<int32_t>(__precision != 0) // possible decimal point
+      + static_cast<int32_t>(__precision); // zeroes after decimal point
+
+    if (_Last - _First < _Total_zero_length) {
+      return { _Last, errc::value_too_large };
+    }
+
+    *_First++ = '0';
+    if (__precision > 0) {
+      *_First++ = '.';
+      _VSTD::memset(_First, '0', __precision);
+      _First += __precision;
+    }
+    return { _First, errc{} };
+  }
+
+  // Decode __bits into mantissa and exponent.
+  const uint64_t __ieeeMantissa = __bits & ((1ull << __DOUBLE_MANTISSA_BITS) - 1);
+  const uint32_t __ieeeExponent = static_cast<uint32_t>(__bits >> __DOUBLE_MANTISSA_BITS);
+
+  int32_t __e2;
+  uint64_t __m2;
+  if (__ieeeExponent == 0) {
+    __e2 = 1 - __DOUBLE_BIAS - __DOUBLE_MANTISSA_BITS;
+    __m2 = __ieeeMantissa;
+  } else {
+    __e2 = static_cast<int32_t>(__ieeeExponent) - __DOUBLE_BIAS - __DOUBLE_MANTISSA_BITS;
+    __m2 = (1ull << __DOUBLE_MANTISSA_BITS) | __ieeeMantissa;
+  }
+
+  bool __nonzero = false;
+  if (__e2 >= -52) {
+    const uint32_t __idx = __e2 < 0 ? 0 : __indexForExponent(static_cast<uint32_t>(__e2));
+    const uint32_t __p10bits = __pow10BitsForIndex(__idx);
+    const int32_t __len = static_cast<int32_t>(__lengthForIndex(__idx));
+    for (int32_t __i = __len - 1; __i >= 0; --__i) {
+      const uint32_t __j = __p10bits - __e2;
+      // Temporary: __j is usually around 128, and by shifting a bit, we push it to 128 or above, which is
+      // a slightly faster code path in __mulShift_mod1e9. Instead, we can just increase the multipliers.
+      const uint32_t __digits = __mulShift_mod1e9(__m2 << 8, __POW10_SPLIT[__POW10_OFFSET[__idx] + __i],
+        static_cast<int32_t>(__j + 8));
+      if (__nonzero) {
+        if (_Last - _First < 9) {
+          return { _Last, errc::value_too_large };
+        }
+        __append_nine_digits(__digits, _First);
+        _First += 9;
+      } else if (__digits != 0) {
+        const uint32_t __olength = __decimalLength9(__digits);
+        if (_Last - _First < static_cast<ptrdiff_t>(__olength)) {
+          return { _Last, errc::value_too_large };
+        }
+        __append_n_digits(__olength, __digits, _First);
+        _First += __olength;
+        __nonzero = true;
+      }
+    }
+  }
+  if (!__nonzero) {
+    if (_First == _Last) {
+      return { _Last, errc::value_too_large };
+    }
+    *_First++ = '0';
+  }
+  if (__precision > 0) {
+    if (_First == _Last) {
+      return { _Last, errc::value_too_large };
+    }
+    *_First++ = '.';
+  }
+  if (__e2 < 0) {
+    const int32_t __idx = -__e2 / 16;
+    const uint32_t __blocks = __precision / 9 + 1;
+    // 0 = don't round up; 1 = round up unconditionally; 2 = round up if odd.
+    int __roundUp = 0;
+    uint32_t __i = 0;
+    if (__blocks <= __MIN_BLOCK_2[__idx]) {
+      __i = __blocks;
+      if (_Last - _First < static_cast<ptrdiff_t>(__precision)) {
+        return { _Last, errc::value_too_large };
+      }
+      _VSTD::memset(_First, '0', __precision);
+      _First += __precision;
+    } else if (__i < __MIN_BLOCK_2[__idx]) {
+      __i = __MIN_BLOCK_2[__idx];
+      if (_Last - _First < static_cast<ptrdiff_t>(9 * __i)) {
+        return { _Last, errc::value_too_large };
+      }
+      _VSTD::memset(_First, '0', 9 * __i);
+      _First += 9 * __i;
+    }
+    for (; __i < __blocks; ++__i) {
+      const int32_t __j = __ADDITIONAL_BITS_2 + (-__e2 - 16 * __idx);
+      const uint32_t __p = __POW10_OFFSET_2[__idx] + __i - __MIN_BLOCK_2[__idx];
+      if (__p >= __POW10_OFFSET_2[__idx + 1]) {
+        // If the remaining digits are all 0, then we might as well use memset.
+        // No rounding required in this case.
+        const uint32_t __fill = __precision - 9 * __i;
+        if (_Last - _First < static_cast<ptrdiff_t>(__fill)) {
+          return { _Last, errc::value_too_large };
+        }
+        _VSTD::memset(_First, '0', __fill);
+        _First += __fill;
+        break;
+      }
+      // Temporary: __j is usually around 128, and by shifting a bit, we push it to 128 or above, which is
+      // a slightly faster code path in __mulShift_mod1e9. Instead, we can just increase the multipliers.
+      uint32_t __digits = __mulShift_mod1e9(__m2 << 8, __POW10_SPLIT_2[__p], __j + 8);
+      if (__i < __blocks - 1) {
+        if (_Last - _First < 9) {
+          return { _Last, errc::value_too_large };
+        }
+        __append_nine_digits(__digits, _First);
+        _First += 9;
+      } else {
+        const uint32_t __maximum = __precision - 9 * __i;
+        uint32_t __lastDigit = 0;
+        for (uint32_t __k = 0; __k < 9 - __maximum; ++__k) {
+          __lastDigit = __digits % 10;
+          __digits /= 10;
+        }
+        if (__lastDigit != 5) {
+          __roundUp = __lastDigit > 5;
+        } else {
+          // Is m * 10^(additionalDigits + 1) / 2^(-__e2) integer?
+          const int32_t __requiredTwos = -__e2 - static_cast<int32_t>(__precision) - 1;
+          const bool __trailingZeros = __requiredTwos <= 0
+            || (__requiredTwos < 60 && __multipleOfPowerOf2(__m2, static_cast<uint32_t>(__requiredTwos)));
+          __roundUp = __trailingZeros ? 2 : 1;
+        }
+        if (__maximum > 0) {
+          if (_Last - _First < static_cast<ptrdiff_t>(__maximum)) {
+            return { _Last, errc::value_too_large };
+          }
+          __append_c_digits(__maximum, __digits, _First);
+          _First += __maximum;
+        }
+        break;
+      }
+    }
+    if (__roundUp != 0) {
+      char* _Round = _First;
+      char* _Dot = _Last;
+      while (true) {
+        if (_Round == _Original_first) {
+          _Round[0] = '1';
+          if (_Dot != _Last) {
+            _Dot[0] = '0';
+            _Dot[1] = '.';
+          }
+          if (_First == _Last) {
+            return { _Last, errc::value_too_large };
+          }
+          *_First++ = '0';
+          break;
+        }
+        --_Round;
+        const char __c = _Round[0];
+        if (__c == '.') {
+          _Dot = _Round;
+        } else if (__c == '9') {
+          _Round[0] = '0';
+          __roundUp = 1;
+        } else {
+          if (__roundUp == 1 || __c % 2 != 0) {
+            _Round[0] = __c + 1;
+          }
+          break;
+        }
+      }
+    }
+  } else {
+    if (_Last - _First < static_cast<ptrdiff_t>(__precision)) {
+      return { _Last, errc::value_too_large };
+    }
+    _VSTD::memset(_First, '0', __precision);
+    _First += __precision;
+  }
+  return { _First, errc{} };
+}
+
+[[nodiscard]] to_chars_result __d2exp_buffered_n(char* _First, char* const _Last, const double __d,
+  uint32_t __precision) {
+  char* const _Original_first = _First;
+
+  const uint64_t __bits = __double_to_bits(__d);
+
+  // Case distinction; exit early for the easy cases.
+  if (__bits == 0) {
+    const int32_t _Total_zero_length = 1 // leading zero
+      + static_cast<int32_t>(__precision != 0) // possible decimal point
+      + static_cast<int32_t>(__precision) // zeroes after decimal point
+      + 4; // "e+00"
+    if (_Last - _First < _Total_zero_length) {
+      return { _Last, errc::value_too_large };
+    }
+    *_First++ = '0';
+    if (__precision > 0) {
+      *_First++ = '.';
+      _VSTD::memset(_First, '0', __precision);
+      _First += __precision;
+    }
+    _VSTD::memcpy(_First, "e+00", 4);
+    _First += 4;
+    return { _First, errc{} };
+  }
+
+  // Decode __bits into mantissa and exponent.
+  const uint64_t __ieeeMantissa = __bits & ((1ull << __DOUBLE_MANTISSA_BITS) - 1);
+  const uint32_t __ieeeExponent = static_cast<uint32_t>(__bits >> __DOUBLE_MANTISSA_BITS);
+
+  int32_t __e2;
+  uint64_t __m2;
+  if (__ieeeExponent == 0) {
+    __e2 = 1 - __DOUBLE_BIAS - __DOUBLE_MANTISSA_BITS;
+    __m2 = __ieeeMantissa;
+  } else {
+    __e2 = static_cast<int32_t>(__ieeeExponent) - __DOUBLE_BIAS - __DOUBLE_MANTISSA_BITS;
+    __m2 = (1ull << __DOUBLE_MANTISSA_BITS) | __ieeeMantissa;
+  }
+
+  const bool __printDecimalPoint = __precision > 0;
+  ++__precision;
+  uint32_t __digits = 0;
+  uint32_t __printedDigits = 0;
+  uint32_t __availableDigits = 0;
+  int32_t __exp = 0;
+  if (__e2 >= -52) {
+    const uint32_t __idx = __e2 < 0 ? 0 : __indexForExponent(static_cast<uint32_t>(__e2));
+    const uint32_t __p10bits = __pow10BitsForIndex(__idx);
+    const int32_t __len = static_cast<int32_t>(__lengthForIndex(__idx));
+    for (int32_t __i = __len - 1; __i >= 0; --__i) {
+      const uint32_t __j = __p10bits - __e2;
+      // Temporary: __j is usually around 128, and by shifting a bit, we push it to 128 or above, which is
+      // a slightly faster code path in __mulShift_mod1e9. Instead, we can just increase the multipliers.
+      __digits = __mulShift_mod1e9(__m2 << 8, __POW10_SPLIT[__POW10_OFFSET[__idx] + __i],
+        static_cast<int32_t>(__j + 8));
+      if (__printedDigits != 0) {
+        if (__printedDigits + 9 > __precision) {
+          __availableDigits = 9;
+          break;
+        }
+        if (_Last - _First < 9) {
+          return { _Last, errc::value_too_large };
+        }
+        __append_nine_digits(__digits, _First);
+        _First += 9;
+        __printedDigits += 9;
+      } else if (__digits != 0) {
+        __availableDigits = __decimalLength9(__digits);
+        __exp = __i * 9 + static_cast<int32_t>(__availableDigits) - 1;
+        if (__availableDigits > __precision) {
+          break;
+        }
+        if (__printDecimalPoint) {
+          if (_Last - _First < static_cast<ptrdiff_t>(__availableDigits + 1)) {
+            return { _Last, errc::value_too_large };
+          }
+          __append_d_digits(__availableDigits, __digits, _First);
+          _First += __availableDigits + 1; // +1 for decimal point
+        } else {
+          if (_First == _Last) {
+            return { _Last, errc::value_too_large };
+          }
+          *_First++ = static_cast<char>('0' + __digits);
+        }
+        __printedDigits = __availableDigits;
+        __availableDigits = 0;
+      }
+    }
+  }
+
+  if (__e2 < 0 && __availableDigits == 0) {
+    const int32_t __idx = -__e2 / 16;
+    for (int32_t __i = __MIN_BLOCK_2[__idx]; __i < 200; ++__i) {
+      const int32_t __j = __ADDITIONAL_BITS_2 + (-__e2 - 16 * __idx);
+      const uint32_t __p = __POW10_OFFSET_2[__idx] + static_cast<uint32_t>(__i) - __MIN_BLOCK_2[__idx];
+      // Temporary: __j is usually around 128, and by shifting a bit, we push it to 128 or above, which is
+      // a slightly faster code path in __mulShift_mod1e9. Instead, we can just increase the multipliers.
+      __digits = (__p >= __POW10_OFFSET_2[__idx + 1]) ? 0 : __mulShift_mod1e9(__m2 << 8, __POW10_SPLIT_2[__p], __j + 8);
+      if (__printedDigits != 0) {
+        if (__printedDigits + 9 > __precision) {
+          __availableDigits = 9;
+          break;
+        }
+        if (_Last - _First < 9) {
+          return { _Last, errc::value_too_large };
+        }
+        __append_nine_digits(__digits, _First);
+        _First += 9;
+        __printedDigits += 9;
+      } else if (__digits != 0) {
+        __availableDigits = __decimalLength9(__digits);
+        __exp = -(__i + 1) * 9 + static_cast<int32_t>(__availableDigits) - 1;
+        if (__availableDigits > __precision) {
+          break;
+        }
+        if (__printDecimalPoint) {
+          if (_Last - _First < static_cast<ptrdiff_t>(__availableDigits + 1)) {
+            return { _Last, errc::value_too_large };
+          }
+          __append_d_digits(__availableDigits, __digits, _First);
+          _First += __availableDigits + 1; // +1 for decimal point
+        } else {
+          if (_First == _Last) {
+            return { _Last, errc::value_too_large };
+          }
+          *_First++ = static_cast<char>('0' + __digits);
+        }
+        __printedDigits = __availableDigits;
+        __availableDigits = 0;
+      }
+    }
+  }
+
+  const uint32_t __maximum = __precision - __printedDigits;
+  if (__availableDigits == 0) {
+    __digits = 0;
+  }
+  uint32_t __lastDigit = 0;
+  if (__availableDigits > __maximum) {
+    for (uint32_t __k = 0; __k < __availableDigits - __maximum; ++__k) {
+      __lastDigit = __digits % 10;
+      __digits /= 10;
+    }
+  }
+  // 0 = don't round up; 1 = round up unconditionally; 2 = round up if odd.
+  int __roundUp = 0;
+  if (__lastDigit != 5) {
+    __roundUp = __lastDigit > 5;
+  } else {
+    // Is m * 2^__e2 * 10^(__precision + 1 - __exp) integer?
+    // __precision was already increased by 1, so we don't need to write + 1 here.
+    const int32_t __rexp = static_cast<int32_t>(__precision) - __exp;
+    const int32_t __requiredTwos = -__e2 - __rexp;
+    bool __trailingZeros = __requiredTwos <= 0
+      || (__requiredTwos < 60 && __multipleOfPowerOf2(__m2, static_cast<uint32_t>(__requiredTwos)));
+    if (__rexp < 0) {
+      const int32_t __requiredFives = -__rexp;
+      __trailingZeros = __trailingZeros && __multipleOfPowerOf5(__m2, static_cast<uint32_t>(__requiredFives));
+    }
+    __roundUp = __trailingZeros ? 2 : 1;
+  }
+  if (__printedDigits != 0) {
+    if (_Last - _First < static_cast<ptrdiff_t>(__maximum)) {
+      return { _Last, errc::value_too_large };
+    }
+    if (__digits == 0) {
+      _VSTD::memset(_First, '0', __maximum);
+    } else {
+      __append_c_digits(__maximum, __digits, _First);
+    }
+    _First += __maximum;
+  } else {
+    if (__printDecimalPoint) {
+      if (_Last - _First < static_cast<ptrdiff_t>(__maximum + 1)) {
+        return { _Last, errc::value_too_large };
+      }
+      __append_d_digits(__maximum, __digits, _First);
+      _First += __maximum + 1; // +1 for decimal point
+    } else {
+      if (_First == _Last) {
+        return { _Last, errc::value_too_large };
+      }
+      *_First++ = static_cast<char>('0' + __digits);
+    }
+  }
+  if (__roundUp != 0) {
+    char* _Round = _First;
+    while (true) {
+      if (_Round == _Original_first) {
+        _Round[0] = '1';
+        ++__exp;
+        break;
+      }
+      --_Round;
+      const char __c = _Round[0];
+      if (__c == '.') {
+        // Keep going.
+      } else if (__c == '9') {
+        _Round[0] = '0';
+        __roundUp = 1;
+      } else {
+        if (__roundUp == 1 || __c % 2 != 0) {
+          _Round[0] = __c + 1;
+        }
+        break;
+      }
+    }
+  }
+
+  char _Sign_character;
+
+  if (__exp < 0) {
+    _Sign_character = '-';
+    __exp = -__exp;
+  } else {
+    _Sign_character = '+';
+  }
+
+  const int _Exponent_part_length = __exp >= 100
+    ? 5 // "e+NNN"
+    : 4; // "e+NN"
+
+  if (_Last - _First < _Exponent_part_length) {
+    return { _Last, errc::value_too_large };
+  }
+
+  *_First++ = 'e';
+  *_First++ = _Sign_character;
+
+  if (__exp >= 100) {
+    const int32_t __c = __exp % 10;
+    _VSTD::memcpy(_First, __DIGIT_TABLE + 2 * (__exp / 10), 2);
+    _First[2] = static_cast<char>('0' + __c);
+    _First += 3;
+  } else {
+    _VSTD::memcpy(_First, __DIGIT_TABLE + 2 * __exp, 2);
+    _First += 2;
+  }
+
+  return { _First, errc{} };
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+// clang-format on
diff --git a/libcxx/src/ryu/d2s.cpp b/libcxx/src/ryu/d2s.cpp
new file mode 100644
index 000000000000..510b4b8aa3af
--- /dev/null
+++ b/libcxx/src/ryu/d2s.cpp
@@ -0,0 +1,782 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// Copyright 2018 Ulf Adams
+// Copyright (c) Microsoft Corporation. All rights reserved.
+
+// Boost Software License - Version 1.0 - August 17th, 2003
+
+// Permission is hereby granted, free of charge, to any person or organization
+// obtaining a copy of the software and accompanying documentation covered by
+// this license (the "Software") to use, reproduce, display, distribute,
+// execute, and transmit the Software, and to prepare derivative works of the
+// Software, and to permit third-parties to whom the Software is furnished to
+// do so, all subject to the following:
+
+// The copyright notices in the Software and this entire statement, including
+// the above license grant, this restriction and the following disclaimer,
+// must be included in all copies of the Software, in whole or in part, and
+// all derivative works of the Software, unless such copies or derivative
+// works are solely in the form of machine-executable object code generated by
+// a source language processor.
+
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+// SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+// FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+// DEALINGS IN THE SOFTWARE.
+
+// Avoid formatting to keep the changes with the original code minimal.
+// clang-format off
+
+#include "__config"
+#include "charconv"
+
+#include "include/ryu/common.h"
+#include "include/ryu/d2fixed.h"
+#include "include/ryu/d2s.h"
+#include "include/ryu/d2s_full_table.h"
+#include "include/ryu/d2s_intrinsics.h"
+#include "include/ryu/digit_table.h"
+#include "include/ryu/ryu.h"
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// We need a 64x128-bit multiplication and a subsequent 128-bit shift.
+// Multiplication:
+//   The 64-bit factor is variable and passed in, the 128-bit factor comes
+//   from a lookup table. We know that the 64-bit factor only has 55
+//   significant bits (i.e., the 9 topmost bits are zeros). The 128-bit
+//   factor only has 124 significant bits (i.e., the 4 topmost bits are
+//   zeros).
+// Shift:
+//   In principle, the multiplication result requires 55 + 124 = 179 bits to
+//   represent. However, we then shift this value to the right by __j, which is
+//   at least __j >= 115, so the result is guaranteed to fit into 179 - 115 = 64
+//   bits. This means that we only need the topmost 64 significant bits of
+//   the 64x128-bit multiplication.
+//
+// There are several ways to do this:
+// 1. Best case: the compiler exposes a 128-bit type.
+//    We perform two 64x64-bit multiplications, add the higher 64 bits of the
+//    lower result to the higher result, and shift by __j - 64 bits.
+//
+//    We explicitly cast from 64-bit to 128-bit, so the compiler can tell
+//    that these are only 64-bit inputs, and can map these to the best
+//    possible sequence of assembly instructions.
+//    x64 machines happen to have matching assembly instructions for
+//    64x64-bit multiplications and 128-bit shifts.
+//
+// 2. Second best case: the compiler exposes intrinsics for the x64 assembly
+//    instructions mentioned in 1.
+//
+// 3. We only have 64x64 bit instructions that return the lower 64 bits of
+//    the result, i.e., we have to use plain C.
+//    Our inputs are less than the full width, so we have three options:
+//    a. Ignore this fact and just implement the intrinsics manually.
+//    b. Split both into 31-bit pieces, which guarantees no internal overflow,
+//       but requires extra work upfront (unless we change the lookup table).
+//    c. Split only the first factor into 31-bit pieces, which also guarantees
+//       no internal overflow, but requires extra work since the intermediate
+//       results are not perfectly aligned.
+#ifdef _LIBCPP_INTRINSIC128
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline uint64_t __mulShift(const uint64_t __m, const uint64_t* const __mul, const int32_t __j) {
+  // __m is maximum 55 bits
+  uint64_t __high1;                                               // 128
+  const uint64_t __low1 = __ryu_umul128(__m, __mul[1], &__high1); // 64
+  uint64_t __high0;                                               // 64
+  (void) __ryu_umul128(__m, __mul[0], &__high0);                  // 0
+  const uint64_t __sum = __high0 + __low1;
+  if (__sum < __high0) {
+    ++__high1; // overflow into __high1
+  }
+  return __ryu_shiftright128(__sum, __high1, static_cast<uint32_t>(__j - 64));
+}
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline uint64_t __mulShiftAll(const uint64_t __m, const uint64_t* const __mul, const int32_t __j,
+  uint64_t* const __vp, uint64_t* const __vm, const uint32_t __mmShift) {
+  *__vp = __mulShift(4 * __m + 2, __mul, __j);
+  *__vm = __mulShift(4 * __m - 1 - __mmShift, __mul, __j);
+  return __mulShift(4 * __m, __mul, __j);
+}
+
+#else // ^^^ intrinsics available ^^^ / vvv intrinsics unavailable vvv
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline _LIBCPP_ALWAYS_INLINE uint64_t __mulShiftAll(uint64_t __m, const uint64_t* const __mul, const int32_t __j,
+  uint64_t* const __vp, uint64_t* const __vm, const uint32_t __mmShift) { // TRANSITION, VSO-634761
+  __m <<= 1;
+  // __m is maximum 55 bits
+  uint64_t __tmp;
+  const uint64_t __lo = __ryu_umul128(__m, __mul[0], &__tmp);
+  uint64_t __hi;
+  const uint64_t __mid = __tmp + __ryu_umul128(__m, __mul[1], &__hi);
+  __hi += __mid < __tmp; // overflow into __hi
+
+  const uint64_t __lo2 = __lo + __mul[0];
+  const uint64_t __mid2 = __mid + __mul[1] + (__lo2 < __lo);
+  const uint64_t __hi2 = __hi + (__mid2 < __mid);
+  *__vp = __ryu_shiftright128(__mid2, __hi2, static_cast<uint32_t>(__j - 64 - 1));
+
+  if (__mmShift == 1) {
+    const uint64_t __lo3 = __lo - __mul[0];
+    const uint64_t __mid3 = __mid - __mul[1] - (__lo3 > __lo);
+    const uint64_t __hi3 = __hi - (__mid3 > __mid);
+    *__vm = __ryu_shiftright128(__mid3, __hi3, static_cast<uint32_t>(__j - 64 - 1));
+  } else {
+    const uint64_t __lo3 = __lo + __lo;
+    const uint64_t __mid3 = __mid + __mid + (__lo3 < __lo);
+    const uint64_t __hi3 = __hi + __hi + (__mid3 < __mid);
+    const uint64_t __lo4 = __lo3 - __mul[0];
+    const uint64_t __mid4 = __mid3 - __mul[1] - (__lo4 > __lo3);
+    const uint64_t __hi4 = __hi3 - (__mid4 > __mid3);
+    *__vm = __ryu_shiftright128(__mid4, __hi4, static_cast<uint32_t>(__j - 64));
+  }
+
+  return __ryu_shiftright128(__mid, __hi, static_cast<uint32_t>(__j - 64 - 1));
+}
+
+#endif // ^^^ intrinsics unavailable ^^^
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline uint32_t __decimalLength17(const uint64_t __v) {
+  // This is slightly faster than a loop.
+  // The average output length is 16.38 digits, so we check high-to-low.
+  // Function precondition: __v is not an 18, 19, or 20-digit number.
+  // (17 digits are sufficient for round-tripping.)
+  _LIBCPP_ASSERT(__v < 100000000000000000u, "");
+  if (__v >= 10000000000000000u) { return 17; }
+  if (__v >= 1000000000000000u) { return 16; }
+  if (__v >= 100000000000000u) { return 15; }
+  if (__v >= 10000000000000u) { return 14; }
+  if (__v >= 1000000000000u) { return 13; }
+  if (__v >= 100000000000u) { return 12; }
+  if (__v >= 10000000000u) { return 11; }
+  if (__v >= 1000000000u) { return 10; }
+  if (__v >= 100000000u) { return 9; }
+  if (__v >= 10000000u) { return 8; }
+  if (__v >= 1000000u) { return 7; }
+  if (__v >= 100000u) { return 6; }
+  if (__v >= 10000u) { return 5; }
+  if (__v >= 1000u) { return 4; }
+  if (__v >= 100u) { return 3; }
+  if (__v >= 10u) { return 2; }
+  return 1;
+}
+
+// A floating decimal representing m * 10^e.
+struct __floating_decimal_64 {
+  uint64_t __mantissa;
+  int32_t __exponent;
+};
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline __floating_decimal_64 __d2d(const uint64_t __ieeeMantissa, const uint32_t __ieeeExponent) {
+  int32_t __e2;
+  uint64_t __m2;
+  if (__ieeeExponent == 0) {
+    // We subtract 2 so that the bounds computation has 2 additional bits.
+    __e2 = 1 - __DOUBLE_BIAS - __DOUBLE_MANTISSA_BITS - 2;
+    __m2 = __ieeeMantissa;
+  } else {
+    __e2 = static_cast<int32_t>(__ieeeExponent) - __DOUBLE_BIAS - __DOUBLE_MANTISSA_BITS - 2;
+    __m2 = (1ull << __DOUBLE_MANTISSA_BITS) | __ieeeMantissa;
+  }
+  const bool __even = (__m2 & 1) == 0;
+  const bool __acceptBounds = __even;
+
+  // Step 2: Determine the interval of valid decimal representations.
+  const uint64_t __mv = 4 * __m2;
+  // Implicit bool -> int conversion. True is 1, false is 0.
+  const uint32_t __mmShift = __ieeeMantissa != 0 || __ieeeExponent <= 1;
+  // We would compute __mp and __mm like this:
+  // uint64_t __mp = 4 * __m2 + 2;
+  // uint64_t __mm = __mv - 1 - __mmShift;
+
+  // Step 3: Convert to a decimal power base using 128-bit arithmetic.
+  uint64_t __vr, __vp, __vm;
+  int32_t __e10;
+  bool __vmIsTrailingZeros = false;
+  bool __vrIsTrailingZeros = false;
+  if (__e2 >= 0) {
+    // I tried special-casing __q == 0, but there was no effect on performance.
+    // This expression is slightly faster than max(0, __log10Pow2(__e2) - 1).
+    const uint32_t __q = __log10Pow2(__e2) - (__e2 > 3);
+    __e10 = static_cast<int32_t>(__q);
+    const int32_t __k = __DOUBLE_POW5_INV_BITCOUNT + __pow5bits(static_cast<int32_t>(__q)) - 1;
+    const int32_t __i = -__e2 + static_cast<int32_t>(__q) + __k;
+    __vr = __mulShiftAll(__m2, __DOUBLE_POW5_INV_SPLIT[__q], __i, &__vp, &__vm, __mmShift);
+    if (__q <= 21) {
+      // This should use __q <= 22, but I think 21 is also safe. Smaller values
+      // may still be safe, but it's more difficult to reason about them.
+      // Only one of __mp, __mv, and __mm can be a multiple of 5, if any.
+      const uint32_t __mvMod5 = static_cast<uint32_t>(__mv) - 5 * static_cast<uint32_t>(__div5(__mv));
+      if (__mvMod5 == 0) {
+        __vrIsTrailingZeros = __multipleOfPowerOf5(__mv, __q);
+      } else if (__acceptBounds) {
+        // Same as min(__e2 + (~__mm & 1), __pow5Factor(__mm)) >= __q
+        // <=> __e2 + (~__mm & 1) >= __q && __pow5Factor(__mm) >= __q
+        // <=> true && __pow5Factor(__mm) >= __q, since __e2 >= __q.
+        __vmIsTrailingZeros = __multipleOfPowerOf5(__mv - 1 - __mmShift, __q);
+      } else {
+        // Same as min(__e2 + 1, __pow5Factor(__mp)) >= __q.
+        __vp -= __multipleOfPowerOf5(__mv + 2, __q);
+      }
+    }
+  } else {
+    // This expression is slightly faster than max(0, __log10Pow5(-__e2) - 1).
+    const uint32_t __q = __log10Pow5(-__e2) - (-__e2 > 1);
+    __e10 = static_cast<int32_t>(__q) + __e2;
+    const int32_t __i = -__e2 - static_cast<int32_t>(__q);
+    const int32_t __k = __pow5bits(__i) - __DOUBLE_POW5_BITCOUNT;
+    const int32_t __j = static_cast<int32_t>(__q) - __k;
+    __vr = __mulShiftAll(__m2, __DOUBLE_POW5_SPLIT[__i], __j, &__vp, &__vm, __mmShift);
+    if (__q <= 1) {
+      // {__vr,__vp,__vm} is trailing zeros if {__mv,__mp,__mm} has at least __q trailing 0 bits.
+      // __mv = 4 * __m2, so it always has at least two trailing 0 bits.
+      __vrIsTrailingZeros = true;
+      if (__acceptBounds) {
+        // __mm = __mv - 1 - __mmShift, so it has 1 trailing 0 bit iff __mmShift == 1.
+        __vmIsTrailingZeros = __mmShift == 1;
+      } else {
+        // __mp = __mv + 2, so it always has at least one trailing 0 bit.
+        --__vp;
+      }
+    } else if (__q < 63) { // TRANSITION(ulfjack): Use a tighter bound here.
+      // We need to compute min(ntz(__mv), __pow5Factor(__mv) - __e2) >= __q - 1
+      // <=> ntz(__mv) >= __q - 1 && __pow5Factor(__mv) - __e2 >= __q - 1
+      // <=> ntz(__mv) >= __q - 1 (__e2 is negative and -__e2 >= __q)
+      // <=> (__mv & ((1 << (__q - 1)) - 1)) == 0
+      // We also need to make sure that the left shift does not overflow.
+      __vrIsTrailingZeros = __multipleOfPowerOf2(__mv, __q - 1);
+    }
+  }
+
+  // Step 4: Find the shortest decimal representation in the interval of valid representations.
+  int32_t __removed = 0;
+  uint8_t __lastRemovedDigit = 0;
+  uint64_t _Output;
+  // On average, we remove ~2 digits.
+  if (__vmIsTrailingZeros || __vrIsTrailingZeros) {
+    // General case, which happens rarely (~0.7%).
+    for (;;) {
+      const uint64_t __vpDiv10 = __div10(__vp);
+      const uint64_t __vmDiv10 = __div10(__vm);
+      if (__vpDiv10 <= __vmDiv10) {
+        break;
+      }
+      const uint32_t __vmMod10 = static_cast<uint32_t>(__vm) - 10 * static_cast<uint32_t>(__vmDiv10);
+      const uint64_t __vrDiv10 = __div10(__vr);
+      const uint32_t __vrMod10 = static_cast<uint32_t>(__vr) - 10 * static_cast<uint32_t>(__vrDiv10);
+      __vmIsTrailingZeros &= __vmMod10 == 0;
+      __vrIsTrailingZeros &= __lastRemovedDigit == 0;
+      __lastRemovedDigit = static_cast<uint8_t>(__vrMod10);
+      __vr = __vrDiv10;
+      __vp = __vpDiv10;
+      __vm = __vmDiv10;
+      ++__removed;
+    }
+    if (__vmIsTrailingZeros) {
+      for (;;) {
+        const uint64_t __vmDiv10 = __div10(__vm);
+        const uint32_t __vmMod10 = static_cast<uint32_t>(__vm) - 10 * static_cast<uint32_t>(__vmDiv10);
+        if (__vmMod10 != 0) {
+          break;
+        }
+        const uint64_t __vpDiv10 = __div10(__vp);
+        const uint64_t __vrDiv10 = __div10(__vr);
+        const uint32_t __vrMod10 = static_cast<uint32_t>(__vr) - 10 * static_cast<uint32_t>(__vrDiv10);
+        __vrIsTrailingZeros &= __lastRemovedDigit == 0;
+        __lastRemovedDigit = static_cast<uint8_t>(__vrMod10);
+        __vr = __vrDiv10;
+        __vp = __vpDiv10;
+        __vm = __vmDiv10;
+        ++__removed;
+      }
+    }
+    if (__vrIsTrailingZeros && __lastRemovedDigit == 5 && __vr % 2 == 0) {
+      // Round even if the exact number is .....50..0.
+      __lastRemovedDigit = 4;
+    }
+    // We need to take __vr + 1 if __vr is outside bounds or we need to round up.
+    _Output = __vr + ((__vr == __vm && (!__acceptBounds || !__vmIsTrailingZeros)) || __lastRemovedDigit >= 5);
+  } else {
+    // Specialized for the common case (~99.3%). Percentages below are relative to this.
+    bool __roundUp = false;
+    const uint64_t __vpDiv100 = __div100(__vp);
+    const uint64_t __vmDiv100 = __div100(__vm);
+    if (__vpDiv100 > __vmDiv100) { // Optimization: remove two digits at a time (~86.2%).
+      const uint64_t __vrDiv100 = __div100(__vr);
+      const uint32_t __vrMod100 = static_cast<uint32_t>(__vr) - 100 * static_cast<uint32_t>(__vrDiv100);
+      __roundUp = __vrMod100 >= 50;
+      __vr = __vrDiv100;
+      __vp = __vpDiv100;
+      __vm = __vmDiv100;
+      __removed += 2;
+    }
+    // Loop iterations below (approximately), without optimization above:
+    // 0: 0.03%, 1: 13.8%, 2: 70.6%, 3: 14.0%, 4: 1.40%, 5: 0.14%, 6+: 0.02%
+    // Loop iterations below (approximately), with optimization above:
+    // 0: 70.6%, 1: 27.8%, 2: 1.40%, 3: 0.14%, 4+: 0.02%
+    for (;;) {
+      const uint64_t __vpDiv10 = __div10(__vp);
+      const uint64_t __vmDiv10 = __div10(__vm);
+      if (__vpDiv10 <= __vmDiv10) {
+        break;
+      }
+      const uint64_t __vrDiv10 = __div10(__vr);
+      const uint32_t __vrMod10 = static_cast<uint32_t>(__vr) - 10 * static_cast<uint32_t>(__vrDiv10);
+      __roundUp = __vrMod10 >= 5;
+      __vr = __vrDiv10;
+      __vp = __vpDiv10;
+      __vm = __vmDiv10;
+      ++__removed;
+    }
+    // We need to take __vr + 1 if __vr is outside bounds or we need to round up.
+    _Output = __vr + (__vr == __vm || __roundUp);
+  }
+  const int32_t __exp = __e10 + __removed;
+
+  __floating_decimal_64 __fd;
+  __fd.__exponent = __exp;
+  __fd.__mantissa = _Output;
+  return __fd;
+}
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline to_chars_result __to_chars(char* const _First, char* const _Last, const __floating_decimal_64 __v,
+  chars_format _Fmt, const double __f) {
+  // Step 5: Print the decimal representation.
+  uint64_t _Output = __v.__mantissa;
+  int32_t _Ryu_exponent = __v.__exponent;
+  const uint32_t __olength = __decimalLength17(_Output);
+  int32_t _Scientific_exponent = _Ryu_exponent + static_cast<int32_t>(__olength) - 1;
+
+  if (_Fmt == chars_format{}) {
+    int32_t _Lower;
+    int32_t _Upper;
+
+    if (__olength == 1) {
+      // Value | Fixed   | Scientific
+      // 1e-3  | "0.001" | "1e-03"
+      // 1e4   | "10000" | "1e+04"
+      _Lower = -3;
+      _Upper = 4;
+    } else {
+      // Value   | Fixed       | Scientific
+      // 1234e-7 | "0.0001234" | "1.234e-04"
+      // 1234e5  | "123400000" | "1.234e+08"
+      _Lower = -static_cast<int32_t>(__olength + 3);
+      _Upper = 5;
+    }
+
+    if (_Lower <= _Ryu_exponent && _Ryu_exponent <= _Upper) {
+      _Fmt = chars_format::fixed;
+    } else {
+      _Fmt = chars_format::scientific;
+    }
+  } else if (_Fmt == chars_format::general) {
+    // C11 7.21.6.1 "The fprintf function"/8:
+    // "Let P equal [...] 6 if the precision is omitted [...].
+    // Then, if a conversion with style E would have an exponent of X:
+    // - if P > X >= -4, the conversion is with style f [...].
+    // - otherwise, the conversion is with style e [...]."
+    if (-4 <= _Scientific_exponent && _Scientific_exponent < 6) {
+      _Fmt = chars_format::fixed;
+    } else {
+      _Fmt = chars_format::scientific;
+    }
+  }
+
+  if (_Fmt == chars_format::fixed) {
+    // Example: _Output == 1729, __olength == 4
+
+    // _Ryu_exponent | Printed  | _Whole_digits | _Total_fixed_length  | Notes
+    // --------------|----------|---------------|----------------------|---------------------------------------
+    //             2 | 172900   |  6            | _Whole_digits        | Ryu can't be used for printing
+    //             1 | 17290    |  5            | (sometimes adjusted) | when the trimmed digits are nonzero.
+    // --------------|----------|---------------|----------------------|---------------------------------------
+    //             0 | 1729     |  4            | _Whole_digits        | Unified length cases.
+    // --------------|----------|---------------|----------------------|---------------------------------------
+    //            -1 | 172.9    |  3            | __olength + 1        | This case can't happen for
+    //            -2 | 17.29    |  2            |                      | __olength == 1, but no additional
+    //            -3 | 1.729    |  1            |                      | code is needed to avoid it.
+    // --------------|----------|---------------|----------------------|---------------------------------------
+    //            -4 | 0.1729   |  0            | 2 - _Ryu_exponent    | C11 7.21.6.1 "The fprintf function"/8:
+    //            -5 | 0.01729  | -1            |                      | "If a decimal-point character appears,
+    //            -6 | 0.001729 | -2            |                      | at least one digit appears before it."
+
+    const int32_t _Whole_digits = static_cast<int32_t>(__olength) + _Ryu_exponent;
+
+    uint32_t _Total_fixed_length;
+    if (_Ryu_exponent >= 0) { // cases "172900" and "1729"
+      _Total_fixed_length = static_cast<uint32_t>(_Whole_digits);
+      if (_Output == 1) {
+        // Rounding can affect the number of digits.
+        // For example, 1e23 is exactly "99999999999999991611392" which is 23 digits instead of 24.
+        // We can use a lookup table to detect this and adjust the total length.
+        static constexpr uint8_t _Adjustment[309] = {
+          0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,1,1,0,1,0,1,1,1,0,1,1,1,0,0,0,0,0,
+          1,1,0,0,1,0,1,1,1,0,0,0,0,1,1,1,1,0,0,0,1,1,1,1,0,0,0,1,1,1,1,0,1,0,1,0,1,1,0,0,0,0,1,1,1,
+          1,0,0,0,0,0,0,0,1,1,0,1,1,0,0,1,0,1,0,1,0,1,1,0,0,0,0,0,1,1,1,0,0,1,1,1,1,1,0,1,0,1,1,0,1,
+          1,0,0,0,0,0,0,0,0,0,1,1,1,0,0,1,0,0,1,0,0,1,1,1,1,0,0,1,1,0,1,1,0,1,1,0,1,0,0,0,1,0,0,0,1,
+          0,1,0,1,0,1,1,1,0,0,0,0,0,0,1,1,1,1,0,0,1,0,1,1,1,0,0,0,1,0,1,1,1,1,1,1,0,1,0,1,1,0,0,0,1,
+          1,1,0,1,1,0,0,0,1,0,0,0,1,0,1,0,0,0,0,0,0,0,1,0,1,1,0,0,1,1,1,0,0,0,1,0,1,0,0,0,0,0,1,1,0,
+          0,1,0,1,1,1,0,0,1,0,0,0,0,1,0,1,0,0,0,0,0,1,0,1,0,1,1,0,1,0,0,0,0,0,1,1,0,1,0 };
+        _Total_fixed_length -= _Adjustment[_Ryu_exponent];
+        // _Whole_digits doesn't need to be adjusted because these cases won't refer to it later.
+      }
+    } else if (_Whole_digits > 0) { // case "17.29"
+      _Total_fixed_length = __olength + 1;
+    } else { // case "0.001729"
+      _Total_fixed_length = static_cast<uint32_t>(2 - _Ryu_exponent);
+    }
+
+    if (_Last - _First < static_cast<ptrdiff_t>(_Total_fixed_length)) {
+      return { _Last, errc::value_too_large };
+    }
+
+    char* _Mid;
+    if (_Ryu_exponent > 0) { // case "172900"
+      bool _Can_use_ryu;
+
+      if (_Ryu_exponent > 22) { // 10^22 is the largest power of 10 that's exactly representable as a double.
+        _Can_use_ryu = false;
+      } else {
+        // Ryu generated X: __v.__mantissa * 10^_Ryu_exponent
+        // __v.__mantissa == 2^_Trailing_zero_bits * (__v.__mantissa >> _Trailing_zero_bits)
+        // 10^_Ryu_exponent == 2^_Ryu_exponent * 5^_Ryu_exponent
+
+        // _Trailing_zero_bits is [0, 56] (aside: because 2^56 is the largest power of 2
+        // with 17 decimal digits, which is double's round-trip limit.)
+        // _Ryu_exponent is [1, 22].
+        // Normalization adds [2, 52] (aside: at least 2 because the pre-normalized mantissa is at least 5).
+        // This adds up to [3, 130], which is well below double's maximum binary exponent 1023.
+
+        // Therefore, we just need to consider (__v.__mantissa >> _Trailing_zero_bits) * 5^_Ryu_exponent.
+
+        // If that product would exceed 53 bits, then X can't be exactly represented as a double.
+        // (That's not a problem for round-tripping, because X is close enough to the original double,
+        // but X isn't mathematically equal to the original double.) This requires a high-precision fallback.
+
+        // If the product is 53 bits or smaller, then X can be exactly represented as a double (and we don't
+        // need to re-synthesize it; the original double must have been X, because Ryu wouldn't produce the
+        // same output for two different doubles X and Y). This allows Ryu's output to be used (zero-filled).
+
+        // (2^53 - 1) / 5^0 (for indexing), (2^53 - 1) / 5^1, ..., (2^53 - 1) / 5^22
+        static constexpr uint64_t _Max_shifted_mantissa[23] = {
+          9007199254740991u, 1801439850948198u, 360287970189639u, 72057594037927u, 14411518807585u,
+          2882303761517u, 576460752303u, 115292150460u, 23058430092u, 4611686018u, 922337203u, 184467440u,
+          36893488u, 7378697u, 1475739u, 295147u, 59029u, 11805u, 2361u, 472u, 94u, 18u, 3u };
+
+        unsigned long _Trailing_zero_bits;
+#ifdef _LIBCPP_HAS_BITSCAN64
+        (void) _BitScanForward64(&_Trailing_zero_bits, __v.__mantissa); // __v.__mantissa is guaranteed nonzero
+#else // ^^^ 64-bit ^^^ / vvv 32-bit vvv
+        const uint32_t _Low_mantissa = static_cast<uint32_t>(__v.__mantissa);
+        if (_Low_mantissa != 0) {
+          (void) _BitScanForward(&_Trailing_zero_bits, _Low_mantissa);
+        } else {
+          const uint32_t _High_mantissa = static_cast<uint32_t>(__v.__mantissa >> 32); // nonzero here
+          (void) _BitScanForward(&_Trailing_zero_bits, _High_mantissa);
+          _Trailing_zero_bits += 32;
+        }
+#endif // ^^^ 32-bit ^^^
+        const uint64_t _Shifted_mantissa = __v.__mantissa >> _Trailing_zero_bits;
+        _Can_use_ryu = _Shifted_mantissa <= _Max_shifted_mantissa[_Ryu_exponent];
+      }
+
+      if (!_Can_use_ryu) {
+        // Print the integer exactly.
+        // Performance note: This will redundantly perform bounds checking.
+        // Performance note: This will redundantly decompose the IEEE representation.
+        return __d2fixed_buffered_n(_First, _Last, __f, 0);
+      }
+
+      // _Can_use_ryu
+      // Print the decimal digits, left-aligned within [_First, _First + _Total_fixed_length).
+      _Mid = _First + __olength;
+    } else { // cases "1729", "17.29", and "0.001729"
+      // Print the decimal digits, right-aligned within [_First, _First + _Total_fixed_length).
+      _Mid = _First + _Total_fixed_length;
+    }
+
+    // We prefer 32-bit operations, even on 64-bit platforms.
+    // We have at most 17 digits, and uint32_t can store 9 digits.
+    // If _Output doesn't fit into uint32_t, we cut off 8 digits,
+    // so the rest will fit into uint32_t.
+    if ((_Output >> 32) != 0) {
+      // Expensive 64-bit division.
+      const uint64_t __q = __div1e8(_Output);
+      uint32_t __output2 = static_cast<uint32_t>(_Output - 100000000 * __q);
+      _Output = __q;
+
+      const uint32_t __c = __output2 % 10000;
+      __output2 /= 10000;
+      const uint32_t __d = __output2 % 10000;
+      const uint32_t __c0 = (__c % 100) << 1;
+      const uint32_t __c1 = (__c / 100) << 1;
+      const uint32_t __d0 = (__d % 100) << 1;
+      const uint32_t __d1 = (__d / 100) << 1;
+
+      _VSTD::memcpy(_Mid -= 2, __DIGIT_TABLE + __c0, 2);
+      _VSTD::memcpy(_Mid -= 2, __DIGIT_TABLE + __c1, 2);
+      _VSTD::memcpy(_Mid -= 2, __DIGIT_TABLE + __d0, 2);
+      _VSTD::memcpy(_Mid -= 2, __DIGIT_TABLE + __d1, 2);
+    }
+    uint32_t __output2 = static_cast<uint32_t>(_Output);
+    while (__output2 >= 10000) {
+#ifdef __clang__ // TRANSITION, LLVM-38217
+      const uint32_t __c = __output2 - 10000 * (__output2 / 10000);
+#else
+      const uint32_t __c = __output2 % 10000;
+#endif
+      __output2 /= 10000;
+      const uint32_t __c0 = (__c % 100) << 1;
+      const uint32_t __c1 = (__c / 100) << 1;
+      _VSTD::memcpy(_Mid -= 2, __DIGIT_TABLE + __c0, 2);
+      _VSTD::memcpy(_Mid -= 2, __DIGIT_TABLE + __c1, 2);
+    }
+    if (__output2 >= 100) {
+      const uint32_t __c = (__output2 % 100) << 1;
+      __output2 /= 100;
+      _VSTD::memcpy(_Mid -= 2, __DIGIT_TABLE + __c, 2);
+    }
+    if (__output2 >= 10) {
+      const uint32_t __c = __output2 << 1;
+      _VSTD::memcpy(_Mid -= 2, __DIGIT_TABLE + __c, 2);
+    } else {
+      *--_Mid = static_cast<char>('0' + __output2);
+    }
+
+    if (_Ryu_exponent > 0) { // case "172900" with _Can_use_ryu
+      // Performance note: it might be more efficient to do this immediately after setting _Mid.
+      _VSTD::memset(_First + __olength, '0', static_cast<size_t>(_Ryu_exponent));
+    } else if (_Ryu_exponent == 0) { // case "1729"
+      // Done!
+    } else if (_Whole_digits > 0) { // case "17.29"
+      // Performance note: moving digits might not be optimal.
+      _VSTD::memmove(_First, _First + 1, static_cast<size_t>(_Whole_digits));
+      _First[_Whole_digits] = '.';
+    } else { // case "0.001729"
+      // Performance note: a larger memset() followed by overwriting '.' might be more efficient.
+      _First[0] = '0';
+      _First[1] = '.';
+      _VSTD::memset(_First + 2, '0', static_cast<size_t>(-_Whole_digits));
+    }
+
+    return { _First + _Total_fixed_length, errc{} };
+  }
+
+  const uint32_t _Total_scientific_length = __olength + (__olength > 1) // digits + possible decimal point
+    + (-100 < _Scientific_exponent && _Scientific_exponent < 100 ? 4 : 5); // + scientific exponent
+  if (_Last - _First < static_cast<ptrdiff_t>(_Total_scientific_length)) {
+    return { _Last, errc::value_too_large };
+  }
+  char* const __result = _First;
+
+  // Print the decimal digits.
+  uint32_t __i = 0;
+  // We prefer 32-bit operations, even on 64-bit platforms.
+  // We have at most 17 digits, and uint32_t can store 9 digits.
+  // If _Output doesn't fit into uint32_t, we cut off 8 digits,
+  // so the rest will fit into uint32_t.
+  if ((_Output >> 32) != 0) {
+    // Expensive 64-bit division.
+    const uint64_t __q = __div1e8(_Output);
+    uint32_t __output2 = static_cast<uint32_t>(_Output) - 100000000 * static_cast<uint32_t>(__q);
+    _Output = __q;
+
+    const uint32_t __c = __output2 % 10000;
+    __output2 /= 10000;
+    const uint32_t __d = __output2 % 10000;
+    const uint32_t __c0 = (__c % 100) << 1;
+    const uint32_t __c1 = (__c / 100) << 1;
+    const uint32_t __d0 = (__d % 100) << 1;
+    const uint32_t __d1 = (__d / 100) << 1;
+    _VSTD::memcpy(__result + __olength - __i - 1, __DIGIT_TABLE + __c0, 2);
+    _VSTD::memcpy(__result + __olength - __i - 3, __DIGIT_TABLE + __c1, 2);
+    _VSTD::memcpy(__result + __olength - __i - 5, __DIGIT_TABLE + __d0, 2);
+    _VSTD::memcpy(__result + __olength - __i - 7, __DIGIT_TABLE + __d1, 2);
+    __i += 8;
+  }
+  uint32_t __output2 = static_cast<uint32_t>(_Output);
+  while (__output2 >= 10000) {
+#ifdef __clang__ // TRANSITION, LLVM-38217
+    const uint32_t __c = __output2 - 10000 * (__output2 / 10000);
+#else
+    const uint32_t __c = __output2 % 10000;
+#endif
+    __output2 /= 10000;
+    const uint32_t __c0 = (__c % 100) << 1;
+    const uint32_t __c1 = (__c / 100) << 1;
+    _VSTD::memcpy(__result + __olength - __i - 1, __DIGIT_TABLE + __c0, 2);
+    _VSTD::memcpy(__result + __olength - __i - 3, __DIGIT_TABLE + __c1, 2);
+    __i += 4;
+  }
+  if (__output2 >= 100) {
+    const uint32_t __c = (__output2 % 100) << 1;
+    __output2 /= 100;
+    _VSTD::memcpy(__result + __olength - __i - 1, __DIGIT_TABLE + __c, 2);
+    __i += 2;
+  }
+  if (__output2 >= 10) {
+    const uint32_t __c = __output2 << 1;
+    // We can't use memcpy here: the decimal dot goes between these two digits.
+    __result[2] = __DIGIT_TABLE[__c + 1];
+    __result[0] = __DIGIT_TABLE[__c];
+  } else {
+    __result[0] = static_cast<char>('0' + __output2);
+  }
+
+  // Print decimal point if needed.
+  uint32_t __index;
+  if (__olength > 1) {
+    __result[1] = '.';
+    __index = __olength + 1;
+  } else {
+    __index = 1;
+  }
+
+  // Print the exponent.
+  __result[__index++] = 'e';
+  if (_Scientific_exponent < 0) {
+    __result[__index++] = '-';
+    _Scientific_exponent = -_Scientific_exponent;
+  } else {
+    __result[__index++] = '+';
+  }
+
+  if (_Scientific_exponent >= 100) {
+    const int32_t __c = _Scientific_exponent % 10;
+    _VSTD::memcpy(__result + __index, __DIGIT_TABLE + 2 * (_Scientific_exponent / 10), 2);
+    __result[__index + 2] = static_cast<char>('0' + __c);
+    __index += 3;
+  } else {
+    _VSTD::memcpy(__result + __index, __DIGIT_TABLE + 2 * _Scientific_exponent, 2);
+    __index += 2;
+  }
+
+  return { _First + _Total_scientific_length, errc{} };
+}
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline bool __d2d_small_int(const uint64_t __ieeeMantissa, const uint32_t __ieeeExponent,
+  __floating_decimal_64* const __v) {
+  const uint64_t __m2 = (1ull << __DOUBLE_MANTISSA_BITS) | __ieeeMantissa;
+  const int32_t __e2 = static_cast<int32_t>(__ieeeExponent) - __DOUBLE_BIAS - __DOUBLE_MANTISSA_BITS;
+
+  if (__e2 > 0) {
+    // f = __m2 * 2^__e2 >= 2^53 is an integer.
+    // Ignore this case for now.
+    return false;
+  }
+
+  if (__e2 < -52) {
+    // f < 1.
+    return false;
+  }
+
+  // Since 2^52 <= __m2 < 2^53 and 0 <= -__e2 <= 52: 1 <= f = __m2 / 2^-__e2 < 2^53.
+  // Test if the lower -__e2 bits of the significand are 0, i.e. whether the fraction is 0.
+  const uint64_t __mask = (1ull << -__e2) - 1;
+  const uint64_t __fraction = __m2 & __mask;
+  if (__fraction != 0) {
+    return false;
+  }
+
+  // f is an integer in the range [1, 2^53).
+  // Note: __mantissa might contain trailing (decimal) 0's.
+  // Note: since 2^53 < 10^16, there is no need to adjust __decimalLength17().
+  __v->__mantissa = __m2 >> -__e2;
+  __v->__exponent = 0;
+  return true;
+}
+
+[[nodiscard]] to_chars_result __d2s_buffered_n(char* const _First, char* const _Last, const double __f,
+  const chars_format _Fmt) {
+
+  // Step 1: Decode the floating-point number, and unify normalized and subnormal cases.
+  const uint64_t __bits = __double_to_bits(__f);
+
+  // Case distinction; exit early for the easy cases.
+  if (__bits == 0) {
+    if (_Fmt == chars_format::scientific) {
+      if (_Last - _First < 5) {
+        return { _Last, errc::value_too_large };
+      }
+
+      _VSTD::memcpy(_First, "0e+00", 5);
+
+      return { _First + 5, errc{} };
+    }
+
+    // Print "0" for chars_format::fixed, chars_format::general, and chars_format{}.
+    if (_First == _Last) {
+      return { _Last, errc::value_too_large };
+    }
+
+    *_First = '0';
+
+    return { _First + 1, errc{} };
+  }
+
+  // Decode __bits into mantissa and exponent.
+  const uint64_t __ieeeMantissa = __bits & ((1ull << __DOUBLE_MANTISSA_BITS) - 1);
+  const uint32_t __ieeeExponent = static_cast<uint32_t>(__bits >> __DOUBLE_MANTISSA_BITS);
+
+  if (_Fmt == chars_format::fixed) {
+    // const uint64_t _Mantissa2 = __ieeeMantissa | (1ull << __DOUBLE_MANTISSA_BITS); // restore implicit bit
+    const int32_t _Exponent2 = static_cast<int32_t>(__ieeeExponent)
+      - __DOUBLE_BIAS - __DOUBLE_MANTISSA_BITS; // bias and normalization
+
+    // Normal values are equal to _Mantissa2 * 2^_Exponent2.
+    // (Subnormals are different, but they'll be rejected by the _Exponent2 test here, so they can be ignored.)
+
+    // For nonzero integers, _Exponent2 >= -52. (The minimum value occurs when _Mantissa2 * 2^_Exponent2 is 1.
+    // In that case, _Mantissa2 is the implicit 1 bit followed by 52 zeros, so _Exponent2 is -52 to shift away
+    // the zeros.) The dense range of exactly representable integers has negative or zero exponents
+    // (as positive exponents make the range non-dense). For that dense range, Ryu will always be used:
+    // every digit is necessary to uniquely identify the value, so Ryu must print them all.
+
+    // Positive exponents are the non-dense range of exactly representable integers. This contains all of the values
+    // for which Ryu can't be used (and a few Ryu-friendly values). We can save time by detecting positive
+    // exponents here and skipping Ryu. Calling __d2fixed_buffered_n() with precision 0 is valid for all integers
+    // (so it's okay if we call it with a Ryu-friendly value).
+    if (_Exponent2 > 0) {
+      return __d2fixed_buffered_n(_First, _Last, __f, 0);
+    }
+  }
+
+  __floating_decimal_64 __v;
+  const bool __isSmallInt = __d2d_small_int(__ieeeMantissa, __ieeeExponent, &__v);
+  if (__isSmallInt) {
+    // For small integers in the range [1, 2^53), __v.__mantissa might contain trailing (decimal) zeros.
+    // For scientific notation we need to move these zeros into the exponent.
+    // (This is not needed for fixed-point notation, so it might be beneficial to trim
+    // trailing zeros in __to_chars only if needed - once fixed-point notation output is implemented.)
+    for (;;) {
+      const uint64_t __q = __div10(__v.__mantissa);
+      const uint32_t __r = static_cast<uint32_t>(__v.__mantissa) - 10 * static_cast<uint32_t>(__q);
+      if (__r != 0) {
+        break;
+      }
+      __v.__mantissa = __q;
+      ++__v.__exponent;
+    }
+  } else {
+    __v = __d2d(__ieeeMantissa, __ieeeExponent);
+  }
+
+  return __to_chars(_First, _Last, __v, _Fmt, __f);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+// clang-format on
diff --git a/libcxx/src/ryu/f2s.cpp b/libcxx/src/ryu/f2s.cpp
new file mode 100644
index 000000000000..7e10b498367e
--- /dev/null
+++ b/libcxx/src/ryu/f2s.cpp
@@ -0,0 +1,715 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// Copyright 2018 Ulf Adams
+// Copyright (c) Microsoft Corporation. All rights reserved.
+
+// Boost Software License - Version 1.0 - August 17th, 2003
+
+// Permission is hereby granted, free of charge, to any person or organization
+// obtaining a copy of the software and accompanying documentation covered by
+// this license (the "Software") to use, reproduce, display, distribute,
+// execute, and transmit the Software, and to prepare derivative works of the
+// Software, and to permit third-parties to whom the Software is furnished to
+// do so, all subject to the following:
+
+// The copyright notices in the Software and this entire statement, including
+// the above license grant, this restriction and the following disclaimer,
+// must be included in all copies of the Software, in whole or in part, and
+// all derivative works of the Software, unless such copies or derivative
+// works are solely in the form of machine-executable object code generated by
+// a source language processor.
+
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+// SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+// FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+// DEALINGS IN THE SOFTWARE.
+
+// Avoid formatting to keep the changes with the original code minimal.
+// clang-format off
+
+#include "__config"
+#include "charconv"
+
+#include "include/ryu/common.h"
+#include "include/ryu/d2fixed.h"
+#include "include/ryu/d2s_intrinsics.h"
+#include "include/ryu/digit_table.h"
+#include "include/ryu/f2s.h"
+#include "include/ryu/ryu.h"
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+inline constexpr int __FLOAT_MANTISSA_BITS = 23;
+inline constexpr int __FLOAT_EXPONENT_BITS = 8;
+inline constexpr int __FLOAT_BIAS = 127;
+
+inline constexpr int __FLOAT_POW5_INV_BITCOUNT = 59;
+inline constexpr uint64_t __FLOAT_POW5_INV_SPLIT[31] = {
+  576460752303423489u, 461168601842738791u, 368934881474191033u, 295147905179352826u,
+  472236648286964522u, 377789318629571618u, 302231454903657294u, 483570327845851670u,
+  386856262276681336u, 309485009821345069u, 495176015714152110u, 396140812571321688u,
+  316912650057057351u, 507060240091291761u, 405648192073033409u, 324518553658426727u,
+  519229685853482763u, 415383748682786211u, 332306998946228969u, 531691198313966350u,
+  425352958651173080u, 340282366920938464u, 544451787073501542u, 435561429658801234u,
+  348449143727040987u, 557518629963265579u, 446014903970612463u, 356811923176489971u,
+  570899077082383953u, 456719261665907162u, 365375409332725730u
+};
+inline constexpr int __FLOAT_POW5_BITCOUNT = 61;
+inline constexpr uint64_t __FLOAT_POW5_SPLIT[47] = {
+  1152921504606846976u, 1441151880758558720u, 1801439850948198400u, 2251799813685248000u,
+  1407374883553280000u, 1759218604441600000u, 2199023255552000000u, 1374389534720000000u,
+  1717986918400000000u, 2147483648000000000u, 1342177280000000000u, 1677721600000000000u,
+  2097152000000000000u, 1310720000000000000u, 1638400000000000000u, 2048000000000000000u,
+  1280000000000000000u, 1600000000000000000u, 2000000000000000000u, 1250000000000000000u,
+  1562500000000000000u, 1953125000000000000u, 1220703125000000000u, 1525878906250000000u,
+  1907348632812500000u, 1192092895507812500u, 1490116119384765625u, 1862645149230957031u,
+  1164153218269348144u, 1455191522836685180u, 1818989403545856475u, 2273736754432320594u,
+  1421085471520200371u, 1776356839400250464u, 2220446049250313080u, 1387778780781445675u,
+  1734723475976807094u, 2168404344971008868u, 1355252715606880542u, 1694065894508600678u,
+  2117582368135750847u, 1323488980084844279u, 1654361225106055349u, 2067951531382569187u,
+  1292469707114105741u, 1615587133892632177u, 2019483917365790221u
+};
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline uint32_t __pow5Factor(uint32_t __value) {
+  uint32_t __count = 0;
+  for (;;) {
+    _LIBCPP_ASSERT(__value != 0, "");
+    const uint32_t __q = __value / 5;
+    const uint32_t __r = __value % 5;
+    if (__r != 0) {
+      break;
+    }
+    __value = __q;
+    ++__count;
+  }
+  return __count;
+}
+
+// Returns true if __value is divisible by 5^__p.
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline bool __multipleOfPowerOf5(const uint32_t __value, const uint32_t __p) {
+  return __pow5Factor(__value) >= __p;
+}
+
+// Returns true if __value is divisible by 2^__p.
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline bool __multipleOfPowerOf2(const uint32_t __value, const uint32_t __p) {
+  _LIBCPP_ASSERT(__value != 0, "");
+  _LIBCPP_ASSERT(__p < 32, "");
+  // __builtin_ctz doesn't appear to be faster here.
+  return (__value & ((1u << __p) - 1)) == 0;
+}
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline uint32_t __mulShift(const uint32_t __m, const uint64_t __factor, const int32_t __shift) {
+  _LIBCPP_ASSERT(__shift > 32, "");
+
+  // The casts here help MSVC to avoid calls to the __allmul library
+  // function.
+  const uint32_t __factorLo = static_cast<uint32_t>(__factor);
+  const uint32_t __factorHi = static_cast<uint32_t>(__factor >> 32);
+  const uint64_t __bits0 = static_cast<uint64_t>(__m) * __factorLo;
+  const uint64_t __bits1 = static_cast<uint64_t>(__m) * __factorHi;
+
+#ifndef _LIBCPP_64_BIT
+  // On 32-bit platforms we can avoid a 64-bit shift-right since we only
+  // need the upper 32 bits of the result and the shift value is > 32.
+  const uint32_t __bits0Hi = static_cast<uint32_t>(__bits0 >> 32);
+  uint32_t __bits1Lo = static_cast<uint32_t>(__bits1);
+  uint32_t __bits1Hi = static_cast<uint32_t>(__bits1 >> 32);
+  __bits1Lo += __bits0Hi;
+  __bits1Hi += (__bits1Lo < __bits0Hi);
+  const int32_t __s = __shift - 32;
+  return (__bits1Hi << (32 - __s)) | (__bits1Lo >> __s);
+#else // ^^^ 32-bit ^^^ / vvv 64-bit vvv
+  const uint64_t __sum = (__bits0 >> 32) + __bits1;
+  const uint64_t __shiftedSum = __sum >> (__shift - 32);
+  _LIBCPP_ASSERT(__shiftedSum <= UINT32_MAX, "");
+  return static_cast<uint32_t>(__shiftedSum);
+#endif // ^^^ 64-bit ^^^
+}
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline uint32_t __mulPow5InvDivPow2(const uint32_t __m, const uint32_t __q, const int32_t __j) {
+  return __mulShift(__m, __FLOAT_POW5_INV_SPLIT[__q], __j);
+}
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline uint32_t __mulPow5divPow2(const uint32_t __m, const uint32_t __i, const int32_t __j) {
+  return __mulShift(__m, __FLOAT_POW5_SPLIT[__i], __j);
+}
+
+// A floating decimal representing m * 10^e.
+struct __floating_decimal_32 {
+  uint32_t __mantissa;
+  int32_t __exponent;
+};
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline __floating_decimal_32 __f2d(const uint32_t __ieeeMantissa, const uint32_t __ieeeExponent) {
+  int32_t __e2;
+  uint32_t __m2;
+  if (__ieeeExponent == 0) {
+    // We subtract 2 so that the bounds computation has 2 additional bits.
+    __e2 = 1 - __FLOAT_BIAS - __FLOAT_MANTISSA_BITS - 2;
+    __m2 = __ieeeMantissa;
+  } else {
+    __e2 = static_cast<int32_t>(__ieeeExponent) - __FLOAT_BIAS - __FLOAT_MANTISSA_BITS - 2;
+    __m2 = (1u << __FLOAT_MANTISSA_BITS) | __ieeeMantissa;
+  }
+  const bool __even = (__m2 & 1) == 0;
+  const bool __acceptBounds = __even;
+
+  // Step 2: Determine the interval of valid decimal representations.
+  const uint32_t __mv = 4 * __m2;
+  const uint32_t __mp = 4 * __m2 + 2;
+  // Implicit bool -> int conversion. True is 1, false is 0.
+  const uint32_t __mmShift = __ieeeMantissa != 0 || __ieeeExponent <= 1;
+  const uint32_t __mm = 4 * __m2 - 1 - __mmShift;
+
+  // Step 3: Convert to a decimal power base using 64-bit arithmetic.
+  uint32_t __vr, __vp, __vm;
+  int32_t __e10;
+  bool __vmIsTrailingZeros = false;
+  bool __vrIsTrailingZeros = false;
+  uint8_t __lastRemovedDigit = 0;
+  if (__e2 >= 0) {
+    const uint32_t __q = __log10Pow2(__e2);
+    __e10 = static_cast<int32_t>(__q);
+    const int32_t __k = __FLOAT_POW5_INV_BITCOUNT + __pow5bits(static_cast<int32_t>(__q)) - 1;
+    const int32_t __i = -__e2 + static_cast<int32_t>(__q) + __k;
+    __vr = __mulPow5InvDivPow2(__mv, __q, __i);
+    __vp = __mulPow5InvDivPow2(__mp, __q, __i);
+    __vm = __mulPow5InvDivPow2(__mm, __q, __i);
+    if (__q != 0 && (__vp - 1) / 10 <= __vm / 10) {
+      // We need to know one removed digit even if we are not going to loop below. We could use
+      // __q = X - 1 above, except that would require 33 bits for the result, and we've found that
+      // 32-bit arithmetic is faster even on 64-bit machines.
+      const int32_t __l = __FLOAT_POW5_INV_BITCOUNT + __pow5bits(static_cast<int32_t>(__q - 1)) - 1;
+      __lastRemovedDigit = static_cast<uint8_t>(__mulPow5InvDivPow2(__mv, __q - 1,
+        -__e2 + static_cast<int32_t>(__q) - 1 + __l) % 10);
+    }
+    if (__q <= 9) {
+      // The largest power of 5 that fits in 24 bits is 5^10, but __q <= 9 seems to be safe as well.
+      // Only one of __mp, __mv, and __mm can be a multiple of 5, if any.
+      if (__mv % 5 == 0) {
+        __vrIsTrailingZeros = __multipleOfPowerOf5(__mv, __q);
+      } else if (__acceptBounds) {
+        __vmIsTrailingZeros = __multipleOfPowerOf5(__mm, __q);
+      } else {
+        __vp -= __multipleOfPowerOf5(__mp, __q);
+      }
+    }
+  } else {
+    const uint32_t __q = __log10Pow5(-__e2);
+    __e10 = static_cast<int32_t>(__q) + __e2;
+    const int32_t __i = -__e2 - static_cast<int32_t>(__q);
+    const int32_t __k = __pow5bits(__i) - __FLOAT_POW5_BITCOUNT;
+    int32_t __j = static_cast<int32_t>(__q) - __k;
+    __vr = __mulPow5divPow2(__mv, static_cast<uint32_t>(__i), __j);
+    __vp = __mulPow5divPow2(__mp, static_cast<uint32_t>(__i), __j);
+    __vm = __mulPow5divPow2(__mm, static_cast<uint32_t>(__i), __j);
+    if (__q != 0 && (__vp - 1) / 10 <= __vm / 10) {
+      __j = static_cast<int32_t>(__q) - 1 - (__pow5bits(__i + 1) - __FLOAT_POW5_BITCOUNT);
+      __lastRemovedDigit = static_cast<uint8_t>(__mulPow5divPow2(__mv, static_cast<uint32_t>(__i + 1), __j) % 10);
+    }
+    if (__q <= 1) {
+      // {__vr,__vp,__vm} is trailing zeros if {__mv,__mp,__mm} has at least __q trailing 0 bits.
+      // __mv = 4 * __m2, so it always has at least two trailing 0 bits.
+      __vrIsTrailingZeros = true;
+      if (__acceptBounds) {
+        // __mm = __mv - 1 - __mmShift, so it has 1 trailing 0 bit iff __mmShift == 1.
+        __vmIsTrailingZeros = __mmShift == 1;
+      } else {
+        // __mp = __mv + 2, so it always has at least one trailing 0 bit.
+        --__vp;
+      }
+    } else if (__q < 31) { // TRANSITION(ulfjack): Use a tighter bound here.
+      __vrIsTrailingZeros = __multipleOfPowerOf2(__mv, __q - 1);
+    }
+  }
+
+  // Step 4: Find the shortest decimal representation in the interval of valid representations.
+  int32_t __removed = 0;
+  uint32_t _Output;
+  if (__vmIsTrailingZeros || __vrIsTrailingZeros) {
+    // General case, which happens rarely (~4.0%).
+    while (__vp / 10 > __vm / 10) {
+#ifdef __clang__ // TRANSITION, LLVM-23106
+      __vmIsTrailingZeros &= __vm - (__vm / 10) * 10 == 0;
+#else
+      __vmIsTrailingZeros &= __vm % 10 == 0;
+#endif
+      __vrIsTrailingZeros &= __lastRemovedDigit == 0;
+      __lastRemovedDigit = static_cast<uint8_t>(__vr % 10);
+      __vr /= 10;
+      __vp /= 10;
+      __vm /= 10;
+      ++__removed;
+    }
+    if (__vmIsTrailingZeros) {
+      while (__vm % 10 == 0) {
+        __vrIsTrailingZeros &= __lastRemovedDigit == 0;
+        __lastRemovedDigit = static_cast<uint8_t>(__vr % 10);
+        __vr /= 10;
+        __vp /= 10;
+        __vm /= 10;
+        ++__removed;
+      }
+    }
+    if (__vrIsTrailingZeros && __lastRemovedDigit == 5 && __vr % 2 == 0) {
+      // Round even if the exact number is .....50..0.
+      __lastRemovedDigit = 4;
+    }
+    // We need to take __vr + 1 if __vr is outside bounds or we need to round up.
+    _Output = __vr + ((__vr == __vm && (!__acceptBounds || !__vmIsTrailingZeros)) || __lastRemovedDigit >= 5);
+  } else {
+    // Specialized for the common case (~96.0%). Percentages below are relative to this.
+    // Loop iterations below (approximately):
+    // 0: 13.6%, 1: 70.7%, 2: 14.1%, 3: 1.39%, 4: 0.14%, 5+: 0.01%
+    while (__vp / 10 > __vm / 10) {
+      __lastRemovedDigit = static_cast<uint8_t>(__vr % 10);
+      __vr /= 10;
+      __vp /= 10;
+      __vm /= 10;
+      ++__removed;
+    }
+    // We need to take __vr + 1 if __vr is outside bounds or we need to round up.
+    _Output = __vr + (__vr == __vm || __lastRemovedDigit >= 5);
+  }
+  const int32_t __exp = __e10 + __removed;
+
+  __floating_decimal_32 __fd;
+  __fd.__exponent = __exp;
+  __fd.__mantissa = _Output;
+  return __fd;
+}
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline to_chars_result _Large_integer_to_chars(char* const _First, char* const _Last,
+  const uint32_t _Mantissa2, const int32_t _Exponent2) {
+
+  // Print the integer _Mantissa2 * 2^_Exponent2 exactly.
+
+  // For nonzero integers, _Exponent2 >= -23. (The minimum value occurs when _Mantissa2 * 2^_Exponent2 is 1.
+  // In that case, _Mantissa2 is the implicit 1 bit followed by 23 zeros, so _Exponent2 is -23 to shift away
+  // the zeros.) The dense range of exactly representable integers has negative or zero exponents
+  // (as positive exponents make the range non-dense). For that dense range, Ryu will always be used:
+  // every digit is necessary to uniquely identify the value, so Ryu must print them all.
+
+  // Positive exponents are the non-dense range of exactly representable integers.
+  // This contains all of the values for which Ryu can't be used (and a few Ryu-friendly values).
+
+  // Performance note: Long division appears to be faster than losslessly widening float to double and calling
+  // __d2fixed_buffered_n(). If __f2fixed_buffered_n() is implemented, it might be faster than long division.
+
+  _LIBCPP_ASSERT(_Exponent2 > 0, "");
+  _LIBCPP_ASSERT(_Exponent2 <= 104, ""); // because __ieeeExponent <= 254
+
+  // Manually represent _Mantissa2 * 2^_Exponent2 as a large integer. _Mantissa2 is always 24 bits
+  // (due to the implicit bit), while _Exponent2 indicates a shift of at most 104 bits.
+  // 24 + 104 equals 128 equals 4 * 32, so we need exactly 4 32-bit elements.
+  // We use a little-endian representation, visualized like this:
+
+  // << left shift <<
+  // most significant
+  // _Data[3] _Data[2] _Data[1] _Data[0]
+  //                   least significant
+  //                   >> right shift >>
+
+  constexpr uint32_t _Data_size = 4;
+  uint32_t _Data[_Data_size]{};
+
+  // _Maxidx is the index of the most significant nonzero element.
+  uint32_t _Maxidx = ((24 + static_cast<uint32_t>(_Exponent2) + 31) / 32) - 1;
+  _LIBCPP_ASSERT(_Maxidx < _Data_size, "");
+
+  const uint32_t _Bit_shift = static_cast<uint32_t>(_Exponent2) % 32;
+  if (_Bit_shift <= 8) { // _Mantissa2's 24 bits don't cross an element boundary
+    _Data[_Maxidx] = _Mantissa2 << _Bit_shift;
+  } else { // _Mantissa2's 24 bits cross an element boundary
+    _Data[_Maxidx - 1] = _Mantissa2 << _Bit_shift;
+    _Data[_Maxidx] = _Mantissa2 >> (32 - _Bit_shift);
+  }
+
+  // If Ryu hasn't determined the total output length, we need to buffer the digits generated from right to left
+  // by long division. The largest possible float is: 340'282346638'528859811'704183484'516925440
+  uint32_t _Blocks[4];
+  int32_t _Filled_blocks = 0;
+  // From left to right, we're going to print:
+  // _Data[0] will be [1, 10] digits.
+  // Then if _Filled_blocks > 0:
+  // _Blocks[_Filled_blocks - 1], ..., _Blocks[0] will be 0-filled 9-digit blocks.
+
+  if (_Maxidx != 0) { // If the integer is actually large, perform long division.
+                      // Otherwise, skip to printing _Data[0].
+    for (;;) {
+      // Loop invariant: _Maxidx != 0 (i.e. the integer is actually large)
+
+      const uint32_t _Most_significant_elem = _Data[_Maxidx];
+      const uint32_t _Initial_remainder = _Most_significant_elem % 1000000000;
+      const uint32_t _Initial_quotient = _Most_significant_elem / 1000000000;
+      _Data[_Maxidx] = _Initial_quotient;
+      uint64_t _Remainder = _Initial_remainder;
+
+      // Process less significant elements.
+      uint32_t _Idx = _Maxidx;
+      do {
+        --_Idx; // Initially, _Remainder is at most 10^9 - 1.
+
+        // Now, _Remainder is at most (10^9 - 1) * 2^32 + 2^32 - 1, simplified to 10^9 * 2^32 - 1.
+        _Remainder = (_Remainder << 32) | _Data[_Idx];
+
+        // floor((10^9 * 2^32 - 1) / 10^9) == 2^32 - 1, so uint32_t _Quotient is lossless.
+        const uint32_t _Quotient = static_cast<uint32_t>(__div1e9(_Remainder));
+
+        // _Remainder is at most 10^9 - 1 again.
+        // For uint32_t truncation, see the __mod1e9() comment in d2s_intrinsics.h.
+        _Remainder = static_cast<uint32_t>(_Remainder) - 1000000000u * _Quotient;
+
+        _Data[_Idx] = _Quotient;
+      } while (_Idx != 0);
+
+      // Store a 0-filled 9-digit block.
+      _Blocks[_Filled_blocks++] = static_cast<uint32_t>(_Remainder);
+
+      if (_Initial_quotient == 0) { // Is the large integer shrinking?
+        --_Maxidx; // log2(10^9) is 29.9, so we can't shrink by more than one element.
+        if (_Maxidx == 0) {
+          break; // We've finished long division. Now we need to print _Data[0].
+        }
+      }
+    }
+  }
+
+  _LIBCPP_ASSERT(_Data[0] != 0, "");
+  for (uint32_t _Idx = 1; _Idx < _Data_size; ++_Idx) {
+    _LIBCPP_ASSERT(_Data[_Idx] == 0, "");
+  }
+
+  const uint32_t _Data_olength = _Data[0] >= 1000000000 ? 10 : __decimalLength9(_Data[0]);
+  const uint32_t _Total_fixed_length = _Data_olength + 9 * _Filled_blocks;
+
+  if (_Last - _First < static_cast<ptrdiff_t>(_Total_fixed_length)) {
+    return { _Last, errc::value_too_large };
+  }
+
+  char* _Result = _First;
+
+  // Print _Data[0]. While it's up to 10 digits,
+  // which is more than Ryu generates, the code below can handle this.
+  __append_n_digits(_Data_olength, _Data[0], _Result);
+  _Result += _Data_olength;
+
+  // Print 0-filled 9-digit blocks.
+  for (int32_t _Idx = _Filled_blocks - 1; _Idx >= 0; --_Idx) {
+    __append_nine_digits(_Blocks[_Idx], _Result);
+    _Result += 9;
+  }
+
+  return { _Result, errc{} };
+}
+
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline to_chars_result __to_chars(char* const _First, char* const _Last, const __floating_decimal_32 __v,
+  chars_format _Fmt, const uint32_t __ieeeMantissa, const uint32_t __ieeeExponent) {
+  // Step 5: Print the decimal representation.
+  uint32_t _Output = __v.__mantissa;
+  int32_t _Ryu_exponent = __v.__exponent;
+  const uint32_t __olength = __decimalLength9(_Output);
+  int32_t _Scientific_exponent = _Ryu_exponent + static_cast<int32_t>(__olength) - 1;
+
+  if (_Fmt == chars_format{}) {
+    int32_t _Lower;
+    int32_t _Upper;
+
+    if (__olength == 1) {
+      // Value | Fixed   | Scientific
+      // 1e-3  | "0.001" | "1e-03"
+      // 1e4   | "10000" | "1e+04"
+      _Lower = -3;
+      _Upper = 4;
+    } else {
+      // Value   | Fixed       | Scientific
+      // 1234e-7 | "0.0001234" | "1.234e-04"
+      // 1234e5  | "123400000" | "1.234e+08"
+      _Lower = -static_cast<int32_t>(__olength + 3);
+      _Upper = 5;
+    }
+
+    if (_Lower <= _Ryu_exponent && _Ryu_exponent <= _Upper) {
+      _Fmt = chars_format::fixed;
+    } else {
+      _Fmt = chars_format::scientific;
+    }
+  } else if (_Fmt == chars_format::general) {
+    // C11 7.21.6.1 "The fprintf function"/8:
+    // "Let P equal [...] 6 if the precision is omitted [...].
+    // Then, if a conversion with style E would have an exponent of X:
+    // - if P > X >= -4, the conversion is with style f [...].
+    // - otherwise, the conversion is with style e [...]."
+    if (-4 <= _Scientific_exponent && _Scientific_exponent < 6) {
+      _Fmt = chars_format::fixed;
+    } else {
+      _Fmt = chars_format::scientific;
+    }
+  }
+
+  if (_Fmt == chars_format::fixed) {
+    // Example: _Output == 1729, __olength == 4
+
+    // _Ryu_exponent | Printed  | _Whole_digits | _Total_fixed_length  | Notes
+    // --------------|----------|---------------|----------------------|---------------------------------------
+    //             2 | 172900   |  6            | _Whole_digits        | Ryu can't be used for printing
+    //             1 | 17290    |  5            | (sometimes adjusted) | when the trimmed digits are nonzero.
+    // --------------|----------|---------------|----------------------|---------------------------------------
+    //             0 | 1729     |  4            | _Whole_digits        | Unified length cases.
+    // --------------|----------|---------------|----------------------|---------------------------------------
+    //            -1 | 172.9    |  3            | __olength + 1        | This case can't happen for
+    //            -2 | 17.29    |  2            |                      | __olength == 1, but no additional
+    //            -3 | 1.729    |  1            |                      | code is needed to avoid it.
+    // --------------|----------|---------------|----------------------|---------------------------------------
+    //            -4 | 0.1729   |  0            | 2 - _Ryu_exponent    | C11 7.21.6.1 "The fprintf function"/8:
+    //            -5 | 0.01729  | -1            |                      | "If a decimal-point character appears,
+    //            -6 | 0.001729 | -2            |                      | at least one digit appears before it."
+
+    const int32_t _Whole_digits = static_cast<int32_t>(__olength) + _Ryu_exponent;
+
+    uint32_t _Total_fixed_length;
+    if (_Ryu_exponent >= 0) { // cases "172900" and "1729"
+      _Total_fixed_length = static_cast<uint32_t>(_Whole_digits);
+      if (_Output == 1) {
+        // Rounding can affect the number of digits.
+        // For example, 1e11f is exactly "99999997952" which is 11 digits instead of 12.
+        // We can use a lookup table to detect this and adjust the total length.
+        static constexpr uint8_t _Adjustment[39] = {
+          0,0,0,0,0,0,0,0,0,0,0,1,1,1,0,1,0,1,1,1,0,0,1,1,0,1,0,1,1,0,0,1,0,1,1,0,1,1,1 };
+        _Total_fixed_length -= _Adjustment[_Ryu_exponent];
+        // _Whole_digits doesn't need to be adjusted because these cases won't refer to it later.
+      }
+    } else if (_Whole_digits > 0) { // case "17.29"
+      _Total_fixed_length = __olength + 1;
+    } else { // case "0.001729"
+      _Total_fixed_length = static_cast<uint32_t>(2 - _Ryu_exponent);
+    }
+
+    if (_Last - _First < static_cast<ptrdiff_t>(_Total_fixed_length)) {
+      return { _Last, errc::value_too_large };
+    }
+
+    char* _Mid;
+    if (_Ryu_exponent > 0) { // case "172900"
+      bool _Can_use_ryu;
+
+      if (_Ryu_exponent > 10) { // 10^10 is the largest power of 10 that's exactly representable as a float.
+        _Can_use_ryu = false;
+      } else {
+        // Ryu generated X: __v.__mantissa * 10^_Ryu_exponent
+        // __v.__mantissa == 2^_Trailing_zero_bits * (__v.__mantissa >> _Trailing_zero_bits)
+        // 10^_Ryu_exponent == 2^_Ryu_exponent * 5^_Ryu_exponent
+
+        // _Trailing_zero_bits is [0, 29] (aside: because 2^29 is the largest power of 2
+        // with 9 decimal digits, which is float's round-trip limit.)
+        // _Ryu_exponent is [1, 10].
+        // Normalization adds [2, 23] (aside: at least 2 because the pre-normalized mantissa is at least 5).
+        // This adds up to [3, 62], which is well below float's maximum binary exponent 127.
+
+        // Therefore, we just need to consider (__v.__mantissa >> _Trailing_zero_bits) * 5^_Ryu_exponent.
+
+        // If that product would exceed 24 bits, then X can't be exactly represented as a float.
+        // (That's not a problem for round-tripping, because X is close enough to the original float,
+        // but X isn't mathematically equal to the original float.) This requires a high-precision fallback.
+
+        // If the product is 24 bits or smaller, then X can be exactly represented as a float (and we don't
+        // need to re-synthesize it; the original float must have been X, because Ryu wouldn't produce the
+        // same output for two different floats X and Y). This allows Ryu's output to be used (zero-filled).
+
+        // (2^24 - 1) / 5^0 (for indexing), (2^24 - 1) / 5^1, ..., (2^24 - 1) / 5^10
+        static constexpr uint32_t _Max_shifted_mantissa[11] = {
+          16777215, 3355443, 671088, 134217, 26843, 5368, 1073, 214, 42, 8, 1 };
+
+        unsigned long _Trailing_zero_bits;
+        (void) _BitScanForward(&_Trailing_zero_bits, __v.__mantissa); // __v.__mantissa is guaranteed nonzero
+        const uint32_t _Shifted_mantissa = __v.__mantissa >> _Trailing_zero_bits;
+        _Can_use_ryu = _Shifted_mantissa <= _Max_shifted_mantissa[_Ryu_exponent];
+      }
+
+      if (!_Can_use_ryu) {
+        const uint32_t _Mantissa2 = __ieeeMantissa | (1u << __FLOAT_MANTISSA_BITS); // restore implicit bit
+        const int32_t _Exponent2 = static_cast<int32_t>(__ieeeExponent)
+          - __FLOAT_BIAS - __FLOAT_MANTISSA_BITS; // bias and normalization
+
+        // Performance note: We've already called Ryu, so this will redundantly perform buffering and bounds checking.
+        return _Large_integer_to_chars(_First, _Last, _Mantissa2, _Exponent2);
+      }
+
+      // _Can_use_ryu
+      // Print the decimal digits, left-aligned within [_First, _First + _Total_fixed_length).
+      _Mid = _First + __olength;
+    } else { // cases "1729", "17.29", and "0.001729"
+      // Print the decimal digits, right-aligned within [_First, _First + _Total_fixed_length).
+      _Mid = _First + _Total_fixed_length;
+    }
+
+    while (_Output >= 10000) {
+#ifdef __clang__ // TRANSITION, LLVM-38217
+      const uint32_t __c = _Output - 10000 * (_Output / 10000);
+#else
+      const uint32_t __c = _Output % 10000;
+#endif
+      _Output /= 10000;
+      const uint32_t __c0 = (__c % 100) << 1;
+      const uint32_t __c1 = (__c / 100) << 1;
+      _VSTD::memcpy(_Mid -= 2, __DIGIT_TABLE + __c0, 2);
+      _VSTD::memcpy(_Mid -= 2, __DIGIT_TABLE + __c1, 2);
+    }
+    if (_Output >= 100) {
+      const uint32_t __c = (_Output % 100) << 1;
+      _Output /= 100;
+      _VSTD::memcpy(_Mid -= 2, __DIGIT_TABLE + __c, 2);
+    }
+    if (_Output >= 10) {
+      const uint32_t __c = _Output << 1;
+      _VSTD::memcpy(_Mid -= 2, __DIGIT_TABLE + __c, 2);
+    } else {
+      *--_Mid = static_cast<char>('0' + _Output);
+    }
+
+    if (_Ryu_exponent > 0) { // case "172900" with _Can_use_ryu
+      // Performance note: it might be more efficient to do this immediately after setting _Mid.
+      _VSTD::memset(_First + __olength, '0', static_cast<size_t>(_Ryu_exponent));
+    } else if (_Ryu_exponent == 0) { // case "1729"
+      // Done!
+    } else if (_Whole_digits > 0) { // case "17.29"
+      // Performance note: moving digits might not be optimal.
+      _VSTD::memmove(_First, _First + 1, static_cast<size_t>(_Whole_digits));
+      _First[_Whole_digits] = '.';
+    } else { // case "0.001729"
+      // Performance note: a larger memset() followed by overwriting '.' might be more efficient.
+      _First[0] = '0';
+      _First[1] = '.';
+      _VSTD::memset(_First + 2, '0', static_cast<size_t>(-_Whole_digits));
+    }
+
+    return { _First + _Total_fixed_length, errc{} };
+  }
+
+  const uint32_t _Total_scientific_length =
+    __olength + (__olength > 1) + 4; // digits + possible decimal point + scientific exponent
+  if (_Last - _First < static_cast<ptrdiff_t>(_Total_scientific_length)) {
+    return { _Last, errc::value_too_large };
+  }
+  char* const __result = _First;
+
+  // Print the decimal digits.
+  uint32_t __i = 0;
+  while (_Output >= 10000) {
+#ifdef __clang__ // TRANSITION, LLVM-38217
+    const uint32_t __c = _Output - 10000 * (_Output / 10000);
+#else
+    const uint32_t __c = _Output % 10000;
+#endif
+    _Output /= 10000;
+    const uint32_t __c0 = (__c % 100) << 1;
+    const uint32_t __c1 = (__c / 100) << 1;
+    _VSTD::memcpy(__result + __olength - __i - 1, __DIGIT_TABLE + __c0, 2);
+    _VSTD::memcpy(__result + __olength - __i - 3, __DIGIT_TABLE + __c1, 2);
+    __i += 4;
+  }
+  if (_Output >= 100) {
+    const uint32_t __c = (_Output % 100) << 1;
+    _Output /= 100;
+    _VSTD::memcpy(__result + __olength - __i - 1, __DIGIT_TABLE + __c, 2);
+    __i += 2;
+  }
+  if (_Output >= 10) {
+    const uint32_t __c = _Output << 1;
+    // We can't use memcpy here: the decimal dot goes between these two digits.
+    __result[2] = __DIGIT_TABLE[__c + 1];
+    __result[0] = __DIGIT_TABLE[__c];
+  } else {
+    __result[0] = static_cast<char>('0' + _Output);
+  }
+
+  // Print decimal point if needed.
+  uint32_t __index;
+  if (__olength > 1) {
+    __result[1] = '.';
+    __index = __olength + 1;
+  } else {
+    __index = 1;
+  }
+
+  // Print the exponent.
+  __result[__index++] = 'e';
+  if (_Scientific_exponent < 0) {
+    __result[__index++] = '-';
+    _Scientific_exponent = -_Scientific_exponent;
+  } else {
+    __result[__index++] = '+';
+  }
+
+  _VSTD::memcpy(__result + __index, __DIGIT_TABLE + 2 * _Scientific_exponent, 2);
+  __index += 2;
+
+  return { _First + _Total_scientific_length, errc{} };
+}
+
+[[nodiscard]] to_chars_result __f2s_buffered_n(char* const _First, char* const _Last, const float __f,
+  const chars_format _Fmt) {
+
+  // Step 1: Decode the floating-point number, and unify normalized and subnormal cases.
+  const uint32_t __bits = __float_to_bits(__f);
+
+  // Case distinction; exit early for the easy cases.
+  if (__bits == 0) {
+    if (_Fmt == chars_format::scientific) {
+      if (_Last - _First < 5) {
+        return { _Last, errc::value_too_large };
+      }
+
+      _VSTD::memcpy(_First, "0e+00", 5);
+
+      return { _First + 5, errc{} };
+    }
+
+    // Print "0" for chars_format::fixed, chars_format::general, and chars_format{}.
+    if (_First == _Last) {
+      return { _Last, errc::value_too_large };
+    }
+
+    *_First = '0';
+
+    return { _First + 1, errc{} };
+  }
+
+  // Decode __bits into mantissa and exponent.
+  const uint32_t __ieeeMantissa = __bits & ((1u << __FLOAT_MANTISSA_BITS) - 1);
+  const uint32_t __ieeeExponent = __bits >> __FLOAT_MANTISSA_BITS;
+
+  // When _Fmt == chars_format::fixed and the floating-point number is a large integer,
+  // it's faster to skip Ryu and immediately print the integer exactly.
+  if (_Fmt == chars_format::fixed) {
+    const uint32_t _Mantissa2 = __ieeeMantissa | (1u << __FLOAT_MANTISSA_BITS); // restore implicit bit
+    const int32_t _Exponent2 = static_cast<int32_t>(__ieeeExponent)
+      - __FLOAT_BIAS - __FLOAT_MANTISSA_BITS; // bias and normalization
+
+    // Normal values are equal to _Mantissa2 * 2^_Exponent2.
+    // (Subnormals are different, but they'll be rejected by the _Exponent2 test here, so they can be ignored.)
+
+    if (_Exponent2 > 0) {
+      return _Large_integer_to_chars(_First, _Last, _Mantissa2, _Exponent2);
+    }
+  }
+
+  const __floating_decimal_32 __v = __f2d(__ieeeMantissa, __ieeeExponent);
+  return __to_chars(_First, _Last, __v, _Fmt, __ieeeMantissa, __ieeeExponent);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+// clang-format on
diff --git a/libunwind/include/libunwind.h b/libunwind/include/libunwind.h
index e187ee27b0db..9a74faa48d6f 100644
--- a/libunwind/include/libunwind.h
+++ b/libunwind/include/libunwind.h
@@ -718,7 +718,8 @@ enum {
   UNW_ARM_WR14 = 126,
   UNW_ARM_WR15 = 127,
   // 128-133 -- SPSR, SPSR_{FIQ|IRQ|ABT|UND|SVC}
-  // 134-143 -- Reserved
+  // 134-142 -- Reserved
+  UNW_ARM_RA_AUTH_CODE = 143,
   // 144-150 -- R8_USR-R14_USR
   // 151-157 -- R8_FIQ-R14_FIQ
   // 158-159 -- R13_IRQ-R14_IRQ
diff --git a/libunwind/include/unwind_arm_ehabi.h b/libunwind/include/unwind_arm_ehabi.h
index dc9d403e264c..6277a1457f89 100644
--- a/libunwind/include/unwind_arm_ehabi.h
+++ b/libunwind/include/unwind_arm_ehabi.h
@@ -87,10 +87,11 @@ extern void _Unwind_Resume(_Unwind_Exception *exception_object);
 extern void _Unwind_DeleteException(_Unwind_Exception *exception_object);
 
 typedef enum {
-  _UVRSC_CORE = 0, /* integer register */
-  _UVRSC_VFP = 1, /* vfp */
+  _UVRSC_CORE = 0,  /* integer register */
+  _UVRSC_VFP = 1,   /* vfp */
   _UVRSC_WMMXD = 3, /* Intel WMMX data register */
-  _UVRSC_WMMXC = 4 /* Intel WMMX control register */
+  _UVRSC_WMMXC = 4, /* Intel WMMX control register */
+  _UVRSC_PSEUDO = 5 /* Special purpose pseudo register */
 } _Unwind_VRS_RegClass;
 
 typedef enum {
diff --git a/libunwind/src/DwarfInstructions.hpp b/libunwind/src/DwarfInstructions.hpp
index 19835aad668f..1c50941680b3 100644
--- a/libunwind/src/DwarfInstructions.hpp
+++ b/libunwind/src/DwarfInstructions.hpp
@@ -242,6 +242,20 @@ int DwarfInstructions<A, R>::stepWithDwarf(A &addressSpace, pint_t pc,
       }
 #endif
 
+#if defined(_LIBUNWIND_IS_NATIVE_ONLY) && defined(_LIBUNWIND_TARGET_ARM) &&    \
+    defined(__ARM_FEATURE_PAUTH)
+      if ((R::getArch() == REGISTERS_ARM) &&
+          prolog.savedRegisters[UNW_ARM_RA_AUTH_CODE].value) {
+        pint_t pac =
+            getSavedRegister(addressSpace, registers, cfa,
+                             prolog.savedRegisters[UNW_ARM_RA_AUTH_CODE]);
+        __asm__ __volatile__("autg %0, %1, %2"
+                             :
+                             : "r"(pac), "r"(returnAddress), "r"(cfa)
+                             :);
+      }
+#endif
+
 #if defined(_LIBUNWIND_TARGET_SPARC)
       if (R::getArch() == REGISTERS_SPARC) {
         // Skip call site instruction and delay slot
diff --git a/libunwind/src/Registers.hpp b/libunwind/src/Registers.hpp
index c2d5327eade3..373d9e49018e 100644
--- a/libunwind/src/Registers.hpp
+++ b/libunwind/src/Registers.hpp
@@ -2140,6 +2140,10 @@ private:
     uint32_t __pc;    // Program counter r15
   };
 
+  struct PseudoRegisters {
+    uint32_t __pac; // Return Authentication Code (PAC)
+  };
+
   static void saveVFPWithFSTMD(void*);
   static void saveVFPWithFSTMX(void*);
   static void saveVFPv3(void*);
@@ -2156,6 +2160,7 @@ private:
 
   // ARM registers
   GPRs _registers;
+  PseudoRegisters _pseudo_registers;
 
   // We save floating point registers lazily because we can't know ahead of
   // time which ones are used. See EHABI #4.7.
@@ -2193,6 +2198,7 @@ inline Registers_arm::Registers_arm(const void *registers)
                 "arm registers do not fit into unw_context_t");
   // See __unw_getcontext() note about data.
   memcpy(&_registers, registers, sizeof(_registers));
+  memset(&_pseudo_registers, 0, sizeof(_pseudo_registers));
   memset(&_vfp_d0_d15_pad, 0, sizeof(_vfp_d0_d15_pad));
   memset(&_vfp_d16_d31, 0, sizeof(_vfp_d16_d31));
 #if defined(__ARM_WMMX)
@@ -2208,6 +2214,7 @@ inline Registers_arm::Registers_arm()
     _saved_vfp_d0_d15(false),
     _saved_vfp_d16_d31(false) {
   memset(&_registers, 0, sizeof(_registers));
+  memset(&_pseudo_registers, 0, sizeof(_pseudo_registers));
   memset(&_vfp_d0_d15_pad, 0, sizeof(_vfp_d0_d15_pad));
   memset(&_vfp_d16_d31, 0, sizeof(_vfp_d16_d31));
 #if defined(__ARM_WMMX)
@@ -2235,6 +2242,11 @@ inline bool Registers_arm::validRegister(int regNum) const {
     return true;
 #endif
 
+#ifdef __ARM_FEATURE_PAUTH
+  if (regNum == UNW_ARM_RA_AUTH_CODE)
+    return true;
+#endif
+
   return false;
 }
 
@@ -2261,6 +2273,11 @@ inline uint32_t Registers_arm::getRegister(int regNum) const {
   }
 #endif
 
+#ifdef __ARM_FEATURE_PAUTH
+  if (regNum == UNW_ARM_RA_AUTH_CODE)
+    return _pseudo_registers.__pac;
+#endif
+
   _LIBUNWIND_ABORT("unsupported arm register");
 }
 
@@ -2296,6 +2313,11 @@ inline void Registers_arm::setRegister(int regNum, uint32_t value) {
   }
 #endif
 
+  if (regNum == UNW_ARM_RA_AUTH_CODE) {
+    _pseudo_registers.__pac = value;
+    return;
+  }
+
   _LIBUNWIND_ABORT("unsupported arm register");
 }
 
diff --git a/libunwind/src/Unwind-EHABI.cpp b/libunwind/src/Unwind-EHABI.cpp
index 5959d2a25fea..257db724c267 100644
--- a/libunwind/src/Unwind-EHABI.cpp
+++ b/libunwind/src/Unwind-EHABI.cpp
@@ -261,6 +261,7 @@ _Unwind_VRS_Interpret(_Unwind_Context *context, const uint32_t *data,
                       size_t offset, size_t len) {
   bool wrotePC = false;
   bool finish = false;
+  bool hasReturnAddrAuthCode = false;
   while (offset < len && !finish) {
     uint8_t byte = getByte(data, offset++);
     if ((byte & 0x80) == 0) {
@@ -347,6 +348,10 @@ _Unwind_VRS_Interpret(_Unwind_Context *context, const uint32_t *data,
               break;
             }
             case 0xb4:
+              hasReturnAddrAuthCode = true;
+              _Unwind_VRS_Pop(context, _UVRSC_PSEUDO,
+                              0 /* Return Address Auth Code */, _UVRSD_UINT32);
+              break;
             case 0xb5:
             case 0xb6:
             case 0xb7:
@@ -422,6 +427,16 @@ _Unwind_VRS_Interpret(_Unwind_Context *context, const uint32_t *data,
   if (!wrotePC) {
     uint32_t lr;
     _Unwind_VRS_Get(context, _UVRSC_CORE, UNW_ARM_LR, _UVRSD_UINT32, &lr);
+#ifdef __ARM_FEATURE_PAUTH
+    if (hasReturnAddrAuthCode) {
+      uint32_t sp;
+      uint32_t pac;
+      _Unwind_VRS_Get(context, _UVRSC_CORE, UNW_ARM_SP, _UVRSD_UINT32, &sp);
+      _Unwind_VRS_Get(context, _UVRSC_PSEUDO, UNW_ARM_RA_AUTH_CODE,
+                      _UVRSD_UINT32, &pac);
+      __asm__ __volatile__("autg %0, %1, %2" : : "r"(pac), "r"(lr), "r"(sp) :);
+    }
+#endif
     _Unwind_VRS_Set(context, _UVRSC_CORE, UNW_ARM_IP, _UVRSD_UINT32, &lr);
   }
   return _URC_CONTINUE_UNWIND;
@@ -941,6 +956,15 @@ _Unwind_VRS_Set(_Unwind_Context *context, _Unwind_VRS_RegClass regclass,
     case _UVRSC_WMMXD:
       break;
 #endif
+    case _UVRSC_PSEUDO:
+      // There's only one pseudo-register, PAC, with regno == 0.
+      if (representation != _UVRSD_UINT32 || regno != 0)
+        return _UVRSR_FAILED;
+      return __unw_set_reg(cursor, (unw_regnum_t)(UNW_ARM_RA_AUTH_CODE),
+                           *(unw_word_t *)valuep) == UNW_ESUCCESS
+                 ? _UVRSR_OK
+                 : _UVRSR_FAILED;
+      break;
   }
   _LIBUNWIND_ABORT("unsupported register class");
 }
@@ -995,6 +1019,15 @@ _Unwind_VRS_Get_Internal(_Unwind_Context *context,
     case _UVRSC_WMMXD:
       break;
 #endif
+    case _UVRSC_PSEUDO:
+      // There's only one pseudo-register, PAC, with regno == 0.
+      if (representation != _UVRSD_UINT32 || regno != 0)
+        return _UVRSR_FAILED;
+      return __unw_get_reg(cursor, (unw_regnum_t)(UNW_ARM_RA_AUTH_CODE),
+                           (unw_word_t *)valuep) == UNW_ESUCCESS
+                 ? _UVRSR_OK
+                 : _UVRSR_FAILED;
+      break;
   }
   _LIBUNWIND_ABORT("unsupported register class");
 }
@@ -1092,6 +1125,20 @@ _Unwind_VRS_Pop(_Unwind_Context *context, _Unwind_VRS_RegClass regclass,
       return _Unwind_VRS_Set(context, _UVRSC_CORE, UNW_ARM_SP, _UVRSD_UINT32,
                              &sp);
     }
+    case _UVRSC_PSEUDO: {
+      if (representation != _UVRSD_UINT32 || discriminator != 0)
+        return _UVRSR_FAILED;
+      // Return Address Authentication code (PAC) - discriminator 0
+      uint32_t *sp;
+      if (_Unwind_VRS_Get(context, _UVRSC_CORE, UNW_ARM_SP, _UVRSD_UINT32,
+                          &sp) != _UVRSR_OK) {
+        return _UVRSR_FAILED;
+      }
+      uint32_t pac = *sp++;
+      _Unwind_VRS_Set(context, _UVRSC_CORE, UNW_ARM_SP, _UVRSD_UINT32, &sp);
+      return _Unwind_VRS_Set(context, _UVRSC_CORE, UNW_ARM_RA_AUTH_CODE,
+                             _UVRSD_UINT32, &pac);
+    }
   }
   _LIBUNWIND_ABORT("unsupported register class");
 }
diff --git a/libunwind/src/UnwindCursor.hpp b/libunwind/src/UnwindCursor.hpp
index 3931df0b3351..c400fdf33d8a 100644
--- a/libunwind/src/UnwindCursor.hpp
+++ b/libunwind/src/UnwindCursor.hpp
@@ -655,7 +655,9 @@ bool UnwindCursor<A, R>::validReg(int regNum) {
 #if defined(_LIBUNWIND_TARGET_X86_64)
   if (regNum >= UNW_X86_64_RAX && regNum <= UNW_X86_64_R15) return true;
 #elif defined(_LIBUNWIND_TARGET_ARM)
-  if (regNum >= UNW_ARM_R0 && regNum <= UNW_ARM_R15) return true;
+  if ((regNum >= UNW_ARM_R0 && regNum <= UNW_ARM_R15) ||
+      regNum == UNW_ARM_RA_AUTH_CODE)
+    return true;
 #elif defined(_LIBUNWIND_TARGET_AARCH64)
   if (regNum >= UNW_AARCH64_X0 && regNum <= UNW_ARM64_X30) return true;
 #endif
diff --git a/libunwind/src/UnwindRegistersRestore.S b/libunwind/src/UnwindRegistersRestore.S
index 497bf46dbdd4..694738b5155a 100644
--- a/libunwind/src/UnwindRegistersRestore.S
+++ b/libunwind/src/UnwindRegistersRestore.S
@@ -660,7 +660,13 @@ DEFINE_LIBUNWIND_FUNCTION(_ZN9libunwind13Registers_arm20restoreCoreAndJumpToEv)
   ldr sp, [lr, #52]
   ldr lr, [lr, #60]  @ restore pc into lr
 #endif
+#if defined(__ARM_FEATURE_BTI_DEFAULT) && !defined(__ARM_ARCH_ISA_ARM)
+  // 'bx' is not BTI setting when used with lr, therefore r12 is used instead
+  mov r12, lr
+  JMP(r12)
+#else
   JMP(lr)
+#endif
 
 @
 @ static void libunwind::Registers_arm::restoreVFPWithFLDMD(unw_fpreg_t* values)
diff --git a/libunwind/src/assembly.h b/libunwind/src/assembly.h
index e38d32336929..b17f2ade590b 100644
--- a/libunwind/src/assembly.h
+++ b/libunwind/src/assembly.h
@@ -81,7 +81,7 @@
 #define PPC64_OPD2
 #endif
 
-#if defined(__ARM_FEATURE_BTI_DEFAULT)
+#if defined(__aarch64__) && defined(__ARM_FEATURE_BTI_DEFAULT)
   .pushsection ".note.gnu.property", "a" SEPARATOR                             \
   .balign 8 SEPARATOR                                                          \
   .long 4 SEPARATOR                                                            \
@@ -99,6 +99,17 @@
 #define AARCH64_BTI
 #endif
 
+#if !defined(__aarch64__)
+#ifdef __ARM_FEATURE_PAC_DEFAULT
+  .eabi_attribute Tag_PAC_extension, 2
+  .eabi_attribute Tag_PACRET_use, 1
+#endif
+#ifdef __ARM_FEATURE_BTI_DEFAULT
+  .eabi_attribute Tag_BTI_extension, 1
+  .eabi_attribute Tag_BTI_use, 1
+#endif
+#endif
+
 #define GLUE2(a, b) a ## b
 #define GLUE(a, b) GLUE2(a, b)
 #define SYMBOL_NAME(name) GLUE(__USER_LABEL_PREFIX__, name)
diff --git a/lld/COFF/Driver.cpp b/lld/COFF/Driver.cpp
index f1b0c5c0707d..07b60673577e 100644
--- a/lld/COFF/Driver.cpp
+++ b/lld/COFF/Driver.cpp
@@ -2085,7 +2085,7 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
   if (args.hasArg(OPT_include_optional)) {
     // Handle /includeoptional
     for (auto *arg : args.filtered(OPT_include_optional))
-      if (dyn_cast_or_null<LazyArchive>(ctx.symtab.find(arg->getValue())))
+      if (isa_and_nonnull<LazyArchive>(ctx.symtab.find(arg->getValue())))
         addUndefined(arg->getValue());
     while (run());
   }
diff --git a/lld/COFF/DriverUtils.cpp b/lld/COFF/DriverUtils.cpp
index cac254ec6828..0921c8e27f5a 100644
--- a/lld/COFF/DriverUtils.cpp
+++ b/lld/COFF/DriverUtils.cpp
@@ -186,12 +186,6 @@ void parsePDBPageSize(StringRef s) {
     return;
   }
 
-  // FIXME: Remove this once other page sizes work.
-  if (v != 4096) {
-    warn("/pdbpagesize: page sizes != 4096 not yet implemented, ignoring flag");
-    v = 4096;
-  }
-
   config->pdbPageSize = v;
 }
 
diff --git a/lld/COFF/SymbolTable.cpp b/lld/COFF/SymbolTable.cpp
index 89bfc5960286..679c91ad06e6 100644
--- a/lld/COFF/SymbolTable.cpp
+++ b/lld/COFF/SymbolTable.cpp
@@ -16,7 +16,7 @@
 #include "lld/Common/ErrorHandler.h"
 #include "lld/Common/Memory.h"
 #include "lld/Common/Timer.h"
-#include "llvm/DebugInfo/Symbolize/Symbolize.h"
+#include "llvm/DebugInfo/DIContext.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/Object/WindowsMachineFlag.h"
diff --git a/lld/ELF/AArch64ErrataFix.cpp b/lld/ELF/AArch64ErrataFix.cpp
index 741ff26a7e6c..50d4c237778b 100644
--- a/lld/ELF/AArch64ErrataFix.cpp
+++ b/lld/ELF/AArch64ErrataFix.cpp
@@ -440,9 +440,8 @@ void AArch64Err843419Patcher::init() {
   };
 
   // Collect mapping symbols for every executable InputSection.
-  for (InputFile *file : objectFiles) {
-    auto *f = cast<ObjFile<ELF64LE>>(file);
-    for (Symbol *b : f->getLocalSymbols()) {
+  for (ELFFileBase *file : objectFiles) {
+    for (Symbol *b : file->getLocalSymbols()) {
       auto *def = dyn_cast<Defined>(b);
       if (!def)
         continue;
diff --git a/lld/ELF/ARMErrataFix.cpp b/lld/ELF/ARMErrataFix.cpp
index fe6ec09bd979..5ad55f1326b3 100644
--- a/lld/ELF/ARMErrataFix.cpp
+++ b/lld/ELF/ARMErrataFix.cpp
@@ -329,9 +329,8 @@ void ARMErr657417Patcher::init() {
   };
 
   // Collect mapping symbols for every executable InputSection.
-  for (InputFile *file : objectFiles) {
-    auto *f = cast<ObjFile<ELF32LE>>(file);
-    for (Symbol *s : f->getLocalSymbols()) {
+  for (ELFFileBase *file : objectFiles) {
+    for (Symbol *s : file->getLocalSymbols()) {
       auto *def = dyn_cast<Defined>(s);
       if (!def)
         continue;
diff --git a/lld/ELF/Arch/AArch64.cpp b/lld/ELF/Arch/AArch64.cpp
index b57fd61b65cc..ca3a6aa58dc5 100644
--- a/lld/ELF/Arch/AArch64.cpp
+++ b/lld/ELF/Arch/AArch64.cpp
@@ -690,11 +690,11 @@ void AArch64BtiPac::writePlt(uint8_t *buf, const Symbol &sym,
   };
   const uint8_t nopData[] = { 0x1f, 0x20, 0x03, 0xd5 }; // nop
 
-  // needsPltAddr indicates a non-ifunc canonical PLT entry whose address may
+  // needsCopy indicates a non-ifunc canonical PLT entry whose address may
   // escape to shared objects. isInIplt indicates a non-preemptible ifunc. Its
   // address may escape if referenced by a direct relocation. The condition is
   // conservative.
-  bool hasBti = btiHeader && (sym.needsPltAddr || sym.isInIplt);
+  bool hasBti = btiHeader && (sym.needsCopy || sym.isInIplt);
   if (hasBti) {
     memcpy(buf, btiData, sizeof(btiData));
     buf += sizeof(btiData);
diff --git a/lld/ELF/Arch/PPC.cpp b/lld/ELF/Arch/PPC.cpp
index 0dda9a40eef7..e28b62329494 100644
--- a/lld/ELF/Arch/PPC.cpp
+++ b/lld/ELF/Arch/PPC.cpp
@@ -20,6 +20,9 @@ using namespace llvm::ELF;
 using namespace lld;
 using namespace lld::elf;
 
+// Undefine the macro predefined by GCC powerpc32.
+#undef PPC
+
 namespace {
 class PPC final : public TargetInfo {
 public:
@@ -74,7 +77,7 @@ void elf::writePPC32GlinkSection(uint8_t *buf, size_t numEntries) {
   // non-GOT-non-PLT relocations referencing external functions for -fpie/-fPIE.
   uint32_t glink = in.plt->getVA(); // VA of .glink
   if (!config->isPic) {
-    for (const Symbol *sym : cast<PPC32GlinkSection>(in.plt)->canonical_plts) {
+    for (const Symbol *sym : cast<PPC32GlinkSection>(*in.plt).canonical_plts) {
       writePPC32PltCallStub(buf, sym->getGotPltVA(), nullptr, 0);
       buf += 16;
       glink += 16;
diff --git a/lld/ELF/CallGraphSort.cpp b/lld/ELF/CallGraphSort.cpp
index aa00d6eadbf9..5b07f0e18c8a 100644
--- a/lld/ELF/CallGraphSort.cpp
+++ b/lld/ELF/CallGraphSort.cpp
@@ -114,8 +114,8 @@ CallGraphSort::CallGraphSort() {
 
   // Create the graph.
   for (std::pair<SectionPair, uint64_t> &c : profile) {
-    const auto *fromSB = cast<InputSectionBase>(c.first.first->repl);
-    const auto *toSB = cast<InputSectionBase>(c.first.second->repl);
+    const auto *fromSB = cast<InputSectionBase>(c.first.first);
+    const auto *toSB = cast<InputSectionBase>(c.first.second);
     uint64_t weight = c.second;
 
     // Ignore edges between input sections belonging to different output
diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h
index c660a8e67c21..b3d5219ff57b 100644
--- a/lld/ELF/Config.h
+++ b/lld/ELF/Config.h
@@ -22,6 +22,7 @@
 #include "llvm/Support/GlobPattern.h"
 #include "llvm/Support/PrettyStackTrace.h"
 #include <atomic>
+#include <memory>
 #include <vector>
 
 namespace lld {
@@ -30,7 +31,7 @@ namespace elf {
 class InputFile;
 class InputSectionBase;
 
-enum ELFKind {
+enum ELFKind : uint8_t {
   ELFNoneKind,
   ELF32LEKind,
   ELF32BEKind,
@@ -128,6 +129,8 @@ struct Configuration {
   llvm::StringRef thinLTOCacheDir;
   llvm::StringRef thinLTOIndexOnlyArg;
   llvm::StringRef whyExtract;
+  StringRef zBtiReport = "none";
+  StringRef zCetReport = "none";
   llvm::StringRef ltoBasicBlockSections;
   std::pair<llvm::StringRef, llvm::StringRef> thinLTOObjectSuffixReplace;
   std::pair<llvm::StringRef, llvm::StringRef> thinLTOPrefixReplace;
@@ -341,7 +344,7 @@ struct Configuration {
 };
 
 // The only instance of Configuration struct.
-extern Configuration *config;
+extern std::unique_ptr<Configuration> config;
 
 // The first two elements of versionDefinitions represent VER_NDX_LOCAL and
 // VER_NDX_GLOBAL. This helper returns other elements.
diff --git a/lld/ELF/DWARF.cpp b/lld/ELF/DWARF.cpp
index 4d84c09a0185..789820ba7a8e 100644
--- a/lld/ELF/DWARF.cpp
+++ b/lld/ELF/DWARF.cpp
@@ -27,8 +27,7 @@ using namespace lld::elf;
 
 template <class ELFT> LLDDwarfObj<ELFT>::LLDDwarfObj(ObjFile<ELFT> *obj) {
   // Get the ELF sections to retrieve sh_flags. See the SHF_GROUP comment below.
-  ArrayRef<typename ELFT::Shdr> objSections =
-      CHECK(obj->getObj().sections(), obj);
+  ArrayRef<typename ELFT::Shdr> objSections = obj->template getELFShdrs<ELFT>();
   assert(objSections.size() == obj->getSections().size());
   for (auto it : llvm::enumerate(obj->getSections())) {
     InputSectionBase *sec = it.value();
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index 1376e6c2c253..6b689f50cce7 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -71,8 +71,8 @@ using namespace llvm::support;
 using namespace lld;
 using namespace lld::elf;
 
-Configuration *elf::config;
-LinkerDriver *elf::driver;
+std::unique_ptr<Configuration> elf::config;
+std::unique_ptr<LinkerDriver> elf::driver;
 
 static void setConfigs(opt::InputArgList &args);
 static void readConfigs(opt::InputArgList &args);
@@ -90,7 +90,7 @@ bool elf::link(ArrayRef<const char *> args, bool canExitEarly,
     archiveFiles.clear();
     binaryFiles.clear();
     bitcodeFiles.clear();
-    lazyObjFiles.clear();
+    lazyBitcodeFiles.clear();
     objectFiles.clear();
     sharedFiles.clear();
     backwardReferences.clear();
@@ -111,10 +111,10 @@ bool elf::link(ArrayRef<const char *> args, bool canExitEarly,
   errorHandler().exitEarly = canExitEarly;
   stderrOS.enable_colors(stderrOS.has_colors());
 
-  config = make<Configuration>();
-  driver = make<LinkerDriver>();
-  script = make<LinkerScript>();
-  symtab = make<SymbolTable>();
+  config = std::make_unique<Configuration>();
+  driver = std::make_unique<LinkerDriver>();
+  script = std::make_unique<LinkerScript>();
+  symtab = std::make_unique<SymbolTable>();
 
   partitions = {Partition()};
 
@@ -248,7 +248,7 @@ void LinkerDriver::addFile(StringRef path, bool withLOption) {
 
       for (const std::pair<MemoryBufferRef, uint64_t> &p :
            getArchiveMembers(mbref))
-        files.push_back(make<LazyObjFile>(p.first, path, p.second));
+        files.push_back(createLazyFile(p.first, path, p.second));
       return;
     }
 
@@ -273,7 +273,7 @@ void LinkerDriver::addFile(StringRef path, bool withLOption) {
   case file_magic::bitcode:
   case file_magic::elf_relocatable:
     if (inLib)
-      files.push_back(make<LazyObjFile>(mbref, "", 0));
+      files.push_back(createLazyFile(mbref, "", 0));
     else
       files.push_back(createObjectFile(mbref));
     break;
@@ -368,7 +368,13 @@ static void checkOptions() {
       error("-z pac-plt only supported on AArch64");
     if (config->zForceBti)
       error("-z force-bti only supported on AArch64");
+    if (config->zBtiReport != "none")
+      error("-z bti-report only supported on AArch64");
   }
+
+  if (config->emachine != EM_386 && config->emachine != EM_X86_64 &&
+      config->zCetReport != "none")
+    error("-z cet-report only supported on X86 and X86_64");
 }
 
 static const char *getReproduceOption(opt::InputArgList &args) {
@@ -455,6 +461,7 @@ static bool isKnownZFlag(StringRef s) {
          s == "rela" || s == "relro" || s == "retpolineplt" ||
          s == "rodynamic" || s == "shstk" || s == "text" || s == "undefs" ||
          s == "wxneeded" || s.startswith("common-page-size=") ||
+         s.startswith("bti-report=") || s.startswith("cet-report=") ||
          s.startswith("dead-reloc-in-nonalloc=") ||
          s.startswith("max-page-size=") || s.startswith("stack-size=") ||
          s.startswith("start-stop-visibility=");
@@ -798,7 +805,7 @@ static std::pair<bool, bool> getPackDynRelocs(opt::InputArgList &args) {
 static void readCallGraph(MemoryBufferRef mb) {
   // Build a map from symbol name to section
   DenseMap<StringRef, Symbol *> map;
-  for (InputFile *file : objectFiles)
+  for (ELFFileBase *file : objectFiles)
     for (Symbol *sym : file->getSymbols())
       map[sym->getName()] = sym;
 
@@ -839,14 +846,13 @@ static bool
 processCallGraphRelocations(SmallVector<uint32_t, 32> &symbolIndices,
                             ArrayRef<typename ELFT::CGProfile> &cgProfile,
                             ObjFile<ELFT> *inputObj) {
-  symbolIndices.clear();
-  const ELFFile<ELFT> &obj = inputObj->getObj();
-  ArrayRef<Elf_Shdr_Impl<ELFT>> objSections =
-      CHECK(obj.sections(), "could not retrieve object sections");
-
   if (inputObj->cgProfileSectionIndex == SHN_UNDEF)
     return false;
 
+  ArrayRef<Elf_Shdr_Impl<ELFT>> objSections =
+      inputObj->template getELFShdrs<ELFT>();
+  symbolIndices.clear();
+  const ELFFile<ELFT> &obj = inputObj->getObj();
   cgProfile =
       check(obj.template getSectionContentsAsArray<typename ELFT::CGProfile>(
           objSections[inputObj->cgProfileSectionIndex]));
@@ -970,6 +976,11 @@ static void parseClangOption(StringRef opt, const Twine &msg) {
   error(msg + ": " + StringRef(err).trim());
 }
 
+// Checks the parameter of the bti-report and cet-report options.
+static bool isValidReportString(StringRef arg) {
+  return arg == "none" || arg == "warning" || arg == "error";
+}
+
 // Initializes Config members by the command line options.
 static void readConfigs(opt::InputArgList &args) {
   errorHandler().verbose = args.hasArg(OPT_verbose);
@@ -1203,6 +1214,23 @@ static void readConfigs(opt::InputArgList &args) {
       error(errPrefix + toString(pat.takeError()));
   }
 
+  auto reports = {std::make_pair("bti-report", &config->zBtiReport),
+                  std::make_pair("cet-report", &config->zCetReport)};
+  for (opt::Arg *arg : args.filtered(OPT_z)) {
+    std::pair<StringRef, StringRef> option =
+        StringRef(arg->getValue()).split('=');
+    for (auto reportArg : reports) {
+      if (option.first != reportArg.first)
+        continue;
+      if (!isValidReportString(option.second)) {
+        error(Twine("-z ") + reportArg.first + "= parameter " + option.second +
+              " is not recognized");
+        continue;
+      }
+      *reportArg.second = option.second;
+    }
+  }
+
   for (opt::Arg *arg : args.filtered(OPT_z)) {
     std::pair<StringRef, StringRef> option =
         StringRef(arg->getValue()).split('=');
@@ -1662,7 +1690,7 @@ static void excludeLibs(opt::InputArgList &args) {
             sym->versionId = VER_NDX_LOCAL;
   };
 
-  for (InputFile *file : objectFiles)
+  for (ELFFileBase *file : objectFiles)
     visit(file);
 
   for (BitcodeFile *file : bitcodeFiles)
@@ -1793,17 +1821,21 @@ static void writeDependencyFile() {
 // symbols of type CommonSymbol.
 static void replaceCommonSymbols() {
   llvm::TimeTraceScope timeScope("Replace common symbols");
-  for (Symbol *sym : symtab->symbols()) {
-    auto *s = dyn_cast<CommonSymbol>(sym);
-    if (!s)
+  for (ELFFileBase *file : objectFiles) {
+    if (!file->hasCommonSyms)
       continue;
+    for (Symbol *sym : file->getGlobalSymbols()) {
+      auto *s = dyn_cast<CommonSymbol>(sym);
+      if (!s)
+        continue;
 
-    auto *bss = make<BssSection>("COMMON", s->size, s->alignment);
-    bss->file = s->file;
-    bss->markDead();
-    inputSections.push_back(bss);
-    s->replace(Defined{s->file, s->getName(), s->binding, s->stOther, s->type,
-                       /*value=*/0, s->size, bss});
+      auto *bss = make<BssSection>("COMMON", s->size, s->alignment);
+      bss->file = s->file;
+      bss->markDead();
+      inputSections.push_back(bss);
+      s->replace(Defined{s->file, s->getName(), s->binding, s->stOther, s->type,
+                         /*value=*/0, s->size, bss});
+    }
   }
 }
 
@@ -1975,7 +2007,7 @@ template <class ELFT> void LinkerDriver::compileBitcodeFiles() {
     if (!config->relocatable)
       for (Symbol *sym : obj->getGlobalSymbols())
         sym->parseSymbolVersion();
-    objectFiles.push_back(file);
+    objectFiles.push_back(obj);
   }
 }
 
@@ -2091,11 +2123,10 @@ static void redirectSymbols(ArrayRef<WrappedSymbol> wrapped) {
     return;
 
   // Update pointers in input files.
-  parallelForEach(objectFiles, [&](InputFile *file) {
-    MutableArrayRef<Symbol *> syms = file->getMutableSymbols();
-    for (size_t i = 0, e = syms.size(); i != e; ++i)
-      if (Symbol *s = map.lookup(syms[i]))
-        syms[i] = s;
+  parallelForEach(objectFiles, [&](ELFFileBase *file) {
+    for (Symbol *&sym : file->getMutableGlobalSymbols())
+      if (Symbol *s = map.lookup(sym))
+        sym = s;
   });
 
   // Update pointers in the symbol table.
@@ -2103,6 +2134,16 @@ static void redirectSymbols(ArrayRef<WrappedSymbol> wrapped) {
     symtab->wrap(w.sym, w.real, w.wrap);
 }
 
+static void checkAndReportMissingFeature(StringRef config, uint32_t features,
+                                         uint32_t mask, const Twine &report) {
+  if (!(features & mask)) {
+    if (config == "error")
+      error(report);
+    else if (config == "warning")
+      warn(report);
+  }
+}
+
 // To enable CET (x86's hardware-assited control flow enforcement), each
 // source file must be compiled with -fcf-protection. Object files compiled
 // with the flag contain feature flags indicating that they are compatible
@@ -2119,14 +2160,32 @@ template <class ELFT> static uint32_t getAndFeatures() {
   uint32_t ret = -1;
   for (InputFile *f : objectFiles) {
     uint32_t features = cast<ObjFile<ELFT>>(f)->andFeatures;
+
+    checkAndReportMissingFeature(
+        config->zBtiReport, features, GNU_PROPERTY_AARCH64_FEATURE_1_BTI,
+        toString(f) + ": -z bti-report: file does not have "
+                      "GNU_PROPERTY_AARCH64_FEATURE_1_BTI property");
+
+    checkAndReportMissingFeature(
+        config->zCetReport, features, GNU_PROPERTY_X86_FEATURE_1_IBT,
+        toString(f) + ": -z cet-report: file does not have "
+                      "GNU_PROPERTY_X86_FEATURE_1_IBT property");
+
+    checkAndReportMissingFeature(
+        config->zCetReport, features, GNU_PROPERTY_X86_FEATURE_1_SHSTK,
+        toString(f) + ": -z cet-report: file does not have "
+                      "GNU_PROPERTY_X86_FEATURE_1_SHSTK property");
+
     if (config->zForceBti && !(features & GNU_PROPERTY_AARCH64_FEATURE_1_BTI)) {
-      warn(toString(f) + ": -z force-bti: file does not have "
-                         "GNU_PROPERTY_AARCH64_FEATURE_1_BTI property");
       features |= GNU_PROPERTY_AARCH64_FEATURE_1_BTI;
+      if (config->zBtiReport == "none")
+        warn(toString(f) + ": -z force-bti: file does not have "
+                           "GNU_PROPERTY_AARCH64_FEATURE_1_BTI property");
     } else if (config->zForceIbt &&
                !(features & GNU_PROPERTY_X86_FEATURE_1_IBT)) {
-      warn(toString(f) + ": -z force-ibt: file does not have "
-                         "GNU_PROPERTY_X86_FEATURE_1_IBT property");
+      if (config->zCetReport == "none")
+        warn(toString(f) + ": -z force-ibt: file does not have "
+                           "GNU_PROPERTY_X86_FEATURE_1_IBT property");
       features |= GNU_PROPERTY_X86_FEATURE_1_IBT;
     }
     if (config->zPacPlt && !(features & GNU_PROPERTY_AARCH64_FEATURE_1_PAC)) {
diff --git a/lld/ELF/Driver.h b/lld/ELF/Driver.h
index 96d040041c5a..5961e1f69472 100644
--- a/lld/ELF/Driver.h
+++ b/lld/ELF/Driver.h
@@ -22,7 +22,7 @@
 namespace lld {
 namespace elf {
 
-extern class LinkerDriver *driver;
+extern std::unique_ptr<class LinkerDriver> driver;
 
 class LinkerDriver {
 public:
diff --git a/lld/ELF/ICF.cpp b/lld/ELF/ICF.cpp
index 0ec748e8f990..ec63d2ef4d6f 100644
--- a/lld/ELF/ICF.cpp
+++ b/lld/ELF/ICF.cpp
@@ -550,6 +550,22 @@ template <class ELFT> void ICF<ELFT>::run() {
     }
   });
 
+  // Change Defined symbol's section field to the canonical one.
+  auto fold = [](Symbol *sym) {
+    if (auto *d = dyn_cast<Defined>(sym))
+      if (auto *sec = dyn_cast_or_null<InputSection>(d->section))
+        if (sec->repl != d->section) {
+          d->section = sec->repl;
+          d->folded = true;
+        }
+  };
+  for (Symbol *sym : symtab->symbols())
+    fold(sym);
+  parallelForEach(objectFiles, [&](ELFFileBase *file) {
+    for (Symbol *sym : file->getLocalSymbols())
+      fold(sym);
+  });
+
   // InputSectionDescription::sections is populated by processSectionCommands().
   // ICF may fold some input sections assigned to output sections. Remove them.
   for (SectionCommand *cmd : script->sectionCommands)
diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp
index 031a8679db41..e321b0d82920 100644
--- a/lld/ELF/InputFiles.cpp
+++ b/lld/ELF/InputFiles.cpp
@@ -43,12 +43,12 @@ using namespace lld::elf;
 bool InputFile::isInGroup;
 uint32_t InputFile::nextGroupId;
 
-std::vector<ArchiveFile *> elf::archiveFiles;
-std::vector<BinaryFile *> elf::binaryFiles;
-std::vector<BitcodeFile *> elf::bitcodeFiles;
-std::vector<LazyObjFile *> elf::lazyObjFiles;
-std::vector<InputFile *> elf::objectFiles;
-std::vector<SharedFile *> elf::sharedFiles;
+SmallVector<ArchiveFile *, 0> elf::archiveFiles;
+SmallVector<BinaryFile *, 0> elf::binaryFiles;
+SmallVector<BitcodeFile *, 0> elf::bitcodeFiles;
+SmallVector<BitcodeFile *, 0> elf::lazyBitcodeFiles;
+SmallVector<ELFFileBase *, 0> elf::objectFiles;
+SmallVector<SharedFile *, 0> elf::sharedFiles;
 
 std::unique_ptr<TarWriter> elf::tar;
 
@@ -59,11 +59,11 @@ std::string lld::toString(const InputFile *f) {
 
   if (f->toStringCache.empty()) {
     if (f->archiveName.empty())
-      f->toStringCache = std::string(f->getName());
+      f->toStringCache = f->getName();
     else
-      f->toStringCache = (f->archiveName + "(" + f->getName() + ")").str();
+      (f->archiveName + "(" + f->getName() + ")").toVector(f->toStringCache);
   }
-  return f->toStringCache;
+  return std::string(f->toStringCache);
 }
 
 static ELFKind getELFKind(MemoryBufferRef mb, StringRef archiveName) {
@@ -186,9 +186,13 @@ template <class ELFT> static void doParseFile(InputFile *file) {
   }
 
   // Lazy object file
-  if (auto *f = dyn_cast<LazyObjFile>(file)) {
-    lazyObjFiles.push_back(f);
-    f->parse<ELFT>();
+  if (file->lazy) {
+    if (auto *f = dyn_cast<BitcodeFile>(file)) {
+      lazyBitcodeFiles.push_back(f);
+      f->parseLazy();
+    } else {
+      cast<ObjFile<ELFT>>(file)->parseLazy();
+    }
     return;
   }
 
@@ -209,7 +213,7 @@ template <class ELFT> static void doParseFile(InputFile *file) {
   }
 
   // Regular object file
-  objectFiles.push_back(file);
+  objectFiles.push_back(cast<ELFFileBase>(file));
   cast<ObjFile<ELFT>>(file)->parse();
 }
 
@@ -366,6 +370,8 @@ template <class ELFT> void ELFFileBase::init() {
   abiVersion = obj.getHeader().e_ident[llvm::ELF::EI_ABIVERSION];
 
   ArrayRef<Elf_Shdr> sections = CHECK(obj.sections(), this);
+  elfShdrs = sections.data();
+  numELFShdrs = sections.size();
 
   // Find a symbol table.
   bool isDSO =
@@ -384,7 +390,7 @@ template <class ELFT> void ELFFileBase::init() {
     fatal(toString(this) + ": invalid sh_info in symbol table");
 
   elfSyms = reinterpret_cast<const void *>(eSyms.data());
-  numELFSyms = eSyms.size();
+  numELFSyms = uint32_t(eSyms.size());
   stringTable = CHECK(obj.getStringTableForSymtab(*symtabSec, sections), this);
 }
 
@@ -421,9 +427,6 @@ StringRef ObjFile<ELFT>::getShtGroupSignature(ArrayRef<Elf_Shdr> sections,
 
 template <class ELFT>
 bool ObjFile<ELFT>::shouldMerge(const Elf_Shdr &sec, StringRef name) {
-  if (!(sec.sh_flags & SHF_MERGE))
-    return false;
-
   // On a regular link we don't merge sections if -O0 (default is -O1). This
   // sometimes makes the linker significantly faster, although the output will
   // be bigger.
@@ -476,8 +479,7 @@ bool ObjFile<ELFT>::shouldMerge(const Elf_Shdr &sec, StringRef name) {
 // When the option is given, we link "just symbols". The section table is
 // initialized with null pointers.
 template <class ELFT> void ObjFile<ELFT>::initializeJustSymbols() {
-  ArrayRef<Elf_Shdr> sections = CHECK(this->getObj().sections(), this);
-  this->sections.resize(sections.size());
+  sections.resize(numELFShdrs);
 }
 
 // An ELF object file may contain a `.deplibs` section. If it exists, the
@@ -543,7 +545,7 @@ template <class ELFT>
 void ObjFile<ELFT>::initializeSections(bool ignoreComdats) {
   const ELFFile<ELFT> &obj = this->getObj();
 
-  ArrayRef<Elf_Shdr> objSections = CHECK(obj.sections(), this);
+  ArrayRef<Elf_Shdr> objSections = getELFShdrs<ELFT>();
   StringRef shstrtab = CHECK(obj.getSectionStringTable(objSections), this);
   uint64_t size = objSections.size();
   this->sections.resize(size);
@@ -555,13 +557,12 @@ void ObjFile<ELFT>::initializeSections(bool ignoreComdats) {
       continue;
     const Elf_Shdr &sec = objSections[i];
 
-    if (sec.sh_type == ELF::SHT_LLVM_CALL_GRAPH_PROFILE)
-      cgProfileSectionIndex = i;
-
     // SHF_EXCLUDE'ed sections are discarded by the linker. However,
     // if -r is given, we'll let the final link discard such sections.
     // This is compatible with GNU.
     if ((sec.sh_flags & SHF_EXCLUDE) && !config->relocatable) {
+      if (sec.sh_type == SHT_LLVM_CALL_GRAPH_PROFILE)
+        cgProfileSectionIndex = i;
       if (sec.sh_type == SHT_LLVM_ADDRSIG) {
         // We ignore the address-significance table if we know that the object
         // file was created by objcopy or ld -r. This is because these tools
@@ -966,54 +967,65 @@ InputSectionBase *ObjFile<ELFT>::createInputSection(uint32_t idx,
   }
   }
 
-  // The GNU linker uses .note.GNU-stack section as a marker indicating
-  // that the code in the object file does not expect that the stack is
-  // executable (in terms of NX bit). If all input files have the marker,
-  // the GNU linker adds a PT_GNU_STACK segment to tells the loader to
-  // make the stack non-executable. Most object files have this section as
-  // of 2017.
-  //
-  // But making the stack non-executable is a norm today for security
-  // reasons. Failure to do so may result in a serious security issue.
-  // Therefore, we make LLD always add PT_GNU_STACK unless it is
-  // explicitly told to do otherwise (by -z execstack). Because the stack
-  // executable-ness is controlled solely by command line options,
-  // .note.GNU-stack sections are simply ignored.
-  if (name == ".note.GNU-stack")
-    return &InputSection::discarded;
+  if (name.startswith(".n")) {
+    // The GNU linker uses .note.GNU-stack section as a marker indicating
+    // that the code in the object file does not expect that the stack is
+    // executable (in terms of NX bit). If all input files have the marker,
+    // the GNU linker adds a PT_GNU_STACK segment to tells the loader to
+    // make the stack non-executable. Most object files have this section as
+    // of 2017.
+    //
+    // But making the stack non-executable is a norm today for security
+    // reasons. Failure to do so may result in a serious security issue.
+    // Therefore, we make LLD always add PT_GNU_STACK unless it is
+    // explicitly told to do otherwise (by -z execstack). Because the stack
+    // executable-ness is controlled solely by command line options,
+    // .note.GNU-stack sections are simply ignored.
+    if (name == ".note.GNU-stack")
+      return &InputSection::discarded;
 
-  // Object files that use processor features such as Intel Control-Flow
-  // Enforcement (CET) or AArch64 Branch Target Identification BTI, use a
-  // .note.gnu.property section containing a bitfield of feature bits like the
-  // GNU_PROPERTY_X86_FEATURE_1_IBT flag. Read a bitmap containing the flag.
-  //
-  // Since we merge bitmaps from multiple object files to create a new
-  // .note.gnu.property containing a single AND'ed bitmap, we discard an input
-  // file's .note.gnu.property section.
-  if (name == ".note.gnu.property") {
-    this->andFeatures = readAndFeatures<ELFT>(InputSection(*this, sec, name));
-    return &InputSection::discarded;
-  }
+    // Object files that use processor features such as Intel Control-Flow
+    // Enforcement (CET) or AArch64 Branch Target Identification BTI, use a
+    // .note.gnu.property section containing a bitfield of feature bits like the
+    // GNU_PROPERTY_X86_FEATURE_1_IBT flag. Read a bitmap containing the flag.
+    //
+    // Since we merge bitmaps from multiple object files to create a new
+    // .note.gnu.property containing a single AND'ed bitmap, we discard an input
+    // file's .note.gnu.property section.
+    if (name == ".note.gnu.property") {
+      this->andFeatures = readAndFeatures<ELFT>(InputSection(*this, sec, name));
+      return &InputSection::discarded;
+    }
 
-  // Split stacks is a feature to support a discontiguous stack,
-  // commonly used in the programming language Go. For the details,
-  // see https://gcc.gnu.org/wiki/SplitStacks. An object file compiled
-  // for split stack will include a .note.GNU-split-stack section.
-  if (name == ".note.GNU-split-stack") {
-    if (config->relocatable) {
-      error("cannot mix split-stack and non-split-stack in a relocatable link");
+    // Split stacks is a feature to support a discontiguous stack,
+    // commonly used in the programming language Go. For the details,
+    // see https://gcc.gnu.org/wiki/SplitStacks. An object file compiled
+    // for split stack will include a .note.GNU-split-stack section.
+    if (name == ".note.GNU-split-stack") {
+      if (config->relocatable) {
+        error(
+            "cannot mix split-stack and non-split-stack in a relocatable link");
+        return &InputSection::discarded;
+      }
+      this->splitStack = true;
       return &InputSection::discarded;
     }
-    this->splitStack = true;
-    return &InputSection::discarded;
-  }
 
-  // An object file cmpiled for split stack, but where some of the
-  // functions were compiled with the no_split_stack_attribute will
-  // include a .note.GNU-no-split-stack section.
-  if (name == ".note.GNU-no-split-stack") {
-    this->someNoSplitStack = true;
-    return &InputSection::discarded;
+    // An object file cmpiled for split stack, but where some of the
+    // functions were compiled with the no_split_stack_attribute will
+    // include a .note.GNU-no-split-stack section.
+    if (name == ".note.GNU-no-split-stack") {
+      this->someNoSplitStack = true;
+      return &InputSection::discarded;
+    }
+
+    // Strip existing .note.gnu.build-id sections so that the output won't have
+    // more than one build-id. This is not usually a problem because input
+    // object files normally don't have .build-id sections, but you can create
+    // such files by "ld.{bfd,gold,lld} -r --build-id", and we want to guard
+    // against it.
+    if (name == ".note.gnu.build-id")
+      return &InputSection::discarded;
   }
 
   // The linkonce feature is a sort of proto-comdat. Some glibc i386 object
@@ -1025,20 +1037,13 @@ InputSectionBase *ObjFile<ELFT>::createInputSection(uint32_t idx,
       name == ".gnu.linkonce.t.__i686.get_pc_thunk.bx")
     return &InputSection::discarded;
 
-  // Strip existing .note.gnu.build-id sections so that the output won't have
-  // more than one build-id. This is not usually a problem because input object
-  // files normally don't have .build-id sections, but you can create such files
-  // by "ld.{bfd,gold,lld} -r --build-id", and we want to guard against it.
-  if (name == ".note.gnu.build-id")
-    return &InputSection::discarded;
-
   // The linker merges EH (exception handling) frames and creates a
   // .eh_frame_hdr section for runtime. So we handle them with a special
   // class. For relocatable outputs, they are just passed through.
   if (name == ".eh_frame" && !config->relocatable)
     return make<EhInputSection>(*this, sec, name);
 
-  if (shouldMerge(sec, name))
+  if ((sec.sh_flags & SHF_MERGE) && shouldMerge(sec, name))
     return make<MergeInputSection>(*this, sec, name);
   return make<InputSection>(*this, sec, name);
 }
@@ -1046,84 +1051,80 @@ InputSectionBase *ObjFile<ELFT>::createInputSection(uint32_t idx,
 // Initialize this->Symbols. this->Symbols is a parallel array as
 // its corresponding ELF symbol table.
 template <class ELFT> void ObjFile<ELFT>::initializeSymbols() {
+  ArrayRef<InputSectionBase *> sections(this->sections);
+  SymbolTable &symtab = *elf::symtab;
+
   ArrayRef<Elf_Sym> eSyms = this->getELFSyms<ELFT>();
-  this->symbols.resize(eSyms.size());
+  symbols.resize(eSyms.size());
+  SymbolUnion *locals =
+      firstGlobal == 0
+          ? nullptr
+          : getSpecificAllocSingleton<SymbolUnion>().Allocate(firstGlobal);
 
-  // Fill in InputFile::symbols. Some entries have been initialized
-  // because of LazyObjFile.
-  for (size_t i = 0, end = eSyms.size(); i != end; ++i) {
-    if (this->symbols[i])
-      continue;
+  for (size_t i = 0, end = firstGlobal; i != end; ++i) {
     const Elf_Sym &eSym = eSyms[i];
     uint32_t secIdx = getSectionIndex(eSym);
-    if (secIdx >= this->sections.size())
+    if (LLVM_UNLIKELY(secIdx >= sections.size()))
       fatal(toString(this) + ": invalid section index: " + Twine(secIdx));
-    if (eSym.getBinding() != STB_LOCAL) {
-      if (i < firstGlobal)
-        error(toString(this) + ": non-local symbol (" + Twine(i) +
-              ") found at index < .symtab's sh_info (" + Twine(firstGlobal) +
-              ")");
-      this->symbols[i] =
-          symtab->insert(CHECK(eSyms[i].getName(this->stringTable), this));
-      continue;
-    }
+    if (LLVM_UNLIKELY(eSym.getBinding() != STB_LOCAL))
+      error(toString(this) + ": non-local symbol (" + Twine(i) +
+            ") found at index < .symtab's sh_info (" + Twine(end) + ")");
 
-    // Handle local symbols. Local symbols are not added to the symbol
-    // table because they are not visible from other object files. We
-    // allocate symbol instances and add their pointers to symbols.
-    if (i >= firstGlobal)
-      errorOrWarn(toString(this) + ": STB_LOCAL symbol (" + Twine(i) +
-                  ") found at index >= .symtab's sh_info (" +
-                  Twine(firstGlobal) + ")");
-
-    InputSectionBase *sec = this->sections[secIdx];
+    InputSectionBase *sec = sections[secIdx];
     uint8_t type = eSym.getType();
     if (type == STT_FILE)
-      sourceFile = CHECK(eSym.getName(this->stringTable), this);
-    if (this->stringTable.size() <= eSym.st_name)
+      sourceFile = CHECK(eSym.getName(stringTable), this);
+    if (LLVM_UNLIKELY(stringTable.size() <= eSym.st_name))
       fatal(toString(this) + ": invalid symbol name offset");
-    StringRefZ name = this->stringTable.data() + eSym.st_name;
-
-    if (eSym.st_shndx == SHN_UNDEF)
-      this->symbols[i] =
-          make<Undefined>(this, name, STB_LOCAL, eSym.st_other, type);
-    else if (sec == &InputSection::discarded)
-      this->symbols[i] =
-          make<Undefined>(this, name, STB_LOCAL, eSym.st_other, type,
-                          /*discardedSecIdx=*/secIdx);
+    StringRefZ name = stringTable.data() + eSym.st_name;
+
+    symbols[i] = reinterpret_cast<Symbol *>(locals + i);
+    if (eSym.st_shndx == SHN_UNDEF || sec == &InputSection::discarded)
+      new (symbols[i]) Undefined(this, name, STB_LOCAL, eSym.st_other, type,
+                                 /*discardedSecIdx=*/secIdx);
     else
-      this->symbols[i] = make<Defined>(this, name, STB_LOCAL, eSym.st_other,
-                                       type, eSym.st_value, eSym.st_size, sec);
+      new (symbols[i]) Defined(this, name, STB_LOCAL, eSym.st_other, type,
+                               eSym.st_value, eSym.st_size, sec);
   }
 
-  // Symbol resolution of non-local symbols.
+  // Some entries have been filled by LazyObjFile.
+  for (size_t i = firstGlobal, end = eSyms.size(); i != end; ++i)
+    if (!symbols[i])
+      symbols[i] = symtab.insert(CHECK(eSyms[i].getName(stringTable), this));
+
+  // Perform symbol resolution on non-local symbols.
   SmallVector<unsigned, 32> undefineds;
   for (size_t i = firstGlobal, end = eSyms.size(); i != end; ++i) {
     const Elf_Sym &eSym = eSyms[i];
     uint8_t binding = eSym.getBinding();
-    if (binding == STB_LOCAL)
-      continue; // Errored above.
-
+    if (LLVM_UNLIKELY(binding == STB_LOCAL)) {
+      errorOrWarn(toString(this) + ": STB_LOCAL symbol (" + Twine(i) +
+                  ") found at index >= .symtab's sh_info (" +
+                  Twine(firstGlobal) + ")");
+      continue;
+    }
     uint32_t secIdx = getSectionIndex(eSym);
-    InputSectionBase *sec = this->sections[secIdx];
+    if (LLVM_UNLIKELY(secIdx >= sections.size()))
+      fatal(toString(this) + ": invalid section index: " + Twine(secIdx));
+    InputSectionBase *sec = sections[secIdx];
     uint8_t stOther = eSym.st_other;
     uint8_t type = eSym.getType();
     uint64_t value = eSym.st_value;
     uint64_t size = eSym.st_size;
-    StringRefZ name = this->stringTable.data() + eSym.st_name;
 
-    // Handle global undefined symbols.
     if (eSym.st_shndx == SHN_UNDEF) {
       undefineds.push_back(i);
       continue;
     }
 
-    // Handle global common symbols.
-    if (eSym.st_shndx == SHN_COMMON) {
+    Symbol *sym = symbols[i];
+    const StringRef name = sym->getName();
+    if (LLVM_UNLIKELY(eSym.st_shndx == SHN_COMMON)) {
       if (value == 0 || value >= UINT32_MAX)
-        fatal(toString(this) + ": common symbol '" + StringRef(name.data) +
+        fatal(toString(this) + ": common symbol '" + name +
               "' has invalid alignment: " + Twine(value));
-      this->symbols[i]->resolve(
+      hasCommonSyms = true;
+      sym->resolve(
           CommonSymbol{this, name, binding, stOther, type, value, size});
       continue;
     }
@@ -1135,16 +1136,14 @@ template <class ELFT> void ObjFile<ELFT>::initializeSymbols() {
     // defined symbol in a .eh_frame becomes dangling symbols.
     if (sec == &InputSection::discarded) {
       Undefined und{this, name, binding, stOther, type, secIdx};
-      Symbol *sym = this->symbols[i];
-      // !ArchiveFile::parsed or LazyObjFile::extracted means that the file
+      // !ArchiveFile::parsed or !LazyObjFile::lazy means that the file
       // containing this object has not finished processing, i.e. this symbol is
       // a result of a lazy symbol extract. We should demote the lazy symbol to
       // an Undefined so that any relocations outside of the group to it will
       // trigger a discarded section error.
       if ((sym->symbolKind == Symbol::LazyArchiveKind &&
            !cast<ArchiveFile>(sym->file)->parsed) ||
-          (sym->symbolKind == Symbol::LazyObjectKind &&
-           cast<LazyObjFile>(sym->file)->extracted)) {
+          (sym->symbolKind == Symbol::LazyObjectKind && !sym->file->lazy)) {
         sym->replace(und);
         // Prevent LTO from internalizing the symbol in case there is a
         // reference to this symbol from this file.
@@ -1157,7 +1156,7 @@ template <class ELFT> void ObjFile<ELFT>::initializeSymbols() {
     // Handle global defined symbols.
     if (binding == STB_GLOBAL || binding == STB_WEAK ||
         binding == STB_GNU_UNIQUE) {
-      this->symbols[i]->resolve(
+      sym->resolve(
           Defined{this, name, binding, stOther, type, value, size, sec});
       continue;
     }
@@ -1173,10 +1172,10 @@ template <class ELFT> void ObjFile<ELFT>::initializeSymbols() {
   // being resolved to different files.
   for (unsigned i : undefineds) {
     const Elf_Sym &eSym = eSyms[i];
-    StringRefZ name = this->stringTable.data() + eSym.st_name;
-    this->symbols[i]->resolve(Undefined{this, name, eSym.getBinding(),
-                                        eSym.st_other, eSym.getType()});
-    this->symbols[i]->referenced = true;
+    Symbol *sym = symbols[i];
+    sym->resolve(Undefined{this, sym->getName(), eSym.getBinding(),
+                           eSym.st_other, eSym.getType()});
+    sym->referenced = true;
   }
 }
 
@@ -1185,8 +1184,9 @@ ArchiveFile::ArchiveFile(std::unique_ptr<Archive> &&file)
       file(std::move(file)) {}
 
 void ArchiveFile::parse() {
+  SymbolTable &symtab = *elf::symtab;
   for (const Archive::Symbol &sym : file->symbols())
-    symtab->addSymbol(LazyArchive{*this, sym});
+    symtab.addSymbol(LazyArchive{*this, sym});
 
   // Inform a future invocation of ObjFile<ELFT>::initializeSymbols() that this
   // archive has been processed.
@@ -1319,26 +1319,21 @@ unsigned SharedFile::vernauxNum;
 // vector whose nth element contains a pointer to the Elf_Verdef for version
 // identifier n. Version identifiers that are not definitions map to nullptr.
 template <typename ELFT>
-static std::vector<const void *> parseVerdefs(const uint8_t *base,
-                                              const typename ELFT::Shdr *sec) {
+static SmallVector<const void *, 0>
+parseVerdefs(const uint8_t *base, const typename ELFT::Shdr *sec) {
   if (!sec)
     return {};
 
-  // We cannot determine the largest verdef identifier without inspecting
-  // every Elf_Verdef, but both bfd and gold assign verdef identifiers
-  // sequentially starting from 1, so we predict that the largest identifier
-  // will be verdefCount.
-  unsigned verdefCount = sec->sh_info;
-  std::vector<const void *> verdefs(verdefCount + 1);
-
   // Build the Verdefs array by following the chain of Elf_Verdef objects
   // from the start of the .gnu.version_d section.
+  SmallVector<const void *, 0> verdefs;
   const uint8_t *verdef = base + sec->sh_offset;
-  for (unsigned i = 0; i != verdefCount; ++i) {
+  for (unsigned i = 0, e = sec->sh_info; i != e; ++i) {
     auto *curVerdef = reinterpret_cast<const typename ELFT::Verdef *>(verdef);
     verdef += curVerdef->vd_next;
     unsigned verdefIndex = curVerdef->vd_ndx;
-    verdefs.resize(verdefIndex + 1);
+    if (verdefIndex >= verdefs.size())
+      verdefs.resize(verdefIndex + 1);
     verdefs[verdefIndex] = curVerdef;
   }
   return verdefs;
@@ -1417,7 +1412,7 @@ template <class ELFT> void SharedFile::parse() {
 
   ArrayRef<Elf_Dyn> dynamicTags;
   const ELFFile<ELFT> obj = this->getObj<ELFT>();
-  ArrayRef<Elf_Shdr> sections = CHECK(obj.sections(), this);
+  ArrayRef<Elf_Shdr> sections = getELFShdrs<ELFT>();
 
   const Elf_Shdr *versymSec = nullptr;
   const Elf_Shdr *verdefSec = nullptr;
@@ -1502,15 +1497,16 @@ template <class ELFT> void SharedFile::parse() {
   SmallString<0> versionedNameBuffer;
 
   // Add symbols to the symbol table.
+  SymbolTable &symtab = *elf::symtab;
   ArrayRef<Elf_Sym> syms = this->getGlobalELFSyms<ELFT>();
-  for (size_t i = 0; i < syms.size(); ++i) {
+  for (size_t i = 0, e = syms.size(); i != e; ++i) {
     const Elf_Sym &sym = syms[i];
 
     // ELF spec requires that all local symbols precede weak or global
     // symbols in each symbol table, and the index of first non-local symbol
     // is stored to sh_info. If a local symbol appears after some non-local
     // symbol, that's a violation of the spec.
-    StringRef name = CHECK(sym.getName(this->stringTable), this);
+    StringRef name = CHECK(sym.getName(stringTable), this);
     if (sym.getBinding() == STB_LOCAL) {
       warn("found local symbol '" + name +
            "' in global part of symbol table in file " + toString(this));
@@ -1528,15 +1524,15 @@ template <class ELFT> void SharedFile::parse() {
                 toString(this));
           continue;
         }
-        StringRef verName = this->stringTable.data() + verneeds[idx];
+        StringRef verName = stringTable.data() + verneeds[idx];
         versionedNameBuffer.clear();
         name =
             saver.save((name + "@" + verName).toStringRef(versionedNameBuffer));
       }
-      Symbol *s = symtab->addSymbol(
+      Symbol *s = symtab.addSymbol(
           Undefined{this, name, sym.getBinding(), sym.st_other, sym.getType()});
       s->exportDynamic = true;
-      if (s->isUndefined() && !s->isWeak() &&
+      if (s->isUndefined() && sym.getBinding() != STB_WEAK &&
           config->unresolvedSymbolsInShlib != UnresolvedPolicy::Ignore)
         requiredSymbols.push_back(s);
       continue;
@@ -1551,9 +1547,9 @@ template <class ELFT> void SharedFile::parse() {
 
     uint32_t alignment = getAlignment<ELFT>(sections, sym);
     if (!(versyms[i] & VERSYM_HIDDEN)) {
-      symtab->addSymbol(SharedSymbol{*this, name, sym.getBinding(),
-                                     sym.st_other, sym.getType(), sym.st_value,
-                                     sym.st_size, alignment, idx});
+      symtab.addSymbol(SharedSymbol{*this, name, sym.getBinding(), sym.st_other,
+                                    sym.getType(), sym.st_value, sym.st_size,
+                                    alignment, idx});
     }
 
     // Also add the symbol with the versioned name to handle undefined symbols
@@ -1569,13 +1565,13 @@ template <class ELFT> void SharedFile::parse() {
     }
 
     StringRef verName =
-        this->stringTable.data() +
+        stringTable.data() +
         reinterpret_cast<const Elf_Verdef *>(verdefs[idx])->getAux()->vda_name;
     versionedNameBuffer.clear();
     name = (name + "@" + verName).toStringRef(versionedNameBuffer);
-    symtab->addSymbol(SharedSymbol{*this, saver.save(name), sym.getBinding(),
-                                   sym.st_other, sym.getType(), sym.st_value,
-                                   sym.st_size, alignment, idx});
+    symtab.addSymbol(SharedSymbol{*this, saver.save(name), sym.getBinding(),
+                                  sym.st_other, sym.getType(), sym.st_value,
+                                  sym.st_size, alignment, idx});
   }
 }
 
@@ -1641,9 +1637,10 @@ static uint8_t getOsAbi(const Triple &t) {
 }
 
 BitcodeFile::BitcodeFile(MemoryBufferRef mb, StringRef archiveName,
-                         uint64_t offsetInArchive)
+                         uint64_t offsetInArchive, bool lazy)
     : InputFile(BitcodeKind, mb) {
-  this->archiveName = std::string(archiveName);
+  this->archiveName = archiveName;
+  this->lazy = lazy;
 
   std::string path = mb.getBufferIdentifier().str();
   if (config->thinLTOIndexOnly)
@@ -1722,13 +1719,22 @@ template <class ELFT> void BitcodeFile::parse() {
             .second);
   }
 
-  for (const lto::InputFile::Symbol &objSym : obj->symbols())
-    symbols.push_back(createBitcodeSymbol<ELFT>(keptComdats, objSym, *this));
+  symbols.assign(obj->symbols().size(), nullptr);
+  for (auto it : llvm::enumerate(obj->symbols()))
+    symbols[it.index()] =
+        createBitcodeSymbol<ELFT>(keptComdats, it.value(), *this);
 
   for (auto l : obj->getDependentLibraries())
     addDependentLibrary(l, this);
 }
 
+void BitcodeFile::parseLazy() {
+  SymbolTable &symtab = *elf::symtab;
+  for (const lto::InputFile::Symbol &sym : obj->symbols())
+    if (!sym.isUndefined())
+      symtab.addSymbol(LazyObject{*this, saver.save(sym.getName())});
+}
+
 void BinaryFile::parse() {
   ArrayRef<uint8_t> data = arrayRefFromStringRef(mb.getBuffer());
   auto *section = make<InputSection>(this, SHF_ALLOC | SHF_WRITE, SHT_PROGBITS,
@@ -1755,7 +1761,7 @@ void BinaryFile::parse() {
 InputFile *elf::createObjectFile(MemoryBufferRef mb, StringRef archiveName,
                                  uint64_t offsetInArchive) {
   if (isBitcode(mb))
-    return make<BitcodeFile>(mb, archiveName, offsetInArchive);
+    return make<BitcodeFile>(mb, archiveName, offsetInArchive, /*lazy=*/false);
 
   switch (getELFKind(mb, archiveName)) {
   case ELF32LEKind:
@@ -1771,80 +1777,40 @@ InputFile *elf::createObjectFile(MemoryBufferRef mb, StringRef archiveName,
   }
 }
 
-void LazyObjFile::extract() {
-  if (extracted)
-    return;
-  extracted = true;
-
-  InputFile *file = createObjectFile(mb, archiveName, offsetInArchive);
-  file->groupId = groupId;
-
-  // Copy symbol vector so that the new InputFile doesn't have to
-  // insert the same defined symbols to the symbol table again.
-  file->symbols = std::move(symbols);
+InputFile *elf::createLazyFile(MemoryBufferRef mb, StringRef archiveName,
+                               uint64_t offsetInArchive) {
+  if (isBitcode(mb))
+    return make<BitcodeFile>(mb, archiveName, offsetInArchive, /*lazy=*/true);
 
-  parseFile(file);
+  auto *file =
+      cast<ELFFileBase>(createObjectFile(mb, archiveName, offsetInArchive));
+  file->lazy = true;
+  return file;
 }
 
-template <class ELFT> void LazyObjFile::parse() {
-  using Elf_Sym = typename ELFT::Sym;
+template <class ELFT> void ObjFile<ELFT>::parseLazy() {
+  const ArrayRef<typename ELFT::Sym> eSyms = this->getELFSyms<ELFT>();
+  SymbolTable &symtab = *elf::symtab;
 
-  // A lazy object file wraps either a bitcode file or an ELF file.
-  if (isBitcode(this->mb)) {
-    std::unique_ptr<lto::InputFile> obj =
-        CHECK(lto::InputFile::create(this->mb), this);
-    for (const lto::InputFile::Symbol &sym : obj->symbols()) {
-      if (sym.isUndefined())
-        continue;
-      symtab->addSymbol(LazyObject{*this, saver.save(sym.getName())});
-    }
-    return;
-  }
-
-  if (getELFKind(this->mb, archiveName) != config->ekind) {
-    error("incompatible file: " + this->mb.getBufferIdentifier());
-    return;
-  }
-
-  // Find a symbol table.
-  ELFFile<ELFT> obj = check(ELFFile<ELFT>::create(mb.getBuffer()));
-  ArrayRef<typename ELFT::Shdr> sections = CHECK(obj.sections(), this);
-
-  for (const typename ELFT::Shdr &sec : sections) {
-    if (sec.sh_type != SHT_SYMTAB)
-      continue;
+  symbols.resize(eSyms.size());
+  for (size_t i = firstGlobal, end = eSyms.size(); i != end; ++i)
+    if (eSyms[i].st_shndx != SHN_UNDEF)
+      symbols[i] = symtab.insert(CHECK(eSyms[i].getName(stringTable), this));
 
-    // A symbol table is found.
-    ArrayRef<Elf_Sym> eSyms = CHECK(obj.symbols(&sec), this);
-    uint32_t firstGlobal = sec.sh_info;
-    StringRef strtab = CHECK(obj.getStringTableForSymtab(sec, sections), this);
-    this->symbols.resize(eSyms.size());
-
-    // Get existing symbols or insert placeholder symbols.
-    for (size_t i = firstGlobal, end = eSyms.size(); i != end; ++i)
-      if (eSyms[i].st_shndx != SHN_UNDEF)
-        this->symbols[i] = symtab->insert(CHECK(eSyms[i].getName(strtab), this));
-
-    // Replace existing symbols with LazyObject symbols.
-    //
-    // resolve() may trigger this->extract() if an existing symbol is an
-    // undefined symbol. If that happens, this LazyObjFile has served
-    // its purpose, and we can exit from the loop early.
-    for (Symbol *sym : this->symbols) {
-      if (!sym)
-        continue;
+  // Replace existing symbols with LazyObject symbols.
+  //
+  // resolve() may trigger this->extract() if an existing symbol is an undefined
+  // symbol. If that happens, this function has served its purpose, and we can
+  // exit from the loop early.
+  for (Symbol *sym : makeArrayRef(symbols).slice(firstGlobal))
+    if (sym) {
       sym->resolve(LazyObject{*this, sym->getName()});
-
-      // If extracted, stop iterating because this->symbols has been transferred
-      // to the instantiated ObjFile.
-      if (extracted)
+      if (!lazy)
         return;
     }
-    return;
-  }
 }
 
-bool LazyObjFile::shouldExtractForCommon(const StringRef &name) {
+bool InputFile::shouldExtractForCommon(StringRef name) {
   if (isBitcode(mb))
     return isBitcodeNonCommonDef(mb, name, archiveName);
 
@@ -1865,11 +1831,6 @@ template void BitcodeFile::parse<ELF32BE>();
 template void BitcodeFile::parse<ELF64LE>();
 template void BitcodeFile::parse<ELF64BE>();
 
-template void LazyObjFile::parse<ELF32LE>();
-template void LazyObjFile::parse<ELF32BE>();
-template void LazyObjFile::parse<ELF64LE>();
-template void LazyObjFile::parse<ELF64BE>();
-
 template class elf::ObjFile<ELF32LE>;
 template class elf::ObjFile<ELF32BE>;
 template class elf::ObjFile<ELF64LE>;
diff --git a/lld/ELF/InputFiles.h b/lld/ELF/InputFiles.h
index 5bbfb7656e47..d622390fcade 100644
--- a/lld/ELF/InputFiles.h
+++ b/lld/ELF/InputFiles.h
@@ -41,6 +41,7 @@ namespace elf {
 
 using llvm::object::Archive;
 
+class InputSection;
 class Symbol;
 
 // If --reproduce is specified, all input files are written to this tar archive.
@@ -54,8 +55,15 @@ void parseFile(InputFile *file);
 
 // The root class of input files.
 class InputFile {
+private:
+  // Cache for getNameForScript().
+  mutable SmallString<0> nameForScriptCache;
+
+protected:
+  SmallVector<InputSectionBase *, 0> sections;
+
 public:
-  enum Kind {
+  enum Kind : uint8_t {
     ObjKind,
     SharedKind,
     LazyObjKind,
@@ -83,9 +91,7 @@ public:
 
   // Returns object file symbols. It is a runtime error to call this
   // function on files of other types.
-  ArrayRef<Symbol *> getSymbols() { return getMutableSymbols(); }
-
-  MutableArrayRef<Symbol *> getMutableSymbols() {
+  ArrayRef<Symbol *> getSymbols() const {
     assert(fileKind == BinaryKind || fileKind == ObjKind ||
            fileKind == BitcodeKind);
     return symbols;
@@ -94,29 +100,51 @@ public:
   // Get filename to use for linker script processing.
   StringRef getNameForScript() const;
 
+  // Check if a non-common symbol should be extracted to override a common
+  // definition.
+  bool shouldExtractForCommon(StringRef name);
+
   // If not empty, this stores the name of the archive containing this file.
   // We use this string for creating error messages.
-  std::string archiveName;
+  SmallString<0> archiveName;
+
+  // Cache for toString(). Only toString() should use this member.
+  mutable SmallString<0> toStringCache;
+
+  SmallVector<Symbol *, 0> symbols;
+
+  // .got2 in the current file. This is used by PPC32 -fPIC/-fPIE to compute
+  // offsets in PLT call stubs.
+  InputSection *ppc32Got2 = nullptr;
+
+  // Index of MIPS GOT built for this file.
+  uint32_t mipsGotIndex = -1;
+
+  // groupId is used for --warn-backrefs which is an optional error
+  // checking feature. All files within the same --{start,end}-group or
+  // --{start,end}-lib get the same group ID. Otherwise, each file gets a new
+  // group ID. For more info, see checkDependency() in SymbolTable.cpp.
+  uint32_t groupId;
+  static bool isInGroup;
+  static uint32_t nextGroupId;
 
   // If this is an architecture-specific file, the following members
   // have ELF type (i.e. ELF{32,64}{LE,BE}) and target machine type.
-  ELFKind ekind = ELFNoneKind;
   uint16_t emachine = llvm::ELF::EM_NONE;
+  const Kind fileKind;
+  ELFKind ekind = ELFNoneKind;
   uint8_t osabi = 0;
   uint8_t abiVersion = 0;
 
-  // Cache for toString(). Only toString() should use this member.
-  mutable std::string toStringCache;
-
-  std::string getSrcMsg(const Symbol &sym, InputSectionBase &sec,
-                        uint64_t offset);
+  // True if this is a relocatable object file/bitcode file between --start-lib
+  // and --end-lib.
+  bool lazy = false;
 
   // True if this is an argument for --just-symbols. Usually false.
   bool justSymbols = false;
 
-  // outSecOff of .got2 in the current file. This is used by PPC32 -fPIC/-fPIE
-  // to compute offsets in PLT call stubs.
-  uint32_t ppc32Got2OutSecOff = 0;
+  std::string getSrcMsg(const Symbol &sym, InputSectionBase &sec,
+                        uint64_t offset);
 
   // On PPC64 we need to keep track of which files contain small code model
   // relocations that access the .toc section. To minimize the chance of a
@@ -133,28 +161,8 @@ public:
   // R_PPC64_TLSLD. Disable TLS relaxation to avoid bad code generation.
   bool ppc64DisableTLSRelax = false;
 
-  // groupId is used for --warn-backrefs which is an optional error
-  // checking feature. All files within the same --{start,end}-group or
-  // --{start,end}-lib get the same group ID. Otherwise, each file gets a new
-  // group ID. For more info, see checkDependency() in SymbolTable.cpp.
-  uint32_t groupId;
-  static bool isInGroup;
-  static uint32_t nextGroupId;
-
-  // Index of MIPS GOT built for this file.
-  llvm::Optional<size_t> mipsGotIndex;
-
-  std::vector<Symbol *> symbols;
-
 protected:
   InputFile(Kind k, MemoryBufferRef m);
-  std::vector<InputSectionBase *> sections;
-
-private:
-  const Kind fileKind;
-
-  // Cache for getNameForScript().
-  mutable std::string nameForScriptCache;
 };
 
 class ELFFileBase : public InputFile {
@@ -176,7 +184,15 @@ public:
   ArrayRef<Symbol *> getGlobalSymbols() {
     return llvm::makeArrayRef(symbols).slice(firstGlobal);
   }
+  MutableArrayRef<Symbol *> getMutableGlobalSymbols() {
+    return llvm::makeMutableArrayRef(symbols.data(), symbols.size())
+        .slice(firstGlobal);
+  }
 
+  template <typename ELFT> typename ELFT::ShdrRange getELFShdrs() const {
+    return typename ELFT::ShdrRange(
+        reinterpret_cast<const typename ELFT::Shdr *>(elfShdrs), numELFShdrs);
+  }
   template <typename ELFT> typename ELFT::SymRange getELFSyms() const {
     return typename ELFT::SymRange(
         reinterpret_cast<const typename ELFT::Sym *>(elfSyms), numELFSyms);
@@ -189,10 +205,15 @@ protected:
   // Initializes this class's member variables.
   template <typename ELFT> void init();
 
+  StringRef stringTable;
+  const void *elfShdrs = nullptr;
   const void *elfSyms = nullptr;
-  size_t numELFSyms = 0;
+  uint32_t numELFShdrs = 0;
+  uint32_t numELFSyms = 0;
   uint32_t firstGlobal = 0;
-  StringRef stringTable;
+
+public:
+  bool hasCommonSyms = false;
 };
 
 // .o file.
@@ -207,10 +228,11 @@ public:
   }
 
   ObjFile(MemoryBufferRef m, StringRef archiveName) : ELFFileBase(ObjKind, m) {
-    this->archiveName = std::string(archiveName);
+    this->archiveName = archiveName;
   }
 
   void parse(bool ignoreComdats = false);
+  void parseLazy();
 
   StringRef getShtGroupSignature(ArrayRef<Elf_Shdr> sections,
                                  const Elf_Shdr &sec);
@@ -231,6 +253,17 @@ public:
   llvm::Optional<llvm::DILineInfo> getDILineInfo(InputSectionBase *, uint64_t);
   llvm::Optional<std::pair<std::string, unsigned>> getVariableLoc(StringRef name);
 
+  // Name of source file obtained from STT_FILE symbol value,
+  // or empty string if there is no such symbol in object file
+  // symbol table.
+  StringRef sourceFile;
+
+  // Pointer to this input file's .llvm_addrsig section, if it has one.
+  const Elf_Shdr *addrsigSec = nullptr;
+
+  // SHT_LLVM_CALL_GRAPH_PROFILE section index.
+  uint32_t cgProfileSectionIndex = 0;
+
   // MIPS GP0 value defined by this file. This value represents the gp value
   // used to create the relocatable object and required to support
   // R_MIPS_GPREL16 / R_MIPS_GPREL32 relocations.
@@ -238,11 +271,6 @@ public:
 
   uint32_t andFeatures = 0;
 
-  // Name of source file obtained from STT_FILE symbol value,
-  // or empty string if there is no such symbol in object file
-  // symbol table.
-  StringRef sourceFile;
-
   // True if the file defines functions compiled with
   // -fsplit-stack. Usually false.
   bool splitStack = false;
@@ -251,12 +279,6 @@ public:
   // but had one or more functions with the no_split_stack attribute.
   bool someNoSplitStack = false;
 
-  // Pointer to this input file's .llvm_addrsig section, if it has one.
-  const Elf_Shdr *addrsigSec = nullptr;
-
-  // SHT_LLVM_CALL_GRAPH_PROFILE section index.
-  uint32_t cgProfileSectionIndex = 0;
-
   // Get cached DWARF information.
   DWARFCache *getDwarf();
 
@@ -294,36 +316,6 @@ private:
   llvm::once_flag initDwarf;
 };
 
-// LazyObjFile is analogous to ArchiveFile in the sense that
-// the file contains lazy symbols. The difference is that
-// LazyObjFile wraps a single file instead of multiple files.
-//
-// This class is used for --start-lib and --end-lib options which
-// instruct the linker to link object files between them with the
-// archive file semantics.
-class LazyObjFile : public InputFile {
-public:
-  LazyObjFile(MemoryBufferRef m, StringRef archiveName,
-              uint64_t offsetInArchive)
-      : InputFile(LazyObjKind, m), offsetInArchive(offsetInArchive) {
-    this->archiveName = std::string(archiveName);
-  }
-
-  static bool classof(const InputFile *f) { return f->kind() == LazyObjKind; }
-
-  template <class ELFT> void parse();
-  void extract();
-
-  // Check if a non-common symbol should be extracted to override a common
-  // definition.
-  bool shouldExtractForCommon(const StringRef &name);
-
-  bool extracted = false;
-
-private:
-  uint64_t offsetInArchive;
-};
-
 // An ArchiveFile object represents a .a file.
 class ArchiveFile : public InputFile {
 public:
@@ -354,9 +346,10 @@ private:
 class BitcodeFile : public InputFile {
 public:
   BitcodeFile(MemoryBufferRef m, StringRef archiveName,
-              uint64_t offsetInArchive);
+              uint64_t offsetInArchive, bool lazy);
   static bool classof(const InputFile *f) { return f->kind() == BitcodeKind; }
   template <class ELFT> void parse();
+  void parseLazy();
   std::unique_ptr<llvm::lto::InputFile> obj;
 };
 
@@ -368,16 +361,16 @@ public:
         isNeeded(!config->asNeeded) {}
 
   // This is actually a vector of Elf_Verdef pointers.
-  std::vector<const void *> verdefs;
+  SmallVector<const void *, 0> verdefs;
 
   // If the output file needs Elf_Verneed data structures for this file, this is
   // a vector of Elf_Vernaux version identifiers that map onto the entries in
   // Verdefs, otherwise it is empty.
-  std::vector<unsigned> vernauxs;
+  SmallVector<uint32_t, 0> vernauxs;
 
   static unsigned vernauxNum;
 
-  std::vector<StringRef> dtNeeded;
+  SmallVector<StringRef, 0> dtNeeded;
   StringRef soName;
 
   static bool classof(const InputFile *f) { return f->kind() == SharedKind; }
@@ -389,7 +382,7 @@ public:
 
   // Non-weak undefined symbols which are not yet resolved when the SO is
   // parsed. Only filled for `--no-allow-shlib-undefined`.
-  std::vector<Symbol *> requiredSymbols;
+  SmallVector<Symbol *, 0> requiredSymbols;
 
 private:
   template <typename ELFT>
@@ -406,6 +399,8 @@ public:
 
 InputFile *createObjectFile(MemoryBufferRef mb, StringRef archiveName = "",
                             uint64_t offsetInArchive = 0);
+InputFile *createLazyFile(MemoryBufferRef mb, StringRef archiveName,
+                          uint64_t offsetInArchive);
 
 inline bool isBitcode(MemoryBufferRef mb) {
   return identify_magic(mb.getBuffer()) == llvm::file_magic::bitcode;
@@ -413,12 +408,12 @@ inline bool isBitcode(MemoryBufferRef mb) {
 
 std::string replaceThinLTOSuffix(StringRef path);
 
-extern std::vector<ArchiveFile *> archiveFiles;
-extern std::vector<BinaryFile *> binaryFiles;
-extern std::vector<BitcodeFile *> bitcodeFiles;
-extern std::vector<LazyObjFile *> lazyObjFiles;
-extern std::vector<InputFile *> objectFiles;
-extern std::vector<SharedFile *> sharedFiles;
+extern SmallVector<ArchiveFile *, 0> archiveFiles;
+extern SmallVector<BinaryFile *, 0> binaryFiles;
+extern SmallVector<BitcodeFile *, 0> bitcodeFiles;
+extern SmallVector<BitcodeFile *, 0> lazyBitcodeFiles;
+extern SmallVector<ELFFileBase *, 0> objectFiles;
+extern SmallVector<SharedFile *, 0> sharedFiles;
 
 } // namespace elf
 } // namespace lld
diff --git a/lld/ELF/InputSection.cpp b/lld/ELF/InputSection.cpp
index 4d5bd1f1e5f2..e1ee3def89f3 100644
--- a/lld/ELF/InputSection.cpp
+++ b/lld/ELF/InputSection.cpp
@@ -40,7 +40,7 @@ using namespace llvm::sys;
 using namespace lld;
 using namespace lld::elf;
 
-std::vector<InputSectionBase *> elf::inputSections;
+SmallVector<InputSectionBase *, 0> elf::inputSections;
 DenseSet<std::pair<const Symbol *, uint64_t>> elf::ppc64noTocRelax;
 
 // Returns a string to construct an error message.
@@ -163,16 +163,16 @@ template <class ELFT> RelsOrRelas<ELFT> InputSectionBase::relsOrRelas() const {
   if (relSecIdx == 0)
     return {};
   RelsOrRelas<ELFT> ret;
-  const ELFFile<ELFT> obj = cast<ELFFileBase>(file)->getObj<ELFT>();
-  typename ELFT::Shdr shdr = cantFail(obj.sections())[relSecIdx];
+  typename ELFT::Shdr shdr =
+      cast<ELFFileBase>(file)->getELFShdrs<ELFT>()[relSecIdx];
   if (shdr.sh_type == SHT_REL) {
     ret.rels = makeArrayRef(reinterpret_cast<const typename ELFT::Rel *>(
-                                obj.base() + shdr.sh_offset),
+                                file->mb.getBufferStart() + shdr.sh_offset),
                             shdr.sh_size / sizeof(typename ELFT::Rel));
   } else {
     assert(shdr.sh_type == SHT_RELA);
     ret.relas = makeArrayRef(reinterpret_cast<const typename ELFT::Rela *>(
-                                 obj.base() + shdr.sh_offset),
+                                 file->mb.getBufferStart() + shdr.sh_offset),
                              shdr.sh_size / sizeof(typename ELFT::Rela));
   }
   return ret;
@@ -325,7 +325,7 @@ std::string InputSectionBase::getObjMsg(uint64_t off) {
 
   std::string archive;
   if (!file->archiveName.empty())
-    archive = " in archive " + file->archiveName;
+    archive = (" in archive " + file->archiveName).str();
 
   // Find a symbol that encloses a given location.
   for (Symbol *b : file->getSymbols())
@@ -395,6 +395,7 @@ InputSectionBase *InputSection::getRelocatedSection() const {
 // for each relocation. So we copy relocations one by one.
 template <class ELFT, class RelTy>
 void InputSection::copyRelocations(uint8_t *buf, ArrayRef<RelTy> rels) {
+  const TargetInfo &target = *elf::target;
   InputSectionBase *sec = getRelocatedSection();
 
   for (const RelTy &rel : rels) {
@@ -433,8 +434,7 @@ void InputSection::copyRelocations(uint8_t *buf, ArrayRef<RelTy> rels) {
             sec->name != ".gcc_except_table" && sec->name != ".got2" &&
             sec->name != ".toc") {
           uint32_t secIdx = cast<Undefined>(sym).discardedSecIdx;
-          Elf_Shdr_Impl<ELFT> sec =
-              CHECK(file->getObj().sections(), file)[secIdx];
+          Elf_Shdr_Impl<ELFT> sec = file->template getELFShdrs<ELFT>()[secIdx];
           warn("relocation refers to a discarded section: " +
                CHECK(file->getObj().getSectionName(sec), file) +
                "\n>>> referenced by " + getObjMsg(p->r_offset));
@@ -442,7 +442,7 @@ void InputSection::copyRelocations(uint8_t *buf, ArrayRef<RelTy> rels) {
         p->setSymbolAndType(0, 0, false);
         continue;
       }
-      SectionBase *section = d->section->repl;
+      SectionBase *section = d->section;
       if (!section->isLive()) {
         p->setSymbolAndType(0, 0, false);
         continue;
@@ -451,10 +451,10 @@ void InputSection::copyRelocations(uint8_t *buf, ArrayRef<RelTy> rels) {
       int64_t addend = getAddend<ELFT>(rel);
       const uint8_t *bufLoc = sec->data().begin() + rel.r_offset;
       if (!RelTy::IsRela)
-        addend = target->getImplicitAddend(bufLoc, type);
+        addend = target.getImplicitAddend(bufLoc, type);
 
       if (config->emachine == EM_MIPS &&
-          target->getRelExpr(type, sym, bufLoc) == R_MIPS_GOTREL) {
+          target.getRelExpr(type, sym, bufLoc) == R_MIPS_GOTREL) {
         // Some MIPS relocations depend on "gp" value. By default,
         // this value has 0x7ff0 offset from a .got section. But
         // relocatable files produced by a compiler or a linker
@@ -471,16 +471,16 @@ void InputSection::copyRelocations(uint8_t *buf, ArrayRef<RelTy> rels) {
 
       if (RelTy::IsRela)
         p->r_addend = sym.getVA(addend) - section->getOutputSection()->addr;
-      else if (config->relocatable && type != target->noneRel)
+      else if (config->relocatable && type != target.noneRel)
         sec->relocations.push_back({R_ABS, type, rel.r_offset, addend, &sym});
     } else if (config->emachine == EM_PPC && type == R_PPC_PLTREL24 &&
-               p->r_addend >= 0x8000) {
+               p->r_addend >= 0x8000 && sec->file->ppc32Got2) {
       // Similar to R_MIPS_GPREL{16,32}. If the addend of R_PPC_PLTREL24
       // indicates that r30 is relative to the input section .got2
       // (r_addend>=0x8000), after linking, r30 should be relative to the output
       // section .got2 . To compensate for the shift, adjust r_addend by
-      // ppc32Got2OutSecOff.
-      p->r_addend += sec->file->ppc32Got2OutSecOff;
+      // ppc32Got->outSecOff.
+      p->r_addend += sec->file->ppc32Got2->outSecOff;
     }
   }
 }
@@ -865,6 +865,7 @@ uint64_t InputSectionBase::getRelocTargetVA(const InputFile *file, RelType type,
 template <class ELFT, class RelTy>
 void InputSection::relocateNonAlloc(uint8_t *buf, ArrayRef<RelTy> rels) {
   const unsigned bits = sizeof(typename ELFT::uint) * 8;
+  const TargetInfo &target = *elf::target;
   const bool isDebug = isDebugSection(*this);
   const bool isDebugLocOrRanges =
       isDebug && (name == ".debug_loc" || name == ".debug_ranges");
@@ -890,16 +891,16 @@ void InputSection::relocateNonAlloc(uint8_t *buf, ArrayRef<RelTy> rels) {
     uint8_t *bufLoc = buf + offset;
     int64_t addend = getAddend<ELFT>(rel);
     if (!RelTy::IsRela)
-      addend += target->getImplicitAddend(bufLoc, type);
+      addend += target.getImplicitAddend(bufLoc, type);
 
     Symbol &sym = getFile<ELFT>()->getRelocTargetSym(rel);
-    RelExpr expr = target->getRelExpr(type, sym, bufLoc);
+    RelExpr expr = target.getRelExpr(type, sym, bufLoc);
     if (expr == R_NONE)
       continue;
 
     if (expr == R_SIZE) {
-      target->relocateNoSym(bufLoc, type,
-                            SignExtend64<bits>(sym.getSize() + addend));
+      target.relocateNoSym(bufLoc, type,
+                           SignExtend64<bits>(sym.getSize() + addend));
       continue;
     }
 
@@ -923,14 +924,14 @@ void InputSection::relocateNonAlloc(uint8_t *buf, ArrayRef<RelTy> rels) {
       // address 0. For bug-compatibilty, we accept them with warnings. We
       // know Steel Bank Common Lisp as of 2018 have this bug.
       warn(msg);
-      target->relocateNoSym(
+      target.relocateNoSym(
           bufLoc, type,
           SignExtend64<bits>(sym.getVA(addend - offset - outSecOff)));
       continue;
     }
 
     if (tombstone ||
-        (isDebug && (type == target->symbolicRel || expr == R_DTPREL))) {
+        (isDebug && (type == target.symbolicRel || expr == R_DTPREL))) {
       // Resolve relocations in .debug_* referencing (discarded symbols or ICF
       // folded section symbols) to a tombstone value. Resolving to addend is
       // unsatisfactory because the result address range may collide with a
@@ -948,10 +949,10 @@ void InputSection::relocateNonAlloc(uint8_t *buf, ArrayRef<RelTy> rels) {
       //
       // If the referenced symbol is discarded (made Undefined), or the
       // section defining the referenced symbol is garbage collected,
-      // sym.getOutputSection() is nullptr. `ds->section->repl != ds->section`
-      // catches the ICF folded case. However, resolving a relocation in
-      // .debug_line to -1 would stop debugger users from setting breakpoints on
-      // the folded-in function, so exclude .debug_line.
+      // sym.getOutputSection() is nullptr. `ds->folded` catches the ICF folded
+      // case. However, resolving a relocation in .debug_line to -1 would stop
+      // debugger users from setting breakpoints on the folded-in function, so
+      // exclude .debug_line.
       //
       // For pre-DWARF-v5 .debug_loc and .debug_ranges, -1 is a reserved value
       // (base address selection entry), use 1 (which is used by GNU ld for
@@ -960,16 +961,15 @@ void InputSection::relocateNonAlloc(uint8_t *buf, ArrayRef<RelTy> rels) {
       // TODO To reduce disruption, we use 0 instead of -1 as the tombstone
       // value. Enable -1 in a future release.
       auto *ds = dyn_cast<Defined>(&sym);
-      if (!sym.getOutputSection() ||
-          (ds && ds->section->repl != ds->section && !isDebugLine)) {
+      if (!sym.getOutputSection() || (ds && ds->folded && !isDebugLine)) {
         // If -z dead-reloc-in-nonalloc= is specified, respect it.
         const uint64_t value = tombstone ? SignExtend64<bits>(*tombstone)
                                          : (isDebugLocOrRanges ? 1 : 0);
-        target->relocateNoSym(bufLoc, type, value);
+        target.relocateNoSym(bufLoc, type, value);
         continue;
       }
     }
-    target->relocateNoSym(bufLoc, type, SignExtend64<bits>(sym.getVA(addend)));
+    target.relocateNoSym(bufLoc, type, SignExtend64<bits>(sym.getVA(addend)));
   }
 }
 
@@ -992,7 +992,7 @@ static void relocateNonAllocForRelocatable(InputSection *sec, uint8_t *buf) {
 
 template <class ELFT>
 void InputSectionBase::relocate(uint8_t *buf, uint8_t *bufEnd) {
-  if (flags & SHF_EXECINSTR)
+  if ((flags & SHF_EXECINSTR) && LLVM_UNLIKELY(getFile<ELFT>()->splitStack))
     adjustSplitStackFunctionPrologues<ELFT>(buf, bufEnd);
 
   if (flags & SHF_ALLOC) {
@@ -1015,6 +1015,7 @@ void InputSectionBase::relocate(uint8_t *buf, uint8_t *bufEnd) {
 void InputSectionBase::relocateAlloc(uint8_t *buf, uint8_t *bufEnd) {
   assert(flags & SHF_ALLOC);
   const unsigned bits = config->wordsize * 8;
+  const TargetInfo &target = *elf::target;
   uint64_t lastPPCRelaxedRelocOff = UINT64_C(-1);
 
   for (const Relocation &rel : relocations) {
@@ -1022,20 +1023,18 @@ void InputSectionBase::relocateAlloc(uint8_t *buf, uint8_t *bufEnd) {
       continue;
     uint64_t offset = rel.offset;
     uint8_t *bufLoc = buf + offset;
-    RelType type = rel.type;
 
     uint64_t addrLoc = getOutputSection()->addr + offset;
     if (auto *sec = dyn_cast<InputSection>(this))
       addrLoc += sec->outSecOff;
-    RelExpr expr = rel.expr;
-    uint64_t targetVA = SignExtend64(
-        getRelocTargetVA(file, type, rel.addend, addrLoc, *rel.sym, expr),
-        bits);
+    const uint64_t targetVA =
+        SignExtend64(getRelocTargetVA(file, rel.type, rel.addend, addrLoc,
+                                      *rel.sym, rel.expr), bits);
 
-    switch (expr) {
+    switch (rel.expr) {
     case R_RELAX_GOT_PC:
     case R_RELAX_GOT_PC_NOPIC:
-      target->relaxGot(bufLoc, rel, targetVA);
+      target.relaxGot(bufLoc, rel, targetVA);
       break;
     case R_PPC64_RELAX_GOT_PC: {
       // The R_PPC64_PCREL_OPT relocation must appear immediately after
@@ -1044,11 +1043,11 @@ void InputSectionBase::relocateAlloc(uint8_t *buf, uint8_t *bufEnd) {
       // the associated R_PPC64_GOT_PCREL34 since only the latter has an
       // associated symbol. So save the offset when relaxing R_PPC64_GOT_PCREL34
       // and only relax the other if the saved offset matches.
-      if (type == R_PPC64_GOT_PCREL34)
+      if (rel.type == R_PPC64_GOT_PCREL34)
         lastPPCRelaxedRelocOff = offset;
-      if (type == R_PPC64_PCREL_OPT && offset != lastPPCRelaxedRelocOff)
+      if (rel.type == R_PPC64_PCREL_OPT && offset != lastPPCRelaxedRelocOff)
         break;
-      target->relaxGot(bufLoc, rel, targetVA);
+      target.relaxGot(bufLoc, rel, targetVA);
       break;
     }
     case R_PPC64_RELAX_TOC:
@@ -1059,25 +1058,25 @@ void InputSectionBase::relocateAlloc(uint8_t *buf, uint8_t *bufEnd) {
       // opportunities but is safe.
       if (ppc64noTocRelax.count({rel.sym, rel.addend}) ||
           !tryRelaxPPC64TocIndirection(rel, bufLoc))
-        target->relocate(bufLoc, rel, targetVA);
+        target.relocate(bufLoc, rel, targetVA);
       break;
     case R_RELAX_TLS_IE_TO_LE:
-      target->relaxTlsIeToLe(bufLoc, rel, targetVA);
+      target.relaxTlsIeToLe(bufLoc, rel, targetVA);
       break;
     case R_RELAX_TLS_LD_TO_LE:
     case R_RELAX_TLS_LD_TO_LE_ABS:
-      target->relaxTlsLdToLe(bufLoc, rel, targetVA);
+      target.relaxTlsLdToLe(bufLoc, rel, targetVA);
       break;
     case R_RELAX_TLS_GD_TO_LE:
     case R_RELAX_TLS_GD_TO_LE_NEG:
-      target->relaxTlsGdToLe(bufLoc, rel, targetVA);
+      target.relaxTlsGdToLe(bufLoc, rel, targetVA);
       break;
     case R_AARCH64_RELAX_TLS_GD_TO_IE_PAGE_PC:
     case R_RELAX_TLS_GD_TO_IE:
     case R_RELAX_TLS_GD_TO_IE_ABS:
     case R_RELAX_TLS_GD_TO_IE_GOT_OFF:
     case R_RELAX_TLS_GD_TO_IE_GOTPLT:
-      target->relaxTlsGdToIe(bufLoc, rel, targetVA);
+      target.relaxTlsGdToIe(bufLoc, rel, targetVA);
       break;
     case R_PPC64_CALL:
       // If this is a call to __tls_get_addr, it may be part of a TLS
@@ -1102,10 +1101,10 @@ void InputSectionBase::relocateAlloc(uint8_t *buf, uint8_t *bufEnd) {
         }
         write32(bufLoc + 4, 0xe8410018); // ld %r2, 24(%r1)
       }
-      target->relocate(bufLoc, rel, targetVA);
+      target.relocate(bufLoc, rel, targetVA);
       break;
     default:
-      target->relocate(bufLoc, rel, targetVA);
+      target.relocate(bufLoc, rel, targetVA);
       break;
     }
   }
@@ -1118,7 +1117,7 @@ void InputSectionBase::relocateAlloc(uint8_t *buf, uint8_t *bufEnd) {
     for (const JumpInstrMod &jumpMod : jumpInstrMods) {
       uint64_t offset = jumpMod.offset;
       uint8_t *bufLoc = buf + offset;
-      target->applyJumpInstrMod(bufLoc, jumpMod.original, jumpMod.size);
+      target.applyJumpInstrMod(bufLoc, jumpMod.original, jumpMod.size);
     }
   }
 }
@@ -1175,8 +1174,6 @@ static bool enclosingPrologueAttempted(uint64_t offset,
 template <class ELFT>
 void InputSectionBase::adjustSplitStackFunctionPrologues(uint8_t *buf,
                                                          uint8_t *end) {
-  if (!getFile<ELFT>()->splitStack)
-    return;
   DenseSet<Defined *> prologues;
   std::vector<Relocation *> morestackCalls;
 
@@ -1229,27 +1226,26 @@ void InputSectionBase::adjustSplitStackFunctionPrologues(uint8_t *buf,
 }
 
 template <class ELFT> void InputSection::writeTo(uint8_t *buf) {
-  if (type == SHT_NOBITS)
-    return;
-
   if (auto *s = dyn_cast<SyntheticSection>(this)) {
     s->writeTo(buf + outSecOff);
     return;
   }
 
+  if (LLVM_UNLIKELY(type == SHT_NOBITS))
+    return;
   // If -r or --emit-relocs is given, then an InputSection
   // may be a relocation section.
-  if (type == SHT_RELA) {
+  if (LLVM_UNLIKELY(type == SHT_RELA)) {
     copyRelocations<ELFT>(buf + outSecOff, getDataAs<typename ELFT::Rela>());
     return;
   }
-  if (type == SHT_REL) {
+  if (LLVM_UNLIKELY(type == SHT_REL)) {
     copyRelocations<ELFT>(buf + outSecOff, getDataAs<typename ELFT::Rel>());
     return;
   }
 
   // If -r is given, we may have a SHT_GROUP section.
-  if (type == SHT_GROUP) {
+  if (LLVM_UNLIKELY(type == SHT_GROUP)) {
     copyShtGroup<ELFT>(buf + outSecOff);
     return;
   }
@@ -1369,7 +1365,7 @@ SyntheticSection *MergeInputSection::getParent() const {
 // null-terminated strings.
 void MergeInputSection::splitStrings(ArrayRef<uint8_t> data, size_t entSize) {
   size_t off = 0;
-  bool isAlloc = flags & SHF_ALLOC;
+  const bool live = !(flags & SHF_ALLOC) || !config->gcSections;
   StringRef s = toStringRef(data);
 
   while (!s.empty()) {
@@ -1378,7 +1374,7 @@ void MergeInputSection::splitStrings(ArrayRef<uint8_t> data, size_t entSize) {
       fatal(toString(this) + ": string is not null terminated");
     size_t size = end + entSize;
 
-    pieces.emplace_back(off, xxHash64(s.substr(0, size)), !isAlloc);
+    pieces.emplace_back(off, xxHash64(s.substr(0, size)), live);
     s = s.substr(size);
     off += size;
   }
@@ -1390,10 +1386,11 @@ void MergeInputSection::splitNonStrings(ArrayRef<uint8_t> data,
                                         size_t entSize) {
   size_t size = data.size();
   assert((size % entSize) == 0);
-  bool isAlloc = flags & SHF_ALLOC;
+  const bool live = !(flags & SHF_ALLOC) || !config->gcSections;
 
-  for (size_t i = 0; i != size; i += entSize)
-    pieces.emplace_back(i, xxHash64(data.slice(i, entSize)), !isAlloc);
+  pieces.assign(size / entSize, SectionPiece(0, 0, false));
+  for (size_t i = 0, j = 0; i != size; i += entSize, j++)
+    pieces[j] = {i, (uint32_t)xxHash64(data.slice(i, entSize)), live};
 }
 
 template <class ELFT>
diff --git a/lld/ELF/InputSection.h b/lld/ELF/InputSection.h
index 7ddc43916a0f..5319830b5d80 100644
--- a/lld/ELF/InputSection.h
+++ b/lld/ELF/InputSection.h
@@ -52,13 +52,6 @@ public:
 
   StringRef name;
 
-  // This pointer points to the "real" instance of this instance.
-  // Usually Repl == this. However, if ICF merges two sections,
-  // Repl pointer of one section points to another section. So,
-  // if you need to get a pointer to this instance, do not use
-  // this but instead this->Repl.
-  SectionBase *repl;
-
   uint8_t sectionKind : 3;
 
   // The next two bit fields are only used by InputSectionBase, but we
@@ -102,9 +95,9 @@ protected:
   constexpr SectionBase(Kind sectionKind, StringRef name, uint64_t flags,
                         uint32_t entsize, uint32_t alignment, uint32_t type,
                         uint32_t info, uint32_t link)
-      : name(name), repl(this), sectionKind(sectionKind), bss(false),
-        keepUnique(false), partition(0), alignment(alignment), flags(flags),
-        entsize(entsize), type(type), link(link), info(info) {}
+      : name(name), sectionKind(sectionKind), bss(false), keepUnique(false),
+        partition(0), alignment(alignment), flags(flags), entsize(entsize),
+        type(type), link(link), info(info) {}
 };
 
 // This corresponds to a section of an input file.
@@ -250,7 +243,7 @@ protected:
 // be found by looking at the next one).
 struct SectionPiece {
   SectionPiece(size_t off, uint32_t hash, bool live)
-      : inputOff(off), live(live || !config->gcSections), hash(hash >> 1) {}
+      : inputOff(off), live(live), hash(hash >> 1) {}
 
   uint32_t inputOff;
   uint32_t live : 1;
@@ -278,7 +271,7 @@ public:
 
   // Splittable sections are handled as a sequence of data
   // rather than a single large blob of data.
-  std::vector<SectionPiece> pieces;
+  SmallVector<SectionPiece, 0> pieces;
 
   // Returns I'th piece's data. This function is very hot when
   // string merging is enabled, so we want to inline.
@@ -367,6 +360,10 @@ public:
   template <class ELFT, class RelTy>
   void relocateNonAlloc(uint8_t *buf, llvm::ArrayRef<RelTy> rels);
 
+  // Points to the canonical section. If ICF folds two sections, repl pointer of
+  // one section points to the other.
+  InputSection *repl = this;
+
   // Used by ICF.
   uint32_t eqClass[2] = {0, 0};
 
@@ -394,7 +391,7 @@ inline bool isDebugSection(const InputSectionBase &sec) {
 }
 
 // The list of all input sections.
-extern std::vector<InputSectionBase *> inputSections;
+extern SmallVector<InputSectionBase *, 0> inputSections;
 
 // The set of TOC entries (.toc + addend) for which we should not apply
 // toc-indirect to toc-relative relaxation. const Symbol * refers to the
diff --git a/lld/ELF/LTO.cpp b/lld/ELF/LTO.cpp
index 46dc77a6789c..65b943c4a54c 100644
--- a/lld/ELF/LTO.cpp
+++ b/lld/ELF/LTO.cpp
@@ -204,6 +204,8 @@ BitcodeCompiler::BitcodeCompiler() {
                                        config->ltoPartitions);
 
   // Initialize usedStartStop.
+  if (bitcodeFiles.empty())
+    return;
   for (Symbol *sym : symtab->symbols()) {
     StringRef s = sym->getName();
     for (StringRef prefix : {"__start_", "__stop_"})
@@ -278,8 +280,8 @@ void BitcodeCompiler::add(BitcodeFile &f) {
 // This is needed because this is what GNU gold plugin does and we have a
 // distributed build system that depends on that behavior.
 static void thinLTOCreateEmptyIndexFiles() {
-  for (LazyObjFile *f : lazyObjFiles) {
-    if (f->extracted || !isBitcode(f->mb))
+  for (BitcodeFile *f : lazyBitcodeFiles) {
+    if (!f->lazy)
       continue;
     std::string path = replaceThinLTOSuffix(getThinLTOOutputFile(f->getName()));
     std::unique_ptr<raw_fd_ostream> os = openFile(path + ".thinlto.bc");
diff --git a/lld/ELF/LinkerScript.cpp b/lld/ELF/LinkerScript.cpp
index cf4da7ab54c9..e8f2ce4fdf1f 100644
--- a/lld/ELF/LinkerScript.cpp
+++ b/lld/ELF/LinkerScript.cpp
@@ -47,10 +47,10 @@ using namespace llvm::support::endian;
 using namespace lld;
 using namespace lld::elf;
 
-LinkerScript *elf::script;
+std::unique_ptr<LinkerScript> elf::script;
 
 static bool isSectionPrefix(StringRef prefix, StringRef name) {
-  return name.startswith(prefix) || name == prefix.drop_back();
+  return name.consume_front(prefix) && (name.empty() || name[0] == '.');
 }
 
 static StringRef getOutputSectionName(const InputSectionBase *s) {
@@ -94,18 +94,21 @@ static StringRef getOutputSectionName(const InputSectionBase *s) {
   // cold parts in .text.split instead of .text.unlikely mitigates against poor
   // profile inaccuracy. Techniques such as hugepage remapping can make
   // conservative decisions at the section granularity.
-  if (config->zKeepTextSectionPrefix)
-    for (StringRef v : {".text.hot.", ".text.unknown.", ".text.unlikely.",
-                        ".text.startup.", ".text.exit.", ".text.split."})
-      if (isSectionPrefix(v, s->name))
-        return v.drop_back();
+  if (isSectionPrefix(".text", s->name)) {
+    if (config->zKeepTextSectionPrefix)
+      for (StringRef v : {".text.hot", ".text.unknown", ".text.unlikely",
+                          ".text.startup", ".text.exit", ".text.split"})
+        if (isSectionPrefix(v.substr(5), s->name.substr(5)))
+          return v;
+    return ".text";
+  }
 
   for (StringRef v :
-       {".text.", ".rodata.", ".data.rel.ro.", ".data.", ".bss.rel.ro.",
-        ".bss.", ".init_array.", ".fini_array.", ".ctors.", ".dtors.", ".tbss.",
-        ".gcc_except_table.", ".tdata.", ".ARM.exidx.", ".ARM.extab."})
+       {".data.rel.ro", ".data", ".rodata", ".bss.rel.ro", ".bss",
+        ".gcc_except_table", ".init_array", ".fini_array", ".tbss", ".tdata",
+        ".ARM.exidx", ".ARM.extab", ".ctors", ".dtors"})
     if (isSectionPrefix(v, s->name))
-      return v.drop_back();
+      return v;
 
   return s->name;
 }
@@ -557,22 +560,22 @@ LinkerScript::computeInputSections(const InputSectionDescription *cmd,
   return ret;
 }
 
-void LinkerScript::discard(InputSectionBase *s) {
-  if (s == in.shStrTab || s == mainPart->relrDyn)
-    error("discarding " + s->name + " section is not allowed");
+void LinkerScript::discard(InputSectionBase &s) {
+  if (&s == in.shStrTab || &s == mainPart->relrDyn)
+    error("discarding " + s.name + " section is not allowed");
 
   // You can discard .hash and .gnu.hash sections by linker scripts. Since
   // they are synthesized sections, we need to handle them differently than
   // other regular sections.
-  if (s == mainPart->gnuHashTab)
+  if (&s == mainPart->gnuHashTab)
     mainPart->gnuHashTab = nullptr;
-  if (s == mainPart->hashTab)
+  if (&s == mainPart->hashTab)
     mainPart->hashTab = nullptr;
 
-  s->markDead();
-  s->parent = nullptr;
-  for (InputSection *ds : s->dependentSections)
-    discard(ds);
+  s.markDead();
+  s.parent = nullptr;
+  for (InputSection *sec : s.dependentSections)
+    discard(*sec);
 }
 
 void LinkerScript::discardSynthetic(OutputSection &outCmd) {
@@ -586,7 +589,7 @@ void LinkerScript::discardSynthetic(OutputSection &outCmd) {
         std::vector<InputSectionBase *> matches =
             computeInputSections(isd, secs);
         for (InputSectionBase *s : matches)
-          discard(s);
+          discard(*s);
       }
   }
 }
@@ -615,7 +618,7 @@ void LinkerScript::processSectionCommands() {
     // Any input section assigned to it is discarded.
     if (osec->name == "/DISCARD/") {
       for (InputSectionBase *s : v)
-        discard(s);
+        discard(*s);
       discardSynthetic(*osec);
       osec->commands.clear();
       return false;
@@ -1335,7 +1338,7 @@ std::vector<PhdrEntry *> LinkerScript::createPhdrs() {
   // Process PHDRS and FILEHDR keywords because they are not
   // real output sections and cannot be added in the following loop.
   for (const PhdrsCommand &cmd : phdrsCommands) {
-    PhdrEntry *phdr = make<PhdrEntry>(cmd.type, cmd.flags ? *cmd.flags : PF_R);
+    PhdrEntry *phdr = make<PhdrEntry>(cmd.type, cmd.flags.getValueOr(PF_R));
 
     if (cmd.hasFilehdr)
       phdr->add(Out::elfHeader);
diff --git a/lld/ELF/LinkerScript.h b/lld/ELF/LinkerScript.h
index badc4d126be8..f385c8320978 100644
--- a/lld/ELF/LinkerScript.h
+++ b/lld/ELF/LinkerScript.h
@@ -312,7 +312,7 @@ public:
 
   bool hasPhdrsCommands() { return !phdrsCommands.empty(); }
   uint64_t getDot() { return dot; }
-  void discard(InputSectionBase *s);
+  void discard(InputSectionBase &s);
 
   ExprValue getSymbolValue(StringRef name, const Twine &loc);
 
@@ -366,7 +366,7 @@ public:
   std::vector<const InputSectionBase *> orphanSections;
 };
 
-extern LinkerScript *script;
+extern std::unique_ptr<LinkerScript> script;
 
 } // end namespace elf
 } // end namespace lld
diff --git a/lld/ELF/MapFile.cpp b/lld/ELF/MapFile.cpp
index 06735802f7f1..1998192bfba6 100644
--- a/lld/ELF/MapFile.cpp
+++ b/lld/ELF/MapFile.cpp
@@ -54,11 +54,11 @@ static void writeHeader(raw_ostream &os, uint64_t vma, uint64_t lma,
 // Returns a list of all symbols that we want to print out.
 static std::vector<Defined *> getSymbols() {
   std::vector<Defined *> v;
-  for (InputFile *file : objectFiles)
+  for (ELFFileBase *file : objectFiles)
     for (Symbol *b : file->getSymbols())
       if (auto *dr = dyn_cast<Defined>(b))
         if (!dr->isSection() && dr->section && dr->section->isLive() &&
-            (dr->file == file || dr->needsPltAddr || dr->section->bss))
+            (dr->file == file || dr->needsCopy || dr->section->bss))
           v.push_back(dr);
   return v;
 }
@@ -72,10 +72,17 @@ static SymbolMapTy getSectionSyms(ArrayRef<Defined *> syms) {
   // Sort symbols by address. We want to print out symbols in the
   // order in the output file rather than the order they appeared
   // in the input files.
-  for (auto &it : ret)
+  SmallPtrSet<Defined *, 4> set;
+  for (auto &it : ret) {
+    // Deduplicate symbols which need a canonical PLT entry/copy relocation.
+    set.clear();
+    llvm::erase_if(it.second,
+                   [&](Defined *sym) { return !set.insert(sym).second; });
+
     llvm::stable_sort(it.second, [](Defined *a, Defined *b) {
       return a->getVA() < b->getVA();
     });
+  }
   return ret;
 }
 
@@ -236,7 +243,7 @@ void elf::writeWhyExtract() {
 static void writeCref(raw_fd_ostream &os) {
   // Collect symbols and files.
   MapVector<Symbol *, SetVector<InputFile *>> map;
-  for (InputFile *file : objectFiles) {
+  for (ELFFileBase *file : objectFiles) {
     for (Symbol *sym : file->getSymbols()) {
       if (isa<SharedSymbol>(sym))
         map[sym].insert(file);
diff --git a/lld/ELF/MarkLive.cpp b/lld/ELF/MarkLive.cpp
index 11e0466b1157..b63f2beb9dcb 100644
--- a/lld/ELF/MarkLive.cpp
+++ b/lld/ELF/MarkLive.cpp
@@ -177,10 +177,11 @@ static bool isReserved(InputSectionBase *sec) {
     // SHT_NOTE sections in a group are subject to garbage collection.
     return !sec->nextInSectionGroup;
   default:
+    // Support SHT_PROGBITS .init_array for a while
+    // (https://golang.org/issue/50295).
     StringRef s = sec->name;
-    return s.startswith(".ctors") || s.startswith(".dtors") ||
-           s.startswith(".init") || s.startswith(".fini") ||
-           s.startswith(".jcr");
+    return s == ".init" || s == ".fini" || s == ".init_array" || s == ".jcr" ||
+           s.startswith(".ctors") || s.startswith(".dtors");
   }
 }
 
@@ -243,8 +244,6 @@ template <class ELFT> void MarkLive<ELFT>::run() {
   for (StringRef s : script->referencedSymbols)
     markSymbol(symtab->find(s));
 
-  // Preserve special sections and those which are specified in linker
-  // script KEEP command.
   for (InputSectionBase *sec : inputSections) {
     // Mark .eh_frame sections as live because there are usually no relocations
     // that point to .eh_frames. Otherwise, the garbage collector would drop
@@ -258,6 +257,7 @@ template <class ELFT> void MarkLive<ELFT>::run() {
         scanEhFrameSection(*eh, rels.rels);
       else if (rels.relas.size())
         scanEhFrameSection(*eh, rels.relas);
+      continue;
     }
 
     if (sec->flags & SHF_GNU_RETAIN) {
@@ -267,6 +267,39 @@ template <class ELFT> void MarkLive<ELFT>::run() {
     if (sec->flags & SHF_LINK_ORDER)
       continue;
 
+    // Usually, non-SHF_ALLOC sections are not removed even if they are
+    // unreachable through relocations because reachability is not a good signal
+    // whether they are garbage or not (e.g. there is usually no section
+    // referring to a .comment section, but we want to keep it.) When a
+    // non-SHF_ALLOC section is retained, we also retain sections dependent on
+    // it.
+    //
+    // Note on SHF_LINK_ORDER: Such sections contain metadata and they
+    // have a reverse dependency on the InputSection they are linked with.
+    // We are able to garbage collect them.
+    //
+    // Note on SHF_REL{,A}: Such sections reach here only when -r
+    // or --emit-reloc were given. And they are subject of garbage
+    // collection because, if we remove a text section, we also
+    // remove its relocation section.
+    //
+    // Note on nextInSectionGroup: The ELF spec says that group sections are
+    // included or omitted as a unit. We take the interpretation that:
+    //
+    // - Group members (nextInSectionGroup != nullptr) are subject to garbage
+    //   collection.
+    // - Groups members are retained or discarded as a unit.
+    if (!(sec->flags & SHF_ALLOC)) {
+      bool isRel = sec->type == SHT_REL || sec->type == SHT_RELA;
+      if (!isRel && !sec->nextInSectionGroup) {
+        sec->markLive();
+        for (InputSection *isec : sec->dependentSections)
+          isec->markLive();
+      }
+    }
+
+    // Preserve special sections and those which are specified in linker
+    // script KEEP command.
     if (isReserved(sec) || script->shouldKeep(sec)) {
       enqueue(sec, 0);
     } else if ((!config->zStartStopGC || sec->name.startswith("__libc_")) &&
@@ -312,7 +345,7 @@ template <class ELFT> void MarkLive<ELFT>::mark() {
 // to from __start_/__stop_ symbols because there will only be one set of
 // symbols for the whole program.
 template <class ELFT> void MarkLive<ELFT>::moveToMain() {
-  for (InputFile *file : objectFiles)
+  for (ELFFileBase *file : objectFiles)
     for (Symbol *s : file->getSymbols())
       if (auto *d = dyn_cast<Defined>(s))
         if ((d->type == STT_GNU_IFUNC || d->type == STT_TLS) && d->section &&
@@ -348,46 +381,6 @@ template <class ELFT> void elf::markLive() {
     return;
   }
 
-  // Otherwise, do mark-sweep GC.
-  //
-  // The --gc-sections option works only for SHF_ALLOC sections (sections that
-  // are memory-mapped at runtime). So we can unconditionally make non-SHF_ALLOC
-  // sections alive except SHF_LINK_ORDER, SHT_REL/SHT_RELA sections, and
-  // sections in a group.
-  //
-  // Usually, non-SHF_ALLOC sections are not removed even if they are
-  // unreachable through relocations because reachability is not a good signal
-  // whether they are garbage or not (e.g. there is usually no section referring
-  // to a .comment section, but we want to keep it.) When a non-SHF_ALLOC
-  // section is retained, we also retain sections dependent on it.
-  //
-  // Note on SHF_LINK_ORDER: Such sections contain metadata and they
-  // have a reverse dependency on the InputSection they are linked with.
-  // We are able to garbage collect them.
-  //
-  // Note on SHF_REL{,A}: Such sections reach here only when -r
-  // or --emit-reloc were given. And they are subject of garbage
-  // collection because, if we remove a text section, we also
-  // remove its relocation section.
-  //
-  // Note on nextInSectionGroup: The ELF spec says that group sections are
-  // included or omitted as a unit. We take the interpretation that:
-  //
-  // - Group members (nextInSectionGroup != nullptr) are subject to garbage
-  //   collection.
-  // - Groups members are retained or discarded as a unit.
-  for (InputSectionBase *sec : inputSections) {
-    bool isAlloc = (sec->flags & SHF_ALLOC);
-    bool isLinkOrder = (sec->flags & SHF_LINK_ORDER);
-    bool isRel = (sec->type == SHT_REL || sec->type == SHT_RELA);
-
-    if (!isAlloc && !isLinkOrder && !isRel && !sec->nextInSectionGroup) {
-      sec->markLive();
-      for (InputSection *isec : sec->dependentSections)
-        isec->markLive();
-    }
-  }
-
   // Follow the graph to mark all live sections.
   for (unsigned curPart = 1; curPart <= partitions.size(); ++curPart)
     MarkLive<ELFT>(curPart).run();
diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td
index f9f9f54a80d8..bddf13a3cb42 100644
--- a/lld/ELF/Options.td
+++ b/lld/ELF/Options.td
@@ -79,9 +79,6 @@ defm split_stack_adjust_size
          "non-split-stack function">,
       MetaVarName<"<value>">;
 
-defm library_path:
-  Eq<"library-path", "Add a directory to the library search path">, MetaVarName<"<dir>">;
-
 def O: JoinedOrSeparate<["-"], "O">, HelpText<"Optimize output file size">;
 
 defm Tbss: Eq<"Tbss", "Same as --section-start with .bss as the sectionname">;
@@ -266,8 +263,10 @@ defm just_symbols: Eq<"just-symbols", "Just link symbols">;
 
 defm keep_unique: Eq<"keep-unique", "Do not fold this symbol during ICF">;
 
-defm library: Eq<"library", "Root name of library to use">,
-  MetaVarName<"<libName>">;
+def library: JoinedOrSeparate<["-"], "l">, MetaVarName<"<libname>">,
+  HelpText<"Search for library <libname>">;
+def library_path: JoinedOrSeparate<["-"], "L">, MetaVarName<"<dir>">,
+  HelpText<"Add <dir> to the library search path">;
 
 def m: JoinedOrSeparate<["-"], "m">, HelpText<"Set target emulation">;
 
@@ -524,8 +523,10 @@ def: JoinedOrSeparate<["-"], "e">, Alias<entry>, HelpText<"Alias for --entry">;
 def: Flag<["-"], "E">, Alias<export_dynamic>, HelpText<"Alias for --export-dynamic">;
 def: Separate<["-"], "F">, Alias<filter>, HelpText<"Alias for --filter">;
 def: Separate<["-"], "b">, Alias<format>, HelpText<"Alias for --format">;
-def: JoinedOrSeparate<["-"], "l">, Alias<library>, HelpText<"Alias for --library">;
-def: JoinedOrSeparate<["-"], "L">, Alias<library_path>, HelpText<"Alias for --library-path">;
+def: Separate<["--", "-"], "library">, Alias<library>;
+def: Joined<["--", "-"], "library=">, Alias<library>;
+def: Separate<["--", "-"], "library-path">, Alias<library_path>;
+def: Joined<["--", "-"], "library-path=">, Alias<library_path>;
 def: Flag<["-"], "n">, Alias<nmagic>, HelpText<"Alias for --nmagic">;
 def: Flag<["-"], "N">, Alias<omagic>, HelpText<"Alias for --omagic">;
 def: Joined<["--"], "output=">, Alias<o>, HelpText<"Alias for -o">;
diff --git a/lld/ELF/OutputSections.cpp b/lld/ELF/OutputSections.cpp
index a17f713b742a..4a03ac387814 100644
--- a/lld/ELF/OutputSections.cpp
+++ b/lld/ELF/OutputSections.cpp
@@ -40,7 +40,7 @@ OutputSection *Out::preinitArray;
 OutputSection *Out::initArray;
 OutputSection *Out::finiArray;
 
-std::vector<OutputSection *> elf::outputSections;
+SmallVector<OutputSection *, 0> elf::outputSections;
 
 uint32_t OutputSection::getPhdrFlags() const {
   uint32_t ret = 0;
@@ -155,6 +155,15 @@ void OutputSection::commitSection(InputSection *isec) {
     entsize = 0;
 }
 
+static MergeSyntheticSection *createMergeSynthetic(StringRef name,
+                                                   uint32_t type,
+                                                   uint64_t flags,
+                                                   uint32_t alignment) {
+  if ((flags & SHF_STRINGS) && config->optimize >= 2)
+    return make<MergeTailSection>(name, type, flags, alignment);
+  return make<MergeNoTailSection>(name, type, flags, alignment);
+}
+
 // This function scans over the InputSectionBase list sectionBases to create
 // InputSectionDescription::sections.
 //
@@ -323,6 +332,7 @@ static void writeInt(uint8_t *buf, uint64_t data, uint64_t size) {
 }
 
 template <class ELFT> void OutputSection::writeTo(uint8_t *buf) {
+  llvm::TimeTraceScope timeScope("Write sections", name);
   if (type == SHT_NOBITS)
     return;
 
@@ -550,7 +560,7 @@ void OutputSection::checkDynRelAddends(const uint8_t *bufStart) {
     if (!sec)
       return;
     for (const DynamicReloc &rel : sec->relocs) {
-      int64_t addend = rel.computeAddend();
+      int64_t addend = rel.addend;
       const OutputSection *relOsec = rel.inputSec->getOutputSection();
       assert(relOsec != nullptr && "missing output section for relocation");
       const uint8_t *relocTarget =
diff --git a/lld/ELF/OutputSections.h b/lld/ELF/OutputSections.h
index a5b05cf28aa8..fb3eb0059909 100644
--- a/lld/ELF/OutputSections.h
+++ b/lld/ELF/OutputSections.h
@@ -138,7 +138,7 @@ struct Out {
 
 uint64_t getHeaderSize();
 
-extern std::vector<OutputSection *> outputSections;
+extern llvm::SmallVector<OutputSection *, 0> outputSections;
 } // namespace elf
 } // namespace lld
 
diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp
index 5136ba2151a3..cfe49007b814 100644
--- a/lld/ELF/Relocations.cpp
+++ b/lld/ELF/Relocations.cpp
@@ -295,18 +295,20 @@ static SmallSet<SharedSymbol *, 4> getSymbolsAt(SharedSymbol &ss) {
 // in .bss and in the case of a canonical plt entry it is in .plt. This function
 // replaces the existing symbol with a Defined pointing to the appropriate
 // location.
-static void replaceWithDefined(Symbol &sym, SectionBase *sec, uint64_t value,
+static void replaceWithDefined(Symbol &sym, SectionBase &sec, uint64_t value,
                                uint64_t size) {
   Symbol old = sym;
 
   sym.replace(Defined{sym.file, sym.getName(), sym.binding, sym.stOther,
-                      sym.type, value, size, sec});
+                      sym.type, value, size, &sec});
 
   sym.pltIndex = old.pltIndex;
   sym.gotIndex = old.gotIndex;
   sym.verdefIndex = old.verdefIndex;
   sym.exportDynamic = true;
   sym.isUsedInRegularObj = true;
+  // A copy relocated alias may need a GOT entry.
+  sym.needsGot = old.needsGot;
 }
 
 // Reserve space in .bss or .bss.rel.ro for copy relocation.
@@ -351,7 +353,7 @@ static void replaceWithDefined(Symbol &sym, SectionBase *sec, uint64_t value,
 // to the variable in .bss. This kind of issue is sometimes very hard to
 // debug. What's a solution? Instead of exporting a variable V from a DSO,
 // define an accessor getV().
-template <class ELFT> static void addCopyRelSymbol(SharedSymbol &ss) {
+template <class ELFT> static void addCopyRelSymbolImpl(SharedSymbol &ss) {
   // Copy relocation against zero-sized symbol doesn't make sense.
   uint64_t symSize = ss.getSize();
   if (symSize == 0 || ss.alignment == 0)
@@ -377,9 +379,29 @@ template <class ELFT> static void addCopyRelSymbol(SharedSymbol &ss) {
   // dynamic symbol for each one. This causes the copy relocation to correctly
   // interpose any aliases.
   for (SharedSymbol *sym : getSymbolsAt<ELFT>(ss))
-    replaceWithDefined(*sym, sec, 0, sym->size);
+    replaceWithDefined(*sym, *sec, 0, sym->size);
 
-  mainPart->relaDyn->addSymbolReloc(target->copyRel, sec, 0, ss);
+  mainPart->relaDyn->addSymbolReloc(target->copyRel, *sec, 0, ss);
+}
+
+static void addCopyRelSymbol(SharedSymbol &ss) {
+  const SharedFile &file = ss.getFile();
+  switch (file.ekind) {
+  case ELF32LEKind:
+    addCopyRelSymbolImpl<ELF32LE>(ss);
+    break;
+  case ELF32BEKind:
+    addCopyRelSymbolImpl<ELF32BE>(ss);
+    break;
+  case ELF64LEKind:
+    addCopyRelSymbolImpl<ELF64LE>(ss);
+    break;
+  case ELF64BEKind:
+    addCopyRelSymbolImpl<ELF64BE>(ss);
+    break;
+  default:
+    llvm_unreachable("");
+  }
 }
 
 // MIPS has an odd notion of "paired" relocations to calculate addends.
@@ -450,8 +472,8 @@ static std::string maybeReportDiscarded(Undefined &sym) {
   if (!file || !sym.discardedSecIdx ||
       file->getSections()[sym.discardedSecIdx] != &InputSection::discarded)
     return "";
-  ArrayRef<Elf_Shdr_Impl<ELFT>> objSections =
-      CHECK(file->getObj().sections(), file);
+  ArrayRef<typename ELFT::Shdr> objSections =
+      file->template getELFShdrs<ELFT>();
 
   std::string msg;
   if (sym.type == ELF::STT_SECTION) {
@@ -680,6 +702,12 @@ static void reportUndefinedSymbol(const UndefinedDiag &undef,
     msg +=
         "\n>>> the vtable symbol may be undefined because the class is missing "
         "its key function (see https://lld.llvm.org/missingkeyfunction)";
+  if (config->gcSections && config->zStartStopGC &&
+      sym.getName().startswith("__start_")) {
+    msg += "\n>>> the encapsulation symbol needs to be retained under "
+           "--gc-sections properly; consider -z nostart-stop-gc "
+           "(see https://lld.llvm.org/ELF/start-stop-gc)";
+  }
 
   if (undef.isWarning)
     warn(msg);
@@ -711,8 +739,6 @@ template <class ELFT> void elf::reportUndefinedSymbols() {
 // Returns true if the undefined symbol will produce an error message.
 static bool maybeReportUndefined(Symbol &sym, InputSectionBase &sec,
                                  uint64_t offset) {
-  if (!sym.isUndefined())
-    return false;
   // If versioned, issue an error (even if the symbol is weak) because we don't
   // know the defining filename which is required to construct a Verneed entry.
   if (*sym.getVersionSuffix() == '@') {
@@ -807,10 +833,10 @@ private:
 };
 } // namespace
 
-static void addRelativeReloc(InputSectionBase *isec, uint64_t offsetInSec,
+static void addRelativeReloc(InputSectionBase &isec, uint64_t offsetInSec,
                              Symbol &sym, int64_t addend, RelExpr expr,
                              RelType type) {
-  Partition &part = isec->getPartition();
+  Partition &part = isec.getPartition();
 
   // Add a relative relocation. If relrDyn section is enabled, and the
   // relocation offset is guaranteed to be even, add the relocation to
@@ -818,9 +844,9 @@ static void addRelativeReloc(InputSectionBase *isec, uint64_t offsetInSec,
   // relrDyn sections don't support odd offsets. Also, relrDyn sections
   // don't store the addend values, so we must write it to the relocated
   // address.
-  if (part.relrDyn && isec->alignment >= 2 && offsetInSec % 2 == 0) {
-    isec->relocations.push_back({expr, type, offsetInSec, addend, &sym});
-    part.relrDyn->relocs.push_back({isec, offsetInSec});
+  if (part.relrDyn && isec.alignment >= 2 && offsetInSec % 2 == 0) {
+    isec.relocations.push_back({expr, type, offsetInSec, addend, &sym});
+    part.relrDyn->relocs.push_back({&isec, offsetInSec});
     return;
   }
   part.relaDyn->addRelativeReloc(target->relativeRel, isec, offsetInSec, sym,
@@ -828,14 +854,14 @@ static void addRelativeReloc(InputSectionBase *isec, uint64_t offsetInSec,
 }
 
 template <class PltSection, class GotPltSection>
-static void addPltEntry(PltSection *plt, GotPltSection *gotPlt,
-                        RelocationBaseSection *rel, RelType type, Symbol &sym) {
-  plt->addEntry(sym);
-  gotPlt->addEntry(sym);
-  rel->addReloc({type, gotPlt, sym.getGotPltOffset(),
-                 sym.isPreemptible ? DynamicReloc::AgainstSymbol
-                                   : DynamicReloc::AddendOnlyWithTargetVA,
-                 sym, 0, R_ABS});
+static void addPltEntry(PltSection &plt, GotPltSection &gotPlt,
+                        RelocationBaseSection &rel, RelType type, Symbol &sym) {
+  plt.addEntry(sym);
+  gotPlt.addEntry(sym);
+  rel.addReloc({type, &gotPlt, sym.getGotPltOffset(),
+                sym.isPreemptible ? DynamicReloc::AgainstSymbol
+                                  : DynamicReloc::AddendOnlyWithTargetVA,
+                sym, 0, R_ABS});
 }
 
 static void addGotEntry(Symbol &sym) {
@@ -854,7 +880,7 @@ static void addGotEntry(Symbol &sym) {
   if (!config->isPic || isAbsolute(sym))
     in.got->relocations.push_back({R_ABS, target->symbolicRel, off, 0, &sym});
   else
-    addRelativeReloc(in.got, off, sym, 0, R_ABS, target->symbolicRel);
+    addRelativeReloc(*in.got, off, sym, 0, R_ABS, target->symbolicRel);
 }
 
 static void addTpOffsetGotEntry(Symbol &sym) {
@@ -865,7 +891,7 @@ static void addTpOffsetGotEntry(Symbol &sym) {
     return;
   }
   mainPart->relaDyn->addAddendOnlyRelocIfNonPreemptible(
-      target->tlsGotRel, in.got, off, sym, target->symbolicRel);
+      target->tlsGotRel, *in.got, off, sym, target->symbolicRel);
 }
 
 // Return true if we can define a symbol in the executable that
@@ -993,12 +1019,12 @@ static void processRelocAux(InputSectionBase &sec, RelExpr expr, RelType type,
   if (canWrite) {
     RelType rel = target->getDynRel(type);
     if (expr == R_GOT || (rel == target->symbolicRel && !sym.isPreemptible)) {
-      addRelativeReloc(&sec, offset, sym, addend, expr, type);
+      addRelativeReloc(sec, offset, sym, addend, expr, type);
       return;
     } else if (rel != 0) {
       if (config->emachine == EM_MIPS && rel == target->symbolicRel)
         rel = target->relativeRel;
-      sec.getPartition().relaDyn->addSymbolReloc(rel, &sec, offset, sym, addend,
+      sec.getPartition().relaDyn->addSymbolReloc(rel, sec, offset, sym, addend,
                                                  type);
 
       // MIPS ABI turns using of GOT and dynamic relocations inside out.
@@ -1039,7 +1065,7 @@ static void processRelocAux(InputSectionBase &sec, RelExpr expr, RelType type,
                 " against symbol '" + toString(*ss) +
                 "'; recompile with -fPIC or remove '-z nocopyreloc'" +
                 getLocation(sec, sym, offset));
-        addCopyRelSymbol<ELFT>(*ss);
+        sym.needsCopy = true;
       }
       sec.relocations.push_back({expr, type, offset, addend, &sym});
       return;
@@ -1077,20 +1103,8 @@ static void processRelocAux(InputSectionBase &sec, RelExpr expr, RelType type,
         errorOrWarn("symbol '" + toString(sym) +
                     "' cannot be preempted; recompile with -fPIE" +
                     getLocation(sec, sym, offset));
-      if (!sym.isInPlt())
-        addPltEntry(in.plt, in.gotPlt, in.relaPlt, target->pltRel, sym);
-      if (!sym.isDefined()) {
-        replaceWithDefined(
-            sym, in.plt,
-            target->pltHeaderSize + target->pltEntrySize * sym.pltIndex, 0);
-        if (config->emachine == EM_PPC) {
-          // PPC32 canonical PLT entries are at the beginning of .glink
-          cast<Defined>(sym).value = in.plt->headerSize;
-          in.plt->headerSize += 16;
-          cast<PPC32GlinkSection>(in.plt)->canonical_plts.push_back(&sym);
-        }
-      }
-      sym.needsPltAddr = true;
+      sym.needsCopy = true;
+      sym.needsPlt = true;
       sec.relocations.push_back({expr, type, offset, addend, &sym});
       return;
     }
@@ -1144,13 +1158,10 @@ handleTlsRelocation(RelType type, Symbol &sym, InputSectionBase &c,
   if (oneof<R_AARCH64_TLSDESC_PAGE, R_TLSDESC, R_TLSDESC_CALL, R_TLSDESC_PC,
             R_TLSDESC_GOTPLT>(expr) &&
       config->shared) {
-    if (in.got->addDynTlsEntry(sym)) {
-      uint64_t off = in.got->getGlobalDynOffset(sym);
-      mainPart->relaDyn->addAddendOnlyRelocIfNonPreemptible(
-          target->tlsDescRel, in.got, off, sym, target->tlsDescRel);
-    }
-    if (expr != R_TLSDESC_CALL)
+    if (expr != R_TLSDESC_CALL) {
+      sym.needsTlsDesc = true;
       c.relocations.push_back({expr, type, offset, addend, &sym});
+    }
     return 1;
   }
 
@@ -1186,14 +1197,7 @@ handleTlsRelocation(RelType type, Symbol &sym, InputSectionBase &c,
     }
     if (expr == R_TLSLD_HINT)
       return 1;
-    if (in.got->addTlsIndex()) {
-      if (isLocalInExecutable)
-        in.got->relocations.push_back(
-            {R_ADDEND, target->symbolicRel, in.got->getTlsIndexOff(), 1, &sym});
-      else
-        mainPart->relaDyn->addReloc(
-            {target->tlsModuleIndexRel, in.got, in.got->getTlsIndexOff()});
-    }
+    sym.needsTlsLd = true;
     c.relocations.push_back({expr, type, offset, addend, &sym});
     return 1;
   }
@@ -1209,12 +1213,7 @@ handleTlsRelocation(RelType type, Symbol &sym, InputSectionBase &c,
   // Local-Dynamic sequence where offset of tls variable relative to dynamic
   // thread pointer is stored in the got. This cannot be relaxed to Local-Exec.
   if (expr == R_TLSLD_GOT_OFF) {
-    if (!sym.isInGot()) {
-      in.got->addEntry(sym);
-      uint64_t off = sym.getGotOffset();
-      in.got->relocations.push_back(
-          {R_ABS, target->tlsOffsetRel, off, 0, &sym});
-    }
+    sym.needsGotDtprel = true;
     c.relocations.push_back({expr, type, offset, addend, &sym});
     return 1;
   }
@@ -1222,27 +1221,7 @@ handleTlsRelocation(RelType type, Symbol &sym, InputSectionBase &c,
   if (oneof<R_AARCH64_TLSDESC_PAGE, R_TLSDESC, R_TLSDESC_CALL, R_TLSDESC_PC,
             R_TLSDESC_GOTPLT, R_TLSGD_GOT, R_TLSGD_GOTPLT, R_TLSGD_PC>(expr)) {
     if (!toExecRelax) {
-      if (in.got->addDynTlsEntry(sym)) {
-        uint64_t off = in.got->getGlobalDynOffset(sym);
-
-        if (isLocalInExecutable)
-          // Write one to the GOT slot.
-          in.got->relocations.push_back(
-              {R_ADDEND, target->symbolicRel, off, 1, &sym});
-        else
-          mainPart->relaDyn->addSymbolReloc(target->tlsModuleIndexRel, in.got,
-                                            off, sym);
-
-        // If the symbol is preemptible we need the dynamic linker to write
-        // the offset too.
-        uint64_t offsetOff = off + config->wordsize;
-        if (sym.isPreemptible)
-          mainPart->relaDyn->addSymbolReloc(target->tlsOffsetRel, in.got,
-                                            offsetOff, sym);
-        else
-          in.got->relocations.push_back(
-              {R_ABS, target->tlsOffsetRel, offsetOff, 0, &sym});
-      }
+      sym.needsTlsGd = true;
       c.relocations.push_back({expr, type, offset, addend, &sym});
       return 1;
     }
@@ -1250,14 +1229,10 @@ handleTlsRelocation(RelType type, Symbol &sym, InputSectionBase &c,
     // Global-Dynamic relocs can be relaxed to Initial-Exec or Local-Exec
     // depending on the symbol being locally defined or not.
     if (sym.isPreemptible) {
+      sym.needsTlsGdToIe = true;
       c.relocations.push_back(
           {target->adjustTlsExpr(type, R_RELAX_TLS_GD_TO_IE), type, offset,
            addend, &sym});
-      if (!sym.isInGot()) {
-        in.got->addEntry(sym);
-        mainPart->relaDyn->addSymbolReloc(target->tlsGotRel, in.got,
-                                          sym.getGotOffset(), sym);
-      }
     } else {
       c.relocations.push_back(
           {target->adjustTlsExpr(type, R_RELAX_TLS_GD_TO_LE), type, offset,
@@ -1274,11 +1249,10 @@ handleTlsRelocation(RelType type, Symbol &sym, InputSectionBase &c,
       c.relocations.push_back(
           {R_RELAX_TLS_IE_TO_LE, type, offset, addend, &sym});
     } else if (expr != R_TLSIE_HINT) {
-      if (!sym.isInGot())
-        addTpOffsetGotEntry(sym);
+      sym.needsTlsIe = true;
       // R_GOT needs a relative relocation for PIC on i386 and Hexagon.
       if (expr == R_GOT && config->isPic && !target->usesOnlyLowPageBits(type))
-        addRelativeReloc(&c, offset, sym, addend, expr, type);
+        addRelativeReloc(c, offset, sym, addend, expr, type);
       else
         c.relocations.push_back({expr, type, offset, addend, &sym});
     }
@@ -1311,7 +1285,8 @@ static void scanReloc(InputSectionBase &sec, OffsetGetter &getOffset, RelTy *&i,
 
   // Error if the target symbol is undefined. Symbol index 0 may be used by
   // marker relocations, e.g. R_*_NONE and R_ARM_V4BX. Don't error on them.
-  if (symIndex != 0 && maybeReportUndefined(sym, sec, rel.r_offset))
+  if (sym.isUndefined() && symIndex != 0 &&
+      maybeReportUndefined(sym, sec, rel.r_offset))
     return;
 
   const uint8_t *relocatedAddr = sec.data().begin() + rel.r_offset;
@@ -1365,8 +1340,8 @@ static void scanReloc(InputSectionBase &sec, OffsetGetter &getOffset, RelTy *&i,
   if (oneof<R_GOTPLTONLY_PC, R_GOTPLTREL, R_GOTPLT, R_PLT_GOTPLT,
             R_TLSDESC_GOTPLT, R_TLSGD_GOTPLT>(expr)) {
     in.gotPlt->hasGotPltOffRel = true;
-  } else if (oneof<R_GOTONLY_PC, R_GOTREL, R_PPC64_TOCBASE, R_PPC64_RELAX_TOC>(
-                 expr)) {
+  } else if (oneof<R_GOTONLY_PC, R_GOTREL, R_PPC32_PLTREL, R_PPC64_TOCBASE,
+                   R_PPC64_RELAX_TOC>(expr)) {
     in.got->hasGotOffRel = true;
   }
 
@@ -1415,120 +1390,27 @@ static void scanReloc(InputSectionBase &sec, OffsetGetter &getOffset, RelTy *&i,
   // direct relocation on through.
   if (sym.isGnuIFunc() && config->zIfuncNoplt) {
     sym.exportDynamic = true;
-    mainPart->relaDyn->addSymbolReloc(type, &sec, offset, sym, addend, type);
+    mainPart->relaDyn->addSymbolReloc(type, sec, offset, sym, addend, type);
     return;
   }
 
-  // Non-preemptible ifuncs require special handling. First, handle the usual
-  // case where the symbol isn't one of these.
-  if (!sym.isGnuIFunc() || sym.isPreemptible) {
-    // If a relocation needs PLT, we create PLT and GOTPLT slots for the symbol.
-    if (needsPlt(expr) && !sym.isInPlt())
-      addPltEntry(in.plt, in.gotPlt, in.relaPlt, target->pltRel, sym);
-
-    // Create a GOT slot if a relocation needs GOT.
-    if (needsGot(expr)) {
-      if (config->emachine == EM_MIPS) {
-        // MIPS ABI has special rules to process GOT entries and doesn't
-        // require relocation entries for them. A special case is TLS
-        // relocations. In that case dynamic loader applies dynamic
-        // relocations to initialize TLS GOT entries.
-        // See "Global Offset Table" in Chapter 5 in the following document
-        // for detailed description:
-        // ftp://www.linux-mips.org/pub/linux/mips/doc/ABI/mipsabi.pdf
-        in.mipsGot->addEntry(*sec.file, sym, addend, expr);
-      } else if (!sym.isInGot()) {
-        addGotEntry(sym);
-      }
+  if (needsGot(expr)) {
+    if (config->emachine == EM_MIPS) {
+      // MIPS ABI has special rules to process GOT entries and doesn't
+      // require relocation entries for them. A special case is TLS
+      // relocations. In that case dynamic loader applies dynamic
+      // relocations to initialize TLS GOT entries.
+      // See "Global Offset Table" in Chapter 5 in the following document
+      // for detailed description:
+      // ftp://www.linux-mips.org/pub/linux/mips/doc/ABI/mipsabi.pdf
+      in.mipsGot->addEntry(*sec.file, sym, addend, expr);
+    } else {
+      sym.needsGot = true;
     }
+  } else if (needsPlt(expr)) {
+    sym.needsPlt = true;
   } else {
-    // Handle a reference to a non-preemptible ifunc. These are special in a
-    // few ways:
-    //
-    // - Unlike most non-preemptible symbols, non-preemptible ifuncs do not have
-    //   a fixed value. But assuming that all references to the ifunc are
-    //   GOT-generating or PLT-generating, the handling of an ifunc is
-    //   relatively straightforward. We create a PLT entry in Iplt, which is
-    //   usually at the end of .plt, which makes an indirect call using a
-    //   matching GOT entry in igotPlt, which is usually at the end of .got.plt.
-    //   The GOT entry is relocated using an IRELATIVE relocation in relaIplt,
-    //   which is usually at the end of .rela.plt. Unlike most relocations in
-    //   .rela.plt, which may be evaluated lazily without -z now, dynamic
-    //   loaders evaluate IRELATIVE relocs eagerly, which means that for
-    //   IRELATIVE relocs only, GOT-generating relocations can point directly to
-    //   .got.plt without requiring a separate GOT entry.
-    //
-    // - Despite the fact that an ifunc does not have a fixed value, compilers
-    //   that are not passed -fPIC will assume that they do, and will emit
-    //   direct (non-GOT-generating, non-PLT-generating) relocations to the
-    //   symbol. This means that if a direct relocation to the symbol is
-    //   seen, the linker must set a value for the symbol, and this value must
-    //   be consistent no matter what type of reference is made to the symbol.
-    //   This can be done by creating a PLT entry for the symbol in the way
-    //   described above and making it canonical, that is, making all references
-    //   point to the PLT entry instead of the resolver. In lld we also store
-    //   the address of the PLT entry in the dynamic symbol table, which means
-    //   that the symbol will also have the same value in other modules.
-    //   Because the value loaded from the GOT needs to be consistent with
-    //   the value computed using a direct relocation, a non-preemptible ifunc
-    //   may end up with two GOT entries, one in .got.plt that points to the
-    //   address returned by the resolver and is used only by the PLT entry,
-    //   and another in .got that points to the PLT entry and is used by
-    //   GOT-generating relocations.
-    //
-    // - The fact that these symbols do not have a fixed value makes them an
-    //   exception to the general rule that a statically linked executable does
-    //   not require any form of dynamic relocation. To handle these relocations
-    //   correctly, the IRELATIVE relocations are stored in an array which a
-    //   statically linked executable's startup code must enumerate using the
-    //   linker-defined symbols __rela?_iplt_{start,end}.
-    if (!sym.isInPlt()) {
-      // Create PLT and GOTPLT slots for the symbol.
-      sym.isInIplt = true;
-
-      // Create a copy of the symbol to use as the target of the IRELATIVE
-      // relocation in the igotPlt. This is in case we make the PLT canonical
-      // later, which would overwrite the original symbol.
-      //
-      // FIXME: Creating a copy of the symbol here is a bit of a hack. All
-      // that's really needed to create the IRELATIVE is the section and value,
-      // so ideally we should just need to copy those.
-      auto *directSym = make<Defined>(cast<Defined>(sym));
-      addPltEntry(in.iplt, in.igotPlt, in.relaIplt, target->iRelativeRel,
-                  *directSym);
-      sym.pltIndex = directSym->pltIndex;
-    }
-    if (needsGot(expr)) {
-      // Redirect GOT accesses to point to the Igot.
-      //
-      // This field is also used to keep track of whether we ever needed a GOT
-      // entry. If we did and we make the PLT canonical later, we'll need to
-      // create a GOT entry pointing to the PLT entry for Sym.
-      sym.gotInIgot = true;
-    } else if (!needsPlt(expr)) {
-      // Make the ifunc's PLT entry canonical by changing the value of its
-      // symbol to redirect all references to point to it.
-      auto &d = cast<Defined>(sym);
-      d.section = in.iplt;
-      d.value = sym.pltIndex * target->ipltEntrySize;
-      d.size = 0;
-      // It's important to set the symbol type here so that dynamic loaders
-      // don't try to call the PLT as if it were an ifunc resolver.
-      d.type = STT_FUNC;
-
-      if (sym.gotInIgot) {
-        // We previously encountered a GOT generating reference that we
-        // redirected to the Igot. Now that the PLT entry is canonical we must
-        // clear the redirection to the Igot and add a GOT entry. As we've
-        // changed the symbol type to STT_FUNC future GOT generating references
-        // will naturally use this GOT entry.
-        //
-        // We don't need to worry about creating a MIPS GOT here because ifuncs
-        // aren't a thing on MIPS.
-        sym.gotInIgot = false;
-        addGotEntry(sym);
-      }
-    }
+    sym.hasDirectReloc = true;
   }
 
   processRelocAux<ELFT>(sec, expr, type, offset, sym, addend);
@@ -1609,6 +1491,179 @@ template <class ELFT> void elf::scanRelocations(InputSectionBase &s) {
     scanRelocs<ELFT>(s, rels.relas);
 }
 
+static bool handleNonPreemptibleIfunc(Symbol &sym) {
+  // Handle a reference to a non-preemptible ifunc. These are special in a
+  // few ways:
+  //
+  // - Unlike most non-preemptible symbols, non-preemptible ifuncs do not have
+  //   a fixed value. But assuming that all references to the ifunc are
+  //   GOT-generating or PLT-generating, the handling of an ifunc is
+  //   relatively straightforward. We create a PLT entry in Iplt, which is
+  //   usually at the end of .plt, which makes an indirect call using a
+  //   matching GOT entry in igotPlt, which is usually at the end of .got.plt.
+  //   The GOT entry is relocated using an IRELATIVE relocation in relaIplt,
+  //   which is usually at the end of .rela.plt. Unlike most relocations in
+  //   .rela.plt, which may be evaluated lazily without -z now, dynamic
+  //   loaders evaluate IRELATIVE relocs eagerly, which means that for
+  //   IRELATIVE relocs only, GOT-generating relocations can point directly to
+  //   .got.plt without requiring a separate GOT entry.
+  //
+  // - Despite the fact that an ifunc does not have a fixed value, compilers
+  //   that are not passed -fPIC will assume that they do, and will emit
+  //   direct (non-GOT-generating, non-PLT-generating) relocations to the
+  //   symbol. This means that if a direct relocation to the symbol is
+  //   seen, the linker must set a value for the symbol, and this value must
+  //   be consistent no matter what type of reference is made to the symbol.
+  //   This can be done by creating a PLT entry for the symbol in the way
+  //   described above and making it canonical, that is, making all references
+  //   point to the PLT entry instead of the resolver. In lld we also store
+  //   the address of the PLT entry in the dynamic symbol table, which means
+  //   that the symbol will also have the same value in other modules.
+  //   Because the value loaded from the GOT needs to be consistent with
+  //   the value computed using a direct relocation, a non-preemptible ifunc
+  //   may end up with two GOT entries, one in .got.plt that points to the
+  //   address returned by the resolver and is used only by the PLT entry,
+  //   and another in .got that points to the PLT entry and is used by
+  //   GOT-generating relocations.
+  //
+  // - The fact that these symbols do not have a fixed value makes them an
+  //   exception to the general rule that a statically linked executable does
+  //   not require any form of dynamic relocation. To handle these relocations
+  //   correctly, the IRELATIVE relocations are stored in an array which a
+  //   statically linked executable's startup code must enumerate using the
+  //   linker-defined symbols __rela?_iplt_{start,end}.
+  if (!sym.isGnuIFunc() || sym.isPreemptible || config->zIfuncNoplt)
+    return false;
+  // Skip unreferenced non-preemptible ifunc.
+  if (!(sym.needsGot || sym.needsPlt || sym.hasDirectReloc))
+    return true;
+
+  sym.isInIplt = true;
+
+  // Create an Iplt and the associated IRELATIVE relocation pointing to the
+  // original section/value pairs. For non-GOT non-PLT relocation case below, we
+  // may alter section/value, so create a copy of the symbol to make
+  // section/value fixed.
+  auto *directSym = makeDefined(cast<Defined>(sym));
+  addPltEntry(*in.iplt, *in.igotPlt, *in.relaIplt, target->iRelativeRel,
+              *directSym);
+  sym.pltIndex = directSym->pltIndex;
+
+  if (sym.hasDirectReloc) {
+    // Change the value to the IPLT and redirect all references to it.
+    auto &d = cast<Defined>(sym);
+    d.section = in.iplt;
+    d.value = sym.pltIndex * target->ipltEntrySize;
+    d.size = 0;
+    // It's important to set the symbol type here so that dynamic loaders
+    // don't try to call the PLT as if it were an ifunc resolver.
+    d.type = STT_FUNC;
+
+    if (sym.needsGot)
+      addGotEntry(sym);
+  } else if (sym.needsGot) {
+    // Redirect GOT accesses to point to the Igot.
+    sym.gotInIgot = true;
+  }
+  return true;
+}
+
+void elf::postScanRelocations() {
+  auto fn = [](Symbol &sym) {
+    if (handleNonPreemptibleIfunc(sym))
+      return;
+    if (sym.needsGot)
+      addGotEntry(sym);
+    if (sym.needsPlt)
+      addPltEntry(*in.plt, *in.gotPlt, *in.relaPlt, target->pltRel, sym);
+    if (sym.needsCopy) {
+      if (sym.isObject()) {
+        addCopyRelSymbol(cast<SharedSymbol>(sym));
+        // needsCopy is cleared for sym and its aliases so that in later
+        // iterations aliases won't cause redundant copies.
+        assert(!sym.needsCopy);
+      } else {
+        assert(sym.isFunc() && sym.needsPlt);
+        if (!sym.isDefined()) {
+          replaceWithDefined(
+              sym, *in.plt,
+              target->pltHeaderSize + target->pltEntrySize * sym.pltIndex, 0);
+          sym.needsCopy = true;
+          if (config->emachine == EM_PPC) {
+            // PPC32 canonical PLT entries are at the beginning of .glink
+            cast<Defined>(sym).value = in.plt->headerSize;
+            in.plt->headerSize += 16;
+            cast<PPC32GlinkSection>(*in.plt).canonical_plts.push_back(&sym);
+          }
+        }
+      }
+    }
+
+    if (!sym.isTls())
+      return;
+    bool isLocalInExecutable = !sym.isPreemptible && !config->shared;
+
+    if (sym.needsTlsDesc) {
+      in.got->addDynTlsEntry(sym);
+      mainPart->relaDyn->addAddendOnlyRelocIfNonPreemptible(
+          target->tlsDescRel, *in.got, in.got->getGlobalDynOffset(sym), sym,
+          target->tlsDescRel);
+    }
+    if (sym.needsTlsGd && !sym.needsTlsDesc) {
+      // TODO Support mixed TLSDESC and TLS GD.
+      in.got->addDynTlsEntry(sym);
+      uint64_t off = in.got->getGlobalDynOffset(sym);
+      if (isLocalInExecutable)
+        // Write one to the GOT slot.
+        in.got->relocations.push_back(
+            {R_ADDEND, target->symbolicRel, off, 1, &sym});
+      else
+        mainPart->relaDyn->addSymbolReloc(target->tlsModuleIndexRel, *in.got,
+                                          off, sym);
+
+      // If the symbol is preemptible we need the dynamic linker to write
+      // the offset too.
+      uint64_t offsetOff = off + config->wordsize;
+      if (sym.isPreemptible)
+        mainPart->relaDyn->addSymbolReloc(target->tlsOffsetRel, *in.got,
+                                          offsetOff, sym);
+      else
+        in.got->relocations.push_back(
+            {R_ABS, target->tlsOffsetRel, offsetOff, 0, &sym});
+    }
+    if (sym.needsTlsGdToIe) {
+      in.got->addEntry(sym);
+      mainPart->relaDyn->addSymbolReloc(target->tlsGotRel, *in.got,
+                                        sym.getGotOffset(), sym);
+    }
+
+    if (sym.needsTlsLd && in.got->addTlsIndex()) {
+      if (isLocalInExecutable)
+        in.got->relocations.push_back(
+            {R_ADDEND, target->symbolicRel, in.got->getTlsIndexOff(), 1, &sym});
+      else
+        mainPart->relaDyn->addReloc(
+            {target->tlsModuleIndexRel, in.got, in.got->getTlsIndexOff()});
+    }
+    if (sym.needsGotDtprel) {
+      in.got->addEntry(sym);
+      in.got->relocations.push_back(
+          {R_ABS, target->tlsOffsetRel, sym.getGotOffset(), 0, &sym});
+    }
+
+    if (sym.needsTlsIe && !sym.needsTlsGdToIe)
+      addTpOffsetGotEntry(sym);
+  };
+  for (Symbol *sym : symtab->symbols())
+    fn(*sym);
+
+  // Local symbols may need the aforementioned non-preemptible ifunc and GOT
+  // handling. They don't need regular PLT.
+  for (ELFFileBase *file : objectFiles)
+    for (Symbol *sym : cast<ELFFileBase>(file)->getLocalSymbols())
+      fn(*sym);
+}
+
 static bool mergeCmp(const InputSection *a, const InputSection *b) {
   // std::merge requires a strict weak ordering.
   if (a->outSecOff < b->outSecOff)
@@ -1964,8 +2019,8 @@ std::pair<Thunk *, bool> ThunkCreator::getThunk(InputSection *isec,
   // non-Thunk target, so we cannot fold offset + addend.
   if (auto *d = dyn_cast<Defined>(rel.sym))
     if (!d->isInPlt() && d->section)
-      thunkVec = &thunkedSymbolsBySectionAndAddend[{
-          {d->section->repl, d->value}, keyAddend}];
+      thunkVec = &thunkedSymbolsBySectionAndAddend[{{d->section, d->value},
+                                                    keyAddend}];
   if (!thunkVec)
     thunkVec = &thunkedSymbols[{rel.sym, keyAddend}];
 
@@ -2119,7 +2174,7 @@ void elf::hexagonTLSSymbolUpdate(ArrayRef<OutputSection *> outputSections) {
           for (Relocation &rel : isec->relocations)
             if (rel.sym->type == llvm::ELF::STT_TLS && rel.expr == R_PLT_PC) {
               if (needEntry) {
-                addPltEntry(in.plt, in.gotPlt, in.relaPlt, target->pltRel,
+                addPltEntry(*in.plt, *in.gotPlt, *in.relaPlt, target->pltRel,
                             *sym);
                 needEntry = false;
               }
diff --git a/lld/ELF/Relocations.h b/lld/ELF/Relocations.h
index 86e6cf4bc1f5..c652c0a5f70f 100644
--- a/lld/ELF/Relocations.h
+++ b/lld/ELF/Relocations.h
@@ -126,6 +126,7 @@ struct JumpInstrMod {
 // Call reportUndefinedSymbols() after calling scanRelocations() to emit
 // the diagnostics.
 template <class ELFT> void scanRelocations(InputSectionBase &);
+void postScanRelocations();
 
 template <class ELFT> void reportUndefinedSymbols();
 
diff --git a/lld/ELF/SymbolTable.cpp b/lld/ELF/SymbolTable.cpp
index e615fb70a40f..a12c5f22c4fe 100644
--- a/lld/ELF/SymbolTable.cpp
+++ b/lld/ELF/SymbolTable.cpp
@@ -29,7 +29,7 @@ using namespace llvm::ELF;
 using namespace lld;
 using namespace lld::elf;
 
-SymbolTable *elf::symtab;
+std::unique_ptr<SymbolTable> elf::symtab;
 
 void SymbolTable::wrap(Symbol *sym, Symbol *real, Symbol *wrap) {
   // Redirect __real_foo to the original foo and foo to the original __wrap_foo.
@@ -64,16 +64,18 @@ Symbol *SymbolTable::insert(StringRef name) {
   // Since this is a hot path, the following string search code is
   // optimized for speed. StringRef::find(char) is much faster than
   // StringRef::find(StringRef).
+  StringRef stem = name;
   size_t pos = name.find('@');
   if (pos != StringRef::npos && pos + 1 < name.size() && name[pos + 1] == '@')
-    name = name.take_front(pos);
-
-  auto p = symMap.insert({CachedHashStringRef(name), (int)symVector.size()});
-  int &symIndex = p.first->second;
-  bool isNew = p.second;
-
-  if (!isNew)
-    return symVector[symIndex];
+    stem = name.take_front(pos);
+
+  auto p = symMap.insert({CachedHashStringRef(stem), (int)symVector.size()});
+  if (!p.second) {
+    Symbol *sym = symVector[p.first->second];
+    if (stem.size() != name.size())
+      sym->setName(name);
+    return sym;
+  }
 
   Symbol *sym = reinterpret_cast<Symbol *>(make<SymbolUnion>());
   symVector.push_back(sym);
@@ -105,10 +107,7 @@ Symbol *SymbolTable::find(StringRef name) {
   auto it = symMap.find(CachedHashStringRef(name));
   if (it == symMap.end())
     return nullptr;
-  Symbol *sym = symVector[it->second];
-  if (sym->isPlaceholder())
-    return nullptr;
-  return sym;
+  return symVector[it->second];
 }
 
 // A version script/dynamic list is only meaningful for a Defined symbol.
@@ -131,7 +130,7 @@ static bool canBeVersioned(const Symbol &sym) {
 // other than trying to match a pattern against all demangled symbols.
 // So, if "extern C++" feature is used, we need to demangle all known
 // symbols.
-StringMap<std::vector<Symbol *>> &SymbolTable::getDemangledSyms() {
+StringMap<SmallVector<Symbol *, 0>> &SymbolTable::getDemangledSyms() {
   if (!demangledSyms) {
     demangledSyms.emplace();
     std::string demangled;
@@ -152,7 +151,7 @@ StringMap<std::vector<Symbol *>> &SymbolTable::getDemangledSyms() {
   return *demangledSyms;
 }
 
-std::vector<Symbol *> SymbolTable::findByVersion(SymbolVersion ver) {
+SmallVector<Symbol *, 0> SymbolTable::findByVersion(SymbolVersion ver) {
   if (ver.isExternCpp)
     return getDemangledSyms().lookup(ver.name);
   if (Symbol *sym = find(ver.name))
@@ -161,9 +160,9 @@ std::vector<Symbol *> SymbolTable::findByVersion(SymbolVersion ver) {
   return {};
 }
 
-std::vector<Symbol *> SymbolTable::findAllByVersion(SymbolVersion ver,
-                                                    bool includeNonDefault) {
-  std::vector<Symbol *> res;
+SmallVector<Symbol *, 0> SymbolTable::findAllByVersion(SymbolVersion ver,
+                                                       bool includeNonDefault) {
+  SmallVector<Symbol *, 0> res;
   SingleStringMatcher m(ver.name);
   auto check = [&](StringRef name) {
     size_t pos = name.find('@');
@@ -189,8 +188,8 @@ std::vector<Symbol *> SymbolTable::findAllByVersion(SymbolVersion ver,
 }
 
 void SymbolTable::handleDynamicList() {
+  SmallVector<Symbol *, 0> syms;
   for (SymbolVersion &ver : config->dynamicList) {
-    std::vector<Symbol *> syms;
     if (ver.hasWildcard)
       syms = findAllByVersion(ver, /*includeNonDefault=*/true);
     else
@@ -207,7 +206,7 @@ bool SymbolTable::assignExactVersion(SymbolVersion ver, uint16_t versionId,
                                      StringRef versionName,
                                      bool includeNonDefault) {
   // Get a list of symbols which we need to assign the version to.
-  std::vector<Symbol *> syms = findByVersion(ver);
+  SmallVector<Symbol *, 0> syms = findByVersion(ver);
 
   auto getName = [](uint16_t ver) -> std::string {
     if (ver == VER_NDX_LOCAL)
@@ -228,7 +227,7 @@ bool SymbolTable::assignExactVersion(SymbolVersion ver, uint16_t versionId,
 
     // If the version has not been assigned, verdefIndex is -1. Use an arbitrary
     // number (0) to indicate the version has been assigned.
-    if (sym->verdefIndex == UINT32_C(-1)) {
+    if (sym->verdefIndex == uint16_t(-1)) {
       sym->verdefIndex = 0;
       sym->versionId = versionId;
     }
@@ -247,7 +246,7 @@ void SymbolTable::assignWildcardVersion(SymbolVersion ver, uint16_t versionId,
   // so we set a version to a symbol only if no version has been assigned
   // to the symbol. This behavior is compatible with GNU.
   for (Symbol *sym : findAllByVersion(ver, includeNonDefault))
-    if (sym->verdefIndex == UINT32_C(-1)) {
+    if (sym->verdefIndex == uint16_t(-1)) {
       sym->verdefIndex = 0;
       sym->versionId = versionId;
     }
@@ -262,7 +261,6 @@ void SymbolTable::scanVersionScript() {
   SmallString<128> buf;
   // First, we assign versions to exact matching symbols,
   // i.e. version definitions not containing any glob meta-characters.
-  std::vector<Symbol *> syms;
   for (VersionDefinition &v : config->versionDefinitions) {
     auto assignExact = [&](SymbolVersion pat, uint16_t id, StringRef ver) {
       bool found =
diff --git a/lld/ELF/SymbolTable.h b/lld/ELF/SymbolTable.h
index 54c4b1169ed1..84d93a3dc786 100644
--- a/lld/ELF/SymbolTable.h
+++ b/lld/ELF/SymbolTable.h
@@ -35,8 +35,9 @@ class SymbolTable {
   struct FilterOutPlaceholder {
     bool operator()(Symbol *S) const { return !S->isPlaceholder(); }
   };
-  using iterator = llvm::filter_iterator<std::vector<Symbol *>::const_iterator,
-                                         FilterOutPlaceholder>;
+  using iterator =
+      llvm::filter_iterator<SmallVector<Symbol *, 0>::const_iterator,
+                            FilterOutPlaceholder>;
 
 public:
   llvm::iterator_range<iterator> symbols() const {
@@ -64,11 +65,11 @@ public:
   llvm::DenseMap<llvm::CachedHashStringRef, const InputFile *> comdatGroups;
 
 private:
-  std::vector<Symbol *> findByVersion(SymbolVersion ver);
-  std::vector<Symbol *> findAllByVersion(SymbolVersion ver,
-                                         bool includeNonDefault);
+  SmallVector<Symbol *, 0> findByVersion(SymbolVersion ver);
+  SmallVector<Symbol *, 0> findAllByVersion(SymbolVersion ver,
+                                            bool includeNonDefault);
 
-  llvm::StringMap<std::vector<Symbol *>> &getDemangledSyms();
+  llvm::StringMap<SmallVector<Symbol *, 0>> &getDemangledSyms();
   bool assignExactVersion(SymbolVersion ver, uint16_t versionId,
                           StringRef versionName, bool includeNonDefault);
   void assignWildcardVersion(SymbolVersion ver, uint16_t versionId,
@@ -82,16 +83,16 @@ private:
   // FIXME: Experiment with passing in a custom hashing or sorting the symbols
   // once symbol resolution is finished.
   llvm::DenseMap<llvm::CachedHashStringRef, int> symMap;
-  std::vector<Symbol *> symVector;
+  SmallVector<Symbol *, 0> symVector;
 
   // A map from demangled symbol names to their symbol objects.
   // This mapping is 1:N because two symbols with different versions
   // can have the same name. We use this map to handle "extern C++ {}"
   // directive in version scripts.
-  llvm::Optional<llvm::StringMap<std::vector<Symbol *>>> demangledSyms;
+  llvm::Optional<llvm::StringMap<SmallVector<Symbol *, 0>>> demangledSyms;
 };
 
-extern SymbolTable *symtab;
+extern std::unique_ptr<SymbolTable> symtab;
 
 } // namespace elf
 } // namespace lld
diff --git a/lld/ELF/Symbols.cpp b/lld/ELF/Symbols.cpp
index 8c410b4d5bfb..20301497a059 100644
--- a/lld/ELF/Symbols.cpp
+++ b/lld/ELF/Symbols.cpp
@@ -67,7 +67,7 @@ DenseMap<const Symbol *, std::pair<const InputFile *, const InputFile *>>
 SmallVector<std::tuple<std::string, const InputFile *, const Symbol &>, 0>
     elf::whyExtract;
 
-static uint64_t getSymVA(const Symbol &sym, int64_t &addend) {
+static uint64_t getSymVA(const Symbol &sym, int64_t addend) {
   switch (sym.kind()) {
   case Symbol::DefinedKind: {
     auto &d = cast<Defined>(sym);
@@ -78,7 +78,6 @@ static uint64_t getSymVA(const Symbol &sym, int64_t &addend) {
       return d.value;
 
     assert(isec != &InputSection::discarded);
-    isec = isec->repl;
 
     uint64_t offset = d.value;
 
@@ -93,10 +92,8 @@ static uint64_t getSymVA(const Symbol &sym, int64_t &addend) {
     // To make this work, we incorporate the addend into the section
     // offset (and zero out the addend for later processing) so that
     // we find the right object in the section.
-    if (d.isSection()) {
+    if (d.isSection())
       offset += addend;
-      addend = 0;
-    }
 
     // In the typical case, this is actually very simple and boils
     // down to adding together 3 numbers:
@@ -109,6 +106,8 @@ static uint64_t getSymVA(const Symbol &sym, int64_t &addend) {
     // line (and how they get built), then you have a pretty good
     // understanding of the linker.
     uint64_t va = isec->getVA(offset);
+    if (d.isSection())
+      va -= addend;
 
     // MIPS relocatable files can mix regular and microMIPS code.
     // Linker needs to distinguish such code. To do so microMIPS
@@ -120,7 +119,7 @@ static uint64_t getSymVA(const Symbol &sym, int64_t &addend) {
     // field etc) do the same trick as compiler uses to mark microMIPS
     // for CPU - set the less-significant bit.
     if (config->emachine == EM_MIPS && isMicroMips() &&
-        ((sym.stOther & STO_MIPS_MICROMIPS) || sym.needsPltAddr))
+        ((sym.stOther & STO_MIPS_MICROMIPS) || sym.needsCopy))
       va |= 1;
 
     if (d.isTls() && !config->relocatable) {
@@ -152,8 +151,7 @@ static uint64_t getSymVA(const Symbol &sym, int64_t &addend) {
 }
 
 uint64_t Symbol::getVA(int64_t addend) const {
-  uint64_t outVA = getSymVA(*this, addend);
-  return outVA + addend;
+  return getSymVA(*this, addend) + addend;
 }
 
 uint64_t Symbol::getGotVA() const {
@@ -201,7 +199,7 @@ uint64_t Symbol::getSize() const {
 OutputSection *Symbol::getOutputSection() const {
   if (auto *s = dyn_cast<Defined>(this)) {
     if (auto *sec = s->section)
-      return sec->repl->getOutputSection();
+      return sec->getOutputSection();
     return nullptr;
   }
   return nullptr;
@@ -215,7 +213,7 @@ void Symbol::parseSymbolVersion() {
     return;
   StringRef s = getName();
   size_t pos = s.find('@');
-  if (pos == 0 || pos == StringRef::npos)
+  if (pos == StringRef::npos)
     return;
   StringRef verstr = s.substr(pos + 1);
   if (verstr.empty())
@@ -257,10 +255,12 @@ void Symbol::parseSymbolVersion() {
 }
 
 void Symbol::extract() const {
-  if (auto *sym = dyn_cast<LazyArchive>(this))
+  if (auto *sym = dyn_cast<LazyArchive>(this)) {
     cast<ArchiveFile>(sym->file)->extract(sym->sym);
-  else
-    cast<LazyObjFile>(this->file)->extract();
+  } else if (file->lazy) {
+    file->lazy = false;
+    parseFile(file);
+  }
 }
 
 MemoryBufferRef LazyArchive::getMemberBuffer() {
@@ -347,7 +347,7 @@ void elf::maybeWarnUnorderableSymbol(const Symbol *sym) {
     report(": unable to order absolute symbol: ");
   else if (d && isa<OutputSection>(d->section))
     report(": unable to order synthetic symbol: ");
-  else if (d && !d->section->repl->isLive())
+  else if (d && !d->section->isLive())
     report(": unable to order discarded symbol: ");
 }
 
@@ -550,7 +550,7 @@ void Symbol::resolveUndefined(const Undefined &other) {
   }
 
   // Undefined symbols in a SharedFile do not change the binding.
-  if (dyn_cast_or_null<SharedFile>(other.file))
+  if (isa_and_nonnull<SharedFile>(other.file))
     return;
 
   if (isUndefined() || isShared()) {
@@ -562,22 +562,6 @@ void Symbol::resolveUndefined(const Undefined &other) {
   }
 }
 
-// Using .symver foo,foo@@VER unfortunately creates two symbols: foo and
-// foo@@VER. We want to effectively ignore foo, so give precedence to
-// foo@@VER.
-// FIXME: If users can transition to using
-// .symver foo,foo@@@VER
-// we can delete this hack.
-static int compareVersion(StringRef a, StringRef b) {
-  bool x = a.contains("@@");
-  bool y = b.contains("@@");
-  if (!x && y)
-    return 1;
-  if (x && !y)
-    return -1;
-  return 0;
-}
-
 // Compare two symbols. Return 1 if the new symbol should win, -1 if
 // the new symbol should lose, or 0 if there is a conflict.
 int Symbol::compare(const Symbol *other) const {
@@ -586,8 +570,16 @@ int Symbol::compare(const Symbol *other) const {
   if (!isDefined() && !isCommon())
     return 1;
 
-  if (int cmp = compareVersion(getName(), other->getName()))
-    return cmp;
+  // .symver foo,foo@@VER unfortunately creates two defined symbols: foo and
+  // foo@@VER. In GNU ld, if foo and foo@@VER are in the same file, foo is
+  // ignored. In our implementation, when this is foo, this->getName() may still
+  // contain @@, return 1 in this case as well.
+  if (file == other->file) {
+    if (other->getName().contains("@@"))
+      return 1;
+    if (getName().contains("@@"))
+      return -1;
+  }
 
   if (other->isWeak())
     return -1;
@@ -616,7 +608,7 @@ int Symbol::compare(const Symbol *other) const {
   auto *oldSym = cast<Defined>(this);
   auto *newSym = cast<Defined>(other);
 
-  if (dyn_cast_or_null<BitcodeFile>(other->file))
+  if (isa_and_nonnull<BitcodeFile>(other->file))
     return 0;
 
   if (!oldSym->section && !newSym->section && oldSym->value == newSym->value &&
@@ -720,8 +712,7 @@ template <class LazyT> void Symbol::resolveLazy(const LazyT &other) {
         return;
       }
     } else if (auto *loSym = dyn_cast<LazyObject>(&other)) {
-      LazyObjFile *obj = cast<LazyObjFile>(loSym->file);
-      if (obj->shouldExtractForCommon(loSym->getName())) {
+      if (loSym->file->shouldExtractForCommon(loSym->getName())) {
         replaceCommon(*this, other);
         return;
       }
diff --git a/lld/ELF/Symbols.h b/lld/ELF/Symbols.h
index cc48ef0ab3b7..27c36eedce80 100644
--- a/lld/ELF/Symbols.h
+++ b/lld/ELF/Symbols.h
@@ -16,6 +16,7 @@
 #include "InputFiles.h"
 #include "InputSection.h"
 #include "lld/Common/LLVM.h"
+#include "lld/Common/Memory.h"
 #include "lld/Common/Strings.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/Object/Archive.h"
@@ -85,7 +86,7 @@ public:
   uint32_t globalDynIndex = -1;
 
   // This field is a index to the symbol's version definition.
-  uint32_t verdefIndex = -1;
+  uint16_t verdefIndex = -1;
 
   // Version definition index.
   uint16_t versionId;
@@ -245,16 +246,14 @@ protected:
         type(type), stOther(stOther), symbolKind(k), visibility(stOther & 3),
         isUsedInRegularObj(!file || file->kind() == InputFile::ObjKind),
         exportDynamic(isExportDynamic(k, visibility)), inDynamicList(false),
-        canInline(false), referenced(false), traced(false), needsPltAddr(false),
-        isInIplt(false), gotInIgot(false), isPreemptible(false),
-        used(!config->gcSections), needsTocRestore(false),
-        scriptDefined(false) {}
+        canInline(false), referenced(false), traced(false), isInIplt(false),
+        gotInIgot(false), isPreemptible(false), used(!config->gcSections),
+        folded(false), needsTocRestore(false), scriptDefined(false),
+        needsCopy(false), needsGot(false), needsPlt(false), needsTlsDesc(false),
+        needsTlsGd(false), needsTlsGdToIe(false), needsTlsLd(false),
+        needsGotDtprel(false), needsTlsIe(false), hasDirectReloc(false) {}
 
 public:
-  // True the symbol should point to its PLT entry.
-  // For SharedSymbol only.
-  uint8_t needsPltAddr : 1;
-
   // True if this symbol is in the Iplt sub-section of the Plt and the Igot
   // sub-section of the .got.plt or .got.
   uint8_t isInIplt : 1;
@@ -272,6 +271,9 @@ public:
   // which are referenced by relocations when -r or --emit-relocs is given.
   uint8_t used : 1;
 
+  // True if defined relative to a section discarded by ICF.
+  uint8_t folded : 1;
+
   // True if a call to this symbol needs to be followed by a restore of the
   // PPC64 toc pointer.
   uint8_t needsTocRestore : 1;
@@ -279,6 +281,22 @@ public:
   // True if this symbol is defined by a linker script.
   uint8_t scriptDefined : 1;
 
+  // True if this symbol needs a canonical PLT entry, or (during
+  // postScanRelocations) a copy relocation.
+  uint8_t needsCopy : 1;
+
+  // Temporary flags used to communicate which symbol entries need PLT and GOT
+  // entries during postScanRelocations();
+  uint8_t needsGot : 1;
+  uint8_t needsPlt : 1;
+  uint8_t needsTlsDesc : 1;
+  uint8_t needsTlsGd : 1;
+  uint8_t needsTlsGdToIe : 1;
+  uint8_t needsTlsLd : 1;
+  uint8_t needsGotDtprel : 1;
+  uint8_t needsTlsIe : 1;
+  uint8_t hasDirectReloc : 1;
+
   // The partition whose dynamic symbol table contains this symbol's definition.
   uint8_t partition = 1;
 
@@ -358,7 +376,7 @@ public:
 
   SharedSymbol(InputFile &file, StringRef name, uint8_t binding,
                uint8_t stOther, uint8_t type, uint64_t value, uint64_t size,
-               uint32_t alignment, uint32_t verdefIndex)
+               uint32_t alignment, uint16_t verdefIndex)
       : Symbol(SharedKind, &file, name, binding, stOther, type), value(value),
         size(size), alignment(alignment) {
     this->verdefIndex = verdefIndex;
@@ -423,7 +441,9 @@ class LazyObject : public Symbol {
 public:
   LazyObject(InputFile &file, StringRef name)
       : Symbol(LazyObjectKind, &file, name, llvm::ELF::STB_GLOBAL,
-               llvm::ELF::STV_DEFAULT, llvm::ELF::STT_NOTYPE) {}
+               llvm::ELF::STV_DEFAULT, llvm::ELF::STT_NOTYPE) {
+    isUsedInRegularObj = false;
+  }
 
   static bool classof(const Symbol *s) { return s->kind() == LazyObjectKind; }
 };
@@ -559,17 +579,18 @@ void Symbol::replace(const Symbol &newSym) {
   scriptDefined = old.scriptDefined;
   partition = old.partition;
 
-  // Symbol length is computed lazily. If we already know a symbol length,
-  // propagate it.
-  if (nameData == old.nameData && nameSize == 0 && old.nameSize != 0)
-    nameSize = old.nameSize;
-
   // Print out a log message if --trace-symbol was specified.
   // This is for debugging.
   if (traced)
     printTraceSymbol(this);
 }
 
+template <typename... T> Defined *makeDefined(T &&...args) {
+  return new (reinterpret_cast<Defined *>(
+      getSpecificAllocSingleton<SymbolUnion>().Allocate()))
+      Defined(std::forward<T>(args)...);
+}
+
 void maybeWarnUnorderableSymbol(const Symbol *sym);
 bool computeIsPreemptible(const Symbol &sym);
 void reportBackrefs();
diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp
index 4078f7e01674..e480118f5ae9 100644
--- a/lld/ELF/SyntheticSections.cpp
+++ b/lld/ELF/SyntheticSections.cpp
@@ -264,8 +264,8 @@ InputSection *elf::createInterpSection() {
 
 Defined *elf::addSyntheticLocal(StringRef name, uint8_t type, uint64_t value,
                                 uint64_t size, InputSectionBase &section) {
-  auto *s = make<Defined>(section.file, name, STB_LOCAL, STV_DEFAULT, type,
-                          value, size, &section);
+  Defined *s = makeDefined(section.file, name, STB_LOCAL, STV_DEFAULT, type,
+                           value, size, &section);
   if (in.symTab)
     in.symTab->addSymbol(s);
   return s;
@@ -391,7 +391,7 @@ Defined *EhFrameSection::isFdeLive(EhSectionPiece &fde, ArrayRef<RelTy> rels) {
   // FDEs for garbage-collected or merged-by-ICF sections, or sections in
   // another partition, are dead.
   if (auto *d = dyn_cast<Defined>(&b))
-    if (d->section && d->section->partition == partition)
+    if (!d->folded && d->section && d->section->partition == partition)
       return d;
   return nullptr;
 }
@@ -763,18 +763,18 @@ size_t MipsGotSection::FileGot::getIndexedEntriesNum() const {
 }
 
 MipsGotSection::FileGot &MipsGotSection::getGot(InputFile &f) {
-  if (!f.mipsGotIndex.hasValue()) {
+  if (f.mipsGotIndex == uint32_t(-1)) {
     gots.emplace_back();
     gots.back().file = &f;
     f.mipsGotIndex = gots.size() - 1;
   }
-  return gots[*f.mipsGotIndex];
+  return gots[f.mipsGotIndex];
 }
 
 uint64_t MipsGotSection::getPageEntryOffset(const InputFile *f,
                                             const Symbol &sym,
                                             int64_t addend) const {
-  const FileGot &g = gots[*f->mipsGotIndex];
+  const FileGot &g = gots[f->mipsGotIndex];
   uint64_t index = 0;
   if (const OutputSection *outSec = sym.getOutputSection()) {
     uint64_t secAddr = getMipsPageAddr(outSec->addr);
@@ -788,7 +788,7 @@ uint64_t MipsGotSection::getPageEntryOffset(const InputFile *f,
 
 uint64_t MipsGotSection::getSymEntryOffset(const InputFile *f, const Symbol &s,
                                            int64_t addend) const {
-  const FileGot &g = gots[*f->mipsGotIndex];
+  const FileGot &g = gots[f->mipsGotIndex];
   Symbol *sym = const_cast<Symbol *>(&s);
   if (sym->isTls())
     return g.tls.lookup(sym) * config->wordsize;
@@ -798,13 +798,13 @@ uint64_t MipsGotSection::getSymEntryOffset(const InputFile *f, const Symbol &s,
 }
 
 uint64_t MipsGotSection::getTlsIndexOffset(const InputFile *f) const {
-  const FileGot &g = gots[*f->mipsGotIndex];
+  const FileGot &g = gots[f->mipsGotIndex];
   return g.dynTlsSymbols.lookup(nullptr) * config->wordsize;
 }
 
 uint64_t MipsGotSection::getGlobalDynOffset(const InputFile *f,
                                             const Symbol &s) const {
-  const FileGot &g = gots[*f->mipsGotIndex];
+  const FileGot &g = gots[f->mipsGotIndex];
   Symbol *sym = const_cast<Symbol *>(&s);
   return g.dynTlsSymbols.lookup(sym) * config->wordsize;
 }
@@ -1007,14 +1007,14 @@ void MipsGotSection::build() {
         // thread-locals that have been marked as local through a linker script)
         if (!s->isPreemptible && !config->shared)
           continue;
-        mainPart->relaDyn->addSymbolReloc(target->tlsModuleIndexRel, this,
+        mainPart->relaDyn->addSymbolReloc(target->tlsModuleIndexRel, *this,
                                           offset, *s);
         // However, we can skip writing the TLS offset reloc for non-preemptible
         // symbols since it is known even in shared libraries
         if (!s->isPreemptible)
           continue;
         offset += config->wordsize;
-        mainPart->relaDyn->addSymbolReloc(target->tlsOffsetRel, this, offset,
+        mainPart->relaDyn->addSymbolReloc(target->tlsOffsetRel, *this, offset,
                                           *s);
       }
     }
@@ -1027,7 +1027,7 @@ void MipsGotSection::build() {
     // Dynamic relocations for "global" entries.
     for (const std::pair<Symbol *, size_t> &p : got.global) {
       uint64_t offset = p.second * config->wordsize;
-      mainPart->relaDyn->addSymbolReloc(target->relativeRel, this, offset,
+      mainPart->relaDyn->addSymbolReloc(target->relativeRel, *this, offset,
                                         *p.first);
     }
     if (!config->isPic)
@@ -1061,10 +1061,9 @@ uint64_t MipsGotSection::getGp(const InputFile *f) const {
   // For files without related GOT or files refer a primary GOT
   // returns "common" _gp value. For secondary GOTs calculate
   // individual _gp values.
-  if (!f || !f->mipsGotIndex.hasValue() || *f->mipsGotIndex == 0)
+  if (!f || f->mipsGotIndex == uint32_t(-1) || f->mipsGotIndex == 0)
     return ElfSym::mipsGp->getVA(0);
-  return getVA() + gots[*f->mipsGotIndex].startIndex * config->wordsize +
-         0x7ff0;
+  return getVA() + gots[f->mipsGotIndex].startIndex * config->wordsize + 0x7ff0;
 }
 
 void MipsGotSection::writeTo(uint8_t *buf) {
@@ -1298,8 +1297,8 @@ DynamicSection<ELFT>::computeContents() {
   auto addInt = [&](int32_t tag, uint64_t val) {
     entries.emplace_back(tag, val);
   };
-  auto addInSec = [&](int32_t tag, const InputSection *sec) {
-    entries.emplace_back(tag, sec->getVA());
+  auto addInSec = [&](int32_t tag, const InputSection &sec) {
+    entries.emplace_back(tag, sec.getVA());
   };
 
   for (StringRef s : config->filterList)
@@ -1375,7 +1374,7 @@ DynamicSection<ELFT>::computeContents() {
   if (part.relaDyn->isNeeded() ||
       (in.relaIplt->isNeeded() &&
        part.relaDyn->getParent() == in.relaIplt->getParent())) {
-    addInSec(part.relaDyn->dynamicTag, part.relaDyn);
+    addInSec(part.relaDyn->dynamicTag, *part.relaDyn);
     entries.emplace_back(part.relaDyn->sizeDynamicTag, addRelaSz(part.relaDyn));
 
     bool isRela = config->isRela;
@@ -1393,7 +1392,7 @@ DynamicSection<ELFT>::computeContents() {
   }
   if (part.relrDyn && !part.relrDyn->relocs.empty()) {
     addInSec(config->useAndroidRelrTags ? DT_ANDROID_RELR : DT_RELR,
-             part.relrDyn);
+             *part.relrDyn);
     addInt(config->useAndroidRelrTags ? DT_ANDROID_RELRSZ : DT_RELRSZ,
            part.relrDyn->getParent()->size);
     addInt(config->useAndroidRelrTags ? DT_ANDROID_RELRENT : DT_RELRENT,
@@ -1406,14 +1405,14 @@ DynamicSection<ELFT>::computeContents() {
   // case, so here we always use relaPlt as marker for the beginning of
   // .rel[a].plt section.
   if (isMain && (in.relaPlt->isNeeded() || in.relaIplt->isNeeded())) {
-    addInSec(DT_JMPREL, in.relaPlt);
+    addInSec(DT_JMPREL, *in.relaPlt);
     entries.emplace_back(DT_PLTRELSZ, addPltRelSz());
     switch (config->emachine) {
     case EM_MIPS:
-      addInSec(DT_MIPS_PLTGOT, in.gotPlt);
+      addInSec(DT_MIPS_PLTGOT, *in.gotPlt);
       break;
     case EM_SPARCV9:
-      addInSec(DT_PLTGOT, in.plt);
+      addInSec(DT_PLTGOT, *in.plt);
       break;
     case EM_AARCH64:
       if (llvm::find_if(in.relaPlt->relocs, [](const DynamicReloc &r) {
@@ -1423,7 +1422,7 @@ DynamicSection<ELFT>::computeContents() {
         addInt(DT_AARCH64_VARIANT_PCS, 0);
       LLVM_FALLTHROUGH;
     default:
-      addInSec(DT_PLTGOT, in.gotPlt);
+      addInSec(DT_PLTGOT, *in.gotPlt);
       break;
     }
     addInt(DT_PLTREL, config->isRela ? DT_RELA : DT_REL);
@@ -1436,16 +1435,16 @@ DynamicSection<ELFT>::computeContents() {
       addInt(DT_AARCH64_PAC_PLT, 0);
   }
 
-  addInSec(DT_SYMTAB, part.dynSymTab);
+  addInSec(DT_SYMTAB, *part.dynSymTab);
   addInt(DT_SYMENT, sizeof(Elf_Sym));
-  addInSec(DT_STRTAB, part.dynStrTab);
+  addInSec(DT_STRTAB, *part.dynStrTab);
   addInt(DT_STRSZ, part.dynStrTab->getSize());
   if (!config->zText)
     addInt(DT_TEXTREL, 0);
   if (part.gnuHashTab)
-    addInSec(DT_GNU_HASH, part.gnuHashTab);
+    addInSec(DT_GNU_HASH, *part.gnuHashTab);
   if (part.hashTab)
-    addInSec(DT_HASH, part.hashTab);
+    addInSec(DT_HASH, *part.hashTab);
 
   if (isMain) {
     if (Out::preinitArray) {
@@ -1470,13 +1469,13 @@ DynamicSection<ELFT>::computeContents() {
   }
 
   if (part.verSym && part.verSym->isNeeded())
-    addInSec(DT_VERSYM, part.verSym);
+    addInSec(DT_VERSYM, *part.verSym);
   if (part.verDef && part.verDef->isLive()) {
-    addInSec(DT_VERDEF, part.verDef);
+    addInSec(DT_VERDEF, *part.verDef);
     addInt(DT_VERDEFNUM, getVerDefNum());
   }
   if (part.verNeed && part.verNeed->isNeeded()) {
-    addInSec(DT_VERNEED, part.verNeed);
+    addInSec(DT_VERNEED, *part.verNeed);
     unsigned needNum = 0;
     for (SharedFile *f : sharedFiles)
       if (!f->vernauxs.empty())
@@ -1495,10 +1494,10 @@ DynamicSection<ELFT>::computeContents() {
       addInt(DT_MIPS_GOTSYM, b->dynsymIndex);
     else
       addInt(DT_MIPS_GOTSYM, part.dynSymTab->getNumSymbols());
-    addInSec(DT_PLTGOT, in.mipsGot);
+    addInSec(DT_PLTGOT, *in.mipsGot);
     if (in.mipsRldMap) {
       if (!config->pie)
-        addInSec(DT_MIPS_RLD_MAP, in.mipsRldMap);
+        addInSec(DT_MIPS_RLD_MAP, *in.mipsRldMap);
       // Store the offset to the .rld_map section
       // relative to the address of the tag.
       addInt(DT_MIPS_RLD_MAP_REL,
@@ -1509,7 +1508,7 @@ DynamicSection<ELFT>::computeContents() {
   // DT_PPC_GOT indicates to glibc Secure PLT is used. If DT_PPC_GOT is absent,
   // glibc assumes the old-style BSS PLT layout which we don't support.
   if (config->emachine == EM_PPC)
-    addInSec(DT_PPC_GOT, in.got);
+    addInSec(DT_PPC_GOT, *in.got);
 
   // Glink dynamic tag is required by the V2 abi if the plt section isn't empty.
   if (config->emachine == EM_PPC64 && in.plt->isNeeded()) {
@@ -1574,7 +1573,7 @@ RelocationBaseSection::RelocationBaseSection(StringRef name, uint32_t type,
       dynamicTag(dynamicTag), sizeDynamicTag(sizeDynamicTag) {}
 
 void RelocationBaseSection::addSymbolReloc(RelType dynType,
-                                           InputSectionBase *isec,
+                                           InputSectionBase &isec,
                                            uint64_t offsetInSec, Symbol &sym,
                                            int64_t addend,
                                            Optional<RelType> addendRelType) {
@@ -1583,7 +1582,7 @@ void RelocationBaseSection::addSymbolReloc(RelType dynType,
 }
 
 void RelocationBaseSection::addRelativeReloc(
-    RelType dynType, InputSectionBase *inputSec, uint64_t offsetInSec,
+    RelType dynType, InputSectionBase &inputSec, uint64_t offsetInSec,
     Symbol &sym, int64_t addend, RelType addendRelType, RelExpr expr) {
   // This function should only be called for non-preemptible symbols or
   // RelExpr values that refer to an address inside the output file (e.g. the
@@ -1596,11 +1595,11 @@ void RelocationBaseSection::addRelativeReloc(
 }
 
 void RelocationBaseSection::addAddendOnlyRelocIfNonPreemptible(
-    RelType dynType, InputSectionBase *isec, uint64_t offsetInSec, Symbol &sym,
+    RelType dynType, InputSectionBase &isec, uint64_t offsetInSec, Symbol &sym,
     RelType addendRelType) {
   // No need to write an addend to the section for preemptible symbols.
   if (sym.isPreemptible)
-    addReloc({dynType, isec, offsetInSec, DynamicReloc::AgainstSymbol, sym, 0,
+    addReloc({dynType, &isec, offsetInSec, DynamicReloc::AgainstSymbol, sym, 0,
               R_ABS});
   else
     addReloc(DynamicReloc::AddendOnlyWithTargetVA, dynType, isec, offsetInSec,
@@ -1608,16 +1607,16 @@ void RelocationBaseSection::addAddendOnlyRelocIfNonPreemptible(
 }
 
 void RelocationBaseSection::addReloc(DynamicReloc::Kind kind, RelType dynType,
-                                     InputSectionBase *inputSec,
+                                     InputSectionBase &inputSec,
                                      uint64_t offsetInSec, Symbol &sym,
                                      int64_t addend, RelExpr expr,
                                      RelType addendRelType) {
   // Write the addends to the relocated address if required. We skip
   // it if the written value would be zero.
   if (config->writeAddends && (expr != R_ADDEND || addend != 0))
-    inputSec->relocations.push_back(
+    inputSec.relocations.push_back(
         {expr, addendRelType, offsetInSec, addend, &sym});
-  addReloc({dynType, inputSec, offsetInSec, kind, sym, addend, expr});
+  addReloc({dynType, &inputSec, offsetInSec, kind, sym, addend, expr});
 }
 
 void RelocationBaseSection::addReloc(const DynamicReloc &reloc) {
@@ -1653,13 +1652,19 @@ RelrBaseSection::RelrBaseSection()
                        config->wordsize, ".relr.dyn") {}
 
 template <class ELFT>
-static void encodeDynamicReloc(SymbolTableBaseSection *symTab,
-                               typename ELFT::Rela *p,
+static void encodeDynamicReloc(typename ELFT::Rela *p,
                                const DynamicReloc &rel) {
+  p->r_offset = rel.r_offset;
+  p->setSymbolAndType(rel.r_sym, rel.type, config->isMips64EL);
   if (config->isRela)
-    p->r_addend = rel.computeAddend();
-  p->r_offset = rel.getOffset();
-  p->setSymbolAndType(rel.getSymIndex(symTab), rel.type, config->isMips64EL);
+    p->r_addend = rel.addend;
+}
+
+void DynamicReloc::computeRaw(SymbolTableBaseSection *symtab) {
+  r_offset = getOffset();
+  r_sym = getSymIndex(symtab);
+  addend = computeAddend();
+  kind = AddendOnly; // Catch errors
 }
 
 template <class ELFT>
@@ -1674,20 +1679,21 @@ RelocationSection<ELFT>::RelocationSection(StringRef name, bool sort)
 template <class ELFT> void RelocationSection<ELFT>::writeTo(uint8_t *buf) {
   SymbolTableBaseSection *symTab = getPartition().dynSymTab;
 
+  parallelForEach(relocs,
+                  [symTab](DynamicReloc &rel) { rel.computeRaw(symTab); });
   // Sort by (!IsRelative,SymIndex,r_offset). DT_REL[A]COUNT requires us to
   // place R_*_RELATIVE first. SymIndex is to improve locality, while r_offset
   // is to make results easier to read.
-  if (sort)
-    llvm::stable_sort(
-        relocs, [&](const DynamicReloc &a, const DynamicReloc &b) {
-          return std::make_tuple(a.type != target->relativeRel,
-                                 a.getSymIndex(symTab), a.getOffset()) <
-                 std::make_tuple(b.type != target->relativeRel,
-                                 b.getSymIndex(symTab), b.getOffset());
-        });
+  if (sort) {
+    const RelType relativeRel = target->relativeRel;
+    parallelSort(relocs, [&](const DynamicReloc &a, const DynamicReloc &b) {
+      return std::make_tuple(a.type != relativeRel, a.r_sym, a.r_offset) <
+             std::make_tuple(b.type != relativeRel, b.r_sym, b.r_offset);
+    });
+  }
 
   for (const DynamicReloc &rel : relocs) {
-    encodeDynamicReloc<ELFT>(symTab, reinterpret_cast<Elf_Rela *>(buf), rel);
+    encodeDynamicReloc<ELFT>(reinterpret_cast<Elf_Rela *>(buf), rel);
     buf += config->isRela ? sizeof(Elf_Rela) : sizeof(Elf_Rel);
   }
 }
@@ -1765,7 +1771,11 @@ bool AndroidPackedRelocationSection<ELFT>::updateAllocSize() {
 
   for (const DynamicReloc &rel : relocs) {
     Elf_Rela r;
-    encodeDynamicReloc<ELFT>(getPartition().dynSymTab, &r, rel);
+    r.r_offset = rel.getOffset();
+    r.setSymbolAndType(rel.getSymIndex(getPartition().dynSymTab), rel.type,
+                       false);
+    if (config->isRela)
+      r.r_addend = rel.computeAddend();
 
     if (r.getType(config->isMips64EL) == target->relativeRel)
       relatives.push_back(r);
@@ -2175,7 +2185,8 @@ static BssSection *getCommonSec(Symbol *sym) {
 static uint32_t getSymSectionIndex(Symbol *sym) {
   if (getCommonSec(sym))
     return SHN_COMMON;
-  if (!isa<Defined>(sym) || sym->needsPltAddr)
+  assert(!(sym->needsCopy && sym->isObject()));
+  if (!isa<Defined>(sym) || sym->needsCopy)
     return SHN_UNDEF;
   if (const OutputSection *os = sym->getOutputSection())
     return os->sectionIndex >= SHN_LORESERVE ? (uint32_t)SHN_XINDEX
@@ -2250,7 +2261,7 @@ template <class ELFT> void SymbolTableSection<ELFT>::writeTo(uint8_t *buf) {
 
     for (SymbolTableEntry &ent : symbols) {
       Symbol *sym = ent.sym;
-      if (sym->isInPlt() && sym->needsPltAddr)
+      if (sym->isInPlt() && sym->needsCopy)
         eSym->st_other |= STO_MIPS_PLT;
       if (isMicroMips()) {
         // We already set the less-significant bit for symbols
@@ -2261,7 +2272,7 @@ template <class ELFT> void SymbolTableSection<ELFT>::writeTo(uint8_t *buf) {
         // like `objdump` will be able to deal with a correct
         // symbol position.
         if (sym->isDefined() &&
-            ((sym->stOther & STO_MIPS_MICROMIPS) || sym->needsPltAddr)) {
+            ((sym->stOther & STO_MIPS_MICROMIPS) || sym->needsCopy)) {
           if (!strTabSec.isDynamic())
             eSym->st_value &= ~1;
           eSym->st_other |= STO_MIPS_MICROMIPS;
@@ -2426,10 +2437,10 @@ static uint32_t hashGnu(StringRef name) {
 // Add symbols to this symbol hash table. Note that this function
 // destructively sort a given vector -- which is needed because
 // GNU-style hash table places some sorting requirements.
-void GnuHashTableSection::addSymbols(std::vector<SymbolTableEntry> &v) {
+void GnuHashTableSection::addSymbols(SmallVectorImpl<SymbolTableEntry> &v) {
   // We cannot use 'auto' for Mid because GCC 6.1 cannot deduce
   // its type correctly.
-  std::vector<SymbolTableEntry>::iterator mid =
+  auto mid =
       std::stable_partition(v.begin(), v.end(), [&](const SymbolTableEntry &s) {
         return !s.sym->isDefined() || s.sym->partition != partition;
       });
@@ -2715,16 +2726,17 @@ void GdbIndexSection::initOutputSize() {
   }
 }
 
-static std::vector<GdbIndexSection::CuEntry> readCuList(DWARFContext &dwarf) {
-  std::vector<GdbIndexSection::CuEntry> ret;
+static SmallVector<GdbIndexSection::CuEntry, 0>
+readCuList(DWARFContext &dwarf) {
+  SmallVector<GdbIndexSection::CuEntry, 0> ret;
   for (std::unique_ptr<DWARFUnit> &cu : dwarf.compile_units())
     ret.push_back({cu->getOffset(), cu->getLength() + 4});
   return ret;
 }
 
-static std::vector<GdbIndexSection::AddressEntry>
+static SmallVector<GdbIndexSection::AddressEntry, 0>
 readAddressAreas(DWARFContext &dwarf, InputSection *sec) {
-  std::vector<GdbIndexSection::AddressEntry> ret;
+  SmallVector<GdbIndexSection::AddressEntry, 0> ret;
 
   uint32_t cuIdx = 0;
   for (std::unique_ptr<DWARFUnit> &cu : dwarf.compile_units()) {
@@ -2757,7 +2769,7 @@ readAddressAreas(DWARFContext &dwarf, InputSection *sec) {
 template <class ELFT>
 static std::vector<GdbIndexSection::NameAttrEntry>
 readPubNamesAndTypes(const LLDDwarfObj<ELFT> &obj,
-                     const std::vector<GdbIndexSection::CuEntry> &cus) {
+                     const SmallVectorImpl<GdbIndexSection::CuEntry> &cus) {
   const LLDDWARFSection &pubNames = obj.getGnuPubnamesSection();
   const LLDDWARFSection &pubTypes = obj.getGnuPubtypesSection();
 
@@ -3254,8 +3266,8 @@ void MergeTailSection::finalizeContents() {
 }
 
 void MergeNoTailSection::writeTo(uint8_t *buf) {
-  for (size_t i = 0; i < numShards; ++i)
-    shards[i].write(buf + shardOffsets[i]);
+  parallelForEachN(0, numShards,
+                   [&](size_t i) { shards[i].write(buf + shardOffsets[i]); });
 }
 
 // This function is very hot (i.e. it can take several seconds to finish)
@@ -3312,15 +3324,6 @@ void MergeNoTailSection::finalizeContents() {
   });
 }
 
-MergeSyntheticSection *elf::createMergeSynthetic(StringRef name, uint32_t type,
-                                                 uint64_t flags,
-                                                 uint32_t alignment) {
-  bool shouldTailMerge = (flags & SHF_STRINGS) && config->optimize >= 2;
-  if (shouldTailMerge)
-    return make<MergeTailSection>(name, type, flags, alignment);
-  return make<MergeNoTailSection>(name, type, flags, alignment);
-}
-
 template <class ELFT> void elf::splitSections() {
   llvm::TimeTraceScope timeScope("Split sections");
   // splitIntoPieces needs to be called on each MergeInputSection
@@ -3617,14 +3620,12 @@ void PPC32Got2Section::finalizeContents() {
   // .got2 . This function computes outSecOff of each .got2 to be used in
   // PPC32PltCallStub::writeTo(). The purpose of this empty synthetic section is
   // to collect input sections named ".got2".
-  uint32_t offset = 0;
   for (SectionCommand *cmd : getParent()->commands)
     if (auto *isd = dyn_cast<InputSectionDescription>(cmd)) {
       for (InputSection *isec : isd->sections) {
-        if (isec == this)
-          continue;
-        isec->file->ppc32Got2OutSecOff = offset;
-        offset += (uint32_t)isec->getSize();
+        // isec->file may be nullptr for MergeSyntheticSection.
+        if (isec != this && isec->file)
+          isec->file->ppc32Got2 = isec;
       }
     }
 }
diff --git a/lld/ELF/SyntheticSections.h b/lld/ELF/SyntheticSections.h
index 3d2e73071d09..c35e19cf2fb4 100644
--- a/lld/ELF/SyntheticSections.h
+++ b/lld/ELF/SyntheticSections.h
@@ -61,7 +61,7 @@ public:
 
 struct CieRecord {
   EhSectionPiece *cie = nullptr;
-  std::vector<EhSectionPiece *> fdes;
+  SmallVector<EhSectionPiece *, 0> fdes;
 };
 
 // Section for .eh_frame.
@@ -79,7 +79,7 @@ public:
 
   void addSection(EhInputSection *sec);
 
-  std::vector<EhInputSection *> sections;
+  SmallVector<EhInputSection *, 0> sections;
   size_t numFdes = 0;
 
   struct FdeData {
@@ -115,7 +115,7 @@ private:
 
   uint64_t getFdePc(uint8_t *buf, size_t off, uint8_t enc) const;
 
-  std::vector<CieRecord *> cieRecords;
+  SmallVector<CieRecord *, 0> cieRecords;
 
   // CIE records are uniquified by their contents and personality functions.
   llvm::DenseMap<std::pair<ArrayRef<uint8_t>, Symbol *>, CieRecord *> cieMap;
@@ -387,7 +387,7 @@ public:
   bool hasGotPltOffRel = false;
 
 private:
-  std::vector<const Symbol *> entries;
+  SmallVector<const Symbol *, 0> entries;
 };
 
 // The IgotPltSection is a Got associated with the PltSection for GNU Ifunc
@@ -403,7 +403,7 @@ public:
   bool isNeeded() const override { return !entries.empty(); }
 
 private:
-  std::vector<const Symbol *> entries;
+  SmallVector<const Symbol *, 0> entries;
 };
 
 class StringTableSection final : public SyntheticSection {
@@ -420,7 +420,7 @@ private:
   uint64_t size = 0;
 
   llvm::DenseMap<StringRef, unsigned> stringMap;
-  std::vector<StringRef> strings;
+  SmallVector<StringRef, 0> strings;
 };
 
 class DynamicReloc {
@@ -449,21 +449,21 @@ public:
   DynamicReloc(RelType type, const InputSectionBase *inputSec,
                uint64_t offsetInSec, Kind kind, Symbol &sym, int64_t addend,
                RelExpr expr)
-      : type(type), sym(&sym), inputSec(inputSec), offsetInSec(offsetInSec),
-        kind(kind), expr(expr), addend(addend) {}
+      : sym(&sym), inputSec(inputSec), offsetInSec(offsetInSec), type(type),
+        addend(addend), kind(kind), expr(expr) {}
   /// This constructor records a relative relocation with no symbol.
   DynamicReloc(RelType type, const InputSectionBase *inputSec,
                uint64_t offsetInSec, int64_t addend = 0)
-      : type(type), sym(nullptr), inputSec(inputSec), offsetInSec(offsetInSec),
-        kind(AddendOnly), expr(R_ADDEND), addend(addend) {}
+      : sym(nullptr), inputSec(inputSec), offsetInSec(offsetInSec), type(type),
+        addend(addend), kind(AddendOnly), expr(R_ADDEND) {}
   /// This constructor records dynamic relocation settings used by the MIPS
   /// multi-GOT implementation.
   DynamicReloc(RelType type, const InputSectionBase *inputSec,
                uint64_t offsetInSec, const OutputSection *outputSec,
                int64_t addend)
-      : type(type), sym(nullptr), inputSec(inputSec), offsetInSec(offsetInSec),
-        kind(MipsMultiGotPage), expr(R_ADDEND), addend(addend),
-        outputSec(outputSec) {}
+      : sym(nullptr), outputSec(outputSec), inputSec(inputSec),
+        offsetInSec(offsetInSec), type(type), addend(addend),
+        kind(MipsMultiGotPage), expr(R_ADDEND) {}
 
   uint64_t getOffset() const;
   uint32_t getSymIndex(SymbolTableBaseSection *symTab) const;
@@ -476,18 +476,24 @@ public:
   /// address/the address of the corresponding GOT entry/etc.
   int64_t computeAddend() const;
 
-  RelType type;
+  void computeRaw(SymbolTableBaseSection *symtab);
+
   Symbol *sym;
+  const OutputSection *outputSec = nullptr;
   const InputSectionBase *inputSec;
   uint64_t offsetInSec;
+  uint64_t r_offset;
+  RelType type;
+  uint32_t r_sym;
+  // Initially input addend, then the output addend after
+  // RelocationSection<ELFT>::writeTo.
+  int64_t addend;
 
 private:
   Kind kind;
   // The kind of expression used to calculate the added (required e.g. for
   // relative GOT relocations).
   RelExpr expr;
-  int64_t addend;
-  const OutputSection *outputSec = nullptr;
 };
 
 template <class ELFT> class DynamicSection final : public SyntheticSection {
@@ -513,22 +519,22 @@ public:
   /// using relocations on the input section (e.g. MipsGotSection::writeTo()).
   void addReloc(const DynamicReloc &reloc);
   /// Add a dynamic relocation against \p sym with an optional addend.
-  void addSymbolReloc(RelType dynType, InputSectionBase *isec,
+  void addSymbolReloc(RelType dynType, InputSectionBase &isec,
                       uint64_t offsetInSec, Symbol &sym, int64_t addend = 0,
                       llvm::Optional<RelType> addendRelType = llvm::None);
   /// Add a relative dynamic relocation that uses the target address of \p sym
   /// (i.e. InputSection::getRelocTargetVA()) + \p addend as the addend.
-  void addRelativeReloc(RelType dynType, InputSectionBase *isec,
+  void addRelativeReloc(RelType dynType, InputSectionBase &isec,
                         uint64_t offsetInSec, Symbol &sym, int64_t addend,
                         RelType addendRelType, RelExpr expr);
   /// Add a dynamic relocation using the target address of \p sym as the addend
   /// if \p sym is non-preemptible. Otherwise add a relocation against \p sym.
   void addAddendOnlyRelocIfNonPreemptible(RelType dynType,
-                                          InputSectionBase *isec,
+                                          InputSectionBase &isec,
                                           uint64_t offsetInSec, Symbol &sym,
                                           RelType addendRelType);
   void addReloc(DynamicReloc::Kind kind, RelType dynType,
-                InputSectionBase *inputSec, uint64_t offsetInSec, Symbol &sym,
+                InputSectionBase &inputSec, uint64_t offsetInSec, Symbol &sym,
                 int64_t addend, RelExpr expr, RelType addendRelType);
   bool isNeeded() const override { return !relocs.empty(); }
   size_t getSize() const override { return relocs.size() * this->entsize; }
@@ -540,7 +546,7 @@ public:
             d->type == llvm::ELF::SHT_RELR);
   }
   int32_t dynamicTag, sizeDynamicTag;
-  std::vector<DynamicReloc> relocs;
+  SmallVector<DynamicReloc, 0> relocs;
 
 protected:
   size_t numRelativeRelocs = 0;
@@ -588,7 +594,7 @@ class RelrBaseSection : public SyntheticSection {
 public:
   RelrBaseSection();
   bool isNeeded() const override { return !relocs.empty(); }
-  std::vector<RelativeReloc> relocs;
+  SmallVector<RelativeReloc, 0> relocs;
 };
 
 // RelrSection is used to encode offsets for relative relocations.
@@ -608,7 +614,7 @@ public:
   }
 
 private:
-  std::vector<Elf_Relr> relrRelocs;
+  SmallVector<Elf_Relr, 0> relrRelocs;
 };
 
 struct SymbolTableEntry {
@@ -630,7 +636,7 @@ protected:
   void sortSymTabSymbols();
 
   // A vector of symbols and their string table offsets.
-  std::vector<SymbolTableEntry> symbols;
+  SmallVector<SymbolTableEntry, 0> symbols;
 
   StringTableSection &strTabSec;
 
@@ -669,7 +675,7 @@ public:
 
   // Adds symbols to the hash table.
   // Sorts the input to satisfy GNU hash section requirements.
-  void addSymbols(std::vector<SymbolTableEntry> &symbols);
+  void addSymbols(llvm::SmallVectorImpl<SymbolTableEntry> &symbols);
 
 private:
   // See the comment in writeBloomFilter.
@@ -682,7 +688,7 @@ private:
     uint32_t bucketIdx;
   };
 
-  std::vector<Entry> symbols;
+  SmallVector<Entry, 0> symbols;
   size_t maskWords;
   size_t nBuckets = 0;
   size_t size = 0;
@@ -722,7 +728,7 @@ public:
 
   size_t headerSize;
 
-  std::vector<const Symbol *> entries;
+  SmallVector<const Symbol *, 0> entries;
 };
 
 // Used for non-preemptible ifuncs. It does not have a header. Each entry is
@@ -730,7 +736,7 @@ public:
 // runtime. PltSection can only contain entries associated with JUMP_SLOT
 // relocations, so IPLT entries are in a separate section.
 class IpltSection final : public SyntheticSection {
-  std::vector<const Symbol *> entries;
+  SmallVector<const Symbol *, 0> entries;
 
 public:
   IpltSection();
@@ -747,7 +753,7 @@ public:
   void writeTo(uint8_t *buf) override;
   size_t getSize() const override;
 
-  std::vector<const Symbol *> canonical_plts;
+  SmallVector<const Symbol *, 0> canonical_plts;
   static constexpr size_t footerSize = 64;
 };
 
@@ -780,13 +786,13 @@ public:
 
   struct GdbChunk {
     InputSection *sec;
-    std::vector<AddressEntry> addressAreas;
-    std::vector<CuEntry> compilationUnits;
+    SmallVector<AddressEntry, 0> addressAreas;
+    SmallVector<CuEntry, 0> compilationUnits;
   };
 
   struct GdbSymbol {
     llvm::CachedHashStringRef name;
-    std::vector<uint32_t> cuVector;
+    SmallVector<uint32_t, 0> cuVector;
     uint32_t nameOff;
     uint32_t cuVectorOff;
   };
@@ -859,7 +865,7 @@ private:
   StringRef getFileDefName();
 
   unsigned fileDefNameOff;
-  std::vector<unsigned> verDefNameOffs;
+  SmallVector<unsigned, 0> verDefNameOffs;
 };
 
 // The .gnu.version section specifies the required version of each symbol in the
@@ -898,7 +904,7 @@ class VersionNeedSection final : public SyntheticSection {
     std::vector<Vernaux> vernauxs;
   };
 
-  std::vector<Verneed> verneeds;
+  SmallVector<Verneed, 0> verneeds;
 
 public:
   VersionNeedSection();
@@ -915,7 +921,7 @@ public:
 class MergeSyntheticSection : public SyntheticSection {
 public:
   void addSection(MergeInputSection *ms);
-  std::vector<MergeInputSection *> sections;
+  SmallVector<MergeInputSection *, 0> sections;
 
 protected:
   MergeSyntheticSection(StringRef name, uint32_t type, uint64_t flags,
@@ -962,7 +968,7 @@ private:
 
   // String table contents
   constexpr static size_t numShards = 32;
-  std::vector<llvm::StringTableBuilder> shards;
+  SmallVector<llvm::StringTableBuilder, 0> shards;
   size_t shardOffsets[numShards];
 };
 
@@ -1120,7 +1126,7 @@ public:
   bool roundUpSizeForErrata = false;
 
 private:
-  std::vector<Thunk *> thunks;
+  SmallVector<Thunk *, 0> thunks;
   size_t size = 0;
 };
 
@@ -1151,7 +1157,7 @@ public:
   void finalizeContents() override { finalized = true; }
 
 private:
-  std::vector<std::pair<const Symbol *, int64_t>> entries;
+  SmallVector<std::pair<const Symbol *, int64_t>, 0> entries;
   llvm::DenseMap<std::pair<const Symbol *, int64_t>, uint32_t> entry_index;
   bool finalized = false;
 };
@@ -1182,8 +1188,6 @@ public:
 
 InputSection *createInterpSection();
 MergeInputSection *createCommentSection();
-MergeSyntheticSection *createMergeSynthetic(StringRef name, uint32_t type,
-                                            uint64_t flags, uint32_t alignment);
 template <class ELFT> void splitSections();
 
 template <typename ELFT> void writeEhdr(uint8_t *buf, Partition &part);
diff --git a/lld/ELF/Thunks.cpp b/lld/ELF/Thunks.cpp
index ffbc8d94a800..38de4db191f4 100644
--- a/lld/ELF/Thunks.cpp
+++ b/lld/ELF/Thunks.cpp
@@ -384,7 +384,7 @@ public:
     if (Optional<uint32_t> index =
             in.ppc64LongBranchTarget->addEntry(&dest, addend)) {
       mainPart->relaDyn->addRelativeReloc(
-          target->relativeRel, in.ppc64LongBranchTarget, *index * UINT64_C(8),
+          target->relativeRel, *in.ppc64LongBranchTarget, *index * UINT64_C(8),
           dest, addend + getPPC64GlobalEntryToLocalEntryOffset(dest.stOther),
           target->symbolicRel, R_ABS);
     }
@@ -806,8 +806,9 @@ void elf::writePPC32PltCallStub(uint8_t *buf, uint64_t gotPltVA,
     // The stub loads an address relative to r30 (.got2+Addend). Addend is
     // almost always 0x8000. The address of .got2 is different in another object
     // file, so a stub cannot be shared.
-    offset = gotPltVA - (in.ppc32Got2->getParent()->getVA() +
-                         file->ppc32Got2OutSecOff + addend);
+    offset = gotPltVA -
+             (in.ppc32Got2->getParent()->getVA() +
+              (file->ppc32Got2 ? file->ppc32Got2->outSecOff : 0) + addend);
   } else {
     // The stub loads an address relative to _GLOBAL_OFFSET_TABLE_ (which is
     // currently the address of .got).
diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp
index 07c5e2303374..497e56886b72 100644
--- a/lld/ELF/Writer.cpp
+++ b/lld/ELF/Writer.cpp
@@ -297,7 +297,7 @@ template <class ELFT> void elf::createSyntheticSections() {
     }
   }
 
-  auto add = [](SyntheticSection *sec) { inputSections.push_back(sec); };
+  auto add = [](SyntheticSection &sec) { inputSections.push_back(&sec); };
 
   in.shStrTab = make<StringTableSection>(".shstrtab", false);
 
@@ -311,7 +311,7 @@ template <class ELFT> void elf::createSyntheticSections() {
   }
 
   in.bss = make<BssSection>(".bss", 0, 1);
-  add(in.bss);
+  add(*in.bss);
 
   // If there is a SECTIONS command and a .data.rel.ro section name use name
   // .data.rel.ro.bss so that we match in the .data.rel.ro output section.
@@ -320,42 +320,42 @@ template <class ELFT> void elf::createSyntheticSections() {
       script->hasSectionsCommand && findSection(".data.rel.ro", 0);
   in.bssRelRo =
       make<BssSection>(hasDataRelRo ? ".data.rel.ro.bss" : ".bss.rel.ro", 0, 1);
-  add(in.bssRelRo);
+  add(*in.bssRelRo);
 
   // Add MIPS-specific sections.
   if (config->emachine == EM_MIPS) {
     if (!config->shared && config->hasDynSymTab) {
       in.mipsRldMap = make<MipsRldMapSection>();
-      add(in.mipsRldMap);
+      add(*in.mipsRldMap);
     }
     if (auto *sec = MipsAbiFlagsSection<ELFT>::create())
-      add(sec);
+      add(*sec);
     if (auto *sec = MipsOptionsSection<ELFT>::create())
-      add(sec);
+      add(*sec);
     if (auto *sec = MipsReginfoSection<ELFT>::create())
-      add(sec);
+      add(*sec);
   }
 
   StringRef relaDynName = config->isRela ? ".rela.dyn" : ".rel.dyn";
 
   for (Partition &part : partitions) {
-    auto add = [&](SyntheticSection *sec) {
-      sec->partition = part.getNumber();
-      inputSections.push_back(sec);
+    auto add = [&](SyntheticSection &sec) {
+      sec.partition = part.getNumber();
+      inputSections.push_back(&sec);
     };
 
     if (!part.name.empty()) {
       part.elfHeader = make<PartitionElfHeaderSection<ELFT>>();
       part.elfHeader->name = part.name;
-      add(part.elfHeader);
+      add(*part.elfHeader);
 
       part.programHeaders = make<PartitionProgramHeadersSection<ELFT>>();
-      add(part.programHeaders);
+      add(*part.programHeaders);
     }
 
     if (config->buildId != BuildIdKind::None) {
       part.buildId = make<BuildIdSection>();
-      add(part.buildId);
+      add(*part.buildId);
     }
 
     part.dynStrTab = make<StringTableSection>(".dynstr", true);
@@ -368,53 +368,53 @@ template <class ELFT> void elf::createSyntheticSections() {
           make<RelocationSection<ELFT>>(relaDynName, config->zCombreloc);
 
     if (config->hasDynSymTab) {
-      add(part.dynSymTab);
+      add(*part.dynSymTab);
 
       part.verSym = make<VersionTableSection>();
-      add(part.verSym);
+      add(*part.verSym);
 
       if (!namedVersionDefs().empty()) {
         part.verDef = make<VersionDefinitionSection>();
-        add(part.verDef);
+        add(*part.verDef);
       }
 
       part.verNeed = make<VersionNeedSection<ELFT>>();
-      add(part.verNeed);
+      add(*part.verNeed);
 
       if (config->gnuHash) {
         part.gnuHashTab = make<GnuHashTableSection>();
-        add(part.gnuHashTab);
+        add(*part.gnuHashTab);
       }
 
       if (config->sysvHash) {
         part.hashTab = make<HashTableSection>();
-        add(part.hashTab);
+        add(*part.hashTab);
       }
 
-      add(part.dynamic);
-      add(part.dynStrTab);
-      add(part.relaDyn);
+      add(*part.dynamic);
+      add(*part.dynStrTab);
+      add(*part.relaDyn);
     }
 
     if (config->relrPackDynRelocs) {
       part.relrDyn = make<RelrSection<ELFT>>();
-      add(part.relrDyn);
+      add(*part.relrDyn);
     }
 
     if (!config->relocatable) {
       if (config->ehFrameHdr) {
         part.ehFrameHdr = make<EhFrameHeader>();
-        add(part.ehFrameHdr);
+        add(*part.ehFrameHdr);
       }
       part.ehFrame = make<EhFrameSection>();
-      add(part.ehFrame);
+      add(*part.ehFrame);
     }
 
     if (config->emachine == EM_ARM && !config->relocatable) {
       // The ARMExidxsyntheticsection replaces all the individual .ARM.exidx
       // InputSections.
       part.armExidx = make<ARMExidxSyntheticSection>();
-      add(part.armExidx);
+      add(*part.armExidx);
     }
   }
 
@@ -424,39 +424,39 @@ template <class ELFT> void elf::createSyntheticSections() {
     // special handling (see createPhdrs() and combineEhSections()).
     in.partEnd = make<BssSection>(".part.end", config->maxPageSize, 1);
     in.partEnd->partition = 255;
-    add(in.partEnd);
+    add(*in.partEnd);
 
     in.partIndex = make<PartitionIndexSection>();
     addOptionalRegular("__part_index_begin", in.partIndex, 0);
     addOptionalRegular("__part_index_end", in.partIndex,
                        in.partIndex->getSize());
-    add(in.partIndex);
+    add(*in.partIndex);
   }
 
   // Add .got. MIPS' .got is so different from the other archs,
   // it has its own class.
   if (config->emachine == EM_MIPS) {
     in.mipsGot = make<MipsGotSection>();
-    add(in.mipsGot);
+    add(*in.mipsGot);
   } else {
     in.got = make<GotSection>();
-    add(in.got);
+    add(*in.got);
   }
 
   if (config->emachine == EM_PPC) {
     in.ppc32Got2 = make<PPC32Got2Section>();
-    add(in.ppc32Got2);
+    add(*in.ppc32Got2);
   }
 
   if (config->emachine == EM_PPC64) {
     in.ppc64LongBranchTarget = make<PPC64LongBranchTargetSection>();
-    add(in.ppc64LongBranchTarget);
+    add(*in.ppc64LongBranchTarget);
   }
 
   in.gotPlt = make<GotPltSection>();
-  add(in.gotPlt);
+  add(*in.gotPlt);
   in.igotPlt = make<IgotPltSection>();
-  add(in.igotPlt);
+  add(*in.igotPlt);
 
   // _GLOBAL_OFFSET_TABLE_ is defined relative to either .got.plt or .got. Treat
   // it as a relocation and ensure the referenced section is created.
@@ -468,13 +468,13 @@ template <class ELFT> void elf::createSyntheticSections() {
   }
 
   if (config->gdbIndex)
-    add(GdbIndexSection::create<ELFT>());
+    add(*GdbIndexSection::create<ELFT>());
 
   // We always need to add rel[a].plt to output if it has entries.
   // Even for static linking it can contain R_[*]_IRELATIVE relocations.
   in.relaPlt = make<RelocationSection<ELFT>>(
       config->isRela ? ".rela.plt" : ".rel.plt", /*sort=*/false);
-  add(in.relaPlt);
+  add(*in.relaPlt);
 
   // The relaIplt immediately follows .rel[a].dyn to ensure that the IRelative
   // relocations are processed last by the dynamic loader. We cannot place the
@@ -485,22 +485,22 @@ template <class ELFT> void elf::createSyntheticSections() {
   in.relaIplt = make<RelocationSection<ELFT>>(
       config->androidPackDynRelocs ? in.relaPlt->name : relaDynName,
       /*sort=*/false);
-  add(in.relaIplt);
+  add(*in.relaIplt);
 
   if ((config->emachine == EM_386 || config->emachine == EM_X86_64) &&
       (config->andFeatures & GNU_PROPERTY_X86_FEATURE_1_IBT)) {
     in.ibtPlt = make<IBTPltSection>();
-    add(in.ibtPlt);
+    add(*in.ibtPlt);
   }
 
   in.plt = config->emachine == EM_PPC ? make<PPC32GlinkSection>()
                                       : make<PltSection>();
-  add(in.plt);
+  add(*in.plt);
   in.iplt = make<IpltSection>();
-  add(in.iplt);
+  add(*in.iplt);
 
   if (config->andFeatures)
-    add(make<GnuPropertySection>());
+    add(*make<GnuPropertySection>());
 
   // .note.GNU-stack is always added when we are creating a re-linkable
   // object file. Other linkers are using the presence of this marker
@@ -508,15 +508,15 @@ template <class ELFT> void elf::createSyntheticSections() {
   // is irrelevant these days. Stack area should always be non-executable
   // by default. So we emit this section unconditionally.
   if (config->relocatable)
-    add(make<GnuStackSection>());
+    add(*make<GnuStackSection>());
 
   if (in.symTab)
-    add(in.symTab);
+    add(*in.symTab);
   if (in.symTabShndx)
-    add(in.symTabShndx);
-  add(in.shStrTab);
+    add(*in.symTabShndx);
+  add(*in.shStrTab);
   if (in.strTab)
-    add(in.strTab);
+    add(*in.strTab);
 }
 
 // The main function of the writer.
@@ -622,7 +622,7 @@ template <class ELFT> static void markUsedLocalSymbols() {
     return;
   // Without --gc-sections, the field is initialized with "true".
   // Drop the flag first and then rise for symbols referenced in relocations.
-  for (InputFile *file : objectFiles) {
+  for (ELFFileBase *file : objectFiles) {
     ObjFile<ELFT> *f = cast<ObjFile<ELFT>>(file);
     for (Symbol *b : f->getLocalSymbols())
       b->used = false;
@@ -674,15 +674,11 @@ static bool shouldKeepInSymtab(const Defined &sym) {
 }
 
 static bool includeInSymtab(const Symbol &b) {
-  if (!b.isLocal() && !b.isUsedInRegularObj)
-    return false;
-
   if (auto *d = dyn_cast<Defined>(&b)) {
     // Always include absolute symbols.
     SectionBase *sec = d->section;
     if (!sec)
       return true;
-    sec = sec->repl;
 
     // Exclude symbols pointing to garbage-collected sections.
     if (isa<InputSectionBase>(sec) && !sec->isLive())
@@ -704,20 +700,16 @@ template <class ELFT> void Writer<ELFT>::copyLocalSymbols() {
   llvm::TimeTraceScope timeScope("Add local symbols");
   if (config->copyRelocs && config->discard != DiscardPolicy::None)
     markUsedLocalSymbols<ELFT>();
-  for (InputFile *file : objectFiles) {
-    ObjFile<ELFT> *f = cast<ObjFile<ELFT>>(file);
-    for (Symbol *b : f->getLocalSymbols()) {
+  for (ELFFileBase *file : objectFiles) {
+    for (Symbol *b : file->getLocalSymbols()) {
       assert(b->isLocal() && "should have been caught in initializeSymbols()");
       auto *dr = dyn_cast<Defined>(b);
 
       // No reason to keep local undefined symbol in symtab.
       if (!dr)
         continue;
-      if (!includeInSymtab(*b))
-        continue;
-      if (!shouldKeepInSymtab(*dr))
-        continue;
-      in.symTab->addSymbol(b);
+      if (includeInSymtab(*b) && shouldKeepInSymtab(*dr))
+        in.symTab->addSymbol(b);
     }
   }
 }
@@ -753,10 +745,9 @@ template <class ELFT> void Writer<ELFT>::addSectionSymbols() {
     // Set the symbol to be relative to the output section so that its st_value
     // equals the output section address. Note, there may be a gap between the
     // start of the output section and isec.
-    auto *sym =
-        make<Defined>(isec->file, "", STB_LOCAL, /*stOther=*/0, STT_SECTION,
-                      /*value=*/0, /*size=*/0, isec->getOutputSection());
-    in.symTab->addSymbol(sym);
+    in.symTab->addSymbol(
+        makeDefined(isec->file, "", STB_LOCAL, /*stOther=*/0, STT_SECTION,
+                    /*value=*/0, /*size=*/0, isec->getOutputSection()));
   }
 }
 
@@ -1242,7 +1233,7 @@ static void maybeShuffle(DenseMap<const InputSectionBase *, int> &order) {
   if (config->shuffleSections.empty())
     return;
 
-  std::vector<InputSectionBase *> matched, sections = inputSections;
+  SmallVector<InputSectionBase *, 0> matched, sections = inputSections;
   matched.reserve(sections.size());
   for (const auto &patAndSeed : config->shuffleSections) {
     matched.clear();
@@ -1310,7 +1301,7 @@ static DenseMap<const InputSectionBase *, int> buildSectionOrder() {
 
     if (auto *d = dyn_cast<Defined>(&sym)) {
       if (auto *sec = dyn_cast_or_null<InputSectionBase>(d->section)) {
-        int &priority = sectionOrder[cast<InputSectionBase>(sec->repl)];
+        int &priority = sectionOrder[cast<InputSectionBase>(sec)];
         priority = std::min(priority, ent.priority);
       }
     }
@@ -1322,7 +1313,7 @@ static DenseMap<const InputSectionBase *, int> buildSectionOrder() {
     if (!sym->isLazy())
       addSym(*sym);
 
-  for (InputFile *file : objectFiles)
+  for (ELFFileBase *file : objectFiles)
     for (Symbol *sym : file->getSymbols()) {
       if (!sym->isLocal())
         break;
@@ -1477,13 +1468,6 @@ template <class ELFT> void Writer<ELFT>::sortSections() {
     if (!os)
       continue;
     os->sortRank = getSectionRank(os);
-
-    // We want to assign rude approximation values to outSecOff fields
-    // to know the relative order of the input sections. We use it for
-    // sorting SHF_LINK_ORDER sections. See resolveShfLinkOrder().
-    uint64_t i = 0;
-    for (InputSection *sec : getInputSections(os))
-      sec->outSecOff = i++;
   }
 
   if (!script->hasSectionsCommand) {
@@ -1740,7 +1724,7 @@ static void fixSymbolsAfterShrinking() {
       if (!sec)
         return;
 
-      const InputSectionBase *inputSec = dyn_cast<InputSectionBase>(sec->repl);
+      const InputSectionBase *inputSec = dyn_cast<InputSectionBase>(sec);
       if (!inputSec || !inputSec->bytesDropped)
         return;
 
@@ -1947,6 +1931,7 @@ template <class ELFT> void Writer<ELFT>::finalizeSections() {
     if (!config->relocatable) {
       forEachRelSec(scanRelocations<ELFT>);
       reportUndefinedSymbols<ELFT>();
+      postScanRelocations();
     }
   }
 
@@ -1986,7 +1971,7 @@ template <class ELFT> void Writer<ELFT>::finalizeSections() {
     // Now that we have defined all possible global symbols including linker-
     // synthesized ones. Visit all symbols to give the finishing touches.
     for (Symbol *sym : symtab->symbols()) {
-      if (!includeInSymtab(*sym))
+      if (!sym->isUsedInRegularObj || !includeInSymtab(*sym))
         continue;
       if (in.symTab)
         in.symTab->addSymbol(sym);
@@ -2894,6 +2879,8 @@ template <class ELFT> void Writer<ELFT>::writeTrapInstr() {
 
 // Write section contents to a mmap'ed file.
 template <class ELFT> void Writer<ELFT>::writeSections() {
+  llvm::TimeTraceScope timeScope("Write sections");
+
   // In -r or --emit-relocs mode, write the relocation sections first as in
   // ELf_Rel targets we might find out that we need to modify the relocated
   // section while doing it.
diff --git a/lld/MachO/ConcatOutputSection.cpp b/lld/MachO/ConcatOutputSection.cpp
index 46cd15a40025..9f71e81b073a 100644
--- a/lld/MachO/ConcatOutputSection.cpp
+++ b/lld/MachO/ConcatOutputSection.cpp
@@ -124,13 +124,13 @@ bool ConcatOutputSection::needsThunks() const {
   if (!target->usesThunks())
     return false;
   uint64_t isecAddr = addr;
-  for (InputSection *isec : inputs)
+  for (ConcatInputSection *isec : inputs)
     isecAddr = alignTo(isecAddr, isec->align) + isec->getSize();
   if (isecAddr - addr + in.stubs->getSize() <=
       std::min(target->backwardBranchRange, target->forwardBranchRange))
     return false;
   // Yes, this program is large enough to need thunks.
-  for (InputSection *isec : inputs) {
+  for (ConcatInputSection *isec : inputs) {
     for (Reloc &r : isec->relocs) {
       if (!target->hasAttr(r.type, RelocAttrBits::BRANCH))
         continue;
@@ -143,9 +143,8 @@ bool ConcatOutputSection::needsThunks() const {
       // might need to create more for this referent at the time we are
       // estimating distance to __stubs in estimateStubsInRangeVA().
       ++thunkInfo.callSiteCount;
-      // Knowing InputSection call site count will help us avoid work on those
-      // that have no BRANCH relocs.
-      ++isec->callSiteCount;
+      // We can avoid work on InputSections that have no BRANCH relocs.
+      isec->hasCallSites = true;
     }
   }
   return true;
@@ -250,7 +249,7 @@ void ConcatOutputSection::finalize() {
                                     isecVA + forwardBranchRange - slop)
       finalizeOne(inputs[finalIdx++]);
 
-    if (isec->callSiteCount == 0)
+    if (!isec->hasCallSites)
       continue;
 
     if (finalIdx == endIdx && stubsInRangeVA == TargetInfo::outOfRangeVA) {
diff --git a/lld/MachO/InputFiles.cpp b/lld/MachO/InputFiles.cpp
index 558de4131cb9..c596b8c1ff32 100644
--- a/lld/MachO/InputFiles.cpp
+++ b/lld/MachO/InputFiles.cpp
@@ -879,8 +879,6 @@ template <class LP> void ObjFile::parse() {
                        sections[i].subsections);
 
   parseDebugInfo();
-  if (config->emitDataInCodeInfo)
-    parseDataInCode();
   if (compactUnwindSection)
     registerCompactUnwind();
 }
@@ -908,19 +906,14 @@ void ObjFile::parseDebugInfo() {
   compileUnit = it->get();
 }
 
-void ObjFile::parseDataInCode() {
+ArrayRef<data_in_code_entry> ObjFile::getDataInCode() const {
   const auto *buf = reinterpret_cast<const uint8_t *>(mb.getBufferStart());
   const load_command *cmd = findCommand(buf, LC_DATA_IN_CODE);
   if (!cmd)
-    return;
+    return {};
   const auto *c = reinterpret_cast<const linkedit_data_command *>(cmd);
-  dataInCodeEntries = {
-      reinterpret_cast<const data_in_code_entry *>(buf + c->dataoff),
-      c->datasize / sizeof(data_in_code_entry)};
-  assert(is_sorted(dataInCodeEntries, [](const data_in_code_entry &lhs,
-                                         const data_in_code_entry &rhs) {
-    return lhs.offset < rhs.offset;
-  }));
+  return {reinterpret_cast<const data_in_code_entry *>(buf + c->dataoff),
+          c->datasize / sizeof(data_in_code_entry)};
 }
 
 // Create pointers from symbols to their associated compact unwind entries.
@@ -1154,16 +1147,34 @@ DylibFile::DylibFile(MemoryBufferRef mb, DylibFile *umbrella,
   exportingFile = isImplicitlyLinked(installName) ? this : this->umbrella;
   if (const load_command *cmd = findCommand(hdr, LC_DYLD_INFO_ONLY)) {
     auto *c = reinterpret_cast<const dyld_info_command *>(cmd);
+    struct TrieEntry {
+      StringRef name;
+      uint64_t flags;
+    };
+
+    std::vector<TrieEntry> entries;
+    // Find all the $ld$* symbols to process first.
     parseTrie(buf + c->export_off, c->export_size,
               [&](const Twine &name, uint64_t flags) {
                 StringRef savedName = saver.save(name);
                 if (handleLDSymbol(savedName))
                   return;
-                bool isWeakDef = flags & EXPORT_SYMBOL_FLAGS_WEAK_DEFINITION;
-                bool isTlv = flags & EXPORT_SYMBOL_FLAGS_KIND_THREAD_LOCAL;
-                symbols.push_back(symtab->addDylib(savedName, exportingFile,
-                                                   isWeakDef, isTlv));
+                entries.push_back({savedName, flags});
               });
+
+    // Process the "normal" symbols.
+    for (TrieEntry &entry : entries) {
+      if (exportingFile->hiddenSymbols.contains(
+              CachedHashStringRef(entry.name)))
+        continue;
+
+      bool isWeakDef = entry.flags & EXPORT_SYMBOL_FLAGS_WEAK_DEFINITION;
+      bool isTlv = entry.flags & EXPORT_SYMBOL_FLAGS_KIND_THREAD_LOCAL;
+
+      symbols.push_back(
+          symtab->addDylib(entry.name, exportingFile, isWeakDef, isTlv));
+    }
+
   } else {
     error("LC_DYLD_INFO_ONLY not found in " + toString(this));
     return;
@@ -1238,20 +1249,36 @@ DylibFile::DylibFile(const InterfaceFile &interface, DylibFile *umbrella,
 
   exportingFile = isImplicitlyLinked(installName) ? this : umbrella;
   auto addSymbol = [&](const Twine &name) -> void {
-    symbols.push_back(symtab->addDylib(saver.save(name), exportingFile,
+    StringRef savedName = saver.save(name);
+    if (exportingFile->hiddenSymbols.contains(CachedHashStringRef(savedName)))
+      return;
+
+    symbols.push_back(symtab->addDylib(savedName, exportingFile,
                                        /*isWeakDef=*/false,
                                        /*isTlv=*/false));
   };
-  // TODO(compnerd) filter out symbols based on the target platform
-  // TODO: handle weak defs, thread locals
+
+  std::vector<const llvm::MachO::Symbol *> normalSymbols;
+  normalSymbols.reserve(interface.symbolsCount());
   for (const auto *symbol : interface.symbols()) {
     if (!symbol->getArchitectures().has(config->arch()))
       continue;
-
     if (handleLDSymbol(symbol->getName()))
       continue;
 
     switch (symbol->getKind()) {
+    case SymbolKind::GlobalSymbol:               // Fallthrough
+    case SymbolKind::ObjectiveCClass:            // Fallthrough
+    case SymbolKind::ObjectiveCClassEHType:      // Fallthrough
+    case SymbolKind::ObjectiveCInstanceVariable: // Fallthrough
+      normalSymbols.push_back(symbol);
+    }
+  }
+
+  // TODO(compnerd) filter out symbols based on the target platform
+  // TODO: handle weak defs, thread locals
+  for (const auto *symbol : normalSymbols) {
+    switch (symbol->getKind()) {
     case SymbolKind::GlobalSymbol:
       addSymbol(symbol->getName());
       break;
@@ -1296,6 +1323,8 @@ bool DylibFile::handleLDSymbol(StringRef originalName) {
     handleLDPreviousSymbol(name, originalName);
   else if (action == "install_name")
     handleLDInstallNameSymbol(name, originalName);
+  else if (action == "hide")
+    handleLDHideSymbol(name, originalName);
   return true;
 }
 
@@ -1364,6 +1393,29 @@ void DylibFile::handleLDInstallNameSymbol(StringRef name,
     this->installName = saver.save(installName);
 }
 
+void DylibFile::handleLDHideSymbol(StringRef name, StringRef originalName) {
+  StringRef symbolName;
+  bool shouldHide = true;
+  if (name.startswith("os")) {
+    // If it's hidden based on versions.
+    name = name.drop_front(2);
+    StringRef minVersion;
+    std::tie(minVersion, symbolName) = name.split('$');
+    VersionTuple versionTup;
+    if (versionTup.tryParse(minVersion)) {
+      warn("Failed to parse hidden version, symbol `" + originalName +
+           "` ignored.");
+      return;
+    }
+    shouldHide = versionTup == config->platformInfo.minimum;
+  } else {
+    symbolName = name;
+  }
+
+  if (shouldHide)
+    exportingFile->hiddenSymbols.insert(CachedHashStringRef(symbolName));
+}
+
 void DylibFile::checkAppExtensionSafety(bool dylibIsAppExtensionSafe) const {
   if (config->applicationExtension && !dylibIsAppExtensionSafe)
     warn("using '-application_extension' with unsafe dylib: " + toString(this));
@@ -1445,9 +1497,8 @@ static macho::Symbol *createBitcodeSymbol(const lto::InputFile::Symbol &objSym,
                                           BitcodeFile &file) {
   StringRef name = saver.save(objSym.getName());
 
-  // TODO: support weak references
   if (objSym.isUndefined())
-    return symtab->addUndefined(name, &file, /*isWeakRef=*/false);
+    return symtab->addUndefined(name, &file, /*isWeakRef=*/objSym.isWeak());
 
   // TODO: Write a test demonstrating why computing isPrivateExtern before
   // LTO compilation is important.
@@ -1478,6 +1529,7 @@ static macho::Symbol *createBitcodeSymbol(const lto::InputFile::Symbol &objSym,
 BitcodeFile::BitcodeFile(MemoryBufferRef mb, StringRef archiveName,
                          uint64_t offsetInArchive)
     : InputFile(BitcodeKind, mb) {
+  this->archiveName = std::string(archiveName);
   std::string path = mb.getBufferIdentifier().str();
   // ThinLTO assumes that all MemoryBufferRefs given to it have a unique
   // name. If two members with the same name are provided, this causes a
diff --git a/lld/MachO/InputFiles.h b/lld/MachO/InputFiles.h
index 93794cb5a4aa..36da70011170 100644
--- a/lld/MachO/InputFiles.h
+++ b/lld/MachO/InputFiles.h
@@ -108,12 +108,13 @@ private:
 class ObjFile final : public InputFile {
 public:
   ObjFile(MemoryBufferRef mb, uint32_t modTime, StringRef archiveName);
+  ArrayRef<llvm::MachO::data_in_code_entry> getDataInCode() const;
+
   static bool classof(const InputFile *f) { return f->kind() == ObjKind; }
 
   llvm::DWARFUnit *compileUnit = nullptr;
   const uint32_t modTime;
   std::vector<ConcatInputSection *> debugSections;
-  ArrayRef<llvm::MachO::data_in_code_entry> dataInCodeEntries;
 
 private:
   Section *compactUnwindSection = nullptr;
@@ -130,7 +131,6 @@ private:
   void parseRelocations(ArrayRef<SectionHeader> sectionHeaders,
                         const SectionHeader &, Subsections &);
   void parseDebugInfo();
-  void parseDataInCode();
   void registerCompactUnwind();
 };
 
@@ -190,7 +190,10 @@ private:
   bool handleLDSymbol(StringRef originalName);
   void handleLDPreviousSymbol(StringRef name, StringRef originalName);
   void handleLDInstallNameSymbol(StringRef name, StringRef originalName);
+  void handleLDHideSymbol(StringRef name, StringRef originalName);
   void checkAppExtensionSafety(bool dylibIsAppExtensionSafe) const;
+
+  llvm::DenseSet<llvm::CachedHashStringRef> hiddenSymbols;
 };
 
 // .a file
diff --git a/lld/MachO/InputSection.h b/lld/MachO/InputSection.h
index 1183e32fbabf..fa137223c426 100644
--- a/lld/MachO/InputSection.h
+++ b/lld/MachO/InputSection.h
@@ -57,9 +57,8 @@ public:
   OutputSection *parent = nullptr;
 
   uint32_t align = 1;
-  uint32_t callSiteCount : 31;
   // is address assigned?
-  uint32_t isFinal : 1;
+  bool isFinal = false;
 
   ArrayRef<uint8_t> data;
   std::vector<Reloc> relocs;
@@ -86,12 +85,11 @@ protected:
 
   InputSection(Kind kind, StringRef segname, StringRef name, InputFile *file,
                ArrayRef<uint8_t> data, uint32_t align, uint32_t flags)
-      : align(align), callSiteCount(0), isFinal(false), data(data),
+      : align(align), data(data),
         shared(make<Shared>(file, name, segname, flags, kind)) {}
 
   InputSection(const InputSection &rhs)
-      : align(rhs.align), callSiteCount(0), isFinal(false), data(rhs.data),
-        shared(rhs.shared) {}
+      : align(rhs.align), data(rhs.data), shared(rhs.shared) {}
 
   const Shared *const shared;
 };
@@ -143,6 +141,7 @@ public:
   // first and not copied to the output.
   bool wasCoalesced = false;
   bool live = !config->deadStrip;
+  bool hasCallSites = false;
   // This variable has two usages. Initially, it represents the input order.
   // After assignAddresses is called, it represents the offset from the
   // beginning of the output section this section was assigned to.
diff --git a/lld/MachO/SymbolTable.cpp b/lld/MachO/SymbolTable.cpp
index c212516a4780..cec717e0cd31 100644
--- a/lld/MachO/SymbolTable.cpp
+++ b/lld/MachO/SymbolTable.cpp
@@ -184,10 +184,18 @@ Symbol *SymbolTable::addLazy(StringRef name, ArchiveFile *file,
   bool wasInserted;
   std::tie(s, wasInserted) = insert(name, file);
 
-  if (wasInserted)
+  if (wasInserted) {
     replaceSymbol<LazySymbol>(s, file, sym);
-  else if (isa<Undefined>(s) || (isa<DylibSymbol>(s) && s->isWeakDef()))
+  } else if (isa<Undefined>(s)) {
     file->fetch(sym);
+  } else if (auto *dysym = dyn_cast<DylibSymbol>(s)) {
+    if (dysym->isWeakDef()) {
+      if (dysym->getRefState() != RefState::Unreferenced)
+        file->fetch(sym);
+      else
+        replaceSymbol<LazySymbol>(s, file, sym);
+    }
+  }
   return s;
 }
 
diff --git a/lld/MachO/SyntheticSections.cpp b/lld/MachO/SyntheticSections.cpp
index 99a15666c8fa..b64a9db485c5 100644
--- a/lld/MachO/SyntheticSections.cpp
+++ b/lld/MachO/SyntheticSections.cpp
@@ -733,35 +733,30 @@ DataInCodeSection::DataInCodeSection()
 
 template <class LP>
 static std::vector<MachO::data_in_code_entry> collectDataInCodeEntries() {
-  using SegmentCommand = typename LP::segment_command;
-  using SectionHeader = typename LP::section;
-
   std::vector<MachO::data_in_code_entry> dataInCodeEntries;
   for (const InputFile *inputFile : inputFiles) {
     if (!isa<ObjFile>(inputFile))
       continue;
     const ObjFile *objFile = cast<ObjFile>(inputFile);
-    const auto *c = reinterpret_cast<const SegmentCommand *>(
-        findCommand(objFile->mb.getBufferStart(), LP::segmentLCType));
-    if (!c)
-      continue;
-    ArrayRef<SectionHeader> sectionHeaders{
-        reinterpret_cast<const SectionHeader *>(c + 1), c->nsects};
-
-    ArrayRef<MachO::data_in_code_entry> entries = objFile->dataInCodeEntries;
+    ArrayRef<MachO::data_in_code_entry> entries = objFile->getDataInCode();
     if (entries.empty())
       continue;
+
+    assert(is_sorted(dataInCodeEntries, [](const data_in_code_entry &lhs,
+                                           const data_in_code_entry &rhs) {
+      return lhs.offset < rhs.offset;
+    }));
     // For each code subsection find 'data in code' entries residing in it.
     // Compute the new offset values as
     // <offset within subsection> + <subsection address> - <__TEXT address>.
-    for (size_t i = 0, n = sectionHeaders.size(); i < n; ++i) {
-      for (const Subsection &subsec : objFile->sections[i].subsections) {
+    for (const Section &section : objFile->sections) {
+      for (const Subsection &subsec : section.subsections) {
         const InputSection *isec = subsec.isec;
         if (!isCodeSection(isec))
           continue;
         if (cast<ConcatInputSection>(isec)->shouldOmitFromOutput())
           continue;
-        const uint64_t beginAddr = sectionHeaders[i].addr + subsec.offset;
+        const uint64_t beginAddr = section.address + subsec.offset;
         auto it = llvm::lower_bound(
             entries, beginAddr,
             [](const MachO::data_in_code_entry &entry, uint64_t addr) {
diff --git a/lld/MachO/Writer.cpp b/lld/MachO/Writer.cpp
index 093a380d175e..8903f0189ef9 100644
--- a/lld/MachO/Writer.cpp
+++ b/lld/MachO/Writer.cpp
@@ -29,6 +29,7 @@
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/Parallel.h"
 #include "llvm/Support/Path.h"
+#include "llvm/Support/ThreadPool.h"
 #include "llvm/Support/TimeProfiler.h"
 #include "llvm/Support/xxhash.h"
 
@@ -64,6 +65,7 @@ public:
 
   template <class LP> void run();
 
+  ThreadPool threadPool;
   std::unique_ptr<FileOutputBuffer> &buffer;
   uint64_t addr = 0;
   uint64_t fileOff = 0;
@@ -1035,10 +1037,14 @@ void Writer::finalizeLinkEditSegment() {
       dataInCodeSection,
       functionStartsSection,
   };
-  parallelForEach(linkEditSections, [](LinkEditSection *osec) {
+  SmallVector<std::shared_future<void>> threadFutures;
+  threadFutures.reserve(linkEditSections.size());
+  for (LinkEditSection *osec : linkEditSections)
     if (osec)
-      osec->finalizeContents();
-  });
+      threadFutures.emplace_back(threadPool.async(
+          [](LinkEditSection *osec) { osec->finalizeContents(); }, osec));
+  for (std::shared_future<void> &future : threadFutures)
+    future.wait();
 
   // Now that __LINKEDIT is filled out, do a proper calculation of its
   // addresses and offsets.
@@ -1091,14 +1097,21 @@ void Writer::writeSections() {
 // values.
 void Writer::writeUuid() {
   TimeTraceScope timeScope("Computing UUID");
+
   ArrayRef<uint8_t> data{buffer->getBufferStart(), buffer->getBufferEnd()};
   unsigned chunkCount = parallel::strategy.compute_thread_count() * 10;
   // Round-up integer division
   size_t chunkSize = (data.size() + chunkCount - 1) / chunkCount;
   std::vector<ArrayRef<uint8_t>> chunks = split(data, chunkSize);
   std::vector<uint64_t> hashes(chunks.size());
-  parallelForEachN(0, chunks.size(),
-                   [&](size_t i) { hashes[i] = xxHash64(chunks[i]); });
+  SmallVector<std::shared_future<void>> threadFutures;
+  threadFutures.reserve(chunks.size());
+  for (size_t i = 0; i < chunks.size(); ++i)
+    threadFutures.emplace_back(threadPool.async(
+        [&](size_t i) { hashes[i] = xxHash64(chunks[i]); }, i));
+  for (std::shared_future<void> &future : threadFutures)
+    future.wait();
+
   uint64_t digest = xxHash64({reinterpret_cast<uint8_t *>(hashes.data()),
                               hashes.size() * sizeof(uint64_t)});
   uuidCommand->writeUuid(digest);
@@ -1147,8 +1160,8 @@ template <class LP> void Writer::run() {
   sortSegmentsAndSections();
   createLoadCommands<LP>();
   finalizeAddresses();
+  threadPool.async(writeMapFile);
   finalizeLinkEditSegment();
-  writeMapFile();
   writeOutputFile();
 }
 
diff --git a/lld/docs/ELF/start-stop-gc.rst b/lld/docs/ELF/start-stop-gc.rst
new file mode 100644
index 000000000000..0baa1cae246a
--- /dev/null
+++ b/lld/docs/ELF/start-stop-gc.rst
@@ -0,0 +1,66 @@
+-z start-stop-gc
+================
+
+If your ``-Wl,--gc-sections`` build fail with a linker error like this:
+
+    error: undefined symbol: __start_meta
+    >>> referenced by {{.*}}
+    >>> the encapsulation symbol needs to be retained under --gc-sections properly; consider -z nostart-stop-gc (see https://lld.llvm.org/start-stop-gc) 
+
+it is likely your C identifier name sections are not properly annotated to
+suffice under ``--gc-sections``.
+
+``__start_meta`` and ``__stop_meta`` are sometimed called encapsulation
+symbols. In October 2015, GNU ld switched behavior and made a ``__start_meta``
+reference from a live section retain all ``meta`` input sections. This
+conservative behavior works for existing code which does not take GC into fair
+consideration, but unnecessarily increases sizes for modern metadata section
+usage which desires precise GC.
+
+GNU ld 2.37 added ``-z start-stop-gc`` to restore the traditional behavior
+ld.lld 13.0.0 defaults to ``-z start-stop-gc`` and supports ``-z nostart-stop-gc``
+to switch to the conservative behavior.
+
+The Apple ld64 linker has a similar ``section$start`` feature and always
+allowed GC (like ``-z start-stop-gc``).
+
+Annotate C identifier name sections
+-----------------------------------
+
+A C identifier name section (``meta``) sometimes depends on another section.
+Let that section reference ``meta`` via a relocation.
+
+.. code-block:: c
+
+  asm(".pushsection .init_array,\"aw\",@init_array\n" \
+      ".reloc ., R_AARCH64_NONE, meta\n"              \
+      ".popsection\n")
+
+If a relocation is inconvenient, consider using ``__attribute__((retain))``
+(GCC 11 with modern binutils, Clang 13).
+
+.. code-block:: c
+
+  #pragma GCC diagnostic push
+  #pragma GCC diagnostic ignored "-Wattributes"
+  __attribute__((retain,used,section("meta")))
+  static const char dummy[0];
+  #pragma GCC diagnostic pop
+
+GCC before 11 and Clang before 13 do not recognize ``__attribute__((retain))``,
+so ``-Wattributes`` may need to be ignored. On ELF targets,
+``__attribute__((used))`` prevents compiler discarding, but does not affect
+linker ``--gc-sections``.
+
+In a macro, you may use:
+
+.. code-block:: c
+
+  _Pragma("GCC diagnostic push")
+  _Pragma("GCC diagnostic ignored \"-Wattributes\"")
+  ...
+  _Pragma("GCC diagnostic pop")
+
+If you use the ``SECTIONS`` command in a linker script, use
+`the ``KEEP`` keyword <https://sourceware.org/binutils/docs/ld/Input-Section-Keep.html>`_, e.g.
+``meta : { KEEP(*(meta)) }``
diff --git a/lld/docs/ReleaseNotes.rst b/lld/docs/ReleaseNotes.rst
index a2456fc46689..f7e099b9cf6e 100644
--- a/lld/docs/ReleaseNotes.rst
+++ b/lld/docs/ReleaseNotes.rst
@@ -31,7 +31,7 @@ ELF Improvements
 * ``e_entry`` no longer falls back to the address of ``.text`` if the entry symbol does not exist.
   Instead, a value of 0 will be written.
   (`D110014 <https://reviews.llvm.org/D110014>`_)
-* If ``-Map`` is specified, ``--cref`` will be printted to the specified file.
+* If ``-Map`` is specified, ``--cref`` will be printed to the specified file.
   (`D114663 <https://reviews.llvm.org/D114663>`_)
 
 Architecture specific changes:
diff --git a/lld/docs/index.rst b/lld/docs/index.rst
index 4b42abadb94a..50131bda54bd 100644
--- a/lld/docs/index.rst
+++ b/lld/docs/index.rst
@@ -175,4 +175,5 @@ document soon.
    Partitions
    ReleaseNotes
    ELF/linker_script
+   ELF/start-stop-gc
    ELF/warn_backrefs
diff --git a/lld/docs/ld.lld.1 b/lld/docs/ld.lld.1
index 0422231d78b5..04f0982b4ced 100644
--- a/lld/docs/ld.lld.1
+++ b/lld/docs/ld.lld.1
@@ -702,6 +702,16 @@ Stack permissions are recorded in the
 .Dv PT_GNU_STACK
 segment.
 .Pp
+.It Cm bti-report Ns = Ns Ar [none|warning|error]
+Specify how to report the missing GNU_PROPERTY_AARCH64_FEATURE_1_BTI property.
+.Cm none
+is the default, linker will not report the missing property otherwise will be reported as a warning or an error.
+.Pp
+.It Cm cet-report Ns = Ns Ar [none|warning|error]
+Specify how to report the missing GNU_PROPERTY_X86_FEATURE_1_IBT or GNU_PROPERTY_X86_FEATURE_1_SHSTK properties.
+.Cm none
+is the default, linker will not report the missing property otherwise will be reported as a warning or an error.
+.Pp
 .It Cm force-bti
 Force enable AArch64 BTI instruction in PLT, warn if Input ELF file does not have GNU_PROPERTY_AARCH64_FEATURE_1_BTI property.
 .Pp
diff --git a/lld/include/lld/Common/Driver.h b/lld/include/lld/Common/Driver.h
index eb5bc7b82b9b..0e505a16463e 100644
--- a/lld/include/lld/Common/Driver.h
+++ b/lld/include/lld/Common/Driver.h
@@ -42,11 +42,6 @@ bool link(llvm::ArrayRef<const char *> args, bool canExitEarly,
           llvm::raw_ostream &stdoutOS, llvm::raw_ostream &stderrOS);
 }
 
-namespace mach_o {
-bool link(llvm::ArrayRef<const char *> args, bool canExitEarly,
-          llvm::raw_ostream &stdoutOS, llvm::raw_ostream &stderrOS);
-}
-
 namespace macho {
 bool link(llvm::ArrayRef<const char *> args, bool canExitEarly,
           llvm::raw_ostream &stdoutOS, llvm::raw_ostream &stderrOS);
diff --git a/lld/include/lld/Core/Reference.h b/lld/include/lld/Core/Reference.h
index b104f8495474..7a6c36aecfc5 100644
--- a/lld/include/lld/Core/Reference.h
+++ b/lld/include/lld/Core/Reference.h
@@ -42,9 +42,8 @@ class Reference {
 public:
   /// Which universe defines the kindValue().
   enum class KindNamespace {
-    all     = 0,
+    all = 0,
     testing = 1,
-    mach_o  = 2,
   };
 
   KindNamespace kindNamespace() const { return (KindNamespace)_kindNamespace; }
diff --git a/lld/include/lld/ReaderWriter/MachOLinkingContext.h b/lld/include/lld/ReaderWriter/MachOLinkingContext.h
deleted file mode 100644
index 974f323bc612..000000000000
--- a/lld/include/lld/ReaderWriter/MachOLinkingContext.h
+++ /dev/null
@@ -1,505 +0,0 @@
-//===- lld/ReaderWriter/MachOLinkingContext.h -----------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLD_READER_WRITER_MACHO_LINKING_CONTEXT_H
-#define LLD_READER_WRITER_MACHO_LINKING_CONTEXT_H
-
-#include "lld/Core/LinkingContext.h"
-#include "lld/Core/Reader.h"
-#include "lld/Core/Writer.h"
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/StringMap.h"
-#include "llvm/ADT/StringSet.h"
-#include "llvm/BinaryFormat/MachO.h"
-#include "llvm/Support/ErrorHandling.h"
-#include <set>
-
-using llvm::MachO::HeaderFileType;
-
-namespace lld {
-
-namespace mach_o {
-class ArchHandler;
-class MachODylibFile;
-class MachOFile;
-class SectCreateFile;
-}
-
-class MachOLinkingContext : public LinkingContext {
-public:
-  MachOLinkingContext();
-  ~MachOLinkingContext() override;
-
-  enum Arch {
-    arch_unknown,
-    arch_ppc,
-    arch_x86,
-    arch_x86_64,
-    arch_armv6,
-    arch_armv7,
-    arch_armv7s,
-    arch_arm64,
-  };
-
-  enum class OS {
-    unknown,
-    macOSX,
-    iOS,
-    iOS_simulator
-  };
-
-  enum class ExportMode {
-    globals,    // Default, all global symbols exported.
-    exported,   // -exported_symbol[s_list], only listed symbols exported.
-    unexported  // -unexported_symbol[s_list], no listed symbol exported.
-  };
-
-  enum class DebugInfoMode {
-    addDebugMap,    // Default
-    noDebugMap      // -S option
-  };
-
-  enum class UndefinedMode {
-    error,
-    warning,
-    suppress,
-    dynamicLookup
-  };
-
-  enum ObjCConstraint {
-    objc_unknown = 0,
-    objc_supports_gc = 2,
-    objc_gc_only = 4,
-    // Image optimized by dyld = 8
-    // GC compaction = 16
-    objc_retainReleaseForSimulator = 32,
-    objc_retainRelease
-  };
-
-  /// Initializes the context to sane default values given the specified output
-  /// file type, arch, os, and minimum os version.  This should be called before
-  /// other setXXX() methods.
-  void configure(HeaderFileType type, Arch arch, OS os, uint32_t minOSVersion,
-                 bool exportDynamicSymbols);
-
-  void addPasses(PassManager &pm) override;
-  bool validateImpl() override;
-  std::string demangle(StringRef symbolName) const override;
-
-  void createImplicitFiles(std::vector<std::unique_ptr<File>> &) override;
-
-  /// Creates a new file which is owned by the context.  Returns a pointer to
-  /// the new file.
-  template <class T, class... Args>
-  typename std::enable_if<!std::is_array<T>::value, T *>::type
-  make_file(Args &&... args) const {
-    auto file = std::unique_ptr<T>(new T(std::forward<Args>(args)...));
-    auto *filePtr = file.get();
-    auto *ctx = const_cast<MachOLinkingContext *>(this);
-    ctx->getNodes().push_back(std::make_unique<FileNode>(std::move(file)));
-    return filePtr;
-  }
-
-  uint32_t getCPUType() const;
-  uint32_t getCPUSubType() const;
-
-  bool addEntryPointLoadCommand() const;
-  bool addUnixThreadLoadCommand() const;
-  bool outputTypeHasEntry() const;
-  bool is64Bit() const;
-
-  virtual uint64_t pageZeroSize() const { return _pageZeroSize; }
-  virtual uint64_t pageSize() const { return _pageSize; }
-
-  mach_o::ArchHandler &archHandler() const;
-
-  HeaderFileType outputMachOType() const { return _outputMachOType; }
-
-  Arch arch() const { return _arch; }
-  StringRef archName() const { return nameFromArch(_arch); }
-  OS os() const { return _os; }
-
-  ExportMode exportMode() const { return _exportMode; }
-  void setExportMode(ExportMode mode) { _exportMode = mode; }
-  void addExportSymbol(StringRef sym);
-  bool exportRestrictMode() const { return _exportMode != ExportMode::globals; }
-  bool exportSymbolNamed(StringRef sym) const;
-
-  DebugInfoMode debugInfoMode() const { return _debugInfoMode; }
-  void setDebugInfoMode(DebugInfoMode mode) {
-    _debugInfoMode = mode;
-  }
-
-  void appendOrderedSymbol(StringRef symbol, StringRef filename);
-
-  bool keepPrivateExterns() const { return _keepPrivateExterns; }
-  void setKeepPrivateExterns(bool v) { _keepPrivateExterns = v; }
-  bool demangleSymbols() const { return _demangle; }
-  void setDemangleSymbols(bool d) { _demangle = d; }
-  bool mergeObjCCategories() const { return _mergeObjCCategories; }
-  void setMergeObjCCategories(bool v) { _mergeObjCCategories = v; }
-  /// Create file at specified path which will contain a binary encoding
-  /// of all input and output file paths.
-  std::error_code createDependencyFile(StringRef path);
-  void addInputFileDependency(StringRef path) const;
-  void addInputFileNotFound(StringRef path) const;
-  void addOutputFileDependency(StringRef path) const;
-
-  bool minOS(StringRef mac, StringRef iOS) const;
-  void setDoNothing(bool value) { _doNothing = value; }
-  bool doNothing() const { return _doNothing; }
-  bool printAtoms() const { return _printAtoms; }
-  bool testingFileUsage() const { return _testingFileUsage; }
-  const StringRefVector &searchDirs() const { return _searchDirs; }
-  const StringRefVector &frameworkDirs() const { return _frameworkDirs; }
-  void setSysLibRoots(const StringRefVector &paths);
-  const StringRefVector &sysLibRoots() const { return _syslibRoots; }
-  bool PIE() const { return _pie; }
-  void setPIE(bool pie) { _pie = pie; }
-  bool generateVersionLoadCommand() const {
-    return _generateVersionLoadCommand;
-  }
-  void setGenerateVersionLoadCommand(bool v) {
-    _generateVersionLoadCommand = v;
-  }
-
-  bool generateFunctionStartsLoadCommand() const {
-    return _generateFunctionStartsLoadCommand;
-  }
-  void setGenerateFunctionStartsLoadCommand(bool v) {
-    _generateFunctionStartsLoadCommand = v;
-  }
-
-  bool generateDataInCodeLoadCommand() const {
-    return _generateDataInCodeLoadCommand;
-  }
-  void setGenerateDataInCodeLoadCommand(bool v) {
-    _generateDataInCodeLoadCommand = v;
-  }
-
-  uint64_t stackSize() const { return _stackSize; }
-  void setStackSize(uint64_t stackSize) { _stackSize = stackSize; }
-
-  uint64_t baseAddress() const { return _baseAddress; }
-  void setBaseAddress(uint64_t baseAddress) { _baseAddress = baseAddress; }
-
-  ObjCConstraint objcConstraint() const { return _objcConstraint; }
-
-  uint32_t osMinVersion() const { return _osMinVersion; }
-
-  uint32_t sdkVersion() const { return _sdkVersion; }
-  void setSdkVersion(uint64_t v) { _sdkVersion = v; }
-
-  uint64_t sourceVersion() const { return _sourceVersion; }
-  void setSourceVersion(uint64_t v) { _sourceVersion = v; }
-
-  uint32_t swiftVersion() const { return _swiftVersion; }
-
-  /// Checks whether a given path on the filesystem exists.
-  ///
-  /// When running in -test_file_usage mode, this method consults an
-  /// internally maintained list of files that exist (provided by -path_exists)
-  /// instead of the actual filesystem.
-  bool pathExists(StringRef path) const;
-
-  /// Like pathExists() but only used on files - not directories.
-  bool fileExists(StringRef path) const;
-
-  /// Adds any library search paths derived from the given base, possibly
-  /// modified by -syslibroots.
-  ///
-  /// The set of paths added consists of approximately all syslibroot-prepended
-  /// versions of libPath that exist, or the original libPath if there are none
-  /// for whatever reason. With various edge-cases for compatibility.
-  void addModifiedSearchDir(StringRef libPath, bool isSystemPath = false);
-
-  /// Determine whether -lFoo can be resolve within the given path, and
-  /// return the filename if so.
-  ///
-  /// The -lFoo option is documented to search for libFoo.dylib and libFoo.a in
-  /// that order, unless Foo ends in ".o", in which case only the exact file
-  /// matches (e.g. -lfoo.o would only find foo.o).
-  llvm::Optional<StringRef> searchDirForLibrary(StringRef path,
-                                                StringRef libName) const;
-
-  /// Iterates through all search path entries looking for libName (as
-  /// specified by -lFoo).
-  llvm::Optional<StringRef> searchLibrary(StringRef libName) const;
-
-  /// Add a framework search path.  Internally, this method may be prepended
-  /// the path with syslibroot.
-  void addFrameworkSearchDir(StringRef fwPath, bool isSystemPath = false);
-
-  /// Iterates through all framework directories looking for
-  /// Foo.framework/Foo (when fwName = "Foo").
-  llvm::Optional<StringRef> findPathForFramework(StringRef fwName) const;
-
-  /// The dylib's binary compatibility version, in the raw uint32 format.
-  ///
-  /// When building a dynamic library, this is the compatibility version that
-  /// gets embedded into the result. Other Mach-O binaries that link against
-  /// this library will store the compatibility version in its load command. At
-  /// runtime, the loader will verify that the binary is compatible with the
-  /// installed dynamic library.
-  uint32_t compatibilityVersion() const { return _compatibilityVersion; }
-
-  /// The dylib's current version, in the raw uint32 format.
-  ///
-  /// When building a dynamic library, this is the current version that gets
-  /// embedded into the result. Other Mach-O binaries that link against
-  /// this library will store the compatibility version in its load command.
-  uint32_t currentVersion() const { return _currentVersion; }
-
-  /// The dylib's install name.
-  ///
-  /// Binaries that link against the dylib will embed this path into the dylib
-  /// load command. When loading the binaries at runtime, this is the location
-  /// on disk that the loader will look for the dylib.
-  StringRef installName() const { return _installName; }
-
-  /// Whether or not the dylib has side effects during initialization.
-  ///
-  /// Dylibs marked as being dead strippable provide the guarantee that loading
-  /// the dylib has no side effects, allowing the linker to strip out the dylib
-  /// when linking a binary that does not use any of its symbols.
-  bool deadStrippableDylib() const { return _deadStrippableDylib; }
-
-  /// Whether or not to use flat namespace.
-  ///
-  /// MachO usually uses a two-level namespace, where each external symbol
-  /// referenced by the target is associated with the dylib that will provide
-  /// the symbol's definition at runtime. Using flat namespace overrides this
-  /// behavior: the linker searches all dylibs on the command line and all
-  /// dylibs those original dylibs depend on, but does not record which dylib
-  /// an external symbol came from. At runtime dyld again searches all images
-  /// and uses the first definition it finds. In addition, any undefines in
-  /// loaded flat_namespace dylibs must be resolvable at build time.
-  bool useFlatNamespace() const { return _flatNamespace; }
-
-  /// How to handle undefined symbols.
-  ///
-  /// Options are:
-  ///  * error: Report an error and terminate linking.
-  ///  * warning: Report a warning, but continue linking.
-  ///  * suppress: Ignore and continue linking.
-  ///  * dynamic_lookup: For use with -twolevel namespace: Records source dylibs
-  ///    for symbols that are defined in a linked dylib at static link time.
-  ///    Undefined symbols are handled by searching all loaded images at
-  ///    runtime.
-  UndefinedMode undefinedMode() const { return _undefinedMode; }
-
-  /// The path to the executable that will load the bundle at runtime.
-  ///
-  /// When building a Mach-O bundle, this executable will be examined if there
-  /// are undefined symbols after the main link phase. It is expected that this
-  /// binary will be loading the bundle at runtime and will provide the symbols
-  /// at that point.
-  StringRef bundleLoader() const { return _bundleLoader; }
-
-  void setCompatibilityVersion(uint32_t vers) { _compatibilityVersion = vers; }
-  void setCurrentVersion(uint32_t vers) { _currentVersion = vers; }
-  void setInstallName(StringRef name) { _installName = name; }
-  void setDeadStrippableDylib(bool deadStrippable) {
-    _deadStrippableDylib = deadStrippable;
-  }
-  void setUseFlatNamespace(bool flatNamespace) {
-    _flatNamespace = flatNamespace;
-  }
-
-  void setUndefinedMode(UndefinedMode undefinedMode) {
-    _undefinedMode = undefinedMode;
-  }
-
-  void setBundleLoader(StringRef loader) { _bundleLoader = loader; }
-  void setPrintAtoms(bool value=true) { _printAtoms = value; }
-  void setTestingFileUsage(bool value = true) {
-    _testingFileUsage = value;
-  }
-  void addExistingPathForDebug(StringRef path) {
-    _existingPaths.insert(path);
-  }
-
-  void addRpath(StringRef rpath);
-  const StringRefVector &rpaths() const { return _rpaths; }
-
-  /// Add section alignment constraint on final layout.
-  void addSectionAlignment(StringRef seg, StringRef sect, uint16_t align);
-
-  /// Add a section based on a command-line sectcreate option.
-  void addSectCreateSection(StringRef seg, StringRef sect,
-                            std::unique_ptr<MemoryBuffer> content);
-
-  /// Returns true if specified section had alignment constraints.
-  bool sectionAligned(StringRef seg, StringRef sect, uint16_t &align) const;
-
-  StringRef dyldPath() const { return "/usr/lib/dyld"; }
-
-  /// Stub creation Pass should be run.
-  bool needsStubsPass() const;
-
-  // GOT creation Pass should be run.
-  bool needsGOTPass() const;
-
-  /// Pass to add TLV sections.
-  bool needsTLVPass() const;
-
-  /// Pass to transform __compact_unwind into __unwind_info should be run.
-  bool needsCompactUnwindPass() const;
-
-  /// Pass to add shims switching between thumb and arm mode.
-  bool needsShimPass() const;
-
-  /// Pass to add objc image info and optimized objc data.
-  bool needsObjCPass() const;
-
-  /// Magic symbol name stubs will need to help lazy bind.
-  StringRef binderSymbolName() const;
-
-  /// Used to keep track of direct and indirect dylibs.
-  void registerDylib(mach_o::MachODylibFile *dylib, bool upward) const;
-
-  // Reads a file from disk to memory. Returns only a needed chunk
-  // if a fat binary.
-  ErrorOr<std::unique_ptr<MemoryBuffer>> getMemoryBuffer(StringRef path);
-
-  /// Used to find indirect dylibs. Instantiates a MachODylibFile if one
-  /// has not already been made for the requested dylib.  Uses -L and -F
-  /// search paths to allow indirect dylibs to be overridden.
-  mach_o::MachODylibFile* findIndirectDylib(StringRef path);
-
-  uint32_t dylibCurrentVersion(StringRef installName) const;
-
-  uint32_t dylibCompatVersion(StringRef installName) const;
-
-  ArrayRef<mach_o::MachODylibFile*> allDylibs() const {
-    return _allDylibs;
-  }
-
-  /// Creates a copy (owned by this MachOLinkingContext) of a string.
-  StringRef copy(StringRef str) { return str.copy(_allocator); }
-
-  /// If the memoryBuffer is a fat file with a slice for the current arch,
-  /// this method will return the offset and size of that slice.
-  bool sliceFromFatFile(MemoryBufferRef mb, uint32_t &offset, uint32_t &size);
-
-  /// Returns if a command line option specified dylib is an upward link.
-  bool isUpwardDylib(StringRef installName) const;
-
-  static bool isThinObjectFile(StringRef path, Arch &arch);
-  static Arch archFromCpuType(uint32_t cputype, uint32_t cpusubtype);
-  static Arch archFromName(StringRef archName);
-  static StringRef nameFromArch(Arch arch);
-  static uint32_t cpuTypeFromArch(Arch arch);
-  static uint32_t cpuSubtypeFromArch(Arch arch);
-  static bool is64Bit(Arch arch);
-  static bool isHostEndian(Arch arch);
-  static bool isBigEndian(Arch arch);
-
-  /// Construct 32-bit value from string "X.Y.Z" where
-  /// bits are xxxx.yy.zz.  Largest number is 65535.255.255
-  static bool parsePackedVersion(StringRef str, uint32_t &result);
-
-  /// Construct 64-bit value from string "A.B.C.D.E" where
-  /// bits are aaaa.bb.cc.dd.ee.  Largest number is 16777215.1023.1023.1023.1023
-  static bool parsePackedVersion(StringRef str, uint64_t &result);
-
-  void finalizeInputFiles() override;
-
-  llvm::Error handleLoadedFile(File &file) override;
-
-  bool customAtomOrderer(const DefinedAtom *left, const DefinedAtom *right,
-                         bool &leftBeforeRight) const;
-
-  /// Return the 'flat namespace' file. This is the file that supplies
-  /// atoms for otherwise undefined symbols when the -flat_namespace or
-  /// -undefined dynamic_lookup options are used.
-  File* flatNamespaceFile() const { return _flatNamespaceFile; }
-
-private:
-  Writer &writer() const override;
-  mach_o::MachODylibFile* loadIndirectDylib(StringRef path);
-  struct ArchInfo {
-    StringRef                 archName;
-    MachOLinkingContext::Arch arch;
-    bool                      littleEndian;
-    uint32_t                  cputype;
-    uint32_t                  cpusubtype;
-  };
-
-  struct SectionAlign {
-    StringRef segmentName;
-    StringRef sectionName;
-    uint16_t  align;
-  };
-
-  struct OrderFileNode {
-    StringRef fileFilter;
-    unsigned  order;
-  };
-
-  static bool findOrderOrdinal(const std::vector<OrderFileNode> &nodes,
-                             const DefinedAtom *atom, unsigned &ordinal);
-
-  static ArchInfo _s_archInfos[];
-
-  std::set<StringRef> _existingPaths; // For testing only.
-  StringRefVector _searchDirs;
-  StringRefVector _syslibRoots;
-  StringRefVector _frameworkDirs;
-  HeaderFileType _outputMachOType = llvm::MachO::MH_EXECUTE;
-  bool _outputMachOTypeStatic = false; // Disambiguate static vs dynamic prog
-  bool _doNothing = false;             // for -help and -v which just print info
-  bool _pie = false;
-  Arch _arch = arch_unknown;
-  OS _os = OS::macOSX;
-  uint32_t _osMinVersion = 0;
-  uint32_t _sdkVersion = 0;
-  uint64_t _sourceVersion = 0;
-  uint64_t _pageZeroSize = 0;
-  uint64_t _pageSize = 4096;
-  uint64_t _baseAddress = 0;
-  uint64_t _stackSize = 0;
-  uint32_t _compatibilityVersion = 0;
-  uint32_t _currentVersion = 0;
-  ObjCConstraint _objcConstraint = objc_unknown;
-  uint32_t _swiftVersion = 0;
-  StringRef _installName;
-  StringRefVector _rpaths;
-  bool _flatNamespace = false;
-  UndefinedMode _undefinedMode = UndefinedMode::error;
-  bool _deadStrippableDylib = false;
-  bool _printAtoms = false;
-  bool _testingFileUsage = false;
-  bool _keepPrivateExterns = false;
-  bool _demangle = false;
-  bool _mergeObjCCategories = true;
-  bool _generateVersionLoadCommand = false;
-  bool _generateFunctionStartsLoadCommand = false;
-  bool _generateDataInCodeLoadCommand = false;
-  StringRef _bundleLoader;
-  mutable std::unique_ptr<mach_o::ArchHandler> _archHandler;
-  mutable std::unique_ptr<Writer> _writer;
-  std::vector<SectionAlign> _sectAligns;
-  mutable llvm::StringMap<mach_o::MachODylibFile*> _pathToDylibMap;
-  mutable std::vector<mach_o::MachODylibFile*> _allDylibs;
-  mutable std::set<mach_o::MachODylibFile*> _upwardDylibs;
-  mutable std::vector<std::unique_ptr<File>> _indirectDylibs;
-  mutable std::mutex _dylibsMutex;
-  ExportMode _exportMode = ExportMode::globals;
-  llvm::StringSet<> _exportedSymbols;
-  DebugInfoMode _debugInfoMode = DebugInfoMode::addDebugMap;
-  std::unique_ptr<llvm::raw_fd_ostream> _dependencyInfo;
-  llvm::StringMap<std::vector<OrderFileNode>> _orderFiles;
-  unsigned _orderFileEntries = 0;
-  File *_flatNamespaceFile = nullptr;
-  mach_o::SectCreateFile *_sectCreateFile = nullptr;
-};
-
-} // end namespace lld
-
-#endif // LLD_READER_WRITER_MACHO_LINKING_CONTEXT_H
diff --git a/lld/include/lld/ReaderWriter/YamlContext.h b/lld/include/lld/ReaderWriter/YamlContext.h
deleted file mode 100644
index dc133e3627de..000000000000
--- a/lld/include/lld/ReaderWriter/YamlContext.h
+++ /dev/null
@@ -1,42 +0,0 @@
-//===- lld/ReaderWriter/YamlContext.h - object used in YAML I/O context ---===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLD_READER_WRITER_YAML_CONTEXT_H
-#define LLD_READER_WRITER_YAML_CONTEXT_H
-
-#include "lld/Common/LLVM.h"
-#include <functional>
-#include <memory>
-#include <vector>
-
-namespace lld {
-class File;
-class LinkingContext;
-class Registry;
-namespace mach_o {
-namespace normalized {
-struct NormalizedFile;
-}
-}
-
-using lld::mach_o::normalized::NormalizedFile;
-
-/// When YAML I/O is used in lld, the yaml context always holds a YamlContext
-/// object.  We need to support hetergenous yaml documents which each require
-/// different context info.  This struct supports all clients.
-struct YamlContext {
-  const LinkingContext *_ctx = nullptr;
-  const Registry *_registry = nullptr;
-  File *_file = nullptr;
-  NormalizedFile *_normalizeMachOFile = nullptr;
-  StringRef _path;
-};
-
-} // end namespace lld
-
-#endif // LLD_READER_WRITER_YAML_CONTEXT_H
diff --git a/lld/lib/Core/DefinedAtom.cpp b/lld/lib/Core/DefinedAtom.cpp
deleted file mode 100644
index 3c1eece16841..000000000000
--- a/lld/lib/Core/DefinedAtom.cpp
+++ /dev/null
@@ -1,81 +0,0 @@
-//===- DefinedAtom.cpp ------------------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Support/ErrorHandling.h"
-#include "lld/Core/DefinedAtom.h"
-#include "lld/Core/File.h"
-
-namespace lld {
-
-DefinedAtom::ContentPermissions DefinedAtom::permissions() const {
-  // By default base permissions on content type.
-  return permissions(this->contentType());
-}
-
-// Utility function for deriving permissions from content type
-DefinedAtom::ContentPermissions DefinedAtom::permissions(ContentType type) {
-  switch (type) {
-  case typeCode:
-  case typeResolver:
-  case typeBranchIsland:
-  case typeBranchShim:
-  case typeStub:
-  case typeStubHelper:
-  case typeMachHeader:
-    return permR_X;
-
-  case typeConstant:
-  case typeCString:
-  case typeUTF16String:
-  case typeCFI:
-  case typeLSDA:
-  case typeLiteral4:
-  case typeLiteral8:
-  case typeLiteral16:
-  case typeDTraceDOF:
-  case typeCompactUnwindInfo:
-  case typeProcessedUnwindInfo:
-  case typeObjCImageInfo:
-  case typeObjCMethodList:
-    return permR__;
-
-  case typeData:
-  case typeDataFast:
-  case typeZeroFill:
-  case typeZeroFillFast:
-  case typeObjC1Class:
-  case typeLazyPointer:
-  case typeLazyDylibPointer:
-  case typeNonLazyPointer:
-  case typeThunkTLV:
-    return permRW_;
-
-  case typeGOT:
-  case typeConstData:
-  case typeCFString:
-  case typeInitializerPtr:
-  case typeTerminatorPtr:
-  case typeCStringPtr:
-  case typeObjCClassPtr:
-  case typeObjC2CategoryList:
-  case typeInterposingTuples:
-  case typeTLVInitialData:
-  case typeTLVInitialZeroFill:
-  case typeTLVInitializerPtr:
-    return permRW_L;
-
-  case typeUnknown:
-  case typeTempLTO:
-  case typeSectCreate:
-  case typeDSOHandle:
-    return permUnknown;
-  }
-  llvm_unreachable("unknown content type");
-}
-
-} // namespace
diff --git a/lld/lib/Core/Error.cpp b/lld/lib/Core/Error.cpp
deleted file mode 100644
index a4f4b1b8af48..000000000000
--- a/lld/lib/Core/Error.cpp
+++ /dev/null
@@ -1,93 +0,0 @@
-//===- Error.cpp - system_error extensions for lld --------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "lld/Core/Error.h"
-#include "llvm/ADT/Twine.h"
-#include "llvm/Support/ErrorHandling.h"
-#include <mutex>
-#include <string>
-#include <vector>
-
-using namespace lld;
-
-namespace {
-class _YamlReaderErrorCategory : public std::error_category {
-public:
-  const char* name() const noexcept override {
-    return "lld.yaml.reader";
-  }
-
-  std::string message(int ev) const override {
-    switch (static_cast<YamlReaderError>(ev)) {
-    case YamlReaderError::unknown_keyword:
-      return "Unknown keyword found in yaml file";
-    case YamlReaderError::illegal_value:
-      return "Bad value found in yaml file";
-    }
-    llvm_unreachable("An enumerator of YamlReaderError does not have a "
-                     "message defined.");
-  }
-};
-} // end anonymous namespace
-
-const std::error_category &lld::YamlReaderCategory() {
-  static _YamlReaderErrorCategory o;
-  return o;
-}
-
-namespace lld {
-
-/// Temporary class to enable make_dynamic_error_code() until
-/// llvm::ErrorOr<> is updated to work with error encapsulations
-/// other than error_code.
-class dynamic_error_category : public std::error_category {
-public:
-  ~dynamic_error_category() override = default;
-
-  const char *name() const noexcept override {
-    return "lld.dynamic_error";
-  }
-
-  std::string message(int ev) const override {
-    assert(ev >= 0);
-    assert(ev < (int)_messages.size());
-    // The value is an index into the string vector.
-    return _messages[ev];
-  }
-
-  int add(std::string msg) {
-    std::lock_guard<std::recursive_mutex> lock(_mutex);
-    // Value zero is always the success value.
-    if (_messages.empty())
-      _messages.push_back("Success");
-    _messages.push_back(msg);
-    // Return the index of the string just appended.
-    return _messages.size() - 1;
-  }
-
-private:
-  std::vector<std::string> _messages;
-  std::recursive_mutex _mutex;
-};
-
-static dynamic_error_category categorySingleton;
-
-std::error_code make_dynamic_error_code(StringRef msg) {
-  return std::error_code(categorySingleton.add(std::string(msg)),
-                         categorySingleton);
-}
-
-char GenericError::ID = 0;
-
-GenericError::GenericError(Twine Msg) : Msg(Msg.str()) { }
-
-void GenericError::log(raw_ostream &OS) const {
-  OS << Msg;
-}
-
-} // namespace lld
diff --git a/lld/lib/Core/File.cpp b/lld/lib/Core/File.cpp
deleted file mode 100644
index ce33923c136e..000000000000
--- a/lld/lib/Core/File.cpp
+++ /dev/null
@@ -1,28 +0,0 @@
-//===- Core/File.cpp - A Container of Atoms -------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "lld/Core/File.h"
-#include <mutex>
-
-namespace lld {
-
-File::~File() = default;
-
-File::AtomVector<DefinedAtom> File::_noDefinedAtoms;
-File::AtomVector<UndefinedAtom> File::_noUndefinedAtoms;
-File::AtomVector<SharedLibraryAtom> File::_noSharedLibraryAtoms;
-File::AtomVector<AbsoluteAtom> File::_noAbsoluteAtoms;
-
-std::error_code File::parse() {
-  std::lock_guard<std::mutex> lock(_parseMutex);
-  if (!_lastError.hasValue())
-    _lastError = doParse();
-  return _lastError.getValue();
-}
-
-} // end namespace lld
diff --git a/lld/lib/Core/LinkingContext.cpp b/lld/lib/Core/LinkingContext.cpp
deleted file mode 100644
index 911ae606678d..000000000000
--- a/lld/lib/Core/LinkingContext.cpp
+++ /dev/null
@@ -1,69 +0,0 @@
-//===- lib/Core/LinkingContext.cpp - Linker Context Object Interface ------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "lld/Core/LinkingContext.h"
-#include "lld/Core/File.h"
-#include "lld/Core/Node.h"
-#include "lld/Core/Simple.h"
-#include "lld/Core/Writer.h"
-#include <algorithm>
-
-namespace lld {
-
-LinkingContext::LinkingContext() = default;
-
-LinkingContext::~LinkingContext() = default;
-
-bool LinkingContext::validate() {
-  return validateImpl();
-}
-
-llvm::Error LinkingContext::writeFile(const File &linkedFile) const {
-  return this->writer().writeFile(linkedFile, _outputPath);
-}
-
-std::unique_ptr<File> LinkingContext::createEntrySymbolFile() const {
-  return createEntrySymbolFile("<command line option -e>");
-}
-
-std::unique_ptr<File>
-LinkingContext::createEntrySymbolFile(StringRef filename) const {
-  if (entrySymbolName().empty())
-    return nullptr;
-  std::unique_ptr<SimpleFile> entryFile(new SimpleFile(filename,
-                                                       File::kindEntryObject));
-  entryFile->addAtom(
-      *(new (_allocator) SimpleUndefinedAtom(*entryFile, entrySymbolName())));
-  return std::move(entryFile);
-}
-
-std::unique_ptr<File> LinkingContext::createUndefinedSymbolFile() const {
-  return createUndefinedSymbolFile("<command line option -u or --defsym>");
-}
-
-std::unique_ptr<File>
-LinkingContext::createUndefinedSymbolFile(StringRef filename) const {
-  if (_initialUndefinedSymbols.empty())
-    return nullptr;
-  std::unique_ptr<SimpleFile> undefinedSymFile(
-    new SimpleFile(filename, File::kindUndefinedSymsObject));
-  for (StringRef undefSym : _initialUndefinedSymbols)
-    undefinedSymFile->addAtom(*(new (_allocator) SimpleUndefinedAtom(
-                                   *undefinedSymFile, undefSym)));
-  return std::move(undefinedSymFile);
-}
-
-void LinkingContext::createInternalFiles(
-    std::vector<std::unique_ptr<File>> &result) const {
-  if (std::unique_ptr<File> file = createEntrySymbolFile())
-    result.push_back(std::move(file));
-  if (std::unique_ptr<File> file = createUndefinedSymbolFile())
-    result.push_back(std::move(file));
-}
-
-} // end namespace lld
diff --git a/lld/lib/Core/Reader.cpp b/lld/lib/Core/Reader.cpp
deleted file mode 100644
index 3592d87ce627..000000000000
--- a/lld/lib/Core/Reader.cpp
+++ /dev/null
@@ -1,113 +0,0 @@
-//===- lib/Core/Reader.cpp ------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "lld/Core/Reader.h"
-#include "lld/Core/File.h"
-#include "lld/Core/Reference.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/BinaryFormat/Magic.h"
-#include "llvm/Support/Errc.h"
-#include "llvm/Support/FileSystem.h"
-#include "llvm/Support/MemoryBuffer.h"
-#include <algorithm>
-#include <memory>
-
-using llvm::file_magic;
-using llvm::identify_magic;
-
-namespace lld {
-
-YamlIOTaggedDocumentHandler::~YamlIOTaggedDocumentHandler() = default;
-
-void Registry::add(std::unique_ptr<Reader> reader) {
-  _readers.push_back(std::move(reader));
-}
-
-void Registry::add(std::unique_ptr<YamlIOTaggedDocumentHandler> handler) {
-  _yamlHandlers.push_back(std::move(handler));
-}
-
-ErrorOr<std::unique_ptr<File>>
-Registry::loadFile(std::unique_ptr<MemoryBuffer> mb) const {
-  // Get file magic.
-  StringRef content(mb->getBufferStart(), mb->getBufferSize());
-  file_magic fileType = identify_magic(content);
-
-  // Ask each registered reader if it can handle this file type or extension.
-  for (const std::unique_ptr<Reader> &reader : _readers) {
-    if (!reader->canParse(fileType, mb->getMemBufferRef()))
-      continue;
-    return reader->loadFile(std::move(mb), *this);
-  }
-
-  // No Reader could parse this file.
-  return make_error_code(llvm::errc::executable_format_error);
-}
-
-static const Registry::KindStrings kindStrings[] = {
-    {Reference::kindLayoutAfter, "layout-after"},
-    {Reference::kindAssociate, "associate"},
-    LLD_KIND_STRING_END};
-
-Registry::Registry() {
-  addKindTable(Reference::KindNamespace::all, Reference::KindArch::all,
-               kindStrings);
-}
-
-bool Registry::handleTaggedDoc(llvm::yaml::IO &io,
-                               const lld::File *&file) const {
-  for (const std::unique_ptr<YamlIOTaggedDocumentHandler> &h : _yamlHandlers)
-    if (h->handledDocTag(io, file))
-      return true;
-  return false;
-}
-
-void Registry::addKindTable(Reference::KindNamespace ns,
-                            Reference::KindArch arch,
-                            const KindStrings array[]) {
-  KindEntry entry = { ns, arch, array };
-  _kindEntries.push_back(entry);
-}
-
-bool Registry::referenceKindFromString(StringRef inputStr,
-                                       Reference::KindNamespace &ns,
-                                       Reference::KindArch &arch,
-                                       Reference::KindValue &value) const {
-  for (const KindEntry &entry : _kindEntries) {
-    for (const KindStrings *pair = entry.array; !pair->name.empty(); ++pair) {
-      if (!inputStr.equals(pair->name))
-        continue;
-      ns = entry.ns;
-      arch = entry.arch;
-      value = pair->value;
-      return true;
-    }
-  }
-  return false;
-}
-
-bool Registry::referenceKindToString(Reference::KindNamespace ns,
-                                     Reference::KindArch arch,
-                                     Reference::KindValue value,
-                                     StringRef &str) const {
-  for (const KindEntry &entry : _kindEntries) {
-    if (entry.ns != ns)
-      continue;
-    if (entry.arch != arch)
-      continue;
-    for (const KindStrings *pair = entry.array; !pair->name.empty(); ++pair) {
-      if (pair->value != value)
-        continue;
-      str = pair->name;
-      return true;
-    }
-  }
-  return false;
-}
-
-} // end namespace lld
diff --git a/lld/lib/Core/Resolver.cpp b/lld/lib/Core/Resolver.cpp
deleted file mode 100644
index 1ed0b1c6e618..000000000000
--- a/lld/lib/Core/Resolver.cpp
+++ /dev/null
@@ -1,496 +0,0 @@
-//===- Core/Resolver.cpp - Resolves Atom References -----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "lld/Core/Resolver.h"
-#include "lld/Common/LLVM.h"
-#include "lld/Core/ArchiveLibraryFile.h"
-#include "lld/Core/Atom.h"
-#include "lld/Core/File.h"
-#include "lld/Core/Instrumentation.h"
-#include "lld/Core/LinkingContext.h"
-#include "lld/Core/SharedLibraryFile.h"
-#include "lld/Core/SymbolTable.h"
-#include "lld/Core/UndefinedAtom.h"
-#include "llvm/ADT/iterator_range.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/Error.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/Format.h"
-#include "llvm/Support/raw_ostream.h"
-#include <algorithm>
-#include <cassert>
-#include <utility>
-#include <vector>
-
-namespace lld {
-
-llvm::Expected<bool> Resolver::handleFile(File &file) {
-  if (auto ec = _ctx.handleLoadedFile(file))
-    return std::move(ec);
-  bool undefAdded = false;
-  for (auto &atom : file.defined().owning_ptrs())
-    doDefinedAtom(std::move(atom));
-  for (auto &atom : file.undefined().owning_ptrs()) {
-    if (doUndefinedAtom(std::move(atom)))
-      undefAdded = true;
-  }
-  for (auto &atom : file.sharedLibrary().owning_ptrs())
-    doSharedLibraryAtom(std::move(atom));
-  for (auto &atom : file.absolute().owning_ptrs())
-    doAbsoluteAtom(std::move(atom));
-  return undefAdded;
-}
-
-llvm::Expected<bool> Resolver::forEachUndefines(File &file,
-                                                UndefCallback callback) {
-  size_t i = _undefineIndex[&file];
-  bool undefAdded = false;
-  do {
-    for (; i < _undefines.size(); ++i) {
-      StringRef undefName = _undefines[i];
-      if (undefName.empty())
-        continue;
-      const Atom *atom = _symbolTable.findByName(undefName);
-      if (!isa<UndefinedAtom>(atom) || _symbolTable.isCoalescedAway(atom)) {
-        // The symbol was resolved by some other file. Cache the result.
-        _undefines[i] = "";
-        continue;
-      }
-      auto undefAddedOrError = callback(undefName);
-      if (auto ec = undefAddedOrError.takeError())
-        return std::move(ec);
-      undefAdded |= undefAddedOrError.get();
-    }
-  } while (i < _undefines.size());
-  _undefineIndex[&file] = i;
-  return undefAdded;
-}
-
-llvm::Expected<bool> Resolver::handleArchiveFile(File &file) {
-  ArchiveLibraryFile *archiveFile = cast<ArchiveLibraryFile>(&file);
-  return forEachUndefines(file,
-                          [&](StringRef undefName) -> llvm::Expected<bool> {
-    if (File *member = archiveFile->find(undefName)) {
-      member->setOrdinal(_ctx.getNextOrdinalAndIncrement());
-      return handleFile(*member);
-    }
-    return false;
-  });
-}
-
-llvm::Error Resolver::handleSharedLibrary(File &file) {
-  // Add all the atoms from the shared library
-  SharedLibraryFile *sharedLibrary = cast<SharedLibraryFile>(&file);
-  auto undefAddedOrError = handleFile(*sharedLibrary);
-  if (auto ec = undefAddedOrError.takeError())
-    return ec;
-  undefAddedOrError =
-      forEachUndefines(file, [&](StringRef undefName) -> llvm::Expected<bool> {
-        auto atom = sharedLibrary->exports(undefName);
-        if (atom.get())
-          doSharedLibraryAtom(std::move(atom));
-        return false;
-      });
-
-  if (auto ec = undefAddedOrError.takeError())
-    return ec;
-  return llvm::Error::success();
-}
-
-bool Resolver::doUndefinedAtom(OwningAtomPtr<UndefinedAtom> atom) {
-  DEBUG_WITH_TYPE("resolver", llvm::dbgs()
-                    << "       UndefinedAtom: "
-                    << llvm::format("0x%09lX", atom.get())
-                    << ", name=" << atom.get()->name() << "\n");
-
-  // tell symbol table
-  bool newUndefAdded = _symbolTable.add(*atom.get());
-  if (newUndefAdded)
-    _undefines.push_back(atom.get()->name());
-
-  // add to list of known atoms
-  _atoms.push_back(OwningAtomPtr<Atom>(atom.release()));
-
-  return newUndefAdded;
-}
-
-// Called on each atom when a file is added. Returns true if a given
-// atom is added to the symbol table.
-void Resolver::doDefinedAtom(OwningAtomPtr<DefinedAtom> atom) {
-  DEBUG_WITH_TYPE("resolver", llvm::dbgs()
-                    << "         DefinedAtom: "
-                    << llvm::format("0x%09lX", atom.get())
-                    << ", file=#"
-                    << atom.get()->file().ordinal()
-                    << ", atom=#"
-                    << atom.get()->ordinal()
-                    << ", name="
-                    << atom.get()->name()
-                    << ", type="
-                    << atom.get()->contentType()
-                    << "\n");
-
-  // An atom that should never be dead-stripped is a dead-strip root.
-  if (_ctx.deadStrip() &&
-      atom.get()->deadStrip() == DefinedAtom::deadStripNever) {
-    _deadStripRoots.insert(atom.get());
-  }
-
-  // add to list of known atoms
-  _symbolTable.add(*atom.get());
-  _atoms.push_back(OwningAtomPtr<Atom>(atom.release()));
-}
-
-void Resolver::doSharedLibraryAtom(OwningAtomPtr<SharedLibraryAtom> atom) {
-  DEBUG_WITH_TYPE("resolver", llvm::dbgs()
-                    << "   SharedLibraryAtom: "
-                    << llvm::format("0x%09lX", atom.get())
-                    << ", name="
-                    << atom.get()->name()
-                    << "\n");
-
-  // tell symbol table
-  _symbolTable.add(*atom.get());
-
-  // add to list of known atoms
-  _atoms.push_back(OwningAtomPtr<Atom>(atom.release()));
-}
-
-void Resolver::doAbsoluteAtom(OwningAtomPtr<AbsoluteAtom> atom) {
-  DEBUG_WITH_TYPE("resolver", llvm::dbgs()
-                    << "       AbsoluteAtom: "
-                    << llvm::format("0x%09lX", atom.get())
-                    << ", name="
-                    << atom.get()->name()
-                    << "\n");
-
-  // tell symbol table
-  if (atom.get()->scope() != Atom::scopeTranslationUnit)
-    _symbolTable.add(*atom.get());
-
-  // add to list of known atoms
-  _atoms.push_back(OwningAtomPtr<Atom>(atom.release()));
-}
-
-// Returns true if at least one of N previous files has created an
-// undefined symbol.
-bool Resolver::undefinesAdded(int begin, int end) {
-  std::vector<std::unique_ptr<Node>> &inputs = _ctx.getNodes();
-  for (int i = begin; i < end; ++i)
-    if (FileNode *node = dyn_cast<FileNode>(inputs[i].get()))
-      if (_newUndefinesAdded[node->getFile()])
-        return true;
-  return false;
-}
-
-File *Resolver::getFile(int &index) {
-  std::vector<std::unique_ptr<Node>> &inputs = _ctx.getNodes();
-  if ((size_t)index >= inputs.size())
-    return nullptr;
-  if (GroupEnd *group = dyn_cast<GroupEnd>(inputs[index].get())) {
-    // We are at the end of the current group. If one or more new
-    // undefined atom has been added in the last groupSize files, we
-    // reiterate over the files.
-    int size = group->getSize();
-    if (undefinesAdded(index - size, index)) {
-      index -= size;
-      return getFile(index);
-    }
-    ++index;
-    return getFile(index);
-  }
-  return cast<FileNode>(inputs[index++].get())->getFile();
-}
-
-// Keep adding atoms until _ctx.getNextFile() returns an error. This
-// function is where undefined atoms are resolved.
-bool Resolver::resolveUndefines() {
-  DEBUG_WITH_TYPE("resolver",
-                  llvm::dbgs() << "******** Resolving undefines:\n");
-  ScopedTask task(getDefaultDomain(), "resolveUndefines");
-  int index = 0;
-  std::set<File *> seen;
-  for (;;) {
-    bool undefAdded = false;
-    DEBUG_WITH_TYPE("resolver",
-                    llvm::dbgs() << "Loading file #" << index << "\n");
-    File *file = getFile(index);
-    if (!file)
-      return true;
-    if (std::error_code ec = file->parse()) {
-      llvm::errs() << "Cannot open " + file->path() << ": " << ec.message()
-                   << "\n";
-      return false;
-    }
-    DEBUG_WITH_TYPE("resolver",
-                    llvm::dbgs() << "Loaded file: " << file->path() << "\n");
-    switch (file->kind()) {
-    case File::kindErrorObject:
-    case File::kindNormalizedObject:
-    case File::kindMachObject:
-    case File::kindCEntryObject:
-    case File::kindHeaderObject:
-    case File::kindEntryObject:
-    case File::kindUndefinedSymsObject:
-    case File::kindStubHelperObject:
-    case File::kindResolverMergedObject:
-    case File::kindSectCreateObject: {
-      // The same file may be visited more than once if the file is
-      // in --start-group and --end-group. Only library files should
-      // be processed more than once.
-      if (seen.count(file))
-        break;
-      seen.insert(file);
-      assert(!file->hasOrdinal());
-      file->setOrdinal(_ctx.getNextOrdinalAndIncrement());
-      auto undefAddedOrError = handleFile(*file);
-      if (auto EC = undefAddedOrError.takeError()) {
-        // FIXME: This should be passed to logAllUnhandledErrors but it needs
-        // to be passed a Twine instead of a string.
-        llvm::errs() << "Error in " + file->path() << ": ";
-        logAllUnhandledErrors(std::move(EC), llvm::errs(), std::string());
-        return false;
-      }
-      undefAdded = undefAddedOrError.get();
-      break;
-    }
-    case File::kindArchiveLibrary: {
-      if (!file->hasOrdinal())
-        file->setOrdinal(_ctx.getNextOrdinalAndIncrement());
-      auto undefAddedOrError = handleArchiveFile(*file);
-      if (auto EC = undefAddedOrError.takeError()) {
-        // FIXME: This should be passed to logAllUnhandledErrors but it needs
-        // to be passed a Twine instead of a string.
-        llvm::errs() << "Error in " + file->path() << ": ";
-        logAllUnhandledErrors(std::move(EC), llvm::errs(), std::string());
-        return false;
-      }
-      undefAdded = undefAddedOrError.get();
-      break;
-    }
-    case File::kindSharedLibrary:
-      if (!file->hasOrdinal())
-        file->setOrdinal(_ctx.getNextOrdinalAndIncrement());
-      if (auto EC = handleSharedLibrary(*file)) {
-        // FIXME: This should be passed to logAllUnhandledErrors but it needs
-        // to be passed a Twine instead of a string.
-        llvm::errs() << "Error in " + file->path() << ": ";
-        logAllUnhandledErrors(std::move(EC), llvm::errs(), std::string());
-        return false;
-      }
-      break;
-    }
-    _newUndefinesAdded[file] = undefAdded;
-  }
-}
-
-// switch all references to undefined or coalesced away atoms
-// to the new defined atom
-void Resolver::updateReferences() {
-  DEBUG_WITH_TYPE("resolver",
-                  llvm::dbgs() << "******** Updating references:\n");
-  ScopedTask task(getDefaultDomain(), "updateReferences");
-  for (const OwningAtomPtr<Atom> &atom : _atoms) {
-    if (const DefinedAtom *defAtom = dyn_cast<DefinedAtom>(atom.get())) {
-      for (const Reference *ref : *defAtom) {
-        // A reference of type kindAssociate shouldn't be updated.
-        // Instead, an atom having such reference will be removed
-        // if the target atom is coalesced away, so that they will
-        // go away as a group.
-        if (ref->kindNamespace() == lld::Reference::KindNamespace::all &&
-            ref->kindValue() == lld::Reference::kindAssociate) {
-          if (_symbolTable.isCoalescedAway(atom.get()))
-            _deadAtoms.insert(ref->target());
-          continue;
-        }
-        const Atom *newTarget = _symbolTable.replacement(ref->target());
-        const_cast<Reference *>(ref)->setTarget(newTarget);
-      }
-    }
-  }
-}
-
-// For dead code stripping, recursively mark atoms "live"
-void Resolver::markLive(const Atom *atom) {
-  // Mark the atom is live. If it's already marked live, then stop recursion.
-  auto exists = _liveAtoms.insert(atom);
-  if (!exists.second)
-    return;
-
-  // Mark all atoms it references as live
-  if (const DefinedAtom *defAtom = dyn_cast<DefinedAtom>(atom)) {
-    for (const Reference *ref : *defAtom)
-      markLive(ref->target());
-    for (auto &p : llvm::make_range(_reverseRef.equal_range(defAtom))) {
-      const Atom *target = p.second;
-      markLive(target);
-    }
-  }
-}
-
-static bool isBackref(const Reference *ref) {
-  if (ref->kindNamespace() != lld::Reference::KindNamespace::all)
-    return false;
-  return (ref->kindValue() == lld::Reference::kindLayoutAfter);
-}
-
-// remove all atoms not actually used
-void Resolver::deadStripOptimize() {
-  DEBUG_WITH_TYPE("resolver",
-                  llvm::dbgs() << "******** Dead stripping unused atoms:\n");
-  ScopedTask task(getDefaultDomain(), "deadStripOptimize");
-  // only do this optimization with -dead_strip
-  if (!_ctx.deadStrip())
-    return;
-
-  // Some type of references prevent referring atoms to be dead-striped.
-  // Make a reverse map of such references before traversing the graph.
-  // While traversing the list of atoms, mark AbsoluteAtoms as live
-  // in order to avoid reclaim.
-  for (const OwningAtomPtr<Atom> &atom : _atoms) {
-    if (const DefinedAtom *defAtom = dyn_cast<DefinedAtom>(atom.get()))
-      for (const Reference *ref : *defAtom)
-        if (isBackref(ref))
-          _reverseRef.insert(std::make_pair(ref->target(), atom.get()));
-    if (const AbsoluteAtom *absAtom = dyn_cast<AbsoluteAtom>(atom.get()))
-      markLive(absAtom);
-  }
-
-  // By default, shared libraries are built with all globals as dead strip roots
-  if (_ctx.globalsAreDeadStripRoots())
-    for (const OwningAtomPtr<Atom> &atom : _atoms)
-      if (const DefinedAtom *defAtom = dyn_cast<DefinedAtom>(atom.get()))
-        if (defAtom->scope() == DefinedAtom::scopeGlobal)
-          _deadStripRoots.insert(defAtom);
-
-  // Or, use list of names that are dead strip roots.
-  for (const StringRef &name : _ctx.deadStripRoots()) {
-    const Atom *symAtom = _symbolTable.findByName(name);
-    assert(symAtom);
-    _deadStripRoots.insert(symAtom);
-  }
-
-  // mark all roots as live, and recursively all atoms they reference
-  for (const Atom *dsrAtom : _deadStripRoots)
-    markLive(dsrAtom);
-
-  // now remove all non-live atoms from _atoms
-  llvm::erase_if(_atoms, [&](OwningAtomPtr<Atom> &a) {
-    return _liveAtoms.count(a.get()) == 0;
-  });
-}
-
-// error out if some undefines remain
-bool Resolver::checkUndefines() {
-  DEBUG_WITH_TYPE("resolver",
-                  llvm::dbgs() << "******** Checking for undefines:\n");
-
-  // build vector of remaining undefined symbols
-  std::vector<const UndefinedAtom *> undefinedAtoms = _symbolTable.undefines();
-  if (_ctx.deadStrip()) {
-    // When dead code stripping, we don't care if dead atoms are undefined.
-    llvm::erase_if(undefinedAtoms,
-                   [&](const Atom *a) { return _liveAtoms.count(a) == 0; });
-  }
-
-  if (undefinedAtoms.empty())
-    return false;
-
-  // Warn about unresolved symbols.
-  bool foundUndefines = false;
-  for (const UndefinedAtom *undef : undefinedAtoms) {
-    // Skip over a weak symbol.
-    if (undef->canBeNull() != UndefinedAtom::canBeNullNever)
-      continue;
-
-    // If this is a library and undefined symbols are allowed on the
-    // target platform, skip over it.
-    if (isa<SharedLibraryFile>(undef->file()) && _ctx.allowShlibUndefines())
-      continue;
-
-    // If the undefine is coalesced away, skip over it.
-    if (_symbolTable.isCoalescedAway(undef))
-      continue;
-
-    // Seems like this symbol is undefined. Warn that.
-    foundUndefines = true;
-    if (_ctx.printRemainingUndefines()) {
-      llvm::errs() << "Undefined symbol: " << undef->file().path() << ": "
-                   << _ctx.demangle(undef->name()) << "\n";
-    }
-  }
-  if (!foundUndefines)
-    return false;
-  if (_ctx.printRemainingUndefines())
-    llvm::errs() << "symbol(s) not found\n";
-  return true;
-}
-
-// Remove from _atoms all coalesced away atoms.
-void Resolver::removeCoalescedAwayAtoms() {
-  DEBUG_WITH_TYPE("resolver",
-                  llvm::dbgs() << "******** Removing coalesced away atoms:\n");
-  ScopedTask task(getDefaultDomain(), "removeCoalescedAwayAtoms");
-  llvm::erase_if(_atoms, [&](OwningAtomPtr<Atom> &a) {
-    return _symbolTable.isCoalescedAway(a.get()) || _deadAtoms.count(a.get());
-  });
-}
-
-bool Resolver::resolve() {
-  DEBUG_WITH_TYPE("resolver",
-                  llvm::dbgs() << "******** Resolving atom references:\n");
-  if (!resolveUndefines())
-    return false;
-  updateReferences();
-  deadStripOptimize();
-  if (checkUndefines()) {
-    DEBUG_WITH_TYPE("resolver", llvm::dbgs() << "Found undefines... ");
-    if (!_ctx.allowRemainingUndefines()) {
-      DEBUG_WITH_TYPE("resolver", llvm::dbgs() << "which we don't allow\n");
-      return false;
-    }
-    DEBUG_WITH_TYPE("resolver", llvm::dbgs() << "which we are ok with\n");
-  }
-  removeCoalescedAwayAtoms();
-  _result->addAtoms(_atoms);
-  DEBUG_WITH_TYPE("resolver", llvm::dbgs() << "******** Finished resolver\n");
-  return true;
-}
-
-void Resolver::MergedFile::addAtoms(
-                              llvm::MutableArrayRef<OwningAtomPtr<Atom>> all) {
-  ScopedTask task(getDefaultDomain(), "addAtoms");
-  DEBUG_WITH_TYPE("resolver", llvm::dbgs() << "Resolver final atom list:\n");
-
-  for (OwningAtomPtr<Atom> &atom : all) {
-#ifndef NDEBUG
-    if (auto *definedAtom = dyn_cast<DefinedAtom>(atom.get())) {
-      DEBUG_WITH_TYPE("resolver", llvm::dbgs()
-                      << llvm::format("    0x%09lX", definedAtom)
-                      << ", file=#"
-                      << definedAtom->file().ordinal()
-                      << ", atom=#"
-                      << definedAtom->ordinal()
-                      << ", name="
-                      << definedAtom->name()
-                      << ", type="
-                      << definedAtom->contentType()
-                      << "\n");
-    } else {
-      DEBUG_WITH_TYPE("resolver", llvm::dbgs()
-                      << llvm::format("    0x%09lX", atom.get())
-                      << ", name="
-                      << atom.get()->name()
-                      << "\n");
-    }
-#endif
-    addAtom(*atom.release());
-  }
-}
-
-} // namespace lld
diff --git a/lld/lib/Core/SymbolTable.cpp b/lld/lib/Core/SymbolTable.cpp
deleted file mode 100644
index 3ce9555aa494..000000000000
--- a/lld/lib/Core/SymbolTable.cpp
+++ /dev/null
@@ -1,284 +0,0 @@
-//===- Core/SymbolTable.cpp - Main Symbol Table ---------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "lld/Core/SymbolTable.h"
-#include "lld/Common/LLVM.h"
-#include "lld/Core/AbsoluteAtom.h"
-#include "lld/Core/Atom.h"
-#include "lld/Core/DefinedAtom.h"
-#include "lld/Core/File.h"
-#include "lld/Core/LinkingContext.h"
-#include "lld/Core/Resolver.h"
-#include "lld/Core/SharedLibraryAtom.h"
-#include "lld/Core/UndefinedAtom.h"
-#include "llvm/ADT/ArrayRef.h"
-#include "llvm/ADT/DenseMapInfo.h"
-#include "llvm/ADT/Hashing.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/raw_ostream.h"
-#include <algorithm>
-#include <cassert>
-#include <cstdlib>
-#include <vector>
-
-namespace lld {
-bool SymbolTable::add(const UndefinedAtom &atom) { return addByName(atom); }
-
-bool SymbolTable::add(const SharedLibraryAtom &atom) { return addByName(atom); }
-
-bool SymbolTable::add(const AbsoluteAtom &atom) { return addByName(atom); }
-
-bool SymbolTable::add(const DefinedAtom &atom) {
-  if (!atom.name().empty() &&
-      atom.scope() != DefinedAtom::scopeTranslationUnit) {
-    // Named atoms cannot be merged by content.
-    assert(atom.merge() != DefinedAtom::mergeByContent);
-    // Track named atoms that are not scoped to file (static).
-    return addByName(atom);
-  }
-  if (atom.merge() == DefinedAtom::mergeByContent) {
-    // Named atoms cannot be merged by content.
-    assert(atom.name().empty());
-    // Currently only read-only constants can be merged.
-    if (atom.permissions() == DefinedAtom::permR__)
-      return addByContent(atom);
-    // TODO: support mergeByContent of data atoms by comparing content & fixups.
-  }
-  return false;
-}
-
-enum NameCollisionResolution {
-  NCR_First,
-  NCR_Second,
-  NCR_DupDef,
-  NCR_DupUndef,
-  NCR_DupShLib,
-  NCR_Error
-};
-
-static NameCollisionResolution cases[4][4] = {
-  //regular     absolute    undef      sharedLib
-  {
-    // first is regular
-    NCR_DupDef, NCR_Error,   NCR_First, NCR_First
-  },
-  {
-    // first is absolute
-    NCR_Error,  NCR_Error,  NCR_First, NCR_First
-  },
-  {
-    // first is undef
-    NCR_Second, NCR_Second, NCR_DupUndef, NCR_Second
-  },
-  {
-    // first is sharedLib
-    NCR_Second, NCR_Second, NCR_First, NCR_DupShLib
-  }
-};
-
-static NameCollisionResolution collide(Atom::Definition first,
-                                       Atom::Definition second) {
-  return cases[first][second];
-}
-
-enum MergeResolution {
-  MCR_First,
-  MCR_Second,
-  MCR_Largest,
-  MCR_SameSize,
-  MCR_Error
-};
-
-static MergeResolution mergeCases[][6] = {
-  // no          tentative      weak          weakAddress   sameNameAndSize largest
-  {MCR_Error,    MCR_First,     MCR_First,    MCR_First,    MCR_SameSize,   MCR_Largest},  // no
-  {MCR_Second,   MCR_Largest,   MCR_Second,   MCR_Second,   MCR_SameSize,   MCR_Largest},  // tentative
-  {MCR_Second,   MCR_First,     MCR_First,    MCR_Second,   MCR_SameSize,   MCR_Largest},  // weak
-  {MCR_Second,   MCR_First,     MCR_First,    MCR_First,    MCR_SameSize,   MCR_Largest},  // weakAddress
-  {MCR_SameSize, MCR_SameSize,  MCR_SameSize, MCR_SameSize, MCR_SameSize,   MCR_SameSize}, // sameSize
-  {MCR_Largest,  MCR_Largest,   MCR_Largest,  MCR_Largest,  MCR_SameSize,   MCR_Largest},  // largest
-};
-
-static MergeResolution mergeSelect(DefinedAtom::Merge first,
-                                   DefinedAtom::Merge second) {
-  assert(first != DefinedAtom::mergeByContent);
-  assert(second != DefinedAtom::mergeByContent);
-  return mergeCases[first][second];
-}
-
-bool SymbolTable::addByName(const Atom &newAtom) {
-  StringRef name = newAtom.name();
-  assert(!name.empty());
-  const Atom *existing = findByName(name);
-  if (existing == nullptr) {
-    // Name is not in symbol table yet, add it associate with this atom.
-    _nameTable[name] = &newAtom;
-    return true;
-  }
-
-  // Do nothing if the same object is added more than once.
-  if (existing == &newAtom)
-    return false;
-
-  // Name is already in symbol table and associated with another atom.
-  bool useNew = true;
-  switch (collide(existing->definition(), newAtom.definition())) {
-  case NCR_First:
-    useNew = false;
-    break;
-  case NCR_Second:
-    useNew = true;
-    break;
-  case NCR_DupDef: {
-    const auto *existingDef = cast<DefinedAtom>(existing);
-    const auto *newDef = cast<DefinedAtom>(&newAtom);
-    switch (mergeSelect(existingDef->merge(), newDef->merge())) {
-    case MCR_First:
-      useNew = false;
-      break;
-    case MCR_Second:
-      useNew = true;
-      break;
-    case MCR_Largest: {
-      uint64_t existingSize = existingDef->sectionSize();
-      uint64_t newSize = newDef->sectionSize();
-      useNew = (newSize >= existingSize);
-      break;
-    }
-    case MCR_SameSize: {
-      uint64_t existingSize = existingDef->sectionSize();
-      uint64_t newSize = newDef->sectionSize();
-      if (existingSize == newSize) {
-        useNew = true;
-        break;
-      }
-      llvm::errs() << "Size mismatch: " << existing->name() << " ("
-                   << existingSize << ") " << newAtom.name() << " (" << newSize
-                   << ")\n";
-      LLVM_FALLTHROUGH;
-    }
-    case MCR_Error:
-      llvm::errs() << "Duplicate symbols: " << existing->name() << ":"
-                   << existing->file().path() << " and " << newAtom.name()
-                   << ":" << newAtom.file().path() << "\n";
-      llvm::report_fatal_error("duplicate symbol error");
-      break;
-    }
-    break;
-  }
-  case NCR_DupUndef: {
-    const UndefinedAtom* existingUndef = cast<UndefinedAtom>(existing);
-    const UndefinedAtom* newUndef = cast<UndefinedAtom>(&newAtom);
-
-    bool sameCanBeNull = (existingUndef->canBeNull() == newUndef->canBeNull());
-    if (sameCanBeNull)
-      useNew = false;
-    else
-      useNew = (newUndef->canBeNull() < existingUndef->canBeNull());
-    break;
-  }
-  case NCR_DupShLib: {
-    useNew = false;
-    break;
-  }
-  case NCR_Error:
-    llvm::errs() << "SymbolTable: error while merging " << name << "\n";
-    llvm::report_fatal_error("duplicate symbol error");
-    break;
-  }
-
-  if (useNew) {
-    // Update name table to use new atom.
-    _nameTable[name] = &newAtom;
-    // Add existing atom to replacement table.
-    _replacedAtoms[existing] = &newAtom;
-  } else {
-    // New atom is not being used.  Add it to replacement table.
-    _replacedAtoms[&newAtom] = existing;
-  }
-  return false;
-}
-
-unsigned SymbolTable::AtomMappingInfo::getHashValue(const DefinedAtom *atom) {
-  auto content = atom->rawContent();
-  return llvm::hash_combine(atom->size(),
-                            atom->contentType(),
-                            llvm::hash_combine_range(content.begin(),
-                                                     content.end()));
-}
-
-bool SymbolTable::AtomMappingInfo::isEqual(const DefinedAtom * const l,
-                                           const DefinedAtom * const r) {
-  if (l == r)
-    return true;
-  if (l == getEmptyKey() || r == getEmptyKey())
-    return false;
-  if (l == getTombstoneKey() || r == getTombstoneKey())
-    return false;
-  if (l->contentType() != r->contentType())
-    return false;
-  if (l->size() != r->size())
-    return false;
-  if (l->sectionChoice() != r->sectionChoice())
-    return false;
-  if (l->sectionChoice() == DefinedAtom::sectionCustomRequired) {
-    if (!l->customSectionName().equals(r->customSectionName()))
-      return false;
-  }
-  ArrayRef<uint8_t> lc = l->rawContent();
-  ArrayRef<uint8_t> rc = r->rawContent();
-  return memcmp(lc.data(), rc.data(), lc.size()) == 0;
-}
-
-bool SymbolTable::addByContent(const DefinedAtom &newAtom) {
-  AtomContentSet::iterator pos = _contentTable.find(&newAtom);
-  if (pos == _contentTable.end()) {
-    _contentTable.insert(&newAtom);
-    return true;
-  }
-  const Atom* existing = *pos;
-  // New atom is not being used.  Add it to replacement table.
-  _replacedAtoms[&newAtom] = existing;
-  return false;
-}
-
-const Atom *SymbolTable::findByName(StringRef sym) {
-  NameToAtom::iterator pos = _nameTable.find(sym);
-  if (pos == _nameTable.end())
-    return nullptr;
-  return pos->second;
-}
-
-const Atom *SymbolTable::replacement(const Atom *atom) {
-  // Find the replacement for a given atom. Atoms in _replacedAtoms
-  // may be chained, so find the last one.
-  for (;;) {
-    AtomToAtom::iterator pos = _replacedAtoms.find(atom);
-    if (pos == _replacedAtoms.end())
-      return atom;
-    atom = pos->second;
-  }
-}
-
-bool SymbolTable::isCoalescedAway(const Atom *atom) {
-  return _replacedAtoms.count(atom) > 0;
-}
-
-std::vector<const UndefinedAtom *> SymbolTable::undefines() {
-  std::vector<const UndefinedAtom *> ret;
-  for (auto it : _nameTable) {
-    const Atom *atom = it.second;
-    assert(atom != nullptr);
-    if (const auto *undef = dyn_cast<const UndefinedAtom>(atom))
-      if (_replacedAtoms.count(undef) == 0)
-        ret.push_back(undef);
-  }
-  return ret;
-}
-
-} // namespace lld
diff --git a/lld/lib/Core/Writer.cpp b/lld/lib/Core/Writer.cpp
deleted file mode 100644
index 12788b187e11..000000000000
--- a/lld/lib/Core/Writer.cpp
+++ /dev/null
@@ -1,17 +0,0 @@
-//===- lib/Core/Writer.cpp ------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "lld/Core/Writer.h"
-
-namespace lld {
-
-Writer::Writer() = default;
-
-Writer::~Writer() = default;
-
-} // end namespace lld
diff --git a/lld/lib/Driver/DarwinLdDriver.cpp b/lld/lib/Driver/DarwinLdDriver.cpp
deleted file mode 100644
index 21d125726192..000000000000
--- a/lld/lib/Driver/DarwinLdDriver.cpp
+++ /dev/null
@@ -1,1229 +0,0 @@
-//===- lib/Driver/DarwinLdDriver.cpp --------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-///
-/// \file
-///
-/// Concrete instance of the Driver for darwin's ld.
-///
-//===----------------------------------------------------------------------===//
-
-#include "lld/Common/Args.h"
-#include "lld/Common/ErrorHandler.h"
-#include "lld/Common/LLVM.h"
-#include "lld/Core/ArchiveLibraryFile.h"
-#include "lld/Core/Error.h"
-#include "lld/Core/File.h"
-#include "lld/Core/Instrumentation.h"
-#include "lld/Core/LinkingContext.h"
-#include "lld/Core/Node.h"
-#include "lld/Core/PassManager.h"
-#include "lld/Core/Resolver.h"
-#include "lld/Core/SharedLibraryFile.h"
-#include "lld/Core/Simple.h"
-#include "lld/ReaderWriter/MachOLinkingContext.h"
-#include "llvm/ADT/ArrayRef.h"
-#include "llvm/ADT/Optional.h"
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/SmallString.h"
-#include "llvm/ADT/StringExtras.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/ADT/Twine.h"
-#include "llvm/BinaryFormat/MachO.h"
-#include "llvm/Option/Arg.h"
-#include "llvm/Option/ArgList.h"
-#include "llvm/Option/OptTable.h"
-#include "llvm/Option/Option.h"
-#include "llvm/Support/Casting.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/Error.h"
-#include "llvm/Support/ErrorOr.h"
-#include "llvm/Support/Format.h"
-#include "llvm/Support/MathExtras.h"
-#include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/Path.h"
-#include "llvm/Support/raw_ostream.h"
-#include <algorithm>
-#include <cstdint>
-#include <memory>
-#include <string>
-#include <system_error>
-#include <utility>
-#include <vector>
-
-using namespace lld;
-
-namespace {
-
-// Create enum with OPT_xxx values for each option in DarwinLdOptions.td
-enum {
-  OPT_INVALID = 0,
-#define OPTION(PREFIX, NAME, ID, KIND, GROUP, ALIAS, ALIASARGS, FLAGS, PARAM,  \
-               HELP, META, VALUES)                                             \
-  OPT_##ID,
-#include "DarwinLdOptions.inc"
-#undef OPTION
-};
-
-// Create prefix string literals used in DarwinLdOptions.td
-#define PREFIX(NAME, VALUE) const char *const NAME[] = VALUE;
-#include "DarwinLdOptions.inc"
-#undef PREFIX
-
-// Create table mapping all options defined in DarwinLdOptions.td
-static const llvm::opt::OptTable::Info InfoTable[] = {
-#define OPTION(PREFIX, NAME, ID, KIND, GROUP, ALIAS, ALIASARGS, FLAGS, PARAM,  \
-               HELPTEXT, METAVAR, VALUES)                                      \
-  {PREFIX,      NAME,      HELPTEXT,                                           \
-   METAVAR,     OPT_##ID,  llvm::opt::Option::KIND##Class,                     \
-   PARAM,       FLAGS,     OPT_##GROUP,                                        \
-   OPT_##ALIAS, ALIASARGS, VALUES},
-#include "DarwinLdOptions.inc"
-#undef OPTION
-};
-
-// Create OptTable class for parsing actual command line arguments
-class DarwinLdOptTable : public llvm::opt::OptTable {
-public:
-  DarwinLdOptTable() : OptTable(InfoTable) {}
-};
-
-static std::vector<std::unique_ptr<File>>
-makeErrorFile(StringRef path, std::error_code ec) {
-  std::vector<std::unique_ptr<File>> result;
-  result.push_back(std::make_unique<ErrorFile>(path, ec));
-  return result;
-}
-
-static std::vector<std::unique_ptr<File>>
-parseMemberFiles(std::unique_ptr<File> file) {
-  std::vector<std::unique_ptr<File>> members;
-  if (auto *archive = dyn_cast<ArchiveLibraryFile>(file.get())) {
-    if (std::error_code ec = archive->parseAllMembers(members))
-      return makeErrorFile(file->path(), ec);
-  } else {
-    members.push_back(std::move(file));
-  }
-  return members;
-}
-
-std::vector<std::unique_ptr<File>> loadFile(MachOLinkingContext &ctx,
-                                            StringRef path, bool wholeArchive,
-                                            bool upwardDylib) {
-  if (ctx.logInputFiles())
-    message(path);
-
-  ErrorOr<std::unique_ptr<MemoryBuffer>> mbOrErr = ctx.getMemoryBuffer(path);
-  if (std::error_code ec = mbOrErr.getError())
-    return makeErrorFile(path, ec);
-  ErrorOr<std::unique_ptr<File>> fileOrErr =
-      ctx.registry().loadFile(std::move(mbOrErr.get()));
-  if (std::error_code ec = fileOrErr.getError())
-    return makeErrorFile(path, ec);
-  std::unique_ptr<File> &file = fileOrErr.get();
-
-  // If file is a dylib, inform LinkingContext about it.
-  if (SharedLibraryFile *shl = dyn_cast<SharedLibraryFile>(file.get())) {
-    if (std::error_code ec = shl->parse())
-      return makeErrorFile(path, ec);
-    ctx.registerDylib(reinterpret_cast<mach_o::MachODylibFile *>(shl),
-                      upwardDylib);
-  }
-  if (wholeArchive)
-    return parseMemberFiles(std::move(file));
-  std::vector<std::unique_ptr<File>> files;
-  files.push_back(std::move(file));
-  return files;
-}
-
-} // end anonymous namespace
-
-// Test may be running on Windows. Canonicalize the path
-// separator to '/' to get consistent outputs for tests.
-static std::string canonicalizePath(StringRef path) {
-  char sep = llvm::sys::path::get_separator().front();
-  if (sep != '/') {
-    std::string fixedPath = std::string(path);
-    std::replace(fixedPath.begin(), fixedPath.end(), sep, '/');
-    return fixedPath;
-  } else {
-    return std::string(path);
-  }
-}
-
-static void addFile(StringRef path, MachOLinkingContext &ctx,
-                    bool loadWholeArchive, bool upwardDylib) {
-  std::vector<std::unique_ptr<File>> files =
-      loadFile(ctx, path, loadWholeArchive, upwardDylib);
-  for (std::unique_ptr<File> &file : files)
-    ctx.getNodes().push_back(std::make_unique<FileNode>(std::move(file)));
-}
-
-// Export lists are one symbol per line.  Blank lines are ignored.
-// Trailing comments start with #.
-static std::error_code parseExportsList(StringRef exportFilePath,
-                                        MachOLinkingContext &ctx) {
-  // Map in export list file.
-  ErrorOr<std::unique_ptr<MemoryBuffer>> mb =
-                                   MemoryBuffer::getFileOrSTDIN(exportFilePath);
-  if (std::error_code ec = mb.getError())
-    return ec;
-  ctx.addInputFileDependency(exportFilePath);
-  StringRef buffer = mb->get()->getBuffer();
-  while (!buffer.empty()) {
-    // Split off each line in the file.
-    std::pair<StringRef, StringRef> lineAndRest = buffer.split('\n');
-    StringRef line = lineAndRest.first;
-    // Ignore trailing # comments.
-    std::pair<StringRef, StringRef> symAndComment = line.split('#');
-    StringRef sym = symAndComment.first.trim();
-    if (!sym.empty())
-      ctx.addExportSymbol(sym);
-    buffer = lineAndRest.second;
-  }
-  return std::error_code();
-}
-
-/// Order files are one symbol per line. Blank lines are ignored.
-/// Trailing comments start with #. Symbol names can be prefixed with an
-/// architecture name and/or .o leaf name.  Examples:
-///     _foo
-///     bar.o:_bar
-///     libfrob.a(bar.o):_bar
-///     x86_64:_foo64
-static std::error_code parseOrderFile(StringRef orderFilePath,
-                                      MachOLinkingContext &ctx) {
-  // Map in order file.
-  ErrorOr<std::unique_ptr<MemoryBuffer>> mb =
-                                   MemoryBuffer::getFileOrSTDIN(orderFilePath);
-  if (std::error_code ec = mb.getError())
-    return ec;
-  ctx.addInputFileDependency(orderFilePath);
-  StringRef buffer = mb->get()->getBuffer();
-  while (!buffer.empty()) {
-    // Split off each line in the file.
-    std::pair<StringRef, StringRef> lineAndRest = buffer.split('\n');
-    StringRef line = lineAndRest.first;
-    buffer = lineAndRest.second;
-    // Ignore trailing # comments.
-    std::pair<StringRef, StringRef> symAndComment = line.split('#');
-    if (symAndComment.first.empty())
-      continue;
-    StringRef sym = symAndComment.first.trim();
-    if (sym.empty())
-      continue;
-    // Check for prefix.
-    StringRef prefix;
-    std::pair<StringRef, StringRef> prefixAndSym = sym.split(':');
-    if (!prefixAndSym.second.empty()) {
-      sym = prefixAndSym.second;
-      prefix = prefixAndSym.first;
-      if (!prefix.endswith(".o") && !prefix.endswith(".o)")) {
-        // If arch name prefix does not match arch being linked, ignore symbol.
-        if (!ctx.archName().equals(prefix))
-          continue;
-        prefix = "";
-      }
-    } else
-     sym = prefixAndSym.first;
-    if (!sym.empty()) {
-      ctx.appendOrderedSymbol(sym, prefix);
-      // llvm::errs() << sym << ", prefix=" << prefix << "\n";
-    }
-  }
-  return std::error_code();
-}
-
-//
-// There are two variants of the  -filelist option:
-//
-//   -filelist <path>
-// In this variant, the path is to a text file which contains one file path
-// per line.  There are no comments or trimming of whitespace.
-//
-//   -fileList <path>,<dir>
-// In this variant, the path is to a text file which contains a partial path
-// per line. The <dir> prefix is prepended to each partial path.
-//
-static llvm::Error loadFileList(StringRef fileListPath,
-                                MachOLinkingContext &ctx, bool forceLoad) {
-  // If there is a comma, split off <dir>.
-  std::pair<StringRef, StringRef> opt = fileListPath.split(',');
-  StringRef filePath = opt.first;
-  StringRef dirName = opt.second;
-  ctx.addInputFileDependency(filePath);
-  // Map in file list file.
-  ErrorOr<std::unique_ptr<MemoryBuffer>> mb =
-                                        MemoryBuffer::getFileOrSTDIN(filePath);
-  if (std::error_code ec = mb.getError())
-    return llvm::errorCodeToError(ec);
-  StringRef buffer = mb->get()->getBuffer();
-  while (!buffer.empty()) {
-    // Split off each line in the file.
-    std::pair<StringRef, StringRef> lineAndRest = buffer.split('\n');
-    StringRef line = lineAndRest.first;
-    StringRef path;
-    if (!dirName.empty()) {
-      // If there is a <dir> then prepend dir to each line.
-      SmallString<256> fullPath;
-      fullPath.assign(dirName);
-      llvm::sys::path::append(fullPath, Twine(line));
-      path = ctx.copy(fullPath.str());
-    } else {
-      // No <dir> use whole line as input file path.
-      path = ctx.copy(line);
-    }
-    if (!ctx.pathExists(path)) {
-      return llvm::make_error<GenericError>(Twine("File not found '")
-                                            + path
-                                            + "'");
-    }
-    if (ctx.testingFileUsage()) {
-      message("Found filelist entry " + canonicalizePath(path));
-    }
-    addFile(path, ctx, forceLoad, false);
-    buffer = lineAndRest.second;
-  }
-  return llvm::Error::success();
-}
-
-/// Parse number assuming it is base 16, but allow 0x prefix.
-static bool parseNumberBase16(StringRef numStr, uint64_t &baseAddress) {
-  if (numStr.startswith_insensitive("0x"))
-    numStr = numStr.drop_front(2);
-  return numStr.getAsInteger(16, baseAddress);
-}
-
-static void parseLLVMOptions(const LinkingContext &ctx) {
-  // Honor -mllvm
-  if (!ctx.llvmOptions().empty()) {
-    unsigned numArgs = ctx.llvmOptions().size();
-    auto **args = new const char *[numArgs + 2];
-    args[0] = "lld (LLVM option parsing)";
-    for (unsigned i = 0; i != numArgs; ++i)
-      args[i + 1] = ctx.llvmOptions()[i];
-    args[numArgs + 1] = nullptr;
-    llvm::cl::ResetAllOptionOccurrences();
-    llvm::cl::ParseCommandLineOptions(numArgs + 1, args);
-  }
-}
-
-namespace lld {
-namespace mach_o {
-
-bool parse(llvm::ArrayRef<const char *> args, MachOLinkingContext &ctx) {
-  // Parse command line options using DarwinLdOptions.td
-  DarwinLdOptTable table;
-  unsigned missingIndex;
-  unsigned missingCount;
-  llvm::opt::InputArgList parsedArgs =
-      table.ParseArgs(args.slice(1), missingIndex, missingCount);
-  if (missingCount) {
-    error("missing arg value for '" +
-          Twine(parsedArgs.getArgString(missingIndex)) + "' expected " +
-          Twine(missingCount) + " argument(s).");
-    return false;
-  }
-
-  for (auto unknownArg : parsedArgs.filtered(OPT_UNKNOWN)) {
-    warn("ignoring unknown argument: " +
-         Twine(unknownArg->getAsString(parsedArgs)));
-  }
-
-  errorHandler().verbose = parsedArgs.hasArg(OPT_v);
-  errorHandler().errorLimit = args::getInteger(parsedArgs, OPT_error_limit, 20);
-
-  // Figure out output kind ( -dylib, -r, -bundle, -preload, or -static )
-  llvm::MachO::HeaderFileType fileType = llvm::MachO::MH_EXECUTE;
-  bool isStaticExecutable = false;
-  if (llvm::opt::Arg *kind = parsedArgs.getLastArg(
-          OPT_dylib, OPT_relocatable, OPT_bundle, OPT_static, OPT_preload)) {
-    switch (kind->getOption().getID()) {
-    case OPT_dylib:
-      fileType = llvm::MachO::MH_DYLIB;
-      break;
-    case OPT_relocatable:
-      fileType = llvm::MachO::MH_OBJECT;
-      break;
-    case OPT_bundle:
-      fileType = llvm::MachO::MH_BUNDLE;
-      break;
-    case OPT_static:
-      fileType = llvm::MachO::MH_EXECUTE;
-      isStaticExecutable = true;
-      break;
-    case OPT_preload:
-      fileType = llvm::MachO::MH_PRELOAD;
-      break;
-    }
-  }
-
-  // Handle -arch xxx
-  MachOLinkingContext::Arch arch = MachOLinkingContext::arch_unknown;
-  if (llvm::opt::Arg *archStr = parsedArgs.getLastArg(OPT_arch)) {
-    arch = MachOLinkingContext::archFromName(archStr->getValue());
-    if (arch == MachOLinkingContext::arch_unknown) {
-      error("unknown arch named '" + Twine(archStr->getValue()) + "'");
-      return false;
-    }
-  }
-  // If no -arch specified, scan input files to find first non-fat .o file.
-  if (arch == MachOLinkingContext::arch_unknown) {
-    for (auto &inFile : parsedArgs.filtered(OPT_INPUT)) {
-      // This is expensive because it opens and maps the file.  But that is
-      // ok because no -arch is rare.
-      if (MachOLinkingContext::isThinObjectFile(inFile->getValue(), arch))
-        break;
-    }
-    if (arch == MachOLinkingContext::arch_unknown &&
-        !parsedArgs.getLastArg(OPT_test_file_usage)) {
-      // If no -arch and no options at all, print usage message.
-      if (parsedArgs.size() == 0) {
-        table.printHelp(llvm::outs(),
-                        (std::string(args[0]) + " [options] file...").c_str(),
-                        "LLVM Linker", false);
-      } else {
-        error("-arch not specified and could not be inferred");
-      }
-      return false;
-    }
-  }
-
-  // Handle -macosx_version_min or -ios_version_min
-  MachOLinkingContext::OS os = MachOLinkingContext::OS::unknown;
-  uint32_t minOSVersion = 0;
-  if (llvm::opt::Arg *minOS =
-          parsedArgs.getLastArg(OPT_macosx_version_min, OPT_ios_version_min,
-                                OPT_ios_simulator_version_min)) {
-    switch (minOS->getOption().getID()) {
-    case OPT_macosx_version_min:
-      os = MachOLinkingContext::OS::macOSX;
-      if (MachOLinkingContext::parsePackedVersion(minOS->getValue(),
-                                                  minOSVersion)) {
-        error("malformed macosx_version_min value");
-        return false;
-      }
-      break;
-    case OPT_ios_version_min:
-      os = MachOLinkingContext::OS::iOS;
-      if (MachOLinkingContext::parsePackedVersion(minOS->getValue(),
-                                                  minOSVersion)) {
-        error("malformed ios_version_min value");
-        return false;
-      }
-      break;
-    case OPT_ios_simulator_version_min:
-      os = MachOLinkingContext::OS::iOS_simulator;
-      if (MachOLinkingContext::parsePackedVersion(minOS->getValue(),
-                                                  minOSVersion)) {
-        error("malformed ios_simulator_version_min value");
-        return false;
-      }
-      break;
-    }
-  } else {
-    // No min-os version on command line, check environment variables
-  }
-
-  // Handle export_dynamic
-  // FIXME: Should we warn when this applies to something other than a static
-  // executable or dylib?  Those are the only cases where this has an effect.
-  // Note, this has to come before ctx.configure() so that we get the correct
-  // value for _globalsAreDeadStripRoots.
-  bool exportDynamicSymbols = parsedArgs.hasArg(OPT_export_dynamic);
-
-  // Now that there's enough information parsed in, let the linking context
-  // set up default values.
-  ctx.configure(fileType, arch, os, minOSVersion, exportDynamicSymbols);
-
-  // Handle -e xxx
-  if (llvm::opt::Arg *entry = parsedArgs.getLastArg(OPT_entry))
-    ctx.setEntrySymbolName(entry->getValue());
-
-  // Handle -o xxx
-  if (llvm::opt::Arg *outpath = parsedArgs.getLastArg(OPT_output))
-    ctx.setOutputPath(outpath->getValue());
-  else
-    ctx.setOutputPath("a.out");
-
-  // Handle -image_base XXX and -seg1addr XXXX
-  if (llvm::opt::Arg *imageBase = parsedArgs.getLastArg(OPT_image_base)) {
-    uint64_t baseAddress;
-    if (parseNumberBase16(imageBase->getValue(), baseAddress)) {
-      error("image_base expects a hex number");
-      return false;
-    } else if (baseAddress < ctx.pageZeroSize()) {
-      error("image_base overlaps with __PAGEZERO");
-      return false;
-    } else if (baseAddress % ctx.pageSize()) {
-      error("image_base must be a multiple of page size (0x" +
-            llvm::utohexstr(ctx.pageSize()) + ")");
-      return false;
-    }
-
-    ctx.setBaseAddress(baseAddress);
-  }
-
-  // Handle -dead_strip
-  if (parsedArgs.getLastArg(OPT_dead_strip))
-    ctx.setDeadStripping(true);
-
-  bool globalWholeArchive = false;
-  // Handle -all_load
-  if (parsedArgs.getLastArg(OPT_all_load))
-    globalWholeArchive = true;
-
-  // Handle -install_name
-  if (llvm::opt::Arg *installName = parsedArgs.getLastArg(OPT_install_name))
-    ctx.setInstallName(installName->getValue());
-  else
-    ctx.setInstallName(ctx.outputPath());
-
-  // Handle -mark_dead_strippable_dylib
-  if (parsedArgs.getLastArg(OPT_mark_dead_strippable_dylib))
-    ctx.setDeadStrippableDylib(true);
-
-  // Handle -compatibility_version and -current_version
-  if (llvm::opt::Arg *vers = parsedArgs.getLastArg(OPT_compatibility_version)) {
-    if (ctx.outputMachOType() != llvm::MachO::MH_DYLIB) {
-      error("-compatibility_version can only be used with -dylib");
-      return false;
-    }
-    uint32_t parsedVers;
-    if (MachOLinkingContext::parsePackedVersion(vers->getValue(), parsedVers)) {
-      error("-compatibility_version value is malformed");
-      return false;
-    }
-    ctx.setCompatibilityVersion(parsedVers);
-  }
-
-  if (llvm::opt::Arg *vers = parsedArgs.getLastArg(OPT_current_version)) {
-    if (ctx.outputMachOType() != llvm::MachO::MH_DYLIB) {
-      error("-current_version can only be used with -dylib");
-      return false;
-    }
-    uint32_t parsedVers;
-    if (MachOLinkingContext::parsePackedVersion(vers->getValue(), parsedVers)) {
-      error("-current_version value is malformed");
-      return false;
-    }
-    ctx.setCurrentVersion(parsedVers);
-  }
-
-  // Handle -bundle_loader
-  if (llvm::opt::Arg *loader = parsedArgs.getLastArg(OPT_bundle_loader))
-    ctx.setBundleLoader(loader->getValue());
-
-  // Handle -sectalign segname sectname align
-  for (auto &alignArg : parsedArgs.filtered(OPT_sectalign)) {
-    const char* segName   = alignArg->getValue(0);
-    const char* sectName  = alignArg->getValue(1);
-    const char* alignStr  = alignArg->getValue(2);
-    if ((alignStr[0] == '0') && (alignStr[1] == 'x'))
-      alignStr += 2;
-    unsigned long long alignValue;
-    if (llvm::getAsUnsignedInteger(alignStr, 16, alignValue)) {
-      error("-sectalign alignment value '" + Twine(alignStr) +
-            "' not a valid number");
-      return false;
-    }
-    uint16_t align = 1 << llvm::countTrailingZeros(alignValue);
-    if (!llvm::isPowerOf2_64(alignValue)) {
-      std::string Msg;
-      llvm::raw_string_ostream OS(Msg);
-      OS << "alignment for '-sectalign " << segName << " " << sectName
-         << llvm::format(" 0x%llX", alignValue)
-         << "' is not a power of two, using " << llvm::format("0x%08X", align);
-      OS.flush();
-      warn(Msg);
-    }
-    ctx.addSectionAlignment(segName, sectName, align);
-  }
-
-  // Handle -mllvm
-  for (auto &llvmArg : parsedArgs.filtered(OPT_mllvm)) {
-    ctx.appendLLVMOption(llvmArg->getValue());
-  }
-
-  // Handle -print_atoms
-  if (parsedArgs.getLastArg(OPT_print_atoms))
-    ctx.setPrintAtoms();
-
-  // Handle -t (trace) option.
-  if (parsedArgs.getLastArg(OPT_t))
-    ctx.setLogInputFiles(true);
-
-  // Handle -demangle option.
-  if (parsedArgs.getLastArg(OPT_demangle))
-    ctx.setDemangleSymbols(true);
-
-  // Handle -keep_private_externs
-  if (parsedArgs.getLastArg(OPT_keep_private_externs)) {
-    ctx.setKeepPrivateExterns(true);
-    if (ctx.outputMachOType() != llvm::MachO::MH_OBJECT)
-      warn("-keep_private_externs only used in -r mode");
-  }
-
-  // Handle -dependency_info <path> used by Xcode.
-  if (llvm::opt::Arg *depInfo = parsedArgs.getLastArg(OPT_dependency_info))
-    if (std::error_code ec = ctx.createDependencyFile(depInfo->getValue()))
-      warn(ec.message() + ", processing '-dependency_info " +
-           depInfo->getValue());
-
-  // In -test_file_usage mode, we'll be given an explicit list of paths that
-  // exist. We'll also be expected to print out information about how we located
-  // libraries and so on that the user specified, but not to actually do any
-  // linking.
-  if (parsedArgs.getLastArg(OPT_test_file_usage)) {
-    ctx.setTestingFileUsage();
-
-    // With paths existing by fiat, linking is not going to end well.
-    ctx.setDoNothing(true);
-
-    // Only bother looking for an existence override if we're going to use it.
-    for (auto existingPath : parsedArgs.filtered(OPT_path_exists)) {
-      ctx.addExistingPathForDebug(existingPath->getValue());
-    }
-  }
-
-  // Register possible input file parsers.
-  if (!ctx.doNothing()) {
-    ctx.registry().addSupportMachOObjects(ctx);
-    ctx.registry().addSupportArchives(ctx.logInputFiles());
-    ctx.registry().addSupportYamlFiles();
-  }
-
-  // Now construct the set of library search directories, following ld64's
-  // baroque set of accumulated hacks. Mostly, the algorithm constructs
-  //     { syslibroots } x { libpaths }
-  //
-  // Unfortunately, there are numerous exceptions:
-  //   1. Only absolute paths get modified by syslibroot options.
-  //   2. If there is just 1 -syslibroot, system paths not found in it are
-  //      skipped.
-  //   3. If the last -syslibroot is "/", all of them are ignored entirely.
-  //   4. If { syslibroots } x path ==  {}, the original path is kept.
-  std::vector<StringRef> sysLibRoots;
-  for (auto syslibRoot : parsedArgs.filtered(OPT_syslibroot)) {
-    sysLibRoots.push_back(syslibRoot->getValue());
-  }
-  if (!sysLibRoots.empty()) {
-    // Ignore all if last -syslibroot is "/".
-    if (sysLibRoots.back() != "/")
-      ctx.setSysLibRoots(sysLibRoots);
-  }
-
-  // Paths specified with -L come first, and are not considered system paths for
-  // the case where there is precisely 1 -syslibroot.
-  for (auto libPath : parsedArgs.filtered(OPT_L)) {
-    ctx.addModifiedSearchDir(libPath->getValue());
-  }
-
-  // Process -F directories (where to look for frameworks).
-  for (auto fwPath : parsedArgs.filtered(OPT_F)) {
-    ctx.addFrameworkSearchDir(fwPath->getValue());
-  }
-
-  // -Z suppresses the standard search paths.
-  if (!parsedArgs.hasArg(OPT_Z)) {
-    ctx.addModifiedSearchDir("/usr/lib", true);
-    ctx.addModifiedSearchDir("/usr/local/lib", true);
-    ctx.addFrameworkSearchDir("/Library/Frameworks", true);
-    ctx.addFrameworkSearchDir("/System/Library/Frameworks", true);
-  }
-
-  // Now that we've constructed the final set of search paths, print out those
-  // search paths in verbose mode.
-  if (errorHandler().verbose) {
-    message("Library search paths:");
-    for (auto path : ctx.searchDirs()) {
-      message("    " + path);
-    }
-    message("Framework search paths:");
-    for (auto path : ctx.frameworkDirs()) {
-      message("    " + path);
-    }
-  }
-
-  // Handle -exported_symbols_list <file>
-  for (auto expFile : parsedArgs.filtered(OPT_exported_symbols_list)) {
-    if (ctx.exportMode() == MachOLinkingContext::ExportMode::unexported) {
-      error("-exported_symbols_list cannot be combined with "
-            "-unexported_symbol[s_list]");
-      return false;
-    }
-    ctx.setExportMode(MachOLinkingContext::ExportMode::exported);
-    if (std::error_code ec = parseExportsList(expFile->getValue(), ctx)) {
-      error(ec.message() + ", processing '-exported_symbols_list " +
-            expFile->getValue());
-      return false;
-    }
-  }
-
-  // Handle -exported_symbol <symbol>
-  for (auto symbol : parsedArgs.filtered(OPT_exported_symbol)) {
-    if (ctx.exportMode() == MachOLinkingContext::ExportMode::unexported) {
-      error("-exported_symbol cannot be combined with "
-            "-unexported_symbol[s_list]");
-      return false;
-    }
-    ctx.setExportMode(MachOLinkingContext::ExportMode::exported);
-    ctx.addExportSymbol(symbol->getValue());
-  }
-
-  // Handle -unexported_symbols_list <file>
-  for (auto expFile : parsedArgs.filtered(OPT_unexported_symbols_list)) {
-    if (ctx.exportMode() == MachOLinkingContext::ExportMode::exported) {
-      error("-unexported_symbols_list cannot be combined with "
-            "-exported_symbol[s_list]");
-      return false;
-    }
-    ctx.setExportMode(MachOLinkingContext::ExportMode::unexported);
-    if (std::error_code ec = parseExportsList(expFile->getValue(), ctx)) {
-      error(ec.message() + ", processing '-unexported_symbols_list " +
-            expFile->getValue());
-      return false;
-    }
-  }
-
-  // Handle -unexported_symbol <symbol>
-  for (auto symbol : parsedArgs.filtered(OPT_unexported_symbol)) {
-    if (ctx.exportMode() == MachOLinkingContext::ExportMode::exported) {
-      error("-unexported_symbol cannot be combined with "
-            "-exported_symbol[s_list]");
-      return false;
-    }
-    ctx.setExportMode(MachOLinkingContext::ExportMode::unexported);
-    ctx.addExportSymbol(symbol->getValue());
-  }
-
-  // Handle obosolete -multi_module and -single_module
-  if (llvm::opt::Arg *mod =
-          parsedArgs.getLastArg(OPT_multi_module, OPT_single_module)) {
-    if (mod->getOption().getID() == OPT_multi_module)
-      warn("-multi_module is obsolete and being ignored");
-    else if (ctx.outputMachOType() != llvm::MachO::MH_DYLIB)
-      warn("-single_module being ignored. It is only for use when producing a "
-           "dylib");
-  }
-
-  // Handle obsolete ObjC options: -objc_gc_compaction, -objc_gc, -objc_gc_only
-  if (parsedArgs.getLastArg(OPT_objc_gc_compaction)) {
-    error("-objc_gc_compaction is not supported");
-    return false;
-  }
-
-  if (parsedArgs.getLastArg(OPT_objc_gc)) {
-    error("-objc_gc is not supported");
-    return false;
-  }
-
-  if (parsedArgs.getLastArg(OPT_objc_gc_only)) {
-    error("-objc_gc_only is not supported");
-    return false;
-  }
-
-  // Handle -pie or -no_pie
-  if (llvm::opt::Arg *pie = parsedArgs.getLastArg(OPT_pie, OPT_no_pie)) {
-    switch (ctx.outputMachOType()) {
-    case llvm::MachO::MH_EXECUTE:
-      switch (ctx.os()) {
-      case MachOLinkingContext::OS::macOSX:
-        if ((minOSVersion < 0x000A0500) &&
-            (pie->getOption().getID() == OPT_pie)) {
-          error("-pie can only be used when targeting Mac OS X 10.5 or later");
-          return false;
-        }
-        break;
-      case MachOLinkingContext::OS::iOS:
-        if ((minOSVersion < 0x00040200) &&
-            (pie->getOption().getID() == OPT_pie)) {
-          error("-pie can only be used when targeting iOS 4.2 or later");
-          return false;
-        }
-        break;
-      case MachOLinkingContext::OS::iOS_simulator:
-        if (pie->getOption().getID() == OPT_no_pie) {
-          error("iOS simulator programs must be built PIE");
-          return false;
-        }
-        break;
-      case MachOLinkingContext::OS::unknown:
-        break;
-      }
-      ctx.setPIE(pie->getOption().getID() == OPT_pie);
-      break;
-    case llvm::MachO::MH_PRELOAD:
-      break;
-    case llvm::MachO::MH_DYLIB:
-    case llvm::MachO::MH_BUNDLE:
-      warn(pie->getSpelling() +
-           " being ignored. It is only used when linking main executables");
-      break;
-    default:
-      error(pie->getSpelling() +
-            " can only used when linking main executables");
-      return false;
-    }
-  }
-
-  // Handle -version_load_command or -no_version_load_command
-  {
-    bool flagOn = false;
-    bool flagOff = false;
-    if (auto *arg = parsedArgs.getLastArg(OPT_version_load_command,
-                                          OPT_no_version_load_command)) {
-      flagOn = arg->getOption().getID() == OPT_version_load_command;
-      flagOff = arg->getOption().getID() == OPT_no_version_load_command;
-    }
-
-    // default to adding version load command for dynamic code,
-    // static code must opt-in
-    switch (ctx.outputMachOType()) {
-      case llvm::MachO::MH_OBJECT:
-        ctx.setGenerateVersionLoadCommand(false);
-        break;
-      case llvm::MachO::MH_EXECUTE:
-        // dynamic executables default to generating a version load command,
-        // while static executables only generate it if required.
-        if (isStaticExecutable) {
-          if (flagOn)
-            ctx.setGenerateVersionLoadCommand(true);
-        } else {
-          if (!flagOff)
-            ctx.setGenerateVersionLoadCommand(true);
-        }
-        break;
-      case llvm::MachO::MH_PRELOAD:
-      case llvm::MachO::MH_KEXT_BUNDLE:
-        if (flagOn)
-          ctx.setGenerateVersionLoadCommand(true);
-        break;
-      case llvm::MachO::MH_DYLINKER:
-      case llvm::MachO::MH_DYLIB:
-      case llvm::MachO::MH_BUNDLE:
-        if (!flagOff)
-          ctx.setGenerateVersionLoadCommand(true);
-        break;
-      case llvm::MachO::MH_FVMLIB:
-      case llvm::MachO::MH_DYLDLINK:
-      case llvm::MachO::MH_DYLIB_STUB:
-      case llvm::MachO::MH_DSYM:
-        // We don't generate load commands for these file types, even if
-        // forced on.
-        break;
-    }
-  }
-
-  // Handle -function_starts or -no_function_starts
-  {
-    bool flagOn = false;
-    bool flagOff = false;
-    if (auto *arg = parsedArgs.getLastArg(OPT_function_starts,
-                                          OPT_no_function_starts)) {
-      flagOn = arg->getOption().getID() == OPT_function_starts;
-      flagOff = arg->getOption().getID() == OPT_no_function_starts;
-    }
-
-    // default to adding functions start for dynamic code, static code must
-    // opt-in
-    switch (ctx.outputMachOType()) {
-      case llvm::MachO::MH_OBJECT:
-        ctx.setGenerateFunctionStartsLoadCommand(false);
-        break;
-      case llvm::MachO::MH_EXECUTE:
-        // dynamic executables default to generating a version load command,
-        // while static executables only generate it if required.
-        if (isStaticExecutable) {
-          if (flagOn)
-            ctx.setGenerateFunctionStartsLoadCommand(true);
-        } else {
-          if (!flagOff)
-            ctx.setGenerateFunctionStartsLoadCommand(true);
-        }
-        break;
-      case llvm::MachO::MH_PRELOAD:
-      case llvm::MachO::MH_KEXT_BUNDLE:
-        if (flagOn)
-          ctx.setGenerateFunctionStartsLoadCommand(true);
-        break;
-      case llvm::MachO::MH_DYLINKER:
-      case llvm::MachO::MH_DYLIB:
-      case llvm::MachO::MH_BUNDLE:
-        if (!flagOff)
-          ctx.setGenerateFunctionStartsLoadCommand(true);
-        break;
-      case llvm::MachO::MH_FVMLIB:
-      case llvm::MachO::MH_DYLDLINK:
-      case llvm::MachO::MH_DYLIB_STUB:
-      case llvm::MachO::MH_DSYM:
-        // We don't generate load commands for these file types, even if
-        // forced on.
-        break;
-    }
-  }
-
-  // Handle -data_in_code_info or -no_data_in_code_info
-  {
-    bool flagOn = false;
-    bool flagOff = false;
-    if (auto *arg = parsedArgs.getLastArg(OPT_data_in_code_info,
-                                          OPT_no_data_in_code_info)) {
-      flagOn = arg->getOption().getID() == OPT_data_in_code_info;
-      flagOff = arg->getOption().getID() == OPT_no_data_in_code_info;
-    }
-
-    // default to adding data in code for dynamic code, static code must
-    // opt-in
-    switch (ctx.outputMachOType()) {
-      case llvm::MachO::MH_OBJECT:
-        if (!flagOff)
-          ctx.setGenerateDataInCodeLoadCommand(true);
-        break;
-      case llvm::MachO::MH_EXECUTE:
-        // dynamic executables default to generating a version load command,
-        // while static executables only generate it if required.
-        if (isStaticExecutable) {
-          if (flagOn)
-            ctx.setGenerateDataInCodeLoadCommand(true);
-        } else {
-          if (!flagOff)
-            ctx.setGenerateDataInCodeLoadCommand(true);
-        }
-        break;
-      case llvm::MachO::MH_PRELOAD:
-      case llvm::MachO::MH_KEXT_BUNDLE:
-        if (flagOn)
-          ctx.setGenerateDataInCodeLoadCommand(true);
-        break;
-      case llvm::MachO::MH_DYLINKER:
-      case llvm::MachO::MH_DYLIB:
-      case llvm::MachO::MH_BUNDLE:
-        if (!flagOff)
-          ctx.setGenerateDataInCodeLoadCommand(true);
-        break;
-      case llvm::MachO::MH_FVMLIB:
-      case llvm::MachO::MH_DYLDLINK:
-      case llvm::MachO::MH_DYLIB_STUB:
-      case llvm::MachO::MH_DSYM:
-        // We don't generate load commands for these file types, even if
-        // forced on.
-        break;
-    }
-  }
-
-  // Handle sdk_version
-  if (llvm::opt::Arg *arg = parsedArgs.getLastArg(OPT_sdk_version)) {
-    uint32_t sdkVersion = 0;
-    if (MachOLinkingContext::parsePackedVersion(arg->getValue(),
-                                                sdkVersion)) {
-      error("malformed sdkVersion value");
-      return false;
-    }
-    ctx.setSdkVersion(sdkVersion);
-  } else if (ctx.generateVersionLoadCommand()) {
-    // If we don't have an sdk version, but were going to emit a load command
-    // with min_version, then we need to give a warning as we have no sdk
-    // version to put in that command.
-    // FIXME: We need to decide whether to make this an error.
-    warn("-sdk_version is required when emitting min version load command.  "
-         "Setting sdk version to match provided min version");
-    ctx.setSdkVersion(ctx.osMinVersion());
-  }
-
-  // Handle source_version
-  if (llvm::opt::Arg *arg = parsedArgs.getLastArg(OPT_source_version)) {
-    uint64_t version = 0;
-    if (MachOLinkingContext::parsePackedVersion(arg->getValue(),
-                                                version)) {
-      error("malformed source_version value");
-      return false;
-    }
-    ctx.setSourceVersion(version);
-  }
-
-  // Handle stack_size
-  if (llvm::opt::Arg *stackSize = parsedArgs.getLastArg(OPT_stack_size)) {
-    uint64_t stackSizeVal;
-    if (parseNumberBase16(stackSize->getValue(), stackSizeVal)) {
-      error("stack_size expects a hex number");
-      return false;
-    }
-    if ((stackSizeVal % ctx.pageSize()) != 0) {
-      error("stack_size must be a multiple of page size (0x" +
-            llvm::utohexstr(ctx.pageSize()) + ")");
-      return false;
-    }
-
-    ctx.setStackSize(stackSizeVal);
-  }
-
-  // Handle debug info handling options: -S
-  if (parsedArgs.hasArg(OPT_S))
-    ctx.setDebugInfoMode(MachOLinkingContext::DebugInfoMode::noDebugMap);
-
-  // Handle -order_file <file>
-  for (auto orderFile : parsedArgs.filtered(OPT_order_file)) {
-    if (std::error_code ec = parseOrderFile(orderFile->getValue(), ctx)) {
-      error(ec.message() + ", processing '-order_file " + orderFile->getValue()
-            + "'");
-      return false;
-    }
-  }
-
-  // Handle -flat_namespace.
-  if (llvm::opt::Arg *ns =
-          parsedArgs.getLastArg(OPT_flat_namespace, OPT_twolevel_namespace)) {
-    if (ns->getOption().getID() == OPT_flat_namespace)
-      ctx.setUseFlatNamespace(true);
-  }
-
-  // Handle -undefined
-  if (llvm::opt::Arg *undef = parsedArgs.getLastArg(OPT_undefined)) {
-    MachOLinkingContext::UndefinedMode UndefMode;
-    if (StringRef(undef->getValue()).equals("error"))
-      UndefMode = MachOLinkingContext::UndefinedMode::error;
-    else if (StringRef(undef->getValue()).equals("warning"))
-      UndefMode = MachOLinkingContext::UndefinedMode::warning;
-    else if (StringRef(undef->getValue()).equals("suppress"))
-      UndefMode = MachOLinkingContext::UndefinedMode::suppress;
-    else if (StringRef(undef->getValue()).equals("dynamic_lookup"))
-      UndefMode = MachOLinkingContext::UndefinedMode::dynamicLookup;
-    else {
-      error("invalid option to -undefined [ warning | error | suppress | "
-            "dynamic_lookup ]");
-      return false;
-    }
-
-    if (ctx.useFlatNamespace()) {
-      // If we're using -flat_namespace then 'warning', 'suppress' and
-      // 'dynamic_lookup' are all equivalent, so map them to 'suppress'.
-      if (UndefMode != MachOLinkingContext::UndefinedMode::error)
-        UndefMode = MachOLinkingContext::UndefinedMode::suppress;
-    } else {
-      // If we're using -twolevel_namespace then 'warning' and 'suppress' are
-      // illegal. Emit a diagnostic if they've been (mis)used.
-      if (UndefMode == MachOLinkingContext::UndefinedMode::warning ||
-          UndefMode == MachOLinkingContext::UndefinedMode::suppress) {
-        error("can't use -undefined warning or suppress with "
-              "-twolevel_namespace");
-        return false;
-      }
-    }
-
-    ctx.setUndefinedMode(UndefMode);
-  }
-
-  // Handle -no_objc_category_merging.
-  if (parsedArgs.getLastArg(OPT_no_objc_category_merging))
-    ctx.setMergeObjCCategories(false);
-
-  // Handle -rpath <path>
-  if (parsedArgs.hasArg(OPT_rpath)) {
-    switch (ctx.outputMachOType()) {
-      case llvm::MachO::MH_EXECUTE:
-      case llvm::MachO::MH_DYLIB:
-      case llvm::MachO::MH_BUNDLE:
-        if (!ctx.minOS("10.5", "2.0")) {
-          if (ctx.os() == MachOLinkingContext::OS::macOSX)
-            error("-rpath can only be used when targeting OS X 10.5 or later");
-          else
-            error("-rpath can only be used when targeting iOS 2.0 or later");
-          return false;
-        }
-        break;
-      default:
-        error("-rpath can only be used when creating a dynamic final linked "
-              "image");
-        return false;
-    }
-
-    for (auto rPath : parsedArgs.filtered(OPT_rpath)) {
-      ctx.addRpath(rPath->getValue());
-    }
-  }
-
-  // Parse the LLVM options before we process files in case the file handling
-  // makes use of things like LLVM_DEBUG().
-  parseLLVMOptions(ctx);
-
-  // Handle input files and sectcreate.
-  for (auto &arg : parsedArgs) {
-    bool upward;
-    llvm::Optional<StringRef> resolvedPath;
-    switch (arg->getOption().getID()) {
-    default:
-      continue;
-    case OPT_INPUT:
-      addFile(arg->getValue(), ctx, globalWholeArchive, false);
-      break;
-    case OPT_upward_library:
-      addFile(arg->getValue(), ctx, false, true);
-      break;
-    case OPT_force_load:
-      addFile(arg->getValue(), ctx, true, false);
-      break;
-    case OPT_l:
-    case OPT_upward_l:
-      upward = (arg->getOption().getID() == OPT_upward_l);
-      resolvedPath = ctx.searchLibrary(arg->getValue());
-      if (!resolvedPath) {
-        error("Unable to find library for " + arg->getSpelling() +
-              arg->getValue());
-        return false;
-      } else if (ctx.testingFileUsage()) {
-        message(Twine("Found ") + (upward ? "upward " : " ") + "library " +
-                canonicalizePath(resolvedPath.getValue()));
-      }
-      addFile(resolvedPath.getValue(), ctx, globalWholeArchive, upward);
-      break;
-    case OPT_framework:
-    case OPT_upward_framework:
-      upward = (arg->getOption().getID() == OPT_upward_framework);
-      resolvedPath = ctx.findPathForFramework(arg->getValue());
-      if (!resolvedPath) {
-        error("Unable to find framework for " + arg->getSpelling() + " " +
-              arg->getValue());
-        return false;
-      } else if (ctx.testingFileUsage()) {
-        message(Twine("Found ") + (upward ? "upward " : " ") + "framework " +
-                canonicalizePath(resolvedPath.getValue()));
-      }
-      addFile(resolvedPath.getValue(), ctx, globalWholeArchive, upward);
-      break;
-    case OPT_filelist:
-      if (auto ec = loadFileList(arg->getValue(), ctx, globalWholeArchive)) {
-        handleAllErrors(std::move(ec), [&](const llvm::ErrorInfoBase &EI) {
-          error(EI.message() + ", processing '-filelist " + arg->getValue());
-        });
-        return false;
-      }
-      break;
-    case OPT_sectcreate: {
-        const char* seg  = arg->getValue(0);
-        const char* sect = arg->getValue(1);
-        const char* fileName = arg->getValue(2);
-
-        ErrorOr<std::unique_ptr<MemoryBuffer>> contentOrErr =
-          MemoryBuffer::getFile(fileName);
-
-        if (!contentOrErr) {
-          error("can't open -sectcreate file " + Twine(fileName));
-          return false;
-        }
-
-        ctx.addSectCreateSection(seg, sect, std::move(*contentOrErr));
-      }
-      break;
-    }
-  }
-
-  if (ctx.getNodes().empty()) {
-    error("No input files");
-    return false;
-  }
-
-  // Validate the combination of options used.
-  return ctx.validate();
-}
-
-static void createFiles(MachOLinkingContext &ctx, bool Implicit) {
-  std::vector<std::unique_ptr<File>> Files;
-  if (Implicit)
-    ctx.createImplicitFiles(Files);
-  else
-    ctx.createInternalFiles(Files);
-  for (auto i = Files.rbegin(), e = Files.rend(); i != e; ++i) {
-    auto &members = ctx.getNodes();
-    members.insert(members.begin(), std::make_unique<FileNode>(std::move(*i)));
-  }
-}
-
-/// This is where the link is actually performed.
-bool link(llvm::ArrayRef<const char *> args, bool CanExitEarly,
-          raw_ostream &StdoutOS, raw_ostream &StderrOS) {
-  lld::stdoutOS = &StdoutOS;
-  lld::stderrOS = &StderrOS;
-
-  errorHandler().logName = args::getFilenameWithoutExe(args[0]);
-  errorHandler().errorLimitExceededMsg =
-      "too many errors emitted, stopping now (use "
-      "'-error-limit 0' to see all errors)";
-  errorHandler().exitEarly = CanExitEarly;
-  StderrOS.enable_colors(StderrOS.has_colors());
-
-  MachOLinkingContext ctx;
-  if (!parse(args, ctx))
-    return false;
-  if (ctx.doNothing())
-    return true;
-  if (ctx.getNodes().empty())
-    return false;
-
-  for (std::unique_ptr<Node> &ie : ctx.getNodes())
-    if (FileNode *node = dyn_cast<FileNode>(ie.get()))
-      node->getFile()->parse();
-
-  createFiles(ctx, false /* Implicit */);
-
-  // Give target a chance to add files
-  createFiles(ctx, true /* Implicit */);
-
-  // Give target a chance to postprocess input files.
-  // Mach-O uses this chance to move all object files before library files.
-  ctx.finalizeInputFiles();
-
-  // Do core linking.
-  ScopedTask resolveTask(getDefaultDomain(), "Resolve");
-  Resolver resolver(ctx);
-  if (!resolver.resolve())
-    return false;
-  SimpleFile *merged = nullptr;
-  {
-    std::unique_ptr<SimpleFile> mergedFile = resolver.resultFile();
-    merged = mergedFile.get();
-    auto &members = ctx.getNodes();
-    members.insert(members.begin(),
-                   std::make_unique<FileNode>(std::move(mergedFile)));
-  }
-  resolveTask.end();
-
-  // Run passes on linked atoms.
-  ScopedTask passTask(getDefaultDomain(), "Passes");
-  PassManager pm;
-  ctx.addPasses(pm);
-  if (auto ec = pm.runOnFile(*merged)) {
-    // FIXME: This should be passed to logAllUnhandledErrors but it needs
-    // to be passed a Twine instead of a string.
-    lld::errs() << "Failed to run passes on file '" << ctx.outputPath()
-                << "': ";
-    logAllUnhandledErrors(std::move(ec), lld::errs(), std::string());
-    return false;
-  }
-
-  passTask.end();
-
-  // Give linked atoms to Writer to generate output file.
-  ScopedTask writeTask(getDefaultDomain(), "Write");
-  if (auto ec = ctx.writeFile(*merged)) {
-    // FIXME: This should be passed to logAllUnhandledErrors but it needs
-    // to be passed a Twine instead of a string.
-    lld::errs() << "Failed to write file '" << ctx.outputPath() << "': ";
-    logAllUnhandledErrors(std::move(ec), lld::errs(), std::string());
-    return false;
-  }
-
-  // Call exit() if we can to avoid calling destructors.
-  if (CanExitEarly)
-    exitLld(errorCount() ? 1 : 0);
-
-
-  return true;
-}
-
-} // end namespace mach_o
-} // end namespace lld
diff --git a/lld/lib/Driver/DarwinLdOptions.td b/lld/lib/Driver/DarwinLdOptions.td
deleted file mode 100644
index 3bbde8bf1c1c..000000000000
--- a/lld/lib/Driver/DarwinLdOptions.td
+++ /dev/null
@@ -1,250 +0,0 @@
-include "llvm/Option/OptParser.td"
-
-
-// output kinds
-def grp_kind : OptionGroup<"outs">, HelpText<"OUTPUT KIND">;
-def relocatable : Flag<["-"], "r">,
-     HelpText<"Create relocatable object file">, Group<grp_kind>;
-def static : Flag<["-"], "static">,
-     HelpText<"Create static executable">, Group<grp_kind>;
-def dynamic : Flag<["-"], "dynamic">,
-     HelpText<"Create dynamic executable (default)">,Group<grp_kind>;
-def dylib : Flag<["-"], "dylib">,
-     HelpText<"Create dynamic library">, Group<grp_kind>;
-def bundle : Flag<["-"], "bundle">,
-     HelpText<"Create dynamic bundle">, Group<grp_kind>;
-def execute : Flag<["-"], "execute">,
-     HelpText<"Create main executable (default)">, Group<grp_kind>;
-def preload : Flag<["-"], "preload">,
-     HelpText<"Create binary for use with embedded systems">, Group<grp_kind>;
-
-// optimizations
-def grp_opts : OptionGroup<"opts">, HelpText<"OPTIMIZATIONS">;
-def dead_strip : Flag<["-"], "dead_strip">,
-     HelpText<"Remove unreference code and data">, Group<grp_opts>;
-def macosx_version_min : Separate<["-"], "macosx_version_min">,
-     MetaVarName<"<version>">,
-     HelpText<"Minimum Mac OS X version">, Group<grp_opts>;
-def ios_version_min : Separate<["-"], "ios_version_min">,
-     MetaVarName<"<version>">,
-     HelpText<"Minimum iOS version">, Group<grp_opts>;
-def iphoneos_version_min : Separate<["-"], "iphoneos_version_min">,
-     Alias<ios_version_min>;
-def ios_simulator_version_min : Separate<["-"], "ios_simulator_version_min">,
-     MetaVarName<"<version>">,
-     HelpText<"Minimum iOS simulator version">, Group<grp_opts>;
-def sdk_version : Separate<["-"], "sdk_version">,
-     MetaVarName<"<version>">,
-     HelpText<"SDK version">, Group<grp_opts>;
-def source_version : Separate<["-"], "source_version">,
-     MetaVarName<"<version>">,
-     HelpText<"Source version">, Group<grp_opts>;
-def version_load_command : Flag<["-"], "version_load_command">,
-     HelpText<"Force generation of a version load command">, Group<grp_opts>;
-def no_version_load_command : Flag<["-"], "no_version_load_command">,
-     HelpText<"Disable generation of a version load command">, Group<grp_opts>;
-def function_starts : Flag<["-"], "function_starts">,
-     HelpText<"Force generation of a function starts load command">,
-     Group<grp_opts>;
-def no_function_starts : Flag<["-"], "no_function_starts">,
-     HelpText<"Disable generation of a function starts load command">,
-     Group<grp_opts>;
-def data_in_code_info : Flag<["-"], "data_in_code_info">,
-     HelpText<"Force generation of a data in code load command">,
-     Group<grp_opts>;
-def no_data_in_code_info : Flag<["-"], "no_data_in_code_info">,
-     HelpText<"Disable generation of a data in code load command">,
-     Group<grp_opts>;
-def mllvm : Separate<["-"], "mllvm">,
-     MetaVarName<"<option>">,
-     HelpText<"Options to pass to LLVM during LTO">, Group<grp_opts>;
-def exported_symbols_list : Separate<["-"], "exported_symbols_list">,
-     MetaVarName<"<file-path>">,
-     HelpText<"Restricts which symbols will be exported">, Group<grp_opts>;
-def exported_symbol : Separate<["-"], "exported_symbol">,
-     MetaVarName<"<symbol>">,
-     HelpText<"Restricts which symbols will be exported">, Group<grp_opts>;
-def unexported_symbols_list : Separate<["-"], "unexported_symbols_list">,
-     MetaVarName<"<file-path>">,
-     HelpText<"Lists symbols that should not be exported">, Group<grp_opts>;
-def unexported_symbol : Separate<["-"], "unexported_symbol">,
-     MetaVarName<"<symbol>">,
-     HelpText<"A symbol which should not be exported">, Group<grp_opts>;
-def keep_private_externs : Flag<["-"], "keep_private_externs">,
-     HelpText<"Private extern (hidden) symbols should not be transformed "
-              "into local symbols">, Group<grp_opts>;
-def order_file : Separate<["-"], "order_file">,
-     MetaVarName<"<file-path>">,
-     HelpText<"re-order and move specified symbols to start of their section">,
-     Group<grp_opts>;
-def flat_namespace : Flag<["-"], "flat_namespace">,
-     HelpText<"Resolves symbols in any (transitively) linked dynamic libraries. "
-              "Source libraries are not recorded: dyld will re-search all "
-              "images at runtime and use the first definition found.">,
-     Group<grp_opts>;
-def twolevel_namespace : Flag<["-"], "twolevel_namespace">,
-     HelpText<"Resolves symbols in listed libraries only. Source libraries are "
-              "recorded in the symbol table.">,
-     Group<grp_opts>;
-def undefined : Separate<["-"], "undefined">,
-                MetaVarName<"<undefined>">,
-                HelpText<"Determines how undefined symbols are handled.">,
-                Group<grp_opts>;
-def no_objc_category_merging : Flag<["-"], "no_objc_category_merging">,
-     HelpText<"Disables the optimisation which merges Objective-C categories "
-              "on a class in to the class itself.">,
-     Group<grp_opts>;
-
-// main executable options
-def grp_main : OptionGroup<"opts">, HelpText<"MAIN EXECUTABLE OPTIONS">;
-def entry : Separate<["-"], "e">,
-     MetaVarName<"<entry-name>">,
-     HelpText<"entry symbol name">,Group<grp_main>;
-def pie : Flag<["-"], "pie">,
-     HelpText<"Create Position Independent Executable (for ASLR)">,
-     Group<grp_main>;
-def no_pie : Flag<["-"], "no_pie">,
-     HelpText<"Do not create Position Independent Executable">,
-     Group<grp_main>;
-def stack_size : Separate<["-"], "stack_size">,
-     HelpText<"Specifies the maximum stack size for the main thread in a program. "
-              "Must be a page-size multiple. (default=8Mb)">,
-     Group<grp_main>;
-def export_dynamic : Flag<["-"], "export_dynamic">,
-     HelpText<"Preserves all global symbols in main executables during LTO">,
-     Group<grp_main>;
-
-// dylib executable options
-def grp_dylib : OptionGroup<"opts">, HelpText<"DYLIB EXECUTABLE OPTIONS">;
-def install_name : Separate<["-"], "install_name">,
-     MetaVarName<"<path>">,
-     HelpText<"The dylib's install name">, Group<grp_dylib>;
-def mark_dead_strippable_dylib : Flag<["-"], "mark_dead_strippable_dylib">,
-     HelpText<"Marks the dylib as having no side effects during initialization">,
-     Group<grp_dylib>;
-def compatibility_version : Separate<["-"], "compatibility_version">,
-     MetaVarName<"<version>">,
-     HelpText<"The dylib's compatibility version">, Group<grp_dylib>;
-def current_version : Separate<["-"], "current_version">,
-     MetaVarName<"<version>">,
-     HelpText<"The dylib's current version">, Group<grp_dylib>;
-
-// dylib executable options - compatibility aliases
-def dylib_install_name : Separate<["-"], "dylib_install_name">,
-     Alias<install_name>;
-def dylib_compatibility_version : Separate<["-"], "dylib_compatibility_version">,
-     MetaVarName<"<version>">, Alias<compatibility_version>;
-def dylib_current_version : Separate<["-"], "dylib_current_version">,
-     MetaVarName<"<version>">, Alias<current_version>;
-
-// bundle executable options
-def grp_bundle : OptionGroup<"opts">, HelpText<"BUNDLE EXECUTABLE OPTIONS">;
-def bundle_loader : Separate<["-"], "bundle_loader">,
-     MetaVarName<"<path>">,
-     HelpText<"The executable that will be loading this Mach-O bundle">,
-     Group<grp_bundle>;
-
-// library options
-def grp_libs : OptionGroup<"libs">, HelpText<"LIBRARY OPTIONS">;
-def L : JoinedOrSeparate<["-"], "L">,
-     MetaVarName<"<dir>">,
-     HelpText<"Add directory to library search path">, Group<grp_libs>;
-def F : JoinedOrSeparate<["-"], "F">,
-     MetaVarName<"<dir>">,
-     HelpText<"Add directory to framework search path">, Group<grp_libs>;
-def Z : Flag<["-"], "Z">,
-     HelpText<"Do not search standard directories for libraries or frameworks">;
-def all_load : Flag<["-"], "all_load">,
-     HelpText<"Forces all members of all static libraries to be loaded">,
-     Group<grp_libs>;
-def force_load : Separate<["-"], "force_load">,
-     MetaVarName<"<library-path>">,
-     HelpText<"Forces all members of specified static libraries to be loaded">,
-     Group<grp_libs>;
-def syslibroot : Separate<["-"], "syslibroot">, MetaVarName<"<dir>">,
-     HelpText<"Add path to SDK to all absolute library search paths">,
-     Group<grp_libs>;
-
-// Input options
-def l : Joined<["-"], "l">,
-     MetaVarName<"<libname>">,
-     HelpText<"Base name of library searched for in -L directories">;
-def upward_l : Joined<["-"], "upward-l">,
-     MetaVarName<"<libname>">,
-     HelpText<"Base name of upward library searched for in -L directories">;
-def framework : Separate<["-"], "framework">,
-     MetaVarName<"<name>">,
-     HelpText<"Base name of framework searched for in -F directories">;
-def upward_framework : Separate<["-"], "upward_framework">,
-     MetaVarName<"<name>">,
-     HelpText<"Base name of upward framework searched for in -F directories">;
-def upward_library : Separate<["-"], "upward_library">,
-     MetaVarName<"<path>">,
-     HelpText<"path to upward dylib to link with">;
-def filelist : Separate<["-"], "filelist">,
-     MetaVarName<"<path>">,
-     HelpText<"file containing paths to input files">;
-
-
-// test case options
-def print_atoms : Flag<["-"], "print_atoms">,
-     HelpText<"Emit output as yaml atoms">;
-def test_file_usage : Flag<["-"], "test_file_usage">,
-     HelpText<"Only files specified by -file_exists are considered to exist. "
-              "Print which files would be used">;
-def path_exists : Separate<["-"], "path_exists">,
-     MetaVarName<"<path>">,
-     HelpText<"Used with -test_file_usage to declare a path">;
-
-
-// general options
-def output : Separate<["-"], "o">,
-     MetaVarName<"<path>">,
-     HelpText<"Output file path">;
-def arch : Separate<["-"], "arch">,
-     MetaVarName<"<arch-name>">,
-     HelpText<"Architecture to link">;
-def sectalign : MultiArg<["-"], "sectalign", 3>,
-     MetaVarName<"<segname> <sectname> <alignment>">,
-     HelpText<"Alignment for segment/section">;
-def sectcreate : MultiArg<["-"], "sectcreate", 3>,
-     MetaVarName<"<segname> <sectname> <file>">,
-     HelpText<"Create section <segname>/<sectname> from contents of <file>">;
-def image_base : Separate<["-"], "image_base">;
-def seg1addr : Separate<["-"], "seg1addr">, Alias<image_base>;
-def demangle : Flag<["-"], "demangle">,
-     HelpText<"Demangles symbol names in errors and warnings">;
-def dependency_info : Separate<["-"], "dependency_info">,
-     MetaVarName<"<file>">,
-     HelpText<"Write binary list of files used during link">;
-def S : Flag<["-"], "S">,
-     HelpText<"Remove debug information (STABS or DWARF) from the output file">;
-def rpath : Separate<["-"], "rpath">,
-     MetaVarName<"<path>">,
-     HelpText<"Add path to the runpath search path list for image being created">;
-
-def t : Flag<["-"], "t">,
-     HelpText<"Print the names of the input files as ld processes them">;
-def v : Flag<["-"], "v">,
-     HelpText<"Print linker information">;
-def error_limit : Separate<["-", "--"], "error-limit">,
-     MetaVarName<"<number>">,
-     HelpText<"Maximum number of errors to emit before stopping (0 = no limit)">;
-
-// Ignored options
-def lto_library : Separate<["-"], "lto_library">,
-    MetaVarName<"<path>">,
-    HelpText<"Ignored for compatibility with other linkers">;
-
-// Obsolete options
-def grp_obsolete : OptionGroup<"obsolete">, HelpText<"OBSOLETE OPTIONS">;
-def single_module : Flag<["-"], "single_module">,
-     HelpText<"Default for dylibs">, Group<grp_obsolete>;
-def multi_module : Flag<["-"], "multi_module">,
-     HelpText<"Unsupported way to build dylibs">, Group<grp_obsolete>;
-def objc_gc_compaction : Flag<["-"], "objc_gc_compaction">,
-     HelpText<"Unsupported ObjC GC option">, Group<grp_obsolete>;
-def objc_gc : Flag<["-"], "objc_gc">,
-     HelpText<"Unsupported ObjC GC option">, Group<grp_obsolete>;
-def objc_gc_only : Flag<["-"], "objc_gc_only">,
-     HelpText<"Unsupported ObjC GC option">, Group<grp_obsolete>;
diff --git a/lld/lib/ReaderWriter/FileArchive.cpp b/lld/lib/ReaderWriter/FileArchive.cpp
deleted file mode 100644
index 98f4d06ee210..000000000000
--- a/lld/lib/ReaderWriter/FileArchive.cpp
+++ /dev/null
@@ -1,227 +0,0 @@
-//===- lib/ReaderWriter/FileArchive.cpp -----------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "lld/Common/LLVM.h"
-#include "lld/Core/ArchiveLibraryFile.h"
-#include "lld/Core/File.h"
-#include "lld/Core/Reader.h"
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/BinaryFormat/Magic.h"
-#include "llvm/Object/Archive.h"
-#include "llvm/Object/Error.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/ErrorOr.h"
-#include "llvm/Support/FileSystem.h"
-#include "llvm/Support/Format.h"
-#include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/raw_ostream.h"
-#include <memory>
-#include <set>
-#include <string>
-#include <system_error>
-#include <unordered_map>
-#include <utility>
-#include <vector>
-
-using llvm::object::Archive;
-using llvm::file_magic;
-using llvm::identify_magic;
-
-namespace lld {
-
-namespace {
-
-/// The FileArchive class represents an Archive Library file
-class FileArchive : public lld::ArchiveLibraryFile {
-public:
-  FileArchive(std::unique_ptr<MemoryBuffer> mb, const Registry &reg,
-              StringRef path, bool logLoading)
-      : ArchiveLibraryFile(path), _mb(std::shared_ptr<MemoryBuffer>(mb.release())),
-        _registry(reg), _logLoading(logLoading) {}
-
-  /// Check if any member of the archive contains an Atom with the
-  /// specified name and return the File object for that member, or nullptr.
-  File *find(StringRef name) override {
-    auto member = _symbolMemberMap.find(name);
-    if (member == _symbolMemberMap.end())
-      return nullptr;
-    Archive::Child c = member->second;
-
-    // Don't return a member already returned
-    Expected<StringRef> buf = c.getBuffer();
-    if (!buf) {
-      // TODO: Actually report errors helpfully.
-      consumeError(buf.takeError());
-      return nullptr;
-    }
-    const char *memberStart = buf->data();
-    if (_membersInstantiated.count(memberStart))
-      return nullptr;
-    _membersInstantiated.insert(memberStart);
-
-    std::unique_ptr<File> result;
-    if (instantiateMember(c, result))
-      return nullptr;
-
-    File *file = result.get();
-    _filesReturned.push_back(std::move(result));
-
-    // Give up the file pointer. It was stored and will be destroyed with destruction of FileArchive
-    return file;
-  }
-
-  /// parse each member
-  std::error_code
-  parseAllMembers(std::vector<std::unique_ptr<File>> &result) override {
-    if (std::error_code ec = parse())
-      return ec;
-    llvm::Error err = llvm::Error::success();
-    for (auto mf = _archive->child_begin(err), me = _archive->child_end();
-         mf != me; ++mf) {
-      std::unique_ptr<File> file;
-      if (std::error_code ec = instantiateMember(*mf, file)) {
-        // err is Success (or we wouldn't be in the loop body) but we can't
-        // return without testing or consuming it.
-        consumeError(std::move(err));
-        return ec;
-      }
-      result.push_back(std::move(file));
-    }
-    if (err)
-      return errorToErrorCode(std::move(err));
-    return std::error_code();
-  }
-
-  const AtomRange<DefinedAtom> defined() const override {
-    return _noDefinedAtoms;
-  }
-
-  const AtomRange<UndefinedAtom> undefined() const override {
-    return _noUndefinedAtoms;
-  }
-
-  const AtomRange<SharedLibraryAtom> sharedLibrary() const override {
-    return _noSharedLibraryAtoms;
-  }
-
-  const AtomRange<AbsoluteAtom> absolute() const override {
-    return _noAbsoluteAtoms;
-  }
-
-  void clearAtoms() override {
-    _noDefinedAtoms.clear();
-    _noUndefinedAtoms.clear();
-    _noSharedLibraryAtoms.clear();
-    _noAbsoluteAtoms.clear();
-  }
-
-protected:
-  std::error_code doParse() override {
-    // Make Archive object which will be owned by FileArchive object.
-    llvm::Error Err = llvm::Error::success();
-    _archive.reset(new Archive(_mb->getMemBufferRef(), Err));
-    if (Err)
-      return errorToErrorCode(std::move(Err));
-    std::error_code ec;
-    if ((ec = buildTableOfContents()))
-      return ec;
-    return std::error_code();
-  }
-
-private:
-  std::error_code instantiateMember(Archive::Child member,
-                                    std::unique_ptr<File> &result) const {
-    Expected<llvm::MemoryBufferRef> mbOrErr = member.getMemoryBufferRef();
-    if (!mbOrErr)
-      return errorToErrorCode(mbOrErr.takeError());
-    llvm::MemoryBufferRef mb = mbOrErr.get();
-    std::string memberPath = (_archive->getFileName() + "("
-                           + mb.getBufferIdentifier() + ")").str();
-
-    if (_logLoading)
-      llvm::errs() << memberPath << "\n";
-
-    std::unique_ptr<MemoryBuffer> memberMB(MemoryBuffer::getMemBuffer(
-        mb.getBuffer(), mb.getBufferIdentifier(), false));
-
-    ErrorOr<std::unique_ptr<File>> fileOrErr =
-        _registry.loadFile(std::move(memberMB));
-    if (std::error_code ec = fileOrErr.getError())
-      return ec;
-    result = std::move(fileOrErr.get());
-    if (std::error_code ec = result->parse())
-      return ec;
-    result->setArchivePath(_archive->getFileName());
-
-    // The memory buffer is co-owned by the archive file and the children,
-    // so that the bufffer is deallocated when all the members are destructed.
-    result->setSharedMemoryBuffer(_mb);
-    return std::error_code();
-  }
-
-  std::error_code buildTableOfContents() {
-    DEBUG_WITH_TYPE("FileArchive", llvm::dbgs()
-                                       << "Table of contents for archive '"
-                                       << _archive->getFileName() << "':\n");
-    for (const Archive::Symbol &sym : _archive->symbols()) {
-      StringRef name = sym.getName();
-      Expected<Archive::Child> memberOrErr = sym.getMember();
-      if (!memberOrErr)
-        return errorToErrorCode(memberOrErr.takeError());
-      Archive::Child member = memberOrErr.get();
-      DEBUG_WITH_TYPE("FileArchive",
-                      llvm::dbgs()
-                          << llvm::format("0x%08llX ",
-                                          member.getBuffer()->data())
-                          << "'" << name << "'\n");
-      _symbolMemberMap.insert(std::make_pair(name, member));
-    }
-    return std::error_code();
-  }
-
-  typedef std::unordered_map<StringRef, Archive::Child> MemberMap;
-  typedef std::set<const char *> InstantiatedSet;
-
-  std::shared_ptr<MemoryBuffer> _mb;
-  const Registry &_registry;
-  std::unique_ptr<Archive> _archive;
-  MemberMap _symbolMemberMap;
-  InstantiatedSet _membersInstantiated;
-  bool _logLoading;
-  std::vector<std::unique_ptr<MemoryBuffer>> _memberBuffers;
-  std::vector<std::unique_ptr<File>> _filesReturned;
-};
-
-class ArchiveReader : public Reader {
-public:
-  ArchiveReader(bool logLoading) : _logLoading(logLoading) {}
-
-  bool canParse(file_magic magic, MemoryBufferRef) const override {
-    return magic == file_magic::archive;
-  }
-
-  ErrorOr<std::unique_ptr<File>> loadFile(std::unique_ptr<MemoryBuffer> mb,
-                                          const Registry &reg) const override {
-    StringRef path = mb->getBufferIdentifier();
-    std::unique_ptr<File> ret =
-        std::make_unique<FileArchive>(std::move(mb), reg, path, _logLoading);
-    return std::move(ret);
-  }
-
-private:
-  bool _logLoading;
-};
-
-} // anonymous namespace
-
-void Registry::addSupportArchives(bool logLoading) {
-  add(std::unique_ptr<Reader>(new ArchiveReader(logLoading)));
-}
-
-} // namespace lld
diff --git a/lld/lib/ReaderWriter/MachO/ArchHandler.cpp b/lld/lib/ReaderWriter/MachO/ArchHandler.cpp
deleted file mode 100644
index c101f3b157bb..000000000000
--- a/lld/lib/ReaderWriter/MachO/ArchHandler.cpp
+++ /dev/null
@@ -1,171 +0,0 @@
-//===- lib/FileFormat/MachO/ArchHandler.cpp -------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-
-#include "ArchHandler.h"
-#include "Atoms.h"
-#include "MachONormalizedFileBinaryUtils.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/ADT/StringSwitch.h"
-#include "llvm/ADT/Triple.h"
-#include "llvm/Support/ErrorHandling.h"
-
-using namespace llvm::MachO;
-using namespace lld::mach_o::normalized;
-
-namespace lld {
-namespace mach_o {
-
-
-ArchHandler::ArchHandler() {
-}
-
-ArchHandler::~ArchHandler() {
-}
-
-std::unique_ptr<mach_o::ArchHandler> ArchHandler::create(
-                                               MachOLinkingContext::Arch arch) {
-  switch (arch) {
-  case MachOLinkingContext::arch_x86_64:
-    return create_x86_64();
-  case MachOLinkingContext::arch_x86:
-    return create_x86();
-  case MachOLinkingContext::arch_armv6:
-  case MachOLinkingContext::arch_armv7:
-  case MachOLinkingContext::arch_armv7s:
-    return create_arm();
-  case MachOLinkingContext::arch_arm64:
-    return create_arm64();
-  default:
-    llvm_unreachable("Unknown arch");
-  }
-}
-
-
-bool ArchHandler::isLazyPointer(const Reference &ref) {
-  // A lazy bind entry is needed for a lazy pointer.
-  const StubInfo &info = stubInfo();
-  if (ref.kindNamespace() != Reference::KindNamespace::mach_o)
-    return false;
-  if (ref.kindArch() != info.lazyPointerReferenceToFinal.arch)
-    return false;
-  return (ref.kindValue() == info.lazyPointerReferenceToFinal.kind);
-}
-
-
-ArchHandler::RelocPattern ArchHandler::relocPattern(const Relocation &reloc) {
-  assert((reloc.type & 0xFFF0) == 0);
-  uint16_t result = reloc.type;
-  if (reloc.scattered)
-    result |= rScattered;
-  if (reloc.pcRel)
-    result |= rPcRel;
-  if (reloc.isExtern)
-    result |= rExtern;
-  switch(reloc.length) {
-  case 0:
-    break;
-  case 1:
-    result |= rLength2;
-    break;
-  case 2:
-    result |= rLength4;
-    break;
-  case 3:
-    result |= rLength8;
-    break;
-  default:
-    llvm_unreachable("bad r_length");
-  }
-  return result;
-}
-
-normalized::Relocation
-ArchHandler::relocFromPattern(ArchHandler::RelocPattern pattern) {
-  normalized::Relocation result;
-  result.offset    = 0;
-  result.scattered = (pattern & rScattered);
-  result.type     = (RelocationInfoType)(pattern & 0xF);
-  result.pcRel    = (pattern & rPcRel);
-  result.isExtern = (pattern & rExtern);
-  result.value    = 0;
-  result.symbol    = 0;
-  switch (pattern & 0x300) {
-  case rLength1:
-    result.length = 0;
-    break;
-  case rLength2:
-    result.length = 1;
-    break;
-  case rLength4:
-    result.length = 2;
-    break;
-  case rLength8:
-    result.length = 3;
-    break;
-  }
-  return result;
-}
-
-void ArchHandler::appendReloc(normalized::Relocations &relocs, uint32_t offset,
-                              uint32_t symbol, uint32_t value,
-                              RelocPattern pattern) {
-  normalized::Relocation reloc = relocFromPattern(pattern);
-  reloc.offset = offset;
-  reloc.symbol = symbol;
-  reloc.value  = value;
-  relocs.push_back(reloc);
-}
-
-
-int16_t ArchHandler::readS16(const uint8_t *addr, bool isBig) {
-    return read16(addr, isBig);
-}
-
-int32_t ArchHandler::readS32(const uint8_t *addr, bool isBig) {
-  return read32(addr, isBig);
-}
-
-uint32_t ArchHandler::readU32(const uint8_t *addr, bool isBig) {
-  return read32(addr, isBig);
-}
-
-  int64_t ArchHandler::readS64(const uint8_t *addr, bool isBig) {
-  return read64(addr, isBig);
-}
-
-bool ArchHandler::isDwarfCIE(bool isBig, const DefinedAtom *atom) {
-  assert(atom->contentType() == DefinedAtom::typeCFI);
-  if (atom->rawContent().size() < sizeof(uint32_t))
-    return false;
-  uint32_t size = read32(atom->rawContent().data(), isBig);
-
-  uint32_t idOffset = sizeof(uint32_t);
-  if (size == 0xffffffffU)
-    idOffset += sizeof(uint64_t);
-
-  return read32(atom->rawContent().data() + idOffset, isBig) == 0;
-}
-
-const Atom *ArchHandler::fdeTargetFunction(const DefinedAtom *fde) {
-  for (auto ref : *fde) {
-    if (ref->kindNamespace() == Reference::KindNamespace::mach_o &&
-        ref->kindValue() == unwindRefToFunctionKind()) {
-      assert(ref->kindArch() == kindArch() && "unexpected Reference arch");
-      return ref->target();
-    }
-  }
-
-  return nullptr;
-}
-
-} // namespace mach_o
-} // namespace lld
-
-
-
diff --git a/lld/lib/ReaderWriter/MachO/ArchHandler.h b/lld/lib/ReaderWriter/MachO/ArchHandler.h
deleted file mode 100644
index 83646c09b1a8..000000000000
--- a/lld/lib/ReaderWriter/MachO/ArchHandler.h
+++ /dev/null
@@ -1,322 +0,0 @@
-//===- lib/FileFormat/MachO/ArchHandler.h ---------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLD_READER_WRITER_MACHO_ARCH_HANDLER_H
-#define LLD_READER_WRITER_MACHO_ARCH_HANDLER_H
-
-#include "Atoms.h"
-#include "File.h"
-#include "MachONormalizedFile.h"
-#include "lld/Common/LLVM.h"
-#include "lld/Core/Error.h"
-#include "lld/Core/Reference.h"
-#include "lld/Core/Simple.h"
-#include "lld/ReaderWriter/MachOLinkingContext.h"
-#include "llvm/ADT/Triple.h"
-
-namespace lld {
-namespace mach_o {
-
-///
-/// The ArchHandler class handles all architecture specific aspects of
-/// mach-o linking.
-///
-class ArchHandler {
-public:
-  virtual ~ArchHandler();
-
-  /// There is no public interface to subclasses of ArchHandler, so this
-  /// is the only way to instantiate an ArchHandler.
-  static std::unique_ptr<ArchHandler> create(MachOLinkingContext::Arch arch);
-
-  /// Get (arch specific) kind strings used by Registry.
-  virtual const Registry::KindStrings *kindStrings() = 0;
-
-  /// Convert mach-o Arch to Reference::KindArch.
-  virtual Reference::KindArch kindArch() = 0;
-
-  /// Used by StubPass to update References to shared library functions
-  /// to be references to a stub.
-  virtual bool isCallSite(const Reference &) = 0;
-
-  /// Used by GOTPass to locate GOT References
-  virtual bool isGOTAccess(const Reference &, bool &canBypassGOT) {
-    return false;
-  }
-
-  /// Used by TLVPass to locate TLV References.
-  virtual bool isTLVAccess(const Reference &) const { return false; }
-
-  /// Used by the TLVPass to update TLV References.
-  virtual void updateReferenceToTLV(const Reference *) {}
-
-  /// Used by ShimPass to insert shims in branches that switch mode.
-  virtual bool isNonCallBranch(const Reference &) = 0;
-
-  /// Used by GOTPass to update GOT References
-  virtual void updateReferenceToGOT(const Reference *, bool targetIsNowGOT) {}
-
-  /// Does this architecture make use of __unwind_info sections for exception
-  /// handling? If so, it will need a separate pass to create them.
-  virtual bool needsCompactUnwind() = 0;
-
-  /// Returns the kind of reference to use to synthesize a 32-bit image-offset
-  /// value, used in the __unwind_info section.
-  virtual Reference::KindValue imageOffsetKind() = 0;
-
-  /// Returns the kind of reference to use to synthesize a 32-bit image-offset
-  /// indirect value. Used for personality functions in the __unwind_info
-  /// section.
-  virtual Reference::KindValue imageOffsetKindIndirect() = 0;
-
-  /// Architecture specific compact unwind type that signals __eh_frame should
-  /// actually be used.
-  virtual uint32_t dwarfCompactUnwindType() = 0;
-
-  /// Reference from an __eh_frame CIE atom to its personality function it's
-  /// describing. Usually pointer-sized and PC-relative, but differs in whether
-  /// it needs to be in relocatable objects.
-  virtual Reference::KindValue unwindRefToPersonalityFunctionKind() = 0;
-
-  /// Reference from an __eh_frame FDE to the CIE it's based on.
-  virtual Reference::KindValue unwindRefToCIEKind() = 0;
-
-  /// Reference from an __eh_frame FDE atom to the function it's
-  /// describing. Usually pointer-sized and PC-relative, but differs in whether
-  /// it needs to be in relocatable objects.
-  virtual Reference::KindValue unwindRefToFunctionKind() = 0;
-
-  /// Reference from an __unwind_info entry of dwarfCompactUnwindType to the
-  /// required __eh_frame entry. On current architectures, the low 24 bits
-  /// represent the offset of the function's FDE entry from the start of
-  /// __eh_frame.
-  virtual Reference::KindValue unwindRefToEhFrameKind() = 0;
-
-  /// Returns a pointer sized reference kind.  On 64-bit targets this will
-  /// likely be something like pointer64, and pointer32 on 32-bit targets.
-  virtual Reference::KindValue pointerKind() = 0;
-
-  virtual const Atom *fdeTargetFunction(const DefinedAtom *fde);
-
-  /// Used by normalizedFromAtoms() to know where to generated rebasing and
-  /// binding info in final executables.
-  virtual bool isPointer(const Reference &) = 0;
-
-  /// Used by normalizedFromAtoms() to know where to generated lazy binding
-  /// info in final executables.
-  virtual bool isLazyPointer(const Reference &);
-
-  /// Reference from an __stub_helper entry to the required offset of the
-  /// lazy bind commands.
-  virtual Reference::KindValue lazyImmediateLocationKind() = 0;
-
-  /// Returns true if the specified relocation is paired to the next relocation.
-  virtual bool isPairedReloc(const normalized::Relocation &) = 0;
-
-  /// Prototype for a helper function.  Given a sectionIndex and address,
-  /// finds the atom and offset with that atom of that address.
-  typedef std::function<llvm::Error (uint32_t sectionIndex, uint64_t addr,
-                        const lld::Atom **, Reference::Addend *)>
-                        FindAtomBySectionAndAddress;
-
-  /// Prototype for a helper function.  Given a symbolIndex, finds the atom
-  /// representing that symbol.
-  typedef std::function<llvm::Error (uint32_t symbolIndex,
-                        const lld::Atom **)> FindAtomBySymbolIndex;
-
-  /// Analyzes a relocation from a .o file and returns the info
-  /// (kind, target, addend) needed to instantiate a Reference.
-  /// Two helper functions are passed as parameters to find the target atom
-  /// given a symbol index or address.
-  virtual llvm::Error
-          getReferenceInfo(const normalized::Relocation &reloc,
-                           const DefinedAtom *inAtom,
-                           uint32_t offsetInAtom,
-                           uint64_t fixupAddress, bool isBigEndian,
-                           FindAtomBySectionAndAddress atomFromAddress,
-                           FindAtomBySymbolIndex atomFromSymbolIndex,
-                           Reference::KindValue *kind,
-                           const lld::Atom **target,
-                           Reference::Addend *addend) = 0;
-
-  /// Analyzes a pair of relocations from a .o file and returns the info
-  /// (kind, target, addend) needed to instantiate a Reference.
-  /// Two helper functions are passed as parameters to find the target atom
-  /// given a symbol index or address.
-  virtual llvm::Error
-      getPairReferenceInfo(const normalized::Relocation &reloc1,
-                           const normalized::Relocation &reloc2,
-                           const DefinedAtom *inAtom,
-                           uint32_t offsetInAtom,
-                           uint64_t fixupAddress, bool isBig, bool scatterable,
-                           FindAtomBySectionAndAddress atomFromAddress,
-                           FindAtomBySymbolIndex atomFromSymbolIndex,
-                           Reference::KindValue *kind,
-                           const lld::Atom **target,
-                           Reference::Addend *addend) = 0;
-
-  /// Prototype for a helper function.  Given an atom, finds the symbol table
-  /// index for it in the output file.
-  typedef std::function<uint32_t (const Atom &atom)> FindSymbolIndexForAtom;
-
-  /// Prototype for a helper function.  Given an atom, finds the index
-  /// of the section that will contain the atom.
-  typedef std::function<uint32_t (const Atom &atom)> FindSectionIndexForAtom;
-
-  /// Prototype for a helper function.  Given an atom, finds the address
-  /// assigned to it in the output file.
-  typedef std::function<uint64_t (const Atom &atom)> FindAddressForAtom;
-
-  /// Some architectures require local symbols on anonymous atoms.
-  virtual bool needsLocalSymbolInRelocatableFile(const DefinedAtom *atom) {
-    return false;
-  }
-
-  /// Copy raw content then apply all fixup References on an Atom.
-  virtual void generateAtomContent(const DefinedAtom &atom, bool relocatable,
-                                   FindAddressForAtom findAddress,
-                                   FindAddressForAtom findSectionAddress,
-                                   uint64_t imageBaseAddress,
-                          llvm::MutableArrayRef<uint8_t> atomContentBuffer) = 0;
-
-  /// Used in -r mode to convert a Reference to a mach-o relocation.
-  virtual void appendSectionRelocations(const DefinedAtom &atom,
-                                        uint64_t atomSectionOffset,
-                                        const Reference &ref,
-                                        FindSymbolIndexForAtom,
-                                        FindSectionIndexForAtom,
-                                        FindAddressForAtom,
-                                        normalized::Relocations&) = 0;
-
-  /// Add arch-specific References.
-  virtual void addAdditionalReferences(MachODefinedAtom &atom) { }
-
-  // Add Reference for data-in-code marker.
-  virtual void addDataInCodeReference(MachODefinedAtom &atom, uint32_t atomOff,
-                                      uint16_t length, uint16_t kind) { }
-
-  /// Returns true if the specificed Reference value marks the start or end
-  /// of a data-in-code range in an atom.
-  virtual bool isDataInCodeTransition(Reference::KindValue refKind) {
-    return false;
-  }
-
-  /// Returns the Reference value for a Reference that marks that start of
-  /// a data-in-code range.
-  virtual Reference::KindValue dataInCodeTransitionStart(
-                                                const MachODefinedAtom &atom) {
-    return 0;
-  }
-
-  /// Returns the Reference value for a Reference that marks that end of
-  /// a data-in-code range.
-  virtual Reference::KindValue dataInCodeTransitionEnd(
-                                                const MachODefinedAtom &atom) {
-    return 0;
-  }
-
-  /// Only relevant for 32-bit arm archs.
-  virtual bool isThumbFunction(const DefinedAtom &atom) { return false; }
-
-  /// Only relevant for 32-bit arm archs.
-  virtual const DefinedAtom *createShim(MachOFile &file, bool thumbToArm,
-                                        const DefinedAtom &) {
-    llvm_unreachable("shims only support on arm");
-  }
-
-  /// Does a given unwind-cfi atom represent a CIE (as opposed to an FDE).
-  static bool isDwarfCIE(bool isBig, const DefinedAtom *atom);
-
-  struct ReferenceInfo {
-    Reference::KindArch arch;
-    uint16_t            kind;
-    uint32_t            offset;
-    int32_t             addend;
-  };
-
-  struct OptionalRefInfo {
-    bool                used;
-    uint16_t            kind;
-    uint32_t            offset;
-    int32_t             addend;
-  };
-
-  /// Table of architecture specific information for creating stubs.
-  struct StubInfo {
-    const char*     binderSymbolName;
-    ReferenceInfo   lazyPointerReferenceToHelper;
-    ReferenceInfo   lazyPointerReferenceToFinal;
-    ReferenceInfo   nonLazyPointerReferenceToBinder;
-    uint8_t         codeAlignment;
-
-    uint32_t        stubSize;
-    uint8_t         stubBytes[16];
-    ReferenceInfo   stubReferenceToLP;
-    OptionalRefInfo optStubReferenceToLP;
-
-    uint32_t        stubHelperSize;
-    uint8_t         stubHelperBytes[16];
-    ReferenceInfo   stubHelperReferenceToImm;
-    ReferenceInfo   stubHelperReferenceToHelperCommon;
-
-    DefinedAtom::ContentType stubHelperImageCacheContentType;
-
-    uint32_t        stubHelperCommonSize;
-    uint8_t         stubHelperCommonAlignment;
-    uint8_t         stubHelperCommonBytes[36];
-    ReferenceInfo   stubHelperCommonReferenceToCache;
-    OptionalRefInfo optStubHelperCommonReferenceToCache;
-    ReferenceInfo   stubHelperCommonReferenceToBinder;
-    OptionalRefInfo optStubHelperCommonReferenceToBinder;
-  };
-
-  virtual const StubInfo &stubInfo() = 0;
-
-protected:
-  ArchHandler();
-
-  static std::unique_ptr<mach_o::ArchHandler> create_x86_64();
-  static std::unique_ptr<mach_o::ArchHandler> create_x86();
-  static std::unique_ptr<mach_o::ArchHandler> create_arm();
-  static std::unique_ptr<mach_o::ArchHandler> create_arm64();
-
-  // Handy way to pack mach-o r_type and other bit fields into one 16-bit value.
-  typedef uint16_t RelocPattern;
-  enum {
-    rScattered = 0x8000,
-    rPcRel     = 0x4000,
-    rExtern    = 0x2000,
-    rLength1   = 0x0000,
-    rLength2   = 0x0100,
-    rLength4   = 0x0200,
-    rLength8   = 0x0300,
-    rLenArmLo  = rLength1,
-    rLenArmHi  = rLength2,
-    rLenThmbLo = rLength4,
-    rLenThmbHi = rLength8
-  };
-  /// Extract RelocPattern from normalized mach-o relocation.
-  static RelocPattern relocPattern(const normalized::Relocation &reloc);
-  /// Create normalized Relocation initialized from pattern.
-  static normalized::Relocation relocFromPattern(RelocPattern pattern);
-  /// One liner to add a relocation.
-  static void appendReloc(normalized::Relocations &relocs, uint32_t offset,
-                          uint32_t symbol, uint32_t value,
-                          RelocPattern pattern);
-
-
-  static int16_t  readS16(const uint8_t *addr, bool isBig);
-  static int32_t  readS32(const uint8_t *addr, bool isBig);
-  static uint32_t readU32(const uint8_t *addr, bool isBig);
-  static int64_t  readS64(const uint8_t *addr, bool isBig);
-};
-
-} // namespace mach_o
-} // namespace lld
-
-#endif // LLD_READER_WRITER_MACHO_ARCH_HANDLER_H
diff --git a/lld/lib/ReaderWriter/MachO/ArchHandler_arm.cpp b/lld/lib/ReaderWriter/MachO/ArchHandler_arm.cpp
deleted file mode 100644
index 06c98ac06fd1..000000000000
--- a/lld/lib/ReaderWriter/MachO/ArchHandler_arm.cpp
+++ /dev/null
@@ -1,1522 +0,0 @@
-//===- lib/FileFormat/MachO/ArchHandler_arm.cpp ---------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "ArchHandler.h"
-#include "Atoms.h"
-#include "MachONormalizedFileBinaryUtils.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/ADT/StringSwitch.h"
-#include "llvm/ADT/Triple.h"
-#include "llvm/Support/Endian.h"
-#include "llvm/Support/ErrorHandling.h"
-
-using namespace llvm::MachO;
-using namespace lld::mach_o::normalized;
-
-namespace lld {
-namespace mach_o {
-
-using llvm::support::ulittle32_t;
-using llvm::support::little32_t;
-
-
-class ArchHandler_arm : public ArchHandler {
-public:
-  ArchHandler_arm() = default;
-  ~ArchHandler_arm() override = default;
-
-  const Registry::KindStrings *kindStrings() override { return _sKindStrings; }
-
-  Reference::KindArch kindArch() override { return Reference::KindArch::ARM; }
-
-  const ArchHandler::StubInfo &stubInfo() override;
-  bool isCallSite(const Reference &) override;
-  bool isPointer(const Reference &) override;
-  bool isPairedReloc(const normalized::Relocation &) override;
-  bool isNonCallBranch(const Reference &) override;
-
-  bool needsCompactUnwind() override {
-    return false;
-  }
-  Reference::KindValue imageOffsetKind() override {
-    return invalid;
-  }
-  Reference::KindValue imageOffsetKindIndirect() override {
-    return invalid;
-  }
-
-  Reference::KindValue unwindRefToPersonalityFunctionKind() override {
-    return invalid;
-  }
-
-  Reference::KindValue unwindRefToCIEKind() override {
-    return invalid;
-  }
-
-  Reference::KindValue unwindRefToFunctionKind() override {
-    return invalid;
-  }
-
-  Reference::KindValue unwindRefToEhFrameKind() override {
-    return invalid;
-  }
-
-  Reference::KindValue lazyImmediateLocationKind() override {
-    return lazyImmediateLocation;
-  }
-
-  Reference::KindValue pointerKind() override {
-    return invalid;
-  }
-
-  uint32_t dwarfCompactUnwindType() override {
-    // FIXME
-    return -1;
-  }
-
-  llvm::Error getReferenceInfo(const normalized::Relocation &reloc,
-                               const DefinedAtom *inAtom,
-                               uint32_t offsetInAtom,
-                               uint64_t fixupAddress, bool swap,
-                               FindAtomBySectionAndAddress atomFromAddress,
-                               FindAtomBySymbolIndex atomFromSymbolIndex,
-                               Reference::KindValue *kind,
-                               const lld::Atom **target,
-                               Reference::Addend *addend) override;
-  llvm::Error
-      getPairReferenceInfo(const normalized::Relocation &reloc1,
-                           const normalized::Relocation &reloc2,
-                           const DefinedAtom *inAtom,
-                           uint32_t offsetInAtom,
-                           uint64_t fixupAddress, bool swap, bool scatterable,
-                           FindAtomBySectionAndAddress atomFromAddress,
-                           FindAtomBySymbolIndex atomFromSymbolIndex,
-                           Reference::KindValue *kind,
-                           const lld::Atom **target,
-                           Reference::Addend *addend) override;
-
-  void generateAtomContent(const DefinedAtom &atom, bool relocatable,
-                           FindAddressForAtom findAddress,
-                           FindAddressForAtom findSectionAddress,
-                           uint64_t imageBaseAddress,
-                           llvm::MutableArrayRef<uint8_t> atomContentBuffer) override;
-
-  void appendSectionRelocations(const DefinedAtom &atom,
-                                uint64_t atomSectionOffset,
-                                const Reference &ref,
-                                FindSymbolIndexForAtom,
-                                FindSectionIndexForAtom,
-                                FindAddressForAtom,
-                                normalized::Relocations &) override;
-
-  void addAdditionalReferences(MachODefinedAtom &atom) override;
-
-  bool isDataInCodeTransition(Reference::KindValue refKind) override {
-    switch (refKind) {
-    case modeThumbCode:
-    case modeArmCode:
-    case modeData:
-      return true;
-    default:
-      return false;
-      break;
-    }
-  }
-
-  Reference::KindValue dataInCodeTransitionStart(
-                                        const MachODefinedAtom &atom) override {
-    return modeData;
-  }
-
-  Reference::KindValue dataInCodeTransitionEnd(
-                                        const MachODefinedAtom &atom) override {
-    return atom.isThumb() ? modeThumbCode : modeArmCode;
-  }
-
-  bool isThumbFunction(const DefinedAtom &atom) override;
-  const DefinedAtom *createShim(MachOFile &file, bool thumbToArm,
-                                const DefinedAtom &) override;
-
-private:
-  friend class Thumb2ToArmShimAtom;
-  friend class ArmToThumbShimAtom;
-
-  static const Registry::KindStrings _sKindStrings[];
-  static const StubInfo              _sStubInfoArmPIC;
-
-  enum ArmKind : Reference::KindValue {
-    invalid,               /// for error condition
-
-    modeThumbCode,         /// Content starting at this offset is thumb.
-    modeArmCode,           /// Content starting at this offset is arm.
-    modeData,              /// Content starting at this offset is data.
-
-    // Kinds found in mach-o .o files:
-    thumb_bl22,            /// ex: bl _foo
-    thumb_b22,             /// ex: b _foo
-    thumb_movw,            /// ex: movw	r1, :lower16:_foo
-    thumb_movt,            /// ex: movt	r1, :lower16:_foo
-    thumb_movw_funcRel,    /// ex: movw	r1, :lower16:(_foo-(L1+4))
-    thumb_movt_funcRel,    /// ex: movt r1, :upper16:(_foo-(L1+4))
-    arm_bl24,              /// ex: bl _foo
-    arm_b24,               /// ex: b _foo
-    arm_movw,              /// ex: movw	r1, :lower16:_foo
-    arm_movt,              /// ex: movt	r1, :lower16:_foo
-    arm_movw_funcRel,      /// ex: movw	r1, :lower16:(_foo-(L1+4))
-    arm_movt_funcRel,      /// ex: movt r1, :upper16:(_foo-(L1+4))
-    pointer32,             /// ex: .long _foo
-    delta32,               /// ex: .long _foo - .
-
-    // Kinds introduced by Passes:
-    lazyPointer,           /// Location contains a lazy pointer.
-    lazyImmediateLocation, /// Location contains immediate value used in stub.
-  };
-
-  // Utility functions for inspecting/updating instructions.
-  static bool isThumbMovw(uint32_t instruction);
-  static bool isThumbMovt(uint32_t instruction);
-  static bool isArmMovw(uint32_t instruction);
-  static bool isArmMovt(uint32_t instruction);
-  static int32_t getDisplacementFromThumbBranch(uint32_t instruction, uint32_t);
-  static int32_t getDisplacementFromArmBranch(uint32_t instruction);
-  static uint16_t getWordFromThumbMov(uint32_t instruction);
-  static uint16_t getWordFromArmMov(uint32_t instruction);
-  static uint32_t clearThumbBit(uint32_t value, const Atom *target);
-  static uint32_t setDisplacementInArmBranch(uint32_t instr, int32_t disp,
-                                             bool targetIsThumb);
-  static uint32_t setDisplacementInThumbBranch(uint32_t instr, uint32_t ia,
-                                               int32_t disp, bool targetThumb);
-  static uint32_t setWordFromThumbMov(uint32_t instruction, uint16_t word);
-  static uint32_t setWordFromArmMov(uint32_t instruction, uint16_t word);
-
-  StringRef stubName(const DefinedAtom &);
-  bool useExternalRelocationTo(const Atom &target);
-
-  void applyFixupFinal(const Reference &ref, uint8_t *location,
-                       uint64_t fixupAddress, uint64_t targetAddress,
-                       uint64_t inAtomAddress, bool &thumbMode,
-                       bool targetIsThumb);
-
-  void applyFixupRelocatable(const Reference &ref, uint8_t *location,
-                             uint64_t fixupAddress,
-                             uint64_t targetAddress,
-                             uint64_t inAtomAddress, bool &thumbMode,
-                             bool targetIsThumb);
-};
-
-//===----------------------------------------------------------------------===//
-//  ArchHandler_arm
-//===----------------------------------------------------------------------===//
-
-const Registry::KindStrings ArchHandler_arm::_sKindStrings[] = {
-  LLD_KIND_STRING_ENTRY(invalid),
-  LLD_KIND_STRING_ENTRY(modeThumbCode),
-  LLD_KIND_STRING_ENTRY(modeArmCode),
-  LLD_KIND_STRING_ENTRY(modeData),
-  LLD_KIND_STRING_ENTRY(thumb_bl22),
-  LLD_KIND_STRING_ENTRY(thumb_b22),
-  LLD_KIND_STRING_ENTRY(thumb_movw),
-  LLD_KIND_STRING_ENTRY(thumb_movt),
-  LLD_KIND_STRING_ENTRY(thumb_movw_funcRel),
-  LLD_KIND_STRING_ENTRY(thumb_movt_funcRel),
-  LLD_KIND_STRING_ENTRY(arm_bl24),
-  LLD_KIND_STRING_ENTRY(arm_b24),
-  LLD_KIND_STRING_ENTRY(arm_movw),
-  LLD_KIND_STRING_ENTRY(arm_movt),
-  LLD_KIND_STRING_ENTRY(arm_movw_funcRel),
-  LLD_KIND_STRING_ENTRY(arm_movt_funcRel),
-  LLD_KIND_STRING_ENTRY(pointer32),
-  LLD_KIND_STRING_ENTRY(delta32),
-  LLD_KIND_STRING_ENTRY(lazyPointer),
-  LLD_KIND_STRING_ENTRY(lazyImmediateLocation),
-  LLD_KIND_STRING_END
-};
-
-const ArchHandler::StubInfo ArchHandler_arm::_sStubInfoArmPIC = {
-  "dyld_stub_binder",
-
-  // References in lazy pointer
-  { Reference::KindArch::ARM, pointer32, 0, 0 },
-  { Reference::KindArch::ARM, lazyPointer, 0, 0 },
-
-  // GOT pointer to dyld_stub_binder
-  { Reference::KindArch::ARM, pointer32, 0, 0 },
-
-  // arm code alignment 2^2
-  2,
-
-  // Stub size and code
-  16,
-  { 0x04, 0xC0, 0x9F, 0xE5,       // 	ldr ip, pc + 12
-    0x0C, 0xC0, 0x8F, 0xE0,       //  add ip, pc, ip
-    0x00, 0xF0, 0x9C, 0xE5,       // 	ldr pc, [ip]
-    0x00, 0x00, 0x00, 0x00 },     // 	.long L_foo$lazy_ptr - (L1$scv + 8)
-  { Reference::KindArch::ARM, delta32, 12, 0 },
-  { false, 0, 0, 0 },
-
-  // Stub Helper size and code
-  12,
-  { 0x00, 0xC0, 0x9F, 0xE5,       // ldr   ip, [pc, #0]
-    0x00, 0x00, 0x00, 0xEA,       // b	     _helperhelper
-    0x00, 0x00, 0x00, 0x00 },     // .long  lazy-info-offset
-  { Reference::KindArch::ARM, lazyImmediateLocation, 8, 0 },
-  { Reference::KindArch::ARM, arm_b24, 4, 0 },
-
-  // Stub helper image cache content type
-  DefinedAtom::typeGOT,
-
-  // Stub Helper-Common size and code
-  36,
-  // Stub helper alignment
-  2,
-	{ // push lazy-info-offset
-    0x04, 0xC0, 0x2D, 0xE5,       // str ip, [sp, #-4]!
-		// push address of dyld_mageLoaderCache
-    0x10, 0xC0, 0x9F, 0xE5,       // ldr	ip, L1
-    0x0C, 0xC0, 0x8F, 0xE0,       // add	ip, pc, ip
-    0x04, 0xC0, 0x2D, 0xE5,       // str ip, [sp, #-4]!
-		// jump through dyld_stub_binder
-    0x08, 0xC0, 0x9F, 0xE5,       // ldr	ip, L2
-    0x0C, 0xC0, 0x8F, 0xE0,       // add	ip, pc, ip
-    0x00, 0xF0, 0x9C, 0xE5,       // ldr	pc, [ip]
-    0x00, 0x00, 0x00, 0x00,       // L1: .long fFastStubGOTAtom - (helper+16)
-    0x00, 0x00, 0x00, 0x00 },     // L2: .long dyld_stub_binder - (helper+28)
-  { Reference::KindArch::ARM, delta32, 28, 0xC },
-  { false, 0, 0, 0 },
-  { Reference::KindArch::ARM, delta32, 32, 0x04 },
-  { false, 0, 0, 0 }
-};
-
-const ArchHandler::StubInfo &ArchHandler_arm::stubInfo() {
-  // If multiple kinds of stubs are supported, select which StubInfo here.
-  return _sStubInfoArmPIC;
-}
-
-bool ArchHandler_arm::isCallSite(const Reference &ref) {
-  switch (ref.kindValue()) {
-  case thumb_b22:
-  case thumb_bl22:
-  case arm_b24:
-  case arm_bl24:
-    return true;
-  default:
-    return false;
-  }
-}
-
-bool ArchHandler_arm::isPointer(const Reference &ref) {
-  return (ref.kindValue() == pointer32);
-}
-
-bool ArchHandler_arm::isNonCallBranch(const Reference &ref) {
-  switch (ref.kindValue()) {
-  case thumb_b22:
-  case arm_b24:
-    return true;
-  default:
-    return false;
-  }
-}
-
-bool ArchHandler_arm::isPairedReloc(const Relocation &reloc) {
-  switch (reloc.type) {
-  case ARM_RELOC_SECTDIFF:
-  case ARM_RELOC_LOCAL_SECTDIFF:
-  case ARM_RELOC_HALF_SECTDIFF:
-  case ARM_RELOC_HALF:
-    return true;
-  default:
-    return false;
-  }
-}
-
-/// Trace references from stub atom to lazy pointer to target and get its name.
-StringRef ArchHandler_arm::stubName(const DefinedAtom &stubAtom) {
-  assert(stubAtom.contentType() == DefinedAtom::typeStub);
-  for (const Reference *ref : stubAtom) {
-    if (const DefinedAtom* lp = dyn_cast<DefinedAtom>(ref->target())) {
-      if (lp->contentType() != DefinedAtom::typeLazyPointer)
-        continue;
-      for (const Reference *ref2 : *lp) {
-        if (ref2->kindValue() != lazyPointer)
-          continue;
-        return ref2->target()->name();
-      }
-    }
-  }
-  return "stub";
-}
-
-/// Extract displacement from an ARM b/bl/blx instruction.
-int32_t ArchHandler_arm::getDisplacementFromArmBranch(uint32_t instruction) {
-  // Sign-extend imm24
-  int32_t displacement = (instruction & 0x00FFFFFF) << 2;
-  if ((displacement & 0x02000000) != 0)
-    displacement |= 0xFC000000;
-  // If this is BLX and H bit set, add 2.
-  if ((instruction & 0xFF000000) == 0xFB000000)
-    displacement += 2;
-  return displacement;
-}
-
-/// Update an ARM b/bl/blx instruction, switching bl <-> blx as needed.
-uint32_t ArchHandler_arm::setDisplacementInArmBranch(uint32_t instruction,
-                                                     int32_t displacement,
-                                                     bool targetIsThumb) {
-  assert((displacement <= 33554428) && (displacement > (-33554432))
-                                              && "arm branch out of range");
-  bool is_blx = ((instruction & 0xF0000000) == 0xF0000000);
-  uint32_t newInstruction = (instruction & 0xFF000000);
-  uint32_t h = 0;
-  if (targetIsThumb) {
-    // Force use of BLX.
-    newInstruction = 0xFA000000;
-    if (!is_blx) {
-      assert(((instruction & 0xF0000000) == 0xE0000000)
-                                                   && "no conditional arm blx");
-      assert(((instruction & 0xFF000000) == 0xEB000000)
-                                             && "no arm pc-rel BX instruction");
-    }
-    if (displacement & 2)
-      h = 1;
-  }
-  else {
-    // Force use of B/BL.
-    if (is_blx)
-      newInstruction = 0xEB000000;
-  }
-  newInstruction |= (h << 24) | ((displacement >> 2) & 0x00FFFFFF);
-  return newInstruction;
-}
-
-/// Extract displacement from a thumb b/bl/blx instruction.
-int32_t ArchHandler_arm::getDisplacementFromThumbBranch(uint32_t instruction,
-                                                        uint32_t instrAddr) {
-  bool is_blx = ((instruction & 0xD000F800) == 0xC000F000);
-  uint32_t s = (instruction >> 10) & 0x1;
-  uint32_t j1 = (instruction >> 29) & 0x1;
-  uint32_t j2 = (instruction >> 27) & 0x1;
-  uint32_t imm10 = instruction & 0x3FF;
-  uint32_t imm11 = (instruction >> 16) & 0x7FF;
-  uint32_t i1 = (j1 == s);
-  uint32_t i2 = (j2 == s);
-  uint32_t dis =
-      (s << 24) | (i1 << 23) | (i2 << 22) | (imm10 << 12) | (imm11 << 1);
-  int32_t sdis = dis;
-  int32_t result = s ? (sdis | 0xFE000000) : sdis;
-  if (is_blx && (instrAddr & 0x2)) {
-    // The thumb blx instruction always has low bit of imm11 as zero.  The way
-    // a 2-byte aligned blx can branch to a 4-byte aligned ARM target is that
-    // the blx instruction always 4-byte aligns the pc before adding the
-    // displacement from the blx.  We must emulate that when decoding this.
-    result -= 2;
-  }
-  return result;
-}
-
-/// Update a thumb b/bl/blx instruction, switching bl <-> blx as needed.
-uint32_t ArchHandler_arm::setDisplacementInThumbBranch(uint32_t instruction,
-                                                       uint32_t instrAddr,
-                                                       int32_t displacement,
-                                                       bool targetIsThumb) {
-  assert((displacement <= 16777214) && (displacement > (-16777216))
-                                              && "thumb branch out of range");
-	bool is_bl = ((instruction & 0xD000F800) == 0xD000F000);
-	bool is_blx = ((instruction & 0xD000F800) == 0xC000F000);
-	bool is_b = ((instruction & 0xD000F800) == 0x9000F000);
-  uint32_t newInstruction = (instruction & 0xD000F800);
-  if (is_bl || is_blx) {
-    if (targetIsThumb) {
-      newInstruction = 0xD000F000; // Use bl
-    } else {
-      newInstruction = 0xC000F000; // Use blx
-      // See note in getDisplacementFromThumbBranch() about blx.
-      if (instrAddr & 0x2)
-        displacement += 2;
-    }
-  } else if (is_b) {
-    assert(targetIsThumb && "no pc-rel thumb branch instruction that "
-                             "switches to arm mode");
-  }
-  else {
-    llvm_unreachable("thumb branch22 reloc on a non-branch instruction");
-  }
-  uint32_t s = (uint32_t)(displacement >> 24) & 0x1;
-  uint32_t i1 = (uint32_t)(displacement >> 23) & 0x1;
-  uint32_t i2 = (uint32_t)(displacement >> 22) & 0x1;
-  uint32_t imm10 = (uint32_t)(displacement >> 12) & 0x3FF;
-  uint32_t imm11 = (uint32_t)(displacement >> 1) & 0x7FF;
-  uint32_t j1 = (i1 == s);
-  uint32_t j2 = (i2 == s);
-  uint32_t nextDisp = (j1 << 13) | (j2 << 11) | imm11;
-  uint32_t firstDisp = (s << 10) | imm10;
-  newInstruction |= (nextDisp << 16) | firstDisp;
-  return newInstruction;
-}
-
-bool ArchHandler_arm::isThumbMovw(uint32_t instruction) {
-  return (instruction & 0x8000FBF0) == 0x0000F240;
-}
-
-bool ArchHandler_arm::isThumbMovt(uint32_t instruction) {
-  return (instruction & 0x8000FBF0) == 0x0000F2C0;
-}
-
-bool ArchHandler_arm::isArmMovw(uint32_t instruction) {
-  return (instruction & 0x0FF00000) == 0x03000000;
-}
-
-bool ArchHandler_arm::isArmMovt(uint32_t instruction) {
-  return (instruction & 0x0FF00000) == 0x03400000;
-}
-
-uint16_t ArchHandler_arm::getWordFromThumbMov(uint32_t instruction) {
-  assert(isThumbMovw(instruction) || isThumbMovt(instruction));
-  uint32_t i = ((instruction & 0x00000400) >> 10);
-  uint32_t imm4 = (instruction & 0x0000000F);
-  uint32_t imm3 = ((instruction & 0x70000000) >> 28);
-  uint32_t imm8 = ((instruction & 0x00FF0000) >> 16);
-  return (imm4 << 12) | (i << 11) | (imm3 << 8) | imm8;
-}
-
-uint16_t ArchHandler_arm::getWordFromArmMov(uint32_t instruction) {
-  assert(isArmMovw(instruction) || isArmMovt(instruction));
-  uint32_t imm4 = ((instruction & 0x000F0000) >> 16);
-  uint32_t imm12 = (instruction & 0x00000FFF);
-  return (imm4 << 12) | imm12;
-}
-
-uint32_t ArchHandler_arm::setWordFromThumbMov(uint32_t instr, uint16_t word) {
-  assert(isThumbMovw(instr) || isThumbMovt(instr));
-  uint32_t imm4 = (word & 0xF000) >> 12;
-  uint32_t i =    (word & 0x0800) >> 11;
-  uint32_t imm3 = (word & 0x0700) >> 8;
-  uint32_t imm8 =  word & 0x00FF;
-	return (instr & 0x8F00FBF0) | imm4 | (i << 10) | (imm3 << 28) | (imm8 << 16);
-}
-
-uint32_t ArchHandler_arm::setWordFromArmMov(uint32_t instr, uint16_t word) {
-  assert(isArmMovw(instr) || isArmMovt(instr));
-  uint32_t imm4 = (word & 0xF000) >> 12;
-  uint32_t imm12 = word & 0x0FFF;
-  return (instr & 0xFFF0F000) | (imm4 << 16) | imm12;
-}
-
-uint32_t ArchHandler_arm::clearThumbBit(uint32_t value, const Atom *target) {
-  // The assembler often adds one to the address of a thumb function.
-  // We need to undo that so it does not look like an addend.
-  if (value & 1) {
-    if (isa<DefinedAtom>(target)) {
-      const MachODefinedAtom *machoTarget =
-          reinterpret_cast<const MachODefinedAtom *>(target);
-      if (machoTarget->isThumb())
-        value &= -2; // mask off thumb-bit
-    }
-  }
-  return value;
-}
-
-llvm::Error ArchHandler_arm::getReferenceInfo(
-    const Relocation &reloc, const DefinedAtom *inAtom, uint32_t offsetInAtom,
-    uint64_t fixupAddress, bool isBig,
-    FindAtomBySectionAndAddress atomFromAddress,
-    FindAtomBySymbolIndex atomFromSymbolIndex, Reference::KindValue *kind,
-    const lld::Atom **target, Reference::Addend *addend) {
-  const uint8_t *fixupContent = &inAtom->rawContent()[offsetInAtom];
-  uint64_t targetAddress;
-  uint32_t instruction = *(const ulittle32_t *)fixupContent;
-  int32_t displacement;
-  switch (relocPattern(reloc)) {
-  case ARM_THUMB_RELOC_BR22 | rPcRel | rExtern | rLength4:
-    // ex: bl _foo (and _foo is undefined)
-    if ((instruction & 0xD000F800) == 0x9000F000)
-      *kind = thumb_b22;
-    else
-      *kind = thumb_bl22;
-    if (auto ec = atomFromSymbolIndex(reloc.symbol, target))
-      return ec;
-    // Instruction contains branch to addend.
-    displacement = getDisplacementFromThumbBranch(instruction, fixupAddress);
-    *addend = fixupAddress + 4 + displacement;
-    return llvm::Error::success();
-  case ARM_THUMB_RELOC_BR22 | rPcRel | rLength4:
-    // ex: bl _foo (and _foo is defined)
-    if ((instruction & 0xD000F800) == 0x9000F000)
-      *kind = thumb_b22;
-    else
-      *kind = thumb_bl22;
-    displacement = getDisplacementFromThumbBranch(instruction, fixupAddress);
-    targetAddress = fixupAddress + 4 + displacement;
-    return atomFromAddress(reloc.symbol, targetAddress, target, addend);
-  case ARM_THUMB_RELOC_BR22 | rScattered | rPcRel | rLength4:
-    // ex: bl _foo+4 (and _foo is defined)
-    if ((instruction & 0xD000F800) == 0x9000F000)
-      *kind = thumb_b22;
-    else
-      *kind = thumb_bl22;
-    displacement = getDisplacementFromThumbBranch(instruction, fixupAddress);
-    targetAddress = fixupAddress + 4 + displacement;
-    if (auto ec = atomFromAddress(0, reloc.value, target, addend))
-      return ec;
-    // reloc.value is target atom's address.  Instruction contains branch
-    // to atom+addend.
-    *addend += (targetAddress - reloc.value);
-    return llvm::Error::success();
-  case ARM_RELOC_BR24 | rPcRel | rExtern | rLength4:
-    // ex: bl _foo (and _foo is undefined)
-    if (((instruction & 0x0F000000) == 0x0A000000)
-        && ((instruction & 0xF0000000) != 0xF0000000))
-      *kind = arm_b24;
-    else
-      *kind = arm_bl24;
-    if (auto ec = atomFromSymbolIndex(reloc.symbol, target))
-      return ec;
-    // Instruction contains branch to addend.
-    displacement = getDisplacementFromArmBranch(instruction);
-    *addend = fixupAddress + 8 + displacement;
-    return llvm::Error::success();
-  case ARM_RELOC_BR24 | rPcRel | rLength4:
-    // ex: bl _foo (and _foo is defined)
-    if (((instruction & 0x0F000000) == 0x0A000000)
-        && ((instruction & 0xF0000000) != 0xF0000000))
-      *kind = arm_b24;
-    else
-      *kind = arm_bl24;
-    displacement = getDisplacementFromArmBranch(instruction);
-    targetAddress = fixupAddress + 8 + displacement;
-    return atomFromAddress(reloc.symbol, targetAddress, target, addend);
-  case ARM_RELOC_BR24 | rScattered | rPcRel | rLength4:
-    // ex: bl _foo+4 (and _foo is defined)
-    if (((instruction & 0x0F000000) == 0x0A000000)
-        && ((instruction & 0xF0000000) != 0xF0000000))
-      *kind = arm_b24;
-    else
-      *kind = arm_bl24;
-    displacement = getDisplacementFromArmBranch(instruction);
-    targetAddress = fixupAddress + 8 + displacement;
-    if (auto ec = atomFromAddress(0, reloc.value, target, addend))
-      return ec;
-    // reloc.value is target atom's address.  Instruction contains branch
-    // to atom+addend.
-    *addend += (targetAddress - reloc.value);
-    return llvm::Error::success();
-  case ARM_RELOC_VANILLA | rExtern | rLength4:
-    // ex: .long _foo (and _foo is undefined)
-    *kind = pointer32;
-    if (auto ec = atomFromSymbolIndex(reloc.symbol, target))
-      return ec;
-    *addend = instruction;
-    return llvm::Error::success();
-  case ARM_RELOC_VANILLA | rLength4:
-    // ex: .long _foo (and _foo is defined)
-    *kind = pointer32;
-    if (auto ec = atomFromAddress(reloc.symbol, instruction, target, addend))
-      return ec;
-    *addend = clearThumbBit((uint32_t) * addend, *target);
-    return llvm::Error::success();
-  case ARM_RELOC_VANILLA | rScattered | rLength4:
-    // ex: .long _foo+a (and _foo is defined)
-    *kind = pointer32;
-    if (auto ec = atomFromAddress(0, reloc.value, target, addend))
-      return ec;
-    *addend += (clearThumbBit(instruction, *target) - reloc.value);
-    return llvm::Error::success();
-  default:
-    return llvm::make_error<GenericError>("unsupported arm relocation type");
-  }
-  return llvm::Error::success();
-}
-
-llvm::Error
-ArchHandler_arm::getPairReferenceInfo(const normalized::Relocation &reloc1,
-                                     const normalized::Relocation &reloc2,
-                                     const DefinedAtom *inAtom,
-                                     uint32_t offsetInAtom,
-                                     uint64_t fixupAddress, bool isBig,
-                                     bool scatterable,
-                                     FindAtomBySectionAndAddress atomFromAddr,
-                                     FindAtomBySymbolIndex atomFromSymbolIndex,
-                                     Reference::KindValue *kind,
-                                     const lld::Atom **target,
-                                     Reference::Addend *addend) {
-  bool pointerDiff = false;
-  bool funcRel;
-  bool top;
-  bool thumbReloc;
-  switch(relocPattern(reloc1) << 16 | relocPattern(reloc2)) {
-  case ((ARM_RELOC_HALF_SECTDIFF  | rScattered | rLenThmbLo) << 16 |
-         ARM_RELOC_PAIR           | rScattered | rLenThmbLo):
-    // ex: movw	r1, :lower16:(_x-L1) [thumb mode]
-    *kind = thumb_movw_funcRel;
-    funcRel = true;
-    top = false;
-    thumbReloc = true;
-    break;
-  case ((ARM_RELOC_HALF_SECTDIFF  | rScattered | rLenThmbHi) << 16 |
-         ARM_RELOC_PAIR           | rScattered | rLenThmbHi):
-    // ex: movt	r1, :upper16:(_x-L1) [thumb mode]
-    *kind = thumb_movt_funcRel;
-    funcRel = true;
-    top = true;
-    thumbReloc = true;
-    break;
-  case ((ARM_RELOC_HALF_SECTDIFF  | rScattered | rLenArmLo) << 16 |
-         ARM_RELOC_PAIR           | rScattered | rLenArmLo):
-    // ex: movw	r1, :lower16:(_x-L1) [arm mode]
-    *kind = arm_movw_funcRel;
-    funcRel = true;
-    top = false;
-    thumbReloc = false;
-    break;
-  case ((ARM_RELOC_HALF_SECTDIFF  | rScattered | rLenArmHi) << 16 |
-         ARM_RELOC_PAIR           | rScattered | rLenArmHi):
-    // ex: movt	r1, :upper16:(_x-L1) [arm mode]
-    *kind = arm_movt_funcRel;
-    funcRel = true;
-    top = true;
-    thumbReloc = false;
-    break;
-  case ((ARM_RELOC_HALF     | rLenThmbLo) << 16 |
-         ARM_RELOC_PAIR     | rLenThmbLo):
-    // ex: movw	r1, :lower16:_x [thumb mode]
-    *kind = thumb_movw;
-    funcRel = false;
-    top = false;
-    thumbReloc = true;
-    break;
-  case ((ARM_RELOC_HALF     | rLenThmbHi) << 16 |
-         ARM_RELOC_PAIR     | rLenThmbHi):
-    // ex: movt	r1, :upper16:_x [thumb mode]
-    *kind = thumb_movt;
-    funcRel = false;
-    top = true;
-    thumbReloc = true;
-    break;
-  case ((ARM_RELOC_HALF     | rLenArmLo) << 16 |
-         ARM_RELOC_PAIR     | rLenArmLo):
-    // ex: movw	r1, :lower16:_x [arm mode]
-    *kind = arm_movw;
-    funcRel = false;
-    top = false;
-    thumbReloc = false;
-    break;
-  case ((ARM_RELOC_HALF     | rLenArmHi) << 16 |
-         ARM_RELOC_PAIR     | rLenArmHi):
-    // ex: movt	r1, :upper16:_x [arm mode]
-    *kind = arm_movt;
-    funcRel = false;
-    top = true;
-    thumbReloc = false;
-    break;
-  case ((ARM_RELOC_HALF | rScattered  | rLenThmbLo) << 16 |
-         ARM_RELOC_PAIR               | rLenThmbLo):
-    // ex: movw	r1, :lower16:_x+a [thumb mode]
-    *kind = thumb_movw;
-    funcRel = false;
-    top = false;
-    thumbReloc = true;
-    break;
-  case ((ARM_RELOC_HALF | rScattered  | rLenThmbHi) << 16 |
-         ARM_RELOC_PAIR               | rLenThmbHi):
-    // ex: movt	r1, :upper16:_x+a [thumb mode]
-    *kind = thumb_movt;
-    funcRel = false;
-    top = true;
-    thumbReloc = true;
-    break;
-  case ((ARM_RELOC_HALF | rScattered  | rLenArmLo) << 16 |
-         ARM_RELOC_PAIR               | rLenArmLo):
-    // ex: movw	r1, :lower16:_x+a [arm mode]
-    *kind = arm_movw;
-    funcRel = false;
-    top = false;
-    thumbReloc = false;
-    break;
-  case ((ARM_RELOC_HALF | rScattered  | rLenArmHi) << 16 |
-         ARM_RELOC_PAIR               | rLenArmHi):
-    // ex: movt	r1, :upper16:_x+a [arm mode]
-    *kind = arm_movt;
-    funcRel = false;
-    top = true;
-    thumbReloc = false;
-    break;
-  case ((ARM_RELOC_HALF | rExtern   | rLenThmbLo) << 16 |
-         ARM_RELOC_PAIR             | rLenThmbLo):
-    // ex: movw	r1, :lower16:_undef [thumb mode]
-    *kind = thumb_movw;
-    funcRel = false;
-    top = false;
-    thumbReloc = true;
-    break;
-  case ((ARM_RELOC_HALF | rExtern   | rLenThmbHi) << 16 |
-         ARM_RELOC_PAIR             | rLenThmbHi):
-    // ex: movt	r1, :upper16:_undef [thumb mode]
-    *kind = thumb_movt;
-    funcRel = false;
-    top = true;
-    thumbReloc = true;
-    break;
-  case ((ARM_RELOC_HALF | rExtern   | rLenArmLo) << 16 |
-         ARM_RELOC_PAIR             | rLenArmLo):
-    // ex: movw	r1, :lower16:_undef [arm mode]
-    *kind = arm_movw;
-    funcRel = false;
-    top = false;
-    thumbReloc = false;
-    break;
-  case ((ARM_RELOC_HALF | rExtern   | rLenArmHi) << 16 |
-         ARM_RELOC_PAIR             | rLenArmHi):
-    // ex: movt	r1, :upper16:_undef [arm mode]
-    *kind = arm_movt;
-    funcRel = false;
-    top = true;
-    thumbReloc = false;
-    break;
-  case ((ARM_RELOC_SECTDIFF       | rScattered | rLength4) << 16 |
-         ARM_RELOC_PAIR           | rScattered | rLength4):
-  case ((ARM_RELOC_LOCAL_SECTDIFF | rScattered | rLength4) << 16 |
-         ARM_RELOC_PAIR           | rScattered | rLength4):
-    // ex: .long _foo - .
-    pointerDiff = true;
-    break;
-  default:
-    return llvm::make_error<GenericError>("unsupported arm relocation pair");
-  }
-  const uint8_t *fixupContent = &inAtom->rawContent()[offsetInAtom];
-  uint32_t instruction = *(const ulittle32_t *)fixupContent;
-  uint32_t value;
-  uint32_t fromAddress;
-  uint32_t toAddress;
-  uint16_t instruction16;
-  uint16_t other16;
-  const lld::Atom *fromTarget;
-  Reference::Addend offsetInTo;
-  Reference::Addend offsetInFrom;
-  if (pointerDiff) {
-    toAddress = reloc1.value;
-    fromAddress = reloc2.value;
-    if (auto ec = atomFromAddr(0, toAddress, target, &offsetInTo))
-      return ec;
-    if (auto ec = atomFromAddr(0, fromAddress, &fromTarget, &offsetInFrom))
-      return ec;
-    if (scatterable && (fromTarget != inAtom))
-      return llvm::make_error<GenericError>(
-          "SECTDIFF relocation where subtrahend label is not in atom");
-    *kind = delta32;
-    value = clearThumbBit(instruction, *target);
-    *addend = (int32_t)(value - (toAddress - fixupAddress));
-  } else if (funcRel) {
-    toAddress = reloc1.value;
-    fromAddress = reloc2.value;
-    if (auto ec = atomFromAddr(0, toAddress, target, &offsetInTo))
-      return ec;
-    if (auto ec = atomFromAddr(0, fromAddress, &fromTarget, &offsetInFrom))
-      return ec;
-    if (fromTarget != inAtom)
-      return llvm::make_error<GenericError>("ARM_RELOC_HALF_SECTDIFF relocation"
-                                     " where subtrahend label is not in atom");
-    other16 = (reloc2.offset & 0xFFFF);
-    if (thumbReloc) {
-      if (top) {
-        if (!isThumbMovt(instruction))
-          return llvm::make_error<GenericError>("expected movt instruction");
-      }
-      else {
-        if (!isThumbMovw(instruction))
-          return llvm::make_error<GenericError>("expected movw instruction");
-      }
-      instruction16 = getWordFromThumbMov(instruction);
-    }
-    else {
-      if (top) {
-        if (!isArmMovt(instruction))
-          return llvm::make_error<GenericError>("expected movt instruction");
-      }
-      else {
-        if (!isArmMovw(instruction))
-          return llvm::make_error<GenericError>("expected movw instruction");
-      }
-      instruction16 = getWordFromArmMov(instruction);
-    }
-    if (top)
-      value = (instruction16 << 16) | other16;
-    else
-      value = (other16 << 16) | instruction16;
-    value = clearThumbBit(value, *target);
-    int64_t ta = (int64_t) value - (toAddress - fromAddress);
-    *addend = ta - offsetInFrom;
-    return llvm::Error::success();
-  } else {
-    uint32_t sectIndex;
-    if (thumbReloc) {
-      if (top) {
-        if (!isThumbMovt(instruction))
-          return llvm::make_error<GenericError>("expected movt instruction");
-      }
-      else {
-        if (!isThumbMovw(instruction))
-          return llvm::make_error<GenericError>("expected movw instruction");
-      }
-      instruction16 = getWordFromThumbMov(instruction);
-    }
-    else {
-      if (top) {
-        if (!isArmMovt(instruction))
-          return llvm::make_error<GenericError>("expected movt instruction");
-      }
-      else {
-        if (!isArmMovw(instruction))
-          return llvm::make_error<GenericError>("expected movw instruction");
-      }
-      instruction16 = getWordFromArmMov(instruction);
-    }
-    other16 = (reloc2.offset & 0xFFFF);
-    if (top)
-      value = (instruction16 << 16) | other16;
-    else
-      value = (other16 << 16) | instruction16;
-    if (reloc1.isExtern) {
-      if (auto ec = atomFromSymbolIndex(reloc1.symbol, target))
-        return ec;
-      *addend = value;
-    } else {
-      if (reloc1.scattered) {
-        toAddress = reloc1.value;
-        sectIndex = 0;
-      } else {
-        toAddress = value;
-        sectIndex = reloc1.symbol;
-      }
-      if (auto ec = atomFromAddr(sectIndex, toAddress, target, &offsetInTo))
-        return ec;
-      *addend = value - toAddress;
-    }
-  }
-
-  return llvm::Error::success();
-}
-
-void ArchHandler_arm::applyFixupFinal(const Reference &ref, uint8_t *loc,
-                                      uint64_t fixupAddress,
-                                      uint64_t targetAddress,
-                                      uint64_t inAtomAddress,
-                                      bool &thumbMode, bool targetIsThumb) {
-  if (ref.kindNamespace() != Reference::KindNamespace::mach_o)
-    return;
-  assert(ref.kindArch() == Reference::KindArch::ARM);
-  ulittle32_t *loc32 = reinterpret_cast<ulittle32_t *>(loc);
-  int32_t displacement;
-  uint16_t value16;
-  uint32_t value32;
-  switch (static_cast<ArmKind>(ref.kindValue())) {
-  case modeThumbCode:
-    thumbMode = true;
-    break;
-  case modeArmCode:
-    thumbMode = false;
-    break;
-  case modeData:
-    break;
-  case thumb_b22:
-  case thumb_bl22:
-    assert(thumbMode);
-    displacement = (targetAddress - (fixupAddress + 4)) + ref.addend();
-    value32 = setDisplacementInThumbBranch(*loc32, fixupAddress,
-                                           displacement, targetIsThumb);
-    *loc32 = value32;
-    break;
-  case thumb_movw:
-    assert(thumbMode);
-    value16 = (targetAddress + ref.addend()) & 0xFFFF;
-    if (targetIsThumb)
-      value16 |= 1;
-    *loc32 = setWordFromThumbMov(*loc32, value16);
-    break;
-  case thumb_movt:
-    assert(thumbMode);
-    value16 = (targetAddress + ref.addend()) >> 16;
-    *loc32 = setWordFromThumbMov(*loc32, value16);
-    break;
-  case thumb_movw_funcRel:
-    assert(thumbMode);
-    value16 = (targetAddress - inAtomAddress + ref.addend()) & 0xFFFF;
-    if (targetIsThumb)
-      value16 |= 1;
-    *loc32 = setWordFromThumbMov(*loc32, value16);
-    break;
-  case thumb_movt_funcRel:
-    assert(thumbMode);
-    value16 = (targetAddress - inAtomAddress + ref.addend()) >> 16;
-    *loc32 = setWordFromThumbMov(*loc32, value16);
-    break;
-  case arm_b24:
-  case arm_bl24:
-   assert(!thumbMode);
-    displacement = (targetAddress - (fixupAddress + 8)) + ref.addend();
-    value32 = setDisplacementInArmBranch(*loc32, displacement, targetIsThumb);
-    *loc32 = value32;
-    break;
-  case arm_movw:
-    assert(!thumbMode);
-    value16 = (targetAddress + ref.addend()) & 0xFFFF;
-    if (targetIsThumb)
-      value16 |= 1;
-    *loc32 = setWordFromArmMov(*loc32, value16);
-    break;
-  case arm_movt:
-    assert(!thumbMode);
-    value16 = (targetAddress + ref.addend()) >> 16;
-    *loc32 = setWordFromArmMov(*loc32, value16);
-    break;
-  case arm_movw_funcRel:
-    assert(!thumbMode);
-    value16 = (targetAddress - inAtomAddress + ref.addend()) & 0xFFFF;
-    if (targetIsThumb)
-      value16 |= 1;
-    *loc32 = setWordFromArmMov(*loc32, value16);
-    break;
-  case arm_movt_funcRel:
-    assert(!thumbMode);
-    value16 = (targetAddress - inAtomAddress + ref.addend()) >> 16;
-    *loc32 = setWordFromArmMov(*loc32, value16);
-    break;
-  case pointer32:
-    if (targetIsThumb)
-      *loc32 = targetAddress + ref.addend() + 1;
-    else
-      *loc32 = targetAddress + ref.addend();
-    break;
-  case delta32:
-    if (targetIsThumb)
-      *loc32 = targetAddress - fixupAddress + ref.addend() + 1;
-    else
-      *loc32 = targetAddress - fixupAddress + ref.addend();
-    break;
-  case lazyPointer:
-    // do nothing
-    break;
-  case lazyImmediateLocation:
-    *loc32 = ref.addend();
-    break;
-  case invalid:
-    llvm_unreachable("invalid ARM Reference Kind");
-    break;
-  }
-}
-
-void ArchHandler_arm::generateAtomContent(const DefinedAtom &atom,
-                                          bool relocatable,
-                                          FindAddressForAtom findAddress,
-                                          FindAddressForAtom findSectionAddress,
-                                          uint64_t imageBaseAddress,
-                            llvm::MutableArrayRef<uint8_t> atomContentBuffer) {
-  // Copy raw bytes.
-  std::copy(atom.rawContent().begin(), atom.rawContent().end(),
-            atomContentBuffer.begin());
-  // Apply fix-ups.
-  bool thumbMode = false;
-  for (const Reference *ref : atom) {
-    uint32_t offset = ref->offsetInAtom();
-    const Atom *target = ref->target();
-    uint64_t targetAddress = 0;
-    bool targetIsThumb = false;
-    if (const DefinedAtom *defTarg = dyn_cast<DefinedAtom>(target)) {
-      targetAddress = findAddress(*target);
-      targetIsThumb = isThumbFunction(*defTarg);
-    }
-    uint64_t atomAddress = findAddress(atom);
-    uint64_t fixupAddress = atomAddress + offset;
-    if (relocatable) {
-      applyFixupRelocatable(*ref, &atomContentBuffer[offset], fixupAddress,
-                            targetAddress, atomAddress, thumbMode,
-                            targetIsThumb);
-    } else {
-      applyFixupFinal(*ref, &atomContentBuffer[offset], fixupAddress,
-                      targetAddress, atomAddress, thumbMode, targetIsThumb);
-    }
-  }
-}
-
-bool ArchHandler_arm::useExternalRelocationTo(const Atom &target) {
-  // Undefined symbols are referenced via external relocations.
-  if (isa<UndefinedAtom>(&target))
-    return true;
-  if (const DefinedAtom *defAtom = dyn_cast<DefinedAtom>(&target)) {
-     switch (defAtom->merge()) {
-     case DefinedAtom::mergeAsTentative:
-       // Tentative definitions are referenced via external relocations.
-       return true;
-     case DefinedAtom::mergeAsWeak:
-     case DefinedAtom::mergeAsWeakAndAddressUsed:
-       // Global weak-defs are referenced via external relocations.
-       return (defAtom->scope() == DefinedAtom::scopeGlobal);
-     default:
-       break;
-    }
-  }
-  // Everything else is reference via an internal relocation.
-  return false;
-}
-
-void ArchHandler_arm::applyFixupRelocatable(const Reference &ref, uint8_t *loc,
-                                            uint64_t fixupAddress,
-                                            uint64_t targetAddress,
-                                            uint64_t inAtomAddress,
-                                            bool &thumbMode,
-                                            bool targetIsThumb) {
-  if (ref.kindNamespace() != Reference::KindNamespace::mach_o)
-    return;
-  assert(ref.kindArch() == Reference::KindArch::ARM);
-  bool useExternalReloc = useExternalRelocationTo(*ref.target());
-  ulittle32_t *loc32 = reinterpret_cast<ulittle32_t *>(loc);
-  int32_t displacement;
-  uint16_t value16;
-  uint32_t value32;
-  bool targetIsUndef = isa<UndefinedAtom>(ref.target());
-  switch (static_cast<ArmKind>(ref.kindValue())) {
-  case modeThumbCode:
-    thumbMode = true;
-    break;
-  case modeArmCode:
-    thumbMode = false;
-    break;
-  case modeData:
-    break;
-  case thumb_b22:
-  case thumb_bl22:
-    assert(thumbMode);
-    if (useExternalReloc)
-      displacement = (ref.addend() - (fixupAddress + 4));
-    else
-      displacement = (targetAddress - (fixupAddress + 4)) + ref.addend();
-    value32 = setDisplacementInThumbBranch(*loc32, fixupAddress,
-                                           displacement,
-                                           targetIsUndef || targetIsThumb);
-    *loc32 = value32;
-    break;
-  case thumb_movw:
-    assert(thumbMode);
-    if (useExternalReloc)
-      value16 = ref.addend() & 0xFFFF;
-    else
-      value16 = (targetAddress + ref.addend()) & 0xFFFF;
-    *loc32 = setWordFromThumbMov(*loc32, value16);
-    break;
-  case thumb_movt:
-    assert(thumbMode);
-    if (useExternalReloc)
-      value16 = ref.addend() >> 16;
-    else
-      value16 = (targetAddress + ref.addend()) >> 16;
-    *loc32 = setWordFromThumbMov(*loc32, value16);
-    break;
-  case thumb_movw_funcRel:
-    assert(thumbMode);
-    value16 = (targetAddress - inAtomAddress + ref.addend()) & 0xFFFF;
-    *loc32 = setWordFromThumbMov(*loc32, value16);
-    break;
-  case thumb_movt_funcRel:
-    assert(thumbMode);
-    value16 = (targetAddress - inAtomAddress + ref.addend()) >> 16;
-    *loc32 = setWordFromThumbMov(*loc32, value16);
-    break;
-  case arm_b24:
-  case arm_bl24:
-    assert(!thumbMode);
-    if (useExternalReloc)
-      displacement = (ref.addend() - (fixupAddress + 8));
-    else
-      displacement = (targetAddress - (fixupAddress + 8)) + ref.addend();
-    value32 = setDisplacementInArmBranch(*loc32, displacement,
-                                         targetIsThumb);
-    *loc32 = value32;
-    break;
-  case arm_movw:
-    assert(!thumbMode);
-    if (useExternalReloc)
-      value16 = ref.addend() & 0xFFFF;
-    else
-      value16 = (targetAddress + ref.addend()) & 0xFFFF;
-    *loc32 = setWordFromArmMov(*loc32, value16);
-    break;
-  case arm_movt:
-    assert(!thumbMode);
-    if (useExternalReloc)
-      value16 = ref.addend() >> 16;
-    else
-      value16 = (targetAddress + ref.addend()) >> 16;
-    *loc32 = setWordFromArmMov(*loc32, value16);
-    break;
-  case arm_movw_funcRel:
-    assert(!thumbMode);
-    value16 = (targetAddress - inAtomAddress + ref.addend()) & 0xFFFF;
-    *loc32 = setWordFromArmMov(*loc32, value16);
-    break;
-  case arm_movt_funcRel:
-    assert(!thumbMode);
-    value16 = (targetAddress - inAtomAddress + ref.addend()) >> 16;
-    *loc32 = setWordFromArmMov(*loc32, value16);
-    break;
-  case pointer32:
-    *loc32 = targetAddress + ref.addend();
-    break;
-  case delta32:
-    *loc32 = targetAddress - fixupAddress + ref.addend();
-    break;
-  case lazyPointer:
-  case lazyImmediateLocation:
-    // do nothing
-    break;
-  case invalid:
-    llvm_unreachable("invalid ARM Reference Kind");
-    break;
-  }
-}
-
-void ArchHandler_arm::appendSectionRelocations(
-                                   const DefinedAtom &atom,
-                                   uint64_t atomSectionOffset,
-                                   const Reference &ref,
-                                   FindSymbolIndexForAtom symbolIndexForAtom,
-                                   FindSectionIndexForAtom sectionIndexForAtom,
-                                   FindAddressForAtom addressForAtom,
-                                   normalized::Relocations &relocs) {
-  if (ref.kindNamespace() != Reference::KindNamespace::mach_o)
-    return;
-  assert(ref.kindArch() == Reference::KindArch::ARM);
-  uint32_t sectionOffset = atomSectionOffset + ref.offsetInAtom();
-  bool useExternalReloc = useExternalRelocationTo(*ref.target());
-  uint32_t targetAtomAddress;
-  uint32_t fromAtomAddress;
-  uint16_t other16;
-  switch (static_cast<ArmKind>(ref.kindValue())) {
-  case modeThumbCode:
-  case modeArmCode:
-  case modeData:
-    // Do nothing.
-    break;
-  case thumb_b22:
-  case thumb_bl22:
-    if (useExternalReloc) {
-      appendReloc(relocs, sectionOffset, symbolIndexForAtom(*ref.target()), 0,
-                  ARM_THUMB_RELOC_BR22 | rExtern    | rPcRel | rLength4);
-    } else {
-      if (ref.addend() != 0)
-        appendReloc(relocs, sectionOffset, 0, addressForAtom(*ref.target()),
-                  ARM_THUMB_RELOC_BR22 | rScattered | rPcRel | rLength4);
-      else
-        appendReloc(relocs, sectionOffset, sectionIndexForAtom(*ref.target()),0,
-                  ARM_THUMB_RELOC_BR22 |              rPcRel | rLength4);
-    }
-    break;
-  case thumb_movw:
-    if (useExternalReloc) {
-      other16 = ref.addend() >> 16;
-      appendReloc(relocs, sectionOffset, symbolIndexForAtom(*ref.target()), 0,
-                  ARM_RELOC_HALF | rExtern    | rLenThmbLo);
-      appendReloc(relocs, other16, 0, 0,
-                  ARM_RELOC_PAIR              | rLenThmbLo);
-    } else {
-      targetAtomAddress = addressForAtom(*ref.target());
-      if (ref.addend() != 0) {
-        other16 = (targetAtomAddress + ref.addend()) >> 16;
-        appendReloc(relocs, sectionOffset, 0, targetAtomAddress,
-                  ARM_RELOC_HALF | rScattered | rLenThmbLo);
-        appendReloc(relocs, other16, 0, 0,
-                  ARM_RELOC_PAIR              | rLenThmbLo);
-      } else {
-        other16 = (targetAtomAddress + ref.addend()) >> 16;
-        appendReloc(relocs, sectionOffset, sectionIndexForAtom(*ref.target()),0,
-                  ARM_RELOC_HALF              | rLenThmbLo);
-        appendReloc(relocs, other16, 0, 0,
-                  ARM_RELOC_PAIR              | rLenThmbLo);
-      }
-    }
-    break;
-  case thumb_movt:
-    if (useExternalReloc) {
-      other16 = ref.addend() & 0xFFFF;
-      appendReloc(relocs, sectionOffset, symbolIndexForAtom(*ref.target()), 0,
-                  ARM_RELOC_HALF | rExtern    | rLenThmbHi);
-      appendReloc(relocs, other16, 0, 0,
-                  ARM_RELOC_PAIR              | rLenThmbHi);
-    } else {
-      targetAtomAddress = addressForAtom(*ref.target());
-      if (ref.addend() != 0) {
-        other16 = (targetAtomAddress + ref.addend()) & 0xFFFF;
-        appendReloc(relocs, sectionOffset, 0, targetAtomAddress,
-                    ARM_RELOC_HALF | rScattered | rLenThmbHi);
-        appendReloc(relocs, other16, 0, 0,
-                    ARM_RELOC_PAIR              | rLenThmbHi);
-      } else {
-        other16 = (targetAtomAddress + ref.addend()) & 0xFFFF;
-        appendReloc(relocs, sectionOffset, sectionIndexForAtom(*ref.target()),0,
-                    ARM_RELOC_HALF              | rLenThmbHi);
-        appendReloc(relocs, other16, 0, 0,
-                    ARM_RELOC_PAIR              | rLenThmbHi);
-      }
-    }
-    break;
-  case thumb_movw_funcRel:
-    fromAtomAddress = addressForAtom(atom);
-    targetAtomAddress = addressForAtom(*ref.target());
-    other16 = (targetAtomAddress - fromAtomAddress + ref.addend()) >> 16;
-    appendReloc(relocs, sectionOffset, 0, targetAtomAddress,
-                ARM_RELOC_HALF_SECTDIFF | rScattered | rLenThmbLo);
-    appendReloc(relocs, other16, 0, fromAtomAddress,
-                ARM_RELOC_PAIR          | rScattered | rLenThmbLo);
-    break;
-  case thumb_movt_funcRel:
-    fromAtomAddress = addressForAtom(atom);
-    targetAtomAddress = addressForAtom(*ref.target());
-    other16 = (targetAtomAddress - fromAtomAddress + ref.addend()) & 0xFFFF;
-    appendReloc(relocs, sectionOffset, 0, targetAtomAddress,
-                ARM_RELOC_HALF_SECTDIFF | rScattered | rLenThmbHi);
-    appendReloc(relocs, other16, 0, fromAtomAddress,
-                ARM_RELOC_PAIR          | rScattered | rLenThmbHi);
-    break;
-  case arm_b24:
-  case arm_bl24:
-    if (useExternalReloc) {
-      appendReloc(relocs, sectionOffset, symbolIndexForAtom(*ref.target()), 0,
-                  ARM_RELOC_BR24 | rExtern    | rPcRel | rLength4);
-    } else {
-      if (ref.addend() != 0)
-        appendReloc(relocs, sectionOffset, 0, addressForAtom(*ref.target()),
-                  ARM_RELOC_BR24 | rScattered | rPcRel | rLength4);
-      else
-        appendReloc(relocs, sectionOffset, sectionIndexForAtom(*ref.target()),0,
-                  ARM_RELOC_BR24 |              rPcRel | rLength4);
-    }
-    break;
-  case arm_movw:
-    if (useExternalReloc) {
-      other16 = ref.addend() >> 16;
-      appendReloc(relocs, sectionOffset, symbolIndexForAtom(*ref.target()), 0,
-                  ARM_RELOC_HALF | rExtern    | rLenArmLo);
-      appendReloc(relocs, other16, 0, 0,
-                  ARM_RELOC_PAIR              | rLenArmLo);
-    } else {
-      targetAtomAddress = addressForAtom(*ref.target());
-      if (ref.addend() != 0) {
-        other16 = (targetAtomAddress + ref.addend()) >> 16;
-        appendReloc(relocs, sectionOffset, 0, targetAtomAddress,
-                  ARM_RELOC_HALF | rScattered | rLenArmLo);
-        appendReloc(relocs, other16, 0, 0,
-                  ARM_RELOC_PAIR              | rLenArmLo);
-      } else {
-        other16 = (targetAtomAddress + ref.addend()) >> 16;
-        appendReloc(relocs, sectionOffset, sectionIndexForAtom(*ref.target()),0,
-                  ARM_RELOC_HALF              | rLenArmLo);
-        appendReloc(relocs, other16, 0, 0,
-                  ARM_RELOC_PAIR              | rLenArmLo);
-      }
-    }
-    break;
-  case arm_movt:
-    if (useExternalReloc) {
-      other16 = ref.addend() & 0xFFFF;
-      appendReloc(relocs, sectionOffset, symbolIndexForAtom(*ref.target()), 0,
-                  ARM_RELOC_HALF | rExtern    | rLenArmHi);
-      appendReloc(relocs, other16, 0, 0,
-                  ARM_RELOC_PAIR              | rLenArmHi);
-    } else {
-      targetAtomAddress = addressForAtom(*ref.target());
-      if (ref.addend() != 0) {
-        other16 = (targetAtomAddress + ref.addend()) & 0xFFFF;
-        appendReloc(relocs, sectionOffset, 0, targetAtomAddress,
-                  ARM_RELOC_HALF | rScattered | rLenArmHi);
-        appendReloc(relocs, other16, 0, 0,
-                  ARM_RELOC_PAIR              | rLenArmHi);
-      } else {
-        other16 = (targetAtomAddress + ref.addend()) & 0xFFFF;
-        appendReloc(relocs, sectionOffset, sectionIndexForAtom(*ref.target()),0,
-                  ARM_RELOC_HALF              | rLenArmHi);
-        appendReloc(relocs, other16, 0, 0,
-                  ARM_RELOC_PAIR              | rLenArmHi);
-      }
-    }
-    break;
-  case arm_movw_funcRel:
-    fromAtomAddress = addressForAtom(atom);
-    targetAtomAddress = addressForAtom(*ref.target());
-    other16 = (targetAtomAddress - fromAtomAddress + ref.addend()) >> 16;
-    appendReloc(relocs, sectionOffset, 0, targetAtomAddress,
-                ARM_RELOC_HALF_SECTDIFF | rScattered | rLenArmLo);
-    appendReloc(relocs, other16, 0, fromAtomAddress,
-                ARM_RELOC_PAIR          | rScattered | rLenArmLo);
-    break;
-  case arm_movt_funcRel:
-    fromAtomAddress = addressForAtom(atom);
-    targetAtomAddress = addressForAtom(*ref.target());
-    other16 = (targetAtomAddress - fromAtomAddress + ref.addend()) & 0xFFFF;
-    appendReloc(relocs, sectionOffset, 0, targetAtomAddress,
-                ARM_RELOC_HALF_SECTDIFF | rScattered | rLenArmHi);
-    appendReloc(relocs, other16, 0, fromAtomAddress,
-                ARM_RELOC_PAIR          | rScattered | rLenArmHi);
-    break;
-  case pointer32:
-    if (useExternalReloc) {
-      appendReloc(relocs, sectionOffset, symbolIndexForAtom(*ref.target()),  0,
-                ARM_RELOC_VANILLA |    rExtern     |  rLength4);
-    }
-    else {
-      if (ref.addend() != 0)
-        appendReloc(relocs, sectionOffset, 0, addressForAtom(*ref.target()),
-                ARM_RELOC_VANILLA |    rScattered  |  rLength4);
-      else
-        appendReloc(relocs, sectionOffset, sectionIndexForAtom(*ref.target()),0,
-                ARM_RELOC_VANILLA |                   rLength4);
-    }
-    break;
-  case delta32:
-    appendReloc(relocs, sectionOffset, 0, addressForAtom(*ref.target()),
-              ARM_RELOC_SECTDIFF  |  rScattered    | rLength4);
-    appendReloc(relocs, sectionOffset, 0, addressForAtom(atom) +
-                                                           ref.offsetInAtom(),
-              ARM_RELOC_PAIR      |  rScattered    | rLength4);
-    break;
-  case lazyPointer:
-  case lazyImmediateLocation:
-    // do nothing
-    break;
-  case invalid:
-    llvm_unreachable("invalid ARM Reference Kind");
-    break;
-  }
-}
-
-void ArchHandler_arm::addAdditionalReferences(MachODefinedAtom &atom) {
-  if (atom.isThumb()) {
-    atom.addReference(Reference::KindNamespace::mach_o,
-                      Reference::KindArch::ARM, modeThumbCode, 0, &atom, 0);
-  }
-}
-
-bool ArchHandler_arm::isThumbFunction(const DefinedAtom &atom) {
-  for (const Reference *ref : atom) {
-    if (ref->offsetInAtom() != 0)
-      return false;
-    if (ref->kindNamespace() != Reference::KindNamespace::mach_o)
-      continue;
-    assert(ref->kindArch() == Reference::KindArch::ARM);
-    if (ref->kindValue() == modeThumbCode)
-      return true;
-  }
-  return false;
-}
-
-class Thumb2ToArmShimAtom : public SimpleDefinedAtom {
-public:
-  Thumb2ToArmShimAtom(MachOFile &file, StringRef targetName,
-                      const DefinedAtom &target)
-      : SimpleDefinedAtom(file) {
-    addReference(Reference::KindNamespace::mach_o, Reference::KindArch::ARM,
-                 ArchHandler_arm::modeThumbCode, 0, this, 0);
-    addReference(Reference::KindNamespace::mach_o, Reference::KindArch::ARM,
-                 ArchHandler_arm::delta32, 8, &target, 0);
-    std::string name = std::string(targetName) + "$shim";
-    StringRef tmp(name);
-    _name = tmp.copy(file.allocator());
-  }
-
-  ~Thumb2ToArmShimAtom() override = default;
-
-  StringRef name() const override {
-    return _name;
-  }
-
-  ContentType contentType() const override {
-    return DefinedAtom::typeCode;
-  }
-
-  Alignment alignment() const override { return 4; }
-
-  uint64_t size() const override {
-    return 12;
-  }
-
-  ContentPermissions permissions() const override {
-    return DefinedAtom::permR_X;
-  }
-
-  ArrayRef<uint8_t> rawContent() const override {
-    static const uint8_t bytes[] =
-    { 0xDF, 0xF8, 0x04, 0xC0,       //  ldr ip, pc + 4
-      0xFF, 0x44,                   //  add ip, pc, ip
-      0x60, 0x47,                   //  ldr pc, [ip]
-      0x00, 0x00, 0x00, 0x00 };     //  .long target - this
-    assert(sizeof(bytes) == size());
-    return llvm::makeArrayRef(bytes, sizeof(bytes));
-  }
-private:
-  StringRef _name;
-};
-
-class ArmToThumbShimAtom : public SimpleDefinedAtom {
-public:
-  ArmToThumbShimAtom(MachOFile &file, StringRef targetName,
-                     const DefinedAtom &target)
-      : SimpleDefinedAtom(file) {
-    addReference(Reference::KindNamespace::mach_o, Reference::KindArch::ARM,
-                 ArchHandler_arm::delta32, 12, &target, 0);
-    std::string name = std::string(targetName) + "$shim";
-    StringRef tmp(name);
-    _name = tmp.copy(file.allocator());
-  }
-
-  ~ArmToThumbShimAtom() override = default;
-
-  StringRef name() const override {
-    return _name;
-  }
-
-  ContentType contentType() const override {
-    return DefinedAtom::typeCode;
-  }
-
-  Alignment alignment() const override { return 4; }
-
-  uint64_t size() const override {
-    return 16;
-  }
-
-  ContentPermissions permissions() const override {
-    return DefinedAtom::permR_X;
-  }
-
-  ArrayRef<uint8_t> rawContent() const override {
-    static const uint8_t bytes[] =
-    { 0x04, 0xC0, 0x9F, 0xE5,       //  ldr ip, pc + 4
-      0x0C, 0xC0, 0x8F, 0xE0,       //  add ip, pc, ip
-      0x1C, 0xFF, 0x2F, 0xE1,       //  ldr pc, [ip]
-      0x00, 0x00, 0x00, 0x00 };     //  .long target - this
-    assert(sizeof(bytes) == size());
-    return llvm::makeArrayRef(bytes, sizeof(bytes));
-  }
-private:
-  StringRef _name;
-};
-
-const DefinedAtom *ArchHandler_arm::createShim(MachOFile &file,
-                                               bool thumbToArm,
-                                               const DefinedAtom &target) {
-  bool isStub = (target.contentType() == DefinedAtom::typeStub);
-  StringRef targetName = isStub ? stubName(target) : target.name();
-  if (thumbToArm)
-    return new (file.allocator()) Thumb2ToArmShimAtom(file, targetName, target);
-  else
-    return new (file.allocator()) ArmToThumbShimAtom(file, targetName, target);
-}
-
-std::unique_ptr<mach_o::ArchHandler> ArchHandler::create_arm() {
-  return std::unique_ptr<mach_o::ArchHandler>(new ArchHandler_arm());
-}
-
-} // namespace mach_o
-} // namespace lld
diff --git a/lld/lib/ReaderWriter/MachO/ArchHandler_arm64.cpp b/lld/lib/ReaderWriter/MachO/ArchHandler_arm64.cpp
deleted file mode 100644
index bee081aec067..000000000000
--- a/lld/lib/ReaderWriter/MachO/ArchHandler_arm64.cpp
+++ /dev/null
@@ -1,897 +0,0 @@
-//===- lib/FileFormat/MachO/ArchHandler_arm64.cpp -------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "ArchHandler.h"
-#include "Atoms.h"
-#include "MachONormalizedFileBinaryUtils.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/ADT/StringSwitch.h"
-#include "llvm/ADT/Triple.h"
-#include "llvm/Support/Endian.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/Format.h"
-
-using namespace llvm::MachO;
-using namespace lld::mach_o::normalized;
-
-namespace lld {
-namespace mach_o {
-
-using llvm::support::ulittle32_t;
-using llvm::support::ulittle64_t;
-
-using llvm::support::little32_t;
-using llvm::support::little64_t;
-
-class ArchHandler_arm64 : public ArchHandler {
-public:
-  ArchHandler_arm64() = default;
-  ~ArchHandler_arm64() override = default;
-
-  const Registry::KindStrings *kindStrings() override { return _sKindStrings; }
-
-  Reference::KindArch kindArch() override {
-    return Reference::KindArch::AArch64;
-  }
-
-  /// Used by GOTPass to locate GOT References
-  bool isGOTAccess(const Reference &ref, bool &canBypassGOT) override {
-    if (ref.kindNamespace() != Reference::KindNamespace::mach_o)
-      return false;
-    assert(ref.kindArch() == Reference::KindArch::AArch64);
-    switch (ref.kindValue()) {
-    case gotPage21:
-    case gotOffset12:
-      canBypassGOT = true;
-      return true;
-    case delta32ToGOT:
-    case unwindCIEToPersonalityFunction:
-    case imageOffsetGot:
-      canBypassGOT = false;
-      return true;
-    default:
-      return false;
-    }
-  }
-
-  /// Used by GOTPass to update GOT References.
-  void updateReferenceToGOT(const Reference *ref, bool targetNowGOT) override {
-    // If GOT slot was instantiated, transform:
-    //   gotPage21/gotOffset12 -> page21/offset12scale8
-    // If GOT slot optimized away, transform:
-    //   gotPage21/gotOffset12 -> page21/addOffset12
-    assert(ref->kindNamespace() == Reference::KindNamespace::mach_o);
-    assert(ref->kindArch() == Reference::KindArch::AArch64);
-    switch (ref->kindValue()) {
-    case gotPage21:
-      const_cast<Reference *>(ref)->setKindValue(page21);
-      break;
-    case gotOffset12:
-      const_cast<Reference *>(ref)->setKindValue(targetNowGOT ?
-                                                 offset12scale8 : addOffset12);
-      break;
-    case delta32ToGOT:
-      const_cast<Reference *>(ref)->setKindValue(delta32);
-      break;
-    case imageOffsetGot:
-      const_cast<Reference *>(ref)->setKindValue(imageOffset);
-      break;
-    default:
-      llvm_unreachable("Not a GOT reference");
-    }
-  }
-
-  const StubInfo &stubInfo() override { return _sStubInfo; }
-
-  bool isCallSite(const Reference &) override;
-  bool isNonCallBranch(const Reference &) override {
-    return false;
-  }
-
-  bool isPointer(const Reference &) override;
-  bool isPairedReloc(const normalized::Relocation &) override;
-
-  bool needsCompactUnwind() override {
-    return true;
-  }
-  Reference::KindValue imageOffsetKind() override {
-    return imageOffset;
-  }
-  Reference::KindValue imageOffsetKindIndirect() override {
-    return imageOffsetGot;
-  }
-
-  Reference::KindValue unwindRefToPersonalityFunctionKind() override {
-    return unwindCIEToPersonalityFunction;
-  }
-
-  Reference::KindValue unwindRefToCIEKind() override {
-    return negDelta32;
-  }
-
-  Reference::KindValue unwindRefToFunctionKind() override {
-    return unwindFDEToFunction;
-  }
-
-  Reference::KindValue unwindRefToEhFrameKind() override {
-    return unwindInfoToEhFrame;
-  }
-
-  Reference::KindValue pointerKind() override {
-    return pointer64;
-  }
-
-  Reference::KindValue lazyImmediateLocationKind() override {
-    return lazyImmediateLocation;
-  }
-
-  uint32_t dwarfCompactUnwindType() override {
-    return 0x03000000;
-  }
-
-  llvm::Error getReferenceInfo(const normalized::Relocation &reloc,
-                               const DefinedAtom *inAtom,
-                               uint32_t offsetInAtom,
-                               uint64_t fixupAddress, bool isBig,
-                               FindAtomBySectionAndAddress atomFromAddress,
-                               FindAtomBySymbolIndex atomFromSymbolIndex,
-                               Reference::KindValue *kind,
-                               const lld::Atom **target,
-                               Reference::Addend *addend) override;
-  llvm::Error
-      getPairReferenceInfo(const normalized::Relocation &reloc1,
-                           const normalized::Relocation &reloc2,
-                           const DefinedAtom *inAtom,
-                           uint32_t offsetInAtom,
-                           uint64_t fixupAddress, bool isBig, bool scatterable,
-                           FindAtomBySectionAndAddress atomFromAddress,
-                           FindAtomBySymbolIndex atomFromSymbolIndex,
-                           Reference::KindValue *kind,
-                           const lld::Atom **target,
-                           Reference::Addend *addend) override;
-
-  bool needsLocalSymbolInRelocatableFile(const DefinedAtom *atom) override {
-    return (atom->contentType() == DefinedAtom::typeCString);
-  }
-
-  void generateAtomContent(const DefinedAtom &atom, bool relocatable,
-                           FindAddressForAtom findAddress,
-                           FindAddressForAtom findSectionAddress,
-                           uint64_t imageBaseAddress,
-                    llvm::MutableArrayRef<uint8_t> atomContentBuffer) override;
-
-  void appendSectionRelocations(const DefinedAtom &atom,
-                                uint64_t atomSectionOffset,
-                                const Reference &ref,
-                                FindSymbolIndexForAtom symbolIndexForAtom,
-                                FindSectionIndexForAtom sectionIndexForAtom,
-                                FindAddressForAtom addressForAtom,
-                                normalized::Relocations &relocs) override;
-
-private:
-  static const Registry::KindStrings _sKindStrings[];
-  static const StubInfo _sStubInfo;
-
-  enum Arm64Kind : Reference::KindValue {
-    invalid,               /// for error condition
-
-    // Kinds found in mach-o .o files:
-    branch26,              /// ex: bl   _foo
-    page21,                /// ex: adrp x1, _foo@PAGE
-    offset12,              /// ex: ldrb w0, [x1, _foo@PAGEOFF]
-    offset12scale2,        /// ex: ldrs w0, [x1, _foo@PAGEOFF]
-    offset12scale4,        /// ex: ldr  w0, [x1, _foo@PAGEOFF]
-    offset12scale8,        /// ex: ldr  x0, [x1, _foo@PAGEOFF]
-    offset12scale16,       /// ex: ldr  q0, [x1, _foo@PAGEOFF]
-    gotPage21,             /// ex: adrp x1, _foo@GOTPAGE
-    gotOffset12,           /// ex: ldr  w0, [x1, _foo@GOTPAGEOFF]
-    tlvPage21,             /// ex: adrp x1, _foo@TLVPAGE
-    tlvOffset12,           /// ex: ldr  w0, [x1, _foo@TLVPAGEOFF]
-
-    pointer64,             /// ex: .quad _foo
-    delta64,               /// ex: .quad _foo - .
-    delta32,               /// ex: .long _foo - .
-    negDelta32,            /// ex: .long . - _foo
-    pointer64ToGOT,        /// ex: .quad _foo@GOT
-    delta32ToGOT,          /// ex: .long _foo@GOT - .
-
-    // Kinds introduced by Passes:
-    addOffset12,           /// Location contains LDR to change into ADD.
-    lazyPointer,           /// Location contains a lazy pointer.
-    lazyImmediateLocation, /// Location contains immediate value used in stub.
-    imageOffset,           /// Location contains offset of atom in final image
-    imageOffsetGot,        /// Location contains offset of GOT entry for atom in
-                           /// final image (typically personality function).
-    unwindCIEToPersonalityFunction,   /// Nearly delta32ToGOT, but cannot be
-                           /// rematerialized in relocatable object
-                           /// (yay for implicit contracts!).
-    unwindFDEToFunction,   /// Nearly delta64, but cannot be rematerialized in
-                           /// relocatable object (yay for implicit contracts!).
-    unwindInfoToEhFrame,   /// Fix low 24 bits of compact unwind encoding to
-                           /// refer to __eh_frame entry.
-  };
-
-  void applyFixupFinal(const Reference &ref, uint8_t *location,
-                       uint64_t fixupAddress, uint64_t targetAddress,
-                       uint64_t inAtomAddress, uint64_t imageBaseAddress,
-                       FindAddressForAtom findSectionAddress);
-
-  void applyFixupRelocatable(const Reference &ref, uint8_t *location,
-                             uint64_t fixupAddress, uint64_t targetAddress,
-                             uint64_t inAtomAddress, bool targetUnnamed);
-
-  // Utility functions for inspecting/updating instructions.
-  static uint32_t setDisplacementInBranch26(uint32_t instr, int32_t disp);
-  static uint32_t setDisplacementInADRP(uint32_t instr, int64_t disp);
-  static Arm64Kind offset12KindFromInstruction(uint32_t instr);
-  static uint32_t setImm12(uint32_t instr, uint32_t offset);
-};
-
-const Registry::KindStrings ArchHandler_arm64::_sKindStrings[] = {
-  LLD_KIND_STRING_ENTRY(invalid),
-  LLD_KIND_STRING_ENTRY(branch26),
-  LLD_KIND_STRING_ENTRY(page21),
-  LLD_KIND_STRING_ENTRY(offset12),
-  LLD_KIND_STRING_ENTRY(offset12scale2),
-  LLD_KIND_STRING_ENTRY(offset12scale4),
-  LLD_KIND_STRING_ENTRY(offset12scale8),
-  LLD_KIND_STRING_ENTRY(offset12scale16),
-  LLD_KIND_STRING_ENTRY(gotPage21),
-  LLD_KIND_STRING_ENTRY(gotOffset12),
-  LLD_KIND_STRING_ENTRY(tlvPage21),
-  LLD_KIND_STRING_ENTRY(tlvOffset12),
-  LLD_KIND_STRING_ENTRY(pointer64),
-  LLD_KIND_STRING_ENTRY(delta64),
-  LLD_KIND_STRING_ENTRY(delta32),
-  LLD_KIND_STRING_ENTRY(negDelta32),
-  LLD_KIND_STRING_ENTRY(pointer64ToGOT),
-  LLD_KIND_STRING_ENTRY(delta32ToGOT),
-
-  LLD_KIND_STRING_ENTRY(addOffset12),
-  LLD_KIND_STRING_ENTRY(lazyPointer),
-  LLD_KIND_STRING_ENTRY(lazyImmediateLocation),
-  LLD_KIND_STRING_ENTRY(imageOffset),
-  LLD_KIND_STRING_ENTRY(imageOffsetGot),
-  LLD_KIND_STRING_ENTRY(unwindCIEToPersonalityFunction),
-  LLD_KIND_STRING_ENTRY(unwindFDEToFunction),
-  LLD_KIND_STRING_ENTRY(unwindInfoToEhFrame),
-
-  LLD_KIND_STRING_END
-};
-
-const ArchHandler::StubInfo ArchHandler_arm64::_sStubInfo = {
-  "dyld_stub_binder",
-
-  // Lazy pointer references
-  { Reference::KindArch::AArch64, pointer64, 0, 0 },
-  { Reference::KindArch::AArch64, lazyPointer, 0, 0 },
-
-  // GOT pointer to dyld_stub_binder
-  { Reference::KindArch::AArch64, pointer64, 0, 0 },
-
-  // arm64 code alignment 2^1
-  1,
-
-  // Stub size and code
-  12,
-  { 0x10, 0x00, 0x00, 0x90,   // ADRP  X16, lazy_pointer@page
-    0x10, 0x02, 0x40, 0xF9,   // LDR   X16, [X16, lazy_pointer@pageoff]
-    0x00, 0x02, 0x1F, 0xD6 }, // BR    X16
-  { Reference::KindArch::AArch64, page21, 0, 0 },
-  { true,                         offset12scale8, 4, 0 },
-
-  // Stub Helper size and code
-  12,
-  { 0x50, 0x00, 0x00, 0x18,   //      LDR   W16, L0
-    0x00, 0x00, 0x00, 0x14,   //      LDR   B  helperhelper
-    0x00, 0x00, 0x00, 0x00 }, // L0: .long 0
-  { Reference::KindArch::AArch64, lazyImmediateLocation, 8, 0 },
-  { Reference::KindArch::AArch64, branch26, 4, 0 },
-
-  // Stub helper image cache content type
-  DefinedAtom::typeGOT,
-
-  // Stub Helper-Common size and code
-  24,
-  // Stub helper alignment
-  2,
-  { 0x11, 0x00, 0x00, 0x90,   //  ADRP  X17, dyld_ImageLoaderCache@page
-    0x31, 0x02, 0x00, 0x91,   //  ADD   X17, X17, dyld_ImageLoaderCache@pageoff
-    0xF0, 0x47, 0xBF, 0xA9,   //  STP   X16/X17, [SP, #-16]!
-    0x10, 0x00, 0x00, 0x90,   //  ADRP  X16, _fast_lazy_bind@page
-    0x10, 0x02, 0x40, 0xF9,   //  LDR   X16, [X16,_fast_lazy_bind@pageoff]
-    0x00, 0x02, 0x1F, 0xD6 }, //  BR    X16
-  { Reference::KindArch::AArch64, page21,   0, 0 },
-  { true,                         offset12, 4, 0 },
-  { Reference::KindArch::AArch64, page21,   12, 0 },
-  { true,                         offset12scale8, 16, 0 }
-};
-
-bool ArchHandler_arm64::isCallSite(const Reference &ref) {
-  if (ref.kindNamespace() != Reference::KindNamespace::mach_o)
-    return false;
-  assert(ref.kindArch() == Reference::KindArch::AArch64);
-  return (ref.kindValue() == branch26);
-}
-
-bool ArchHandler_arm64::isPointer(const Reference &ref) {
-  if (ref.kindNamespace() != Reference::KindNamespace::mach_o)
-    return false;
-  assert(ref.kindArch() == Reference::KindArch::AArch64);
-  Reference::KindValue kind = ref.kindValue();
-  return (kind == pointer64);
-}
-
-bool ArchHandler_arm64::isPairedReloc(const Relocation &r) {
-  return ((r.type == ARM64_RELOC_ADDEND) || (r.type == ARM64_RELOC_SUBTRACTOR));
-}
-
-uint32_t ArchHandler_arm64::setDisplacementInBranch26(uint32_t instr,
-                                                      int32_t displacement) {
-  assert((displacement <= 134217727) && (displacement > (-134217728)) &&
-         "arm64 branch out of range");
-  return (instr & 0xFC000000) | ((uint32_t)(displacement >> 2) & 0x03FFFFFF);
-}
-
-uint32_t ArchHandler_arm64::setDisplacementInADRP(uint32_t instruction,
-                                                  int64_t displacement) {
-  assert((displacement <= 0x100000000LL) && (displacement > (-0x100000000LL)) &&
-         "arm64 ADRP out of range");
-  assert(((instruction & 0x9F000000) == 0x90000000) &&
-         "reloc not on ADRP instruction");
-  uint32_t immhi = (displacement >> 9) & (0x00FFFFE0);
-  uint32_t immlo = (displacement << 17) & (0x60000000);
-  return (instruction & 0x9F00001F) | immlo | immhi;
-}
-
-ArchHandler_arm64::Arm64Kind
-ArchHandler_arm64::offset12KindFromInstruction(uint32_t instruction) {
-  if (instruction & 0x08000000) {
-    switch ((instruction >> 30) & 0x3) {
-    case 0:
-      if ((instruction & 0x04800000) == 0x04800000)
-        return offset12scale16;
-      return offset12;
-    case 1:
-      return offset12scale2;
-    case 2:
-      return offset12scale4;
-    case 3:
-      return offset12scale8;
-    }
-  }
-  return offset12;
-}
-
-uint32_t ArchHandler_arm64::setImm12(uint32_t instruction, uint32_t offset) {
-  assert(((offset & 0xFFFFF000) == 0) && "imm12 offset out of range");
-  uint32_t imm12 = offset << 10;
-  return (instruction & 0xFFC003FF) | imm12;
-}
-
-llvm::Error ArchHandler_arm64::getReferenceInfo(
-    const Relocation &reloc, const DefinedAtom *inAtom, uint32_t offsetInAtom,
-    uint64_t fixupAddress, bool isBig,
-    FindAtomBySectionAndAddress atomFromAddress,
-    FindAtomBySymbolIndex atomFromSymbolIndex, Reference::KindValue *kind,
-    const lld::Atom **target, Reference::Addend *addend) {
-  const uint8_t *fixupContent = &inAtom->rawContent()[offsetInAtom];
-  switch (relocPattern(reloc)) {
-  case ARM64_RELOC_BRANCH26           | rPcRel | rExtern | rLength4:
-    // ex: bl _foo
-    *kind = branch26;
-    if (auto ec = atomFromSymbolIndex(reloc.symbol, target))
-      return ec;
-    *addend = 0;
-    return llvm::Error::success();
-  case ARM64_RELOC_PAGE21             | rPcRel | rExtern | rLength4:
-    // ex: adrp x1, _foo@PAGE
-    *kind = page21;
-    if (auto ec = atomFromSymbolIndex(reloc.symbol, target))
-      return ec;
-    *addend = 0;
-    return llvm::Error::success();
-  case ARM64_RELOC_PAGEOFF12                   | rExtern | rLength4:
-    // ex: ldr x0, [x1, _foo@PAGEOFF]
-    *kind = offset12KindFromInstruction(*(const little32_t *)fixupContent);
-    if (auto ec = atomFromSymbolIndex(reloc.symbol, target))
-      return ec;
-    *addend = 0;
-    return llvm::Error::success();
-  case ARM64_RELOC_GOT_LOAD_PAGE21    | rPcRel | rExtern | rLength4:
-    // ex: adrp x1, _foo@GOTPAGE
-    *kind = gotPage21;
-    if (auto ec = atomFromSymbolIndex(reloc.symbol, target))
-      return ec;
-    *addend = 0;
-    return llvm::Error::success();
-  case ARM64_RELOC_GOT_LOAD_PAGEOFF12          | rExtern | rLength4:
-    // ex: ldr x0, [x1, _foo@GOTPAGEOFF]
-    *kind = gotOffset12;
-    if (auto ec = atomFromSymbolIndex(reloc.symbol, target))
-      return ec;
-    *addend = 0;
-    return llvm::Error::success();
-  case ARM64_RELOC_TLVP_LOAD_PAGE21   | rPcRel | rExtern | rLength4:
-    // ex: adrp x1, _foo@TLVPAGE
-    *kind = tlvPage21;
-    if (auto ec = atomFromSymbolIndex(reloc.symbol, target))
-      return ec;
-    *addend = 0;
-    return llvm::Error::success();
-  case ARM64_RELOC_TLVP_LOAD_PAGEOFF12         | rExtern | rLength4:
-    // ex: ldr x0, [x1, _foo@TLVPAGEOFF]
-    *kind = tlvOffset12;
-    if (auto ec = atomFromSymbolIndex(reloc.symbol, target))
-      return ec;
-    *addend = 0;
-    return llvm::Error::success();
-  case ARM64_RELOC_UNSIGNED                    | rExtern | rLength8:
-    // ex: .quad _foo + N
-    *kind = pointer64;
-    if (auto ec = atomFromSymbolIndex(reloc.symbol, target))
-      return ec;
-    *addend = *(const little64_t *)fixupContent;
-    return llvm::Error::success();
-  case ARM64_RELOC_UNSIGNED                              | rLength8:
-     // ex: .quad Lfoo + N
-     *kind = pointer64;
-     return atomFromAddress(reloc.symbol, *(const little64_t *)fixupContent,
-                            target, addend);
-  case ARM64_RELOC_POINTER_TO_GOT              | rExtern | rLength8:
-    // ex: .quad _foo@GOT
-    *kind = pointer64ToGOT;
-    if (auto ec = atomFromSymbolIndex(reloc.symbol, target))
-      return ec;
-    *addend = 0;
-    return llvm::Error::success();
-  case ARM64_RELOC_POINTER_TO_GOT     | rPcRel | rExtern | rLength4:
-    // ex: .long _foo@GOT - .
-
-    // If we are in an .eh_frame section, then the kind of the relocation should
-    // not be delta32ToGOT.  It may instead be unwindCIEToPersonalityFunction.
-    if (inAtom->contentType() == DefinedAtom::typeCFI)
-      *kind = unwindCIEToPersonalityFunction;
-    else
-      *kind = delta32ToGOT;
-
-    if (auto ec = atomFromSymbolIndex(reloc.symbol, target))
-      return ec;
-    *addend = 0;
-    return llvm::Error::success();
-  default:
-    return llvm::make_error<GenericError>("unsupported arm64 relocation type");
-  }
-}
-
-llvm::Error ArchHandler_arm64::getPairReferenceInfo(
-    const normalized::Relocation &reloc1, const normalized::Relocation &reloc2,
-    const DefinedAtom *inAtom, uint32_t offsetInAtom, uint64_t fixupAddress,
-    bool swap, bool scatterable, FindAtomBySectionAndAddress atomFromAddress,
-    FindAtomBySymbolIndex atomFromSymbolIndex, Reference::KindValue *kind,
-    const lld::Atom **target, Reference::Addend *addend) {
-  const uint8_t *fixupContent = &inAtom->rawContent()[offsetInAtom];
-  switch (relocPattern(reloc1) << 16 | relocPattern(reloc2)) {
-  case ((ARM64_RELOC_ADDEND                                | rLength4) << 16 |
-         ARM64_RELOC_BRANCH26           | rPcRel | rExtern | rLength4):
-    // ex: bl _foo+8
-    *kind = branch26;
-    if (auto ec = atomFromSymbolIndex(reloc2.symbol, target))
-      return ec;
-    *addend = reloc1.symbol;
-    return llvm::Error::success();
-  case ((ARM64_RELOC_ADDEND                                | rLength4) << 16 |
-         ARM64_RELOC_PAGE21             | rPcRel | rExtern | rLength4):
-    // ex: adrp x1, _foo@PAGE
-    *kind = page21;
-    if (auto ec = atomFromSymbolIndex(reloc2.symbol, target))
-      return ec;
-    *addend = reloc1.symbol;
-    return llvm::Error::success();
-  case ((ARM64_RELOC_ADDEND                                | rLength4) << 16 |
-         ARM64_RELOC_PAGEOFF12                   | rExtern | rLength4): {
-    // ex: ldr w0, [x1, _foo@PAGEOFF]
-    uint32_t cont32 = (int32_t)*(const little32_t *)fixupContent;
-    *kind = offset12KindFromInstruction(cont32);
-    if (auto ec = atomFromSymbolIndex(reloc2.symbol, target))
-      return ec;
-    *addend = reloc1.symbol;
-    return llvm::Error::success();
-  }
-  case ((ARM64_RELOC_SUBTRACTOR                  | rExtern | rLength8) << 16 |
-         ARM64_RELOC_UNSIGNED                    | rExtern | rLength8):
-    // ex: .quad _foo - .
-    if (auto ec = atomFromSymbolIndex(reloc2.symbol, target))
-      return ec;
-
-    // If we are in an .eh_frame section, then the kind of the relocation should
-    // not be delta64.  It may instead be unwindFDEToFunction.
-    if (inAtom->contentType() == DefinedAtom::typeCFI)
-      *kind = unwindFDEToFunction;
-    else
-      *kind = delta64;
-
-    // The offsets of the 2 relocations must match
-    if (reloc1.offset != reloc2.offset)
-      return llvm::make_error<GenericError>(
-                                    "paired relocs must have the same offset");
-    *addend = (int64_t)*(const little64_t *)fixupContent + offsetInAtom;
-    return llvm::Error::success();
-  case ((ARM64_RELOC_SUBTRACTOR                  | rExtern | rLength4) << 16 |
-         ARM64_RELOC_UNSIGNED                    | rExtern | rLength4):
-    // ex: .quad _foo - .
-    *kind = delta32;
-    if (auto ec = atomFromSymbolIndex(reloc2.symbol, target))
-      return ec;
-    *addend = (int32_t)*(const little32_t *)fixupContent + offsetInAtom;
-    return llvm::Error::success();
-  default:
-    return llvm::make_error<GenericError>("unsupported arm64 relocation pair");
-  }
-}
-
-void ArchHandler_arm64::generateAtomContent(
-    const DefinedAtom &atom, bool relocatable, FindAddressForAtom findAddress,
-    FindAddressForAtom findSectionAddress, uint64_t imageBaseAddress,
-    llvm::MutableArrayRef<uint8_t> atomContentBuffer) {
-  // Copy raw bytes.
-  std::copy(atom.rawContent().begin(), atom.rawContent().end(),
-            atomContentBuffer.begin());
-  // Apply fix-ups.
-#ifndef NDEBUG
-  if (atom.begin() != atom.end()) {
-    DEBUG_WITH_TYPE("atom-content", llvm::dbgs()
-                    << "Applying fixups to atom:\n"
-                    << "   address="
-                    << llvm::format("    0x%09lX", &atom)
-                    << ", file=#"
-                    << atom.file().ordinal()
-                    << ", atom=#"
-                    << atom.ordinal()
-                    << ", name="
-                    << atom.name()
-                    << ", type="
-                    << atom.contentType()
-                    << "\n");
-  }
-#endif
-  for (const Reference *ref : atom) {
-    uint32_t offset = ref->offsetInAtom();
-    const Atom *target = ref->target();
-    bool targetUnnamed = target->name().empty();
-    uint64_t targetAddress = 0;
-    if (isa<DefinedAtom>(target))
-      targetAddress = findAddress(*target);
-    uint64_t atomAddress = findAddress(atom);
-    uint64_t fixupAddress = atomAddress + offset;
-    if (relocatable) {
-      applyFixupRelocatable(*ref, &atomContentBuffer[offset], fixupAddress,
-                            targetAddress, atomAddress, targetUnnamed);
-    } else {
-      applyFixupFinal(*ref, &atomContentBuffer[offset], fixupAddress,
-                      targetAddress, atomAddress, imageBaseAddress,
-                      findSectionAddress);
-    }
-  }
-}
-
-void ArchHandler_arm64::applyFixupFinal(const Reference &ref, uint8_t *loc,
-                                        uint64_t fixupAddress,
-                                        uint64_t targetAddress,
-                                        uint64_t inAtomAddress,
-                                        uint64_t imageBaseAddress,
-                                        FindAddressForAtom findSectionAddress) {
-  if (ref.kindNamespace() != Reference::KindNamespace::mach_o)
-    return;
-  assert(ref.kindArch() == Reference::KindArch::AArch64);
-  ulittle32_t *loc32 = reinterpret_cast<ulittle32_t *>(loc);
-  ulittle64_t *loc64 = reinterpret_cast<ulittle64_t *>(loc);
-  int32_t displacement;
-  uint32_t instruction;
-  uint32_t value32;
-  uint32_t value64;
-  switch (static_cast<Arm64Kind>(ref.kindValue())) {
-  case branch26:
-    displacement = (targetAddress - fixupAddress) + ref.addend();
-    *loc32 = setDisplacementInBranch26(*loc32, displacement);
-    return;
-  case page21:
-  case gotPage21:
-  case tlvPage21:
-    displacement =
-        ((targetAddress + ref.addend()) & (-4096)) - (fixupAddress & (-4096));
-    *loc32 = setDisplacementInADRP(*loc32, displacement);
-    return;
-  case offset12:
-  case gotOffset12:
-  case tlvOffset12:
-    displacement = (targetAddress + ref.addend()) & 0x00000FFF;
-    *loc32 = setImm12(*loc32, displacement);
-    return;
-  case offset12scale2:
-    displacement = (targetAddress + ref.addend()) & 0x00000FFF;
-    assert(((displacement & 0x1) == 0) &&
-           "scaled imm12 not accessing 2-byte aligneds");
-    *loc32 = setImm12(*loc32, displacement >> 1);
-    return;
-  case offset12scale4:
-    displacement = (targetAddress + ref.addend()) & 0x00000FFF;
-    assert(((displacement & 0x3) == 0) &&
-           "scaled imm12 not accessing 4-byte aligned");
-    *loc32 = setImm12(*loc32, displacement >> 2);
-    return;
-  case offset12scale8:
-    displacement = (targetAddress + ref.addend()) & 0x00000FFF;
-    assert(((displacement & 0x7) == 0) &&
-           "scaled imm12 not accessing 8-byte aligned");
-    *loc32 = setImm12(*loc32, displacement >> 3);
-    return;
-  case offset12scale16:
-    displacement = (targetAddress + ref.addend()) & 0x00000FFF;
-    assert(((displacement & 0xF) == 0) &&
-           "scaled imm12 not accessing 16-byte aligned");
-    *loc32 = setImm12(*loc32, displacement >> 4);
-    return;
-  case addOffset12:
-    instruction = *loc32;
-    assert(((instruction & 0xFFC00000) == 0xF9400000) &&
-           "GOT reloc is not an LDR instruction");
-    displacement = (targetAddress + ref.addend()) & 0x00000FFF;
-    value32 = 0x91000000 | (instruction & 0x000003FF);
-    instruction = setImm12(value32, displacement);
-    *loc32 = instruction;
-    return;
-  case pointer64:
-  case pointer64ToGOT:
-    *loc64 = targetAddress + ref.addend();
-    return;
-  case delta64:
-  case unwindFDEToFunction:
-    *loc64 = (targetAddress - fixupAddress) + ref.addend();
-    return;
-  case delta32:
-  case delta32ToGOT:
-  case unwindCIEToPersonalityFunction:
-    *loc32 = (targetAddress - fixupAddress) + ref.addend();
-    return;
-  case negDelta32:
-    *loc32 = fixupAddress - targetAddress + ref.addend();
-    return;
-  case lazyPointer:
-    // Do nothing
-    return;
-  case lazyImmediateLocation:
-    *loc32 = ref.addend();
-    return;
-  case imageOffset:
-    *loc32 = (targetAddress - imageBaseAddress) + ref.addend();
-    return;
-  case imageOffsetGot:
-    llvm_unreachable("imageOffsetGot should have been changed to imageOffset");
-    break;
-  case unwindInfoToEhFrame:
-    value64 = targetAddress - findSectionAddress(*ref.target()) + ref.addend();
-    assert(value64 < 0xffffffU && "offset in __eh_frame too large");
-    *loc32 = (*loc32 & 0xff000000U) | value64;
-    return;
-  case invalid:
-    // Fall into llvm_unreachable().
-    break;
-  }
-  llvm_unreachable("invalid arm64 Reference Kind");
-}
-
-void ArchHandler_arm64::applyFixupRelocatable(const Reference &ref,
-                                              uint8_t *loc,
-                                              uint64_t fixupAddress,
-                                              uint64_t targetAddress,
-                                              uint64_t inAtomAddress,
-                                              bool targetUnnamed) {
-  if (ref.kindNamespace() != Reference::KindNamespace::mach_o)
-    return;
-  assert(ref.kindArch() == Reference::KindArch::AArch64);
-  ulittle32_t *loc32 = reinterpret_cast<ulittle32_t *>(loc);
-  ulittle64_t *loc64 = reinterpret_cast<ulittle64_t *>(loc);
-  switch (static_cast<Arm64Kind>(ref.kindValue())) {
-  case branch26:
-    *loc32 = setDisplacementInBranch26(*loc32, 0);
-    return;
-  case page21:
-  case gotPage21:
-  case tlvPage21:
-    *loc32 = setDisplacementInADRP(*loc32, 0);
-    return;
-  case offset12:
-  case offset12scale2:
-  case offset12scale4:
-  case offset12scale8:
-  case offset12scale16:
-  case gotOffset12:
-  case tlvOffset12:
-    *loc32 = setImm12(*loc32, 0);
-    return;
-  case pointer64:
-    if (targetUnnamed)
-      *loc64 = targetAddress + ref.addend();
-    else
-      *loc64 = ref.addend();
-    return;
-  case delta64:
-    *loc64 = ref.addend() + inAtomAddress - fixupAddress;
-    return;
-  case unwindFDEToFunction:
-    // We don't emit unwindFDEToFunction in -r mode as they are implicitly
-    // generated from the data in the __eh_frame section.  So here we need
-    // to use the targetAddress so that we can generate the full relocation
-    // when we parse again later.
-    *loc64 = targetAddress - fixupAddress;
-    return;
-  case delta32:
-    *loc32 = ref.addend() + inAtomAddress - fixupAddress;
-    return;
-  case negDelta32:
-    // We don't emit negDelta32 in -r mode as they are implicitly
-    // generated from the data in the __eh_frame section.  So here we need
-    // to use the targetAddress so that we can generate the full relocation
-    // when we parse again later.
-    *loc32 = fixupAddress - targetAddress + ref.addend();
-    return;
-  case pointer64ToGOT:
-    *loc64 = 0;
-    return;
-  case delta32ToGOT:
-    *loc32 = inAtomAddress - fixupAddress;
-    return;
-  case unwindCIEToPersonalityFunction:
-    // We don't emit unwindCIEToPersonalityFunction in -r mode as they are
-    // implicitly generated from the data in the __eh_frame section.  So here we
-    // need to use the targetAddress so that we can generate the full relocation
-    // when we parse again later.
-    *loc32 = targetAddress - fixupAddress;
-    return;
-  case addOffset12:
-    llvm_unreachable("lazy reference kind implies GOT pass was run");
-  case lazyPointer:
-  case lazyImmediateLocation:
-    llvm_unreachable("lazy reference kind implies Stubs pass was run");
-  case imageOffset:
-  case imageOffsetGot:
-  case unwindInfoToEhFrame:
-    llvm_unreachable("fixup implies __unwind_info");
-    return;
-  case invalid:
-    // Fall into llvm_unreachable().
-    break;
-  }
-  llvm_unreachable("unknown arm64 Reference Kind");
-}
-
-void ArchHandler_arm64::appendSectionRelocations(
-    const DefinedAtom &atom, uint64_t atomSectionOffset, const Reference &ref,
-    FindSymbolIndexForAtom symbolIndexForAtom,
-    FindSectionIndexForAtom sectionIndexForAtom,
-    FindAddressForAtom addressForAtom, normalized::Relocations &relocs) {
-  if (ref.kindNamespace() != Reference::KindNamespace::mach_o)
-    return;
-  assert(ref.kindArch() == Reference::KindArch::AArch64);
-  uint32_t sectionOffset = atomSectionOffset + ref.offsetInAtom();
-  switch (static_cast<Arm64Kind>(ref.kindValue())) {
-  case branch26:
-    if (ref.addend()) {
-      appendReloc(relocs, sectionOffset, ref.addend(), 0,
-                  ARM64_RELOC_ADDEND | rLength4);
-      appendReloc(relocs, sectionOffset, symbolIndexForAtom(*ref.target()), 0,
-                  ARM64_RELOC_BRANCH26 | rPcRel | rExtern | rLength4);
-     } else {
-      appendReloc(relocs, sectionOffset, symbolIndexForAtom(*ref.target()), 0,
-                  ARM64_RELOC_BRANCH26 | rPcRel | rExtern | rLength4);
-    }
-    return;
-  case page21:
-    if (ref.addend()) {
-      appendReloc(relocs, sectionOffset, ref.addend(), 0,
-                  ARM64_RELOC_ADDEND | rLength4);
-      appendReloc(relocs, sectionOffset, symbolIndexForAtom(*ref.target()), 0,
-                  ARM64_RELOC_PAGE21 | rPcRel | rExtern | rLength4);
-     } else {
-      appendReloc(relocs, sectionOffset, symbolIndexForAtom(*ref.target()), 0,
-                  ARM64_RELOC_PAGE21 | rPcRel | rExtern | rLength4);
-    }
-    return;
-  case offset12:
-  case offset12scale2:
-  case offset12scale4:
-  case offset12scale8:
-  case offset12scale16:
-    if (ref.addend()) {
-      appendReloc(relocs, sectionOffset, ref.addend(), 0,
-                  ARM64_RELOC_ADDEND | rLength4);
-      appendReloc(relocs, sectionOffset, symbolIndexForAtom(*ref.target()), 0,
-                  ARM64_RELOC_PAGEOFF12  | rExtern | rLength4);
-     } else {
-      appendReloc(relocs, sectionOffset, symbolIndexForAtom(*ref.target()), 0,
-                  ARM64_RELOC_PAGEOFF12 | rExtern | rLength4);
-    }
-    return;
-  case gotPage21:
-    assert(ref.addend() == 0);
-    appendReloc(relocs, sectionOffset, symbolIndexForAtom(*ref.target()), 0,
-                  ARM64_RELOC_GOT_LOAD_PAGE21 | rPcRel | rExtern | rLength4);
-    return;
-  case gotOffset12:
-    assert(ref.addend() == 0);
-    appendReloc(relocs, sectionOffset, symbolIndexForAtom(*ref.target()), 0,
-                  ARM64_RELOC_GOT_LOAD_PAGEOFF12 | rExtern | rLength4);
-    return;
-  case tlvPage21:
-    assert(ref.addend() == 0);
-    appendReloc(relocs, sectionOffset, symbolIndexForAtom(*ref.target()), 0,
-                  ARM64_RELOC_TLVP_LOAD_PAGE21 | rPcRel | rExtern | rLength4);
-    return;
-  case tlvOffset12:
-    assert(ref.addend() == 0);
-    appendReloc(relocs, sectionOffset, symbolIndexForAtom(*ref.target()), 0,
-                  ARM64_RELOC_TLVP_LOAD_PAGEOFF12 | rExtern | rLength4);
-    return;
-  case pointer64:
-    if (ref.target()->name().empty())
-      appendReloc(relocs, sectionOffset, sectionIndexForAtom(*ref.target()), 0,
-                  ARM64_RELOC_UNSIGNED           | rLength8);
-    else
-      appendReloc(relocs, sectionOffset, symbolIndexForAtom(*ref.target()), 0,
-                  ARM64_RELOC_UNSIGNED | rExtern | rLength8);
-    return;
-  case delta64:
-    appendReloc(relocs, sectionOffset, symbolIndexForAtom(atom), 0,
-                ARM64_RELOC_SUBTRACTOR | rExtern | rLength8);
-    appendReloc(relocs, sectionOffset, symbolIndexForAtom(*ref.target()), 0,
-                ARM64_RELOC_UNSIGNED  | rExtern | rLength8);
-    return;
-  case delta32:
-    appendReloc(relocs, sectionOffset, symbolIndexForAtom(atom), 0,
-                ARM64_RELOC_SUBTRACTOR | rExtern | rLength4 );
-    appendReloc(relocs, sectionOffset, symbolIndexForAtom(*ref.target()), 0,
-                ARM64_RELOC_UNSIGNED   | rExtern | rLength4 );
-    return;
-  case pointer64ToGOT:
-    assert(ref.addend() == 0);
-    appendReloc(relocs, sectionOffset, symbolIndexForAtom(*ref.target()), 0,
-                  ARM64_RELOC_POINTER_TO_GOT | rExtern | rLength8);
-    return;
-  case delta32ToGOT:
-    assert(ref.addend() == 0);
-    appendReloc(relocs, sectionOffset, symbolIndexForAtom(*ref.target()), 0,
-                  ARM64_RELOC_POINTER_TO_GOT | rPcRel | rExtern | rLength4);
-    return;
-  case addOffset12:
-    llvm_unreachable("lazy reference kind implies GOT pass was run");
-  case lazyPointer:
-  case lazyImmediateLocation:
-    llvm_unreachable("lazy reference kind implies Stubs pass was run");
-  case imageOffset:
-  case imageOffsetGot:
-    llvm_unreachable("deltas from mach_header can only be in final images");
-  case unwindCIEToPersonalityFunction:
-  case unwindFDEToFunction:
-  case unwindInfoToEhFrame:
-  case negDelta32:
-    // Do nothing.
-    return;
-  case invalid:
-    // Fall into llvm_unreachable().
-    break;
-  }
-  llvm_unreachable("unknown arm64 Reference Kind");
-}
-
-std::unique_ptr<mach_o::ArchHandler> ArchHandler::create_arm64() {
-  return std::unique_ptr<mach_o::ArchHandler>(new ArchHandler_arm64());
-}
-
-} // namespace mach_o
-} // namespace lld
diff --git a/lld/lib/ReaderWriter/MachO/ArchHandler_x86.cpp b/lld/lib/ReaderWriter/MachO/ArchHandler_x86.cpp
deleted file mode 100644
index 6ea8e8c42e80..000000000000
--- a/lld/lib/ReaderWriter/MachO/ArchHandler_x86.cpp
+++ /dev/null
@@ -1,643 +0,0 @@
-//===- lib/FileFormat/MachO/ArchHandler_x86.cpp ---------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "ArchHandler.h"
-#include "Atoms.h"
-#include "MachONormalizedFileBinaryUtils.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/ADT/StringSwitch.h"
-#include "llvm/ADT/Triple.h"
-#include "llvm/Support/Endian.h"
-#include "llvm/Support/ErrorHandling.h"
-
-using namespace llvm::MachO;
-using namespace lld::mach_o::normalized;
-
-namespace lld {
-namespace mach_o {
-
-using llvm::support::ulittle16_t;
-using llvm::support::ulittle32_t;
-
-using llvm::support::little16_t;
-using llvm::support::little32_t;
-
-class ArchHandler_x86 : public ArchHandler {
-public:
-  ArchHandler_x86() = default;
-  ~ArchHandler_x86() override = default;
-
-  const Registry::KindStrings *kindStrings() override { return _sKindStrings; }
-
-  Reference::KindArch kindArch() override { return Reference::KindArch::x86; }
-
-  const StubInfo &stubInfo() override { return _sStubInfo; }
-  bool isCallSite(const Reference &) override;
-  bool isNonCallBranch(const Reference &) override {
-    return false;
-  }
-
-  bool isPointer(const Reference &) override;
-  bool isPairedReloc(const normalized::Relocation &) override;
-
-  bool needsCompactUnwind() override {
-    return false;
-  }
-
-  Reference::KindValue imageOffsetKind() override {
-    return invalid;
-  }
-
-  Reference::KindValue imageOffsetKindIndirect() override {
-    return invalid;
-  }
-
-  Reference::KindValue unwindRefToPersonalityFunctionKind() override {
-    return invalid;
-  }
-
-  Reference::KindValue unwindRefToCIEKind() override {
-    return negDelta32;
-  }
-
-  Reference::KindValue unwindRefToFunctionKind() override{
-    return delta32;
-  }
-
-  Reference::KindValue lazyImmediateLocationKind() override {
-    return lazyImmediateLocation;
-  }
-
-  Reference::KindValue unwindRefToEhFrameKind() override {
-    return invalid;
-  }
-
-  Reference::KindValue pointerKind() override {
-    return invalid;
-  }
-
-  uint32_t dwarfCompactUnwindType() override {
-    return 0x04000000U;
-  }
-
-  llvm::Error getReferenceInfo(const normalized::Relocation &reloc,
-                               const DefinedAtom *inAtom,
-                               uint32_t offsetInAtom,
-                               uint64_t fixupAddress, bool swap,
-                               FindAtomBySectionAndAddress atomFromAddress,
-                               FindAtomBySymbolIndex atomFromSymbolIndex,
-                               Reference::KindValue *kind,
-                               const lld::Atom **target,
-                               Reference::Addend *addend) override;
-  llvm::Error
-      getPairReferenceInfo(const normalized::Relocation &reloc1,
-                           const normalized::Relocation &reloc2,
-                           const DefinedAtom *inAtom,
-                           uint32_t offsetInAtom,
-                           uint64_t fixupAddress, bool swap, bool scatterable,
-                           FindAtomBySectionAndAddress atomFromAddress,
-                           FindAtomBySymbolIndex atomFromSymbolIndex,
-                           Reference::KindValue *kind,
-                           const lld::Atom **target,
-                           Reference::Addend *addend) override;
-
-  void generateAtomContent(const DefinedAtom &atom, bool relocatable,
-                           FindAddressForAtom findAddress,
-                           FindAddressForAtom findSectionAddress,
-                           uint64_t imageBaseAddress,
-                    llvm::MutableArrayRef<uint8_t> atomContentBuffer) override;
-
-  void appendSectionRelocations(const DefinedAtom &atom,
-                                uint64_t atomSectionOffset,
-                                const Reference &ref,
-                                FindSymbolIndexForAtom symbolIndexForAtom,
-                                FindSectionIndexForAtom sectionIndexForAtom,
-                                FindAddressForAtom addressForAtom,
-                                normalized::Relocations &relocs) override;
-
-  bool isDataInCodeTransition(Reference::KindValue refKind) override {
-    return refKind == modeCode || refKind == modeData;
-  }
-
-  Reference::KindValue dataInCodeTransitionStart(
-                                        const MachODefinedAtom &atom) override {
-    return modeData;
-  }
-
-  Reference::KindValue dataInCodeTransitionEnd(
-                                        const MachODefinedAtom &atom) override {
-    return modeCode;
-  }
-
-private:
-  static const Registry::KindStrings _sKindStrings[];
-  static const StubInfo              _sStubInfo;
-
-  enum X86Kind : Reference::KindValue {
-    invalid,               /// for error condition
-
-    modeCode,              /// Content starting at this offset is code.
-    modeData,              /// Content starting at this offset is data.
-
-    // Kinds found in mach-o .o files:
-    branch32,              /// ex: call _foo
-    branch16,              /// ex: callw _foo
-    abs32,                 /// ex: movl _foo, %eax
-    funcRel32,             /// ex: movl _foo-L1(%eax), %eax
-    pointer32,             /// ex: .long _foo
-    delta32,               /// ex: .long _foo - .
-    negDelta32,            /// ex: .long . - _foo
-
-    // Kinds introduced by Passes:
-    lazyPointer,           /// Location contains a lazy pointer.
-    lazyImmediateLocation, /// Location contains immediate value used in stub.
-  };
-
-  static bool useExternalRelocationTo(const Atom &target);
-
-  void applyFixupFinal(const Reference &ref, uint8_t *location,
-                       uint64_t fixupAddress, uint64_t targetAddress,
-                       uint64_t inAtomAddress);
-
-  void applyFixupRelocatable(const Reference &ref, uint8_t *location,
-                             uint64_t fixupAddress,
-                             uint64_t targetAddress,
-                             uint64_t inAtomAddress);
-};
-
-//===----------------------------------------------------------------------===//
-//  ArchHandler_x86
-//===----------------------------------------------------------------------===//
-
-const Registry::KindStrings ArchHandler_x86::_sKindStrings[] = {
-  LLD_KIND_STRING_ENTRY(invalid),
-  LLD_KIND_STRING_ENTRY(modeCode),
-  LLD_KIND_STRING_ENTRY(modeData),
-  LLD_KIND_STRING_ENTRY(branch32),
-  LLD_KIND_STRING_ENTRY(branch16),
-  LLD_KIND_STRING_ENTRY(abs32),
-  LLD_KIND_STRING_ENTRY(funcRel32),
-  LLD_KIND_STRING_ENTRY(pointer32),
-  LLD_KIND_STRING_ENTRY(delta32),
-  LLD_KIND_STRING_ENTRY(negDelta32),
-  LLD_KIND_STRING_ENTRY(lazyPointer),
-  LLD_KIND_STRING_ENTRY(lazyImmediateLocation),
-  LLD_KIND_STRING_END
-};
-
-const ArchHandler::StubInfo ArchHandler_x86::_sStubInfo = {
-  "dyld_stub_binder",
-
-  // Lazy pointer references
-  { Reference::KindArch::x86, pointer32, 0, 0 },
-  { Reference::KindArch::x86, lazyPointer, 0, 0 },
-
-  // GOT pointer to dyld_stub_binder
-  { Reference::KindArch::x86, pointer32, 0, 0 },
-
-  // x86 code alignment
-  1,
-
-  // Stub size and code
-  6,
-  { 0xff, 0x25, 0x00, 0x00, 0x00, 0x00 },       // jmp *lazyPointer
-  { Reference::KindArch::x86, abs32, 2, 0 },
-  { false, 0, 0, 0 },
-
-  // Stub Helper size and code
-  10,
-  { 0x68, 0x00, 0x00, 0x00, 0x00,               // pushl $lazy-info-offset
-    0xE9, 0x00, 0x00, 0x00, 0x00 },             // jmp helperhelper
-  { Reference::KindArch::x86, lazyImmediateLocation, 1, 0 },
-  { Reference::KindArch::x86, branch32, 6, 0 },
-
-  // Stub helper image cache content type
-  DefinedAtom::typeNonLazyPointer,
-
-  // Stub Helper-Common size and code
-  12,
-  // Stub helper alignment
-  2,
-  { 0x68, 0x00, 0x00, 0x00, 0x00,               // pushl $dyld_ImageLoaderCache
-    0xFF, 0x25, 0x00, 0x00, 0x00, 0x00,         // jmp *_fast_lazy_bind
-    0x90 },                                     // nop
-  { Reference::KindArch::x86, abs32, 1, 0 },
-  { false, 0, 0, 0 },
-  { Reference::KindArch::x86, abs32, 7, 0 },
-  { false, 0, 0, 0 }
-};
-
-bool ArchHandler_x86::isCallSite(const Reference &ref) {
-  return (ref.kindValue() == branch32);
-}
-
-bool ArchHandler_x86::isPointer(const Reference &ref) {
-  return (ref.kindValue() == pointer32);
-}
-
-bool ArchHandler_x86::isPairedReloc(const Relocation &reloc) {
-  if (!reloc.scattered)
-    return false;
-  return (reloc.type == GENERIC_RELOC_LOCAL_SECTDIFF) ||
-         (reloc.type == GENERIC_RELOC_SECTDIFF);
-}
-
-llvm::Error
-ArchHandler_x86::getReferenceInfo(const Relocation &reloc,
-                                  const DefinedAtom *inAtom,
-                                  uint32_t offsetInAtom,
-                                  uint64_t fixupAddress, bool swap,
-                                  FindAtomBySectionAndAddress atomFromAddress,
-                                  FindAtomBySymbolIndex atomFromSymbolIndex,
-                                  Reference::KindValue *kind,
-                                  const lld::Atom **target,
-                                  Reference::Addend *addend) {
-  DefinedAtom::ContentPermissions perms;
-  const uint8_t *fixupContent = &inAtom->rawContent()[offsetInAtom];
-  uint64_t targetAddress;
-  switch (relocPattern(reloc)) {
-  case GENERIC_RELOC_VANILLA | rPcRel | rExtern | rLength4:
-    // ex: call _foo (and _foo undefined)
-    *kind = branch32;
-    if (auto ec = atomFromSymbolIndex(reloc.symbol, target))
-      return ec;
-    *addend = fixupAddress + 4 + (int32_t)*(const little32_t *)fixupContent;
-    break;
-  case GENERIC_RELOC_VANILLA | rPcRel | rLength4:
-    // ex: call _foo (and _foo defined)
-    *kind = branch32;
-    targetAddress =
-        fixupAddress + 4 + (int32_t) * (const little32_t *)fixupContent;
-    return atomFromAddress(reloc.symbol, targetAddress, target, addend);
-    break;
-  case GENERIC_RELOC_VANILLA | rScattered | rPcRel | rLength4:
-    // ex: call _foo+n (and _foo defined)
-    *kind = branch32;
-    targetAddress =
-        fixupAddress + 4 + (int32_t) * (const little32_t *)fixupContent;
-    if (auto ec = atomFromAddress(0, reloc.value, target, addend))
-      return ec;
-    *addend = targetAddress - reloc.value;
-    break;
-  case GENERIC_RELOC_VANILLA | rPcRel | rExtern | rLength2:
-    // ex: callw _foo (and _foo undefined)
-    *kind = branch16;
-    if (auto ec = atomFromSymbolIndex(reloc.symbol, target))
-      return ec;
-    *addend = fixupAddress + 2 + (int16_t)*(const little16_t *)fixupContent;
-    break;
-  case GENERIC_RELOC_VANILLA | rPcRel | rLength2:
-    // ex: callw _foo (and _foo defined)
-    *kind = branch16;
-    targetAddress =
-        fixupAddress + 2 + (int16_t) * (const little16_t *)fixupContent;
-    return atomFromAddress(reloc.symbol, targetAddress, target, addend);
-    break;
-  case GENERIC_RELOC_VANILLA | rScattered | rPcRel | rLength2:
-    // ex: callw _foo+n (and _foo defined)
-    *kind = branch16;
-    targetAddress =
-        fixupAddress + 2 + (int16_t) * (const little16_t *)fixupContent;
-    if (auto ec = atomFromAddress(0, reloc.value, target, addend))
-      return ec;
-    *addend = targetAddress - reloc.value;
-    break;
-  case GENERIC_RELOC_VANILLA | rExtern | rLength4:
-    // ex: movl	_foo, %eax   (and _foo undefined)
-    // ex: .long _foo        (and _foo undefined)
-    perms = inAtom->permissions();
-    *kind =
-        ((perms & DefinedAtom::permR_X) == DefinedAtom::permR_X) ? abs32
-                                                                 : pointer32;
-    if (auto ec = atomFromSymbolIndex(reloc.symbol, target))
-      return ec;
-    *addend = *(const ulittle32_t *)fixupContent;
-    break;
-  case GENERIC_RELOC_VANILLA | rLength4:
-    // ex: movl	_foo, %eax   (and _foo defined)
-    // ex: .long _foo        (and _foo defined)
-    perms = inAtom->permissions();
-    *kind =
-        ((perms & DefinedAtom::permR_X) == DefinedAtom::permR_X) ? abs32
-                                                                 : pointer32;
-    targetAddress = *(const ulittle32_t *)fixupContent;
-    return atomFromAddress(reloc.symbol, targetAddress, target, addend);
-    break;
-  case GENERIC_RELOC_VANILLA | rScattered | rLength4:
-    // ex: .long _foo+n      (and _foo defined)
-    perms = inAtom->permissions();
-    *kind =
-        ((perms & DefinedAtom::permR_X) == DefinedAtom::permR_X) ? abs32
-                                                                 : pointer32;
-    if (auto ec = atomFromAddress(0, reloc.value, target, addend))
-      return ec;
-    *addend = *(const ulittle32_t *)fixupContent - reloc.value;
-    break;
-  default:
-    return llvm::make_error<GenericError>("unsupported i386 relocation type");
-  }
-  return llvm::Error::success();
-}
-
-llvm::Error
-ArchHandler_x86::getPairReferenceInfo(const normalized::Relocation &reloc1,
-                                      const normalized::Relocation &reloc2,
-                                      const DefinedAtom *inAtom,
-                                      uint32_t offsetInAtom,
-                                      uint64_t fixupAddress, bool swap,
-                                      bool scatterable,
-                                      FindAtomBySectionAndAddress atomFromAddr,
-                                      FindAtomBySymbolIndex atomFromSymbolIndex,
-                                      Reference::KindValue *kind,
-                                      const lld::Atom **target,
-                                      Reference::Addend *addend) {
-  const uint8_t *fixupContent = &inAtom->rawContent()[offsetInAtom];
-  DefinedAtom::ContentPermissions perms = inAtom->permissions();
-  uint32_t fromAddress;
-  uint32_t toAddress;
-  uint32_t value;
-  const lld::Atom *fromTarget;
-  Reference::Addend offsetInTo;
-  Reference::Addend offsetInFrom;
-  switch (relocPattern(reloc1) << 16 | relocPattern(reloc2)) {
-  case ((GENERIC_RELOC_SECTDIFF | rScattered | rLength4) << 16 |
-         GENERIC_RELOC_PAIR | rScattered | rLength4):
-  case ((GENERIC_RELOC_LOCAL_SECTDIFF | rScattered | rLength4) << 16 |
-         GENERIC_RELOC_PAIR | rScattered | rLength4):
-    toAddress = reloc1.value;
-    fromAddress = reloc2.value;
-    value = *(const little32_t *)fixupContent;
-    if (auto ec = atomFromAddr(0, toAddress, target, &offsetInTo))
-      return ec;
-    if (auto ec = atomFromAddr(0, fromAddress, &fromTarget, &offsetInFrom))
-      return ec;
-    if (fromTarget != inAtom) {
-      if (*target != inAtom)
-        return llvm::make_error<GenericError>(
-            "SECTDIFF relocation where neither target is in atom");
-      *kind = negDelta32;
-      *addend = toAddress - value - fromAddress;
-      *target = fromTarget;
-    } else {
-      if ((perms & DefinedAtom::permR_X) == DefinedAtom::permR_X) {
-        // SECTDIFF relocations are used in i386 codegen where the function
-        // prolog does a CALL to the next instruction which POPs the return
-        // address into EBX which becomes the pic-base register.  The POP
-        // instruction is label the used for the subtrahend in expressions.
-        // The funcRel32 kind represents the 32-bit delta to some symbol from
-        // the start of the function (atom) containing the funcRel32.
-        *kind = funcRel32;
-        uint32_t ta = fromAddress + value - toAddress;
-        *addend = ta - offsetInFrom;
-      } else {
-        *kind = delta32;
-        *addend = fromAddress + value - toAddress;
-      }
-    }
-    return llvm::Error::success();
-    break;
-  default:
-    return llvm::make_error<GenericError>("unsupported i386 relocation type");
-  }
-}
-
-void ArchHandler_x86::generateAtomContent(const DefinedAtom &atom,
-                                          bool relocatable,
-                                          FindAddressForAtom findAddress,
-                                          FindAddressForAtom findSectionAddress,
-                                          uint64_t imageBaseAddress,
-                            llvm::MutableArrayRef<uint8_t> atomContentBuffer) {
-  // Copy raw bytes.
-  std::copy(atom.rawContent().begin(), atom.rawContent().end(),
-            atomContentBuffer.begin());
-  // Apply fix-ups.
-  for (const Reference *ref : atom) {
-    uint32_t offset = ref->offsetInAtom();
-    const Atom *target = ref->target();
-    uint64_t targetAddress = 0;
-    if (isa<DefinedAtom>(target))
-      targetAddress = findAddress(*target);
-    uint64_t atomAddress = findAddress(atom);
-    uint64_t fixupAddress = atomAddress + offset;
-    if (relocatable) {
-      applyFixupRelocatable(*ref, &atomContentBuffer[offset],
-                                        fixupAddress, targetAddress,
-                                        atomAddress);
-    } else {
-      applyFixupFinal(*ref, &atomContentBuffer[offset],
-                                  fixupAddress, targetAddress,
-                                  atomAddress);
-    }
-  }
-}
-
-void ArchHandler_x86::applyFixupFinal(const Reference &ref, uint8_t *loc,
-                                      uint64_t fixupAddress,
-                                      uint64_t targetAddress,
-                                      uint64_t inAtomAddress) {
-  if (ref.kindNamespace() != Reference::KindNamespace::mach_o)
-    return;
-  assert(ref.kindArch() == Reference::KindArch::x86);
-  ulittle32_t *loc32 = reinterpret_cast<ulittle32_t *>(loc);
-  switch (static_cast<X86Kind>(ref.kindValue())) {
-  case branch32:
-    *loc32 = (targetAddress - (fixupAddress + 4)) + ref.addend();
-    break;
-  case branch16:
-    *loc32 = (targetAddress - (fixupAddress + 2)) + ref.addend();
-    break;
-  case pointer32:
-  case abs32:
-    *loc32 = targetAddress + ref.addend();
-    break;
-  case funcRel32:
-    *loc32 = targetAddress - inAtomAddress + ref.addend();
-    break;
-  case delta32:
-    *loc32 = targetAddress - fixupAddress + ref.addend();
-    break;
-  case negDelta32:
-    *loc32 = fixupAddress - targetAddress + ref.addend();
-    break;
-  case modeCode:
-  case modeData:
-  case lazyPointer:
-    // do nothing
-    break;
-  case lazyImmediateLocation:
-    *loc32 = ref.addend();
-    break;
-  case invalid:
-    llvm_unreachable("invalid x86 Reference Kind");
-    break;
-  }
-}
-
-void ArchHandler_x86::applyFixupRelocatable(const Reference &ref,
-                                               uint8_t *loc,
-                                               uint64_t fixupAddress,
-                                               uint64_t targetAddress,
-                                               uint64_t inAtomAddress) {
-  if (ref.kindNamespace() != Reference::KindNamespace::mach_o)
-    return;
-  assert(ref.kindArch() == Reference::KindArch::x86);
-  bool useExternalReloc = useExternalRelocationTo(*ref.target());
-  ulittle16_t *loc16 = reinterpret_cast<ulittle16_t *>(loc);
-  ulittle32_t *loc32 = reinterpret_cast<ulittle32_t *>(loc);
-  switch (static_cast<X86Kind>(ref.kindValue())) {
-  case branch32:
-    if (useExternalReloc)
-      *loc32 = ref.addend() - (fixupAddress + 4);
-    else
-      *loc32  =(targetAddress - (fixupAddress+4)) + ref.addend();
-    break;
-  case branch16:
-    if (useExternalReloc)
-      *loc16 = ref.addend() - (fixupAddress + 2);
-    else
-      *loc16 = (targetAddress - (fixupAddress+2)) + ref.addend();
-    break;
-  case pointer32:
-  case abs32:
-    *loc32 = targetAddress + ref.addend();
-    break;
-  case funcRel32:
-    *loc32 = targetAddress - inAtomAddress + ref.addend(); // FIXME
-    break;
-  case delta32:
-    *loc32 = targetAddress - fixupAddress + ref.addend();
-    break;
-  case negDelta32:
-    *loc32 = fixupAddress - targetAddress + ref.addend();
-    break;
-  case modeCode:
-  case modeData:
-  case lazyPointer:
-  case lazyImmediateLocation:
-    // do nothing
-    break;
-  case invalid:
-    llvm_unreachable("invalid x86 Reference Kind");
-    break;
-  }
-}
-
-bool ArchHandler_x86::useExternalRelocationTo(const Atom &target) {
-  // Undefined symbols are referenced via external relocations.
-  if (isa<UndefinedAtom>(&target))
-    return true;
-  if (const DefinedAtom *defAtom = dyn_cast<DefinedAtom>(&target)) {
-     switch (defAtom->merge()) {
-     case DefinedAtom::mergeAsTentative:
-       // Tentative definitions are referenced via external relocations.
-       return true;
-     case DefinedAtom::mergeAsWeak:
-     case DefinedAtom::mergeAsWeakAndAddressUsed:
-       // Global weak-defs are referenced via external relocations.
-       return (defAtom->scope() == DefinedAtom::scopeGlobal);
-     default:
-       break;
-    }
-  }
-  // Everything else is reference via an internal relocation.
-  return false;
-}
-
-void ArchHandler_x86::appendSectionRelocations(
-                                   const DefinedAtom &atom,
-                                   uint64_t atomSectionOffset,
-                                   const Reference &ref,
-                                   FindSymbolIndexForAtom symbolIndexForAtom,
-                                   FindSectionIndexForAtom sectionIndexForAtom,
-                                   FindAddressForAtom addressForAtom,
-                                   normalized::Relocations &relocs) {
-  if (ref.kindNamespace() != Reference::KindNamespace::mach_o)
-    return;
-  assert(ref.kindArch() == Reference::KindArch::x86);
-  uint32_t sectionOffset = atomSectionOffset + ref.offsetInAtom();
-  bool useExternalReloc = useExternalRelocationTo(*ref.target());
-  switch (static_cast<X86Kind>(ref.kindValue())) {
-  case modeCode:
-  case modeData:
-    break;
-  case branch32:
-    if (useExternalReloc) {
-      appendReloc(relocs, sectionOffset, symbolIndexForAtom(*ref.target()), 0,
-                  GENERIC_RELOC_VANILLA | rExtern    | rPcRel | rLength4);
-    } else {
-      if (ref.addend() != 0)
-        appendReloc(relocs, sectionOffset, 0, addressForAtom(*ref.target()),
-                  GENERIC_RELOC_VANILLA | rScattered | rPcRel |  rLength4);
-      else
-        appendReloc(relocs, sectionOffset, sectionIndexForAtom(*ref.target()),0,
-                  GENERIC_RELOC_VANILLA |              rPcRel | rLength4);
-    }
-    break;
-  case branch16:
-    if (useExternalReloc) {
-      appendReloc(relocs, sectionOffset, symbolIndexForAtom(*ref.target()), 0,
-                  GENERIC_RELOC_VANILLA | rExtern    | rPcRel | rLength2);
-    } else {
-      if (ref.addend() != 0)
-        appendReloc(relocs, sectionOffset, 0, addressForAtom(*ref.target()),
-                  GENERIC_RELOC_VANILLA | rScattered | rPcRel |  rLength2);
-      else
-        appendReloc(relocs, sectionOffset, sectionIndexForAtom(*ref.target()),0,
-                  GENERIC_RELOC_VANILLA |              rPcRel | rLength2);
-    }
-    break;
-  case pointer32:
-  case abs32:
-    if (useExternalReloc)
-      appendReloc(relocs, sectionOffset, symbolIndexForAtom(*ref.target()),  0,
-                GENERIC_RELOC_VANILLA |    rExtern     |  rLength4);
-    else {
-      if (ref.addend() != 0)
-        appendReloc(relocs, sectionOffset, 0, addressForAtom(*ref.target()),
-                GENERIC_RELOC_VANILLA |    rScattered  |  rLength4);
-      else
-        appendReloc(relocs, sectionOffset, sectionIndexForAtom(*ref.target()), 0,
-                GENERIC_RELOC_VANILLA |                   rLength4);
-    }
-    break;
-  case funcRel32:
-    appendReloc(relocs, sectionOffset, 0, addressForAtom(*ref.target()),
-              GENERIC_RELOC_SECTDIFF |  rScattered    | rLength4);
-    appendReloc(relocs, sectionOffset, 0, addressForAtom(atom) - ref.addend(),
-              GENERIC_RELOC_PAIR     |  rScattered    | rLength4);
-    break;
-  case delta32:
-    appendReloc(relocs, sectionOffset, 0, addressForAtom(*ref.target()),
-              GENERIC_RELOC_SECTDIFF |  rScattered    | rLength4);
-    appendReloc(relocs, sectionOffset, 0, addressForAtom(atom) +
-                                                           ref.offsetInAtom(),
-              GENERIC_RELOC_PAIR     |  rScattered    | rLength4);
-    break;
-  case negDelta32:
-    appendReloc(relocs, sectionOffset, 0, addressForAtom(atom) +
-                                                           ref.offsetInAtom(),
-              GENERIC_RELOC_SECTDIFF |  rScattered    | rLength4);
-    appendReloc(relocs, sectionOffset, 0, addressForAtom(*ref.target()),
-              GENERIC_RELOC_PAIR     |  rScattered    | rLength4);
-    break;
-  case lazyPointer:
-  case lazyImmediateLocation:
-    llvm_unreachable("lazy reference kind implies Stubs pass was run");
-    break;
-  case invalid:
-    llvm_unreachable("unknown x86 Reference Kind");
-    break;
-  }
-}
-
-std::unique_ptr<mach_o::ArchHandler> ArchHandler::create_x86() {
-  return std::unique_ptr<mach_o::ArchHandler>(new ArchHandler_x86());
-}
-
-} // namespace mach_o
-} // namespace lld
diff --git a/lld/lib/ReaderWriter/MachO/ArchHandler_x86_64.cpp b/lld/lib/ReaderWriter/MachO/ArchHandler_x86_64.cpp
deleted file mode 100644
index 687407049d4b..000000000000
--- a/lld/lib/ReaderWriter/MachO/ArchHandler_x86_64.cpp
+++ /dev/null
@@ -1,899 +0,0 @@
-//===- lib/FileFormat/MachO/ArchHandler_x86_64.cpp ------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "ArchHandler.h"
-#include "Atoms.h"
-#include "MachONormalizedFileBinaryUtils.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/ADT/StringSwitch.h"
-#include "llvm/ADT/Triple.h"
-#include "llvm/Support/Endian.h"
-#include "llvm/Support/ErrorHandling.h"
-
-using namespace llvm::MachO;
-using namespace lld::mach_o::normalized;
-
-namespace lld {
-namespace mach_o {
-
-using llvm::support::ulittle32_t;
-using llvm::support::ulittle64_t;
-
-using llvm::support::little32_t;
-using llvm::support::little64_t;
-
-class ArchHandler_x86_64 : public ArchHandler {
-public:
-  ArchHandler_x86_64() = default;
-  ~ArchHandler_x86_64() override = default;
-
-  const Registry::KindStrings *kindStrings() override { return _sKindStrings; }
-
-  Reference::KindArch kindArch() override {
-    return Reference::KindArch::x86_64;
-  }
-
-  /// Used by GOTPass to locate GOT References
-  bool isGOTAccess(const Reference &ref, bool &canBypassGOT) override {
-    if (ref.kindNamespace() != Reference::KindNamespace::mach_o)
-      return false;
-    assert(ref.kindArch() == Reference::KindArch::x86_64);
-    switch (ref.kindValue()) {
-    case ripRel32GotLoad:
-      canBypassGOT = true;
-      return true;
-    case ripRel32Got:
-      canBypassGOT = false;
-      return true;
-    case imageOffsetGot:
-      canBypassGOT = false;
-      return true;
-    default:
-      return false;
-    }
-  }
-
-  bool isTLVAccess(const Reference &ref) const override {
-    assert(ref.kindNamespace() == Reference::KindNamespace::mach_o);
-    assert(ref.kindArch() == Reference::KindArch::x86_64);
-    return ref.kindValue() == ripRel32Tlv;
-  }
-
-  void updateReferenceToTLV(const Reference *ref) override {
-    assert(ref->kindNamespace() == Reference::KindNamespace::mach_o);
-    assert(ref->kindArch() == Reference::KindArch::x86_64);
-    assert(ref->kindValue() == ripRel32Tlv);
-    const_cast<Reference*>(ref)->setKindValue(ripRel32);
-  }
-
-  /// Used by GOTPass to update GOT References
-  void updateReferenceToGOT(const Reference *ref, bool targetNowGOT) override {
-    assert(ref->kindNamespace() == Reference::KindNamespace::mach_o);
-    assert(ref->kindArch() == Reference::KindArch::x86_64);
-
-    switch (ref->kindValue()) {
-    case ripRel32Got:
-      assert(targetNowGOT && "target must be GOT");
-      LLVM_FALLTHROUGH;
-    case ripRel32GotLoad:
-      const_cast<Reference *>(ref)
-        ->setKindValue(targetNowGOT ? ripRel32 : ripRel32GotLoadNowLea);
-      break;
-    case imageOffsetGot:
-      const_cast<Reference *>(ref)->setKindValue(imageOffset);
-      break;
-    default:
-      llvm_unreachable("unknown GOT reference kind");
-    }
-  }
-
-  bool needsCompactUnwind() override {
-    return true;
-  }
-
-  Reference::KindValue imageOffsetKind() override {
-    return imageOffset;
-  }
-
-  Reference::KindValue imageOffsetKindIndirect() override {
-    return imageOffsetGot;
-  }
-
-  Reference::KindValue unwindRefToPersonalityFunctionKind() override {
-    return ripRel32Got;
-  }
-
-  Reference::KindValue unwindRefToCIEKind() override {
-    return negDelta32;
-  }
-
-  Reference::KindValue unwindRefToFunctionKind() override{
-    return unwindFDEToFunction;
-  }
-
-  Reference::KindValue lazyImmediateLocationKind() override {
-    return lazyImmediateLocation;
-  }
-
-  Reference::KindValue unwindRefToEhFrameKind() override {
-    return unwindInfoToEhFrame;
-  }
-
-  Reference::KindValue pointerKind() override {
-    return pointer64;
-  }
-
-  uint32_t dwarfCompactUnwindType() override {
-    return 0x04000000U;
-  }
-
-  const StubInfo &stubInfo() override { return _sStubInfo; }
-
-  bool isNonCallBranch(const Reference &) override {
-    return false;
-  }
-
-  bool isCallSite(const Reference &) override;
-  bool isPointer(const Reference &) override;
-  bool isPairedReloc(const normalized::Relocation &) override;
-
-  llvm::Error getReferenceInfo(const normalized::Relocation &reloc,
-                               const DefinedAtom *inAtom,
-                               uint32_t offsetInAtom,
-                               uint64_t fixupAddress, bool swap,
-                               FindAtomBySectionAndAddress atomFromAddress,
-                               FindAtomBySymbolIndex atomFromSymbolIndex,
-                               Reference::KindValue *kind,
-                               const lld::Atom **target,
-                               Reference::Addend *addend) override;
-  llvm::Error
-      getPairReferenceInfo(const normalized::Relocation &reloc1,
-                           const normalized::Relocation &reloc2,
-                           const DefinedAtom *inAtom,
-                           uint32_t offsetInAtom,
-                           uint64_t fixupAddress, bool swap, bool scatterable,
-                           FindAtomBySectionAndAddress atomFromAddress,
-                           FindAtomBySymbolIndex atomFromSymbolIndex,
-                           Reference::KindValue *kind,
-                           const lld::Atom **target,
-                           Reference::Addend *addend) override;
-
-  bool needsLocalSymbolInRelocatableFile(const DefinedAtom *atom) override {
-    return (atom->contentType() == DefinedAtom::typeCString);
-  }
-
-  void generateAtomContent(const DefinedAtom &atom, bool relocatable,
-                           FindAddressForAtom findAddress,
-                           FindAddressForAtom findSectionAddress,
-                           uint64_t imageBase,
-                    llvm::MutableArrayRef<uint8_t> atomContentBuffer) override;
-
-  void appendSectionRelocations(const DefinedAtom &atom,
-                                uint64_t atomSectionOffset,
-                                const Reference &ref,
-                                FindSymbolIndexForAtom symbolIndexForAtom,
-                                FindSectionIndexForAtom sectionIndexForAtom,
-                                FindAddressForAtom addressForAtom,
-                                normalized::Relocations &relocs) override;
-
-  bool isDataInCodeTransition(Reference::KindValue refKind) override {
-    return refKind == modeCode || refKind == modeData;
-  }
-
-  Reference::KindValue dataInCodeTransitionStart(
-                                        const MachODefinedAtom &atom) override {
-    return modeData;
-  }
-
-  Reference::KindValue dataInCodeTransitionEnd(
-                                        const MachODefinedAtom &atom) override {
-    return modeCode;
-  }
-
-private:
-  static const Registry::KindStrings _sKindStrings[];
-  static const StubInfo              _sStubInfo;
-
-  enum X86_64Kind: Reference::KindValue {
-    invalid,               /// for error condition
-
-    modeCode,              /// Content starting at this offset is code.
-    modeData,              /// Content starting at this offset is data.
-
-    // Kinds found in mach-o .o files:
-    branch32,              /// ex: call _foo
-    ripRel32,              /// ex: movq _foo(%rip), %rax
-    ripRel32Minus1,        /// ex: movb $0x12, _foo(%rip)
-    ripRel32Minus2,        /// ex: movw $0x1234, _foo(%rip)
-    ripRel32Minus4,        /// ex: movl $0x12345678, _foo(%rip)
-    ripRel32Anon,          /// ex: movq L1(%rip), %rax
-    ripRel32Minus1Anon,    /// ex: movb $0x12, L1(%rip)
-    ripRel32Minus2Anon,    /// ex: movw $0x1234, L1(%rip)
-    ripRel32Minus4Anon,    /// ex: movw $0x12345678, L1(%rip)
-    ripRel32GotLoad,       /// ex: movq  _foo@GOTPCREL(%rip), %rax
-    ripRel32Got,           /// ex: pushq _foo@GOTPCREL(%rip)
-    ripRel32Tlv,           /// ex: movq  _foo@TLVP(%rip), %rdi
-    pointer64,             /// ex: .quad _foo
-    pointer64Anon,         /// ex: .quad L1
-    delta64,               /// ex: .quad _foo - .
-    delta32,               /// ex: .long _foo - .
-    delta64Anon,           /// ex: .quad L1 - .
-    delta32Anon,           /// ex: .long L1 - .
-    negDelta64,            /// ex: .quad . - _foo
-    negDelta32,            /// ex: .long . - _foo
-
-    // Kinds introduced by Passes:
-    ripRel32GotLoadNowLea, /// Target of GOT load is in linkage unit so
-                           ///  "movq  _foo@GOTPCREL(%rip), %rax" can be changed
-                           /// to "leaq _foo(%rip), %rax
-    lazyPointer,           /// Location contains a lazy pointer.
-    lazyImmediateLocation, /// Location contains immediate value used in stub.
-
-    imageOffset,           /// Location contains offset of atom in final image
-    imageOffsetGot,        /// Location contains offset of GOT entry for atom in
-                           /// final image (typically personality function).
-    unwindFDEToFunction,   /// Nearly delta64, but cannot be rematerialized in
-                           /// relocatable object (yay for implicit contracts!).
-    unwindInfoToEhFrame,   /// Fix low 24 bits of compact unwind encoding to
-                           /// refer to __eh_frame entry.
-    tlvInitSectionOffset   /// Location contains offset tlv init-value atom
-                           /// within the __thread_data section.
-  };
-
-  Reference::KindValue kindFromReloc(const normalized::Relocation &reloc);
-
-  void applyFixupFinal(const Reference &ref, uint8_t *location,
-                       uint64_t fixupAddress, uint64_t targetAddress,
-                       uint64_t inAtomAddress, uint64_t imageBaseAddress,
-                       FindAddressForAtom findSectionAddress);
-
-  void applyFixupRelocatable(const Reference &ref, uint8_t *location,
-                             uint64_t fixupAddress,
-                             uint64_t targetAddress,
-                             uint64_t inAtomAddress);
-};
-
-const Registry::KindStrings ArchHandler_x86_64::_sKindStrings[] = {
-  LLD_KIND_STRING_ENTRY(invalid),
-  LLD_KIND_STRING_ENTRY(modeCode),
-  LLD_KIND_STRING_ENTRY(modeData),
-  LLD_KIND_STRING_ENTRY(branch32),
-  LLD_KIND_STRING_ENTRY(ripRel32),
-  LLD_KIND_STRING_ENTRY(ripRel32Minus1),
-  LLD_KIND_STRING_ENTRY(ripRel32Minus2),
-  LLD_KIND_STRING_ENTRY(ripRel32Minus4),
-  LLD_KIND_STRING_ENTRY(ripRel32Anon),
-  LLD_KIND_STRING_ENTRY(ripRel32Minus1Anon),
-  LLD_KIND_STRING_ENTRY(ripRel32Minus2Anon),
-  LLD_KIND_STRING_ENTRY(ripRel32Minus4Anon),
-  LLD_KIND_STRING_ENTRY(ripRel32GotLoad),
-  LLD_KIND_STRING_ENTRY(ripRel32GotLoadNowLea),
-  LLD_KIND_STRING_ENTRY(ripRel32Got),
-  LLD_KIND_STRING_ENTRY(ripRel32Tlv),
-  LLD_KIND_STRING_ENTRY(lazyPointer),
-  LLD_KIND_STRING_ENTRY(lazyImmediateLocation),
-  LLD_KIND_STRING_ENTRY(pointer64),
-  LLD_KIND_STRING_ENTRY(pointer64Anon),
-  LLD_KIND_STRING_ENTRY(delta32),
-  LLD_KIND_STRING_ENTRY(delta64),
-  LLD_KIND_STRING_ENTRY(delta32Anon),
-  LLD_KIND_STRING_ENTRY(delta64Anon),
-  LLD_KIND_STRING_ENTRY(negDelta64),
-  LLD_KIND_STRING_ENTRY(negDelta32),
-  LLD_KIND_STRING_ENTRY(imageOffset),
-  LLD_KIND_STRING_ENTRY(imageOffsetGot),
-  LLD_KIND_STRING_ENTRY(unwindFDEToFunction),
-  LLD_KIND_STRING_ENTRY(unwindInfoToEhFrame),
-  LLD_KIND_STRING_ENTRY(tlvInitSectionOffset),
-  LLD_KIND_STRING_END
-};
-
-const ArchHandler::StubInfo ArchHandler_x86_64::_sStubInfo = {
-  "dyld_stub_binder",
-
-  // Lazy pointer references
-  { Reference::KindArch::x86_64, pointer64, 0, 0 },
-  { Reference::KindArch::x86_64, lazyPointer, 0, 0 },
-
-  // GOT pointer to dyld_stub_binder
-  { Reference::KindArch::x86_64, pointer64, 0, 0 },
-
-  // x86_64 code alignment 2^1
-  1,
-
-  // Stub size and code
-  6,
-  { 0xff, 0x25, 0x00, 0x00, 0x00, 0x00 },       // jmp *lazyPointer
-  { Reference::KindArch::x86_64, ripRel32, 2, 0 },
-  { false, 0, 0, 0 },
-
-  // Stub Helper size and code
-  10,
-  { 0x68, 0x00, 0x00, 0x00, 0x00,               // pushq $lazy-info-offset
-    0xE9, 0x00, 0x00, 0x00, 0x00 },             // jmp helperhelper
-  { Reference::KindArch::x86_64, lazyImmediateLocation, 1, 0 },
-  { Reference::KindArch::x86_64, branch32, 6, 0 },
-
-  // Stub helper image cache content type
-  DefinedAtom::typeNonLazyPointer,
-
-  // Stub Helper-Common size and code
-  16,
-  // Stub helper alignment
-  2,
-  { 0x4C, 0x8D, 0x1D, 0x00, 0x00, 0x00, 0x00,   // leaq cache(%rip),%r11
-    0x41, 0x53,                                 // push %r11
-    0xFF, 0x25, 0x00, 0x00, 0x00, 0x00,         // jmp *binder(%rip)
-    0x90 },                                     // nop
-  { Reference::KindArch::x86_64, ripRel32, 3, 0 },
-  { false, 0, 0, 0 },
-  { Reference::KindArch::x86_64, ripRel32, 11, 0 },
-  { false, 0, 0, 0 }
-
-};
-
-bool ArchHandler_x86_64::isCallSite(const Reference &ref) {
-  if (ref.kindNamespace() != Reference::KindNamespace::mach_o)
-    return false;
-  assert(ref.kindArch() == Reference::KindArch::x86_64);
-  return (ref.kindValue() == branch32);
-}
-
-bool ArchHandler_x86_64::isPointer(const Reference &ref) {
-  if (ref.kindNamespace() != Reference::KindNamespace::mach_o)
-    return false;
-  assert(ref.kindArch() == Reference::KindArch::x86_64);
-  Reference::KindValue kind = ref.kindValue();
-  return (kind == pointer64 || kind == pointer64Anon);
-}
-
-bool ArchHandler_x86_64::isPairedReloc(const Relocation &reloc) {
-  return (reloc.type == X86_64_RELOC_SUBTRACTOR);
-}
-
-Reference::KindValue
-ArchHandler_x86_64::kindFromReloc(const Relocation &reloc) {
-  switch(relocPattern(reloc)) {
-  case X86_64_RELOC_BRANCH   | rPcRel | rExtern | rLength4:
-    return branch32;
-  case X86_64_RELOC_SIGNED   | rPcRel | rExtern | rLength4:
-    return ripRel32;
-  case X86_64_RELOC_SIGNED   | rPcRel |           rLength4:
-    return ripRel32Anon;
-  case X86_64_RELOC_SIGNED_1 | rPcRel | rExtern | rLength4:
-    return ripRel32Minus1;
-  case X86_64_RELOC_SIGNED_1 | rPcRel |           rLength4:
-    return ripRel32Minus1Anon;
-  case X86_64_RELOC_SIGNED_2 | rPcRel | rExtern | rLength4:
-    return ripRel32Minus2;
-  case X86_64_RELOC_SIGNED_2 | rPcRel |           rLength4:
-    return ripRel32Minus2Anon;
-  case X86_64_RELOC_SIGNED_4 | rPcRel | rExtern | rLength4:
-    return ripRel32Minus4;
-  case X86_64_RELOC_SIGNED_4 | rPcRel |           rLength4:
-    return ripRel32Minus4Anon;
-  case X86_64_RELOC_GOT_LOAD | rPcRel | rExtern | rLength4:
-    return ripRel32GotLoad;
-  case X86_64_RELOC_GOT      | rPcRel | rExtern | rLength4:
-    return ripRel32Got;
-  case X86_64_RELOC_TLV      | rPcRel | rExtern | rLength4:
-    return ripRel32Tlv;
-  case X86_64_RELOC_UNSIGNED          | rExtern | rLength8:
-    return pointer64;
-  case X86_64_RELOC_UNSIGNED                    | rLength8:
-    return pointer64Anon;
-  default:
-    return invalid;
-  }
-}
-
-llvm::Error
-ArchHandler_x86_64::getReferenceInfo(const Relocation &reloc,
-                                    const DefinedAtom *inAtom,
-                                    uint32_t offsetInAtom,
-                                    uint64_t fixupAddress, bool swap,
-                                    FindAtomBySectionAndAddress atomFromAddress,
-                                    FindAtomBySymbolIndex atomFromSymbolIndex,
-                                    Reference::KindValue *kind,
-                                    const lld::Atom **target,
-                                    Reference::Addend *addend) {
-  *kind = kindFromReloc(reloc);
-  if (*kind == invalid)
-    return llvm::make_error<GenericError>("unknown type");
-  const uint8_t *fixupContent = &inAtom->rawContent()[offsetInAtom];
-  uint64_t targetAddress;
-  switch (*kind) {
-  case branch32:
-  case ripRel32:
-    if (auto ec = atomFromSymbolIndex(reloc.symbol, target))
-      return ec;
-    *addend = *(const little32_t *)fixupContent;
-    return llvm::Error::success();
-  case ripRel32Minus1:
-    if (auto ec = atomFromSymbolIndex(reloc.symbol, target))
-      return ec;
-    *addend = (int32_t)*(const little32_t *)fixupContent + 1;
-    return llvm::Error::success();
-  case ripRel32Minus2:
-    if (auto ec = atomFromSymbolIndex(reloc.symbol, target))
-      return ec;
-    *addend = (int32_t)*(const little32_t *)fixupContent + 2;
-    return llvm::Error::success();
-  case ripRel32Minus4:
-    if (auto ec = atomFromSymbolIndex(reloc.symbol, target))
-      return ec;
-    *addend = (int32_t)*(const little32_t *)fixupContent + 4;
-    return llvm::Error::success();
-  case ripRel32Anon:
-    targetAddress = fixupAddress + 4 + *(const little32_t *)fixupContent;
-    return atomFromAddress(reloc.symbol, targetAddress, target, addend);
-  case ripRel32Minus1Anon:
-    targetAddress = fixupAddress + 5 + *(const little32_t *)fixupContent;
-    return atomFromAddress(reloc.symbol, targetAddress, target, addend);
-  case ripRel32Minus2Anon:
-    targetAddress = fixupAddress + 6 + *(const little32_t *)fixupContent;
-    return atomFromAddress(reloc.symbol, targetAddress, target, addend);
-  case ripRel32Minus4Anon:
-    targetAddress = fixupAddress + 8 + *(const little32_t *)fixupContent;
-    return atomFromAddress(reloc.symbol, targetAddress, target, addend);
-  case ripRel32GotLoad:
-  case ripRel32Got:
-  case ripRel32Tlv:
-    if (auto ec = atomFromSymbolIndex(reloc.symbol, target))
-      return ec;
-    *addend = *(const little32_t *)fixupContent;
-    return llvm::Error::success();
-  case tlvInitSectionOffset:
-  case pointer64:
-    if (auto ec = atomFromSymbolIndex(reloc.symbol, target))
-      return ec;
-    // If this is the 3rd pointer of a tlv-thunk (i.e. the pointer to the TLV's
-    // initial value) we need to handle it specially.
-    if (inAtom->contentType() == DefinedAtom::typeThunkTLV &&
-        offsetInAtom == 16) {
-      *kind = tlvInitSectionOffset;
-      assert(*addend == 0 && "TLV-init has non-zero addend?");
-    } else
-      *addend = *(const little64_t *)fixupContent;
-    return llvm::Error::success();
-  case pointer64Anon:
-    targetAddress = *(const little64_t *)fixupContent;
-    return atomFromAddress(reloc.symbol, targetAddress, target, addend);
-  default:
-    llvm_unreachable("bad reloc kind");
-  }
-}
-
-llvm::Error
-ArchHandler_x86_64::getPairReferenceInfo(const normalized::Relocation &reloc1,
-                                   const normalized::Relocation &reloc2,
-                                   const DefinedAtom *inAtom,
-                                   uint32_t offsetInAtom,
-                                   uint64_t fixupAddress, bool swap,
-                                   bool scatterable,
-                                   FindAtomBySectionAndAddress atomFromAddress,
-                                   FindAtomBySymbolIndex atomFromSymbolIndex,
-                                   Reference::KindValue *kind,
-                                   const lld::Atom **target,
-                                   Reference::Addend *addend) {
-  const uint8_t *fixupContent = &inAtom->rawContent()[offsetInAtom];
-  uint64_t targetAddress;
-  const lld::Atom *fromTarget;
-  if (auto ec = atomFromSymbolIndex(reloc1.symbol, &fromTarget))
-    return ec;
-
-  switch(relocPattern(reloc1) << 16 | relocPattern(reloc2)) {
-  case ((X86_64_RELOC_SUBTRACTOR | rExtern | rLength8) << 16 |
-        X86_64_RELOC_UNSIGNED    | rExtern | rLength8): {
-    if (auto ec = atomFromSymbolIndex(reloc2.symbol, target))
-      return ec;
-    uint64_t encodedAddend = (int64_t)*(const little64_t *)fixupContent;
-    if (inAtom == fromTarget) {
-      if (inAtom->contentType() == DefinedAtom::typeCFI)
-        *kind = unwindFDEToFunction;
-      else
-        *kind = delta64;
-      *addend = encodedAddend + offsetInAtom;
-    } else if (inAtom == *target) {
-      *kind = negDelta64;
-      *addend = encodedAddend - offsetInAtom;
-      *target = fromTarget;
-    } else
-      return llvm::make_error<GenericError>("Invalid pointer diff");
-    return llvm::Error::success();
-  }
-  case ((X86_64_RELOC_SUBTRACTOR | rExtern | rLength4) << 16 |
-        X86_64_RELOC_UNSIGNED    | rExtern | rLength4): {
-    if (auto ec = atomFromSymbolIndex(reloc2.symbol, target))
-      return ec;
-    uint32_t encodedAddend = (int32_t)*(const little32_t *)fixupContent;
-    if (inAtom == fromTarget) {
-      *kind = delta32;
-      *addend = encodedAddend + offsetInAtom;
-    } else if (inAtom == *target) {
-      *kind = negDelta32;
-      *addend = encodedAddend - offsetInAtom;
-      *target = fromTarget;
-    } else
-      return llvm::make_error<GenericError>("Invalid pointer diff");
-    return llvm::Error::success();
-  }
-  case ((X86_64_RELOC_SUBTRACTOR | rExtern | rLength8) << 16 |
-        X86_64_RELOC_UNSIGNED              | rLength8):
-    if (fromTarget != inAtom)
-      return llvm::make_error<GenericError>("pointer diff not in base atom");
-    *kind = delta64Anon;
-    targetAddress = offsetInAtom + (int64_t)*(const little64_t *)fixupContent;
-    return atomFromAddress(reloc2.symbol, targetAddress, target, addend);
-  case ((X86_64_RELOC_SUBTRACTOR | rExtern | rLength4) << 16 |
-        X86_64_RELOC_UNSIGNED              | rLength4):
-    if (fromTarget != inAtom)
-      return llvm::make_error<GenericError>("pointer diff not in base atom");
-    *kind = delta32Anon;
-    targetAddress = offsetInAtom + (int32_t)*(const little32_t *)fixupContent;
-    return atomFromAddress(reloc2.symbol, targetAddress, target, addend);
-  default:
-    return llvm::make_error<GenericError>("unknown pair");
-  }
-}
-
-void ArchHandler_x86_64::generateAtomContent(
-    const DefinedAtom &atom, bool relocatable, FindAddressForAtom findAddress,
-    FindAddressForAtom findSectionAddress, uint64_t imageBaseAddress,
-    llvm::MutableArrayRef<uint8_t> atomContentBuffer) {
-  // Copy raw bytes.
-  std::copy(atom.rawContent().begin(), atom.rawContent().end(),
-            atomContentBuffer.begin());
-  // Apply fix-ups.
-  for (const Reference *ref : atom) {
-    uint32_t offset = ref->offsetInAtom();
-    const Atom *target = ref->target();
-    uint64_t targetAddress = 0;
-    if (isa<DefinedAtom>(target))
-      targetAddress = findAddress(*target);
-    uint64_t atomAddress = findAddress(atom);
-    uint64_t fixupAddress = atomAddress + offset;
-    if (relocatable) {
-      applyFixupRelocatable(*ref, &atomContentBuffer[offset],
-                                        fixupAddress, targetAddress,
-                                        atomAddress);
-    } else {
-      applyFixupFinal(*ref, &atomContentBuffer[offset],
-                      fixupAddress, targetAddress,
-                      atomAddress, imageBaseAddress, findSectionAddress);
-    }
-  }
-}
-
-void ArchHandler_x86_64::applyFixupFinal(
-    const Reference &ref, uint8_t *loc, uint64_t fixupAddress,
-    uint64_t targetAddress, uint64_t inAtomAddress, uint64_t imageBaseAddress,
-    FindAddressForAtom findSectionAddress) {
-  if (ref.kindNamespace() != Reference::KindNamespace::mach_o)
-    return;
-  assert(ref.kindArch() == Reference::KindArch::x86_64);
-  ulittle32_t *loc32 = reinterpret_cast<ulittle32_t *>(loc);
-  ulittle64_t *loc64 = reinterpret_cast<ulittle64_t *>(loc);
-  switch (static_cast<X86_64Kind>(ref.kindValue())) {
-  case branch32:
-  case ripRel32:
-  case ripRel32Anon:
-  case ripRel32Got:
-  case ripRel32GotLoad:
-  case ripRel32Tlv:
-    *loc32 = targetAddress - (fixupAddress + 4) + ref.addend();
-    return;
-  case pointer64:
-  case pointer64Anon:
-    *loc64 = targetAddress + ref.addend();
-    return;
-  case tlvInitSectionOffset:
-    *loc64 = targetAddress - findSectionAddress(*ref.target()) + ref.addend();
-    return;
-  case ripRel32Minus1:
-  case ripRel32Minus1Anon:
-    *loc32 = targetAddress - (fixupAddress + 5) + ref.addend();
-    return;
-  case ripRel32Minus2:
-  case ripRel32Minus2Anon:
-    *loc32 = targetAddress - (fixupAddress + 6) + ref.addend();
-    return;
-  case ripRel32Minus4:
-  case ripRel32Minus4Anon:
-    *loc32 = targetAddress - (fixupAddress + 8) + ref.addend();
-    return;
-  case delta32:
-  case delta32Anon:
-    *loc32 = targetAddress - fixupAddress + ref.addend();
-    return;
-  case delta64:
-  case delta64Anon:
-  case unwindFDEToFunction:
-    *loc64 = targetAddress - fixupAddress + ref.addend();
-    return;
-  case ripRel32GotLoadNowLea:
-    // Change MOVQ to LEA
-    assert(loc[-2] == 0x8B);
-    loc[-2] = 0x8D;
-    *loc32 = targetAddress - (fixupAddress + 4) + ref.addend();
-    return;
-  case negDelta64:
-    *loc64 = fixupAddress - targetAddress + ref.addend();
-    return;
-  case negDelta32:
-    *loc32 = fixupAddress - targetAddress + ref.addend();
-    return;
-  case modeCode:
-  case modeData:
-  case lazyPointer:
-    // Do nothing
-    return;
-  case lazyImmediateLocation:
-    *loc32 = ref.addend();
-    return;
-  case imageOffset:
-  case imageOffsetGot:
-    *loc32 = (targetAddress - imageBaseAddress) + ref.addend();
-    return;
-  case unwindInfoToEhFrame: {
-    uint64_t val = targetAddress - findSectionAddress(*ref.target()) + ref.addend();
-    assert(val < 0xffffffU && "offset in __eh_frame too large");
-    *loc32 = (*loc32 & 0xff000000U) | val;
-    return;
-  }
-  case invalid:
-    // Fall into llvm_unreachable().
-    break;
-  }
-  llvm_unreachable("invalid x86_64 Reference Kind");
-}
-
-void ArchHandler_x86_64::applyFixupRelocatable(const Reference &ref,
-                                               uint8_t *loc,
-                                               uint64_t fixupAddress,
-                                               uint64_t targetAddress,
-                                               uint64_t inAtomAddress)  {
-  if (ref.kindNamespace() != Reference::KindNamespace::mach_o)
-    return;
-  assert(ref.kindArch() == Reference::KindArch::x86_64);
-  ulittle32_t *loc32 = reinterpret_cast<ulittle32_t *>(loc);
-  ulittle64_t *loc64 = reinterpret_cast<ulittle64_t *>(loc);
-  switch (static_cast<X86_64Kind>(ref.kindValue())) {
-  case branch32:
-  case ripRel32:
-  case ripRel32Got:
-  case ripRel32GotLoad:
-  case ripRel32Tlv:
-    *loc32 = ref.addend();
-    return;
-  case ripRel32Anon:
-    *loc32 = (targetAddress - (fixupAddress + 4)) + ref.addend();
-    return;
-  case tlvInitSectionOffset:
-  case pointer64:
-    *loc64 = ref.addend();
-    return;
-  case pointer64Anon:
-    *loc64 = targetAddress + ref.addend();
-    return;
-  case ripRel32Minus1:
-    *loc32 = ref.addend() - 1;
-    return;
-  case ripRel32Minus1Anon:
-    *loc32 = (targetAddress - (fixupAddress + 5)) + ref.addend();
-    return;
-  case ripRel32Minus2:
-    *loc32 = ref.addend() - 2;
-    return;
-  case ripRel32Minus2Anon:
-    *loc32 = (targetAddress - (fixupAddress + 6)) + ref.addend();
-    return;
-  case ripRel32Minus4:
-    *loc32 = ref.addend() - 4;
-    return;
-  case ripRel32Minus4Anon:
-    *loc32 = (targetAddress - (fixupAddress + 8)) + ref.addend();
-    return;
-  case delta32:
-    *loc32 = ref.addend() + inAtomAddress - fixupAddress;
-    return;
-  case delta32Anon:
-    // The value we write here should be the delta to the target
-    // after taking in to account the difference from the fixup back to the
-    // last defined label
-    // ie, if we have:
-    // _base: ...
-    // Lfixup: .quad Ltarget - .
-    // ...
-    // Ltarget:
-    //
-    // Then we want to encode the value (Ltarget + addend) - (LFixup - _base)
-    *loc32 = (targetAddress + ref.addend()) - (fixupAddress - inAtomAddress);
-    return;
-  case delta64:
-    *loc64 = ref.addend() + inAtomAddress - fixupAddress;
-    return;
-  case delta64Anon:
-    // The value we write here should be the delta to the target
-    // after taking in to account the difference from the fixup back to the
-    // last defined label
-    // ie, if we have:
-    // _base: ...
-    // Lfixup: .quad Ltarget - .
-    // ...
-    // Ltarget:
-    //
-    // Then we want to encode the value (Ltarget + addend) - (LFixup - _base)
-    *loc64 = (targetAddress + ref.addend()) - (fixupAddress - inAtomAddress);
-    return;
-  case negDelta64:
-    *loc64 = ref.addend() + fixupAddress - inAtomAddress;
-    return;
-  case negDelta32:
-    *loc32 = ref.addend() + fixupAddress - inAtomAddress;
-    return;
-  case ripRel32GotLoadNowLea:
-    llvm_unreachable("ripRel32GotLoadNowLea implies GOT pass was run");
-    return;
-  case lazyPointer:
-  case lazyImmediateLocation:
-    llvm_unreachable("lazy reference kind implies Stubs pass was run");
-    return;
-  case imageOffset:
-  case imageOffsetGot:
-  case unwindInfoToEhFrame:
-    llvm_unreachable("fixup implies __unwind_info");
-    return;
-  case modeCode:
-  case modeData:
-  case unwindFDEToFunction:
-    // Do nothing for now
-    return;
-  case invalid:
-    // Fall into llvm_unreachable().
-    break;
-  }
-  llvm_unreachable("unknown x86_64 Reference Kind");
-}
-
-void ArchHandler_x86_64::appendSectionRelocations(
-                                   const DefinedAtom &atom,
-                                   uint64_t atomSectionOffset,
-                                   const Reference &ref,
-                                   FindSymbolIndexForAtom symbolIndexForAtom,
-                                   FindSectionIndexForAtom sectionIndexForAtom,
-                                   FindAddressForAtom addressForAtom,
-                                   normalized::Relocations &relocs) {
-  if (ref.kindNamespace() != Reference::KindNamespace::mach_o)
-    return;
-  assert(ref.kindArch() == Reference::KindArch::x86_64);
-  uint32_t sectionOffset = atomSectionOffset + ref.offsetInAtom();
-  switch (static_cast<X86_64Kind>(ref.kindValue())) {
-  case modeCode:
-  case modeData:
-    return;
-  case branch32:
-    appendReloc(relocs, sectionOffset, symbolIndexForAtom(*ref.target()), 0,
-                X86_64_RELOC_BRANCH | rPcRel | rExtern | rLength4);
-    return;
-  case ripRel32:
-    appendReloc(relocs, sectionOffset, symbolIndexForAtom(*ref.target()), 0,
-                X86_64_RELOC_SIGNED | rPcRel | rExtern | rLength4 );
-    return;
-  case ripRel32Anon:
-    appendReloc(relocs, sectionOffset, sectionIndexForAtom(*ref.target()), 0,
-                X86_64_RELOC_SIGNED | rPcRel           | rLength4 );
-    return;
-  case ripRel32Got:
-    appendReloc(relocs, sectionOffset, symbolIndexForAtom(*ref.target()), 0,
-                X86_64_RELOC_GOT | rPcRel | rExtern | rLength4 );
-    return;
-  case ripRel32GotLoad:
-    appendReloc(relocs, sectionOffset, symbolIndexForAtom(*ref.target()), 0,
-                X86_64_RELOC_GOT_LOAD | rPcRel | rExtern | rLength4 );
-    return;
-  case ripRel32Tlv:
-    appendReloc(relocs, sectionOffset, symbolIndexForAtom(*ref.target()), 0,
-                X86_64_RELOC_TLV | rPcRel | rExtern | rLength4 );
-    return;
-  case tlvInitSectionOffset:
-  case pointer64:
-    appendReloc(relocs, sectionOffset, symbolIndexForAtom(*ref.target()), 0,
-                X86_64_RELOC_UNSIGNED  | rExtern | rLength8);
-    return;
-  case pointer64Anon:
-    appendReloc(relocs, sectionOffset, sectionIndexForAtom(*ref.target()), 0,
-                X86_64_RELOC_UNSIGNED | rLength8);
-    return;
-  case ripRel32Minus1:
-    appendReloc(relocs, sectionOffset, symbolIndexForAtom(*ref.target()), 0,
-                X86_64_RELOC_SIGNED_1 | rPcRel | rExtern | rLength4 );
-    return;
-  case ripRel32Minus1Anon:
-    appendReloc(relocs, sectionOffset, sectionIndexForAtom(*ref.target()), 0,
-                X86_64_RELOC_SIGNED_1 | rPcRel           | rLength4 );
-    return;
-  case ripRel32Minus2:
-    appendReloc(relocs, sectionOffset, symbolIndexForAtom(*ref.target()), 0,
-                X86_64_RELOC_SIGNED_2 | rPcRel | rExtern | rLength4 );
-    return;
-  case ripRel32Minus2Anon:
-    appendReloc(relocs, sectionOffset, sectionIndexForAtom(*ref.target()), 0,
-                X86_64_RELOC_SIGNED_2 | rPcRel           | rLength4 );
-    return;
-  case ripRel32Minus4:
-    appendReloc(relocs, sectionOffset, symbolIndexForAtom(*ref.target()), 0,
-                X86_64_RELOC_SIGNED_4 | rPcRel | rExtern | rLength4 );
-    return;
-  case ripRel32Minus4Anon:
-    appendReloc(relocs, sectionOffset, sectionIndexForAtom(*ref.target()), 0,
-                X86_64_RELOC_SIGNED_4 | rPcRel           | rLength4 );
-    return;
-  case delta32:
-    appendReloc(relocs, sectionOffset, symbolIndexForAtom(atom), 0,
-                X86_64_RELOC_SUBTRACTOR | rExtern | rLength4 );
-    appendReloc(relocs, sectionOffset, symbolIndexForAtom(*ref.target()), 0,
-                X86_64_RELOC_UNSIGNED   | rExtern | rLength4 );
-    return;
-  case delta32Anon:
-    appendReloc(relocs, sectionOffset, symbolIndexForAtom(atom), 0,
-                X86_64_RELOC_SUBTRACTOR | rExtern | rLength4 );
-    appendReloc(relocs, sectionOffset, sectionIndexForAtom(*ref.target()), 0,
-                X86_64_RELOC_UNSIGNED             | rLength4 );
-    return;
-  case delta64:
-    appendReloc(relocs, sectionOffset, symbolIndexForAtom(atom), 0,
-                X86_64_RELOC_SUBTRACTOR | rExtern | rLength8 );
-    appendReloc(relocs, sectionOffset, symbolIndexForAtom(*ref.target()), 0,
-                X86_64_RELOC_UNSIGNED   | rExtern | rLength8 );
-    return;
-  case delta64Anon:
-    appendReloc(relocs, sectionOffset, symbolIndexForAtom(atom), 0,
-                X86_64_RELOC_SUBTRACTOR | rExtern | rLength8 );
-    appendReloc(relocs, sectionOffset, sectionIndexForAtom(*ref.target()), 0,
-                X86_64_RELOC_UNSIGNED             | rLength8 );
-    return;
-  case unwindFDEToFunction:
-  case unwindInfoToEhFrame:
-    return;
-  case negDelta32:
-    appendReloc(relocs, sectionOffset, symbolIndexForAtom(*ref.target()), 0,
-                X86_64_RELOC_SUBTRACTOR | rExtern | rLength4 );
-    appendReloc(relocs, sectionOffset, symbolIndexForAtom(atom), 0,
-                X86_64_RELOC_UNSIGNED   | rExtern | rLength4 );
-    return;
-  case negDelta64:
-    appendReloc(relocs, sectionOffset, symbolIndexForAtom(*ref.target()), 0,
-                X86_64_RELOC_SUBTRACTOR | rExtern | rLength8 );
-    appendReloc(relocs, sectionOffset, symbolIndexForAtom(atom), 0,
-                X86_64_RELOC_UNSIGNED   | rExtern | rLength8 );
-    return;
-  case ripRel32GotLoadNowLea:
-    llvm_unreachable("ripRel32GotLoadNowLea implies GOT pass was run");
-    return;
-  case lazyPointer:
-  case lazyImmediateLocation:
-    llvm_unreachable("lazy reference kind implies Stubs pass was run");
-    return;
-  case imageOffset:
-  case imageOffsetGot:
-    llvm_unreachable("__unwind_info references should have been resolved");
-    return;
-  case invalid:
-    // Fall into llvm_unreachable().
-    break;
-  }
-  llvm_unreachable("unknown x86_64 Reference Kind");
-}
-
-std::unique_ptr<mach_o::ArchHandler> ArchHandler::create_x86_64() {
-  return std::unique_ptr<mach_o::ArchHandler>(new ArchHandler_x86_64());
-}
-
-} // namespace mach_o
-} // namespace lld
diff --git a/lld/lib/ReaderWriter/MachO/Atoms.h b/lld/lib/ReaderWriter/MachO/Atoms.h
deleted file mode 100644
index c61aaa88e8df..000000000000
--- a/lld/lib/ReaderWriter/MachO/Atoms.h
+++ /dev/null
@@ -1,180 +0,0 @@
-//===- lib/ReaderWriter/MachO/Atoms.h ---------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLD_READER_WRITER_MACHO_ATOMS_H
-#define LLD_READER_WRITER_MACHO_ATOMS_H
-
-#include "lld/Core/Atom.h"
-#include "lld/Core/DefinedAtom.h"
-#include "lld/Core/SharedLibraryAtom.h"
-#include "lld/Core/Simple.h"
-#include "llvm/ADT/ArrayRef.h"
-#include "llvm/ADT/StringRef.h"
-#include <cstdint>
-#include <string>
-
-namespace lld {
-
-class File;
-
-namespace mach_o {
-
-class MachODefinedAtom : public SimpleDefinedAtom {
-public:
-  MachODefinedAtom(const File &f, const StringRef name, Scope scope,
-                   ContentType type, Merge merge, bool thumb, bool noDeadStrip,
-                   const ArrayRef<uint8_t> content, Alignment align)
-      : SimpleDefinedAtom(f), _name(name), _content(content),
-        _align(align), _contentType(type), _scope(scope), _merge(merge),
-        _thumb(thumb), _noDeadStrip(noDeadStrip) {}
-
-  // Constructor for zero-fill content
-  MachODefinedAtom(const File &f, const StringRef name, Scope scope,
-                   ContentType type, uint64_t size, bool noDeadStrip,
-                   Alignment align)
-      : SimpleDefinedAtom(f), _name(name),
-        _content(ArrayRef<uint8_t>(nullptr, size)), _align(align),
-        _contentType(type), _scope(scope), _merge(mergeNo), _thumb(false),
-        _noDeadStrip(noDeadStrip) {}
-
-  ~MachODefinedAtom() override = default;
-
-  uint64_t size() const override { return _content.size(); }
-
-  ContentType contentType() const override { return _contentType; }
-
-  Alignment alignment() const override { return _align; }
-
-  StringRef name() const override { return _name; }
-
-  Scope scope() const override { return _scope; }
-
-  Merge merge() const override { return _merge; }
-
-  DeadStripKind deadStrip() const override {
-    if (_contentType == DefinedAtom::typeInitializerPtr)
-      return deadStripNever;
-    if (_contentType == DefinedAtom::typeTerminatorPtr)
-      return deadStripNever;
-    if (_noDeadStrip)
-      return deadStripNever;
-    return deadStripNormal;
-  }
-
-  ArrayRef<uint8_t> rawContent() const override {
-    // Note: Zerofill atoms have a content pointer which is null.
-    return _content;
-  }
-
-  bool isThumb() const { return _thumb; }
-
-private:
-  const StringRef _name;
-  const ArrayRef<uint8_t> _content;
-  const DefinedAtom::Alignment _align;
-  const ContentType _contentType;
-  const Scope _scope;
-  const Merge _merge;
-  const bool _thumb;
-  const bool _noDeadStrip;
-};
-
-class MachODefinedCustomSectionAtom : public MachODefinedAtom {
-public:
-  MachODefinedCustomSectionAtom(const File &f, const StringRef name,
-                                Scope scope, ContentType type, Merge merge,
-                                bool thumb, bool noDeadStrip,
-                                const ArrayRef<uint8_t> content,
-                                StringRef sectionName, Alignment align)
-      : MachODefinedAtom(f, name, scope, type, merge, thumb, noDeadStrip,
-                         content, align),
-        _sectionName(sectionName) {}
-
-  ~MachODefinedCustomSectionAtom() override = default;
-
-  SectionChoice sectionChoice() const override {
-    return DefinedAtom::sectionCustomRequired;
-  }
-
-  StringRef customSectionName() const override {
-    return _sectionName;
-  }
-private:
-  StringRef _sectionName;
-};
-
-class MachOTentativeDefAtom : public SimpleDefinedAtom {
-public:
-  MachOTentativeDefAtom(const File &f, const StringRef name, Scope scope,
-                        uint64_t size, DefinedAtom::Alignment align)
-      : SimpleDefinedAtom(f), _name(std::string(name)), _scope(scope),
-        _size(size), _align(align) {}
-
-  ~MachOTentativeDefAtom() override = default;
-
-  uint64_t size() const override { return _size; }
-
-  Merge merge() const override { return DefinedAtom::mergeAsTentative; }
-
-  ContentType contentType() const override { return DefinedAtom::typeZeroFill; }
-
-  Alignment alignment() const override { return _align; }
-
-  StringRef name() const override { return _name; }
-
-  Scope scope() const override { return _scope; }
-
-  ArrayRef<uint8_t> rawContent() const override { return ArrayRef<uint8_t>(); }
-
-private:
-  const std::string _name;
-  const Scope _scope;
-  const uint64_t _size;
-  const DefinedAtom::Alignment _align;
-};
-
-class MachOSharedLibraryAtom : public SharedLibraryAtom {
-public:
-  MachOSharedLibraryAtom(const File &file, StringRef name,
-                         StringRef dylibInstallName, bool weakDef)
-      : SharedLibraryAtom(), _file(file), _name(name),
-        _dylibInstallName(dylibInstallName) {}
-  ~MachOSharedLibraryAtom() override = default;
-
-  StringRef loadName() const override { return _dylibInstallName; }
-
-  bool canBeNullAtRuntime() const override {
-    // FIXME: this may actually be changeable. For now, all symbols are strongly
-    // defined though.
-    return false;
-  }
-
-  const File &file() const override { return _file; }
-
-  StringRef name() const override { return _name; }
-
-  Type type() const override {
-    // Unused in MachO (I think).
-    return Type::Unknown;
-  }
-
-  uint64_t size() const override {
-    // Unused in MachO (I think)
-    return 0;
-  }
-
-private:
-  const File &_file;
-  StringRef _name;
-  StringRef _dylibInstallName;
-};
-
-} // end namespace mach_o
-} // end namespace lld
-
-#endif // LLD_READER_WRITER_MACHO_ATOMS_H
diff --git a/lld/lib/ReaderWriter/MachO/CompactUnwindPass.cpp b/lld/lib/ReaderWriter/MachO/CompactUnwindPass.cpp
deleted file mode 100644
index f3636feb217b..000000000000
--- a/lld/lib/ReaderWriter/MachO/CompactUnwindPass.cpp
+++ /dev/null
@@ -1,580 +0,0 @@
-//===- lib/ReaderWriter/MachO/CompactUnwindPass.cpp -------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-///
-/// \file A pass to convert MachO's __compact_unwind sections into the final
-/// __unwind_info format used during runtime. See
-/// mach-o/compact_unwind_encoding.h for more details on the formats involved.
-///
-//===----------------------------------------------------------------------===//
-
-#include "ArchHandler.h"
-#include "File.h"
-#include "MachONormalizedFileBinaryUtils.h"
-#include "MachOPasses.h"
-#include "lld/Common/LLVM.h"
-#include "lld/Core/DefinedAtom.h"
-#include "lld/Core/File.h"
-#include "lld/Core/Reference.h"
-#include "lld/Core/Simple.h"
-#include "llvm/ADT/DenseMap.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/Format.h"
-
-#define DEBUG_TYPE "macho-compact-unwind"
-
-namespace lld {
-namespace mach_o {
-
-namespace {
-struct CompactUnwindEntry {
-  const Atom *rangeStart;
-  const Atom *personalityFunction;
-  const Atom *lsdaLocation;
-  const Atom *ehFrame;
-
-  uint32_t rangeLength;
-
-  // There are 3 types of compact unwind entry, distinguished by the encoding
-  // value: 0 indicates a function with no unwind info;
-  // _archHandler.dwarfCompactUnwindType() indicates that the entry defers to
-  // __eh_frame, and that the ehFrame entry will be valid; any other value is a
-  // real compact unwind entry -- personalityFunction will be set and
-  // lsdaLocation may be.
-  uint32_t encoding;
-
-  CompactUnwindEntry(const DefinedAtom *function)
-      : rangeStart(function), personalityFunction(nullptr),
-        lsdaLocation(nullptr), ehFrame(nullptr), rangeLength(function->size()),
-        encoding(0) {}
-
-  CompactUnwindEntry()
-      : rangeStart(nullptr), personalityFunction(nullptr),
-        lsdaLocation(nullptr), ehFrame(nullptr), rangeLength(0), encoding(0) {}
-};
-
-struct UnwindInfoPage {
-  ArrayRef<CompactUnwindEntry> entries;
-};
-}
-
-class UnwindInfoAtom : public SimpleDefinedAtom {
-public:
-  UnwindInfoAtom(ArchHandler &archHandler, const File &file, bool isBig,
-                 std::vector<const Atom *> &personalities,
-                 std::vector<uint32_t> &commonEncodings,
-                 std::vector<UnwindInfoPage> &pages, uint32_t numLSDAs)
-      : SimpleDefinedAtom(file), _archHandler(archHandler),
-        _commonEncodingsOffset(7 * sizeof(uint32_t)),
-        _personalityArrayOffset(_commonEncodingsOffset +
-                                commonEncodings.size() * sizeof(uint32_t)),
-        _topLevelIndexOffset(_personalityArrayOffset +
-                             personalities.size() * sizeof(uint32_t)),
-        _lsdaIndexOffset(_topLevelIndexOffset +
-                         3 * (pages.size() + 1) * sizeof(uint32_t)),
-        _firstPageOffset(_lsdaIndexOffset + 2 * numLSDAs * sizeof(uint32_t)),
-        _isBig(isBig) {
-
-    addHeader(commonEncodings.size(), personalities.size(), pages.size());
-    addCommonEncodings(commonEncodings);
-    addPersonalityFunctions(personalities);
-    addTopLevelIndexes(pages);
-    addLSDAIndexes(pages, numLSDAs);
-    addSecondLevelPages(pages);
-  }
-
-  ~UnwindInfoAtom() override = default;
-
-  ContentType contentType() const override {
-    return DefinedAtom::typeProcessedUnwindInfo;
-  }
-
-  Alignment alignment() const override { return 4; }
-
-  uint64_t size() const override { return _contents.size(); }
-
-  ContentPermissions permissions() const override {
-    return DefinedAtom::permR__;
-  }
-
-  ArrayRef<uint8_t> rawContent() const override { return _contents; }
-
-  void addHeader(uint32_t numCommon, uint32_t numPersonalities,
-                 uint32_t numPages) {
-    using normalized::write32;
-
-    uint32_t headerSize = 7 * sizeof(uint32_t);
-    _contents.resize(headerSize);
-
-    uint8_t *headerEntries = _contents.data();
-    // version
-    write32(headerEntries, 1, _isBig);
-    // commonEncodingsArraySectionOffset
-    write32(headerEntries + sizeof(uint32_t), _commonEncodingsOffset, _isBig);
-    // commonEncodingsArrayCount
-    write32(headerEntries + 2 * sizeof(uint32_t), numCommon, _isBig);
-    // personalityArraySectionOffset
-    write32(headerEntries + 3 * sizeof(uint32_t), _personalityArrayOffset,
-            _isBig);
-    // personalityArrayCount
-    write32(headerEntries + 4 * sizeof(uint32_t), numPersonalities, _isBig);
-    // indexSectionOffset
-    write32(headerEntries + 5 * sizeof(uint32_t), _topLevelIndexOffset, _isBig);
-    // indexCount
-    write32(headerEntries + 6 * sizeof(uint32_t), numPages + 1, _isBig);
-  }
-
-  /// Add the list of common encodings to the section; this is simply an array
-  /// of uint32_t compact values. Size has already been specified in the header.
-  void addCommonEncodings(std::vector<uint32_t> &commonEncodings) {
-    using normalized::write32;
-
-    _contents.resize(_commonEncodingsOffset +
-                     commonEncodings.size() * sizeof(uint32_t));
-    uint8_t *commonEncodingsArea =
-        reinterpret_cast<uint8_t *>(_contents.data() + _commonEncodingsOffset);
-
-    for (uint32_t encoding : commonEncodings) {
-      write32(commonEncodingsArea, encoding, _isBig);
-      commonEncodingsArea += sizeof(uint32_t);
-    }
-  }
-
-  void addPersonalityFunctions(std::vector<const Atom *> personalities) {
-    _contents.resize(_personalityArrayOffset +
-                     personalities.size() * sizeof(uint32_t));
-
-    for (unsigned i = 0; i < personalities.size(); ++i)
-      addImageReferenceIndirect(_personalityArrayOffset + i * sizeof(uint32_t),
-                                personalities[i]);
-  }
-
-  void addTopLevelIndexes(std::vector<UnwindInfoPage> &pages) {
-    using normalized::write32;
-
-    uint32_t numIndexes = pages.size() + 1;
-    _contents.resize(_topLevelIndexOffset + numIndexes * 3 * sizeof(uint32_t));
-
-    uint32_t pageLoc = _firstPageOffset;
-
-    // The most difficult job here is calculating the LSDAs; everything else
-    // follows fairly naturally, but we can't state where the first
-    uint8_t *indexData = &_contents[_topLevelIndexOffset];
-    uint32_t numLSDAs = 0;
-    for (unsigned i = 0; i < pages.size(); ++i) {
-      // functionOffset
-      addImageReference(_topLevelIndexOffset + 3 * i * sizeof(uint32_t),
-                        pages[i].entries[0].rangeStart);
-      // secondLevelPagesSectionOffset
-      write32(indexData + (3 * i + 1) * sizeof(uint32_t), pageLoc, _isBig);
-      write32(indexData + (3 * i + 2) * sizeof(uint32_t),
-              _lsdaIndexOffset + numLSDAs * 2 * sizeof(uint32_t), _isBig);
-
-      for (auto &entry : pages[i].entries)
-        if (entry.lsdaLocation)
-          ++numLSDAs;
-    }
-
-    // Finally, write out the final sentinel index
-    auto &finalEntry = pages[pages.size() - 1].entries.back();
-    addImageReference(_topLevelIndexOffset +
-                          3 * pages.size() * sizeof(uint32_t),
-                      finalEntry.rangeStart, finalEntry.rangeLength);
-    // secondLevelPagesSectionOffset => 0
-    write32(indexData + (3 * pages.size() + 2) * sizeof(uint32_t),
-            _lsdaIndexOffset + numLSDAs * 2 * sizeof(uint32_t), _isBig);
-  }
-
-  void addLSDAIndexes(std::vector<UnwindInfoPage> &pages, uint32_t numLSDAs) {
-    _contents.resize(_lsdaIndexOffset + numLSDAs * 2 * sizeof(uint32_t));
-
-    uint32_t curOffset = _lsdaIndexOffset;
-    for (auto &page : pages) {
-      for (auto &entry : page.entries) {
-        if (!entry.lsdaLocation)
-          continue;
-
-        addImageReference(curOffset, entry.rangeStart);
-        addImageReference(curOffset + sizeof(uint32_t), entry.lsdaLocation);
-        curOffset += 2 * sizeof(uint32_t);
-      }
-    }
-  }
-
-  void addSecondLevelPages(std::vector<UnwindInfoPage> &pages) {
-    for (auto &page : pages) {
-      addRegularSecondLevelPage(page);
-    }
-  }
-
-  void addRegularSecondLevelPage(const UnwindInfoPage &page) {
-    uint32_t curPageOffset = _contents.size();
-    const int16_t headerSize = sizeof(uint32_t) + 2 * sizeof(uint16_t);
-    uint32_t curPageSize =
-        headerSize + 2 * page.entries.size() * sizeof(uint32_t);
-    _contents.resize(curPageOffset + curPageSize);
-
-    using normalized::write32;
-    using normalized::write16;
-    // 2 => regular page
-    write32(&_contents[curPageOffset], 2, _isBig);
-    // offset of 1st entry
-    write16(&_contents[curPageOffset + 4], headerSize, _isBig);
-    write16(&_contents[curPageOffset + 6], page.entries.size(), _isBig);
-
-    uint32_t pagePos = curPageOffset + headerSize;
-    for (auto &entry : page.entries) {
-      addImageReference(pagePos, entry.rangeStart);
-
-      write32(_contents.data() + pagePos + sizeof(uint32_t), entry.encoding,
-              _isBig);
-      if ((entry.encoding & 0x0f000000U) ==
-          _archHandler.dwarfCompactUnwindType())
-        addEhFrameReference(pagePos + sizeof(uint32_t), entry.ehFrame);
-
-      pagePos += 2 * sizeof(uint32_t);
-    }
-  }
-
-  void addEhFrameReference(uint32_t offset, const Atom *dest,
-                           Reference::Addend addend = 0) {
-    addReference(Reference::KindNamespace::mach_o, _archHandler.kindArch(),
-                 _archHandler.unwindRefToEhFrameKind(), offset, dest, addend);
-  }
-
-  void addImageReference(uint32_t offset, const Atom *dest,
-                         Reference::Addend addend = 0) {
-    addReference(Reference::KindNamespace::mach_o, _archHandler.kindArch(),
-                 _archHandler.imageOffsetKind(), offset, dest, addend);
-  }
-
-  void addImageReferenceIndirect(uint32_t offset, const Atom *dest) {
-    addReference(Reference::KindNamespace::mach_o, _archHandler.kindArch(),
-                 _archHandler.imageOffsetKindIndirect(), offset, dest, 0);
-  }
-
-private:
-  mach_o::ArchHandler &_archHandler;
-  std::vector<uint8_t> _contents;
-  uint32_t _commonEncodingsOffset;
-  uint32_t _personalityArrayOffset;
-  uint32_t _topLevelIndexOffset;
-  uint32_t _lsdaIndexOffset;
-  uint32_t _firstPageOffset;
-  bool _isBig;
-};
-
-/// Pass for instantiating and optimizing GOT slots.
-///
-class CompactUnwindPass : public Pass {
-public:
-  CompactUnwindPass(const MachOLinkingContext &context)
-      : _ctx(context), _archHandler(_ctx.archHandler()),
-        _file(*_ctx.make_file<MachOFile>("<mach-o Compact Unwind Pass>")),
-        _isBig(MachOLinkingContext::isBigEndian(_ctx.arch())) {
-    _file.setOrdinal(_ctx.getNextOrdinalAndIncrement());
-  }
-
-private:
-  llvm::Error perform(SimpleFile &mergedFile) override {
-    LLVM_DEBUG(llvm::dbgs() << "MachO Compact Unwind pass\n");
-
-    std::map<const Atom *, CompactUnwindEntry> unwindLocs;
-    std::map<const Atom *, const Atom *> dwarfFrames;
-    std::vector<const Atom *> personalities;
-    uint32_t numLSDAs = 0;
-
-    // First collect all __compact_unwind and __eh_frame entries, addressable by
-    // the function referred to.
-    collectCompactUnwindEntries(mergedFile, unwindLocs, personalities,
-                                numLSDAs);
-
-    collectDwarfFrameEntries(mergedFile, dwarfFrames);
-
-    // Skip rest of pass if no unwind info.
-    if (unwindLocs.empty() && dwarfFrames.empty())
-      return llvm::Error::success();
-
-    // FIXME: if there are more than 4 personality functions then we need to
-    // defer to DWARF info for the ones we don't put in the list. They should
-    // also probably be sorted by frequency.
-    assert(personalities.size() <= 4);
-
-    // TODO: Find common encodings for use by compressed pages.
-    std::vector<uint32_t> commonEncodings;
-
-    // Now sort the entries by final address and fixup the compact encoding to
-    // its final form (i.e. set personality function bits & create DWARF
-    // references where needed).
-    std::vector<CompactUnwindEntry> unwindInfos = createUnwindInfoEntries(
-        mergedFile, unwindLocs, personalities, dwarfFrames);
-
-    // Remove any unused eh-frame atoms.
-    pruneUnusedEHFrames(mergedFile, unwindInfos, unwindLocs, dwarfFrames);
-
-    // Finally, we can start creating pages based on these entries.
-
-    LLVM_DEBUG(llvm::dbgs() << "  Splitting entries into pages\n");
-    // FIXME: we split the entries into pages naively: lots of 4k pages followed
-    // by a small one. ld64 tried to minimize space and align them to real 4k
-    // boundaries. That might be worth doing, or perhaps we could perform some
-    // minor balancing for expected number of lookups.
-    std::vector<UnwindInfoPage> pages;
-    auto remainingInfos = llvm::makeArrayRef(unwindInfos);
-    do {
-      pages.push_back(UnwindInfoPage());
-
-      // FIXME: we only create regular pages at the moment. These can hold up to
-      // 1021 entries according to the documentation.
-      unsigned entriesInPage = std::min(1021U, (unsigned)remainingInfos.size());
-
-      pages.back().entries = remainingInfos.slice(0, entriesInPage);
-      remainingInfos = remainingInfos.slice(entriesInPage);
-
-      LLVM_DEBUG(llvm::dbgs()
-                 << "    Page from "
-                 << pages.back().entries[0].rangeStart->name() << " to "
-                 << pages.back().entries.back().rangeStart->name() << " + "
-                 << llvm::format("0x%x",
-                                 pages.back().entries.back().rangeLength)
-                 << " has " << entriesInPage << " entries\n");
-    } while (!remainingInfos.empty());
-
-    auto *unwind = new (_file.allocator())
-        UnwindInfoAtom(_archHandler, _file, _isBig, personalities,
-                       commonEncodings, pages, numLSDAs);
-    mergedFile.addAtom(*unwind);
-
-    // Finally, remove all __compact_unwind atoms now that we've processed them.
-    mergedFile.removeDefinedAtomsIf([](const DefinedAtom *atom) {
-      return atom->contentType() == DefinedAtom::typeCompactUnwindInfo;
-    });
-
-    return llvm::Error::success();
-  }
-
-  void collectCompactUnwindEntries(
-      const SimpleFile &mergedFile,
-      std::map<const Atom *, CompactUnwindEntry> &unwindLocs,
-      std::vector<const Atom *> &personalities, uint32_t &numLSDAs) {
-    LLVM_DEBUG(llvm::dbgs() << "  Collecting __compact_unwind entries\n");
-
-    for (const DefinedAtom *atom : mergedFile.defined()) {
-      if (atom->contentType() != DefinedAtom::typeCompactUnwindInfo)
-        continue;
-
-      auto unwindEntry = extractCompactUnwindEntry(atom);
-      unwindLocs.insert(std::make_pair(unwindEntry.rangeStart, unwindEntry));
-
-      LLVM_DEBUG(llvm::dbgs() << "    Entry for "
-                              << unwindEntry.rangeStart->name() << ", encoding="
-                              << llvm::format("0x%08x", unwindEntry.encoding));
-      if (unwindEntry.personalityFunction)
-        LLVM_DEBUG(llvm::dbgs()
-                   << ", personality="
-                   << unwindEntry.personalityFunction->name()
-                   << ", lsdaLoc=" << unwindEntry.lsdaLocation->name());
-      LLVM_DEBUG(llvm::dbgs() << '\n');
-
-      // Count number of LSDAs we see, since we need to know how big the index
-      // will be while laying out the section.
-      if (unwindEntry.lsdaLocation)
-        ++numLSDAs;
-
-      // Gather the personality functions now, so that they're in deterministic
-      // order (derived from the DefinedAtom order).
-      if (unwindEntry.personalityFunction &&
-          !llvm::count(personalities, unwindEntry.personalityFunction))
-        personalities.push_back(unwindEntry.personalityFunction);
-    }
-  }
-
-  CompactUnwindEntry extractCompactUnwindEntry(const DefinedAtom *atom) {
-    CompactUnwindEntry entry;
-
-    for (const Reference *ref : *atom) {
-      switch (ref->offsetInAtom()) {
-      case 0:
-        // FIXME: there could legitimately be functions with multiple encoding
-        // entries. However, nothing produces them at the moment.
-        assert(ref->addend() == 0 && "unexpected offset into function");
-        entry.rangeStart = ref->target();
-        break;
-      case 0x10:
-        assert(ref->addend() == 0 && "unexpected offset into personality fn");
-        entry.personalityFunction = ref->target();
-        break;
-      case 0x18:
-        assert(ref->addend() == 0 && "unexpected offset into LSDA atom");
-        entry.lsdaLocation = ref->target();
-        break;
-      }
-    }
-
-    if (atom->rawContent().size() < 4 * sizeof(uint32_t))
-      return entry;
-
-    using normalized::read32;
-    entry.rangeLength =
-        read32(atom->rawContent().data() + 2 * sizeof(uint32_t), _isBig);
-    entry.encoding =
-        read32(atom->rawContent().data() + 3 * sizeof(uint32_t), _isBig);
-    return entry;
-  }
-
-  void
-  collectDwarfFrameEntries(const SimpleFile &mergedFile,
-                           std::map<const Atom *, const Atom *> &dwarfFrames) {
-    for (const DefinedAtom *ehFrameAtom : mergedFile.defined()) {
-      if (ehFrameAtom->contentType() != DefinedAtom::typeCFI)
-        continue;
-      if (ArchHandler::isDwarfCIE(_isBig, ehFrameAtom))
-        continue;
-
-      if (const Atom *function = _archHandler.fdeTargetFunction(ehFrameAtom))
-        dwarfFrames[function] = ehFrameAtom;
-    }
-  }
-
-  /// Every atom defined in __TEXT,__text needs an entry in the final
-  /// __unwind_info section (in order). These comes from two sources:
-  ///   + Input __compact_unwind sections where possible (after adding the
-  ///      personality function offset which is only known now).
-  ///   + A synthesised reference to __eh_frame if there's no __compact_unwind
-  ///     or too many personality functions to be accommodated.
-  std::vector<CompactUnwindEntry> createUnwindInfoEntries(
-      const SimpleFile &mergedFile,
-      const std::map<const Atom *, CompactUnwindEntry> &unwindLocs,
-      const std::vector<const Atom *> &personalities,
-      const std::map<const Atom *, const Atom *> &dwarfFrames) {
-    std::vector<CompactUnwindEntry> unwindInfos;
-
-    LLVM_DEBUG(llvm::dbgs() << "  Creating __unwind_info entries\n");
-    // The final order in the __unwind_info section must be derived from the
-    // order of typeCode atoms, since that's how they'll be put into the object
-    // file eventually (yuck!).
-    for (const DefinedAtom *atom : mergedFile.defined()) {
-      if (atom->contentType() != DefinedAtom::typeCode)
-        continue;
-
-      unwindInfos.push_back(finalizeUnwindInfoEntryForAtom(
-          atom, unwindLocs, personalities, dwarfFrames));
-
-      LLVM_DEBUG(llvm::dbgs()
-                 << "    Entry for " << atom->name() << ", final encoding="
-                 << llvm::format("0x%08x", unwindInfos.back().encoding)
-                 << '\n');
-    }
-
-    return unwindInfos;
-  }
-
-  /// Remove unused EH frames.
-  ///
-  /// An EH frame is considered unused if there is a corresponding compact
-  /// unwind atom that doesn't require the EH frame.
-  void pruneUnusedEHFrames(
-                   SimpleFile &mergedFile,
-                   const std::vector<CompactUnwindEntry> &unwindInfos,
-                   const std::map<const Atom *, CompactUnwindEntry> &unwindLocs,
-                   const std::map<const Atom *, const Atom *> &dwarfFrames) {
-
-    // Worklist of all 'used' FDEs.
-    std::vector<const DefinedAtom *> usedDwarfWorklist;
-
-    // We have to check two conditions when building the worklist:
-    // (1) EH frames used by compact unwind entries.
-    for (auto &entry : unwindInfos)
-      if (entry.ehFrame)
-        usedDwarfWorklist.push_back(cast<DefinedAtom>(entry.ehFrame));
-
-    // (2) EH frames that reference functions with no corresponding compact
-    //     unwind info.
-    for (auto &entry : dwarfFrames)
-      if (!unwindLocs.count(entry.first))
-        usedDwarfWorklist.push_back(cast<DefinedAtom>(entry.second));
-
-    // Add all transitively referenced CFI atoms by processing the worklist.
-    std::set<const Atom *> usedDwarfFrames;
-    while (!usedDwarfWorklist.empty()) {
-      const DefinedAtom *cfiAtom = usedDwarfWorklist.back();
-      usedDwarfWorklist.pop_back();
-      usedDwarfFrames.insert(cfiAtom);
-      for (const auto *ref : *cfiAtom) {
-        const DefinedAtom *cfiTarget = dyn_cast<DefinedAtom>(ref->target());
-        if (cfiTarget->contentType() == DefinedAtom::typeCFI)
-          usedDwarfWorklist.push_back(cfiTarget);
-      }
-    }
-
-    // Finally, delete all unreferenced CFI atoms.
-    mergedFile.removeDefinedAtomsIf([&](const DefinedAtom *atom) {
-      if ((atom->contentType() == DefinedAtom::typeCFI) &&
-          !usedDwarfFrames.count(atom))
-        return true;
-      return false;
-    });
-  }
-
-  CompactUnwindEntry finalizeUnwindInfoEntryForAtom(
-      const DefinedAtom *function,
-      const std::map<const Atom *, CompactUnwindEntry> &unwindLocs,
-      const std::vector<const Atom *> &personalities,
-      const std::map<const Atom *, const Atom *> &dwarfFrames) {
-    auto unwindLoc = unwindLocs.find(function);
-
-    CompactUnwindEntry entry;
-    if (unwindLoc == unwindLocs.end()) {
-      // Default entry has correct encoding (0 => no unwind), but we need to
-      // synthesise the function.
-      entry.rangeStart = function;
-      entry.rangeLength = function->size();
-    } else
-      entry = unwindLoc->second;
-
-
-    // If there's no __compact_unwind entry, or it explicitly says to use
-    // __eh_frame, we need to try and fill in the correct DWARF atom.
-    if (entry.encoding == _archHandler.dwarfCompactUnwindType() ||
-        entry.encoding == 0) {
-      auto dwarfFrame = dwarfFrames.find(function);
-      if (dwarfFrame != dwarfFrames.end()) {
-        entry.encoding = _archHandler.dwarfCompactUnwindType();
-        entry.ehFrame = dwarfFrame->second;
-      }
-    }
-
-    auto personality = llvm::find(personalities, entry.personalityFunction);
-    uint32_t personalityIdx = personality == personalities.end()
-                                  ? 0
-                                  : personality - personalities.begin() + 1;
-
-    // FIXME: We should also use DWARF when there isn't enough room for the
-    // personality function in the compact encoding.
-    assert(personalityIdx < 4 && "too many personality functions");
-
-    entry.encoding |= personalityIdx << 28;
-
-    if (entry.lsdaLocation)
-      entry.encoding |= 1U << 30;
-
-    return entry;
-  }
-
-  const MachOLinkingContext &_ctx;
-  mach_o::ArchHandler &_archHandler;
-  MachOFile &_file;
-  bool _isBig;
-};
-
-void addCompactUnwindPass(PassManager &pm, const MachOLinkingContext &ctx) {
-  assert(ctx.needsCompactUnwindPass());
-  pm.add(std::make_unique<CompactUnwindPass>(ctx));
-}
-
-} // end namespace mach_o
-} // end namespace lld
diff --git a/lld/lib/ReaderWriter/MachO/DebugInfo.h b/lld/lib/ReaderWriter/MachO/DebugInfo.h
deleted file mode 100644
index 591dd1ebad86..000000000000
--- a/lld/lib/ReaderWriter/MachO/DebugInfo.h
+++ /dev/null
@@ -1,106 +0,0 @@
-//===- lib/ReaderWriter/MachO/File.h ----------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLD_READER_WRITER_MACHO_DEBUGINFO_H
-#define LLD_READER_WRITER_MACHO_DEBUGINFO_H
-
-#include "lld/Core/Atom.h"
-#include <vector>
-
-#include "llvm/Support/Allocator.h"
-#include "llvm/Support/Format.h"
-#include "llvm/Support/raw_ostream.h"
-
-
-namespace lld {
-namespace mach_o {
-
-class DebugInfo {
-public:
-  enum class Kind {
-    Dwarf,
-    Stabs
-  };
-
-  Kind kind() const { return _kind; }
-
-  void setAllocator(std::unique_ptr<llvm::BumpPtrAllocator> allocator) {
-    _allocator = std::move(allocator);
-  }
-
-protected:
-  DebugInfo(Kind kind) : _kind(kind) {}
-
-private:
-  std::unique_ptr<llvm::BumpPtrAllocator> _allocator;
-  Kind _kind;
-};
-
-struct TranslationUnitSource {
-  StringRef name;
-  StringRef path;
-};
-
-class DwarfDebugInfo : public DebugInfo {
-public:
-  DwarfDebugInfo(TranslationUnitSource tu)
-    : DebugInfo(Kind::Dwarf), _tu(std::move(tu)) {}
-
-  static inline bool classof(const DebugInfo *di) {
-    return di->kind() == Kind::Dwarf;
-  }
-
-  const TranslationUnitSource &translationUnitSource() const { return _tu; }
-
-private:
-  TranslationUnitSource _tu;
-};
-
-struct Stab {
-  Stab(const Atom* atom, uint8_t type, uint8_t other, uint16_t desc,
-       uint32_t value, StringRef str)
-    : atom(atom), type(type), other(other), desc(desc), value(value),
-      str(str) {}
-
-  const class Atom*   atom;
-  uint8_t             type;
-  uint8_t             other;
-  uint16_t            desc;
-  uint32_t            value;
-  StringRef           str;
-};
-
-inline raw_ostream& operator<<(raw_ostream &os, Stab &s) {
-  os << "Stab -- atom: " << llvm::format("%p", s.atom) << ", type: " << (uint32_t)s.type
-     << ", other: " << (uint32_t)s.other << ", desc: " << s.desc << ", value: " << s.value
-     << ", str: '" << s.str << "'";
-  return os;
-}
-
-class StabsDebugInfo : public DebugInfo {
-public:
-
-  typedef std::vector<Stab> StabsList;
-
-  StabsDebugInfo(StabsList stabs)
-    : DebugInfo(Kind::Stabs), _stabs(std::move(stabs)) {}
-
-  static inline bool classof(const DebugInfo *di) {
-    return di->kind() == Kind::Stabs;
-  }
-
-  const StabsList& stabs() const { return _stabs; }
-
-public:
-  StabsList _stabs;
-};
-
-} // end namespace mach_o
-} // end namespace lld
-
-#endif // LLD_READER_WRITER_MACHO_DEBUGINFO_H
diff --git a/lld/lib/ReaderWriter/MachO/ExecutableAtoms.h b/lld/lib/ReaderWriter/MachO/ExecutableAtoms.h
deleted file mode 100644
index ce94be457026..000000000000
--- a/lld/lib/ReaderWriter/MachO/ExecutableAtoms.h
+++ /dev/null
@@ -1,154 +0,0 @@
-//===- lib/ReaderWriter/MachO/ExecutableAtoms.h ---------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLD_READER_WRITER_MACHO_EXECUTABLE_ATOMS_H
-#define LLD_READER_WRITER_MACHO_EXECUTABLE_ATOMS_H
-
-#include "Atoms.h"
-#include "File.h"
-
-#include "llvm/BinaryFormat/MachO.h"
-
-#include "lld/Core/DefinedAtom.h"
-#include "lld/Core/File.h"
-#include "lld/Core/LinkingContext.h"
-#include "lld/Core/Reference.h"
-#include "lld/Core/Simple.h"
-#include "lld/Core/UndefinedAtom.h"
-#include "lld/ReaderWriter/MachOLinkingContext.h"
-
-namespace lld {
-namespace mach_o {
-
-
-//
-// CEntryFile adds an UndefinedAtom for "_main" so that the Resolving
-// phase will fail if "_main" is undefined.
-//
-class CEntryFile : public SimpleFile {
-public:
-  CEntryFile(const MachOLinkingContext &context)
-      : SimpleFile("C entry", kindCEntryObject),
-       _undefMain(*this, context.entrySymbolName()) {
-    this->addAtom(_undefMain);
-  }
-
-private:
-  SimpleUndefinedAtom   _undefMain;
-};
-
-
-//
-// StubHelperFile adds an UndefinedAtom for "dyld_stub_binder" so that
-// the Resolveing phase will fail if "dyld_stub_binder" is undefined.
-//
-class StubHelperFile : public SimpleFile {
-public:
-  StubHelperFile(const MachOLinkingContext &context)
-      : SimpleFile("stub runtime", kindStubHelperObject),
-        _undefBinder(*this, context.binderSymbolName()) {
-    this->addAtom(_undefBinder);
-  }
-
-private:
-  SimpleUndefinedAtom   _undefBinder;
-};
-
-
-//
-// MachHeaderAliasFile lazily instantiates the magic symbols that mark the start
-// of the mach_header for final linked images.
-//
-class MachHeaderAliasFile : public SimpleFile {
-public:
-  MachHeaderAliasFile(const MachOLinkingContext &context)
-    : SimpleFile("mach_header symbols", kindHeaderObject) {
-    StringRef machHeaderSymbolName;
-    DefinedAtom::Scope symbolScope = DefinedAtom::scopeLinkageUnit;
-    StringRef dsoHandleName;
-    switch (context.outputMachOType()) {
-    case llvm::MachO::MH_OBJECT:
-      machHeaderSymbolName = "__mh_object_header";
-      break;
-    case llvm::MachO::MH_EXECUTE:
-      machHeaderSymbolName = "__mh_execute_header";
-      symbolScope = DefinedAtom::scopeGlobal;
-      dsoHandleName = "___dso_handle";
-      break;
-    case llvm::MachO::MH_FVMLIB:
-      llvm_unreachable("no mach_header symbol for file type");
-    case llvm::MachO::MH_CORE:
-      llvm_unreachable("no mach_header symbol for file type");
-    case llvm::MachO::MH_PRELOAD:
-      llvm_unreachable("no mach_header symbol for file type");
-    case llvm::MachO::MH_DYLIB:
-      machHeaderSymbolName = "__mh_dylib_header";
-      dsoHandleName = "___dso_handle";
-      break;
-    case llvm::MachO::MH_DYLINKER:
-      machHeaderSymbolName = "__mh_dylinker_header";
-      dsoHandleName = "___dso_handle";
-      break;
-    case llvm::MachO::MH_BUNDLE:
-      machHeaderSymbolName = "__mh_bundle_header";
-      dsoHandleName = "___dso_handle";
-      break;
-    case llvm::MachO::MH_DYLIB_STUB:
-      llvm_unreachable("no mach_header symbol for file type");
-    case llvm::MachO::MH_DSYM:
-      llvm_unreachable("no mach_header symbol for file type");
-    case llvm::MachO::MH_KEXT_BUNDLE:
-      dsoHandleName = "___dso_handle";
-      break;
-    }
-    if (!machHeaderSymbolName.empty())
-      _definedAtoms.push_back(new (allocator()) MachODefinedAtom(
-          *this, machHeaderSymbolName, symbolScope,
-          DefinedAtom::typeMachHeader, DefinedAtom::mergeNo, false,
-          true /* noDeadStrip */,
-          ArrayRef<uint8_t>(), DefinedAtom::Alignment(4096)));
-
-    if (!dsoHandleName.empty())
-      _definedAtoms.push_back(new (allocator()) MachODefinedAtom(
-          *this, dsoHandleName, DefinedAtom::scopeLinkageUnit,
-          DefinedAtom::typeDSOHandle, DefinedAtom::mergeNo, false,
-          true /* noDeadStrip */,
-          ArrayRef<uint8_t>(), DefinedAtom::Alignment(1)));
-  }
-
-  const AtomRange<DefinedAtom> defined() const override {
-    return _definedAtoms;
-  }
-  const AtomRange<UndefinedAtom> undefined() const override {
-    return _noUndefinedAtoms;
-  }
-
-  const AtomRange<SharedLibraryAtom> sharedLibrary() const override {
-    return _noSharedLibraryAtoms;
-  }
-
-  const AtomRange<AbsoluteAtom> absolute() const override {
-    return _noAbsoluteAtoms;
-  }
-
-  void clearAtoms() override {
-    _definedAtoms.clear();
-    _noUndefinedAtoms.clear();
-    _noSharedLibraryAtoms.clear();
-    _noAbsoluteAtoms.clear();
-  }
-
-
-private:
-  mutable AtomVector<DefinedAtom> _definedAtoms;
-};
-
-} // namespace mach_o
-} // namespace lld
-
-#endif // LLD_READER_WRITER_MACHO_EXECUTABLE_ATOMS_H
diff --git a/lld/lib/ReaderWriter/MachO/File.h b/lld/lib/ReaderWriter/MachO/File.h
deleted file mode 100644
index 77832969c6b3..000000000000
--- a/lld/lib/ReaderWriter/MachO/File.h
+++ /dev/null
@@ -1,467 +0,0 @@
-//===- lib/ReaderWriter/MachO/File.h ----------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLD_READER_WRITER_MACHO_FILE_H
-#define LLD_READER_WRITER_MACHO_FILE_H
-
-#include "Atoms.h"
-#include "DebugInfo.h"
-#include "MachONormalizedFile.h"
-#include "lld/Core/SharedLibraryFile.h"
-#include "lld/Core/Simple.h"
-#include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/StringMap.h"
-#include "llvm/Support/Format.h"
-#include "llvm/TextAPI/InterfaceFile.h"
-#include "llvm/TextAPI/TextAPIReader.h"
-#include <unordered_map>
-
-namespace lld {
-namespace mach_o {
-
-using lld::mach_o::normalized::Section;
-
-class MachOFile : public SimpleFile {
-public:
-
-  /// Real file constructor - for on-disk files.
-  MachOFile(std::unique_ptr<MemoryBuffer> mb, MachOLinkingContext *ctx)
-    : SimpleFile(mb->getBufferIdentifier(), File::kindMachObject),
-      _mb(std::move(mb)), _ctx(ctx) {}
-
-  /// Dummy file constructor - for virtual files.
-  MachOFile(StringRef path)
-    : SimpleFile(path, File::kindMachObject) {}
-
-  void addDefinedAtom(StringRef name, Atom::Scope scope,
-                      DefinedAtom::ContentType type, DefinedAtom::Merge merge,
-                      uint64_t sectionOffset, uint64_t contentSize, bool thumb,
-                      bool noDeadStrip, bool copyRefs,
-                      const Section *inSection) {
-    assert(sectionOffset+contentSize <= inSection->content.size());
-    ArrayRef<uint8_t> content = inSection->content.slice(sectionOffset,
-                                                        contentSize);
-    if (copyRefs) {
-      // Make a copy of the atom's name and content that is owned by this file.
-      name = name.copy(allocator());
-      content = content.copy(allocator());
-    }
-    DefinedAtom::Alignment align(
-        inSection->alignment,
-        sectionOffset % inSection->alignment);
-    auto *atom =
-        new (allocator()) MachODefinedAtom(*this, name, scope, type, merge,
-                                           thumb, noDeadStrip, content, align);
-    addAtomForSection(inSection, atom, sectionOffset);
-  }
-
-  void addDefinedAtomInCustomSection(StringRef name, Atom::Scope scope,
-                      DefinedAtom::ContentType type, DefinedAtom::Merge merge,
-                      bool thumb, bool noDeadStrip, uint64_t sectionOffset,
-                      uint64_t contentSize, StringRef sectionName,
-                      bool copyRefs, const Section *inSection) {
-    assert(sectionOffset+contentSize <= inSection->content.size());
-    ArrayRef<uint8_t> content = inSection->content.slice(sectionOffset,
-                                                        contentSize);
-   if (copyRefs) {
-      // Make a copy of the atom's name and content that is owned by this file.
-      name = name.copy(allocator());
-      content = content.copy(allocator());
-      sectionName = sectionName.copy(allocator());
-    }
-    DefinedAtom::Alignment align(
-        inSection->alignment,
-        sectionOffset % inSection->alignment);
-    auto *atom =
-        new (allocator()) MachODefinedCustomSectionAtom(*this, name, scope, type,
-                                                        merge, thumb,
-                                                        noDeadStrip, content,
-                                                        sectionName, align);
-    addAtomForSection(inSection, atom, sectionOffset);
-  }
-
-  void addZeroFillDefinedAtom(StringRef name, Atom::Scope scope,
-                              uint64_t sectionOffset, uint64_t size,
-                              bool noDeadStrip, bool copyRefs,
-                              const Section *inSection) {
-    if (copyRefs) {
-      // Make a copy of the atom's name and content that is owned by this file.
-      name = name.copy(allocator());
-    }
-    DefinedAtom::Alignment align(
-        inSection->alignment,
-        sectionOffset % inSection->alignment);
-
-    DefinedAtom::ContentType type = DefinedAtom::typeUnknown;
-    switch (inSection->type) {
-    case llvm::MachO::S_ZEROFILL:
-      type = DefinedAtom::typeZeroFill;
-      break;
-    case llvm::MachO::S_THREAD_LOCAL_ZEROFILL:
-      type = DefinedAtom::typeTLVInitialZeroFill;
-      break;
-    default:
-      llvm_unreachable("Unrecognized zero-fill section");
-    }
-
-    auto *atom =
-        new (allocator()) MachODefinedAtom(*this, name, scope, type, size,
-                                           noDeadStrip, align);
-    addAtomForSection(inSection, atom, sectionOffset);
-  }
-
-  void addUndefinedAtom(StringRef name, bool copyRefs) {
-    if (copyRefs) {
-      // Make a copy of the atom's name that is owned by this file.
-      name = name.copy(allocator());
-    }
-    auto *atom = new (allocator()) SimpleUndefinedAtom(*this, name);
-    addAtom(*atom);
-    _undefAtoms[name] = atom;
-  }
-
-  void addTentativeDefAtom(StringRef name, Atom::Scope scope, uint64_t size,
-                           DefinedAtom::Alignment align, bool copyRefs) {
-    if (copyRefs) {
-      // Make a copy of the atom's name that is owned by this file.
-      name = name.copy(allocator());
-    }
-    auto *atom =
-        new (allocator()) MachOTentativeDefAtom(*this, name, scope, size, align);
-    addAtom(*atom);
-    _undefAtoms[name] = atom;
-  }
-
-  /// Search this file for the atom from 'section' that covers
-  /// 'offsetInSect'.  Returns nullptr is no atom found.
-  MachODefinedAtom *findAtomCoveringAddress(const Section &section,
-                                            uint64_t offsetInSect,
-                                            uint32_t *foundOffsetAtom=nullptr) {
-    const auto &pos = _sectionAtoms.find(&section);
-    if (pos == _sectionAtoms.end())
-      return nullptr;
-    const auto &vec = pos->second;
-    assert(offsetInSect < section.content.size());
-    // Vector of atoms for section are already sorted, so do binary search.
-    const auto &atomPos = std::lower_bound(vec.begin(), vec.end(), offsetInSect,
-        [offsetInSect](const SectionOffsetAndAtom &ao,
-                       uint64_t targetAddr) -> bool {
-          // Each atom has a start offset of its slice of the
-          // section's content. This compare function must return true
-          // iff the atom's range is before the offset being searched for.
-          uint64_t atomsEndOffset = ao.offset+ao.atom->rawContent().size();
-          return (atomsEndOffset <= offsetInSect);
-        });
-    if (atomPos == vec.end())
-      return nullptr;
-    if (foundOffsetAtom)
-      *foundOffsetAtom = offsetInSect - atomPos->offset;
-    return atomPos->atom;
-  }
-
-  /// Searches this file for an UndefinedAtom named 'name'. Returns
-  /// nullptr is no such atom found.
-  const lld::Atom *findUndefAtom(StringRef name) {
-    auto pos = _undefAtoms.find(name);
-    if (pos == _undefAtoms.end())
-      return nullptr;
-    return pos->second;
-  }
-
-  typedef std::function<void (MachODefinedAtom* atom)> DefinedAtomVisitor;
-
-  void eachDefinedAtom(DefinedAtomVisitor vistor) {
-    for (auto &sectAndAtoms : _sectionAtoms) {
-      for (auto &offAndAtom : sectAndAtoms.second) {
-        vistor(offAndAtom.atom);
-      }
-    }
-  }
-
-  typedef std::function<void(MachODefinedAtom *atom, uint64_t offset)>
-      SectionAtomVisitor;
-
-  void eachAtomInSection(const Section &section, SectionAtomVisitor visitor) {
-    auto pos = _sectionAtoms.find(&section);
-    if (pos == _sectionAtoms.end())
-      return;
-    auto vec = pos->second;
-
-    for (auto &offAndAtom : vec)
-      visitor(offAndAtom.atom, offAndAtom.offset);
-  }
-
-  MachOLinkingContext::Arch arch() const { return _arch; }
-  void setArch(MachOLinkingContext::Arch arch) { _arch = arch; }
-
-  MachOLinkingContext::OS OS() const { return _os; }
-  void setOS(MachOLinkingContext::OS os) { _os = os; }
-
-  MachOLinkingContext::ObjCConstraint objcConstraint() const {
-    return _objcConstraint;
-  }
-  void setObjcConstraint(MachOLinkingContext::ObjCConstraint v) {
-    _objcConstraint = v;
-  }
-
-  uint32_t minVersion() const { return _minVersion; }
-  void setMinVersion(uint32_t v) { _minVersion = v; }
-
-  LoadCommandType minVersionLoadCommandKind() const {
-    return _minVersionLoadCommandKind;
-  }
-  void setMinVersionLoadCommandKind(LoadCommandType v) {
-    _minVersionLoadCommandKind = v;
-  }
-
-  uint32_t swiftVersion() const { return _swiftVersion; }
-  void setSwiftVersion(uint32_t v) { _swiftVersion = v; }
-
-  bool subsectionsViaSymbols() const {
-    return _flags & llvm::MachO::MH_SUBSECTIONS_VIA_SYMBOLS;
-  }
-  void setFlags(normalized::FileFlags v) { _flags = v; }
-
-  /// Methods for support type inquiry through isa, cast, and dyn_cast:
-  static inline bool classof(const File *F) {
-    return F->kind() == File::kindMachObject;
-  }
-
-  void setDebugInfo(std::unique_ptr<DebugInfo> debugInfo) {
-    _debugInfo = std::move(debugInfo);
-  }
-
-  DebugInfo* debugInfo() const { return _debugInfo.get(); }
-  std::unique_ptr<DebugInfo> takeDebugInfo() { return std::move(_debugInfo); }
-
-protected:
-  std::error_code doParse() override {
-    // Convert binary file to normalized mach-o.
-    auto normFile = normalized::readBinary(_mb, _ctx->arch());
-    if (auto ec = normFile.takeError())
-      return llvm::errorToErrorCode(std::move(ec));
-    // Convert normalized mach-o to atoms.
-    if (auto ec = normalized::normalizedObjectToAtoms(this, **normFile, false))
-      return llvm::errorToErrorCode(std::move(ec));
-    return std::error_code();
-  }
-
-private:
-  struct SectionOffsetAndAtom { uint64_t offset;  MachODefinedAtom *atom; };
-
-  void addAtomForSection(const Section *inSection, MachODefinedAtom* atom,
-                         uint64_t sectionOffset) {
-    SectionOffsetAndAtom offAndAtom;
-    offAndAtom.offset = sectionOffset;
-    offAndAtom.atom   = atom;
-     _sectionAtoms[inSection].push_back(offAndAtom);
-    addAtom(*atom);
-  }
-
-  typedef llvm::DenseMap<const normalized::Section *,
-                         std::vector<SectionOffsetAndAtom>>  SectionToAtoms;
-  typedef llvm::StringMap<const lld::Atom *> NameToAtom;
-
-  std::unique_ptr<MemoryBuffer> _mb;
-  MachOLinkingContext          *_ctx;
-  SectionToAtoms                _sectionAtoms;
-  NameToAtom                     _undefAtoms;
-  MachOLinkingContext::Arch      _arch = MachOLinkingContext::arch_unknown;
-  MachOLinkingContext::OS        _os = MachOLinkingContext::OS::unknown;
-  uint32_t                       _minVersion = 0;
-  LoadCommandType               _minVersionLoadCommandKind = (LoadCommandType)0;
-  MachOLinkingContext::ObjCConstraint _objcConstraint =
-      MachOLinkingContext::objc_unknown;
-  uint32_t                       _swiftVersion = 0;
-  normalized::FileFlags        _flags = llvm::MachO::MH_SUBSECTIONS_VIA_SYMBOLS;
-  std::unique_ptr<DebugInfo>   _debugInfo;
-};
-
-class MachODylibFile : public SharedLibraryFile {
-public:
-  MachODylibFile(std::unique_ptr<MemoryBuffer> mb, MachOLinkingContext *ctx)
-      : SharedLibraryFile(mb->getBufferIdentifier()),
-        _mb(std::move(mb)), _ctx(ctx) {}
-
-  MachODylibFile(StringRef path) : SharedLibraryFile(path) {}
-
-  OwningAtomPtr<SharedLibraryAtom> exports(StringRef name) const override {
-    // Pass down _installName so that if this requested symbol
-    // is re-exported through this dylib, the SharedLibraryAtom's loadName()
-    // is this dylib installName and not the implementation dylib's.
-    // NOTE: isData is not needed for dylibs (it matters for static libs).
-    return exports(name, _installName);
-  }
-
-  /// Adds symbol name that this dylib exports. The corresponding
-  /// SharedLibraryAtom is created lazily (since most symbols are not used).
-  void addExportedSymbol(StringRef name, bool weakDef, bool copyRefs) {
-    if (copyRefs) {
-      name = name.copy(allocator());
-    }
-    AtomAndFlags info(weakDef);
-    _nameToAtom[name] = info;
-  }
-
-  void addReExportedDylib(StringRef dylibPath) {
-    _reExportedDylibs.emplace_back(dylibPath);
-  }
-
-  StringRef installName() const { return _installName; }
-  uint32_t currentVersion() { return _currentVersion; }
-  uint32_t compatVersion() { return _compatVersion; }
-
-  void setInstallName(StringRef name) { _installName = name; }
-  void setCompatVersion(uint32_t version) { _compatVersion = version; }
-  void setCurrentVersion(uint32_t version) { _currentVersion = version; }
-
-  typedef std::function<MachODylibFile *(StringRef)> FindDylib;
-
-  void loadReExportedDylibs(FindDylib find) {
-    for (ReExportedDylib &entry : _reExportedDylibs) {
-      if (!entry.file)
-        entry.file = find(entry.path);
-    }
-  }
-
-  StringRef getDSOName() const override { return _installName; }
-
-  std::error_code doParse() override {
-    // Convert binary file to normalized mach-o.
-    auto normFile = normalized::readBinary(_mb, _ctx->arch());
-    if (auto ec = normFile.takeError())
-      return llvm::errorToErrorCode(std::move(ec));
-    // Convert normalized mach-o to atoms.
-    if (auto ec = normalized::normalizedDylibToAtoms(this, **normFile, false))
-      return llvm::errorToErrorCode(std::move(ec));
-    return std::error_code();
-  }
-
-protected:
-  OwningAtomPtr<SharedLibraryAtom> exports(StringRef name,
-                                   StringRef installName) const {
-    // First, check if requested symbol is directly implemented by this dylib.
-    auto entry = _nameToAtom.find(name);
-    if (entry != _nameToAtom.end()) {
-      // FIXME: Make this map a set and only used in assert builds.
-      // Note, its safe to assert here as the resolver is the only client of
-      // this API and it only requests exports for undefined symbols.
-      // If we return from here we are no longer undefined so we should never
-      // get here again.
-      assert(!entry->second.atom && "Duplicate shared library export");
-      bool weakDef = entry->second.weakDef;
-      auto *atom = new (allocator()) MachOSharedLibraryAtom(*this, name,
-                                                            installName,
-                                                            weakDef);
-      entry->second.atom = atom;
-      return atom;
-    }
-
-    // Next, check if symbol is implemented in some re-exported dylib.
-    for (const ReExportedDylib &dylib : _reExportedDylibs) {
-      assert(dylib.file);
-      auto atom = dylib.file->exports(name, installName);
-      if (atom.get())
-        return atom;
-    }
-
-    // Symbol not exported or re-exported by this dylib.
-    return nullptr;
-  }
-
-  struct ReExportedDylib {
-    ReExportedDylib(StringRef p) : path(p), file(nullptr) { }
-    ReExportedDylib(StringRef p, MachODylibFile *file) : path(p), file(file) { }
-    StringRef       path;
-    MachODylibFile *file;
-  };
-
-  struct AtomAndFlags {
-    AtomAndFlags() : atom(nullptr), weakDef(false) { }
-    AtomAndFlags(bool weak) : atom(nullptr), weakDef(weak) { }
-    const SharedLibraryAtom  *atom;
-    bool                      weakDef;
-  };
-
-  std::unique_ptr<MemoryBuffer>              _mb;
-  MachOLinkingContext                       *_ctx;
-  StringRef                                  _installName;
-  uint32_t                                   _currentVersion;
-  uint32_t                                   _compatVersion;
-  std::vector<ReExportedDylib>               _reExportedDylibs;
-  mutable std::unordered_map<StringRef, AtomAndFlags> _nameToAtom;
-};
-
-class TAPIFile : public MachODylibFile {
-public:
-
-  TAPIFile(std::unique_ptr<MemoryBuffer> mb, MachOLinkingContext *ctx)
-      : MachODylibFile(std::move(mb), ctx) {}
-
-  std::error_code doParse() override {
-
-    llvm::Expected<std::unique_ptr<llvm::MachO::InterfaceFile>> result =
-        llvm::MachO::TextAPIReader::get(*_mb);
-    if (!result)
-      return std::make_error_code(std::errc::invalid_argument);
-
-    std::unique_ptr<llvm::MachO::InterfaceFile> interface{std::move(*result)};
-    return loadFromInterface(*interface);
-  }
-
-private:
-  std::error_code loadFromInterface(llvm::MachO::InterfaceFile &interface) {
-    llvm::MachO::Architecture arch;
-    switch(_ctx->arch()) {
-    case MachOLinkingContext::arch_x86:
-      arch = llvm::MachO::AK_i386;
-      break;
-    case MachOLinkingContext::arch_x86_64:
-      arch = llvm::MachO::AK_x86_64;
-      break;
-    case MachOLinkingContext::arch_arm64:
-      arch = llvm::MachO::AK_arm64;
-      break;
-    default:
-      return std::make_error_code(std::errc::invalid_argument);
-    }
-
-    setInstallName(interface.getInstallName().copy(allocator()));
-    // TODO(compnerd) filter out symbols based on the target platform
-    for (const auto symbol : interface.symbols())
-      if (symbol->getArchitectures().has(arch))
-        addExportedSymbol(symbol->getName(), symbol->isWeakDefined(), true);
-
-    for (const llvm::MachO::InterfaceFileRef &reexport :
-         interface.reexportedLibraries())
-      addReExportedDylib(reexport.getInstallName().copy(allocator()));
-
-    for (const auto& document : interface.documents()) {
-      for (auto& reexport : _reExportedDylibs) {
-        if (reexport.path != document->getInstallName())
-          continue;
-        assert(!reexport.file);
-        _ownedFiles.push_back(std::make_unique<TAPIFile>(
-            MemoryBuffer::getMemBuffer("", _mb->getBufferIdentifier()), _ctx));
-        reexport.file = _ownedFiles.back().get();
-        std::error_code err = _ownedFiles.back()->loadFromInterface(*document);
-        if (err)
-          return err;
-      }
-    }
-
-    return std::error_code();
-  }
-
-  std::vector<std::unique_ptr<TAPIFile>> _ownedFiles;
-};
-
-} // end namespace mach_o
-} // end namespace lld
-
-#endif // LLD_READER_WRITER_MACHO_FILE_H
diff --git a/lld/lib/ReaderWriter/MachO/FlatNamespaceFile.h b/lld/lib/ReaderWriter/MachO/FlatNamespaceFile.h
deleted file mode 100644
index 1885effef49f..000000000000
--- a/lld/lib/ReaderWriter/MachO/FlatNamespaceFile.h
+++ /dev/null
@@ -1,62 +0,0 @@
-//===- lib/ReaderWriter/MachO/FlatNamespaceFile.h -------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLD_READER_WRITER_MACHO_FLAT_NAMESPACE_FILE_H
-#define LLD_READER_WRITER_MACHO_FLAT_NAMESPACE_FILE_H
-
-#include "Atoms.h"
-#include "lld/Core/SharedLibraryFile.h"
-#include "lld/ReaderWriter/MachOLinkingContext.h"
-#include "llvm/Support/Debug.h"
-
-namespace lld {
-namespace mach_o {
-
-//
-// A FlateNamespaceFile instance may be added as a resolution source of last
-// resort, depending on how -flat_namespace and -undefined are set.
-//
-class FlatNamespaceFile : public SharedLibraryFile {
-public:
-  FlatNamespaceFile(const MachOLinkingContext &context)
-    : SharedLibraryFile("flat namespace") { }
-
-  OwningAtomPtr<SharedLibraryAtom> exports(StringRef name) const override {
-    return new (allocator()) MachOSharedLibraryAtom(*this, name, getDSOName(),
-                                                    false);
-  }
-
-  StringRef getDSOName() const override { return "flat-namespace"; }
-
-  const AtomRange<DefinedAtom> defined() const override {
-    return _noDefinedAtoms;
-  }
-  const AtomRange<UndefinedAtom> undefined() const override {
-    return _noUndefinedAtoms;
-  }
-
-  const AtomRange<SharedLibraryAtom> sharedLibrary() const override {
-    return _noSharedLibraryAtoms;
-  }
-
-  const AtomRange<AbsoluteAtom> absolute() const override {
-    return _noAbsoluteAtoms;
-  }
-
-  void clearAtoms() override {
-    _noDefinedAtoms.clear();
-    _noUndefinedAtoms.clear();
-    _noSharedLibraryAtoms.clear();
-    _noAbsoluteAtoms.clear();
-  }
-};
-
-} // namespace mach_o
-} // namespace lld
-
-#endif // LLD_READER_WRITER_MACHO_FLAT_NAMESPACE_FILE_H
diff --git a/lld/lib/ReaderWriter/MachO/GOTPass.cpp b/lld/lib/ReaderWriter/MachO/GOTPass.cpp
deleted file mode 100644
index 9cb5ab5eab12..000000000000
--- a/lld/lib/ReaderWriter/MachO/GOTPass.cpp
+++ /dev/null
@@ -1,183 +0,0 @@
-//===- lib/ReaderWriter/MachO/GOTPass.cpp -----------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-///
-/// \file
-/// This linker pass transforms all GOT kind references to real references.
-/// That is, in assembly you can write something like:
-///     movq foo@GOTPCREL(%rip), %rax
-/// which means you want to load a pointer to "foo" out of the GOT (global
-/// Offsets Table). In the object file, the Atom containing this instruction
-/// has a Reference whose target is an Atom named "foo" and the Reference
-/// kind is a GOT load.  The linker needs to instantiate a pointer sized
-/// GOT entry.  This is done be creating a GOT Atom to represent that pointer
-/// sized data in this pass, and altering the Atom graph so the Reference now
-/// points to the GOT Atom entry (corresponding to "foo") and changing the
-/// Reference Kind to reflect it is now pointing to a GOT entry (rather
-/// then needing a GOT entry).
-///
-/// There is one optimization the linker can do here.  If the target of the GOT
-/// is in the same linkage unit and does not need to be interposable, and
-/// the GOT use is just a load (not some other operation), this pass can
-/// transform that load into an LEA (add).  This optimizes away one memory load
-/// which at runtime that could stall the pipeline.  This optimization only
-/// works for architectures in which a (GOT) load instruction can be change to
-/// an LEA instruction that is the same size.  The method isGOTAccess() should
-/// only return true for "canBypassGOT" if this optimization is supported.
-///
-//===----------------------------------------------------------------------===//
-
-#include "ArchHandler.h"
-#include "File.h"
-#include "MachOPasses.h"
-#include "lld/Common/LLVM.h"
-#include "lld/Core/DefinedAtom.h"
-#include "lld/Core/File.h"
-#include "lld/Core/Reference.h"
-#include "lld/Core/Simple.h"
-#include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/STLExtras.h"
-
-namespace lld {
-namespace mach_o {
-
-//
-//  GOT Entry Atom created by the GOT pass.
-//
-class GOTEntryAtom : public SimpleDefinedAtom {
-public:
-  GOTEntryAtom(const File &file, bool is64, StringRef name)
-    : SimpleDefinedAtom(file), _is64(is64), _name(name) { }
-
-  ~GOTEntryAtom() override = default;
-
-  ContentType contentType() const override {
-    return DefinedAtom::typeGOT;
-  }
-
-  Alignment alignment() const override {
-    return _is64 ? 8 : 4;
-  }
-
-  uint64_t size() const override {
-    return _is64 ? 8 : 4;
-  }
-
-  ContentPermissions permissions() const override {
-    return DefinedAtom::permRW_;
-  }
-
-  ArrayRef<uint8_t> rawContent() const override {
-    static const uint8_t zeros[] =
-        { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
-    return llvm::makeArrayRef(zeros, size());
-  }
-
-  StringRef slotName() const {
-    return _name;
-  }
-
-private:
-  const bool _is64;
-  StringRef _name;
-};
-
-/// Pass for instantiating and optimizing GOT slots.
-///
-class GOTPass : public Pass {
-public:
-  GOTPass(const MachOLinkingContext &context)
-      : _ctx(context), _archHandler(_ctx.archHandler()),
-        _file(*_ctx.make_file<MachOFile>("<mach-o GOT Pass>")) {
-    _file.setOrdinal(_ctx.getNextOrdinalAndIncrement());
-  }
-
-private:
-  llvm::Error perform(SimpleFile &mergedFile) override {
-    // Scan all references in all atoms.
-    for (const DefinedAtom *atom : mergedFile.defined()) {
-      for (const Reference *ref : *atom) {
-        // Look at instructions accessing the GOT.
-        bool canBypassGOT;
-        if (!_archHandler.isGOTAccess(*ref, canBypassGOT))
-          continue;
-        const Atom *target = ref->target();
-        assert(target != nullptr);
-
-        if (!shouldReplaceTargetWithGOTAtom(target, canBypassGOT)) {
-          // Update reference kind to reflect that target is a direct access.
-          _archHandler.updateReferenceToGOT(ref, false);
-        } else {
-          // Replace the target with a reference to a GOT entry.
-          const DefinedAtom *gotEntry = makeGOTEntry(target);
-          const_cast<Reference *>(ref)->setTarget(gotEntry);
-          // Update reference kind to reflect that target is now a GOT entry.
-          _archHandler.updateReferenceToGOT(ref, true);
-        }
-      }
-    }
-
-    // Sort and add all created GOT Atoms to merged file
-    std::vector<const GOTEntryAtom *> entries;
-    entries.reserve(_targetToGOT.size());
-    for (auto &it : _targetToGOT)
-      entries.push_back(it.second);
-    std::sort(entries.begin(), entries.end(),
-              [](const GOTEntryAtom *left, const GOTEntryAtom *right) {
-      return (left->slotName().compare(right->slotName()) < 0);
-    });
-    for (const GOTEntryAtom *slot : entries)
-      mergedFile.addAtom(*slot);
-
-    return llvm::Error::success();
-  }
-
-  bool shouldReplaceTargetWithGOTAtom(const Atom *target, bool canBypassGOT) {
-    // Accesses to shared library symbols must go through GOT.
-    if (isa<SharedLibraryAtom>(target))
-      return true;
-    // Accesses to interposable symbols in same linkage unit must also go
-    // through GOT.
-    const DefinedAtom *defTarget = dyn_cast<DefinedAtom>(target);
-    if (defTarget != nullptr &&
-        defTarget->interposable() != DefinedAtom::interposeNo) {
-      assert(defTarget->scope() != DefinedAtom::scopeTranslationUnit);
-      return true;
-    }
-    // Target does not require indirection.  So, if instruction allows GOT to be
-    // by-passed, do that optimization and don't create GOT entry.
-    return !canBypassGOT;
-  }
-
-  const DefinedAtom *makeGOTEntry(const Atom *target) {
-    auto pos = _targetToGOT.find(target);
-    if (pos == _targetToGOT.end()) {
-      auto *gotEntry = new (_file.allocator())
-          GOTEntryAtom(_file, _ctx.is64Bit(), target->name());
-      _targetToGOT[target] = gotEntry;
-      const ArchHandler::ReferenceInfo &nlInfo = _archHandler.stubInfo().
-                                                nonLazyPointerReferenceToBinder;
-      gotEntry->addReference(Reference::KindNamespace::mach_o, nlInfo.arch,
-                             nlInfo.kind, 0, target, 0);
-      return gotEntry;
-    }
-    return pos->second;
-  }
-
-  const MachOLinkingContext &_ctx;
-  mach_o::ArchHandler                             &_archHandler;
-  MachOFile                                       &_file;
-  llvm::DenseMap<const Atom*, const GOTEntryAtom*> _targetToGOT;
-};
-
-void addGOTPass(PassManager &pm, const MachOLinkingContext &ctx) {
-  assert(ctx.needsGOTPass());
-  pm.add(std::make_unique<GOTPass>(ctx));
-}
-
-} // end namespace mach_o
-} // end namespace lld
diff --git a/lld/lib/ReaderWriter/MachO/LayoutPass.cpp b/lld/lib/ReaderWriter/MachO/LayoutPass.cpp
deleted file mode 100644
index e92fdf1b4913..000000000000
--- a/lld/lib/ReaderWriter/MachO/LayoutPass.cpp
+++ /dev/null
@@ -1,490 +0,0 @@
-//===-- ReaderWriter/MachO/LayoutPass.cpp - Layout atoms ------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "LayoutPass.h"
-#include "lld/Core/Instrumentation.h"
-#include "lld/Core/PassManager.h"
-#include "lld/ReaderWriter/MachOLinkingContext.h"
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/Twine.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/Parallel.h"
-#include <algorithm>
-#include <set>
-#include <utility>
-
-using namespace lld;
-
-#define DEBUG_TYPE "LayoutPass"
-
-namespace lld {
-namespace mach_o {
-
-static bool compareAtoms(const LayoutPass::SortKey &,
-                         const LayoutPass::SortKey &,
-                         LayoutPass::SortOverride customSorter);
-
-#ifndef NDEBUG
-// Return "reason (leftval, rightval)"
-static std::string formatReason(StringRef reason, int leftVal, int rightVal) {
-  return (Twine(reason) + " (" + Twine(leftVal) + ", " + Twine(rightVal) + ")")
-      .str();
-}
-
-// Less-than relationship of two atoms must be transitive, which is, if a < b
-// and b < c, a < c must be true. This function checks the transitivity by
-// checking the sort results.
-static void checkTransitivity(std::vector<LayoutPass::SortKey> &vec,
-                              LayoutPass::SortOverride customSorter) {
-  for (auto i = vec.begin(), e = vec.end(); (i + 1) != e; ++i) {
-    for (auto j = i + 1; j != e; ++j) {
-      assert(compareAtoms(*i, *j, customSorter));
-      assert(!compareAtoms(*j, *i, customSorter));
-    }
-  }
-}
-
-// Helper functions to check follow-on graph.
-typedef llvm::DenseMap<const DefinedAtom *, const DefinedAtom *> AtomToAtomT;
-
-static std::string atomToDebugString(const Atom *atom) {
-  const DefinedAtom *definedAtom = dyn_cast<DefinedAtom>(atom);
-  std::string str;
-  llvm::raw_string_ostream s(str);
-  if (definedAtom->name().empty())
-    s << "<anonymous " << definedAtom << ">";
-  else
-    s << definedAtom->name();
-  s << " in ";
-  if (definedAtom->customSectionName().empty())
-    s << "<anonymous>";
-  else
-    s << definedAtom->customSectionName();
-  s.flush();
-  return str;
-}
-
-static void showCycleDetectedError(const Registry &registry,
-                                   AtomToAtomT &followOnNexts,
-                                   const DefinedAtom *atom) {
-  const DefinedAtom *start = atom;
-  llvm::dbgs() << "There's a cycle in a follow-on chain!\n";
-  do {
-    llvm::dbgs() << "  " << atomToDebugString(atom) << "\n";
-    for (const Reference *ref : *atom) {
-      StringRef kindValStr;
-      if (!registry.referenceKindToString(ref->kindNamespace(), ref->kindArch(),
-                                          ref->kindValue(), kindValStr)) {
-        kindValStr = "<unknown>";
-      }
-      llvm::dbgs() << "    " << kindValStr
-                   << ": " << atomToDebugString(ref->target()) << "\n";
-    }
-    atom = followOnNexts[atom];
-  } while (atom != start);
-  llvm::report_fatal_error("Cycle detected");
-}
-
-/// Exit if there's a cycle in a followon chain reachable from the
-/// given root atom. Uses the tortoise and hare algorithm to detect a
-/// cycle.
-static void checkNoCycleInFollowonChain(const Registry &registry,
-                                        AtomToAtomT &followOnNexts,
-                                        const DefinedAtom *root) {
-  const DefinedAtom *tortoise = root;
-  const DefinedAtom *hare = followOnNexts[root];
-  while (true) {
-    if (!tortoise || !hare)
-      return;
-    if (tortoise == hare)
-      showCycleDetectedError(registry, followOnNexts, tortoise);
-    tortoise = followOnNexts[tortoise];
-    hare = followOnNexts[followOnNexts[hare]];
-  }
-}
-
-static void checkReachabilityFromRoot(AtomToAtomT &followOnRoots,
-                                      const DefinedAtom *atom) {
-  if (!atom) return;
-  auto i = followOnRoots.find(atom);
-  if (i == followOnRoots.end()) {
-    llvm_unreachable(((Twine("Atom <") + atomToDebugString(atom) +
-                       "> has no follow-on root!"))
-                         .str()
-                         .c_str());
-  }
-  const DefinedAtom *ap = i->second;
-  while (true) {
-    const DefinedAtom *next = followOnRoots[ap];
-    if (!next) {
-      llvm_unreachable((Twine("Atom <" + atomToDebugString(atom) +
-                              "> is not reachable from its root!"))
-                           .str()
-                           .c_str());
-    }
-    if (next == ap)
-      return;
-    ap = next;
-  }
-}
-
-static void printDefinedAtoms(const File::AtomRange<DefinedAtom> &atomRange) {
-  for (const DefinedAtom *atom : atomRange) {
-    llvm::dbgs() << "  file=" << atom->file().path()
-                 << ", name=" << atom->name()
-                 << ", size=" << atom->size()
-                 << ", type=" << atom->contentType()
-                 << ", ordinal=" << atom->ordinal()
-                 << "\n";
-  }
-}
-
-/// Verify that the followon chain is sane. Should not be called in
-/// release binary.
-void LayoutPass::checkFollowonChain(const File::AtomRange<DefinedAtom> &range) {
-  ScopedTask task(getDefaultDomain(), "LayoutPass::checkFollowonChain");
-
-  // Verify that there's no cycle in follow-on chain.
-  std::set<const DefinedAtom *> roots;
-  for (const auto &ai : _followOnRoots)
-    roots.insert(ai.second);
-  for (const DefinedAtom *root : roots)
-    checkNoCycleInFollowonChain(_registry, _followOnNexts, root);
-
-  // Verify that all the atoms in followOnNexts have references to
-  // their roots.
-  for (const auto &ai : _followOnNexts) {
-    checkReachabilityFromRoot(_followOnRoots, ai.first);
-    checkReachabilityFromRoot(_followOnRoots, ai.second);
-  }
-}
-#endif // #ifndef NDEBUG
-
-/// The function compares atoms by sorting atoms in the following order
-/// a) Sorts atoms by their ordinal overrides (layout-after/ingroup)
-/// b) Sorts atoms by their permissions
-/// c) Sorts atoms by their content
-/// d) Sorts atoms by custom sorter
-/// e) Sorts atoms on how they appear using File Ordinality
-/// f) Sorts atoms on how they appear within the File
-static bool compareAtomsSub(const LayoutPass::SortKey &lc,
-                            const LayoutPass::SortKey &rc,
-                            LayoutPass::SortOverride customSorter,
-                            std::string &reason) {
-  const DefinedAtom *left = lc._atom.get();
-  const DefinedAtom *right = rc._atom.get();
-  if (left == right) {
-    reason = "same";
-    return false;
-  }
-
-  // Find the root of the chain if it is a part of a follow-on chain.
-  const DefinedAtom *leftRoot = lc._root;
-  const DefinedAtom *rightRoot = rc._root;
-
-  // Sort atoms by their ordinal overrides only if they fall in the same
-  // chain.
-  if (leftRoot == rightRoot) {
-    LLVM_DEBUG(reason = formatReason("override", lc._override, rc._override));
-    return lc._override < rc._override;
-  }
-
-  // Sort same permissions together.
-  DefinedAtom::ContentPermissions leftPerms = leftRoot->permissions();
-  DefinedAtom::ContentPermissions rightPerms = rightRoot->permissions();
-
-  if (leftPerms != rightPerms) {
-    LLVM_DEBUG(
-        reason = formatReason("contentPerms", (int)leftPerms, (int)rightPerms));
-    return leftPerms < rightPerms;
-  }
-
-  // Sort same content types together.
-  DefinedAtom::ContentType leftType = leftRoot->contentType();
-  DefinedAtom::ContentType rightType = rightRoot->contentType();
-
-  if (leftType != rightType) {
-    LLVM_DEBUG(reason =
-                   formatReason("contentType", (int)leftType, (int)rightType));
-    return leftType < rightType;
-  }
-
-  // Use custom sorter if supplied.
-  if (customSorter) {
-    bool leftBeforeRight;
-    if (customSorter(leftRoot, rightRoot, leftBeforeRight))
-      return leftBeforeRight;
-  }
-
-  // Sort by .o order.
-  const File *leftFile = &leftRoot->file();
-  const File *rightFile = &rightRoot->file();
-
-  if (leftFile != rightFile) {
-    LLVM_DEBUG(reason = formatReason(".o order", (int)leftFile->ordinal(),
-                                     (int)rightFile->ordinal()));
-    return leftFile->ordinal() < rightFile->ordinal();
-  }
-
-  // Sort by atom order with .o file.
-  uint64_t leftOrdinal = leftRoot->ordinal();
-  uint64_t rightOrdinal = rightRoot->ordinal();
-
-  if (leftOrdinal != rightOrdinal) {
-    LLVM_DEBUG(reason = formatReason("ordinal", (int)leftRoot->ordinal(),
-                                     (int)rightRoot->ordinal()));
-    return leftOrdinal < rightOrdinal;
-  }
-
-  llvm::errs() << "Unordered: <" << left->name() << "> <" << right->name()
-               << ">\n";
-  llvm_unreachable("Atoms with Same Ordinal!");
-}
-
-static bool compareAtoms(const LayoutPass::SortKey &lc,
-                         const LayoutPass::SortKey &rc,
-                         LayoutPass::SortOverride customSorter) {
-  std::string reason;
-  bool result = compareAtomsSub(lc, rc, customSorter, reason);
-  LLVM_DEBUG({
-    StringRef comp = result ? "<" : ">=";
-    llvm::dbgs() << "Layout: '" << lc._atom.get()->name()
-                 << "' " << comp << " '"
-                 << rc._atom.get()->name() << "' (" << reason << ")\n";
-  });
-  return result;
-}
-
-LayoutPass::LayoutPass(const Registry &registry, SortOverride sorter)
-    : _registry(registry), _customSorter(std::move(sorter)) {}
-
-// Returns the atom immediately followed by the given atom in the followon
-// chain.
-const DefinedAtom *LayoutPass::findAtomFollowedBy(
-    const DefinedAtom *targetAtom) {
-  // Start from the beginning of the chain and follow the chain until
-  // we find the targetChain.
-  const DefinedAtom *atom = _followOnRoots[targetAtom];
-  while (true) {
-    const DefinedAtom *prevAtom = atom;
-    AtomToAtomT::iterator targetFollowOnAtomsIter = _followOnNexts.find(atom);
-    // The target atom must be in the chain of its root.
-    assert(targetFollowOnAtomsIter != _followOnNexts.end());
-    atom = targetFollowOnAtomsIter->second;
-    if (atom == targetAtom)
-      return prevAtom;
-  }
-}
-
-// Check if all the atoms followed by the given target atom are of size zero.
-// When this method is called, an atom being added is not of size zero and
-// will be added to the head of the followon chain. All the atoms between the
-// atom and the targetAtom (specified by layout-after) need to be of size zero
-// in this case. Otherwise the desired layout is impossible.
-bool LayoutPass::checkAllPrevAtomsZeroSize(const DefinedAtom *targetAtom) {
-  const DefinedAtom *atom = _followOnRoots[targetAtom];
-  while (true) {
-    if (atom == targetAtom)
-      return true;
-    if (atom->size() != 0)
-      // TODO: print warning that an impossible layout is being desired by the
-      // user.
-      return false;
-    AtomToAtomT::iterator targetFollowOnAtomsIter = _followOnNexts.find(atom);
-    // The target atom must be in the chain of its root.
-    assert(targetFollowOnAtomsIter != _followOnNexts.end());
-    atom = targetFollowOnAtomsIter->second;
-  }
-}
-
-// Set the root of all atoms in targetAtom's chain to the given root.
-void LayoutPass::setChainRoot(const DefinedAtom *targetAtom,
-                              const DefinedAtom *root) {
-  // Walk through the followon chain and override each node's root.
-  while (true) {
-    _followOnRoots[targetAtom] = root;
-    AtomToAtomT::iterator targetFollowOnAtomsIter =
-        _followOnNexts.find(targetAtom);
-    if (targetFollowOnAtomsIter == _followOnNexts.end())
-      return;
-    targetAtom = targetFollowOnAtomsIter->second;
-  }
-}
-
-/// This pass builds the followon tables described by two DenseMaps
-/// followOnRoots and followonNexts.
-/// The followOnRoots map contains a mapping of a DefinedAtom to its root
-/// The followOnNexts map contains a mapping of what DefinedAtom follows the
-/// current Atom
-/// The algorithm follows a very simple approach
-/// a) If the atom is first seen, then make that as the root atom
-/// b) The targetAtom which this Atom contains, has the root thats set to the
-///    root of the current atom
-/// c) If the targetAtom is part of a different tree and the root of the
-///    targetAtom is itself, Chain all the atoms that are contained in the tree
-///    to the current Tree
-/// d) If the targetAtom is part of a different chain and the root of the
-///    targetAtom until the targetAtom has all atoms of size 0, then chain the
-///    targetAtoms and its tree to the current chain
-void LayoutPass::buildFollowOnTable(const File::AtomRange<DefinedAtom> &range) {
-  ScopedTask task(getDefaultDomain(), "LayoutPass::buildFollowOnTable");
-  // Set the initial size of the followon and the followonNext hash to the
-  // number of atoms that we have.
-  _followOnRoots.reserve(range.size());
-  _followOnNexts.reserve(range.size());
-  for (const DefinedAtom *ai : range) {
-    for (const Reference *r : *ai) {
-      if (r->kindNamespace() != lld::Reference::KindNamespace::all ||
-          r->kindValue() != lld::Reference::kindLayoutAfter)
-        continue;
-      const DefinedAtom *targetAtom = dyn_cast<DefinedAtom>(r->target());
-      _followOnNexts[ai] = targetAtom;
-
-      // If we find a followon for the first time, let's make that atom as the
-      // root atom.
-      if (_followOnRoots.count(ai) == 0)
-        _followOnRoots[ai] = ai;
-
-      auto iter = _followOnRoots.find(targetAtom);
-      if (iter == _followOnRoots.end()) {
-        // If the targetAtom is not a root of any chain, let's make the root of
-        // the targetAtom to the root of the current chain.
-
-        // The expression m[i] = m[j] where m is a DenseMap and i != j is not
-        // safe. m[j] returns a reference, which would be invalidated when a
-        // rehashing occurs. If rehashing occurs to make room for m[i], m[j]
-        // becomes invalid, and that invalid reference would be used as the RHS
-        // value of the expression.
-        // Copy the value to workaround.
-        const DefinedAtom *tmp = _followOnRoots[ai];
-        _followOnRoots[targetAtom] = tmp;
-        continue;
-      }
-      if (iter->second == targetAtom) {
-        // If the targetAtom is the root of a chain, the chain becomes part of
-        // the current chain. Rewrite the subchain's root to the current
-        // chain's root.
-        setChainRoot(targetAtom, _followOnRoots[ai]);
-        continue;
-      }
-      // The targetAtom is already a part of a chain. If the current atom is
-      // of size zero, we can insert it in the middle of the chain just
-      // before the target atom, while not breaking other atom's followon
-      // relationships. If it's not, we can only insert the current atom at
-      // the beginning of the chain. All the atoms followed by the target
-      // atom must be of size zero in that case to satisfy the followon
-      // relationships.
-      size_t currentAtomSize = ai->size();
-      if (currentAtomSize == 0) {
-        const DefinedAtom *targetPrevAtom = findAtomFollowedBy(targetAtom);
-        _followOnNexts[targetPrevAtom] = ai;
-        const DefinedAtom *tmp = _followOnRoots[targetPrevAtom];
-        _followOnRoots[ai] = tmp;
-        continue;
-      }
-      if (!checkAllPrevAtomsZeroSize(targetAtom))
-        break;
-      _followOnNexts[ai] = _followOnRoots[targetAtom];
-      setChainRoot(_followOnRoots[targetAtom], _followOnRoots[ai]);
-    }
-  }
-}
-
-/// Build an ordinal override map by traversing the followon chain, and
-/// assigning ordinals to each atom, if the atoms have their ordinals
-/// already assigned skip the atom and move to the next. This is the
-/// main map thats used to sort the atoms while comparing two atoms together
-void
-LayoutPass::buildOrdinalOverrideMap(const File::AtomRange<DefinedAtom> &range) {
-  ScopedTask task(getDefaultDomain(), "LayoutPass::buildOrdinalOverrideMap");
-  uint64_t index = 0;
-  for (const DefinedAtom *ai : range) {
-    const DefinedAtom *atom = ai;
-    if (_ordinalOverrideMap.find(atom) != _ordinalOverrideMap.end())
-      continue;
-    AtomToAtomT::iterator start = _followOnRoots.find(atom);
-    if (start == _followOnRoots.end())
-      continue;
-    for (const DefinedAtom *nextAtom = start->second; nextAtom;
-         nextAtom = _followOnNexts[nextAtom]) {
-      AtomToOrdinalT::iterator pos = _ordinalOverrideMap.find(nextAtom);
-      if (pos == _ordinalOverrideMap.end())
-        _ordinalOverrideMap[nextAtom] = index++;
-    }
-  }
-}
-
-std::vector<LayoutPass::SortKey>
-LayoutPass::decorate(File::AtomRange<DefinedAtom> &atomRange) const {
-  std::vector<SortKey> ret;
-  for (OwningAtomPtr<DefinedAtom> &atom : atomRange.owning_ptrs()) {
-    auto ri = _followOnRoots.find(atom.get());
-    auto oi = _ordinalOverrideMap.find(atom.get());
-    const auto *root = (ri == _followOnRoots.end()) ? atom.get() : ri->second;
-    uint64_t override = (oi == _ordinalOverrideMap.end()) ? 0 : oi->second;
-    ret.push_back(SortKey(std::move(atom), root, override));
-  }
-  return ret;
-}
-
-void LayoutPass::undecorate(File::AtomRange<DefinedAtom> &atomRange,
-                            std::vector<SortKey> &keys) const {
-  size_t i = 0;
-  for (SortKey &k : keys)
-    atomRange[i++] = std::move(k._atom);
-}
-
-/// Perform the actual pass
-llvm::Error LayoutPass::perform(SimpleFile &mergedFile) {
-  LLVM_DEBUG(llvm::dbgs() << "******** Laying out atoms:\n");
-  // sort the atoms
-  ScopedTask task(getDefaultDomain(), "LayoutPass");
-  File::AtomRange<DefinedAtom> atomRange = mergedFile.defined();
-
-  // Build follow on tables
-  buildFollowOnTable(atomRange);
-
-  // Check the structure of followon graph if running in debug mode.
-  LLVM_DEBUG(checkFollowonChain(atomRange));
-
-  // Build override maps
-  buildOrdinalOverrideMap(atomRange);
-
-  LLVM_DEBUG({
-    llvm::dbgs() << "unsorted atoms:\n";
-    printDefinedAtoms(atomRange);
-  });
-
-  std::vector<LayoutPass::SortKey> vec = decorate(atomRange);
-  llvm::parallelSort(
-      vec,
-      [&](const LayoutPass::SortKey &l, const LayoutPass::SortKey &r) -> bool {
-        return compareAtoms(l, r, _customSorter);
-      });
-  LLVM_DEBUG(checkTransitivity(vec, _customSorter));
-  undecorate(atomRange, vec);
-
-  LLVM_DEBUG({
-    llvm::dbgs() << "sorted atoms:\n";
-    printDefinedAtoms(atomRange);
-  });
-
-  LLVM_DEBUG(llvm::dbgs() << "******** Finished laying out atoms\n");
-  return llvm::Error::success();
-}
-
-void addLayoutPass(PassManager &pm, const MachOLinkingContext &ctx) {
-  pm.add(std::make_unique<LayoutPass>(
-      ctx.registry(), [&](const DefinedAtom * left, const DefinedAtom * right,
-                          bool & leftBeforeRight) ->bool {
-    return ctx.customAtomOrderer(left, right, leftBeforeRight);
-  }));
-}
-
-} // namespace mach_o
-} // namespace lld
diff --git a/lld/lib/ReaderWriter/MachO/LayoutPass.h b/lld/lib/ReaderWriter/MachO/LayoutPass.h
deleted file mode 100644
index 904e16b7fb0e..000000000000
--- a/lld/lib/ReaderWriter/MachO/LayoutPass.h
+++ /dev/null
@@ -1,118 +0,0 @@
-//===------ lib/ReaderWriter/MachO/LayoutPass.h - Handles Layout of atoms -===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLD_READER_WRITER_MACHO_LAYOUT_PASS_H
-#define LLD_READER_WRITER_MACHO_LAYOUT_PASS_H
-
-#include "lld/Core/File.h"
-#include "lld/Core/Pass.h"
-#include "lld/Core/Reader.h"
-#include "lld/Core/Simple.h"
-#include "llvm/ADT/DenseMap.h"
-#include <map>
-#include <string>
-#include <vector>
-
-namespace lld {
-class DefinedAtom;
-class SimpleFile;
-
-namespace mach_o {
-
-/// This linker pass does the layout of the atoms. The pass is done after the
-/// order their .o files were found on the command line, then by order of the
-/// atoms (address) in the .o file.  But some atoms have a preferred location
-/// in their section (such as pinned to the start or end of the section), so
-/// the sort must take that into account too.
-class LayoutPass : public Pass {
-public:
-  struct SortKey {
-    SortKey(OwningAtomPtr<DefinedAtom> &&atom,
-            const DefinedAtom *root, uint64_t override)
-    : _atom(std::move(atom)), _root(root), _override(override) {}
-    OwningAtomPtr<DefinedAtom> _atom;
-    const DefinedAtom *_root;
-    uint64_t _override;
-
-    // Note, these are only here to appease MSVC bots which didn't like
-    // the same methods being implemented/deleted in OwningAtomPtr.
-    SortKey(SortKey &&key) : _atom(std::move(key._atom)), _root(key._root),
-                             _override(key._override) {
-      key._root = nullptr;
-    }
-
-    SortKey &operator=(SortKey &&key) {
-      _atom = std::move(key._atom);
-      _root = key._root;
-      key._root = nullptr;
-      _override = key._override;
-      return *this;
-    }
-
-  private:
-    SortKey(const SortKey &) = delete;
-    void operator=(const SortKey&) = delete;
-  };
-
-  typedef std::function<bool (const DefinedAtom *left, const DefinedAtom *right,
-                              bool &leftBeforeRight)> SortOverride;
-
-  LayoutPass(const Registry &registry, SortOverride sorter);
-
-  /// Sorts atoms in mergedFile by content type then by command line order.
-  llvm::Error perform(SimpleFile &mergedFile) override;
-
-  ~LayoutPass() override = default;
-
-private:
-  // Build the followOn atoms chain as specified by the kindLayoutAfter
-  // reference type
-  void buildFollowOnTable(const File::AtomRange<DefinedAtom> &range);
-
-  // Build a map of Atoms to ordinals for sorting the atoms
-  void buildOrdinalOverrideMap(const File::AtomRange<DefinedAtom> &range);
-
-  const Registry &_registry;
-  SortOverride _customSorter;
-
-  typedef llvm::DenseMap<const DefinedAtom *, const DefinedAtom *> AtomToAtomT;
-  typedef llvm::DenseMap<const DefinedAtom *, uint64_t> AtomToOrdinalT;
-
-  // A map to be used to sort atoms. It represents the order of atoms in the
-  // result; if Atom X is mapped to atom Y in this map, X will be located
-  // immediately before Y in the output file. Y might be mapped to another
-  // atom, constructing a follow-on chain. An atom cannot be mapped to more
-  // than one atom unless all but one atom are of size zero.
-  AtomToAtomT _followOnNexts;
-
-  // A map to be used to sort atoms. It's a map from an atom to its root of
-  // follow-on chain. A root atom is mapped to itself. If an atom is not in
-  // _followOnNexts, the atom is not in this map, and vice versa.
-  AtomToAtomT _followOnRoots;
-
-  AtomToOrdinalT _ordinalOverrideMap;
-
-  // Helper methods for buildFollowOnTable().
-  const DefinedAtom *findAtomFollowedBy(const DefinedAtom *targetAtom);
-  bool checkAllPrevAtomsZeroSize(const DefinedAtom *targetAtom);
-
-  void setChainRoot(const DefinedAtom *targetAtom, const DefinedAtom *root);
-
-  std::vector<SortKey> decorate(File::AtomRange<DefinedAtom> &atomRange) const;
-
-  void undecorate(File::AtomRange<DefinedAtom> &atomRange,
-                  std::vector<SortKey> &keys) const;
-
-  // Check if the follow-on graph is a correct structure. For debugging only.
-  void checkFollowonChain(const File::AtomRange<DefinedAtom> &range);
-};
-
-} // namespace mach_o
-} // namespace lld
-
-#endif // LLD_READER_WRITER_MACHO_LAYOUT_PASS_H
diff --git a/lld/lib/ReaderWriter/MachO/MachOLinkingContext.cpp b/lld/lib/ReaderWriter/MachO/MachOLinkingContext.cpp
deleted file mode 100644
index acd919e4d411..000000000000
--- a/lld/lib/ReaderWriter/MachO/MachOLinkingContext.cpp
+++ /dev/null
@@ -1,1104 +0,0 @@
-//===- lib/ReaderWriter/MachO/MachOLinkingContext.cpp ---------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "lld/Common/ErrorHandler.h"
-#include "lld/ReaderWriter/MachOLinkingContext.h"
-#include "ArchHandler.h"
-#include "File.h"
-#include "FlatNamespaceFile.h"
-#include "MachONormalizedFile.h"
-#include "MachOPasses.h"
-#include "SectCreateFile.h"
-#include "lld/Common/Driver.h"
-#include "lld/Core/ArchiveLibraryFile.h"
-#include "lld/Core/PassManager.h"
-#include "lld/Core/Reader.h"
-#include "lld/Core/Writer.h"
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/StringExtras.h"
-#include "llvm/ADT/Triple.h"
-#include "llvm/BinaryFormat/MachO.h"
-#include "llvm/Demangle/Demangle.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/Errc.h"
-#include "llvm/Support/Host.h"
-#include "llvm/Support/Path.h"
-#include <algorithm>
-
-using lld::mach_o::ArchHandler;
-using lld::mach_o::MachOFile;
-using lld::mach_o::MachODylibFile;
-using namespace llvm::MachO;
-
-namespace lld {
-
-bool MachOLinkingContext::parsePackedVersion(StringRef str, uint32_t &result) {
-  result = 0;
-
-  if (str.empty())
-    return false;
-
-  SmallVector<StringRef, 3> parts;
-  llvm::SplitString(str, parts, ".");
-
-  unsigned long long num;
-  if (llvm::getAsUnsignedInteger(parts[0], 10, num))
-    return true;
-  if (num > 65535)
-    return true;
-  result = num << 16;
-
-  if (parts.size() > 1) {
-    if (llvm::getAsUnsignedInteger(parts[1], 10, num))
-      return true;
-    if (num > 255)
-      return true;
-    result |= (num << 8);
-  }
-
-  if (parts.size() > 2) {
-    if (llvm::getAsUnsignedInteger(parts[2], 10, num))
-      return true;
-    if (num > 255)
-      return true;
-    result |= num;
-  }
-
-  return false;
-}
-
-bool MachOLinkingContext::parsePackedVersion(StringRef str, uint64_t &result) {
-  result = 0;
-
-  if (str.empty())
-    return false;
-
-  SmallVector<StringRef, 5> parts;
-  llvm::SplitString(str, parts, ".");
-
-  unsigned long long num;
-  if (llvm::getAsUnsignedInteger(parts[0], 10, num))
-    return true;
-  if (num > 0xFFFFFF)
-    return true;
-  result = num << 40;
-
-  unsigned Shift = 30;
-  for (StringRef str : llvm::makeArrayRef(parts).slice(1)) {
-    if (llvm::getAsUnsignedInteger(str, 10, num))
-      return true;
-    if (num > 0x3FF)
-      return true;
-    result |= (num << Shift);
-    Shift -= 10;
-  }
-
-  return false;
-}
-
-MachOLinkingContext::ArchInfo MachOLinkingContext::_s_archInfos[] = {
-  { "x86_64", arch_x86_64, true,  CPU_TYPE_X86_64,  CPU_SUBTYPE_X86_64_ALL },
-  { "i386",   arch_x86,    true,  CPU_TYPE_I386,    CPU_SUBTYPE_X86_ALL },
-  { "ppc",    arch_ppc,    false, CPU_TYPE_POWERPC, CPU_SUBTYPE_POWERPC_ALL },
-  { "armv6",  arch_armv6,  true,  CPU_TYPE_ARM,     CPU_SUBTYPE_ARM_V6 },
-  { "armv7",  arch_armv7,  true,  CPU_TYPE_ARM,     CPU_SUBTYPE_ARM_V7 },
-  { "armv7s", arch_armv7s, true,  CPU_TYPE_ARM,     CPU_SUBTYPE_ARM_V7S },
-  { "arm64",  arch_arm64,  true,  CPU_TYPE_ARM64,   CPU_SUBTYPE_ARM64_ALL },
-  { "",       arch_unknown,false, 0,                0 }
-};
-
-MachOLinkingContext::Arch
-MachOLinkingContext::archFromCpuType(uint32_t cputype, uint32_t cpusubtype) {
-  for (ArchInfo *info = _s_archInfos; !info->archName.empty(); ++info) {
-    if ((info->cputype == cputype) && (info->cpusubtype == cpusubtype))
-      return info->arch;
-  }
-  return arch_unknown;
-}
-
-MachOLinkingContext::Arch
-MachOLinkingContext::archFromName(StringRef archName) {
-  for (ArchInfo *info = _s_archInfos; !info->archName.empty(); ++info) {
-    if (info->archName.equals(archName))
-      return info->arch;
-  }
-  return arch_unknown;
-}
-
-StringRef MachOLinkingContext::nameFromArch(Arch arch) {
-  for (ArchInfo *info = _s_archInfos; !info->archName.empty(); ++info) {
-    if (info->arch == arch)
-      return info->archName;
-  }
-  return "<unknown>";
-}
-
-uint32_t MachOLinkingContext::cpuTypeFromArch(Arch arch) {
-  assert(arch != arch_unknown);
-  for (ArchInfo *info = _s_archInfos; !info->archName.empty(); ++info) {
-    if (info->arch == arch)
-      return info->cputype;
-  }
-  llvm_unreachable("Unknown arch type");
-}
-
-uint32_t MachOLinkingContext::cpuSubtypeFromArch(Arch arch) {
-  assert(arch != arch_unknown);
-  for (ArchInfo *info = _s_archInfos; !info->archName.empty(); ++info) {
-    if (info->arch == arch)
-      return info->cpusubtype;
-  }
-  llvm_unreachable("Unknown arch type");
-}
-
-bool MachOLinkingContext::isThinObjectFile(StringRef path, Arch &arch) {
-  return mach_o::normalized::isThinObjectFile(path, arch);
-}
-
-bool MachOLinkingContext::sliceFromFatFile(MemoryBufferRef mb, uint32_t &offset,
-                                           uint32_t &size) {
-  return mach_o::normalized::sliceFromFatFile(mb, _arch, offset, size);
-}
-
-MachOLinkingContext::MachOLinkingContext() {}
-
-MachOLinkingContext::~MachOLinkingContext() {
-  // Atoms are allocated on BumpPtrAllocator's on File's.
-  // As we transfer atoms from one file to another, we need to clear all of the
-  // atoms before we remove any of the BumpPtrAllocator's.
-  auto &nodes = getNodes();
-  for (unsigned i = 0, e = nodes.size(); i != e; ++i) {
-    FileNode *node = dyn_cast<FileNode>(nodes[i].get());
-    if (!node)
-      continue;
-    File *file = node->getFile();
-    file->clearAtoms();
-  }
-}
-
-void MachOLinkingContext::configure(HeaderFileType type, Arch arch, OS os,
-                                    uint32_t minOSVersion,
-                                    bool exportDynamicSymbols) {
-  _outputMachOType = type;
-  _arch = arch;
-  _os = os;
-  _osMinVersion = minOSVersion;
-
-  // If min OS not specified on command line, use reasonable defaults.
-  // Note that we only do sensible defaults when emitting something other than
-  // object and preload.
-  if (_outputMachOType != llvm::MachO::MH_OBJECT &&
-      _outputMachOType != llvm::MachO::MH_PRELOAD) {
-    if (minOSVersion == 0) {
-      switch (_arch) {
-      case arch_x86_64:
-      case arch_x86:
-        parsePackedVersion("10.8", _osMinVersion);
-        _os = MachOLinkingContext::OS::macOSX;
-        break;
-      case arch_armv6:
-      case arch_armv7:
-      case arch_armv7s:
-      case arch_arm64:
-        parsePackedVersion("7.0", _osMinVersion);
-        _os = MachOLinkingContext::OS::iOS;
-        break;
-      default:
-        break;
-      }
-    }
-  }
-
-  switch (_outputMachOType) {
-  case llvm::MachO::MH_EXECUTE:
-    // If targeting newer OS, use _main
-    if (minOS("10.8", "6.0")) {
-      _entrySymbolName = "_main";
-    } else {
-      // If targeting older OS, use start (in crt1.o)
-      _entrySymbolName = "start";
-    }
-
-    // __PAGEZERO defaults to 4GB on 64-bit (except for PP64 which lld does not
-    // support) and 4KB on 32-bit.
-    if (is64Bit(_arch)) {
-      _pageZeroSize = 0x100000000;
-    } else {
-      _pageZeroSize = 0x1000;
-    }
-
-    // Initial base address is __PAGEZERO size.
-    _baseAddress = _pageZeroSize;
-
-    // Make PIE by default when targetting newer OSs.
-    switch (os) {
-      case OS::macOSX:
-        if (minOSVersion >= 0x000A0700) // MacOSX 10.7
-          _pie = true;
-        break;
-      case OS::iOS:
-        if (minOSVersion >= 0x00040300) // iOS 4.3
-          _pie = true;
-       break;
-       case OS::iOS_simulator:
-        _pie = true;
-       break;
-       case OS::unknown:
-       break;
-    }
-    setGlobalsAreDeadStripRoots(exportDynamicSymbols);
-    break;
-  case llvm::MachO::MH_DYLIB:
-    setGlobalsAreDeadStripRoots(exportDynamicSymbols);
-    break;
-  case llvm::MachO::MH_BUNDLE:
-    break;
-  case llvm::MachO::MH_OBJECT:
-    _printRemainingUndefines = false;
-    _allowRemainingUndefines = true;
-    break;
-  default:
-    break;
-  }
-
-  // Set default segment page sizes based on arch.
-  if (arch == arch_arm64)
-    _pageSize = 4*4096;
-}
-
-uint32_t MachOLinkingContext::getCPUType() const {
-  return cpuTypeFromArch(_arch);
-}
-
-uint32_t MachOLinkingContext::getCPUSubType() const {
-  return cpuSubtypeFromArch(_arch);
-}
-
-bool MachOLinkingContext::is64Bit(Arch arch) {
-  for (ArchInfo *info = _s_archInfos; !info->archName.empty(); ++info) {
-    if (info->arch == arch) {
-      return (info->cputype & CPU_ARCH_ABI64);
-    }
-  }
-  // unknown archs are not 64-bit.
-  return false;
-}
-
-bool MachOLinkingContext::isHostEndian(Arch arch) {
-  assert(arch != arch_unknown);
-  for (ArchInfo *info = _s_archInfos; !info->archName.empty(); ++info) {
-    if (info->arch == arch) {
-      return (info->littleEndian == llvm::sys::IsLittleEndianHost);
-    }
-  }
-  llvm_unreachable("Unknown arch type");
-}
-
-bool MachOLinkingContext::isBigEndian(Arch arch) {
-  assert(arch != arch_unknown);
-  for (ArchInfo *info = _s_archInfos; !info->archName.empty(); ++info) {
-    if (info->arch == arch) {
-      return ! info->littleEndian;
-    }
-  }
-  llvm_unreachable("Unknown arch type");
-}
-
-bool MachOLinkingContext::is64Bit() const {
-  return is64Bit(_arch);
-}
-
-bool MachOLinkingContext::outputTypeHasEntry() const {
-  switch (_outputMachOType) {
-  case MH_EXECUTE:
-  case MH_DYLINKER:
-  case MH_PRELOAD:
-    return true;
-  default:
-    return false;
-  }
-}
-
-bool MachOLinkingContext::needsStubsPass() const {
-  switch (_outputMachOType) {
-  case MH_EXECUTE:
-    return !_outputMachOTypeStatic;
-  case MH_DYLIB:
-  case MH_BUNDLE:
-    return true;
-  default:
-    return false;
-  }
-}
-
-bool MachOLinkingContext::needsGOTPass() const {
-  // GOT pass not used in -r mode.
-  if (_outputMachOType == MH_OBJECT)
-    return false;
-  // Only some arches use GOT pass.
-  switch (_arch) {
-    case arch_x86_64:
-    case arch_arm64:
-      return true;
-    default:
-      return false;
-  }
-}
-
-bool MachOLinkingContext::needsCompactUnwindPass() const {
-  switch (_outputMachOType) {
-  case MH_EXECUTE:
-  case MH_DYLIB:
-  case MH_BUNDLE:
-    return archHandler().needsCompactUnwind();
-  default:
-    return false;
-  }
-}
-
-bool MachOLinkingContext::needsObjCPass() const {
-  // ObjC pass is only needed if any of the inputs were ObjC.
-  return _objcConstraint != objc_unknown;
-}
-
-bool MachOLinkingContext::needsShimPass() const {
-  // Shim pass only used in final executables.
-  if (_outputMachOType == MH_OBJECT)
-    return false;
-  // Only 32-bit arm arches use Shim pass.
-  switch (_arch) {
-  case arch_armv6:
-  case arch_armv7:
-  case arch_armv7s:
-    return true;
-  default:
-    return false;
-  }
-}
-
-bool MachOLinkingContext::needsTLVPass() const {
-  switch (_outputMachOType) {
-  case MH_BUNDLE:
-  case MH_EXECUTE:
-  case MH_DYLIB:
-    return true;
-  default:
-    return false;
-  }
-}
-
-StringRef MachOLinkingContext::binderSymbolName() const {
-  return archHandler().stubInfo().binderSymbolName;
-}
-
-bool MachOLinkingContext::minOS(StringRef mac, StringRef iOS) const {
-  uint32_t parsedVersion;
-  switch (_os) {
-  case OS::macOSX:
-    if (parsePackedVersion(mac, parsedVersion))
-      return false;
-    return _osMinVersion >= parsedVersion;
-  case OS::iOS:
-  case OS::iOS_simulator:
-    if (parsePackedVersion(iOS, parsedVersion))
-      return false;
-    return _osMinVersion >= parsedVersion;
-  case OS::unknown:
-    // If we don't know the target, then assume that we don't meet the min OS.
-    // This matches the ld64 behaviour
-    return false;
-  }
-  llvm_unreachable("invalid OS enum");
-}
-
-bool MachOLinkingContext::addEntryPointLoadCommand() const {
-  if ((_outputMachOType == MH_EXECUTE) && !_outputMachOTypeStatic) {
-    return minOS("10.8", "6.0");
-  }
-  return false;
-}
-
-bool MachOLinkingContext::addUnixThreadLoadCommand() const {
-  switch (_outputMachOType) {
-  case MH_EXECUTE:
-    if (_outputMachOTypeStatic)
-      return true;
-    else
-      return !minOS("10.8", "6.0");
-    break;
-  case MH_DYLINKER:
-  case MH_PRELOAD:
-    return true;
-  default:
-    return false;
-  }
-}
-
-bool MachOLinkingContext::pathExists(StringRef path) const {
-  if (!_testingFileUsage)
-    return llvm::sys::fs::exists(path.str());
-
-  // Otherwise, we're in test mode: only files explicitly provided on the
-  // command-line exist.
-  std::string key = path.str();
-  std::replace(key.begin(), key.end(), '\\', '/');
-  return _existingPaths.find(key) != _existingPaths.end();
-}
-
-bool MachOLinkingContext::fileExists(StringRef path) const {
-  bool found = pathExists(path);
-  // Log search misses.
-  if (!found)
-    addInputFileNotFound(path);
-
-  // When testing, file is never opened, so logging is done here.
-  if (_testingFileUsage && found)
-    addInputFileDependency(path);
-
-  return found;
-}
-
-void MachOLinkingContext::setSysLibRoots(const StringRefVector &paths) {
-  _syslibRoots = paths;
-}
-
-void MachOLinkingContext::addRpath(StringRef rpath) {
-  _rpaths.push_back(rpath);
-}
-
-void MachOLinkingContext::addModifiedSearchDir(StringRef libPath,
-                                               bool isSystemPath) {
-  bool addedModifiedPath = false;
-
-  // -syslibroot only applies to absolute paths.
-  if (libPath.startswith("/")) {
-    for (auto syslibRoot : _syslibRoots) {
-      SmallString<256> path(syslibRoot);
-      llvm::sys::path::append(path, libPath);
-      if (pathExists(path)) {
-        _searchDirs.push_back(path.str().copy(_allocator));
-        addedModifiedPath = true;
-      }
-    }
-  }
-
-  if (addedModifiedPath)
-    return;
-
-  // Finally, if only one -syslibroot is given, system paths which aren't in it
-  // get suppressed.
-  if (_syslibRoots.size() != 1 || !isSystemPath) {
-    if (pathExists(libPath)) {
-      _searchDirs.push_back(libPath);
-    }
-  }
-}
-
-void MachOLinkingContext::addFrameworkSearchDir(StringRef fwPath,
-                                                bool isSystemPath) {
-  bool pathAdded = false;
-
-  // -syslibroot only used with to absolute framework search paths.
-  if (fwPath.startswith("/")) {
-    for (auto syslibRoot : _syslibRoots) {
-      SmallString<256> path(syslibRoot);
-      llvm::sys::path::append(path, fwPath);
-      if (pathExists(path)) {
-        _frameworkDirs.push_back(path.str().copy(_allocator));
-        pathAdded = true;
-      }
-    }
-  }
-  // If fwPath found in any -syslibroot, then done.
-  if (pathAdded)
-    return;
-
-  // If only one -syslibroot, system paths not in that SDK are suppressed.
-  if (isSystemPath && (_syslibRoots.size() == 1))
-    return;
-
-  // Only use raw fwPath if that directory exists.
-  if (pathExists(fwPath))
-    _frameworkDirs.push_back(fwPath);
-}
-
-llvm::Optional<StringRef>
-MachOLinkingContext::searchDirForLibrary(StringRef path,
-                                         StringRef libName) const {
-  SmallString<256> fullPath;
-  if (libName.endswith(".o")) {
-    // A request ending in .o is special: just search for the file directly.
-    fullPath.assign(path);
-    llvm::sys::path::append(fullPath, libName);
-    if (fileExists(fullPath))
-      return fullPath.str().copy(_allocator);
-    return llvm::None;
-  }
-
-  // Search for stub library
-  fullPath.assign(path);
-  llvm::sys::path::append(fullPath, Twine("lib") + libName + ".tbd");
-  if (fileExists(fullPath))
-    return fullPath.str().copy(_allocator);
-
-  // Search for dynamic library
-  fullPath.assign(path);
-  llvm::sys::path::append(fullPath, Twine("lib") + libName + ".dylib");
-  if (fileExists(fullPath))
-    return fullPath.str().copy(_allocator);
-
-  // If not, try for a static library
-  fullPath.assign(path);
-  llvm::sys::path::append(fullPath, Twine("lib") + libName + ".a");
-  if (fileExists(fullPath))
-    return fullPath.str().copy(_allocator);
-
-  return llvm::None;
-}
-
-llvm::Optional<StringRef>
-MachOLinkingContext::searchLibrary(StringRef libName) const {
-  SmallString<256> path;
-  for (StringRef dir : searchDirs()) {
-    llvm::Optional<StringRef> searchDir = searchDirForLibrary(dir, libName);
-    if (searchDir)
-      return searchDir;
-  }
-
-  return llvm::None;
-}
-
-llvm::Optional<StringRef>
-MachOLinkingContext::findPathForFramework(StringRef fwName) const{
-  SmallString<256> fullPath;
-  for (StringRef dir : frameworkDirs()) {
-    fullPath.assign(dir);
-    llvm::sys::path::append(fullPath, Twine(fwName) + ".framework", fwName);
-    if (fileExists(fullPath))
-      return fullPath.str().copy(_allocator);
-  }
-
-  return llvm::None;
-}
-
-bool MachOLinkingContext::validateImpl() {
-  // TODO: if -arch not specified, look at arch of first .o file.
-
-  if (_currentVersion && _outputMachOType != MH_DYLIB) {
-    error("-current_version can only be used with dylibs");
-    return false;
-  }
-
-  if (_compatibilityVersion && _outputMachOType != MH_DYLIB) {
-    error("-compatibility_version can only be used with dylibs");
-    return false;
-  }
-
-  if (_deadStrippableDylib && _outputMachOType != MH_DYLIB) {
-    error("-mark_dead_strippable_dylib can only be used with dylibs");
-    return false;
-  }
-
-  if (!_bundleLoader.empty() && outputMachOType() != MH_BUNDLE) {
-    error("-bundle_loader can only be used with Mach-O bundles");
-    return false;
-  }
-
-  // If -exported_symbols_list used, all exported symbols must be defined.
-  if (_exportMode == ExportMode::exported) {
-    for (const auto &symbol : _exportedSymbols)
-      addInitialUndefinedSymbol(symbol.getKey());
-  }
-
-  // If -dead_strip, set up initial live symbols.
-  if (deadStrip()) {
-    // Entry point is live.
-    if (outputTypeHasEntry())
-      addDeadStripRoot(entrySymbolName());
-    // Lazy binding helper is live.
-    if (needsStubsPass())
-      addDeadStripRoot(binderSymbolName());
-    // If using -exported_symbols_list, make all exported symbols live.
-    if (_exportMode == ExportMode::exported) {
-      setGlobalsAreDeadStripRoots(false);
-      for (const auto &symbol : _exportedSymbols)
-        addDeadStripRoot(symbol.getKey());
-    }
-  }
-
-  addOutputFileDependency(outputPath());
-
-  return true;
-}
-
-void MachOLinkingContext::addPasses(PassManager &pm) {
-  // objc pass should be before layout pass.  Otherwise test cases may contain
-  // no atoms which confuses the layout pass.
-  if (needsObjCPass())
-    mach_o::addObjCPass(pm, *this);
-  mach_o::addLayoutPass(pm, *this);
-  if (needsStubsPass())
-    mach_o::addStubsPass(pm, *this);
-  if (needsCompactUnwindPass())
-    mach_o::addCompactUnwindPass(pm, *this);
-  if (needsGOTPass())
-    mach_o::addGOTPass(pm, *this);
-  if (needsTLVPass())
-    mach_o::addTLVPass(pm, *this);
-  if (needsShimPass())
-    mach_o::addShimPass(pm, *this); // Shim pass must run after stubs pass.
-}
-
-Writer &MachOLinkingContext::writer() const {
-  if (!_writer)
-    _writer = createWriterMachO(*this);
-  return *_writer;
-}
-
-ErrorOr<std::unique_ptr<MemoryBuffer>>
-MachOLinkingContext::getMemoryBuffer(StringRef path) {
-  addInputFileDependency(path);
-
-  ErrorOr<std::unique_ptr<MemoryBuffer>> mbOrErr =
-    MemoryBuffer::getFileOrSTDIN(path);
-  if (std::error_code ec = mbOrErr.getError())
-    return ec;
-  std::unique_ptr<MemoryBuffer> mb = std::move(mbOrErr.get());
-
-  // If buffer contains a fat file, find required arch in fat buffer
-  // and switch buffer to point to just that required slice.
-  uint32_t offset;
-  uint32_t size;
-  if (sliceFromFatFile(mb->getMemBufferRef(), offset, size))
-    return MemoryBuffer::getFileSlice(path, size, offset);
-  return std::move(mb);
-}
-
-MachODylibFile* MachOLinkingContext::loadIndirectDylib(StringRef path) {
-  ErrorOr<std::unique_ptr<MemoryBuffer>> mbOrErr = getMemoryBuffer(path);
-  if (mbOrErr.getError())
-    return nullptr;
-
-  ErrorOr<std::unique_ptr<File>> fileOrErr =
-      registry().loadFile(std::move(mbOrErr.get()));
-  if (!fileOrErr)
-    return nullptr;
-  std::unique_ptr<File> &file = fileOrErr.get();
-  file->parse();
-  MachODylibFile *result = reinterpret_cast<MachODylibFile *>(file.get());
-  // Node object now owned by _indirectDylibs vector.
-  _indirectDylibs.push_back(std::move(file));
-  return result;
-}
-
-MachODylibFile* MachOLinkingContext::findIndirectDylib(StringRef path) {
-  // See if already loaded.
-  auto pos = _pathToDylibMap.find(path);
-  if (pos != _pathToDylibMap.end())
-    return pos->second;
-
-  // Search -L paths if of the form "libXXX.dylib"
-  std::pair<StringRef, StringRef> split = path.rsplit('/');
-  StringRef leafName = split.second;
-  if (leafName.startswith("lib") && leafName.endswith(".dylib")) {
-    // FIXME: Need to enhance searchLibrary() to only look for .dylib
-    auto libPath = searchLibrary(leafName);
-    if (libPath)
-      return loadIndirectDylib(libPath.getValue());
-  }
-
-  // Try full path with sysroot.
-  for (StringRef sysPath : _syslibRoots) {
-    SmallString<256> fullPath;
-    fullPath.assign(sysPath);
-    llvm::sys::path::append(fullPath, path);
-    if (pathExists(fullPath))
-      return loadIndirectDylib(fullPath);
-  }
-
-  // Try full path.
-  if (pathExists(path)) {
-    return loadIndirectDylib(path);
-  }
-
-  return nullptr;
-}
-
-uint32_t MachOLinkingContext::dylibCurrentVersion(StringRef installName) const {
-  auto pos = _pathToDylibMap.find(installName);
-  if (pos != _pathToDylibMap.end())
-    return pos->second->currentVersion();
-  else
-    return 0x10000; // 1.0
-}
-
-uint32_t MachOLinkingContext::dylibCompatVersion(StringRef installName) const {
-  auto pos = _pathToDylibMap.find(installName);
-  if (pos != _pathToDylibMap.end())
-    return pos->second->compatVersion();
-  else
-    return 0x10000; // 1.0
-}
-
-void MachOLinkingContext::createImplicitFiles(
-                            std::vector<std::unique_ptr<File> > &result) {
-  // Add indirect dylibs by asking each linked dylib to add its indirects.
-  // Iterate until no more dylibs get loaded.
-  size_t dylibCount = 0;
-  while (dylibCount != _allDylibs.size()) {
-    dylibCount = _allDylibs.size();
-    for (MachODylibFile *dylib : _allDylibs) {
-      dylib->loadReExportedDylibs([this] (StringRef path) -> MachODylibFile* {
-                                  return findIndirectDylib(path); });
-    }
-  }
-
-  // Let writer add output type specific extras.
-  writer().createImplicitFiles(result);
-
-  // If undefinedMode is != error, add a FlatNamespaceFile instance. This will
-  // provide a SharedLibraryAtom for symbols that aren't defined elsewhere.
-  if (undefinedMode() != UndefinedMode::error) {
-    result.emplace_back(new mach_o::FlatNamespaceFile(*this));
-    _flatNamespaceFile = result.back().get();
-  }
-}
-
-void MachOLinkingContext::registerDylib(MachODylibFile *dylib,
-                                        bool upward) const {
-  std::lock_guard<std::mutex> lock(_dylibsMutex);
-
-  if (!llvm::count(_allDylibs, dylib))
-    _allDylibs.push_back(dylib);
-  _pathToDylibMap[dylib->installName()] = dylib;
-  // If path is different than install name, register path too.
-  if (!dylib->path().equals(dylib->installName()))
-    _pathToDylibMap[dylib->path()] = dylib;
-  if (upward)
-    _upwardDylibs.insert(dylib);
-}
-
-bool MachOLinkingContext::isUpwardDylib(StringRef installName) const {
-  for (MachODylibFile *dylib : _upwardDylibs) {
-    if (dylib->installName().equals(installName))
-      return true;
-  }
-  return false;
-}
-
-ArchHandler &MachOLinkingContext::archHandler() const {
-  if (!_archHandler)
-    _archHandler = ArchHandler::create(_arch);
-  return *_archHandler;
-}
-
-void MachOLinkingContext::addSectionAlignment(StringRef seg, StringRef sect,
-                                              uint16_t align) {
-  SectionAlign entry = { seg, sect, align };
-  _sectAligns.push_back(entry);
-}
-
-void MachOLinkingContext::addSectCreateSection(
-                                        StringRef seg, StringRef sect,
-                                        std::unique_ptr<MemoryBuffer> content) {
-
-  if (!_sectCreateFile) {
-    auto sectCreateFile = std::make_unique<mach_o::SectCreateFile>();
-    _sectCreateFile = sectCreateFile.get();
-    getNodes().push_back(std::make_unique<FileNode>(std::move(sectCreateFile)));
-  }
-
-  assert(_sectCreateFile && "sectcreate file does not exist.");
-  _sectCreateFile->addSection(seg, sect, std::move(content));
-}
-
-bool MachOLinkingContext::sectionAligned(StringRef seg, StringRef sect,
-                                         uint16_t &align) const {
-  for (const SectionAlign &entry : _sectAligns) {
-    if (seg.equals(entry.segmentName) && sect.equals(entry.sectionName)) {
-      align = entry.align;
-      return true;
-    }
-  }
-  return false;
-}
-
-void MachOLinkingContext::addExportSymbol(StringRef sym) {
-  // Support old crufty export lists with bogus entries.
-  if (sym.endswith(".eh") || sym.startswith(".objc_category_name_")) {
-    llvm::errs() << "warning: ignoring " << sym << " in export list\n";
-    return;
-  }
-  // Only i386 MacOSX uses old ABI, so don't change those.
-  if ((_os != OS::macOSX) || (_arch != arch_x86)) {
-    // ObjC has two different ABIs.  Be nice and allow one export list work for
-    // both ABIs by renaming symbols.
-    if (sym.startswith(".objc_class_name_")) {
-      std::string abi2className("_OBJC_CLASS_$_");
-      abi2className += sym.substr(17);
-      _exportedSymbols.insert(copy(abi2className));
-      std::string abi2metaclassName("_OBJC_METACLASS_$_");
-      abi2metaclassName += sym.substr(17);
-      _exportedSymbols.insert(copy(abi2metaclassName));
-      return;
-    }
-  }
-
-  // FIXME: Support wildcards.
-  _exportedSymbols.insert(sym);
-}
-
-bool MachOLinkingContext::exportSymbolNamed(StringRef sym) const {
-  switch (_exportMode) {
-  case ExportMode::globals:
-    llvm_unreachable("exportSymbolNamed() should not be called in this mode");
-    break;
-  case ExportMode::exported:
-    return _exportedSymbols.count(sym);
-  case ExportMode::unexported:
-    return !_exportedSymbols.count(sym);
-  }
-  llvm_unreachable("_exportMode unknown enum value");
-}
-
-std::string MachOLinkingContext::demangle(StringRef symbolName) const {
-  // Only try to demangle symbols if -demangle on command line
-  if (!demangleSymbols())
-    return std::string(symbolName);
-
-  // Only try to demangle symbols that look like C++ symbols
-  if (!symbolName.startswith("__Z"))
-    return std::string(symbolName);
-
-  SmallString<256> symBuff;
-  StringRef nullTermSym = Twine(symbolName).toNullTerminatedStringRef(symBuff);
-  // Mach-O has extra leading underscore that needs to be removed.
-  const char *cstr = nullTermSym.data() + 1;
-  int status;
-  char *demangled = llvm::itaniumDemangle(cstr, nullptr, nullptr, &status);
-  if (demangled) {
-    std::string result(demangled);
-    // __cxa_demangle() always uses a malloc'ed buffer to return the result.
-    free(demangled);
-    return result;
-  }
-
-  return std::string(symbolName);
-}
-
-static void addDependencyInfoHelper(llvm::raw_fd_ostream *DepInfo,
-                                    char Opcode, StringRef Path) {
-  if (!DepInfo)
-    return;
-
-  *DepInfo << Opcode;
-  *DepInfo << Path;
-  *DepInfo << '\0';
-}
-
-std::error_code MachOLinkingContext::createDependencyFile(StringRef path) {
-  std::error_code ec;
-  _dependencyInfo = std::unique_ptr<llvm::raw_fd_ostream>(
-      new llvm::raw_fd_ostream(path, ec, llvm::sys::fs::OF_None));
-  if (ec) {
-    _dependencyInfo.reset();
-    return ec;
-  }
-
-  addDependencyInfoHelper(_dependencyInfo.get(), 0x00, "lld" /*FIXME*/);
-  return std::error_code();
-}
-
-void MachOLinkingContext::addInputFileDependency(StringRef path) const {
-  addDependencyInfoHelper(_dependencyInfo.get(), 0x10, path);
-}
-
-void MachOLinkingContext::addInputFileNotFound(StringRef path) const {
-  addDependencyInfoHelper(_dependencyInfo.get(), 0x11, path);
-}
-
-void MachOLinkingContext::addOutputFileDependency(StringRef path) const {
-  addDependencyInfoHelper(_dependencyInfo.get(), 0x40, path);
-}
-
-void MachOLinkingContext::appendOrderedSymbol(StringRef symbol,
-                                              StringRef filename) {
-  // To support sorting static functions which may have the same name in
-  // multiple .o files, _orderFiles maps the symbol name to a vector
-  // of OrderFileNode each of which can specify a file prefix.
-  OrderFileNode info;
-  if (!filename.empty())
-    info.fileFilter = copy(filename);
-  info.order = _orderFileEntries++;
-  _orderFiles[symbol].push_back(info);
-}
-
-bool
-MachOLinkingContext::findOrderOrdinal(const std::vector<OrderFileNode> &nodes,
-                                      const DefinedAtom *atom,
-                                      unsigned &ordinal) {
-  const File *objFile = &atom->file();
-  assert(objFile);
-  StringRef objName = objFile->path();
-  std::pair<StringRef, StringRef> dirAndLeaf = objName.rsplit('/');
-  if (!dirAndLeaf.second.empty())
-    objName = dirAndLeaf.second;
-  for (const OrderFileNode &info : nodes) {
-    if (info.fileFilter.empty()) {
-      // Have unprefixed symbol name in order file that matches this atom.
-      ordinal = info.order;
-      return true;
-    }
-    if (info.fileFilter.equals(objName)) {
-      // Have prefixed symbol name in order file that matches atom's path.
-      ordinal = info.order;
-      return true;
-    }
-  }
-  return false;
-}
-
-bool MachOLinkingContext::customAtomOrderer(const DefinedAtom *left,
-                                            const DefinedAtom *right,
-                                            bool &leftBeforeRight) const {
-  // No custom sorting if no order file entries.
-  if (!_orderFileEntries)
-    return false;
-
-  // Order files can only order named atoms.
-  StringRef leftName = left->name();
-  StringRef rightName = right->name();
-  if (leftName.empty() || rightName.empty())
-    return false;
-
-  // If neither is in order file list, no custom sorter.
-  auto leftPos = _orderFiles.find(leftName);
-  auto rightPos = _orderFiles.find(rightName);
-  bool leftIsOrdered = (leftPos != _orderFiles.end());
-  bool rightIsOrdered = (rightPos != _orderFiles.end());
-  if (!leftIsOrdered && !rightIsOrdered)
-    return false;
-
-  // There could be multiple symbols with same name but different file prefixes.
-  unsigned leftOrder;
-  unsigned rightOrder;
-  bool foundLeft =
-      leftIsOrdered && findOrderOrdinal(leftPos->getValue(), left, leftOrder);
-  bool foundRight = rightIsOrdered &&
-                    findOrderOrdinal(rightPos->getValue(), right, rightOrder);
-  if (!foundLeft && !foundRight)
-    return false;
-
-  // If only one is in order file list, ordered one goes first.
-  if (foundLeft != foundRight)
-    leftBeforeRight = foundLeft;
-  else
-    leftBeforeRight = (leftOrder < rightOrder);
-
-  return true;
-}
-
-static bool isLibrary(const std::unique_ptr<Node> &elem) {
-  if (FileNode *node = dyn_cast<FileNode>(const_cast<Node *>(elem.get()))) {
-    File *file = node->getFile();
-    return isa<SharedLibraryFile>(file) || isa<ArchiveLibraryFile>(file);
-  }
-  return false;
-}
-
-// The darwin linker processes input files in two phases.  The first phase
-// links in all object (.o) files in command line order. The second phase
-// links in libraries in command line order.
-// In this function we reorder the input files so that all the object files
-// comes before any library file. We also make a group for the library files
-// so that the Resolver will reiterate over the libraries as long as we find
-// new undefines from libraries.
-void MachOLinkingContext::finalizeInputFiles() {
-  std::vector<std::unique_ptr<Node>> &elements = getNodes();
-  llvm::stable_sort(elements, [](const std::unique_ptr<Node> &a,
-                                 const std::unique_ptr<Node> &b) {
-    return !isLibrary(a) && isLibrary(b);
-  });
-  size_t numLibs = std::count_if(elements.begin(), elements.end(), isLibrary);
-  elements.push_back(std::make_unique<GroupEnd>(numLibs));
-}
-
-llvm::Error MachOLinkingContext::handleLoadedFile(File &file) {
-  auto *machoFile = dyn_cast<MachOFile>(&file);
-  if (!machoFile)
-    return llvm::Error::success();
-
-  // Check that the arch of the context matches that of the file.
-  // Also set the arch of the context if it didn't have one.
-  if (_arch == arch_unknown) {
-    _arch = machoFile->arch();
-  } else if (machoFile->arch() != arch_unknown && machoFile->arch() != _arch) {
-    // Archs are different.
-    return llvm::make_error<GenericError>(file.path() +
-                  Twine(" cannot be linked due to incompatible architecture"));
-  }
-
-  // Check that the OS of the context matches that of the file.
-  // Also set the OS of the context if it didn't have one.
-  if (_os == OS::unknown) {
-    _os = machoFile->OS();
-  } else if (machoFile->OS() != OS::unknown && machoFile->OS() != _os) {
-    // OSes are different.
-    return llvm::make_error<GenericError>(file.path() +
-              Twine(" cannot be linked due to incompatible operating systems"));
-  }
-
-  // Check that if the objc info exists, that it is compatible with the target
-  // OS.
-  switch (machoFile->objcConstraint()) {
-    case objc_unknown:
-      // The file is not compiled with objc, so skip the checks.
-      break;
-    case objc_gc_only:
-    case objc_supports_gc:
-      llvm_unreachable("GC support should already have thrown an error");
-    case objc_retainReleaseForSimulator:
-      // The file is built with simulator objc, so make sure that the context
-      // is also building with simulator support.
-      if (_os != OS::iOS_simulator)
-        return llvm::make_error<GenericError>(file.path() +
-          Twine(" cannot be linked.  It contains ObjC built for the simulator"
-                " while we are linking a non-simulator target"));
-      assert((_objcConstraint == objc_unknown ||
-              _objcConstraint == objc_retainReleaseForSimulator) &&
-             "Must be linking with retain/release for the simulator");
-      _objcConstraint = objc_retainReleaseForSimulator;
-      break;
-    case objc_retainRelease:
-      // The file is built without simulator objc, so make sure that the
-      // context is also building without simulator support.
-      if (_os == OS::iOS_simulator)
-        return llvm::make_error<GenericError>(file.path() +
-          Twine(" cannot be linked.  It contains ObjC built for a non-simulator"
-                " target while we are linking a simulator target"));
-      assert((_objcConstraint == objc_unknown ||
-              _objcConstraint == objc_retainRelease) &&
-             "Must be linking with retain/release for a non-simulator target");
-      _objcConstraint = objc_retainRelease;
-      break;
-  }
-
-  // Check that the swift version of the context matches that of the file.
-  // Also set the swift version of the context if it didn't have one.
-  if (!_swiftVersion) {
-    _swiftVersion = machoFile->swiftVersion();
-  } else if (machoFile->swiftVersion() &&
-             machoFile->swiftVersion() != _swiftVersion) {
-    // Swift versions are different.
-    return llvm::make_error<GenericError>("different swift versions");
-  }
-
-  return llvm::Error::success();
-}
-
-} // end namespace lld
diff --git a/lld/lib/ReaderWriter/MachO/MachONormalizedFile.h b/lld/lib/ReaderWriter/MachO/MachONormalizedFile.h
deleted file mode 100644
index 3ef2949addab..000000000000
--- a/lld/lib/ReaderWriter/MachO/MachONormalizedFile.h
+++ /dev/null
@@ -1,336 +0,0 @@
-//===- lib/ReaderWriter/MachO/MachONormalizedFile.h -----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-///
-/// \file These data structures comprise the "normalized" view of
-/// mach-o object files. The normalized view is an in-memory only data structure
-/// which is always in native endianness and pointer size.
-///
-/// The normalized view easily converts to and from YAML using YAML I/O.
-///
-/// The normalized view converts to and from binary mach-o object files using
-/// the writeBinary() and readBinary() functions.
-///
-/// The normalized view converts to and from lld::Atoms using the
-/// normalizedToAtoms() and normalizedFromAtoms().
-///
-/// Overall, the conversion paths available look like:
-///
-///                 +---------------+
-///                 | binary mach-o |
-///                 +---------------+
-///                        ^
-///                        |
-///                        v
-///                  +------------+         +------+
-///                  | normalized |   <->   | yaml |
-///                  +------------+         +------+
-///                        ^
-///                        |
-///                        v
-///                    +-------+
-///                    | Atoms |
-///                    +-------+
-///
-
-#ifndef LLD_READER_WRITER_MACHO_NORMALIZE_FILE_H
-#define LLD_READER_WRITER_MACHO_NORMALIZE_FILE_H
-
-#include "DebugInfo.h"
-#include "lld/Common/LLVM.h"
-#include "lld/Core/Error.h"
-#include "lld/ReaderWriter/MachOLinkingContext.h"
-#include "llvm/ADT/SmallString.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/BinaryFormat/MachO.h"
-#include "llvm/Support/Allocator.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/ErrorOr.h"
-#include "llvm/Support/YAMLTraits.h"
-
-using llvm::BumpPtrAllocator;
-using llvm::yaml::Hex64;
-using llvm::yaml::Hex32;
-using llvm::yaml::Hex16;
-using llvm::yaml::Hex8;
-using llvm::yaml::SequenceTraits;
-using llvm::MachO::HeaderFileType;
-using llvm::MachO::BindType;
-using llvm::MachO::RebaseType;
-using llvm::MachO::NListType;
-using llvm::MachO::RelocationInfoType;
-using llvm::MachO::SectionType;
-using llvm::MachO::LoadCommandType;
-using llvm::MachO::ExportSymbolKind;
-using llvm::MachO::DataRegionType;
-
-namespace lld {
-namespace mach_o {
-namespace normalized {
-
-
-/// The real mach-o relocation record is 8-bytes on disk and is
-/// encoded in one of two different bit-field patterns.  This
-/// normalized form has the union of all possible fields.
-struct Relocation {
-  Relocation() : offset(0), scattered(false),
-                 type(llvm::MachO::GENERIC_RELOC_VANILLA),
-                 length(0), pcRel(false), isExtern(false), value(0),
-                 symbol(0) { }
-
-  Hex32               offset;
-  bool                scattered;
-  RelocationInfoType  type;
-  uint8_t             length;
-  bool                pcRel;
-  bool                isExtern;
-  Hex32               value;
-  uint32_t            symbol;
-};
-
-/// A typedef so that YAML I/O can treat this vector as a sequence.
-typedef std::vector<Relocation> Relocations;
-
-/// A typedef so that YAML I/O can process the raw bytes in a section.
-typedef std::vector<Hex8> ContentBytes;
-
-/// A typedef so that YAML I/O can treat indirect symbols as a flow sequence.
-typedef std::vector<uint32_t> IndirectSymbols;
-
-/// A typedef so that YAML I/O can encode/decode section attributes.
-LLVM_YAML_STRONG_TYPEDEF(uint32_t, SectionAttr)
-
-/// A typedef so that YAML I/O can encode/decode section alignment.
-LLVM_YAML_STRONG_TYPEDEF(uint16_t, SectionAlignment)
-
-/// Mach-O has a 32-bit and 64-bit section record.  This normalized form
-/// can support either kind.
-struct Section {
-  Section() : type(llvm::MachO::S_REGULAR),
-              attributes(0), alignment(1), address(0) { }
-
-  StringRef       segmentName;
-  StringRef       sectionName;
-  SectionType     type;
-  SectionAttr     attributes;
-  SectionAlignment        alignment;
-  Hex64           address;
-  ArrayRef<uint8_t> content;
-  Relocations     relocations;
-  IndirectSymbols indirectSymbols;
-};
-
-
-/// A typedef so that YAML I/O can encode/decode the scope bits of an nlist.
-LLVM_YAML_STRONG_TYPEDEF(uint8_t, SymbolScope)
-
-/// A typedef so that YAML I/O can encode/decode the desc bits of an nlist.
-LLVM_YAML_STRONG_TYPEDEF(uint16_t, SymbolDesc)
-
-/// Mach-O has a 32-bit and 64-bit symbol table entry (nlist), and the symbol
-/// type and scope and mixed in the same n_type field.  This normalized form
-/// works for any pointer size and separates out the type and scope.
-struct Symbol {
-  Symbol() : type(llvm::MachO::N_UNDF), scope(0), sect(0), desc(0), value(0) { }
-
-  StringRef     name;
-  NListType     type;
-  SymbolScope   scope;
-  uint8_t       sect;
-  SymbolDesc    desc;
-  Hex64         value;
-};
-
-/// Check whether the given section type indicates a zero-filled section.
-// FIXME: Utility functions of this kind should probably be moved into
-//        llvm/Support.
-inline bool isZeroFillSection(SectionType T) {
-  return (T == llvm::MachO::S_ZEROFILL ||
-          T == llvm::MachO::S_THREAD_LOCAL_ZEROFILL);
-}
-
-/// A typedef so that YAML I/O can (de/en)code the protection bits of a segment.
-LLVM_YAML_STRONG_TYPEDEF(uint32_t, VMProtect)
-
-/// A typedef to hold verions X.Y.X packed into 32-bit xxxx.yy.zz
-LLVM_YAML_STRONG_TYPEDEF(uint32_t, PackedVersion)
-
-/// Segments are only used in normalized final linked images (not in relocatable
-/// object files). They specify how a range of the file is loaded.
-struct Segment {
-  StringRef     name;
-  Hex64         address;
-  Hex64         size;
-  VMProtect     init_access;
-  VMProtect     max_access;
-};
-
-/// Only used in normalized final linked images to specify on which dylibs
-/// it depends.
-struct DependentDylib {
-  StringRef       path;
-  LoadCommandType kind;
-  PackedVersion   compatVersion;
-  PackedVersion   currentVersion;
-};
-
-/// A normalized rebasing entry.  Only used in normalized final linked images.
-struct RebaseLocation {
-  Hex32         segOffset;
-  uint8_t       segIndex;
-  RebaseType    kind;
-};
-
-/// A normalized binding entry.  Only used in normalized final linked images.
-struct BindLocation {
-  Hex32           segOffset;
-  uint8_t         segIndex;
-  BindType        kind;
-  bool            canBeNull;
-  int             ordinal;
-  StringRef       symbolName;
-  Hex64           addend;
-};
-
-/// A typedef so that YAML I/O can encode/decode export flags.
-LLVM_YAML_STRONG_TYPEDEF(uint32_t, ExportFlags)
-
-/// A normalized export entry.  Only used in normalized final linked images.
-struct Export {
-  StringRef         name;
-  Hex64             offset;
-  ExportSymbolKind  kind;
-  ExportFlags       flags;
-  Hex32             otherOffset;
-  StringRef         otherName;
-};
-
-/// A normalized data-in-code entry.
-struct DataInCode {
-  Hex32           offset;
-  Hex16           length;
-  DataRegionType  kind;
-};
-
-/// A typedef so that YAML I/O can encode/decode mach_header.flags.
-LLVM_YAML_STRONG_TYPEDEF(uint32_t, FileFlags)
-
-///
-struct NormalizedFile {
-  MachOLinkingContext::Arch   arch = MachOLinkingContext::arch_unknown;
-  HeaderFileType              fileType = llvm::MachO::MH_OBJECT;
-  FileFlags                   flags = 0;
-  std::vector<Segment>        segments; // Not used in object files.
-  std::vector<Section>        sections;
-
-  // Symbols sorted by kind.
-  std::vector<Symbol>         localSymbols;
-  std::vector<Symbol>         globalSymbols;
-  std::vector<Symbol>         undefinedSymbols;
-  std::vector<Symbol>         stabsSymbols;
-
-  // Maps to load commands with no LINKEDIT content (final linked images only).
-  std::vector<DependentDylib> dependentDylibs;
-  StringRef                   installName;        // dylibs only
-  PackedVersion               compatVersion = 0;  // dylibs only
-  PackedVersion               currentVersion = 0; // dylibs only
-  bool                        hasUUID = false;
-  bool                        hasMinVersionLoadCommand = false;
-  bool                        generateDataInCodeLoadCommand = false;
-  std::vector<StringRef>      rpaths;
-  Hex64                       entryAddress = 0;
-  Hex64                       stackSize = 0;
-  MachOLinkingContext::OS     os = MachOLinkingContext::OS::unknown;
-  Hex64                       sourceVersion = 0;
-  PackedVersion               minOSverson = 0;
-  PackedVersion               sdkVersion = 0;
-  LoadCommandType             minOSVersionKind = (LoadCommandType)0;
-
-  // Maps to load commands with LINKEDIT content (final linked images only).
-  Hex32                       pageSize = 0;
-  std::vector<RebaseLocation> rebasingInfo;
-  std::vector<BindLocation>   bindingInfo;
-  std::vector<BindLocation>   weakBindingInfo;
-  std::vector<BindLocation>   lazyBindingInfo;
-  std::vector<Export>         exportInfo;
-  std::vector<uint8_t>        functionStarts;
-  std::vector<DataInCode>     dataInCode;
-
-  // TODO:
-  // code-signature
-  // split-seg-info
-  // function-starts
-
-  // For any allocations in this struct which need to be owned by this struct.
-  BumpPtrAllocator            ownedAllocations;
-};
-
-/// Tests if a file is a non-fat mach-o object file.
-bool isThinObjectFile(StringRef path, MachOLinkingContext::Arch &arch);
-
-/// If the buffer is a fat file with the request arch, then this function
-/// returns true with 'offset' and 'size' set to location of the arch slice
-/// within the buffer.  Otherwise returns false;
-bool sliceFromFatFile(MemoryBufferRef mb, MachOLinkingContext::Arch arch,
-                      uint32_t &offset, uint32_t &size);
-
-/// Reads a mach-o file and produces an in-memory normalized view.
-llvm::Expected<std::unique_ptr<NormalizedFile>>
-readBinary(std::unique_ptr<MemoryBuffer> &mb,
-           const MachOLinkingContext::Arch arch);
-
-/// Takes in-memory normalized view and writes a mach-o object file.
-llvm::Error writeBinary(const NormalizedFile &file, StringRef path);
-
-size_t headerAndLoadCommandsSize(const NormalizedFile &file,
-                                 bool includeFunctionStarts);
-
-
-/// Parses a yaml encoded mach-o file to produce an in-memory normalized view.
-llvm::Expected<std::unique_ptr<NormalizedFile>>
-readYaml(std::unique_ptr<MemoryBuffer> &mb);
-
-/// Writes a yaml encoded mach-o files given an in-memory normalized view.
-std::error_code writeYaml(const NormalizedFile &file, raw_ostream &out);
-
-llvm::Error
-normalizedObjectToAtoms(MachOFile *file,
-                        const NormalizedFile &normalizedFile,
-                        bool copyRefs);
-
-llvm::Error
-normalizedDylibToAtoms(MachODylibFile *file,
-                       const NormalizedFile &normalizedFile,
-                       bool copyRefs);
-
-/// Takes in-memory normalized dylib or object and parses it into lld::File
-llvm::Expected<std::unique_ptr<lld::File>>
-normalizedToAtoms(const NormalizedFile &normalizedFile, StringRef path,
-                  bool copyRefs);
-
-/// Takes atoms and generates a normalized macho-o view.
-llvm::Expected<std::unique_ptr<NormalizedFile>>
-normalizedFromAtoms(const lld::File &atomFile, const MachOLinkingContext &ctxt);
-
-
-} // namespace normalized
-
-/// Class for interfacing mach-o yaml files into generic yaml parsing
-class MachOYamlIOTaggedDocumentHandler : public YamlIOTaggedDocumentHandler {
-public:
-  MachOYamlIOTaggedDocumentHandler(MachOLinkingContext::Arch arch)
-    : _arch(arch) { }
-  bool handledDocTag(llvm::yaml::IO &io, const lld::File *&file) const override;
-private:
-  const MachOLinkingContext::Arch _arch;
-};
-
-} // namespace mach_o
-} // namespace lld
-
-#endif // LLD_READER_WRITER_MACHO_NORMALIZE_FILE_H
diff --git a/lld/lib/ReaderWriter/MachO/MachONormalizedFileBinaryReader.cpp b/lld/lib/ReaderWriter/MachO/MachONormalizedFileBinaryReader.cpp
deleted file mode 100644
index 87601ca1be8b..000000000000
--- a/lld/lib/ReaderWriter/MachO/MachONormalizedFileBinaryReader.cpp
+++ /dev/null
@@ -1,614 +0,0 @@
-//===- lib/ReaderWriter/MachO/MachONormalizedFileBinaryReader.cpp ---------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-///
-/// \file For mach-o object files, this implementation converts from
-/// mach-o on-disk binary format to in-memory normalized mach-o.
-///
-///                 +---------------+
-///                 | binary mach-o |
-///                 +---------------+
-///                        |
-///                        |
-///                        v
-///                  +------------+
-///                  | normalized |
-///                  +------------+
-
-#include "ArchHandler.h"
-#include "MachONormalizedFile.h"
-#include "MachONormalizedFileBinaryUtils.h"
-#include "lld/Common/LLVM.h"
-#include "lld/Core/Error.h"
-#include "lld/Core/SharedLibraryFile.h"
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/SmallString.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/ADT/StringSwitch.h"
-#include "llvm/ADT/Twine.h"
-#include "llvm/BinaryFormat/MachO.h"
-#include "llvm/BinaryFormat/Magic.h"
-#include "llvm/Object/MachO.h"
-#include "llvm/Support/Casting.h"
-#include "llvm/Support/Errc.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/FileOutputBuffer.h"
-#include "llvm/Support/Host.h"
-#include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/raw_ostream.h"
-#include <functional>
-#include <system_error>
-
-using namespace llvm::MachO;
-using llvm::object::ExportEntry;
-using llvm::file_magic;
-using llvm::object::MachOObjectFile;
-
-namespace lld {
-namespace mach_o {
-namespace normalized {
-
-// Utility to call a lambda expression on each load command.
-static llvm::Error forEachLoadCommand(
-    StringRef lcRange, unsigned lcCount, bool isBig, bool is64,
-    std::function<bool(uint32_t cmd, uint32_t size, const char *lc)> func) {
-  const char* p = lcRange.begin();
-  for (unsigned i=0; i < lcCount; ++i) {
-    const load_command *lc = reinterpret_cast<const load_command*>(p);
-    load_command lcCopy;
-    const load_command *slc = lc;
-    if (isBig != llvm::sys::IsBigEndianHost) {
-      memcpy(&lcCopy, lc, sizeof(load_command));
-      swapStruct(lcCopy);
-      slc = &lcCopy;
-    }
-    if ( (p + slc->cmdsize) > lcRange.end() )
-      return llvm::make_error<GenericError>("Load command exceeds range");
-
-    if (func(slc->cmd, slc->cmdsize, p))
-      return llvm::Error::success();
-
-    p += slc->cmdsize;
-  }
-
-  return llvm::Error::success();
-}
-
-static std::error_code appendRelocations(Relocations &relocs, StringRef buffer,
-                                         bool bigEndian,
-                                         uint32_t reloff, uint32_t nreloc) {
-  if ((reloff + nreloc*8) > buffer.size())
-    return make_error_code(llvm::errc::executable_format_error);
-  const any_relocation_info* relocsArray =
-            reinterpret_cast<const any_relocation_info*>(buffer.begin()+reloff);
-
-  for(uint32_t i=0; i < nreloc; ++i) {
-    relocs.push_back(unpackRelocation(relocsArray[i], bigEndian));
-  }
-  return std::error_code();
-}
-
-static std::error_code
-appendIndirectSymbols(IndirectSymbols &isyms, StringRef buffer, bool isBig,
-                      uint32_t istOffset, uint32_t istCount,
-                      uint32_t startIndex, uint32_t count) {
-  if ((istOffset + istCount*4) > buffer.size())
-    return make_error_code(llvm::errc::executable_format_error);
-  if (startIndex+count  > istCount)
-    return make_error_code(llvm::errc::executable_format_error);
-  const uint8_t *indirectSymbolArray = (const uint8_t *)buffer.data();
-
-  for(uint32_t i=0; i < count; ++i) {
-    isyms.push_back(read32(
-        indirectSymbolArray + (startIndex + i) * sizeof(uint32_t), isBig));
-  }
-  return std::error_code();
-}
-
-
-template <typename T> static T readBigEndian(T t) {
-  if (llvm::sys::IsLittleEndianHost)
-    llvm::sys::swapByteOrder(t);
-  return t;
-}
-
-
-static bool isMachOHeader(const mach_header *mh, bool &is64, bool &isBig) {
-  switch (read32(&mh->magic, false)) {
-  case llvm::MachO::MH_MAGIC:
-    is64 = false;
-    isBig = false;
-    return true;
-  case llvm::MachO::MH_MAGIC_64:
-    is64 = true;
-    isBig = false;
-    return true;
-  case llvm::MachO::MH_CIGAM:
-    is64 = false;
-    isBig = true;
-    return true;
-  case llvm::MachO::MH_CIGAM_64:
-    is64 = true;
-    isBig = true;
-    return true;
-  default:
-    return false;
-  }
-}
-
-
-bool isThinObjectFile(StringRef path, MachOLinkingContext::Arch &arch) {
-  // Try opening and mapping file at path.
-  ErrorOr<std::unique_ptr<MemoryBuffer>> b = MemoryBuffer::getFileOrSTDIN(path);
-  if (b.getError())
-    return false;
-
-  // If file length < 32 it is too small to be mach-o object file.
-  StringRef fileBuffer = b->get()->getBuffer();
-  if (fileBuffer.size() < 32)
-    return false;
-
-  // If file buffer does not start with MH_MAGIC (and variants), not obj file.
-  const mach_header *mh = reinterpret_cast<const mach_header *>(
-                                                            fileBuffer.begin());
-  bool is64, isBig;
-  if (!isMachOHeader(mh, is64, isBig))
-    return false;
-
-  // If not MH_OBJECT, not object file.
-  if (read32(&mh->filetype, isBig) != MH_OBJECT)
-    return false;
-
-  // Lookup up arch from cpu/subtype pair.
-  arch = MachOLinkingContext::archFromCpuType(
-      read32(&mh->cputype, isBig),
-      read32(&mh->cpusubtype, isBig));
-  return true;
-}
-
-bool sliceFromFatFile(MemoryBufferRef mb, MachOLinkingContext::Arch arch,
-                      uint32_t &offset, uint32_t &size) {
-  const char *start = mb.getBufferStart();
-  const llvm::MachO::fat_header *fh =
-      reinterpret_cast<const llvm::MachO::fat_header *>(start);
-  if (readBigEndian(fh->magic) != llvm::MachO::FAT_MAGIC)
-    return false;
-  uint32_t nfat_arch = readBigEndian(fh->nfat_arch);
-  const fat_arch *fstart =
-      reinterpret_cast<const fat_arch *>(start + sizeof(fat_header));
-  const fat_arch *fend =
-      reinterpret_cast<const fat_arch *>(start + sizeof(fat_header) +
-                                         sizeof(fat_arch) * nfat_arch);
-  const uint32_t reqCpuType = MachOLinkingContext::cpuTypeFromArch(arch);
-  const uint32_t reqCpuSubtype = MachOLinkingContext::cpuSubtypeFromArch(arch);
-  for (const fat_arch *fa = fstart; fa < fend; ++fa) {
-    if ((readBigEndian(fa->cputype) == reqCpuType) &&
-        (readBigEndian(fa->cpusubtype) == reqCpuSubtype)) {
-      offset = readBigEndian(fa->offset);
-      size = readBigEndian(fa->size);
-      if ((offset + size) > mb.getBufferSize())
-        return false;
-      return true;
-    }
-  }
-  return false;
-}
-
-/// Reads a mach-o file and produces an in-memory normalized view.
-llvm::Expected<std::unique_ptr<NormalizedFile>>
-readBinary(std::unique_ptr<MemoryBuffer> &mb,
-           const MachOLinkingContext::Arch arch) {
-  // Make empty NormalizedFile.
-  std::unique_ptr<NormalizedFile> f(new NormalizedFile());
-
-  const char *start = mb->getBufferStart();
-  size_t objSize = mb->getBufferSize();
-  const mach_header *mh = reinterpret_cast<const mach_header *>(start);
-
-  uint32_t sliceOffset;
-  uint32_t sliceSize;
-  if (sliceFromFatFile(mb->getMemBufferRef(), arch, sliceOffset, sliceSize)) {
-    start = &start[sliceOffset];
-    objSize = sliceSize;
-    mh = reinterpret_cast<const mach_header *>(start);
-  }
-
-  // Determine endianness and pointer size for mach-o file.
-  bool is64, isBig;
-  if (!isMachOHeader(mh, is64, isBig))
-    return llvm::make_error<GenericError>("File is not a mach-o");
-
-  // Endian swap header, if needed.
-  mach_header headerCopy;
-  const mach_header *smh = mh;
-  if (isBig != llvm::sys::IsBigEndianHost) {
-    memcpy(&headerCopy, mh, sizeof(mach_header));
-    swapStruct(headerCopy);
-    smh = &headerCopy;
-  }
-
-  // Validate head and load commands fit in buffer.
-  const uint32_t lcCount = smh->ncmds;
-  const char *lcStart =
-      start + (is64 ? sizeof(mach_header_64) : sizeof(mach_header));
-  StringRef lcRange(lcStart, smh->sizeofcmds);
-  if (lcRange.end() > (start + objSize))
-    return llvm::make_error<GenericError>("Load commands exceed file size");
-
-  // Get architecture from mach_header.
-  f->arch = MachOLinkingContext::archFromCpuType(smh->cputype, smh->cpusubtype);
-  if (f->arch != arch) {
-    return llvm::make_error<GenericError>(
-                                  Twine("file is wrong architecture. Expected "
-                                  "(" + MachOLinkingContext::nameFromArch(arch)
-                                  + ") found ("
-                                  + MachOLinkingContext::nameFromArch(f->arch)
-                                  + ")" ));
-  }
-  // Copy file type and flags
-  f->fileType = HeaderFileType(smh->filetype);
-  f->flags = smh->flags;
-
-
-  // Pre-scan load commands looking for indirect symbol table.
-  uint32_t indirectSymbolTableOffset = 0;
-  uint32_t indirectSymbolTableCount = 0;
-  auto ec = forEachLoadCommand(lcRange, lcCount, isBig, is64,
-                               [&](uint32_t cmd, uint32_t size,
-                                   const char *lc) -> bool {
-    if (cmd == LC_DYSYMTAB) {
-      const dysymtab_command *d = reinterpret_cast<const dysymtab_command*>(lc);
-      indirectSymbolTableOffset = read32(&d->indirectsymoff, isBig);
-      indirectSymbolTableCount = read32(&d->nindirectsyms, isBig);
-      return true;
-    }
-    return false;
-  });
-  if (ec)
-    return std::move(ec);
-
-  // Walk load commands looking for segments/sections and the symbol table.
-  const data_in_code_entry *dataInCode = nullptr;
-  const dyld_info_command *dyldInfo = nullptr;
-  uint32_t dataInCodeSize = 0;
-  ec = forEachLoadCommand(lcRange, lcCount, isBig, is64,
-                    [&] (uint32_t cmd, uint32_t size, const char* lc) -> bool {
-    switch(cmd) {
-    case LC_SEGMENT_64:
-      if (is64) {
-        const segment_command_64 *seg =
-                              reinterpret_cast<const segment_command_64*>(lc);
-        const unsigned sectionCount = read32(&seg->nsects, isBig);
-        const section_64 *sects = reinterpret_cast<const section_64*>
-                                  (lc + sizeof(segment_command_64));
-        const unsigned lcSize = sizeof(segment_command_64)
-                                              + sectionCount*sizeof(section_64);
-        // Verify sections don't extend beyond end of segment load command.
-        if (lcSize > size)
-          return true;
-        for (unsigned i=0; i < sectionCount; ++i) {
-          const section_64 *sect = &sects[i];
-          Section section;
-          section.segmentName = getString16(sect->segname);
-          section.sectionName = getString16(sect->sectname);
-          section.type = (SectionType)(read32(&sect->flags, isBig) &
-                                       SECTION_TYPE);
-          section.attributes  = read32(&sect->flags, isBig) & SECTION_ATTRIBUTES;
-          section.alignment   = 1 << read32(&sect->align, isBig);
-          section.address     = read64(&sect->addr, isBig);
-          const uint8_t *content =
-            (const uint8_t *)start + read32(&sect->offset, isBig);
-          size_t contentSize = read64(&sect->size, isBig);
-          // Note: this assign() is copying the content bytes.  Ideally,
-          // we can use a custom allocator for vector to avoid the copy.
-          section.content = llvm::makeArrayRef(content, contentSize);
-          appendRelocations(section.relocations, mb->getBuffer(), isBig,
-                            read32(&sect->reloff, isBig),
-                            read32(&sect->nreloc, isBig));
-          if (section.type == S_NON_LAZY_SYMBOL_POINTERS) {
-            appendIndirectSymbols(section.indirectSymbols, mb->getBuffer(),
-                                  isBig,
-                                  indirectSymbolTableOffset,
-                                  indirectSymbolTableCount,
-                                  read32(&sect->reserved1, isBig),
-                                  contentSize/4);
-          }
-          f->sections.push_back(section);
-        }
-      }
-      break;
-    case LC_SEGMENT:
-      if (!is64) {
-        const segment_command *seg =
-                              reinterpret_cast<const segment_command*>(lc);
-        const unsigned sectionCount = read32(&seg->nsects, isBig);
-        const section *sects = reinterpret_cast<const section*>
-                                  (lc + sizeof(segment_command));
-        const unsigned lcSize = sizeof(segment_command)
-                                              + sectionCount*sizeof(section);
-        // Verify sections don't extend beyond end of segment load command.
-        if (lcSize > size)
-          return true;
-        for (unsigned i=0; i < sectionCount; ++i) {
-          const section *sect = &sects[i];
-          Section section;
-          section.segmentName = getString16(sect->segname);
-          section.sectionName = getString16(sect->sectname);
-          section.type = (SectionType)(read32(&sect->flags, isBig) &
-                                       SECTION_TYPE);
-          section.attributes =
-              read32((const uint8_t *)&sect->flags, isBig) & SECTION_ATTRIBUTES;
-          section.alignment   = 1 << read32(&sect->align, isBig);
-          section.address     = read32(&sect->addr, isBig);
-          const uint8_t *content =
-            (const uint8_t *)start + read32(&sect->offset, isBig);
-          size_t contentSize = read32(&sect->size, isBig);
-          // Note: this assign() is copying the content bytes.  Ideally,
-          // we can use a custom allocator for vector to avoid the copy.
-          section.content = llvm::makeArrayRef(content, contentSize);
-          appendRelocations(section.relocations, mb->getBuffer(), isBig,
-                            read32(&sect->reloff, isBig),
-                            read32(&sect->nreloc, isBig));
-          if (section.type == S_NON_LAZY_SYMBOL_POINTERS) {
-            appendIndirectSymbols(
-                section.indirectSymbols, mb->getBuffer(), isBig,
-                indirectSymbolTableOffset, indirectSymbolTableCount,
-                read32(&sect->reserved1, isBig), contentSize / 4);
-          }
-          f->sections.push_back(section);
-        }
-      }
-      break;
-    case LC_SYMTAB: {
-      const symtab_command *st = reinterpret_cast<const symtab_command*>(lc);
-      const char *strings = start + read32(&st->stroff, isBig);
-      const uint32_t strSize = read32(&st->strsize, isBig);
-      // Validate string pool and symbol table all in buffer.
-      if (read32((const uint8_t *)&st->stroff, isBig) +
-              read32((const uint8_t *)&st->strsize, isBig) >
-          objSize)
-        return true;
-      if (is64) {
-        const uint32_t symOffset = read32(&st->symoff, isBig);
-        const uint32_t symCount = read32(&st->nsyms, isBig);
-        if ( symOffset+(symCount*sizeof(nlist_64)) > objSize)
-          return true;
-        const nlist_64 *symbols =
-            reinterpret_cast<const nlist_64 *>(start + symOffset);
-        // Convert each nlist_64 to a lld::mach_o::normalized::Symbol.
-        for(uint32_t i=0; i < symCount; ++i) {
-          nlist_64 tempSym;
-          memcpy(&tempSym, &symbols[i], sizeof(nlist_64));
-          const nlist_64 *sin = &tempSym;
-          if (isBig != llvm::sys::IsBigEndianHost)
-            swapStruct(tempSym);
-          Symbol sout;
-          if (sin->n_strx > strSize)
-            return true;
-          sout.name  = &strings[sin->n_strx];
-          sout.type = static_cast<NListType>(sin->n_type & (N_STAB|N_TYPE));
-          sout.scope = (sin->n_type & (N_PEXT|N_EXT));
-          sout.sect  = sin->n_sect;
-          sout.desc  = sin->n_desc;
-          sout.value = sin->n_value;
-          if (sin->n_type & N_STAB)
-            f->stabsSymbols.push_back(sout);
-          else if (sout.type == N_UNDF)
-            f->undefinedSymbols.push_back(sout);
-          else if (sin->n_type & N_EXT)
-            f->globalSymbols.push_back(sout);
-          else
-            f->localSymbols.push_back(sout);
-        }
-      } else {
-        const uint32_t symOffset = read32(&st->symoff, isBig);
-        const uint32_t symCount = read32(&st->nsyms, isBig);
-        if ( symOffset+(symCount*sizeof(nlist)) > objSize)
-          return true;
-        const nlist *symbols =
-            reinterpret_cast<const nlist *>(start + symOffset);
-        // Convert each nlist to a lld::mach_o::normalized::Symbol.
-        for(uint32_t i=0; i < symCount; ++i) {
-          const nlist *sin = &symbols[i];
-          nlist tempSym;
-          if (isBig != llvm::sys::IsBigEndianHost) {
-            tempSym = *sin; swapStruct(tempSym); sin = &tempSym;
-          }
-          Symbol sout;
-          if (sin->n_strx > strSize)
-            return true;
-          sout.name  = &strings[sin->n_strx];
-          sout.type  = (NListType)(sin->n_type & N_TYPE);
-          sout.scope = (sin->n_type & (N_PEXT|N_EXT));
-          sout.sect  = sin->n_sect;
-          sout.desc  = sin->n_desc;
-          sout.value = sin->n_value;
-          if (sout.type == N_UNDF)
-            f->undefinedSymbols.push_back(sout);
-          else if (sout.scope == (SymbolScope)N_EXT)
-            f->globalSymbols.push_back(sout);
-          else if (sin->n_type & N_STAB)
-            f->stabsSymbols.push_back(sout);
-          else
-            f->localSymbols.push_back(sout);
-        }
-      }
-      }
-      break;
-    case LC_ID_DYLIB: {
-      const dylib_command *dl = reinterpret_cast<const dylib_command*>(lc);
-      f->installName = lc + read32(&dl->dylib.name, isBig);
-      f->currentVersion = read32(&dl->dylib.current_version, isBig);
-      f->compatVersion = read32(&dl->dylib.compatibility_version, isBig);
-      }
-      break;
-    case LC_DATA_IN_CODE: {
-      const linkedit_data_command *ldc =
-                            reinterpret_cast<const linkedit_data_command*>(lc);
-      dataInCode = reinterpret_cast<const data_in_code_entry *>(
-          start + read32(&ldc->dataoff, isBig));
-      dataInCodeSize = read32(&ldc->datasize, isBig);
-      }
-      break;
-    case LC_LOAD_DYLIB:
-    case LC_LOAD_WEAK_DYLIB:
-    case LC_REEXPORT_DYLIB:
-    case LC_LOAD_UPWARD_DYLIB: {
-      const dylib_command *dl = reinterpret_cast<const dylib_command*>(lc);
-      DependentDylib entry;
-      entry.path = lc + read32(&dl->dylib.name, isBig);
-      entry.kind = LoadCommandType(cmd);
-      entry.compatVersion = read32(&dl->dylib.compatibility_version, isBig);
-      entry.currentVersion = read32(&dl->dylib.current_version, isBig);
-      f->dependentDylibs.push_back(entry);
-     }
-      break;
-    case LC_RPATH: {
-      const rpath_command *rpc = reinterpret_cast<const rpath_command *>(lc);
-      f->rpaths.push_back(lc + read32(&rpc->path, isBig));
-     }
-      break;
-    case LC_DYLD_INFO:
-    case LC_DYLD_INFO_ONLY:
-      dyldInfo = reinterpret_cast<const dyld_info_command*>(lc);
-      break;
-    case LC_VERSION_MIN_MACOSX:
-    case LC_VERSION_MIN_IPHONEOS:
-    case LC_VERSION_MIN_WATCHOS:
-    case LC_VERSION_MIN_TVOS:
-      // If we are emitting an object file, then we may take the load command
-      // kind from these commands and pass it on to the output
-      // file.
-      f->minOSVersionKind = (LoadCommandType)cmd;
-      break;
-    }
-    return false;
-  });
-  if (ec)
-    return std::move(ec);
-
-  if (dataInCode) {
-    // Convert on-disk data_in_code_entry array to DataInCode vector.
-    for (unsigned i=0; i < dataInCodeSize/sizeof(data_in_code_entry); ++i) {
-      DataInCode entry;
-      entry.offset = read32(&dataInCode[i].offset, isBig);
-      entry.length = read16(&dataInCode[i].length, isBig);
-      entry.kind =
-          (DataRegionType)read16((const uint8_t *)&dataInCode[i].kind, isBig);
-      f->dataInCode.push_back(entry);
-    }
-  }
-
-  if (dyldInfo) {
-    // If any exports, extract and add to normalized exportInfo vector.
-    if (dyldInfo->export_size) {
-      const uint8_t *trieStart = reinterpret_cast<const uint8_t *>(
-          start + read32(&dyldInfo->export_off, isBig));
-      ArrayRef<uint8_t> trie(trieStart, read32(&dyldInfo->export_size, isBig));
-      Error Err = Error::success();
-      for (const ExportEntry &trieExport : MachOObjectFile::exports(Err, trie)) {
-        Export normExport;
-        normExport.name = trieExport.name().copy(f->ownedAllocations);
-        normExport.offset = trieExport.address();
-        normExport.kind = ExportSymbolKind(trieExport.flags() & EXPORT_SYMBOL_FLAGS_KIND_MASK);
-        normExport.flags = trieExport.flags() & ~EXPORT_SYMBOL_FLAGS_KIND_MASK;
-        normExport.otherOffset = trieExport.other();
-        if (!trieExport.otherName().empty())
-          normExport.otherName = trieExport.otherName().copy(f->ownedAllocations);
-        f->exportInfo.push_back(normExport);
-      }
-      if (Err)
-        return std::move(Err);
-    }
-  }
-
-  return std::move(f);
-}
-
-class MachOObjectReader : public Reader {
-public:
-  MachOObjectReader(MachOLinkingContext &ctx) : _ctx(ctx) {}
-
-  bool canParse(file_magic magic, MemoryBufferRef mb) const override {
-    return (magic == file_magic::macho_object && mb.getBufferSize() > 32);
-  }
-
-  ErrorOr<std::unique_ptr<File>>
-  loadFile(std::unique_ptr<MemoryBuffer> mb,
-           const Registry &registry) const override {
-    std::unique_ptr<File> ret =
-      std::make_unique<MachOFile>(std::move(mb), &_ctx);
-    return std::move(ret);
-  }
-
-private:
-  MachOLinkingContext &_ctx;
-};
-
-class MachODylibReader : public Reader {
-public:
-  MachODylibReader(MachOLinkingContext &ctx) : _ctx(ctx) {}
-
-  bool canParse(file_magic magic, MemoryBufferRef mb) const override {
-    switch (magic) {
-    case file_magic::macho_dynamically_linked_shared_lib:
-    case file_magic::macho_dynamically_linked_shared_lib_stub:
-      return mb.getBufferSize() > 32;
-    default:
-      return false;
-    }
-  }
-
-  ErrorOr<std::unique_ptr<File>>
-  loadFile(std::unique_ptr<MemoryBuffer> mb,
-           const Registry &registry) const override {
-    std::unique_ptr<File> ret =
-        std::make_unique<MachODylibFile>(std::move(mb), &_ctx);
-    return std::move(ret);
-  }
-
-private:
-  MachOLinkingContext &_ctx;
-};
-
-class MachOTAPIReader : public Reader {
-public:
-  MachOTAPIReader(MachOLinkingContext &ctx) : _ctx(ctx) {}
-
-  bool canParse(file_magic magic, MemoryBufferRef mb) const override {
-    return magic == file_magic::tapi_file;
-  }
-
-  ErrorOr<std::unique_ptr<File>>
-  loadFile(std::unique_ptr<MemoryBuffer> mb,
-           const Registry &registry) const override {
-    std::unique_ptr<File> ret =
-        std::make_unique<TAPIFile>(std::move(mb), &_ctx);
-    return std::move(ret);
-  }
-
-private:
-  MachOLinkingContext &_ctx;
-};
-
-} // namespace normalized
-} // namespace mach_o
-
-void Registry::addSupportMachOObjects(MachOLinkingContext &ctx) {
-  MachOLinkingContext::Arch arch = ctx.arch();
-  add(std::unique_ptr<Reader>(new mach_o::normalized::MachOObjectReader(ctx)));
-  add(std::unique_ptr<Reader>(new mach_o::normalized::MachODylibReader(ctx)));
-  add(std::unique_ptr<Reader>(new mach_o::normalized::MachOTAPIReader(ctx)));
-  addKindTable(Reference::KindNamespace::mach_o, ctx.archHandler().kindArch(),
-               ctx.archHandler().kindStrings());
-  add(std::unique_ptr<YamlIOTaggedDocumentHandler>(
-                           new mach_o::MachOYamlIOTaggedDocumentHandler(arch)));
-}
-
-
-} // namespace lld
diff --git a/lld/lib/ReaderWriter/MachO/MachONormalizedFileBinaryUtils.h b/lld/lib/ReaderWriter/MachO/MachONormalizedFileBinaryUtils.h
deleted file mode 100644
index aeb04ef4508a..000000000000
--- a/lld/lib/ReaderWriter/MachO/MachONormalizedFileBinaryUtils.h
+++ /dev/null
@@ -1,213 +0,0 @@
-//===- lib/ReaderWriter/MachO/MachONormalizedFileBinaryUtils.h ------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLD_READER_WRITER_MACHO_NORMALIZED_FILE_BINARY_UTILS_H
-#define LLD_READER_WRITER_MACHO_NORMALIZED_FILE_BINARY_UTILS_H
-
-#include "MachONormalizedFile.h"
-#include "lld/Common/LLVM.h"
-#include "lld/Core/Error.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/BinaryFormat/MachO.h"
-#include "llvm/Support/Casting.h"
-#include "llvm/Support/Endian.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/Host.h"
-#include "llvm/Support/LEB128.h"
-#include <system_error>
-
-namespace lld {
-namespace mach_o {
-namespace normalized {
-
-class ByteBuffer {
-public:
-  ByteBuffer() : _ostream(_bytes) { }
-
-  void append_byte(uint8_t b) {
-    _ostream << b;
-  }
-  void append_uleb128(uint64_t value) {
-    llvm::encodeULEB128(value, _ostream);
-  }
-  void append_uleb128Fixed(uint64_t value, unsigned byteCount) {
-    unsigned min = llvm::getULEB128Size(value);
-    assert(min <= byteCount);
-    unsigned pad = byteCount - min;
-    llvm::encodeULEB128(value, _ostream, pad);
-  }
-  void append_sleb128(int64_t value) {
-    llvm::encodeSLEB128(value, _ostream);
-  }
-  void append_string(StringRef str) {
-    _ostream << str;
-    append_byte(0);
-  }
-  void align(unsigned alignment) {
-    while ( (_ostream.tell() % alignment) != 0 )
-      append_byte(0);
-  }
-  size_t size() {
-    return _ostream.tell();
-  }
-  const uint8_t *bytes() {
-    return reinterpret_cast<const uint8_t*>(_ostream.str().data());
-  }
-
-private:
-  SmallVector<char, 128>        _bytes;
-  // Stream ivar must be after SmallVector ivar to construct properly.
-  llvm::raw_svector_ostream     _ostream;
-};
-
-using namespace llvm::support::endian;
-using llvm::sys::getSwappedBytes;
-
-template<typename T>
-static inline uint16_t read16(const T *loc, bool isBig) {
-  assert((uint64_t)loc % alignof(T) == 0 && "invalid pointer alignment");
-  return isBig ? read16be(loc) : read16le(loc);
-}
-
-template<typename T>
-static inline uint32_t read32(const T *loc, bool isBig) {
-  assert((uint64_t)loc % alignof(T) == 0 && "invalid pointer alignment");
-  return isBig ? read32be(loc) : read32le(loc);
-}
-
-template<typename T>
-static inline uint64_t read64(const T *loc, bool isBig) {
-  assert((uint64_t)loc % alignof(T) == 0 && "invalid pointer alignment");
-  return isBig ? read64be(loc) : read64le(loc);
-}
-
-inline void write16(uint8_t *loc, uint16_t value, bool isBig) {
-  if (isBig)
-    write16be(loc, value);
-  else
-    write16le(loc, value);
-}
-
-inline void write32(uint8_t *loc, uint32_t value, bool isBig) {
-  if (isBig)
-    write32be(loc, value);
-  else
-    write32le(loc, value);
-}
-
-inline void write64(uint8_t *loc, uint64_t value, bool isBig) {
-  if (isBig)
-    write64be(loc, value);
-  else
-    write64le(loc, value);
-}
-
-inline uint32_t
-bitFieldExtract(uint32_t value, bool isBigEndianBigField, uint8_t firstBit,
-                                                          uint8_t bitCount) {
-  const uint32_t mask = ((1<<bitCount)-1);
-  const uint8_t shift = isBigEndianBigField ? (32-firstBit-bitCount) : firstBit;
-  return (value >> shift) & mask;
-}
-
-inline void
-bitFieldSet(uint32_t &bits, bool isBigEndianBigField, uint32_t newBits,
-                            uint8_t firstBit, uint8_t bitCount) {
-  const uint32_t mask = ((1<<bitCount)-1);
-  assert((newBits & mask) == newBits);
-  const uint8_t shift = isBigEndianBigField ? (32-firstBit-bitCount) : firstBit;
-  bits &= ~(mask << shift);
-  bits |= (newBits << shift);
-}
-
-inline Relocation unpackRelocation(const llvm::MachO::any_relocation_info &r,
-                                   bool isBigEndian) {
-  uint32_t r0 = read32(&r.r_word0, isBigEndian);
-  uint32_t r1 = read32(&r.r_word1, isBigEndian);
-
-  Relocation result;
-  if (r0 & llvm::MachO::R_SCATTERED) {
-    // scattered relocation record always laid out like big endian bit field
-    result.offset     = bitFieldExtract(r0, true, 8, 24);
-    result.scattered  = true;
-    result.type       = (RelocationInfoType)
-                        bitFieldExtract(r0, true, 4, 4);
-    result.length     = bitFieldExtract(r0, true, 2, 2);
-    result.pcRel      = bitFieldExtract(r0, true, 1, 1);
-    result.isExtern   = false;
-    result.value      = r1;
-    result.symbol     = 0;
-  } else {
-    result.offset     = r0;
-    result.scattered  = false;
-    result.type       = (RelocationInfoType)
-                        bitFieldExtract(r1, isBigEndian, 28, 4);
-    result.length     = bitFieldExtract(r1, isBigEndian, 25, 2);
-    result.pcRel      = bitFieldExtract(r1, isBigEndian, 24, 1);
-    result.isExtern   = bitFieldExtract(r1, isBigEndian, 27, 1);
-    result.value      = 0;
-    result.symbol     = bitFieldExtract(r1, isBigEndian, 0, 24);
-  }
-  return result;
-}
-
-
-inline llvm::MachO::any_relocation_info
-packRelocation(const Relocation &r, bool swap, bool isBigEndian) {
-  uint32_t r0 = 0;
-  uint32_t r1 = 0;
-
-  if (r.scattered) {
-    r1 = r.value;
-    bitFieldSet(r0, true, r.offset,    8, 24);
-    bitFieldSet(r0, true, r.type,      4, 4);
-    bitFieldSet(r0, true, r.length,    2, 2);
-    bitFieldSet(r0, true, r.pcRel,     1, 1);
-    bitFieldSet(r0, true, r.scattered, 0, 1); // R_SCATTERED
-  } else {
-    r0 = r.offset;
-    bitFieldSet(r1, isBigEndian, r.type,     28, 4);
-    bitFieldSet(r1, isBigEndian, r.isExtern, 27, 1);
-    bitFieldSet(r1, isBigEndian, r.length,   25, 2);
-    bitFieldSet(r1, isBigEndian, r.pcRel,    24, 1);
-    bitFieldSet(r1, isBigEndian, r.symbol,   0,  24);
-  }
-
-  llvm::MachO::any_relocation_info result;
-  result.r_word0 = swap ? getSwappedBytes(r0) : r0;
-  result.r_word1 = swap ? getSwappedBytes(r1) : r1;
-  return result;
-}
-
-inline StringRef getString16(const char s[16]) {
-  // The StringRef(const char *) constructor passes the const char * to
-  // strlen(), so we can't use this constructor here, because if there is no
-  // null terminator in s, then strlen() will read past the end of the array.
-  return StringRef(s, strnlen(s, 16));
-}
-
-inline void setString16(StringRef str, char s[16]) {
-  memset(s, 0, 16);
-  memcpy(s, str.begin(), (str.size() > 16) ? 16: str.size());
-}
-
-// Implemented in normalizedToAtoms() and used by normalizedFromAtoms() so
-// that the same table can be used to map mach-o sections to and from
-// DefinedAtom::ContentType.
-void relocatableSectionInfoForContentType(DefinedAtom::ContentType atomType,
-                                          StringRef &segmentName,
-                                          StringRef &sectionName,
-                                          SectionType &sectionType,
-                                          SectionAttr &sectionAttrs,
-                                          bool &relocsToDefinedCanBeImplicit);
-
-} // namespace normalized
-} // namespace mach_o
-} // namespace lld
-
-#endif // LLD_READER_WRITER_MACHO_NORMALIZED_FILE_BINARY_UTILS_H
diff --git a/lld/lib/ReaderWriter/MachO/MachONormalizedFileBinaryWriter.cpp b/lld/lib/ReaderWriter/MachO/MachONormalizedFileBinaryWriter.cpp
deleted file mode 100644
index 17b45b9ca827..000000000000
--- a/lld/lib/ReaderWriter/MachO/MachONormalizedFileBinaryWriter.cpp
+++ /dev/null
@@ -1,1560 +0,0 @@
-//===- lib/ReaderWriter/MachO/MachONormalizedFileBinaryWriter.cpp ---------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-///
-/// \file For mach-o object files, this implementation converts normalized
-/// mach-o in memory to mach-o binary on disk.
-///
-///                 +---------------+
-///                 | binary mach-o |
-///                 +---------------+
-///                        ^
-///                        |
-///                        |
-///                  +------------+
-///                  | normalized |
-///                  +------------+
-
-#include "MachONormalizedFile.h"
-#include "MachONormalizedFileBinaryUtils.h"
-#include "lld/Common/LLVM.h"
-#include "lld/Core/Error.h"
-#include "llvm/ADT/SmallString.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/ADT/ilist.h"
-#include "llvm/ADT/ilist_node.h"
-#include "llvm/BinaryFormat/MachO.h"
-#include "llvm/Support/Casting.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/Errc.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/FileOutputBuffer.h"
-#include "llvm/Support/Format.h"
-#include "llvm/Support/Host.h"
-#include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/raw_ostream.h"
-#include <functional>
-#include <list>
-#include <map>
-#include <system_error>
-
-using namespace llvm::MachO;
-
-namespace lld {
-namespace mach_o {
-namespace normalized {
-
-struct TrieNode; // Forward declaration.
-
-struct TrieEdge : public llvm::ilist_node<TrieEdge> {
-  TrieEdge(StringRef s, TrieNode *node) : _subString(s), _child(node) {}
-
-  StringRef          _subString;
-  struct TrieNode   *_child;
-};
-
-} // namespace normalized
-} // namespace mach_o
-} // namespace lld
-
-
-namespace llvm {
-using lld::mach_o::normalized::TrieEdge;
-template <>
-struct ilist_alloc_traits<TrieEdge> : ilist_noalloc_traits<TrieEdge> {};
-} // namespace llvm
-
-
-namespace lld {
-namespace mach_o {
-namespace normalized {
-
-struct TrieNode {
-  typedef llvm::ilist<TrieEdge> TrieEdgeList;
-
-  TrieNode(StringRef s)
-      : _cummulativeString(s), _address(0), _flags(0), _other(0),
-        _trieOffset(0), _hasExportInfo(false) {}
-  ~TrieNode() = default;
-
-  void addSymbol(const Export &entry, BumpPtrAllocator &allocator,
-                 std::vector<TrieNode *> &allNodes);
-
-  void addOrderedNodes(const Export &entry,
-                       std::vector<TrieNode *> &allNodes);
-  bool updateOffset(uint32_t &offset);
-  void appendToByteBuffer(ByteBuffer &out);
-
-private:
-  StringRef                 _cummulativeString;
-  TrieEdgeList              _children;
-  uint64_t                  _address;
-  uint64_t                  _flags;
-  uint64_t                  _other;
-  StringRef                 _importedName;
-  uint32_t                  _trieOffset;
-  bool                      _hasExportInfo;
-  bool                      _ordered = false;
-};
-
-/// Utility class for writing a mach-o binary file given an in-memory
-/// normalized file.
-class MachOFileLayout {
-public:
-  /// All layout computation is done in the constructor.
-  MachOFileLayout(const NormalizedFile &file, bool alwaysIncludeFunctionStarts);
-
-  /// Returns the final file size as computed in the constructor.
-  size_t      size() const;
-
-  // Returns size of the mach_header and load commands.
-  size_t      headerAndLoadCommandsSize() const;
-
-  /// Writes the normalized file as a binary mach-o file to the specified
-  /// path.  This does not have a stream interface because the generated
-  /// file may need the 'x' bit set.
-  llvm::Error writeBinary(StringRef path);
-
-private:
-  uint32_t    loadCommandsSize(uint32_t &count,
-                               bool alwaysIncludeFunctionStarts);
-  void        buildFileOffsets();
-  void        writeMachHeader();
-  llvm::Error writeLoadCommands();
-  void        writeSectionContent();
-  void        writeRelocations();
-  void        writeSymbolTable();
-  void        writeRebaseInfo();
-  void        writeBindingInfo();
-  void        writeLazyBindingInfo();
-  void        writeExportInfo();
-  void        writeFunctionStartsInfo();
-  void        writeDataInCodeInfo();
-  void        writeLinkEditContent();
-  void        buildLinkEditInfo();
-  void        buildRebaseInfo();
-  void        buildBindInfo();
-  void        buildLazyBindInfo();
-  void        buildExportTrie();
-  void        computeFunctionStartsSize();
-  void        computeDataInCodeSize();
-  void        computeSymbolTableSizes();
-  void        buildSectionRelocations();
-  void        appendSymbols(const std::vector<Symbol> &symbols,
-                                      uint32_t &symOffset, uint32_t &strOffset);
-  uint32_t    indirectSymbolIndex(const Section &sect, uint32_t &index);
-  uint32_t    indirectSymbolElementSize(const Section &sect);
-
-  // For use as template parameter to load command methods.
-  struct MachO64Trait {
-    typedef llvm::MachO::segment_command_64 command;
-    typedef llvm::MachO::section_64         section;
-    enum { LC = llvm::MachO::LC_SEGMENT_64 };
-  };
-
-  // For use as template parameter to load command methods.
-  struct MachO32Trait {
-    typedef llvm::MachO::segment_command   command;
-    typedef llvm::MachO::section           section;
-    enum { LC = llvm::MachO::LC_SEGMENT };
-  };
-
-  template <typename T>
-  llvm::Error writeSingleSegmentLoadCommand(uint8_t *&lc);
-  template <typename T> llvm::Error writeSegmentLoadCommands(uint8_t *&lc);
-
-  uint32_t pointerAlign(uint32_t value);
-  static StringRef dyldPath();
-
-  struct SegExtraInfo {
-    uint32_t                    fileOffset;
-    uint32_t                    fileSize;
-    std::vector<const Section*> sections;
-  };
-  typedef std::map<const Segment*, SegExtraInfo> SegMap;
-  struct SectionExtraInfo {
-    uint32_t                    fileOffset;
-  };
-  typedef std::map<const Section*, SectionExtraInfo> SectionMap;
-
-  const NormalizedFile &_file;
-  std::error_code _ec;
-  uint8_t              *_buffer;
-  const bool            _is64;
-  const bool            _swap;
-  const bool            _bigEndianArch;
-  uint64_t              _seg1addr;
-  uint32_t              _startOfLoadCommands;
-  uint32_t              _countOfLoadCommands;
-  uint32_t              _endOfLoadCommands;
-  uint32_t              _startOfRelocations;
-  uint32_t              _startOfFunctionStarts;
-  uint32_t              _startOfDataInCode;
-  uint32_t              _startOfSymbols;
-  uint32_t              _startOfIndirectSymbols;
-  uint32_t              _startOfSymbolStrings;
-  uint32_t              _endOfSymbolStrings;
-  uint32_t              _symbolTableLocalsStartIndex;
-  uint32_t              _symbolTableGlobalsStartIndex;
-  uint32_t              _symbolTableUndefinesStartIndex;
-  uint32_t              _symbolStringPoolSize;
-  uint32_t              _symbolTableSize;
-  uint32_t              _functionStartsSize;
-  uint32_t              _dataInCodeSize;
-  uint32_t              _indirectSymbolTableCount;
-  // Used in object file creation only
-  uint32_t              _startOfSectionsContent;
-  uint32_t              _endOfSectionsContent;
-  // Used in final linked image only
-  uint32_t              _startOfLinkEdit;
-  uint32_t              _startOfRebaseInfo;
-  uint32_t              _endOfRebaseInfo;
-  uint32_t              _startOfBindingInfo;
-  uint32_t              _endOfBindingInfo;
-  uint32_t              _startOfLazyBindingInfo;
-  uint32_t              _endOfLazyBindingInfo;
-  uint32_t              _startOfExportTrie;
-  uint32_t              _endOfExportTrie;
-  uint32_t              _endOfLinkEdit;
-  uint64_t              _addressOfLinkEdit;
-  SegMap                _segInfo;
-  SectionMap            _sectInfo;
-  ByteBuffer            _rebaseInfo;
-  ByteBuffer            _bindingInfo;
-  ByteBuffer            _lazyBindingInfo;
-  ByteBuffer            _weakBindingInfo;
-  ByteBuffer            _exportTrie;
-};
-
-size_t headerAndLoadCommandsSize(const NormalizedFile &file,
-                                 bool includeFunctionStarts) {
-  MachOFileLayout layout(file, includeFunctionStarts);
-  return layout.headerAndLoadCommandsSize();
-}
-
-StringRef MachOFileLayout::dyldPath() {
-  return "/usr/lib/dyld";
-}
-
-uint32_t MachOFileLayout::pointerAlign(uint32_t value) {
-  return llvm::alignTo(value, _is64 ? 8 : 4);
-}
-
-
-size_t MachOFileLayout::headerAndLoadCommandsSize() const {
-  return _endOfLoadCommands;
-}
-
-MachOFileLayout::MachOFileLayout(const NormalizedFile &file,
-                                 bool alwaysIncludeFunctionStarts)
-    : _file(file),
-      _is64(MachOLinkingContext::is64Bit(file.arch)),
-      _swap(!MachOLinkingContext::isHostEndian(file.arch)),
-      _bigEndianArch(MachOLinkingContext::isBigEndian(file.arch)),
-      _seg1addr(INT64_MAX) {
-  _startOfLoadCommands = _is64 ? sizeof(mach_header_64) : sizeof(mach_header);
-  const size_t segCommandBaseSize =
-          (_is64 ? sizeof(segment_command_64) : sizeof(segment_command));
-  const size_t sectsSize = (_is64 ? sizeof(section_64) : sizeof(section));
-  if (file.fileType == llvm::MachO::MH_OBJECT) {
-    // object files have just one segment load command containing all sections
-    _endOfLoadCommands = _startOfLoadCommands
-                               + segCommandBaseSize
-                               + file.sections.size() * sectsSize
-                               + sizeof(symtab_command);
-    _countOfLoadCommands = 2;
-    if (file.hasMinVersionLoadCommand) {
-      _endOfLoadCommands += sizeof(version_min_command);
-      _countOfLoadCommands++;
-    }
-    if (!_file.functionStarts.empty() || alwaysIncludeFunctionStarts) {
-      _endOfLoadCommands += sizeof(linkedit_data_command);
-      _countOfLoadCommands++;
-    }
-    if (_file.generateDataInCodeLoadCommand) {
-      _endOfLoadCommands += sizeof(linkedit_data_command);
-      _countOfLoadCommands++;
-    }
-    // Assign file offsets to each section.
-    _startOfSectionsContent = _endOfLoadCommands;
-    unsigned relocCount = 0;
-    uint64_t offset = _startOfSectionsContent;
-    for (const Section &sect : file.sections) {
-      if (isZeroFillSection(sect.type))
-        _sectInfo[&sect].fileOffset = 0;
-      else {
-        offset = llvm::alignTo(offset, sect.alignment);
-        _sectInfo[&sect].fileOffset = offset;
-        offset += sect.content.size();
-      }
-      relocCount += sect.relocations.size();
-    }
-    _endOfSectionsContent = offset;
-
-    computeSymbolTableSizes();
-    computeFunctionStartsSize();
-    computeDataInCodeSize();
-
-    // Align start of relocations.
-    _startOfRelocations = pointerAlign(_endOfSectionsContent);
-    _startOfFunctionStarts = _startOfRelocations + relocCount * 8;
-    _startOfDataInCode = _startOfFunctionStarts + _functionStartsSize;
-    _startOfSymbols = _startOfDataInCode + _dataInCodeSize;
-    // Add Indirect symbol table.
-    _startOfIndirectSymbols = _startOfSymbols + _symbolTableSize;
-    // Align start of symbol table and symbol strings.
-    _startOfSymbolStrings = _startOfIndirectSymbols
-                  + pointerAlign(_indirectSymbolTableCount * sizeof(uint32_t));
-    _endOfSymbolStrings = _startOfSymbolStrings
-                          + pointerAlign(_symbolStringPoolSize);
-    _endOfLinkEdit = _endOfSymbolStrings;
-    DEBUG_WITH_TYPE("MachOFileLayout",
-                  llvm::dbgs() << "MachOFileLayout()\n"
-      << "  startOfLoadCommands=" << _startOfLoadCommands << "\n"
-      << "  countOfLoadCommands=" << _countOfLoadCommands << "\n"
-      << "  endOfLoadCommands=" << _endOfLoadCommands << "\n"
-      << "  startOfRelocations=" << _startOfRelocations << "\n"
-      << "  startOfSymbols=" << _startOfSymbols << "\n"
-      << "  startOfSymbolStrings=" << _startOfSymbolStrings << "\n"
-      << "  endOfSymbolStrings=" << _endOfSymbolStrings << "\n"
-      << "  startOfSectionsContent=" << _startOfSectionsContent << "\n"
-      << "  endOfSectionsContent=" << _endOfSectionsContent << "\n");
-  } else {
-    // Final linked images have one load command per segment.
-    _endOfLoadCommands = _startOfLoadCommands
-                          + loadCommandsSize(_countOfLoadCommands,
-                                             alwaysIncludeFunctionStarts);
-
-    // Assign section file offsets.
-    buildFileOffsets();
-    buildLinkEditInfo();
-
-    // LINKEDIT of final linked images has in order:
-    // rebase info, binding info, lazy binding info, weak binding info,
-    // data-in-code, symbol table, indirect symbol table, symbol table strings.
-    _startOfRebaseInfo = _startOfLinkEdit;
-    _endOfRebaseInfo = _startOfRebaseInfo + _rebaseInfo.size();
-    _startOfBindingInfo = _endOfRebaseInfo;
-    _endOfBindingInfo = _startOfBindingInfo + _bindingInfo.size();
-    _startOfLazyBindingInfo = _endOfBindingInfo;
-    _endOfLazyBindingInfo = _startOfLazyBindingInfo + _lazyBindingInfo.size();
-    _startOfExportTrie = _endOfLazyBindingInfo;
-    _endOfExportTrie = _startOfExportTrie + _exportTrie.size();
-    _startOfFunctionStarts = _endOfExportTrie;
-    _startOfDataInCode = _startOfFunctionStarts + _functionStartsSize;
-    _startOfSymbols = _startOfDataInCode + _dataInCodeSize;
-    _startOfIndirectSymbols = _startOfSymbols + _symbolTableSize;
-    _startOfSymbolStrings = _startOfIndirectSymbols
-                  + pointerAlign(_indirectSymbolTableCount * sizeof(uint32_t));
-    _endOfSymbolStrings = _startOfSymbolStrings
-                          + pointerAlign(_symbolStringPoolSize);
-    _endOfLinkEdit = _endOfSymbolStrings;
-    DEBUG_WITH_TYPE("MachOFileLayout",
-                  llvm::dbgs() << "MachOFileLayout()\n"
-      << "  startOfLoadCommands=" << _startOfLoadCommands << "\n"
-      << "  countOfLoadCommands=" << _countOfLoadCommands << "\n"
-      << "  endOfLoadCommands=" << _endOfLoadCommands << "\n"
-      << "  startOfLinkEdit=" << _startOfLinkEdit << "\n"
-      << "  startOfRebaseInfo=" << _startOfRebaseInfo << "\n"
-      << "  endOfRebaseInfo=" << _endOfRebaseInfo << "\n"
-      << "  startOfBindingInfo=" << _startOfBindingInfo << "\n"
-      << "  endOfBindingInfo=" << _endOfBindingInfo << "\n"
-      << "  startOfLazyBindingInfo=" << _startOfLazyBindingInfo << "\n"
-      << "  endOfLazyBindingInfo=" << _endOfLazyBindingInfo << "\n"
-      << "  startOfExportTrie=" << _startOfExportTrie << "\n"
-      << "  endOfExportTrie=" << _endOfExportTrie << "\n"
-      << "  startOfFunctionStarts=" << _startOfFunctionStarts << "\n"
-      << "  startOfDataInCode=" << _startOfDataInCode << "\n"
-      << "  startOfSymbols=" << _startOfSymbols << "\n"
-      << "  startOfSymbolStrings=" << _startOfSymbolStrings << "\n"
-      << "  endOfSymbolStrings=" << _endOfSymbolStrings << "\n"
-      << "  addressOfLinkEdit=" << _addressOfLinkEdit << "\n");
-  }
-}
-
-uint32_t MachOFileLayout::loadCommandsSize(uint32_t &count,
-                                           bool alwaysIncludeFunctionStarts) {
-  uint32_t size = 0;
-  count = 0;
-
-  const size_t segCommandSize =
-          (_is64 ? sizeof(segment_command_64) : sizeof(segment_command));
-  const size_t sectionSize = (_is64 ? sizeof(section_64) : sizeof(section));
-
-  // Add LC_SEGMENT for each segment.
-  size += _file.segments.size() * segCommandSize;
-  count += _file.segments.size();
-  // Add section record for each section.
-  size += _file.sections.size() * sectionSize;
-
-  // If creating a dylib, add LC_ID_DYLIB.
-  if (_file.fileType == llvm::MachO::MH_DYLIB) {
-    size += sizeof(dylib_command) + pointerAlign(_file.installName.size() + 1);
-    ++count;
-  }
-
-  // Add LC_DYLD_INFO
-  size += sizeof(dyld_info_command);
-  ++count;
-
-  // Add LC_SYMTAB
-  size += sizeof(symtab_command);
-  ++count;
-
-  // Add LC_DYSYMTAB
-  if (_file.fileType != llvm::MachO::MH_PRELOAD) {
-    size += sizeof(dysymtab_command);
-    ++count;
-  }
-
-  // If main executable add LC_LOAD_DYLINKER
-  if (_file.fileType == llvm::MachO::MH_EXECUTE) {
-    size += pointerAlign(sizeof(dylinker_command) + dyldPath().size()+1);
-    ++count;
-  }
-
-  // Add LC_VERSION_MIN_MACOSX, LC_VERSION_MIN_IPHONEOS, LC_VERSION_MIN_WATCHOS,
-  // LC_VERSION_MIN_TVOS
-  if (_file.hasMinVersionLoadCommand) {
-    size += sizeof(version_min_command);
-    ++count;
-  }
-
-  // Add LC_SOURCE_VERSION
-  size += sizeof(source_version_command);
-  ++count;
-
-  // If main executable add LC_MAIN
-  if (_file.fileType == llvm::MachO::MH_EXECUTE) {
-    size += sizeof(entry_point_command);
-    ++count;
-  }
-
-  // Add LC_LOAD_DYLIB for each dependent dylib.
-  for (const DependentDylib &dep : _file.dependentDylibs) {
-    size += sizeof(dylib_command) + pointerAlign(dep.path.size()+1);
-    ++count;
-  }
-
-  // Add LC_RPATH
-  for (const StringRef &path : _file.rpaths) {
-    size += pointerAlign(sizeof(rpath_command) + path.size() + 1);
-    ++count;
-  }
-
-  // Add LC_FUNCTION_STARTS if needed
-  if (!_file.functionStarts.empty() || alwaysIncludeFunctionStarts) {
-    size += sizeof(linkedit_data_command);
-    ++count;
-  }
-
-  // Add LC_DATA_IN_CODE if requested.  Note, we do encode zero length entries.
-  // FIXME: Zero length entries is only to match ld64.  Should we change this?
-  if (_file.generateDataInCodeLoadCommand) {
-    size += sizeof(linkedit_data_command);
-    ++count;
-  }
-
-  return size;
-}
-
-static bool overlaps(const Segment &s1, const Segment &s2) {
-  if (s2.address >= s1.address+s1.size)
-    return false;
-  if (s1.address >= s2.address+s2.size)
-    return false;
-  return true;
-}
-
-static bool overlaps(const Section &s1, const Section &s2) {
-  if (s2.address >= s1.address+s1.content.size())
-    return false;
-  if (s1.address >= s2.address+s2.content.size())
-    return false;
-  return true;
-}
-
-void MachOFileLayout::buildFileOffsets() {
-  // Verify no segments overlap
-  for (const Segment &sg1 : _file.segments) {
-    for (const Segment &sg2 : _file.segments) {
-      if (&sg1 == &sg2)
-        continue;
-      if (overlaps(sg1,sg2)) {
-        _ec = make_error_code(llvm::errc::executable_format_error);
-        return;
-      }
-    }
-  }
-
-  // Verify no sections overlap
-  for (const Section &s1 : _file.sections) {
-    for (const Section &s2 : _file.sections) {
-      if (&s1 == &s2)
-        continue;
-      if (overlaps(s1,s2)) {
-        _ec = make_error_code(llvm::errc::executable_format_error);
-        return;
-      }
-    }
-  }
-
-  // Build side table of extra info about segments and sections.
-  SegExtraInfo t;
-  t.fileOffset = 0;
-  for (const Segment &sg : _file.segments) {
-    _segInfo[&sg] = t;
-  }
-  SectionExtraInfo t2;
-  t2.fileOffset = 0;
-  // Assign sections to segments.
-  for (const Section &s : _file.sections) {
-    _sectInfo[&s] = t2;
-    bool foundSegment = false;
-    for (const Segment &sg : _file.segments) {
-      if (sg.name.equals(s.segmentName)) {
-        if ((s.address >= sg.address)
-                        && (s.address+s.content.size() <= sg.address+sg.size)) {
-          _segInfo[&sg].sections.push_back(&s);
-          foundSegment = true;
-          break;
-        }
-      }
-    }
-    if (!foundSegment) {
-      _ec = make_error_code(llvm::errc::executable_format_error);
-      return;
-    }
-  }
-
-  // Assign file offsets.
-  uint32_t fileOffset = 0;
-  DEBUG_WITH_TYPE("MachOFileLayout",
-                  llvm::dbgs() << "buildFileOffsets()\n");
-  for (const Segment &sg : _file.segments) {
-    _segInfo[&sg].fileOffset = fileOffset;
-    if ((_seg1addr == INT64_MAX) && sg.init_access)
-      _seg1addr = sg.address;
-    DEBUG_WITH_TYPE("MachOFileLayout",
-                  llvm::dbgs() << "  segment=" << sg.name
-                  << ", fileOffset=" << _segInfo[&sg].fileOffset << "\n");
-
-    uint32_t segFileSize = 0;
-    // A segment that is not zero-fill must use a least one page of disk space.
-    if (sg.init_access)
-      segFileSize = _file.pageSize;
-    for (const Section *s : _segInfo[&sg].sections) {
-      uint32_t sectOffset = s->address - sg.address;
-      uint32_t sectFileSize =
-        isZeroFillSection(s->type) ? 0 : s->content.size();
-      segFileSize = std::max(segFileSize, sectOffset + sectFileSize);
-
-      _sectInfo[s].fileOffset = _segInfo[&sg].fileOffset + sectOffset;
-      DEBUG_WITH_TYPE("MachOFileLayout",
-                  llvm::dbgs() << "    section=" << s->sectionName
-                  << ", fileOffset=" << fileOffset << "\n");
-    }
-
-    // round up all segments to page aligned, except __LINKEDIT
-    if (!sg.name.equals("__LINKEDIT")) {
-      _segInfo[&sg].fileSize = llvm::alignTo(segFileSize, _file.pageSize);
-      fileOffset = llvm::alignTo(fileOffset + segFileSize, _file.pageSize);
-    }
-    _addressOfLinkEdit = sg.address + sg.size;
-  }
-  _startOfLinkEdit = fileOffset;
-}
-
-size_t MachOFileLayout::size() const {
-  return _endOfSymbolStrings;
-}
-
-void MachOFileLayout::writeMachHeader() {
-  auto cpusubtype = MachOLinkingContext::cpuSubtypeFromArch(_file.arch);
-  // dynamic x86 executables on newer OS version should also set the
-  // CPU_SUBTYPE_LIB64 mask in the CPU subtype.
-  // FIXME: Check that this is a dynamic executable, not a static one.
-  if (_file.fileType == llvm::MachO::MH_EXECUTE &&
-      cpusubtype == CPU_SUBTYPE_X86_64_ALL &&
-      _file.os == MachOLinkingContext::OS::macOSX) {
-    uint32_t version;
-    bool failed = MachOLinkingContext::parsePackedVersion("10.5", version);
-    if (!failed && _file.minOSverson >= version)
-      cpusubtype |= CPU_SUBTYPE_LIB64;
-  }
-
-  mach_header *mh = reinterpret_cast<mach_header*>(_buffer);
-  mh->magic = _is64 ? llvm::MachO::MH_MAGIC_64 : llvm::MachO::MH_MAGIC;
-  mh->cputype =  MachOLinkingContext::cpuTypeFromArch(_file.arch);
-  mh->cpusubtype = cpusubtype;
-  mh->filetype = _file.fileType;
-  mh->ncmds = _countOfLoadCommands;
-  mh->sizeofcmds = _endOfLoadCommands - _startOfLoadCommands;
-  mh->flags = _file.flags;
-  if (_swap)
-    swapStruct(*mh);
-}
-
-uint32_t MachOFileLayout::indirectSymbolIndex(const Section &sect,
-                                                   uint32_t &index) {
-  if (sect.indirectSymbols.empty())
-    return 0;
-  uint32_t result = index;
-  index += sect.indirectSymbols.size();
-  return result;
-}
-
-uint32_t MachOFileLayout::indirectSymbolElementSize(const Section &sect) {
-  if (sect.indirectSymbols.empty())
-    return 0;
-  if (sect.type != S_SYMBOL_STUBS)
-    return 0;
-  return sect.content.size() / sect.indirectSymbols.size();
-}
-
-template <typename T>
-llvm::Error MachOFileLayout::writeSingleSegmentLoadCommand(uint8_t *&lc) {
-  typename T::command* seg = reinterpret_cast<typename T::command*>(lc);
-  seg->cmd = T::LC;
-  seg->cmdsize = sizeof(typename T::command)
-                          + _file.sections.size() * sizeof(typename T::section);
-  uint8_t *next = lc + seg->cmdsize;
-  memset(seg->segname, 0, 16);
-  seg->flags = 0;
-  seg->vmaddr = 0;
-  seg->fileoff = _endOfLoadCommands;
-  seg->maxprot = VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE;
-  seg->initprot = VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE;
-  seg->nsects = _file.sections.size();
-  if (seg->nsects) {
-    seg->vmsize = _file.sections.back().address
-                + _file.sections.back().content.size();
-    seg->filesize = _sectInfo[&_file.sections.back()].fileOffset +
-                    _file.sections.back().content.size() -
-                    _sectInfo[&_file.sections.front()].fileOffset;
-  }
-  if (_swap)
-    swapStruct(*seg);
-  typename T::section *sout = reinterpret_cast<typename T::section*>
-                                              (lc+sizeof(typename T::command));
-  uint32_t relOffset = _startOfRelocations;
-  uint32_t indirectSymRunningIndex = 0;
-  for (const Section &sin : _file.sections) {
-    setString16(sin.sectionName, sout->sectname);
-    setString16(sin.segmentName, sout->segname);
-    sout->addr = sin.address;
-    sout->size = sin.content.size();
-    sout->offset = _sectInfo[&sin].fileOffset;
-    sout->align = llvm::Log2_32(sin.alignment);
-    sout->reloff = sin.relocations.empty() ? 0 : relOffset;
-    sout->nreloc = sin.relocations.size();
-    sout->flags = sin.type | sin.attributes;
-    sout->reserved1 = indirectSymbolIndex(sin, indirectSymRunningIndex);
-    sout->reserved2 = indirectSymbolElementSize(sin);
-    relOffset += sin.relocations.size() * sizeof(any_relocation_info);
-    if (_swap)
-      swapStruct(*sout);
-    ++sout;
-  }
-  lc = next;
-  return llvm::Error::success();
-}
-
-template <typename T>
-llvm::Error MachOFileLayout::writeSegmentLoadCommands(uint8_t *&lc) {
-  uint32_t indirectSymRunningIndex = 0;
-  for (const Segment &seg : _file.segments) {
-    // Link edit has no sections and a custom range of address, so handle it
-    // specially.
-    SegExtraInfo &segInfo = _segInfo[&seg];
-    if (seg.name.equals("__LINKEDIT")) {
-      size_t linkeditSize = _endOfLinkEdit - _startOfLinkEdit;
-      typename T::command* cmd = reinterpret_cast<typename T::command*>(lc);
-      cmd->cmd = T::LC;
-      cmd->cmdsize = sizeof(typename T::command);
-      uint8_t *next = lc + cmd->cmdsize;
-      setString16("__LINKEDIT", cmd->segname);
-      cmd->vmaddr   = _addressOfLinkEdit;
-      cmd->vmsize   = llvm::alignTo(linkeditSize, _file.pageSize);
-      cmd->fileoff  = _startOfLinkEdit;
-      cmd->filesize = linkeditSize;
-      cmd->initprot = seg.init_access;
-      cmd->maxprot  = seg.max_access;
-      cmd->nsects   = 0;
-      cmd->flags    = 0;
-      if (_swap)
-        swapStruct(*cmd);
-      lc = next;
-      continue;
-    }
-    // Write segment command with trailing sections.
-    typename T::command* cmd = reinterpret_cast<typename T::command*>(lc);
-    cmd->cmd = T::LC;
-    cmd->cmdsize = sizeof(typename T::command)
-                        + segInfo.sections.size() * sizeof(typename T::section);
-    uint8_t *next = lc + cmd->cmdsize;
-    setString16(seg.name, cmd->segname);
-    cmd->vmaddr   = seg.address;
-    cmd->vmsize   = seg.size;
-    cmd->fileoff  = segInfo.fileOffset;
-    cmd->filesize = segInfo.fileSize;
-    cmd->initprot = seg.init_access;
-    cmd->maxprot  = seg.max_access;
-    cmd->nsects   = segInfo.sections.size();
-    cmd->flags    = 0;
-    if (_swap)
-      swapStruct(*cmd);
-    typename T::section *sect = reinterpret_cast<typename T::section*>
-                                               (lc+sizeof(typename T::command));
-    for (const Section *section : segInfo.sections) {
-      setString16(section->sectionName, sect->sectname);
-      setString16(section->segmentName, sect->segname);
-      sect->addr      = section->address;
-      sect->size      = section->content.size();
-      if (isZeroFillSection(section->type))
-        sect->offset  = 0;
-      else
-        sect->offset  = section->address - seg.address + segInfo.fileOffset;
-      sect->align     = llvm::Log2_32(section->alignment);
-      sect->reloff    = 0;
-      sect->nreloc    = 0;
-      sect->flags     = section->type | section->attributes;
-      sect->reserved1 = indirectSymbolIndex(*section, indirectSymRunningIndex);
-      sect->reserved2 = indirectSymbolElementSize(*section);
-      if (_swap)
-        swapStruct(*sect);
-      ++sect;
-    }
-    lc = reinterpret_cast<uint8_t*>(next);
-  }
-  return llvm::Error::success();
-}
-
-static void writeVersionMinLoadCommand(const NormalizedFile &_file,
-                                       bool _swap,
-                                       uint8_t *&lc) {
-  if (!_file.hasMinVersionLoadCommand)
-    return;
-  version_min_command *vm = reinterpret_cast<version_min_command*>(lc);
-  switch (_file.os) {
-    case MachOLinkingContext::OS::unknown:
-      vm->cmd     = _file.minOSVersionKind;
-      vm->cmdsize = sizeof(version_min_command);
-      vm->version = _file.minOSverson;
-      vm->sdk     = 0;
-      break;
-    case MachOLinkingContext::OS::macOSX:
-      vm->cmd     = LC_VERSION_MIN_MACOSX;
-      vm->cmdsize = sizeof(version_min_command);
-      vm->version = _file.minOSverson;
-      vm->sdk     = _file.sdkVersion;
-      break;
-    case MachOLinkingContext::OS::iOS:
-    case MachOLinkingContext::OS::iOS_simulator:
-      vm->cmd     = LC_VERSION_MIN_IPHONEOS;
-      vm->cmdsize = sizeof(version_min_command);
-      vm->version = _file.minOSverson;
-      vm->sdk     = _file.sdkVersion;
-      break;
-  }
-  if (_swap)
-    swapStruct(*vm);
-  lc += sizeof(version_min_command);
-}
-
-llvm::Error MachOFileLayout::writeLoadCommands() {
-  uint8_t *lc = &_buffer[_startOfLoadCommands];
-  if (_file.fileType == llvm::MachO::MH_OBJECT) {
-    // Object files have one unnamed segment which holds all sections.
-    if (_is64) {
-     if (auto ec = writeSingleSegmentLoadCommand<MachO64Trait>(lc))
-       return ec;
-    } else {
-      if (auto ec = writeSingleSegmentLoadCommand<MachO32Trait>(lc))
-        return ec;
-    }
-    // Add LC_SYMTAB with symbol table info
-    symtab_command* st = reinterpret_cast<symtab_command*>(lc);
-    st->cmd     = LC_SYMTAB;
-    st->cmdsize = sizeof(symtab_command);
-    st->symoff  = _startOfSymbols;
-    st->nsyms   = _file.stabsSymbols.size() + _file.localSymbols.size() +
-                  _file.globalSymbols.size() + _file.undefinedSymbols.size();
-    st->stroff  = _startOfSymbolStrings;
-    st->strsize = _endOfSymbolStrings - _startOfSymbolStrings;
-    if (_swap)
-      swapStruct(*st);
-    lc += sizeof(symtab_command);
-
-    // Add LC_VERSION_MIN_MACOSX, LC_VERSION_MIN_IPHONEOS,
-    // LC_VERSION_MIN_WATCHOS, LC_VERSION_MIN_TVOS
-    writeVersionMinLoadCommand(_file, _swap, lc);
-
-    // Add LC_FUNCTION_STARTS if needed.
-    if (_functionStartsSize != 0) {
-      linkedit_data_command* dl = reinterpret_cast<linkedit_data_command*>(lc);
-      dl->cmd      = LC_FUNCTION_STARTS;
-      dl->cmdsize  = sizeof(linkedit_data_command);
-      dl->dataoff  = _startOfFunctionStarts;
-      dl->datasize = _functionStartsSize;
-      if (_swap)
-        swapStruct(*dl);
-      lc += sizeof(linkedit_data_command);
-    }
-
-    // Add LC_DATA_IN_CODE if requested.
-    if (_file.generateDataInCodeLoadCommand) {
-      linkedit_data_command* dl = reinterpret_cast<linkedit_data_command*>(lc);
-      dl->cmd      = LC_DATA_IN_CODE;
-      dl->cmdsize  = sizeof(linkedit_data_command);
-      dl->dataoff  = _startOfDataInCode;
-      dl->datasize = _dataInCodeSize;
-      if (_swap)
-        swapStruct(*dl);
-      lc += sizeof(linkedit_data_command);
-    }
-  } else {
-    // Final linked images have sections under segments.
-    if (_is64) {
-      if (auto ec = writeSegmentLoadCommands<MachO64Trait>(lc))
-        return ec;
-    } else {
-      if (auto ec = writeSegmentLoadCommands<MachO32Trait>(lc))
-        return ec;
-    }
-
-    // Add LC_ID_DYLIB command for dynamic libraries.
-    if (_file.fileType == llvm::MachO::MH_DYLIB) {
-      dylib_command *dc = reinterpret_cast<dylib_command*>(lc);
-      StringRef path = _file.installName;
-      uint32_t size = sizeof(dylib_command) + pointerAlign(path.size() + 1);
-      dc->cmd                         = LC_ID_DYLIB;
-      dc->cmdsize                     = size;
-      dc->dylib.name                  = sizeof(dylib_command); // offset
-      // needs to be some constant value different than the one in LC_LOAD_DYLIB
-      dc->dylib.timestamp             = 1;
-      dc->dylib.current_version       = _file.currentVersion;
-      dc->dylib.compatibility_version = _file.compatVersion;
-      if (_swap)
-        swapStruct(*dc);
-      memcpy(lc + sizeof(dylib_command), path.begin(), path.size());
-      lc[sizeof(dylib_command) + path.size()] = '\0';
-      lc += size;
-    }
-
-    // Add LC_DYLD_INFO_ONLY.
-    dyld_info_command* di = reinterpret_cast<dyld_info_command*>(lc);
-    di->cmd            = LC_DYLD_INFO_ONLY;
-    di->cmdsize        = sizeof(dyld_info_command);
-    di->rebase_off     = _rebaseInfo.size() ? _startOfRebaseInfo : 0;
-    di->rebase_size    = _rebaseInfo.size();
-    di->bind_off       = _bindingInfo.size() ? _startOfBindingInfo : 0;
-    di->bind_size      = _bindingInfo.size();
-    di->weak_bind_off  = 0;
-    di->weak_bind_size = 0;
-    di->lazy_bind_off  = _lazyBindingInfo.size() ? _startOfLazyBindingInfo : 0;
-    di->lazy_bind_size = _lazyBindingInfo.size();
-    di->export_off     = _exportTrie.size() ? _startOfExportTrie : 0;
-    di->export_size    = _exportTrie.size();
-    if (_swap)
-      swapStruct(*di);
-    lc += sizeof(dyld_info_command);
-
-    // Add LC_SYMTAB with symbol table info.
-    symtab_command* st = reinterpret_cast<symtab_command*>(lc);
-    st->cmd     = LC_SYMTAB;
-    st->cmdsize = sizeof(symtab_command);
-    st->symoff  = _startOfSymbols;
-    st->nsyms   = _file.stabsSymbols.size() + _file.localSymbols.size() +
-                  _file.globalSymbols.size() + _file.undefinedSymbols.size();
-    st->stroff  = _startOfSymbolStrings;
-    st->strsize = _endOfSymbolStrings - _startOfSymbolStrings;
-    if (_swap)
-      swapStruct(*st);
-    lc += sizeof(symtab_command);
-
-    // Add LC_DYSYMTAB
-    if (_file.fileType != llvm::MachO::MH_PRELOAD) {
-      dysymtab_command* dst = reinterpret_cast<dysymtab_command*>(lc);
-      dst->cmd            = LC_DYSYMTAB;
-      dst->cmdsize        = sizeof(dysymtab_command);
-      dst->ilocalsym      = _symbolTableLocalsStartIndex;
-      dst->nlocalsym      = _file.stabsSymbols.size() +
-                            _file.localSymbols.size();
-      dst->iextdefsym     = _symbolTableGlobalsStartIndex;
-      dst->nextdefsym     = _file.globalSymbols.size();
-      dst->iundefsym      = _symbolTableUndefinesStartIndex;
-      dst->nundefsym      = _file.undefinedSymbols.size();
-      dst->tocoff         = 0;
-      dst->ntoc           = 0;
-      dst->modtaboff      = 0;
-      dst->nmodtab        = 0;
-      dst->extrefsymoff   = 0;
-      dst->nextrefsyms    = 0;
-      dst->indirectsymoff = _startOfIndirectSymbols;
-      dst->nindirectsyms  = _indirectSymbolTableCount;
-      dst->extreloff      = 0;
-      dst->nextrel        = 0;
-      dst->locreloff      = 0;
-      dst->nlocrel        = 0;
-      if (_swap)
-        swapStruct(*dst);
-      lc += sizeof(dysymtab_command);
-    }
-
-    // If main executable, add LC_LOAD_DYLINKER
-    if (_file.fileType == llvm::MachO::MH_EXECUTE) {
-      // Build LC_LOAD_DYLINKER load command.
-      uint32_t size=pointerAlign(sizeof(dylinker_command)+dyldPath().size()+1);
-      dylinker_command* dl = reinterpret_cast<dylinker_command*>(lc);
-      dl->cmd              = LC_LOAD_DYLINKER;
-      dl->cmdsize          = size;
-      dl->name             = sizeof(dylinker_command); // offset
-      if (_swap)
-        swapStruct(*dl);
-      memcpy(lc+sizeof(dylinker_command), dyldPath().data(), dyldPath().size());
-      lc[sizeof(dylinker_command)+dyldPath().size()] = '\0';
-      lc += size;
-    }
-
-    // Add LC_VERSION_MIN_MACOSX, LC_VERSION_MIN_IPHONEOS, LC_VERSION_MIN_WATCHOS,
-    // LC_VERSION_MIN_TVOS
-    writeVersionMinLoadCommand(_file, _swap, lc);
-
-    // Add LC_SOURCE_VERSION
-    {
-      // Note, using a temporary here to appease UB as we may not be aligned
-      // enough for a struct containing a uint64_t when emitting a 32-bit binary
-      source_version_command sv;
-      sv.cmd       = LC_SOURCE_VERSION;
-      sv.cmdsize   = sizeof(source_version_command);
-      sv.version   = _file.sourceVersion;
-      if (_swap)
-        swapStruct(sv);
-      memcpy(lc, &sv, sizeof(source_version_command));
-      lc += sizeof(source_version_command);
-    }
-
-    // If main executable, add LC_MAIN.
-    if (_file.fileType == llvm::MachO::MH_EXECUTE) {
-      // Build LC_MAIN load command.
-      // Note, using a temporary here to appease UB as we may not be aligned
-      // enough for a struct containing a uint64_t when emitting a 32-bit binary
-      entry_point_command ep;
-      ep.cmd       = LC_MAIN;
-      ep.cmdsize   = sizeof(entry_point_command);
-      ep.entryoff  = _file.entryAddress - _seg1addr;
-      ep.stacksize = _file.stackSize;
-      if (_swap)
-        swapStruct(ep);
-      memcpy(lc, &ep, sizeof(entry_point_command));
-      lc += sizeof(entry_point_command);
-    }
-
-    // Add LC_LOAD_DYLIB commands
-    for (const DependentDylib &dep : _file.dependentDylibs) {
-      dylib_command* dc = reinterpret_cast<dylib_command*>(lc);
-      uint32_t size = sizeof(dylib_command) + pointerAlign(dep.path.size()+1);
-      dc->cmd                         = dep.kind;
-      dc->cmdsize                     = size;
-      dc->dylib.name                  = sizeof(dylib_command); // offset
-      // needs to be some constant value different than the one in LC_ID_DYLIB
-      dc->dylib.timestamp             = 2;
-      dc->dylib.current_version       = dep.currentVersion;
-      dc->dylib.compatibility_version = dep.compatVersion;
-      if (_swap)
-        swapStruct(*dc);
-      memcpy(lc+sizeof(dylib_command), dep.path.begin(), dep.path.size());
-      lc[sizeof(dylib_command)+dep.path.size()] = '\0';
-      lc += size;
-    }
-
-    // Add LC_RPATH
-    for (const StringRef &path : _file.rpaths) {
-      rpath_command *rpc = reinterpret_cast<rpath_command *>(lc);
-      uint32_t size = pointerAlign(sizeof(rpath_command) + path.size() + 1);
-      rpc->cmd                         = LC_RPATH;
-      rpc->cmdsize                     = size;
-      rpc->path                        = sizeof(rpath_command); // offset
-      if (_swap)
-        swapStruct(*rpc);
-      memcpy(lc+sizeof(rpath_command), path.begin(), path.size());
-      lc[sizeof(rpath_command)+path.size()] = '\0';
-      lc += size;
-    }
-
-    // Add LC_FUNCTION_STARTS if needed.
-    if (_functionStartsSize != 0) {
-      linkedit_data_command* dl = reinterpret_cast<linkedit_data_command*>(lc);
-      dl->cmd      = LC_FUNCTION_STARTS;
-      dl->cmdsize  = sizeof(linkedit_data_command);
-      dl->dataoff  = _startOfFunctionStarts;
-      dl->datasize = _functionStartsSize;
-      if (_swap)
-        swapStruct(*dl);
-      lc += sizeof(linkedit_data_command);
-    }
-
-    // Add LC_DATA_IN_CODE if requested.
-    if (_file.generateDataInCodeLoadCommand) {
-      linkedit_data_command* dl = reinterpret_cast<linkedit_data_command*>(lc);
-      dl->cmd      = LC_DATA_IN_CODE;
-      dl->cmdsize  = sizeof(linkedit_data_command);
-      dl->dataoff  = _startOfDataInCode;
-      dl->datasize = _dataInCodeSize;
-      if (_swap)
-        swapStruct(*dl);
-      lc += sizeof(linkedit_data_command);
-    }
-  }
-  assert(lc == &_buffer[_endOfLoadCommands]);
-  return llvm::Error::success();
-}
-
-void MachOFileLayout::writeSectionContent() {
-  for (const Section &s : _file.sections) {
-    // Copy all section content to output buffer.
-    if (isZeroFillSection(s.type))
-      continue;
-    if (s.content.empty())
-      continue;
-    uint32_t offset = _sectInfo[&s].fileOffset;
-    assert(offset >= _endOfLoadCommands);
-    uint8_t *p = &_buffer[offset];
-    memcpy(p, &s.content[0], s.content.size());
-    p += s.content.size();
-  }
-}
-
-void MachOFileLayout::writeRelocations() {
-  uint32_t relOffset = _startOfRelocations;
-  for (Section sect : _file.sections) {
-    for (Relocation r : sect.relocations) {
-      any_relocation_info* rb = reinterpret_cast<any_relocation_info*>(
-                                                           &_buffer[relOffset]);
-      *rb = packRelocation(r, _swap, _bigEndianArch);
-      relOffset += sizeof(any_relocation_info);
-    }
-  }
-}
-
-void MachOFileLayout::appendSymbols(const std::vector<Symbol> &symbols,
-                                   uint32_t &symOffset, uint32_t &strOffset) {
-  for (const Symbol &sym : symbols) {
-    if (_is64) {
-      nlist_64* nb = reinterpret_cast<nlist_64*>(&_buffer[symOffset]);
-      nb->n_strx = strOffset - _startOfSymbolStrings;
-      nb->n_type = sym.type | sym.scope;
-      nb->n_sect = sym.sect;
-      nb->n_desc = sym.desc;
-      nb->n_value = sym.value;
-      if (_swap)
-        swapStruct(*nb);
-      symOffset += sizeof(nlist_64);
-    } else {
-      nlist* nb = reinterpret_cast<nlist*>(&_buffer[symOffset]);
-      nb->n_strx = strOffset - _startOfSymbolStrings;
-      nb->n_type = sym.type | sym.scope;
-      nb->n_sect = sym.sect;
-      nb->n_desc = sym.desc;
-      nb->n_value = sym.value;
-      if (_swap)
-        swapStruct(*nb);
-      symOffset += sizeof(nlist);
-    }
-    memcpy(&_buffer[strOffset], sym.name.begin(), sym.name.size());
-    strOffset += sym.name.size();
-    _buffer[strOffset++] ='\0'; // Strings in table have nul terminator.
-  }
-}
-
-void MachOFileLayout::writeFunctionStartsInfo() {
-  if (!_functionStartsSize)
-    return;
-  memcpy(&_buffer[_startOfFunctionStarts], _file.functionStarts.data(),
-         _functionStartsSize);
-}
-
-void MachOFileLayout::writeDataInCodeInfo() {
-  uint32_t offset = _startOfDataInCode;
-  for (const DataInCode &entry : _file.dataInCode) {
-    data_in_code_entry *dst = reinterpret_cast<data_in_code_entry*>(
-                                                             &_buffer[offset]);
-    dst->offset = entry.offset;
-    dst->length = entry.length;
-    dst->kind   = entry.kind;
-    if (_swap)
-      swapStruct(*dst);
-    offset += sizeof(data_in_code_entry);
-  }
-}
-
-void MachOFileLayout::writeSymbolTable() {
-  // Write symbol table and symbol strings in parallel.
-  uint32_t symOffset = _startOfSymbols;
-  uint32_t strOffset = _startOfSymbolStrings;
-  // Reserve n_strx offset of zero to mean no name.
-  _buffer[strOffset++] = ' ';
-  _buffer[strOffset++] = '\0';
-  appendSymbols(_file.stabsSymbols, symOffset, strOffset);
-  appendSymbols(_file.localSymbols, symOffset, strOffset);
-  appendSymbols(_file.globalSymbols, symOffset, strOffset);
-  appendSymbols(_file.undefinedSymbols, symOffset, strOffset);
-  // Write indirect symbol table array.
-  uint32_t *indirects = reinterpret_cast<uint32_t*>
-                                            (&_buffer[_startOfIndirectSymbols]);
-  if (_file.fileType == llvm::MachO::MH_OBJECT) {
-    // Object files have sections in same order as input normalized file.
-    for (const Section &section : _file.sections) {
-      for (uint32_t index : section.indirectSymbols) {
-        if (_swap)
-          *indirects++ = llvm::sys::getSwappedBytes(index);
-        else
-          *indirects++ = index;
-      }
-    }
-  } else {
-    // Final linked images must sort sections from normalized file.
-    for (const Segment &seg : _file.segments) {
-      SegExtraInfo &segInfo = _segInfo[&seg];
-      for (const Section *section : segInfo.sections) {
-        for (uint32_t index : section->indirectSymbols) {
-          if (_swap)
-            *indirects++ = llvm::sys::getSwappedBytes(index);
-          else
-            *indirects++ = index;
-        }
-      }
-    }
-  }
-}
-
-void MachOFileLayout::writeRebaseInfo() {
-  memcpy(&_buffer[_startOfRebaseInfo], _rebaseInfo.bytes(), _rebaseInfo.size());
-}
-
-void MachOFileLayout::writeBindingInfo() {
-  memcpy(&_buffer[_startOfBindingInfo],
-                                    _bindingInfo.bytes(), _bindingInfo.size());
-}
-
-void MachOFileLayout::writeLazyBindingInfo() {
-  memcpy(&_buffer[_startOfLazyBindingInfo],
-                            _lazyBindingInfo.bytes(), _lazyBindingInfo.size());
-}
-
-void MachOFileLayout::writeExportInfo() {
-  memcpy(&_buffer[_startOfExportTrie], _exportTrie.bytes(), _exportTrie.size());
-}
-
-void MachOFileLayout::buildLinkEditInfo() {
-  buildRebaseInfo();
-  buildBindInfo();
-  buildLazyBindInfo();
-  buildExportTrie();
-  computeSymbolTableSizes();
-  computeFunctionStartsSize();
-  computeDataInCodeSize();
-}
-
-void MachOFileLayout::buildSectionRelocations() {
-
-}
-
-void MachOFileLayout::buildRebaseInfo() {
-  // TODO: compress rebasing info.
-  for (const RebaseLocation& entry : _file.rebasingInfo) {
-    _rebaseInfo.append_byte(REBASE_OPCODE_SET_TYPE_IMM | entry.kind);
-    _rebaseInfo.append_byte(REBASE_OPCODE_SET_SEGMENT_AND_OFFSET_ULEB
-                            | entry.segIndex);
-    _rebaseInfo.append_uleb128(entry.segOffset);
-    _rebaseInfo.append_uleb128(REBASE_OPCODE_DO_REBASE_IMM_TIMES | 1);
-  }
-  _rebaseInfo.append_byte(REBASE_OPCODE_DONE);
-  _rebaseInfo.align(_is64 ? 8 : 4);
-}
-
-void MachOFileLayout::buildBindInfo() {
-  // TODO: compress bind info.
-  uint64_t lastAddend = 0;
-  int lastOrdinal = 0x80000000;
-  StringRef lastSymbolName;
-  BindType lastType = (BindType)0;
-  Hex32 lastSegOffset = ~0U;
-  uint8_t lastSegIndex = (uint8_t)~0U;
-  for (const BindLocation& entry : _file.bindingInfo) {
-    if (entry.ordinal != lastOrdinal) {
-      if (entry.ordinal <= 0)
-        _bindingInfo.append_byte(BIND_OPCODE_SET_DYLIB_SPECIAL_IMM |
-                                 (entry.ordinal & BIND_IMMEDIATE_MASK));
-      else if (entry.ordinal <= BIND_IMMEDIATE_MASK)
-        _bindingInfo.append_byte(BIND_OPCODE_SET_DYLIB_ORDINAL_IMM |
-                                 entry.ordinal);
-      else {
-        _bindingInfo.append_byte(BIND_OPCODE_SET_DYLIB_ORDINAL_ULEB);
-        _bindingInfo.append_uleb128(entry.ordinal);
-      }
-      lastOrdinal = entry.ordinal;
-    }
-
-    if (lastSymbolName != entry.symbolName) {
-      _bindingInfo.append_byte(BIND_OPCODE_SET_SYMBOL_TRAILING_FLAGS_IMM);
-      _bindingInfo.append_string(entry.symbolName);
-      lastSymbolName = entry.symbolName;
-    }
-
-    if (lastType != entry.kind) {
-      _bindingInfo.append_byte(BIND_OPCODE_SET_TYPE_IMM | entry.kind);
-      lastType = entry.kind;
-    }
-
-    if (lastSegIndex != entry.segIndex || lastSegOffset != entry.segOffset) {
-      _bindingInfo.append_byte(BIND_OPCODE_SET_SEGMENT_AND_OFFSET_ULEB
-                               | entry.segIndex);
-      _bindingInfo.append_uleb128(entry.segOffset);
-      lastSegIndex = entry.segIndex;
-      lastSegOffset = entry.segOffset;
-    }
-    if (entry.addend != lastAddend) {
-      _bindingInfo.append_byte(BIND_OPCODE_SET_ADDEND_SLEB);
-      _bindingInfo.append_sleb128(entry.addend);
-      lastAddend = entry.addend;
-    }
-    _bindingInfo.append_byte(BIND_OPCODE_DO_BIND);
-  }
-  _bindingInfo.append_byte(BIND_OPCODE_DONE);
-  _bindingInfo.align(_is64 ? 8 : 4);
-}
-
-void MachOFileLayout::buildLazyBindInfo() {
-  for (const BindLocation& entry : _file.lazyBindingInfo) {
-    _lazyBindingInfo.append_byte(BIND_OPCODE_SET_SEGMENT_AND_OFFSET_ULEB
-                            | entry.segIndex);
-    _lazyBindingInfo.append_uleb128(entry.segOffset);
-    if (entry.ordinal <= 0)
-      _lazyBindingInfo.append_byte(BIND_OPCODE_SET_DYLIB_SPECIAL_IMM |
-                                   (entry.ordinal & BIND_IMMEDIATE_MASK));
-    else if (entry.ordinal <= BIND_IMMEDIATE_MASK)
-      _lazyBindingInfo.append_byte(BIND_OPCODE_SET_DYLIB_ORDINAL_IMM |
-                                   entry.ordinal);
-    else {
-      _lazyBindingInfo.append_byte(BIND_OPCODE_SET_DYLIB_ORDINAL_ULEB);
-      _lazyBindingInfo.append_uleb128(entry.ordinal);
-    }
-    // FIXME: We need to | the opcode here with flags.
-    _lazyBindingInfo.append_byte(BIND_OPCODE_SET_SYMBOL_TRAILING_FLAGS_IMM);
-    _lazyBindingInfo.append_string(entry.symbolName);
-    _lazyBindingInfo.append_byte(BIND_OPCODE_DO_BIND);
-    _lazyBindingInfo.append_byte(BIND_OPCODE_DONE);
-  }
-  _lazyBindingInfo.align(_is64 ? 8 : 4);
-}
-
-void TrieNode::addSymbol(const Export& entry,
-                         BumpPtrAllocator &allocator,
-                         std::vector<TrieNode*> &allNodes) {
-  StringRef partialStr = entry.name.drop_front(_cummulativeString.size());
-  for (TrieEdge &edge : _children) {
-    StringRef edgeStr = edge._subString;
-    if (partialStr.startswith(edgeStr)) {
-      // Already have matching edge, go down that path.
-      edge._child->addSymbol(entry, allocator, allNodes);
-      return;
-    }
-    // See if string has common prefix with existing edge.
-    for (int n=edgeStr.size()-1; n > 0; --n) {
-      if (partialStr.substr(0, n).equals(edgeStr.substr(0, n))) {
-        // Splice in new node:  was A -> C,  now A -> B -> C
-        StringRef bNodeStr = edge._child->_cummulativeString;
-        bNodeStr = bNodeStr.drop_back(edgeStr.size()-n).copy(allocator);
-        auto *bNode = new (allocator) TrieNode(bNodeStr);
-        allNodes.push_back(bNode);
-        TrieNode* cNode = edge._child;
-        StringRef abEdgeStr = edgeStr.substr(0,n).copy(allocator);
-        StringRef bcEdgeStr = edgeStr.substr(n).copy(allocator);
-        DEBUG_WITH_TYPE("trie-builder", llvm::dbgs()
-                        << "splice in TrieNode('" << bNodeStr
-                        << "') between edge '"
-                        << abEdgeStr << "' and edge='"
-                        << bcEdgeStr<< "'\n");
-        TrieEdge& abEdge = edge;
-        abEdge._subString = abEdgeStr;
-        abEdge._child = bNode;
-        auto *bcEdge = new (allocator) TrieEdge(bcEdgeStr, cNode);
-        bNode->_children.insert(bNode->_children.end(), bcEdge);
-        bNode->addSymbol(entry, allocator, allNodes);
-        return;
-      }
-    }
-  }
-  if (entry.flags & EXPORT_SYMBOL_FLAGS_REEXPORT) {
-    assert(entry.otherOffset != 0);
-  }
-  if (entry.flags & EXPORT_SYMBOL_FLAGS_STUB_AND_RESOLVER) {
-    assert(entry.otherOffset != 0);
-  }
-  // No commonality with any existing child, make a new edge.
-  auto *newNode = new (allocator) TrieNode(entry.name.copy(allocator));
-  auto *newEdge = new (allocator) TrieEdge(partialStr, newNode);
-  _children.insert(_children.end(), newEdge);
-  DEBUG_WITH_TYPE("trie-builder", llvm::dbgs()
-                   << "new TrieNode('" << entry.name << "') with edge '"
-                   << partialStr << "' from node='"
-                   << _cummulativeString << "'\n");
-  newNode->_address = entry.offset;
-  newNode->_flags = entry.flags | entry.kind;
-  newNode->_other = entry.otherOffset;
-  if ((entry.flags & EXPORT_SYMBOL_FLAGS_REEXPORT) && !entry.otherName.empty())
-    newNode->_importedName = entry.otherName.copy(allocator);
-  newNode->_hasExportInfo = true;
-  allNodes.push_back(newNode);
-}
-
-void TrieNode::addOrderedNodes(const Export& entry,
-                               std::vector<TrieNode*> &orderedNodes) {
-  if (!_ordered) {
-    orderedNodes.push_back(this);
-    _ordered = true;
-  }
-
-  StringRef partialStr = entry.name.drop_front(_cummulativeString.size());
-  for (TrieEdge &edge : _children) {
-    StringRef edgeStr = edge._subString;
-    if (partialStr.startswith(edgeStr)) {
-      // Already have matching edge, go down that path.
-      edge._child->addOrderedNodes(entry, orderedNodes);
-      return;
-    }
-  }
-}
-
-bool TrieNode::updateOffset(uint32_t& offset) {
-  uint32_t nodeSize = 1; // Length when no export info
-  if (_hasExportInfo) {
-    if (_flags & EXPORT_SYMBOL_FLAGS_REEXPORT) {
-      nodeSize = llvm::getULEB128Size(_flags);
-      nodeSize += llvm::getULEB128Size(_other); // Other contains ordinal.
-      nodeSize += _importedName.size();
-      ++nodeSize; // Trailing zero in imported name.
-    } else {
-      nodeSize = llvm::getULEB128Size(_flags) + llvm::getULEB128Size(_address);
-      if (_flags & EXPORT_SYMBOL_FLAGS_STUB_AND_RESOLVER)
-        nodeSize += llvm::getULEB128Size(_other);
-    }
-    // Overall node size so far is uleb128 of export info + actual export info.
-    nodeSize += llvm::getULEB128Size(nodeSize);
-  }
-  // Compute size of all child edges.
-  ++nodeSize; // Byte for number of children.
-  for (TrieEdge &edge : _children) {
-    nodeSize += edge._subString.size() + 1 // String length.
-              + llvm::getULEB128Size(edge._child->_trieOffset); // Offset len.
-  }
-  // On input, 'offset' is new prefered location for this node.
-  bool result = (_trieOffset != offset);
-  // Store new location in node object for use by parents.
-  _trieOffset = offset;
-  // Update offset for next iteration.
-  offset += nodeSize;
-  // Return true if _trieOffset was changed.
-  return result;
-}
-
-void TrieNode::appendToByteBuffer(ByteBuffer &out) {
-  if (_hasExportInfo) {
-    if (_flags & EXPORT_SYMBOL_FLAGS_REEXPORT) {
-      if (!_importedName.empty()) {
-        // nodes with re-export info: size, flags, ordinal, import-name
-        uint32_t nodeSize = llvm::getULEB128Size(_flags)
-                          + llvm::getULEB128Size(_other)
-                          + _importedName.size() + 1;
-        assert(nodeSize < 256);
-        out.append_byte(nodeSize);
-        out.append_uleb128(_flags);
-        out.append_uleb128(_other);
-        out.append_string(_importedName);
-      } else {
-        // nodes without re-export info: size, flags, ordinal, empty-string
-        uint32_t nodeSize = llvm::getULEB128Size(_flags)
-                          + llvm::getULEB128Size(_other) + 1;
-        assert(nodeSize < 256);
-        out.append_byte(nodeSize);
-        out.append_uleb128(_flags);
-        out.append_uleb128(_other);
-        out.append_byte(0);
-      }
-    } else if ( _flags & EXPORT_SYMBOL_FLAGS_STUB_AND_RESOLVER ) {
-      // Nodes with export info: size, flags, address, other
-      uint32_t nodeSize = llvm::getULEB128Size(_flags)
-                        + llvm::getULEB128Size(_address)
-                        + llvm::getULEB128Size(_other);
-      assert(nodeSize < 256);
-      out.append_byte(nodeSize);
-      out.append_uleb128(_flags);
-      out.append_uleb128(_address);
-      out.append_uleb128(_other);
-    } else {
-      // Nodes with export info: size, flags, address
-      uint32_t nodeSize = llvm::getULEB128Size(_flags)
-                        + llvm::getULEB128Size(_address);
-      assert(nodeSize < 256);
-      out.append_byte(nodeSize);
-      out.append_uleb128(_flags);
-      out.append_uleb128(_address);
-    }
-  } else {
-    // Node with no export info.
-    uint32_t nodeSize = 0;
-    out.append_byte(nodeSize);
-  }
-  // Add number of children.
-  assert(_children.size() < 256);
-  out.append_byte(_children.size());
-  // Append each child edge substring and node offset.
-  for (TrieEdge &edge : _children) {
-    out.append_string(edge._subString);
-    out.append_uleb128(edge._child->_trieOffset);
-  }
-}
-
-void MachOFileLayout::buildExportTrie() {
-  if (_file.exportInfo.empty())
-    return;
-
-  // For all temporary strings and objects used building trie.
-  BumpPtrAllocator allocator;
-
-  // Build trie of all exported symbols.
-  auto *rootNode = new (allocator) TrieNode(StringRef());
-  std::vector<TrieNode*> allNodes;
-  allNodes.reserve(_file.exportInfo.size()*2);
-  allNodes.push_back(rootNode);
-  for (const Export& entry : _file.exportInfo) {
-    rootNode->addSymbol(entry, allocator, allNodes);
-  }
-
-  std::vector<TrieNode*> orderedNodes;
-  orderedNodes.reserve(allNodes.size());
-
-  for (const Export& entry : _file.exportInfo)
-    rootNode->addOrderedNodes(entry, orderedNodes);
-
-  // Assign each node in the vector an offset in the trie stream, iterating
-  // until all uleb128 sizes have stabilized.
-  bool more;
-  do {
-    uint32_t offset = 0;
-    more = false;
-    for (TrieNode* node : orderedNodes) {
-      if (node->updateOffset(offset))
-        more = true;
-    }
-  } while (more);
-
-  // Serialize trie to ByteBuffer.
-  for (TrieNode* node : orderedNodes) {
-    node->appendToByteBuffer(_exportTrie);
-  }
-  _exportTrie.align(_is64 ? 8 : 4);
-}
-
-void MachOFileLayout::computeSymbolTableSizes() {
-  // MachO symbol tables have three ranges: locals, globals, and undefines
-  const size_t nlistSize = (_is64 ? sizeof(nlist_64) : sizeof(nlist));
-  _symbolTableSize = nlistSize * (_file.stabsSymbols.size()
-                                + _file.localSymbols.size()
-                                + _file.globalSymbols.size()
-                                + _file.undefinedSymbols.size());
-  // Always reserve 1-byte for the empty string and 1-byte for its terminator.
-  _symbolStringPoolSize = 2;
-  for (const Symbol &sym : _file.stabsSymbols) {
-    _symbolStringPoolSize += (sym.name.size()+1);
-  }
-  for (const Symbol &sym : _file.localSymbols) {
-    _symbolStringPoolSize += (sym.name.size()+1);
-  }
-  for (const Symbol &sym : _file.globalSymbols) {
-    _symbolStringPoolSize += (sym.name.size()+1);
-  }
-  for (const Symbol &sym : _file.undefinedSymbols) {
-    _symbolStringPoolSize += (sym.name.size()+1);
-  }
-  _symbolTableLocalsStartIndex = 0;
-  _symbolTableGlobalsStartIndex = _file.stabsSymbols.size() +
-                                  _file.localSymbols.size();
-  _symbolTableUndefinesStartIndex = _symbolTableGlobalsStartIndex
-                                    + _file.globalSymbols.size();
-
-  _indirectSymbolTableCount = 0;
-  for (const Section &sect : _file.sections) {
-    _indirectSymbolTableCount += sect.indirectSymbols.size();
-  }
-}
-
-void MachOFileLayout::computeFunctionStartsSize() {
-  _functionStartsSize = _file.functionStarts.size();
-}
-
-void MachOFileLayout::computeDataInCodeSize() {
-  _dataInCodeSize = _file.dataInCode.size() * sizeof(data_in_code_entry);
-}
-
-void MachOFileLayout::writeLinkEditContent() {
-  if (_file.fileType == llvm::MachO::MH_OBJECT) {
-    writeRelocations();
-    writeFunctionStartsInfo();
-    writeDataInCodeInfo();
-    writeSymbolTable();
-  } else {
-    writeRebaseInfo();
-    writeBindingInfo();
-    writeLazyBindingInfo();
-    // TODO: add weak binding info
-    writeExportInfo();
-    writeFunctionStartsInfo();
-    writeDataInCodeInfo();
-    writeSymbolTable();
-  }
-}
-
-llvm::Error MachOFileLayout::writeBinary(StringRef path) {
-  // Check for pending error from constructor.
-  if (_ec)
-    return llvm::errorCodeToError(_ec);
-  // Create FileOutputBuffer with calculated size.
-  unsigned flags = 0;
-  if (_file.fileType != llvm::MachO::MH_OBJECT)
-    flags = llvm::FileOutputBuffer::F_executable;
-  Expected<std::unique_ptr<llvm::FileOutputBuffer>> fobOrErr =
-      llvm::FileOutputBuffer::create(path, size(), flags);
-  if (Error E = fobOrErr.takeError())
-    return E;
-  std::unique_ptr<llvm::FileOutputBuffer> &fob = *fobOrErr;
-  // Write content.
-  _buffer = fob->getBufferStart();
-  writeMachHeader();
-  if (auto ec = writeLoadCommands())
-    return ec;
-  writeSectionContent();
-  writeLinkEditContent();
-  if (Error E = fob->commit())
-    return E;
-
-  return llvm::Error::success();
-}
-
-/// Takes in-memory normalized view and writes a mach-o object file.
-llvm::Error writeBinary(const NormalizedFile &file, StringRef path) {
-  MachOFileLayout layout(file, false);
-  return layout.writeBinary(path);
-}
-
-} // namespace normalized
-} // namespace mach_o
-} // namespace lld
diff --git a/lld/lib/ReaderWriter/MachO/MachONormalizedFileFromAtoms.cpp b/lld/lib/ReaderWriter/MachO/MachONormalizedFileFromAtoms.cpp
deleted file mode 100644
index ddfd1764f7e1..000000000000
--- a/lld/lib/ReaderWriter/MachO/MachONormalizedFileFromAtoms.cpp
+++ /dev/null
@@ -1,1657 +0,0 @@
-//===- lib/ReaderWriter/MachO/MachONormalizedFileFromAtoms.cpp ------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-///
-/// \file Converts from in-memory Atoms to in-memory normalized mach-o.
-///
-///                  +------------+
-///                  | normalized |
-///                  +------------+
-///                        ^
-///                        |
-///                        |
-///                    +-------+
-///                    | Atoms |
-///                    +-------+
-
-#include "ArchHandler.h"
-#include "DebugInfo.h"
-#include "MachONormalizedFile.h"
-#include "MachONormalizedFileBinaryUtils.h"
-#include "lld/Common/LLVM.h"
-#include "lld/Core/Error.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/ADT/StringSwitch.h"
-#include "llvm/BinaryFormat/MachO.h"
-#include "llvm/Support/Casting.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/Format.h"
-#include <map>
-#include <system_error>
-#include <unordered_set>
-
-using llvm::StringRef;
-using llvm::isa;
-using namespace llvm::MachO;
-using namespace lld::mach_o::normalized;
-using namespace lld;
-
-namespace {
-
-struct AtomInfo {
-  const DefinedAtom  *atom;
-  uint64_t            offsetInSection;
-};
-
-struct SectionInfo {
-  SectionInfo(StringRef seg, StringRef sect, SectionType type,
-              const MachOLinkingContext &ctxt, uint32_t attr,
-              bool relocsToDefinedCanBeImplicit);
-
-  StringRef                 segmentName;
-  StringRef                 sectionName;
-  SectionType               type;
-  uint32_t                  attributes;
-  uint64_t                  address;
-  uint64_t                  size;
-  uint16_t                  alignment;
-
-  /// If this is set, the any relocs in this section which point to defined
-  /// addresses can be implicitly generated.  This is the case for the
-  /// __eh_frame section where references to the function can be implicit if the
-  /// function is defined.
-  bool                      relocsToDefinedCanBeImplicit;
-
-
-  std::vector<AtomInfo>     atomsAndOffsets;
-  uint32_t                  normalizedSectionIndex;
-  uint32_t                  finalSectionIndex;
-};
-
-SectionInfo::SectionInfo(StringRef sg, StringRef sct, SectionType t,
-                         const MachOLinkingContext &ctxt, uint32_t attrs,
-                         bool relocsToDefinedCanBeImplicit)
- : segmentName(sg), sectionName(sct), type(t), attributes(attrs),
-                 address(0), size(0), alignment(1),
-                 relocsToDefinedCanBeImplicit(relocsToDefinedCanBeImplicit),
-                 normalizedSectionIndex(0), finalSectionIndex(0) {
-  uint16_t align = 1;
-  if (ctxt.sectionAligned(segmentName, sectionName, align)) {
-    alignment = align;
-  }
-}
-
-struct SegmentInfo {
-  SegmentInfo(StringRef name);
-
-  StringRef                  name;
-  uint64_t                   address;
-  uint64_t                   size;
-  uint32_t                   init_access;
-  uint32_t                   max_access;
-  std::vector<SectionInfo*>  sections;
-  uint32_t                   normalizedSegmentIndex;
-};
-
-SegmentInfo::SegmentInfo(StringRef n)
- : name(n), address(0), size(0), init_access(0), max_access(0),
-   normalizedSegmentIndex(0) {
-}
-
-class Util {
-public:
-  Util(const MachOLinkingContext &ctxt)
-      : _ctx(ctxt), _archHandler(ctxt.archHandler()), _entryAtom(nullptr),
-        _hasTLVDescriptors(false), _subsectionsViaSymbols(true) {}
-  ~Util();
-
-  void      processDefinedAtoms(const lld::File &atomFile);
-  void      processAtomAttributes(const DefinedAtom *atom);
-  void      assignAtomToSection(const DefinedAtom *atom);
-  void      organizeSections();
-  void      assignAddressesToSections(const NormalizedFile &file);
-  uint32_t  fileFlags();
-  void      copySegmentInfo(NormalizedFile &file);
-  void      copySectionInfo(NormalizedFile &file);
-  void      updateSectionInfo(NormalizedFile &file);
-  void      buildAtomToAddressMap();
-  llvm::Error synthesizeDebugNotes(NormalizedFile &file);
-  llvm::Error addSymbols(const lld::File &atomFile, NormalizedFile &file);
-  void      addIndirectSymbols(const lld::File &atomFile, NormalizedFile &file);
-  void      addRebaseAndBindingInfo(const lld::File &, NormalizedFile &file);
-  void      addExportInfo(const lld::File &, NormalizedFile &file);
-  void      addSectionRelocs(const lld::File &, NormalizedFile &file);
-  void      addFunctionStarts(const lld::File &, NormalizedFile &file);
-  void      buildDataInCodeArray(const lld::File &, NormalizedFile &file);
-  void      addDependentDylibs(const lld::File &, NormalizedFile &file);
-  void      copyEntryPointAddress(NormalizedFile &file);
-  void      copySectionContent(NormalizedFile &file);
-
-  bool allSourceFilesHaveMinVersions() const {
-    return _allSourceFilesHaveMinVersions;
-  }
-
-  uint32_t minVersion() const {
-    return _minVersion;
-  }
-
-  LoadCommandType minVersionCommandType() const {
-    return _minVersionCommandType;
-  }
-
-private:
-  typedef std::map<DefinedAtom::ContentType, SectionInfo*> TypeToSection;
-  typedef llvm::DenseMap<const Atom*, uint64_t> AtomToAddress;
-
-  struct DylibInfo { int ordinal; bool hasWeak; bool hasNonWeak; };
-  typedef llvm::StringMap<DylibInfo> DylibPathToInfo;
-
-  SectionInfo *sectionForAtom(const DefinedAtom*);
-  SectionInfo *getRelocatableSection(DefinedAtom::ContentType type);
-  SectionInfo *getFinalSection(DefinedAtom::ContentType type);
-  void         appendAtom(SectionInfo *sect, const DefinedAtom *atom);
-  SegmentInfo *segmentForName(StringRef segName);
-  void         layoutSectionsInSegment(SegmentInfo *seg, uint64_t &addr);
-  void         layoutSectionsInTextSegment(size_t, SegmentInfo *, uint64_t &);
-  void         copySectionContent(SectionInfo *si, ContentBytes &content);
-  uint16_t     descBits(const DefinedAtom* atom);
-  int          dylibOrdinal(const SharedLibraryAtom *sa);
-  void         segIndexForSection(const SectionInfo *sect,
-                             uint8_t &segmentIndex, uint64_t &segmentStartAddr);
-  const Atom  *targetOfLazyPointer(const DefinedAtom *lpAtom);
-  const Atom  *targetOfStub(const DefinedAtom *stubAtom);
-  llvm::Error getSymbolTableRegion(const DefinedAtom* atom,
-                                   bool &inGlobalsRegion,
-                                   SymbolScope &symbolScope);
-  void         appendSection(SectionInfo *si, NormalizedFile &file);
-  uint32_t     sectionIndexForAtom(const Atom *atom);
-  void fixLazyReferenceImm(const DefinedAtom *atom, uint32_t offset,
-                           NormalizedFile &file);
-
-  typedef llvm::DenseMap<const Atom*, uint32_t> AtomToIndex;
-  struct AtomAndIndex { const Atom *atom; uint32_t index; SymbolScope scope; };
-  struct AtomSorter {
-    bool operator()(const AtomAndIndex &left, const AtomAndIndex &right);
-  };
-  struct SegmentSorter {
-    bool operator()(const SegmentInfo *left, const SegmentInfo *right);
-    static unsigned weight(const SegmentInfo *);
-  };
-  struct TextSectionSorter {
-    bool operator()(const SectionInfo *left, const SectionInfo *right);
-    static unsigned weight(const SectionInfo *);
-  };
-
-  const MachOLinkingContext &_ctx;
-  mach_o::ArchHandler          &_archHandler;
-  llvm::BumpPtrAllocator        _allocator;
-  std::vector<SectionInfo*>     _sectionInfos;
-  std::vector<SegmentInfo*>     _segmentInfos;
-  TypeToSection                 _sectionMap;
-  std::vector<SectionInfo*>     _customSections;
-  AtomToAddress                 _atomToAddress;
-  DylibPathToInfo               _dylibInfo;
-  const DefinedAtom            *_entryAtom;
-  AtomToIndex                   _atomToSymbolIndex;
-  std::vector<const Atom *>     _machHeaderAliasAtoms;
-  bool                          _hasTLVDescriptors;
-  bool                          _subsectionsViaSymbols;
-  bool                          _allSourceFilesHaveMinVersions = true;
-  LoadCommandType               _minVersionCommandType = (LoadCommandType)0;
-  uint32_t                      _minVersion = 0;
-  std::vector<lld::mach_o::Stab> _stabs;
-};
-
-Util::~Util() {
-  // The SectionInfo structs are BumpPtr allocated, but atomsAndOffsets needs
-  // to be deleted.
-  for (SectionInfo *si : _sectionInfos) {
-    // clear() destroys vector elements, but does not deallocate.
-    // Instead use swap() to deallocate vector buffer.
-    std::vector<AtomInfo> empty;
-    si->atomsAndOffsets.swap(empty);
-  }
-  // The SegmentInfo structs are BumpPtr allocated, but sections needs
-  // to be deleted.
-  for (SegmentInfo *sgi : _segmentInfos) {
-    std::vector<SectionInfo*> empty2;
-    sgi->sections.swap(empty2);
-  }
-}
-
-SectionInfo *Util::getRelocatableSection(DefinedAtom::ContentType type) {
-  StringRef segmentName;
-  StringRef sectionName;
-  SectionType sectionType;
-  SectionAttr sectionAttrs;
-  bool relocsToDefinedCanBeImplicit;
-
-  // Use same table used by when parsing .o files.
-  relocatableSectionInfoForContentType(type, segmentName, sectionName,
-                                       sectionType, sectionAttrs,
-                                       relocsToDefinedCanBeImplicit);
-  // If we already have a SectionInfo with this name, re-use it.
-  // This can happen if two ContentType map to the same mach-o section.
-  for (auto sect : _sectionMap) {
-    if (sect.second->sectionName.equals(sectionName) &&
-        sect.second->segmentName.equals(segmentName)) {
-      return sect.second;
-    }
-  }
-  // Otherwise allocate new SectionInfo object.
-  auto *sect = new (_allocator)
-      SectionInfo(segmentName, sectionName, sectionType, _ctx, sectionAttrs,
-                  relocsToDefinedCanBeImplicit);
-  _sectionInfos.push_back(sect);
-  _sectionMap[type] = sect;
-  return sect;
-}
-
-#define ENTRY(seg, sect, type, atomType) \
-  {seg, sect, type, DefinedAtom::atomType }
-
-struct MachOFinalSectionFromAtomType {
-  StringRef                 segmentName;
-  StringRef                 sectionName;
-  SectionType               sectionType;
-  DefinedAtom::ContentType  atomType;
-};
-
-const MachOFinalSectionFromAtomType sectsToAtomType[] = {
-  ENTRY("__TEXT", "__text",           S_REGULAR,          typeCode),
-  ENTRY("__TEXT", "__text",           S_REGULAR,          typeMachHeader),
-  ENTRY("__TEXT", "__cstring",        S_CSTRING_LITERALS, typeCString),
-  ENTRY("__TEXT", "__ustring",        S_REGULAR,          typeUTF16String),
-  ENTRY("__TEXT", "__const",          S_REGULAR,          typeConstant),
-  ENTRY("__TEXT", "__const",          S_4BYTE_LITERALS,   typeLiteral4),
-  ENTRY("__TEXT", "__const",          S_8BYTE_LITERALS,   typeLiteral8),
-  ENTRY("__TEXT", "__const",          S_16BYTE_LITERALS,  typeLiteral16),
-  ENTRY("__TEXT", "__stubs",          S_SYMBOL_STUBS,     typeStub),
-  ENTRY("__TEXT", "__stub_helper",    S_REGULAR,          typeStubHelper),
-  ENTRY("__TEXT", "__gcc_except_tab", S_REGULAR,          typeLSDA),
-  ENTRY("__TEXT", "__eh_frame",       S_COALESCED,        typeCFI),
-  ENTRY("__TEXT", "__unwind_info",    S_REGULAR,          typeProcessedUnwindInfo),
-  ENTRY("__DATA", "__data",           S_REGULAR,          typeData),
-  ENTRY("__DATA", "__const",          S_REGULAR,          typeConstData),
-  ENTRY("__DATA", "__cfstring",       S_REGULAR,          typeCFString),
-  ENTRY("__DATA", "__la_symbol_ptr",  S_LAZY_SYMBOL_POINTERS,
-                                                          typeLazyPointer),
-  ENTRY("__DATA", "__mod_init_func",  S_MOD_INIT_FUNC_POINTERS,
-                                                          typeInitializerPtr),
-  ENTRY("__DATA", "__mod_term_func",  S_MOD_TERM_FUNC_POINTERS,
-                                                          typeTerminatorPtr),
-  ENTRY("__DATA", "__got",            S_NON_LAZY_SYMBOL_POINTERS,
-                                                          typeGOT),
-  ENTRY("__DATA", "__nl_symbol_ptr",  S_NON_LAZY_SYMBOL_POINTERS,
-                                                          typeNonLazyPointer),
-  ENTRY("__DATA", "__thread_vars",    S_THREAD_LOCAL_VARIABLES,
-                                                          typeThunkTLV),
-  ENTRY("__DATA", "__thread_data",    S_THREAD_LOCAL_REGULAR,
-                                                          typeTLVInitialData),
-  ENTRY("__DATA", "__thread_ptrs",    S_THREAD_LOCAL_VARIABLE_POINTERS,
-                                                          typeTLVInitializerPtr),
-  ENTRY("__DATA", "__thread_bss",     S_THREAD_LOCAL_ZEROFILL,
-                                                         typeTLVInitialZeroFill),
-  ENTRY("__DATA", "__bss",            S_ZEROFILL,         typeZeroFill),
-  ENTRY("__DATA", "__interposing",    S_INTERPOSING,      typeInterposingTuples),
-};
-#undef ENTRY
-
-SectionInfo *Util::getFinalSection(DefinedAtom::ContentType atomType) {
-  for (auto &p : sectsToAtomType) {
-    if (p.atomType != atomType)
-      continue;
-    SectionAttr sectionAttrs = 0;
-    switch (atomType) {
-    case DefinedAtom::typeMachHeader:
-    case DefinedAtom::typeCode:
-    case DefinedAtom::typeStub:
-    case DefinedAtom::typeStubHelper:
-      sectionAttrs = S_ATTR_PURE_INSTRUCTIONS | S_ATTR_SOME_INSTRUCTIONS;
-      break;
-    case DefinedAtom::typeThunkTLV:
-      _hasTLVDescriptors = true;
-      break;
-    default:
-      break;
-    }
-    // If we already have a SectionInfo with this name, re-use it.
-    // This can happen if two ContentType map to the same mach-o section.
-    for (auto sect : _sectionMap) {
-      if (sect.second->sectionName.equals(p.sectionName) &&
-          sect.second->segmentName.equals(p.segmentName)) {
-        return sect.second;
-      }
-    }
-    // Otherwise allocate new SectionInfo object.
-    auto *sect = new (_allocator) SectionInfo(
-        p.segmentName, p.sectionName, p.sectionType, _ctx, sectionAttrs,
-        /* relocsToDefinedCanBeImplicit */ false);
-    _sectionInfos.push_back(sect);
-    _sectionMap[atomType] = sect;
-    return sect;
-  }
-  llvm_unreachable("content type not yet supported");
-}
-
-SectionInfo *Util::sectionForAtom(const DefinedAtom *atom) {
-  if (atom->sectionChoice() == DefinedAtom::sectionBasedOnContent) {
-    // Section for this atom is derived from content type.
-    DefinedAtom::ContentType type = atom->contentType();
-    auto pos = _sectionMap.find(type);
-    if ( pos != _sectionMap.end() )
-      return pos->second;
-    bool rMode = (_ctx.outputMachOType() == llvm::MachO::MH_OBJECT);
-    return rMode ? getRelocatableSection(type) : getFinalSection(type);
-  } else {
-    // This atom needs to be in a custom section.
-    StringRef customName = atom->customSectionName();
-    // Look to see if we have already allocated the needed custom section.
-    for(SectionInfo *sect : _customSections) {
-      const DefinedAtom *firstAtom = sect->atomsAndOffsets.front().atom;
-      if (firstAtom->customSectionName().equals(customName)) {
-        return sect;
-      }
-    }
-    // Not found, so need to create a new custom section.
-    size_t seperatorIndex = customName.find('/');
-    assert(seperatorIndex != StringRef::npos);
-    StringRef segName = customName.slice(0, seperatorIndex);
-    StringRef sectName = customName.drop_front(seperatorIndex + 1);
-    auto *sect =
-        new (_allocator) SectionInfo(segName, sectName, S_REGULAR, _ctx,
-                                     0, /* relocsToDefinedCanBeImplicit */ false);
-    _customSections.push_back(sect);
-    _sectionInfos.push_back(sect);
-    return sect;
-  }
-}
-
-void Util::appendAtom(SectionInfo *sect, const DefinedAtom *atom) {
-  // Figure out offset for atom in this section given alignment constraints.
-  uint64_t offset = sect->size;
-  DefinedAtom::Alignment atomAlign = atom->alignment();
-  uint64_t align = atomAlign.value;
-  uint64_t requiredModulus = atomAlign.modulus;
-  uint64_t currentModulus = (offset % align);
-  if ( currentModulus != requiredModulus ) {
-    if ( requiredModulus > currentModulus )
-      offset += requiredModulus-currentModulus;
-    else
-      offset += align+requiredModulus-currentModulus;
-  }
-  // Record max alignment of any atom in this section.
-  if (align > sect->alignment)
-    sect->alignment = atomAlign.value;
-  // Assign atom to this section with this offset.
-  AtomInfo ai = {atom, offset};
-  sect->atomsAndOffsets.push_back(ai);
-  // Update section size to include this atom.
-  sect->size = offset + atom->size();
-}
-
-void Util::processDefinedAtoms(const lld::File &atomFile) {
-  for (const DefinedAtom *atom : atomFile.defined()) {
-    processAtomAttributes(atom);
-    assignAtomToSection(atom);
-  }
-}
-
-void Util::processAtomAttributes(const DefinedAtom *atom) {
-  if (auto *machoFile = dyn_cast<mach_o::MachOFile>(&atom->file())) {
-    // If the file doesn't use subsections via symbols, then make sure we don't
-    // add that flag to the final output file if we have a relocatable file.
-    if (!machoFile->subsectionsViaSymbols())
-      _subsectionsViaSymbols = false;
-
-    // All the source files must have min versions for us to output an object
-    // file with a min version.
-    if (auto v = machoFile->minVersion())
-      _minVersion = std::max(_minVersion, v);
-    else
-      _allSourceFilesHaveMinVersions = false;
-
-    // If we don't have a platform load command, but one of the source files
-    // does, then take the one from the file.
-    if (!_minVersionCommandType)
-      if (auto v = machoFile->minVersionLoadCommandKind())
-        _minVersionCommandType = v;
-  }
-}
-
-void Util::assignAtomToSection(const DefinedAtom *atom) {
-  if (atom->contentType() == DefinedAtom::typeMachHeader) {
-    _machHeaderAliasAtoms.push_back(atom);
-    // Assign atom to this section with this offset.
-    AtomInfo ai = {atom, 0};
-    sectionForAtom(atom)->atomsAndOffsets.push_back(ai);
-  } else if (atom->contentType() == DefinedAtom::typeDSOHandle)
-    _machHeaderAliasAtoms.push_back(atom);
-  else
-    appendAtom(sectionForAtom(atom), atom);
-}
-
-SegmentInfo *Util::segmentForName(StringRef segName) {
-  for (SegmentInfo *si : _segmentInfos) {
-    if ( si->name.equals(segName) )
-      return si;
-  }
-  auto *info = new (_allocator) SegmentInfo(segName);
-
-  // Set the initial segment protection.
-  if (segName.equals("__TEXT"))
-    info->init_access = VM_PROT_READ | VM_PROT_EXECUTE;
-  else if (segName.equals("__PAGEZERO"))
-    info->init_access = 0;
-  else if (segName.equals("__LINKEDIT"))
-    info->init_access = VM_PROT_READ;
-  else {
-    // All others default to read-write
-    info->init_access = VM_PROT_READ | VM_PROT_WRITE;
-  }
-
-  // Set max segment protection
-  // Note, its overkill to use a switch statement here, but makes it so much
-  // easier to use switch coverage to catch new cases.
-  switch (_ctx.os()) {
-    case lld::MachOLinkingContext::OS::unknown:
-    case lld::MachOLinkingContext::OS::macOSX:
-    case lld::MachOLinkingContext::OS::iOS_simulator:
-      if (segName.equals("__PAGEZERO")) {
-        info->max_access = 0;
-        break;
-      }
-      // All others default to all
-      info->max_access = VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE;
-      break;
-    case lld::MachOLinkingContext::OS::iOS:
-      // iPhoneOS always uses same protection for max and initial
-      info->max_access = info->init_access;
-      break;
-  }
-  _segmentInfos.push_back(info);
-  return info;
-}
-
-unsigned Util::SegmentSorter::weight(const SegmentInfo *seg) {
- return llvm::StringSwitch<unsigned>(seg->name)
-    .Case("__PAGEZERO",  1)
-    .Case("__TEXT",      2)
-    .Case("__DATA",      3)
-    .Default(100);
-}
-
-bool Util::SegmentSorter::operator()(const SegmentInfo *left,
-                                  const SegmentInfo *right) {
-  return (weight(left) < weight(right));
-}
-
-unsigned Util::TextSectionSorter::weight(const SectionInfo *sect) {
- return llvm::StringSwitch<unsigned>(sect->sectionName)
-    .Case("__text",         1)
-    .Case("__stubs",        2)
-    .Case("__stub_helper",  3)
-    .Case("__const",        4)
-    .Case("__cstring",      5)
-    .Case("__unwind_info",  98)
-    .Case("__eh_frame",     99)
-    .Default(10);
-}
-
-bool Util::TextSectionSorter::operator()(const SectionInfo *left,
-                                         const SectionInfo *right) {
-  return (weight(left) < weight(right));
-}
-
-void Util::organizeSections() {
-  // NOTE!: Keep this in sync with assignAddressesToSections.
-  switch (_ctx.outputMachOType()) {
-    case llvm::MachO::MH_EXECUTE:
-      // Main executables, need a zero-page segment
-      segmentForName("__PAGEZERO");
-      // Fall into next case.
-      LLVM_FALLTHROUGH;
-    case llvm::MachO::MH_DYLIB:
-    case llvm::MachO::MH_BUNDLE:
-      // All dynamic code needs TEXT segment to hold the load commands.
-      segmentForName("__TEXT");
-      break;
-    default:
-      break;
-  }
-  segmentForName("__LINKEDIT");
-
-  // Group sections into segments.
-  for (SectionInfo *si : _sectionInfos) {
-    SegmentInfo *seg = segmentForName(si->segmentName);
-    seg->sections.push_back(si);
-  }
-  // Sort segments.
-  std::sort(_segmentInfos.begin(), _segmentInfos.end(), SegmentSorter());
-
-  // Sort sections within segments.
-  for (SegmentInfo *seg : _segmentInfos) {
-    if (seg->name.equals("__TEXT")) {
-      std::sort(seg->sections.begin(), seg->sections.end(),
-                TextSectionSorter());
-    }
-  }
-
-  // Record final section indexes.
-  uint32_t segmentIndex = 0;
-  uint32_t sectionIndex = 1;
-  for (SegmentInfo *seg : _segmentInfos) {
-    seg->normalizedSegmentIndex = segmentIndex++;
-    for (SectionInfo *sect : seg->sections)
-      sect->finalSectionIndex = sectionIndex++;
-  }
-}
-
-void Util::layoutSectionsInSegment(SegmentInfo *seg, uint64_t &addr) {
-  seg->address = addr;
-  for (SectionInfo *sect : seg->sections) {
-    sect->address = llvm::alignTo(addr, sect->alignment);
-    addr = sect->address + sect->size;
-  }
-  seg->size = llvm::alignTo(addr - seg->address, _ctx.pageSize());
-}
-
-// __TEXT segment lays out backwards so padding is at front after load commands.
-void Util::layoutSectionsInTextSegment(size_t hlcSize, SegmentInfo *seg,
-                                                               uint64_t &addr) {
-  seg->address = addr;
-  // Walks sections starting at end to calculate padding for start.
-  int64_t taddr = 0;
-  for (auto it = seg->sections.rbegin(); it != seg->sections.rend(); ++it) {
-    SectionInfo *sect = *it;
-    taddr -= sect->size;
-    taddr = taddr & (0 - sect->alignment);
-  }
-  int64_t padding = taddr - hlcSize;
-  while (padding < 0)
-    padding += _ctx.pageSize();
-  // Start assigning section address starting at padded offset.
-  addr += (padding + hlcSize);
-  for (SectionInfo *sect : seg->sections) {
-    sect->address = llvm::alignTo(addr, sect->alignment);
-    addr = sect->address + sect->size;
-  }
-  seg->size = llvm::alignTo(addr - seg->address, _ctx.pageSize());
-}
-
-void Util::assignAddressesToSections(const NormalizedFile &file) {
-  // NOTE!: Keep this in sync with organizeSections.
-  size_t hlcSize = headerAndLoadCommandsSize(file,
-                                      _ctx.generateFunctionStartsLoadCommand());
-  uint64_t address = 0;
-  for (SegmentInfo *seg : _segmentInfos) {
-    if (seg->name.equals("__PAGEZERO")) {
-      seg->size = _ctx.pageZeroSize();
-      address += seg->size;
-    }
-    else if (seg->name.equals("__TEXT")) {
-      // _ctx.baseAddress()  == 0 implies it was either unspecified or
-      // pageZeroSize is also 0. In either case resetting address is safe.
-      address = _ctx.baseAddress() ? _ctx.baseAddress() : address;
-      layoutSectionsInTextSegment(hlcSize, seg, address);
-    } else
-      layoutSectionsInSegment(seg, address);
-
-    address = llvm::alignTo(address, _ctx.pageSize());
-  }
-  DEBUG_WITH_TYPE("WriterMachO-norm",
-    llvm::dbgs() << "assignAddressesToSections()\n";
-    for (SegmentInfo *sgi : _segmentInfos) {
-      llvm::dbgs()  << "   address=" << llvm::format("0x%08llX", sgi->address)
-                    << ", size="  << llvm::format("0x%08llX", sgi->size)
-                    << ", segment-name='" << sgi->name
-                    << "'\n";
-      for (SectionInfo *si : sgi->sections) {
-        llvm::dbgs()<< "      addr="  << llvm::format("0x%08llX", si->address)
-                    << ", size="  << llvm::format("0x%08llX", si->size)
-                    << ", section-name='" << si->sectionName
-                    << "\n";
-      }
-    }
-  );
-}
-
-void Util::copySegmentInfo(NormalizedFile &file) {
-  for (SegmentInfo *sgi : _segmentInfos) {
-    Segment seg;
-    seg.name    = sgi->name;
-    seg.address = sgi->address;
-    seg.size    = sgi->size;
-    seg.init_access  = sgi->init_access;
-    seg.max_access  = sgi->max_access;
-    file.segments.push_back(seg);
-  }
-}
-
-void Util::appendSection(SectionInfo *si, NormalizedFile &file) {
-   // Add new empty section to end of file.sections.
-  Section temp;
-  file.sections.push_back(std::move(temp));
-  Section* normSect = &file.sections.back();
-  // Copy fields to normalized section.
-  normSect->segmentName   = si->segmentName;
-  normSect->sectionName   = si->sectionName;
-  normSect->type          = si->type;
-  normSect->attributes    = si->attributes;
-  normSect->address       = si->address;
-  normSect->alignment     = si->alignment;
-  // Record where normalized section is.
-  si->normalizedSectionIndex = file.sections.size()-1;
-}
-
-void Util::copySectionContent(NormalizedFile &file) {
-  const bool r = (_ctx.outputMachOType() == llvm::MachO::MH_OBJECT);
-
-  // Utility function for ArchHandler to find address of atom in output file.
-  auto addrForAtom = [&] (const Atom &atom) -> uint64_t {
-    auto pos = _atomToAddress.find(&atom);
-    assert(pos != _atomToAddress.end());
-    return pos->second;
-  };
-
-  auto sectionAddrForAtom = [&] (const Atom &atom) -> uint64_t {
-    for (const SectionInfo *sectInfo : _sectionInfos)
-      for (const AtomInfo &atomInfo : sectInfo->atomsAndOffsets)
-        if (atomInfo.atom == &atom)
-          return sectInfo->address;
-    llvm_unreachable("atom not assigned to section");
-  };
-
-  for (SectionInfo *si : _sectionInfos) {
-    Section *normSect = &file.sections[si->normalizedSectionIndex];
-    if (isZeroFillSection(si->type)) {
-      const uint8_t *empty = nullptr;
-      normSect->content = llvm::makeArrayRef(empty, si->size);
-      continue;
-    }
-    // Copy content from atoms to content buffer for section.
-    llvm::MutableArrayRef<uint8_t> sectionContent;
-    if (si->size) {
-      uint8_t *sectContent = file.ownedAllocations.Allocate<uint8_t>(si->size);
-      sectionContent = llvm::MutableArrayRef<uint8_t>(sectContent, si->size);
-      normSect->content = sectionContent;
-    }
-    for (AtomInfo &ai : si->atomsAndOffsets) {
-      if (!ai.atom->size()) {
-        assert(ai.atom->begin() == ai.atom->end() &&
-               "Cannot have references without content");
-        continue;
-      }
-      auto atomContent = sectionContent.slice(ai.offsetInSection,
-                                              ai.atom->size());
-      _archHandler.generateAtomContent(*ai.atom, r, addrForAtom,
-                                       sectionAddrForAtom, _ctx.baseAddress(),
-                                       atomContent);
-    }
-  }
-}
-
-void Util::copySectionInfo(NormalizedFile &file) {
-  file.sections.reserve(_sectionInfos.size());
-  // Write sections grouped by segment.
-  for (SegmentInfo *sgi : _segmentInfos) {
-    for (SectionInfo *si : sgi->sections) {
-      appendSection(si, file);
-    }
-  }
-}
-
-void Util::updateSectionInfo(NormalizedFile &file) {
-  file.sections.reserve(_sectionInfos.size());
-  // sections grouped by segment.
-  for (SegmentInfo *sgi : _segmentInfos) {
-    Segment *normSeg = &file.segments[sgi->normalizedSegmentIndex];
-    normSeg->address = sgi->address;
-    normSeg->size = sgi->size;
-    for (SectionInfo *si : sgi->sections) {
-      Section *normSect = &file.sections[si->normalizedSectionIndex];
-      normSect->address = si->address;
-    }
-  }
-}
-
-void Util::copyEntryPointAddress(NormalizedFile &nFile) {
-  if (!_entryAtom) {
-    nFile.entryAddress = 0;
-    return;
-  }
-
-  if (_ctx.outputTypeHasEntry()) {
-    if (_archHandler.isThumbFunction(*_entryAtom))
-      nFile.entryAddress = (_atomToAddress[_entryAtom] | 1);
-    else
-      nFile.entryAddress = _atomToAddress[_entryAtom];
-  }
-}
-
-void Util::buildAtomToAddressMap() {
-  DEBUG_WITH_TYPE("WriterMachO-address", llvm::dbgs()
-                   << "assign atom addresses:\n");
-  const bool lookForEntry = _ctx.outputTypeHasEntry();
-  for (SectionInfo *sect : _sectionInfos) {
-    for (const AtomInfo &info : sect->atomsAndOffsets) {
-      _atomToAddress[info.atom] = sect->address + info.offsetInSection;
-      if (lookForEntry && (info.atom->contentType() == DefinedAtom::typeCode) &&
-          (info.atom->size() != 0) &&
-          info.atom->name() == _ctx.entrySymbolName()) {
-        _entryAtom = info.atom;
-      }
-      DEBUG_WITH_TYPE("WriterMachO-address", llvm::dbgs()
-                      << "   address="
-                      << llvm::format("0x%016X", _atomToAddress[info.atom])
-                      << llvm::format("    0x%09lX", info.atom)
-                      << ", file=#"
-                      << info.atom->file().ordinal()
-                      << ", atom=#"
-                      << info.atom->ordinal()
-                      << ", name="
-                      << info.atom->name()
-                      << ", type="
-                      << info.atom->contentType()
-                      << "\n");
-    }
-  }
-  DEBUG_WITH_TYPE("WriterMachO-address", llvm::dbgs()
-                  << "assign header alias atom addresses:\n");
-  for (const Atom *atom : _machHeaderAliasAtoms) {
-    _atomToAddress[atom] = _ctx.baseAddress();
-#ifndef NDEBUG
-    if (auto *definedAtom = dyn_cast<DefinedAtom>(atom)) {
-      DEBUG_WITH_TYPE("WriterMachO-address", llvm::dbgs()
-                      << "   address="
-                      << llvm::format("0x%016X", _atomToAddress[atom])
-                      << llvm::format("    0x%09lX", atom)
-                      << ", file=#"
-                      << definedAtom->file().ordinal()
-                      << ", atom=#"
-                      << definedAtom->ordinal()
-                      << ", name="
-                      << definedAtom->name()
-                      << ", type="
-                      << definedAtom->contentType()
-                      << "\n");
-    } else {
-      DEBUG_WITH_TYPE("WriterMachO-address", llvm::dbgs()
-                      << "   address="
-                      << llvm::format("0x%016X", _atomToAddress[atom])
-                      << " atom=" << atom
-                      << " name=" << atom->name() << "\n");
-    }
-#endif
-  }
-}
-
-llvm::Error Util::synthesizeDebugNotes(NormalizedFile &file) {
-
-  // Bail out early if we don't need to generate a debug map.
-  if (_ctx.debugInfoMode() == MachOLinkingContext::DebugInfoMode::noDebugMap)
-    return llvm::Error::success();
-
-  std::vector<const DefinedAtom*> atomsNeedingDebugNotes;
-  std::set<const mach_o::MachOFile*> filesWithStabs;
-  bool objFileHasDwarf = false;
-  const File *objFile = nullptr;
-
-  for (SectionInfo *sect : _sectionInfos) {
-    for (const AtomInfo &info : sect->atomsAndOffsets) {
-      if (const DefinedAtom *atom = dyn_cast<DefinedAtom>(info.atom)) {
-
-        // FIXME: No stabs/debug-notes for symbols that wouldn't be in the
-        //        symbol table.
-        // FIXME: No stabs/debug-notes for kernel dtrace probes.
-
-        if (atom->contentType() == DefinedAtom::typeCFI ||
-            atom->contentType() == DefinedAtom::typeCString)
-          continue;
-
-        // Whenever we encounter a new file, update the 'objfileHasDwarf' flag.
-        if (&info.atom->file() != objFile) {
-          objFileHasDwarf = false;
-          if (const mach_o::MachOFile *atomFile =
-              dyn_cast<mach_o::MachOFile>(&info.atom->file())) {
-            if (atomFile->debugInfo()) {
-              if (isa<mach_o::DwarfDebugInfo>(atomFile->debugInfo()))
-                objFileHasDwarf = true;
-              else if (isa<mach_o::StabsDebugInfo>(atomFile->debugInfo()))
-                filesWithStabs.insert(atomFile);
-            }
-          }
-        }
-
-        // If this atom is from a file that needs dwarf, add it to the list.
-        if (objFileHasDwarf)
-          atomsNeedingDebugNotes.push_back(info.atom);
-      }
-    }
-  }
-
-  // Sort atoms needing debug notes by file ordinal, then atom ordinal.
-  std::sort(atomsNeedingDebugNotes.begin(), atomsNeedingDebugNotes.end(),
-            [](const DefinedAtom *lhs, const DefinedAtom *rhs) {
-              if (lhs->file().ordinal() != rhs->file().ordinal())
-                return (lhs->file().ordinal() < rhs->file().ordinal());
-              return (lhs->ordinal() < rhs->ordinal());
-            });
-
-  // FIXME: Handle <rdar://problem/17689030>: Add -add_ast_path option to \
-  //        linker which add N_AST stab entry to output
-  // See OutputFile::synthesizeDebugNotes in ObjectFile.cpp in ld64.
-
-  StringRef oldFileName = "";
-  StringRef oldDirPath = "";
-  bool wroteStartSO = false;
-  std::unordered_set<std::string> seenFiles;
-  for (const DefinedAtom *atom : atomsNeedingDebugNotes) {
-    const auto &atomFile = cast<mach_o::MachOFile>(atom->file());
-    assert(dyn_cast_or_null<lld::mach_o::DwarfDebugInfo>(atomFile.debugInfo())
-           && "file for atom needing debug notes does not contain dwarf");
-    auto &dwarf = cast<lld::mach_o::DwarfDebugInfo>(*atomFile.debugInfo());
-
-    auto &tu = dwarf.translationUnitSource();
-    StringRef newFileName = tu.name;
-    StringRef newDirPath = tu.path;
-
-    // Add an SO whenever the TU source file changes.
-    if (newFileName != oldFileName || newDirPath != oldDirPath) {
-      // Translation unit change, emit ending SO
-      if (oldFileName != "")
-        _stabs.push_back(mach_o::Stab(nullptr, N_SO, 1, 0, 0, ""));
-
-      oldFileName = newFileName;
-      oldDirPath = newDirPath;
-
-      // If newDirPath doesn't end with a '/' we need to add one:
-      if (newDirPath.back() != '/') {
-        char *p =
-          file.ownedAllocations.Allocate<char>(newDirPath.size() + 2);
-        memcpy(p, newDirPath.data(), newDirPath.size());
-        p[newDirPath.size()] = '/';
-        p[newDirPath.size() + 1] = '\0';
-        newDirPath = p;
-      }
-
-      // New translation unit, emit start SOs:
-      _stabs.push_back(mach_o::Stab(nullptr, N_SO, 0, 0, 0, newDirPath));
-      _stabs.push_back(mach_o::Stab(nullptr, N_SO, 0, 0, 0, newFileName));
-
-      // Synthesize OSO for start of file.
-      char *fullPath = nullptr;
-      {
-        SmallString<1024> pathBuf(atomFile.path());
-        if (auto EC = llvm::sys::fs::make_absolute(pathBuf))
-          return llvm::errorCodeToError(EC);
-        fullPath = file.ownedAllocations.Allocate<char>(pathBuf.size() + 1);
-        memcpy(fullPath, pathBuf.c_str(), pathBuf.size() + 1);
-      }
-
-      // Get mod time.
-      uint32_t modTime = 0;
-      llvm::sys::fs::file_status stat;
-      if (!llvm::sys::fs::status(fullPath, stat))
-        if (llvm::sys::fs::exists(stat))
-          modTime = llvm::sys::toTimeT(stat.getLastModificationTime());
-
-      _stabs.push_back(mach_o::Stab(nullptr, N_OSO, _ctx.getCPUSubType(), 1,
-                                    modTime, fullPath));
-      // <rdar://problem/6337329> linker should put cpusubtype in n_sect field
-      // of nlist entry for N_OSO debug note entries.
-      wroteStartSO = true;
-    }
-
-    if (atom->contentType() == DefinedAtom::typeCode) {
-      // Synthesize BNSYM and start FUN stabs.
-      _stabs.push_back(mach_o::Stab(atom, N_BNSYM, 1, 0, 0, ""));
-      _stabs.push_back(mach_o::Stab(atom, N_FUN, 1, 0, 0, atom->name()));
-      // Synthesize any SOL stabs needed
-      // FIXME: add SOL stabs.
-      _stabs.push_back(mach_o::Stab(nullptr, N_FUN, 0, 0,
-                                    atom->rawContent().size(), ""));
-      _stabs.push_back(mach_o::Stab(nullptr, N_ENSYM, 1, 0,
-                                    atom->rawContent().size(), ""));
-    } else {
-      if (atom->scope() == Atom::scopeTranslationUnit)
-        _stabs.push_back(mach_o::Stab(atom, N_STSYM, 1, 0, 0, atom->name()));
-      else
-        _stabs.push_back(mach_o::Stab(nullptr, N_GSYM, 1, 0, 0, atom->name()));
-    }
-  }
-
-  // Emit ending SO if necessary.
-  if (wroteStartSO)
-    _stabs.push_back(mach_o::Stab(nullptr, N_SO, 1, 0, 0, ""));
-
-  // Copy any stabs from .o file.
-  for (const auto *objFile : filesWithStabs) {
-    const auto &stabsList =
-      cast<mach_o::StabsDebugInfo>(objFile->debugInfo())->stabs();
-    for (auto &stab : stabsList) {
-      // FIXME: Drop stabs whose atoms have been dead-stripped.
-      _stabs.push_back(stab);
-    }
-  }
-
-  return llvm::Error::success();
-}
-
-uint16_t Util::descBits(const DefinedAtom* atom) {
-  uint16_t desc = 0;
-  switch (atom->merge()) {
-  case lld::DefinedAtom::mergeNo:
-  case lld::DefinedAtom::mergeAsTentative:
-    break;
-  case lld::DefinedAtom::mergeAsWeak:
-  case lld::DefinedAtom::mergeAsWeakAndAddressUsed:
-    desc |= N_WEAK_DEF;
-    break;
-  case lld::DefinedAtom::mergeSameNameAndSize:
-  case lld::DefinedAtom::mergeByLargestSection:
-  case lld::DefinedAtom::mergeByContent:
-    llvm_unreachable("Unsupported DefinedAtom::merge()");
-    break;
-  }
-  if (atom->contentType() == lld::DefinedAtom::typeResolver)
-    desc |= N_SYMBOL_RESOLVER;
-  if (atom->contentType() == lld::DefinedAtom::typeMachHeader)
-    desc |= REFERENCED_DYNAMICALLY;
-  if (_archHandler.isThumbFunction(*atom))
-    desc |= N_ARM_THUMB_DEF;
-  if (atom->deadStrip() == DefinedAtom::deadStripNever &&
-      _ctx.outputMachOType() == llvm::MachO::MH_OBJECT) {
-    if ((atom->contentType() != DefinedAtom::typeInitializerPtr)
-     && (atom->contentType() != DefinedAtom::typeTerminatorPtr))
-    desc |= N_NO_DEAD_STRIP;
-  }
-  return desc;
-}
-
-bool Util::AtomSorter::operator()(const AtomAndIndex &left,
-                                  const AtomAndIndex &right) {
-  return (left.atom->name().compare(right.atom->name()) < 0);
-}
-
-llvm::Error Util::getSymbolTableRegion(const DefinedAtom* atom,
-                                       bool &inGlobalsRegion,
-                                       SymbolScope &scope) {
-  bool rMode = (_ctx.outputMachOType() == llvm::MachO::MH_OBJECT);
-  switch (atom->scope()) {
-  case Atom::scopeTranslationUnit:
-    scope = 0;
-    inGlobalsRegion = false;
-    return llvm::Error::success();
-  case Atom::scopeLinkageUnit:
-    if ((_ctx.exportMode() == MachOLinkingContext::ExportMode::exported) &&
-        _ctx.exportSymbolNamed(atom->name())) {
-      return llvm::make_error<GenericError>(
-                          Twine("cannot export hidden symbol ") + atom->name());
-    }
-    if (rMode) {
-      if (_ctx.keepPrivateExterns()) {
-        // -keep_private_externs means keep in globals region as N_PEXT.
-        scope = N_PEXT | N_EXT;
-        inGlobalsRegion = true;
-        return llvm::Error::success();
-      }
-    }
-    // scopeLinkageUnit symbols are no longer global once linked.
-    scope = N_PEXT;
-    inGlobalsRegion = false;
-    return llvm::Error::success();
-  case Atom::scopeGlobal:
-    if (_ctx.exportRestrictMode()) {
-      if (_ctx.exportSymbolNamed(atom->name())) {
-        scope = N_EXT;
-        inGlobalsRegion = true;
-        return llvm::Error::success();
-      } else {
-        scope = N_PEXT;
-        inGlobalsRegion = false;
-        return llvm::Error::success();
-      }
-    } else {
-      scope = N_EXT;
-      inGlobalsRegion = true;
-      return llvm::Error::success();
-    }
-    break;
-  }
-  llvm_unreachable("atom->scope() unknown enum value");
-}
-
-
-
-llvm::Error Util::addSymbols(const lld::File &atomFile,
-                             NormalizedFile &file) {
-  bool rMode = (_ctx.outputMachOType() == llvm::MachO::MH_OBJECT);
-  // Mach-O symbol table has four regions: stabs, locals, globals, undefs.
-
-  // Add all stabs.
-  for (auto &stab : _stabs) {
-    lld::mach_o::normalized::Symbol sym;
-    sym.type = static_cast<NListType>(stab.type);
-    sym.scope = 0;
-    sym.sect = stab.other;
-    sym.desc = stab.desc;
-    if (stab.atom)
-      sym.value = _atomToAddress[stab.atom];
-    else
-      sym.value = stab.value;
-    sym.name = stab.str;
-    file.stabsSymbols.push_back(sym);
-  }
-
-  // Add all local (non-global) symbols in address order
-  std::vector<AtomAndIndex> globals;
-  globals.reserve(512);
-  for (SectionInfo *sect : _sectionInfos) {
-    for (const AtomInfo &info : sect->atomsAndOffsets) {
-      const DefinedAtom *atom = info.atom;
-      if (!atom->name().empty()) {
-        SymbolScope symbolScope;
-        bool inGlobalsRegion;
-        if (auto ec = getSymbolTableRegion(atom, inGlobalsRegion, symbolScope)){
-          return ec;
-        }
-        if (inGlobalsRegion) {
-          AtomAndIndex ai = { atom, sect->finalSectionIndex, symbolScope };
-          globals.push_back(ai);
-        } else {
-          lld::mach_o::normalized::Symbol sym;
-          sym.name  = atom->name();
-          sym.type  = N_SECT;
-          sym.scope = symbolScope;
-          sym.sect  = sect->finalSectionIndex;
-          sym.desc  = descBits(atom);
-          sym.value = _atomToAddress[atom];
-          _atomToSymbolIndex[atom] = file.localSymbols.size();
-          file.localSymbols.push_back(sym);
-        }
-      } else if (rMode && _archHandler.needsLocalSymbolInRelocatableFile(atom)){
-        // Create 'Lxxx' labels for anonymous atoms if archHandler says so.
-        static unsigned tempNum = 1;
-        char tmpName[16];
-        sprintf(tmpName, "L%04u", tempNum++);
-        StringRef tempRef(tmpName);
-        lld::mach_o::normalized::Symbol sym;
-        sym.name  = tempRef.copy(file.ownedAllocations);
-        sym.type  = N_SECT;
-        sym.scope = 0;
-        sym.sect  = sect->finalSectionIndex;
-        sym.desc  = 0;
-        sym.value = _atomToAddress[atom];
-        _atomToSymbolIndex[atom] = file.localSymbols.size();
-        file.localSymbols.push_back(sym);
-      }
-    }
-  }
-
-  // Sort global symbol alphabetically, then add to symbol table.
-  std::sort(globals.begin(), globals.end(), AtomSorter());
-  const uint32_t globalStartIndex = file.localSymbols.size();
-  for (AtomAndIndex &ai : globals) {
-    lld::mach_o::normalized::Symbol sym;
-    sym.name  = ai.atom->name();
-    sym.type  = N_SECT;
-    sym.scope = ai.scope;
-    sym.sect  = ai.index;
-    sym.desc  = descBits(static_cast<const DefinedAtom*>(ai.atom));
-    sym.value = _atomToAddress[ai.atom];
-    _atomToSymbolIndex[ai.atom] = globalStartIndex + file.globalSymbols.size();
-    file.globalSymbols.push_back(sym);
-  }
-
-  // Sort undefined symbol alphabetically, then add to symbol table.
-  std::vector<AtomAndIndex> undefs;
-  undefs.reserve(128);
-  for (const UndefinedAtom *atom : atomFile.undefined()) {
-    AtomAndIndex ai = { atom, 0, N_EXT };
-    undefs.push_back(ai);
-  }
-  for (const SharedLibraryAtom *atom : atomFile.sharedLibrary()) {
-    AtomAndIndex ai = { atom, 0, N_EXT };
-    undefs.push_back(ai);
-  }
-  std::sort(undefs.begin(), undefs.end(), AtomSorter());
-  const uint32_t start = file.globalSymbols.size() + file.localSymbols.size();
-  for (AtomAndIndex &ai : undefs) {
-    lld::mach_o::normalized::Symbol sym;
-    uint16_t desc = 0;
-    if (!rMode) {
-      uint8_t ordinal = 0;
-      if (!_ctx.useFlatNamespace())
-        ordinal = dylibOrdinal(dyn_cast<SharedLibraryAtom>(ai.atom));
-      llvm::MachO::SET_LIBRARY_ORDINAL(desc, ordinal);
-    }
-    sym.name  = ai.atom->name();
-    sym.type  = N_UNDF;
-    sym.scope = ai.scope;
-    sym.sect  = 0;
-    sym.desc  = desc;
-    sym.value = 0;
-    _atomToSymbolIndex[ai.atom] = file.undefinedSymbols.size() + start;
-    file.undefinedSymbols.push_back(sym);
-  }
-
-  return llvm::Error::success();
-}
-
-const Atom *Util::targetOfLazyPointer(const DefinedAtom *lpAtom) {
-  for (const Reference *ref : *lpAtom) {
-    if (_archHandler.isLazyPointer(*ref)) {
-      return ref->target();
-    }
-  }
-  return nullptr;
-}
-
-const Atom *Util::targetOfStub(const DefinedAtom *stubAtom) {
-  for (const Reference *ref : *stubAtom) {
-    if (const Atom *ta = ref->target()) {
-      if (const DefinedAtom *lpAtom = dyn_cast<DefinedAtom>(ta)) {
-        const Atom *target = targetOfLazyPointer(lpAtom);
-        if (target)
-          return target;
-      }
-    }
-  }
-  return nullptr;
-}
-
-void Util::addIndirectSymbols(const lld::File &atomFile, NormalizedFile &file) {
-  for (SectionInfo *si : _sectionInfos) {
-    Section &normSect = file.sections[si->normalizedSectionIndex];
-    switch (si->type) {
-    case llvm::MachO::S_NON_LAZY_SYMBOL_POINTERS:
-      for (const AtomInfo &info : si->atomsAndOffsets) {
-        bool foundTarget = false;
-        for (const Reference *ref : *info.atom) {
-          const Atom *target = ref->target();
-          if (target) {
-            if (isa<const SharedLibraryAtom>(target)) {
-              uint32_t index = _atomToSymbolIndex[target];
-              normSect.indirectSymbols.push_back(index);
-              foundTarget = true;
-            } else {
-              normSect.indirectSymbols.push_back(
-                                            llvm::MachO::INDIRECT_SYMBOL_LOCAL);
-            }
-          }
-        }
-        if (!foundTarget) {
-          normSect.indirectSymbols.push_back(
-                                             llvm::MachO::INDIRECT_SYMBOL_ABS);
-        }
-      }
-      break;
-    case llvm::MachO::S_LAZY_SYMBOL_POINTERS:
-      for (const AtomInfo &info : si->atomsAndOffsets) {
-        const Atom *target = targetOfLazyPointer(info.atom);
-        if (target) {
-          uint32_t index = _atomToSymbolIndex[target];
-          normSect.indirectSymbols.push_back(index);
-        }
-      }
-      break;
-    case llvm::MachO::S_SYMBOL_STUBS:
-      for (const AtomInfo &info : si->atomsAndOffsets) {
-        const Atom *target = targetOfStub(info.atom);
-        if (target) {
-          uint32_t index = _atomToSymbolIndex[target];
-          normSect.indirectSymbols.push_back(index);
-        }
-      }
-      break;
-    default:
-      break;
-    }
-  }
-}
-
-void Util::addDependentDylibs(const lld::File &atomFile,
-                              NormalizedFile &nFile) {
-  // Scan all imported symbols and build up list of dylibs they are from.
-  int ordinal = 1;
-  for (const auto *dylib : _ctx.allDylibs()) {
-    DylibPathToInfo::iterator pos = _dylibInfo.find(dylib->installName());
-    if (pos == _dylibInfo.end()) {
-      DylibInfo info;
-      bool flatNamespaceAtom = dylib == _ctx.flatNamespaceFile();
-
-      // If we're in -flat_namespace mode (or this atom came from the flat
-      // namespace file under -undefined dynamic_lookup) then use the flat
-      // lookup ordinal.
-      if (flatNamespaceAtom || _ctx.useFlatNamespace())
-        info.ordinal = BIND_SPECIAL_DYLIB_FLAT_LOOKUP;
-      else
-        info.ordinal = ordinal++;
-      info.hasWeak = false;
-      info.hasNonWeak = !info.hasWeak;
-      _dylibInfo[dylib->installName()] = info;
-
-      // Unless this was a flat_namespace atom, record the source dylib.
-      if (!flatNamespaceAtom) {
-        DependentDylib depInfo;
-        depInfo.path = dylib->installName();
-        depInfo.kind = llvm::MachO::LC_LOAD_DYLIB;
-        depInfo.currentVersion = _ctx.dylibCurrentVersion(dylib->path());
-        depInfo.compatVersion = _ctx.dylibCompatVersion(dylib->path());
-        nFile.dependentDylibs.push_back(depInfo);
-      }
-    } else {
-      pos->second.hasWeak = false;
-      pos->second.hasNonWeak = !pos->second.hasWeak;
-    }
-  }
-  // Automatically weak link dylib in which all symbols are weak (canBeNull).
-  for (DependentDylib &dep : nFile.dependentDylibs) {
-    DylibInfo &info = _dylibInfo[dep.path];
-    if (info.hasWeak && !info.hasNonWeak)
-      dep.kind = llvm::MachO::LC_LOAD_WEAK_DYLIB;
-    else if (_ctx.isUpwardDylib(dep.path))
-      dep.kind = llvm::MachO::LC_LOAD_UPWARD_DYLIB;
-  }
-}
-
-int Util::dylibOrdinal(const SharedLibraryAtom *sa) {
-  return _dylibInfo[sa->loadName()].ordinal;
-}
-
-void Util::segIndexForSection(const SectionInfo *sect, uint8_t &segmentIndex,
-                                                  uint64_t &segmentStartAddr) {
-  segmentIndex = 0;
-  for (const SegmentInfo *seg : _segmentInfos) {
-    if ((seg->address <= sect->address)
-      && (seg->address+seg->size >= sect->address+sect->size)) {
-      segmentStartAddr = seg->address;
-      return;
-    }
-    ++segmentIndex;
-  }
-  llvm_unreachable("section not in any segment");
-}
-
-uint32_t Util::sectionIndexForAtom(const Atom *atom) {
-  uint64_t address = _atomToAddress[atom];
-  for (const SectionInfo *si : _sectionInfos) {
-    if ((si->address <= address) && (address < si->address+si->size))
-      return si->finalSectionIndex;
-  }
-  llvm_unreachable("atom not in any section");
-}
-
-void Util::addSectionRelocs(const lld::File &, NormalizedFile &file) {
-  if (_ctx.outputMachOType() != llvm::MachO::MH_OBJECT)
-    return;
-
-  // Utility function for ArchHandler to find symbol index for an atom.
-  auto symIndexForAtom = [&] (const Atom &atom) -> uint32_t {
-    auto pos = _atomToSymbolIndex.find(&atom);
-    assert(pos != _atomToSymbolIndex.end());
-    return pos->second;
-  };
-
-  // Utility function for ArchHandler to find section index for an atom.
-  auto sectIndexForAtom = [&] (const Atom &atom) -> uint32_t {
-    return sectionIndexForAtom(&atom);
-  };
-
-  // Utility function for ArchHandler to find address of atom in output file.
-  auto addressForAtom = [&] (const Atom &atom) -> uint64_t {
-    auto pos = _atomToAddress.find(&atom);
-    assert(pos != _atomToAddress.end());
-    return pos->second;
-  };
-
-  for (SectionInfo *si : _sectionInfos) {
-    Section &normSect = file.sections[si->normalizedSectionIndex];
-    for (const AtomInfo &info : si->atomsAndOffsets) {
-      const DefinedAtom *atom = info.atom;
-      for (const Reference *ref : *atom) {
-        // Skip emitting relocs for sections which are always able to be
-        // implicitly regenerated and where the relocation targets an address
-        // which is defined.
-        if (si->relocsToDefinedCanBeImplicit && isa<DefinedAtom>(ref->target()))
-          continue;
-        _archHandler.appendSectionRelocations(*atom, info.offsetInSection, *ref,
-                                              symIndexForAtom,
-                                              sectIndexForAtom,
-                                              addressForAtom,
-                                              normSect.relocations);
-      }
-    }
-  }
-}
-
-void Util::addFunctionStarts(const lld::File &, NormalizedFile &file) {
-  if (!_ctx.generateFunctionStartsLoadCommand())
-    return;
-  file.functionStarts.reserve(8192);
-  // Delta compress function starts, starting with the mach header symbol.
-  const uint64_t badAddress = ~0ULL;
-  uint64_t addr = badAddress;
-  for (SectionInfo *si : _sectionInfos) {
-    for (const AtomInfo &info : si->atomsAndOffsets) {
-      auto type = info.atom->contentType();
-      if (type == DefinedAtom::typeMachHeader) {
-        addr = _atomToAddress[info.atom];
-        continue;
-      }
-      if (type != DefinedAtom::typeCode)
-        continue;
-      assert(addr != badAddress && "Missing mach header symbol");
-      // Skip atoms which have 0 size.  This is so that LC_FUNCTION_STARTS
-      // can't spill in to the next section.
-      if (!info.atom->size())
-        continue;
-      uint64_t nextAddr = _atomToAddress[info.atom];
-      if (_archHandler.isThumbFunction(*info.atom))
-        nextAddr |= 1;
-      uint64_t delta = nextAddr - addr;
-      if (delta) {
-        ByteBuffer buffer;
-        buffer.append_uleb128(delta);
-        file.functionStarts.insert(file.functionStarts.end(), buffer.bytes(),
-                                   buffer.bytes() + buffer.size());
-      }
-      addr = nextAddr;
-    }
-  }
-
-  // Null terminate, and pad to pointer size for this arch.
-  file.functionStarts.push_back(0);
-
-  auto size = file.functionStarts.size();
-  for (unsigned i = size, e = llvm::alignTo(size, _ctx.is64Bit() ? 8 : 4);
-       i != e; ++i)
-    file.functionStarts.push_back(0);
-}
-
-void Util::buildDataInCodeArray(const lld::File &, NormalizedFile &file) {
-  if (!_ctx.generateDataInCodeLoadCommand())
-    return;
-  for (SectionInfo *si : _sectionInfos) {
-    for (const AtomInfo &info : si->atomsAndOffsets) {
-      // Atoms that contain data-in-code have "transition" references
-      // which mark a point where the embedded data starts of ends.
-      // This needs to be converted to the mach-o format which is an array
-      // of data-in-code ranges.
-      uint32_t startOffset = 0;
-      DataRegionType mode = DataRegionType(0);
-      for (const Reference *ref : *info.atom) {
-        if (ref->kindNamespace() != Reference::KindNamespace::mach_o)
-          continue;
-        if (_archHandler.isDataInCodeTransition(ref->kindValue())) {
-          DataRegionType nextMode = (DataRegionType)ref->addend();
-          if (mode != nextMode) {
-            if (mode != 0) {
-              // Found end data range, so make range entry.
-              DataInCode entry;
-              entry.offset = si->address + info.offsetInSection + startOffset;
-              entry.length = ref->offsetInAtom() - startOffset;
-              entry.kind   = mode;
-              file.dataInCode.push_back(entry);
-            }
-          }
-          mode = nextMode;
-          startOffset = ref->offsetInAtom();
-        }
-      }
-      if (mode != 0) {
-        // Function ends with data (no end transition).
-        DataInCode entry;
-        entry.offset = si->address + info.offsetInSection + startOffset;
-        entry.length = info.atom->size() - startOffset;
-        entry.kind   = mode;
-        file.dataInCode.push_back(entry);
-      }
-    }
-  }
-}
-
-void Util::addRebaseAndBindingInfo(const lld::File &atomFile,
-                                                        NormalizedFile &nFile) {
-  if (_ctx.outputMachOType() == llvm::MachO::MH_OBJECT)
-    return;
-
-  uint8_t segmentIndex;
-  uint64_t segmentStartAddr;
-  uint32_t offsetInBindInfo = 0;
-
-  for (SectionInfo *sect : _sectionInfos) {
-    segIndexForSection(sect, segmentIndex, segmentStartAddr);
-    for (const AtomInfo &info : sect->atomsAndOffsets) {
-      const DefinedAtom *atom = info.atom;
-      for (const Reference *ref : *atom) {
-        uint64_t segmentOffset = _atomToAddress[atom] + ref->offsetInAtom()
-                                - segmentStartAddr;
-        const Atom* targ = ref->target();
-        if (_archHandler.isPointer(*ref)) {
-          // A pointer to a DefinedAtom requires rebasing.
-          if (isa<DefinedAtom>(targ)) {
-            RebaseLocation rebase;
-            rebase.segIndex = segmentIndex;
-            rebase.segOffset = segmentOffset;
-            rebase.kind = llvm::MachO::REBASE_TYPE_POINTER;
-            nFile.rebasingInfo.push_back(rebase);
-          }
-          // A pointer to an SharedLibraryAtom requires binding.
-          if (const SharedLibraryAtom *sa = dyn_cast<SharedLibraryAtom>(targ)) {
-            BindLocation bind;
-            bind.segIndex = segmentIndex;
-            bind.segOffset = segmentOffset;
-            bind.kind = llvm::MachO::BIND_TYPE_POINTER;
-            bind.canBeNull = sa->canBeNullAtRuntime();
-            bind.ordinal = dylibOrdinal(sa);
-            bind.symbolName = targ->name();
-            bind.addend = ref->addend();
-            nFile.bindingInfo.push_back(bind);
-          }
-        }
-        else if (_archHandler.isLazyPointer(*ref)) {
-          BindLocation bind;
-          if (const SharedLibraryAtom *sa = dyn_cast<SharedLibraryAtom>(targ)) {
-            bind.ordinal = dylibOrdinal(sa);
-          } else {
-            bind.ordinal = llvm::MachO::BIND_SPECIAL_DYLIB_SELF;
-          }
-          bind.segIndex = segmentIndex;
-          bind.segOffset = segmentOffset;
-          bind.kind = llvm::MachO::BIND_TYPE_POINTER;
-          bind.canBeNull = false; //sa->canBeNullAtRuntime();
-          bind.symbolName = targ->name();
-          bind.addend = ref->addend();
-          nFile.lazyBindingInfo.push_back(bind);
-
-          // Now that we know the segmentOffset and the ordinal attribute,
-          // we can fix the helper's code
-
-          fixLazyReferenceImm(atom, offsetInBindInfo, nFile);
-
-          // 5 bytes for opcodes + variable sizes (target name + \0 and offset
-          // encode's size)
-          offsetInBindInfo +=
-              6 + targ->name().size() + llvm::getULEB128Size(bind.segOffset);
-          if (bind.ordinal > BIND_IMMEDIATE_MASK)
-            offsetInBindInfo += llvm::getULEB128Size(bind.ordinal);
-        }
-      }
-    }
-  }
-}
-
-void Util::fixLazyReferenceImm(const DefinedAtom *atom, uint32_t offset,
-                               NormalizedFile &file) {
-  for (const Reference *ref : *atom) {
-    const DefinedAtom *da = dyn_cast<DefinedAtom>(ref->target());
-    if (da == nullptr)
-      return;
-
-    const Reference *helperRef = nullptr;
-    for (const Reference *hr : *da) {
-      if (hr->kindValue() == _archHandler.lazyImmediateLocationKind()) {
-        helperRef = hr;
-        break;
-      }
-    }
-    if (helperRef == nullptr)
-      continue;
-
-    // TODO: maybe get the fixed atom content from _archHandler ?
-    for (SectionInfo *sectInfo : _sectionInfos) {
-      for (const AtomInfo &atomInfo : sectInfo->atomsAndOffsets) {
-        if (atomInfo.atom == helperRef->target()) {
-          auto sectionContent =
-              file.sections[sectInfo->normalizedSectionIndex].content;
-          uint8_t *rawb =
-              file.ownedAllocations.Allocate<uint8_t>(sectionContent.size());
-          llvm::MutableArrayRef<uint8_t> newContent{rawb,
-                                                    sectionContent.size()};
-          std::copy(sectionContent.begin(), sectionContent.end(),
-                    newContent.begin());
-          llvm::support::ulittle32_t *loc =
-              reinterpret_cast<llvm::support::ulittle32_t *>(
-                  &newContent[atomInfo.offsetInSection +
-                              helperRef->offsetInAtom()]);
-          *loc = offset;
-          file.sections[sectInfo->normalizedSectionIndex].content = newContent;
-        }
-      }
-    }
-  }
-}
-
-void Util::addExportInfo(const lld::File &atomFile, NormalizedFile &nFile) {
-  if (_ctx.outputMachOType() == llvm::MachO::MH_OBJECT)
-    return;
-
-  for (SectionInfo *sect : _sectionInfos) {
-    for (const AtomInfo &info : sect->atomsAndOffsets) {
-      const DefinedAtom *atom = info.atom;
-      if (atom->scope() != Atom::scopeGlobal)
-        continue;
-      if (_ctx.exportRestrictMode()) {
-        if (!_ctx.exportSymbolNamed(atom->name()))
-          continue;
-      }
-      Export exprt;
-      exprt.name = atom->name();
-      exprt.offset = _atomToAddress[atom] - _ctx.baseAddress();
-      exprt.kind = EXPORT_SYMBOL_FLAGS_KIND_REGULAR;
-      if (atom->merge() == DefinedAtom::mergeAsWeak)
-        exprt.flags = EXPORT_SYMBOL_FLAGS_WEAK_DEFINITION;
-      else
-        exprt.flags = 0;
-      exprt.otherOffset = 0;
-      exprt.otherName = StringRef();
-      nFile.exportInfo.push_back(exprt);
-    }
-  }
-}
-
-uint32_t Util::fileFlags() {
-  // FIXME: these need to determined at runtime.
-  if (_ctx.outputMachOType() == MH_OBJECT) {
-    return _subsectionsViaSymbols ? (uint32_t)MH_SUBSECTIONS_VIA_SYMBOLS : 0;
-  } else {
-    uint32_t flags = MH_DYLDLINK;
-    if (!_ctx.useFlatNamespace())
-        flags |= MH_TWOLEVEL | MH_NOUNDEFS;
-    if ((_ctx.outputMachOType() == MH_EXECUTE) && _ctx.PIE())
-      flags |= MH_PIE;
-    if (_hasTLVDescriptors)
-      flags |= (MH_PIE | MH_HAS_TLV_DESCRIPTORS);
-    return flags;
-  }
-}
-
-} // end anonymous namespace
-
-namespace lld {
-namespace mach_o {
-namespace normalized {
-
-/// Convert a set of Atoms into a normalized mach-o file.
-llvm::Expected<std::unique_ptr<NormalizedFile>>
-normalizedFromAtoms(const lld::File &atomFile,
-                                           const MachOLinkingContext &context) {
-  // The util object buffers info until the normalized file can be made.
-  Util util(context);
-  util.processDefinedAtoms(atomFile);
-  util.organizeSections();
-
-  std::unique_ptr<NormalizedFile> f(new NormalizedFile());
-  NormalizedFile &normFile = *f.get();
-  normFile.arch = context.arch();
-  normFile.fileType = context.outputMachOType();
-  normFile.flags = util.fileFlags();
-  normFile.stackSize = context.stackSize();
-  normFile.installName = context.installName();
-  normFile.currentVersion = context.currentVersion();
-  normFile.compatVersion = context.compatibilityVersion();
-  normFile.os = context.os();
-
-  // If we are emitting an object file, then the min version is the maximum
-  // of the min's of all the source files and the cmdline.
-  if (normFile.fileType == llvm::MachO::MH_OBJECT)
-    normFile.minOSverson = std::max(context.osMinVersion(), util.minVersion());
-  else
-    normFile.minOSverson = context.osMinVersion();
-
-  normFile.minOSVersionKind = util.minVersionCommandType();
-
-  normFile.sdkVersion = context.sdkVersion();
-  normFile.sourceVersion = context.sourceVersion();
-
-  if (context.generateVersionLoadCommand() &&
-      context.os() != MachOLinkingContext::OS::unknown)
-    normFile.hasMinVersionLoadCommand = true;
-  else if (normFile.fileType == llvm::MachO::MH_OBJECT &&
-           util.allSourceFilesHaveMinVersions() &&
-           ((normFile.os != MachOLinkingContext::OS::unknown) ||
-            util.minVersionCommandType())) {
-    // If we emit an object file, then it should contain a min version load
-    // command if all of the source files also contained min version commands.
-    // Also, we either need to have a platform, or found a platform from the
-    // source object files.
-    normFile.hasMinVersionLoadCommand = true;
-  }
-  normFile.generateDataInCodeLoadCommand =
-    context.generateDataInCodeLoadCommand();
-  normFile.pageSize = context.pageSize();
-  normFile.rpaths = context.rpaths();
-  util.addDependentDylibs(atomFile, normFile);
-  util.copySegmentInfo(normFile);
-  util.copySectionInfo(normFile);
-  util.assignAddressesToSections(normFile);
-  util.buildAtomToAddressMap();
-  if (auto err = util.synthesizeDebugNotes(normFile))
-    return std::move(err);
-  util.updateSectionInfo(normFile);
-  util.copySectionContent(normFile);
-  if (auto ec = util.addSymbols(atomFile, normFile)) {
-    return std::move(ec);
-  }
-  util.addIndirectSymbols(atomFile, normFile);
-  util.addRebaseAndBindingInfo(atomFile, normFile);
-  util.addExportInfo(atomFile, normFile);
-  util.addSectionRelocs(atomFile, normFile);
-  util.addFunctionStarts(atomFile, normFile);
-  util.buildDataInCodeArray(atomFile, normFile);
-  util.copyEntryPointAddress(normFile);
-
-  return std::move(f);
-}
-
-} // namespace normalized
-} // namespace mach_o
-} // namespace lld
diff --git a/lld/lib/ReaderWriter/MachO/MachONormalizedFileToAtoms.cpp b/lld/lib/ReaderWriter/MachO/MachONormalizedFileToAtoms.cpp
deleted file mode 100644
index 164a283b972b..000000000000
--- a/lld/lib/ReaderWriter/MachO/MachONormalizedFileToAtoms.cpp
+++ /dev/null
@@ -1,1635 +0,0 @@
-//===- lib/ReaderWriter/MachO/MachONormalizedFileToAtoms.cpp --------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-///
-/// \file Converts from in-memory normalized mach-o to in-memory Atoms.
-///
-///                  +------------+
-///                  | normalized |
-///                  +------------+
-///                        |
-///                        |
-///                        v
-///                    +-------+
-///                    | Atoms |
-///                    +-------+
-
-#include "ArchHandler.h"
-#include "Atoms.h"
-#include "File.h"
-#include "MachONormalizedFile.h"
-#include "MachONormalizedFileBinaryUtils.h"
-#include "lld/Common/LLVM.h"
-#include "lld/Core/Error.h"
-#include "llvm/BinaryFormat/Dwarf.h"
-#include "llvm/BinaryFormat/MachO.h"
-#include "llvm/DebugInfo/DWARF/DWARFFormValue.h"
-#include "llvm/Support/DataExtractor.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/Error.h"
-#include "llvm/Support/Format.h"
-#include "llvm/Support/LEB128.h"
-#include "llvm/Support/raw_ostream.h"
-
-using namespace llvm::MachO;
-using namespace lld::mach_o::normalized;
-
-#define DEBUG_TYPE "normalized-file-to-atoms"
-
-namespace lld {
-namespace mach_o {
-
-
-namespace { // anonymous
-
-
-#define ENTRY(seg, sect, type, atomType) \
-  {seg, sect, type, DefinedAtom::atomType }
-
-struct MachORelocatableSectionToAtomType {
-  StringRef                 segmentName;
-  StringRef                 sectionName;
-  SectionType               sectionType;
-  DefinedAtom::ContentType  atomType;
-};
-
-const MachORelocatableSectionToAtomType sectsToAtomType[] = {
-  ENTRY("__TEXT", "__text",           S_REGULAR,          typeCode),
-  ENTRY("__TEXT", "__text",           S_REGULAR,          typeResolver),
-  ENTRY("__TEXT", "__cstring",        S_CSTRING_LITERALS, typeCString),
-  ENTRY("",       "",                 S_CSTRING_LITERALS, typeCString),
-  ENTRY("__TEXT", "__ustring",        S_REGULAR,          typeUTF16String),
-  ENTRY("__TEXT", "__const",          S_REGULAR,          typeConstant),
-  ENTRY("__TEXT", "__const_coal",     S_COALESCED,        typeConstant),
-  ENTRY("__TEXT", "__eh_frame",       S_COALESCED,        typeCFI),
-  ENTRY("__TEXT", "__eh_frame",       S_REGULAR,          typeCFI),
-  ENTRY("__TEXT", "__literal4",       S_4BYTE_LITERALS,   typeLiteral4),
-  ENTRY("__TEXT", "__literal8",       S_8BYTE_LITERALS,   typeLiteral8),
-  ENTRY("__TEXT", "__literal16",      S_16BYTE_LITERALS,  typeLiteral16),
-  ENTRY("__TEXT", "__gcc_except_tab", S_REGULAR,          typeLSDA),
-  ENTRY("__DATA", "__data",           S_REGULAR,          typeData),
-  ENTRY("__DATA", "__datacoal_nt",    S_COALESCED,        typeData),
-  ENTRY("__DATA", "__const",          S_REGULAR,          typeConstData),
-  ENTRY("__DATA", "__cfstring",       S_REGULAR,          typeCFString),
-  ENTRY("__DATA", "__mod_init_func",  S_MOD_INIT_FUNC_POINTERS,
-                                                          typeInitializerPtr),
-  ENTRY("__DATA", "__mod_term_func",  S_MOD_TERM_FUNC_POINTERS,
-                                                          typeTerminatorPtr),
-  ENTRY("__DATA", "__got",            S_NON_LAZY_SYMBOL_POINTERS,
-                                                          typeGOT),
-  ENTRY("__DATA", "__bss",            S_ZEROFILL,         typeZeroFill),
-  ENTRY("",       "",                 S_NON_LAZY_SYMBOL_POINTERS,
-                                                          typeGOT),
-  ENTRY("__DATA", "__interposing",    S_INTERPOSING,      typeInterposingTuples),
-  ENTRY("__DATA", "__thread_vars",    S_THREAD_LOCAL_VARIABLES,
-                                                          typeThunkTLV),
-  ENTRY("__DATA", "__thread_data", S_THREAD_LOCAL_REGULAR, typeTLVInitialData),
-  ENTRY("__DATA", "__thread_bss",     S_THREAD_LOCAL_ZEROFILL,
-                                                        typeTLVInitialZeroFill),
-  ENTRY("__DATA", "__objc_imageinfo", S_REGULAR,          typeObjCImageInfo),
-  ENTRY("__DATA", "__objc_catlist",   S_REGULAR,          typeObjC2CategoryList),
-  ENTRY("",       "",                 S_INTERPOSING,      typeInterposingTuples),
-  ENTRY("__LD",   "__compact_unwind", S_REGULAR,
-                                                         typeCompactUnwindInfo),
-  ENTRY("",       "",                 S_REGULAR,          typeUnknown)
-};
-#undef ENTRY
-
-
-/// Figures out ContentType of a mach-o section.
-DefinedAtom::ContentType atomTypeFromSection(const Section &section,
-                                             bool &customSectionName) {
-  // First look for match of name and type. Empty names in table are wildcards.
-  customSectionName = false;
-  for (const MachORelocatableSectionToAtomType *p = sectsToAtomType ;
-                                 p->atomType != DefinedAtom::typeUnknown; ++p) {
-    if (p->sectionType != section.type)
-      continue;
-    if (!p->segmentName.equals(section.segmentName) && !p->segmentName.empty())
-      continue;
-    if (!p->sectionName.equals(section.sectionName) && !p->sectionName.empty())
-      continue;
-    customSectionName = p->segmentName.empty() && p->sectionName.empty();
-    return p->atomType;
-  }
-  // Look for code denoted by section attributes
-  if (section.attributes & S_ATTR_PURE_INSTRUCTIONS)
-    return DefinedAtom::typeCode;
-
-  return DefinedAtom::typeUnknown;
-}
-
-enum AtomizeModel {
-  atomizeAtSymbols,
-  atomizeFixedSize,
-  atomizePointerSize,
-  atomizeUTF8,
-  atomizeUTF16,
-  atomizeCFI,
-  atomizeCU,
-  atomizeCFString
-};
-
-/// Returns info on how to atomize a section of the specified ContentType.
-void sectionParseInfo(DefinedAtom::ContentType atomType,
-                      unsigned int &sizeMultiple,
-                      DefinedAtom::Scope &scope,
-                      DefinedAtom::Merge &merge,
-                      AtomizeModel &atomizeModel) {
-  struct ParseInfo {
-    DefinedAtom::ContentType  atomType;
-    unsigned int              sizeMultiple;
-    DefinedAtom::Scope        scope;
-    DefinedAtom::Merge        merge;
-    AtomizeModel              atomizeModel;
-  };
-
-  #define ENTRY(type, size, scope, merge, model) \
-    {DefinedAtom::type, size, DefinedAtom::scope, DefinedAtom::merge, model }
-
-  static const ParseInfo parseInfo[] = {
-    ENTRY(typeCode,              1, scopeGlobal,          mergeNo,
-                                                            atomizeAtSymbols),
-    ENTRY(typeData,              1, scopeGlobal,          mergeNo,
-                                                            atomizeAtSymbols),
-    ENTRY(typeConstData,         1, scopeGlobal,          mergeNo,
-                                                            atomizeAtSymbols),
-    ENTRY(typeZeroFill,          1, scopeGlobal,          mergeNo,
-                                                            atomizeAtSymbols),
-    ENTRY(typeConstant,          1, scopeGlobal,          mergeNo,
-                                                            atomizeAtSymbols),
-    ENTRY(typeCString,           1, scopeLinkageUnit,     mergeByContent,
-                                                            atomizeUTF8),
-    ENTRY(typeUTF16String,       1, scopeLinkageUnit,     mergeByContent,
-                                                            atomizeUTF16),
-    ENTRY(typeCFI,               4, scopeTranslationUnit, mergeNo,
-                                                            atomizeCFI),
-    ENTRY(typeLiteral4,          4, scopeLinkageUnit,     mergeByContent,
-                                                            atomizeFixedSize),
-    ENTRY(typeLiteral8,          8, scopeLinkageUnit,     mergeByContent,
-                                                            atomizeFixedSize),
-    ENTRY(typeLiteral16,        16, scopeLinkageUnit,     mergeByContent,
-                                                            atomizeFixedSize),
-    ENTRY(typeCFString,          4, scopeLinkageUnit,     mergeByContent,
-                                                            atomizeCFString),
-    ENTRY(typeInitializerPtr,    4, scopeTranslationUnit, mergeNo,
-                                                            atomizePointerSize),
-    ENTRY(typeTerminatorPtr,     4, scopeTranslationUnit, mergeNo,
-                                                            atomizePointerSize),
-    ENTRY(typeCompactUnwindInfo, 4, scopeTranslationUnit, mergeNo,
-                                                            atomizeCU),
-    ENTRY(typeGOT,               4, scopeLinkageUnit,     mergeByContent,
-                                                            atomizePointerSize),
-    ENTRY(typeObjC2CategoryList, 4, scopeTranslationUnit, mergeByContent,
-                                                            atomizePointerSize),
-    ENTRY(typeUnknown,           1, scopeGlobal,          mergeNo,
-                                                            atomizeAtSymbols)
-  };
-  #undef ENTRY
-  const int tableLen = sizeof(parseInfo) / sizeof(ParseInfo);
-  for (int i=0; i < tableLen; ++i) {
-    if (parseInfo[i].atomType == atomType) {
-      sizeMultiple = parseInfo[i].sizeMultiple;
-      scope        = parseInfo[i].scope;
-      merge        = parseInfo[i].merge;
-      atomizeModel = parseInfo[i].atomizeModel;
-      return;
-    }
-  }
-
-  // Unknown type is atomized by symbols.
-  sizeMultiple = 1;
-  scope = DefinedAtom::scopeGlobal;
-  merge = DefinedAtom::mergeNo;
-  atomizeModel = atomizeAtSymbols;
-}
-
-
-Atom::Scope atomScope(uint8_t scope) {
-  switch (scope) {
-  case N_EXT:
-    return Atom::scopeGlobal;
-  case N_PEXT:
-  case N_PEXT | N_EXT:
-    return Atom::scopeLinkageUnit;
-  case 0:
-    return Atom::scopeTranslationUnit;
-  }
-  llvm_unreachable("unknown scope value!");
-}
-
-void appendSymbolsInSection(
-    const std::vector<lld::mach_o::normalized::Symbol> &inSymbols,
-    uint32_t sectionIndex,
-    SmallVector<const lld::mach_o::normalized::Symbol *, 64> &outSyms) {
-  for (const lld::mach_o::normalized::Symbol &sym : inSymbols) {
-    // Only look at definition symbols.
-    if ((sym.type & N_TYPE) != N_SECT)
-      continue;
-    if (sym.sect != sectionIndex)
-      continue;
-    outSyms.push_back(&sym);
-  }
-}
-
-void atomFromSymbol(DefinedAtom::ContentType atomType, const Section &section,
-                    MachOFile &file, uint64_t symbolAddr, StringRef symbolName,
-                    uint16_t symbolDescFlags, Atom::Scope symbolScope,
-                    uint64_t nextSymbolAddr, bool scatterable, bool copyRefs) {
-  // Mach-O symbol table does have size in it. Instead the size is the
-  // difference between this and the next symbol.
-  uint64_t size = nextSymbolAddr - symbolAddr;
-  uint64_t offset = symbolAddr - section.address;
-  bool noDeadStrip = (symbolDescFlags & N_NO_DEAD_STRIP) || !scatterable;
-  if (isZeroFillSection(section.type)) {
-    file.addZeroFillDefinedAtom(symbolName, symbolScope, offset, size,
-                                noDeadStrip, copyRefs, &section);
-  } else {
-    DefinedAtom::Merge merge = (symbolDescFlags & N_WEAK_DEF)
-                              ? DefinedAtom::mergeAsWeak : DefinedAtom::mergeNo;
-    bool thumb = (symbolDescFlags & N_ARM_THUMB_DEF);
-    if (atomType == DefinedAtom::typeUnknown) {
-      // Mach-O needs a segment and section name.  Concatenate those two
-      // with a / separator (e.g. "seg/sect") to fit into the lld model
-      // of just a section name.
-      std::string segSectName = section.segmentName.str()
-                                + "/" + section.sectionName.str();
-      file.addDefinedAtomInCustomSection(symbolName, symbolScope, atomType,
-                                         merge, thumb, noDeadStrip, offset,
-                                         size, segSectName, true, &section);
-    } else {
-      if ((atomType == lld::DefinedAtom::typeCode) &&
-          (symbolDescFlags & N_SYMBOL_RESOLVER)) {
-        atomType = lld::DefinedAtom::typeResolver;
-      }
-      file.addDefinedAtom(symbolName, symbolScope, atomType, merge,
-                          offset, size, thumb, noDeadStrip, copyRefs, &section);
-    }
-  }
-}
-
-llvm::Error processSymboledSection(DefinedAtom::ContentType atomType,
-                                   const Section &section,
-                                   const NormalizedFile &normalizedFile,
-                                   MachOFile &file, bool scatterable,
-                                   bool copyRefs) {
-  // Find section's index.
-  uint32_t sectIndex = 1;
-  for (auto &sect : normalizedFile.sections) {
-    if (&sect == &section)
-      break;
-    ++sectIndex;
-  }
-
-  // Find all symbols in this section.
-  SmallVector<const lld::mach_o::normalized::Symbol *, 64> symbols;
-  appendSymbolsInSection(normalizedFile.globalSymbols, sectIndex, symbols);
-  appendSymbolsInSection(normalizedFile.localSymbols,  sectIndex, symbols);
-
-  // Sort symbols.
-  std::sort(symbols.begin(), symbols.end(),
-            [](const lld::mach_o::normalized::Symbol *lhs,
-               const lld::mach_o::normalized::Symbol *rhs) -> bool {
-              if (lhs == rhs)
-                return false;
-              // First by address.
-              uint64_t lhsAddr = lhs->value;
-              uint64_t rhsAddr = rhs->value;
-              if (lhsAddr != rhsAddr)
-                return lhsAddr < rhsAddr;
-              // If same address, one is an alias so sort by scope.
-              Atom::Scope lScope = atomScope(lhs->scope);
-              Atom::Scope rScope = atomScope(rhs->scope);
-              if (lScope != rScope)
-                return lScope < rScope;
-              // If same address and scope, see if one might be better as
-              // the alias.
-              bool lPrivate = (lhs->name.front() == 'l');
-              bool rPrivate = (rhs->name.front() == 'l');
-              if (lPrivate != rPrivate)
-                return lPrivate;
-              // If same address and scope, sort by name.
-              return lhs->name < rhs->name;
-            });
-
-  // Debug logging of symbols.
-  // for (const Symbol *sym : symbols)
-  //  llvm::errs() << "  sym: "
-  //    << llvm::format("0x%08llx ", (uint64_t)sym->value)
-  //    << ", " << sym->name << "\n";
-
-  // If section has no symbols and no content, there are no atoms.
-  if (symbols.empty() && section.content.empty())
-    return llvm::Error::success();
-
-  if (symbols.empty()) {
-    // Section has no symbols, put all content in one anonymous atom.
-    atomFromSymbol(atomType, section, file, section.address, StringRef(),
-                  0, Atom::scopeTranslationUnit,
-                  section.address + section.content.size(),
-                  scatterable, copyRefs);
-  }
-  else if (symbols.front()->value != section.address) {
-    // Section has anonymous content before first symbol.
-    atomFromSymbol(atomType, section, file, section.address, StringRef(),
-                   0, Atom::scopeTranslationUnit, symbols.front()->value,
-                   scatterable, copyRefs);
-  }
-
-  const lld::mach_o::normalized::Symbol *lastSym = nullptr;
-  for (const lld::mach_o::normalized::Symbol *sym : symbols) {
-    if (lastSym != nullptr) {
-      // Ignore any assembler added "ltmpNNN" symbol at start of section
-      // if there is another symbol at the start.
-      if ((lastSym->value != sym->value)
-          || lastSym->value != section.address
-          || !lastSym->name.startswith("ltmp")) {
-        atomFromSymbol(atomType, section, file, lastSym->value, lastSym->name,
-                       lastSym->desc, atomScope(lastSym->scope), sym->value,
-                       scatterable, copyRefs);
-      }
-    }
-    lastSym = sym;
-  }
-  if (lastSym != nullptr) {
-    atomFromSymbol(atomType, section, file, lastSym->value, lastSym->name,
-                   lastSym->desc, atomScope(lastSym->scope),
-                   section.address + section.content.size(),
-                   scatterable, copyRefs);
-  }
-
-  // If object built without .subsections_via_symbols, add reference chain.
-  if (!scatterable) {
-    MachODefinedAtom *prevAtom = nullptr;
-    file.eachAtomInSection(section,
-                           [&](MachODefinedAtom *atom, uint64_t offset)->void {
-      if (prevAtom)
-        prevAtom->addReference(Reference::KindNamespace::all,
-                               Reference::KindArch::all,
-                               Reference::kindLayoutAfter, 0, atom, 0);
-      prevAtom = atom;
-    });
-  }
-
-  return llvm::Error::success();
-}
-
-llvm::Error processSection(DefinedAtom::ContentType atomType,
-                           const Section &section,
-                           bool customSectionName,
-                           const NormalizedFile &normalizedFile,
-                           MachOFile &file, bool scatterable,
-                           bool copyRefs) {
-  const bool is64 = MachOLinkingContext::is64Bit(normalizedFile.arch);
-  const bool isBig = MachOLinkingContext::isBigEndian(normalizedFile.arch);
-
-  // Get info on how to atomize section.
-  unsigned int       sizeMultiple;
-  DefinedAtom::Scope scope;
-  DefinedAtom::Merge merge;
-  AtomizeModel       atomizeModel;
-  sectionParseInfo(atomType, sizeMultiple, scope, merge, atomizeModel);
-
-  // Validate section size.
-  if ((section.content.size() % sizeMultiple) != 0)
-    return llvm::make_error<GenericError>(Twine("Section ")
-                                          + section.segmentName
-                                          + "/" + section.sectionName
-                                          + " has size ("
-                                          + Twine(section.content.size())
-                                          + ") which is not a multiple of "
-                                          + Twine(sizeMultiple));
-
-  if (atomizeModel == atomizeAtSymbols) {
-    // Break section up into atoms each with a fixed size.
-    return processSymboledSection(atomType, section, normalizedFile, file,
-                                  scatterable, copyRefs);
-  } else {
-    unsigned int size;
-    for (unsigned int offset = 0, e = section.content.size(); offset != e;) {
-      switch (atomizeModel) {
-      case atomizeFixedSize:
-        // Break section up into atoms each with a fixed size.
-        size = sizeMultiple;
-        break;
-      case atomizePointerSize:
-        // Break section up into atoms each the size of a pointer.
-        size = is64 ? 8 : 4;
-        break;
-      case atomizeUTF8:
-        // Break section up into zero terminated c-strings.
-        size = 0;
-        for (unsigned int i = offset; i < e; ++i) {
-          if (section.content[i] == 0) {
-            size = i + 1 - offset;
-            break;
-          }
-        }
-        break;
-      case atomizeUTF16:
-        // Break section up into zero terminated UTF16 strings.
-        size = 0;
-        for (unsigned int i = offset; i < e; i += 2) {
-          if ((section.content[i] == 0) && (section.content[i + 1] == 0)) {
-            size = i + 2 - offset;
-            break;
-          }
-        }
-        break;
-      case atomizeCFI:
-        // Break section up into dwarf unwind CFIs (FDE or CIE).
-        size = read32(&section.content[offset], isBig) + 4;
-        if (offset+size > section.content.size()) {
-          return llvm::make_error<GenericError>(Twine("Section ")
-                                                + section.segmentName
-                                                + "/" + section.sectionName
-                                                + " is malformed.  Size of CFI "
-                                                "starting at offset ("
-                                                + Twine(offset)
-                                                + ") is past end of section.");
-        }
-        break;
-      case atomizeCU:
-        // Break section up into compact unwind entries.
-        size = is64 ? 32 : 20;
-        break;
-      case atomizeCFString:
-        // Break section up into NS/CFString objects.
-        size = is64 ? 32 : 16;
-        break;
-      case atomizeAtSymbols:
-        break;
-      }
-      if (size == 0) {
-        return llvm::make_error<GenericError>(Twine("Section ")
-                                              + section.segmentName
-                                              + "/" + section.sectionName
-                                              + " is malformed.  The last atom "
-                                              "is not zero terminated.");
-      }
-      if (customSectionName) {
-        // Mach-O needs a segment and section name.  Concatenate those two
-        // with a / separator (e.g. "seg/sect") to fit into the lld model
-        // of just a section name.
-        std::string segSectName = section.segmentName.str()
-                                  + "/" + section.sectionName.str();
-        file.addDefinedAtomInCustomSection(StringRef(), scope, atomType,
-                                           merge, false, false, offset,
-                                           size, segSectName, true, &section);
-      } else {
-        file.addDefinedAtom(StringRef(), scope, atomType, merge, offset, size,
-                            false, false, copyRefs, &section);
-      }
-      offset += size;
-    }
-  }
-  return llvm::Error::success();
-}
-
-const Section* findSectionCoveringAddress(const NormalizedFile &normalizedFile,
-                                          uint64_t address) {
-  for (const Section &s : normalizedFile.sections) {
-    uint64_t sAddr = s.address;
-    if ((sAddr <= address) && (address < sAddr+s.content.size())) {
-      return &s;
-    }
-  }
-  return nullptr;
-}
-
-const MachODefinedAtom *
-findAtomCoveringAddress(const NormalizedFile &normalizedFile, MachOFile &file,
-                        uint64_t addr, Reference::Addend &addend) {
-  const Section *sect = nullptr;
-  sect = findSectionCoveringAddress(normalizedFile, addr);
-  if (!sect)
-    return nullptr;
-
-  uint32_t offsetInTarget;
-  uint64_t offsetInSect = addr - sect->address;
-  auto atom =
-      file.findAtomCoveringAddress(*sect, offsetInSect, &offsetInTarget);
-  addend = offsetInTarget;
-  return atom;
-}
-
-// Walks all relocations for a section in a normalized .o file and
-// creates corresponding lld::Reference objects.
-llvm::Error convertRelocs(const Section &section,
-                          const NormalizedFile &normalizedFile,
-                          bool scatterable,
-                          MachOFile &file,
-                          ArchHandler &handler) {
-  // Utility function for ArchHandler to find atom by its address.
-  auto atomByAddr = [&] (uint32_t sectIndex, uint64_t addr,
-                         const lld::Atom **atom, Reference::Addend *addend)
-                         -> llvm::Error {
-    if (sectIndex > normalizedFile.sections.size())
-      return llvm::make_error<GenericError>(Twine("out of range section "
-                                     "index (") + Twine(sectIndex) + ")");
-    const Section *sect = nullptr;
-    if (sectIndex == 0) {
-      sect = findSectionCoveringAddress(normalizedFile, addr);
-      if (!sect)
-        return llvm::make_error<GenericError>(Twine("address (" + Twine(addr)
-                                       + ") is not in any section"));
-    } else {
-      sect = &normalizedFile.sections[sectIndex-1];
-    }
-    uint32_t offsetInTarget;
-    uint64_t offsetInSect = addr - sect->address;
-    *atom = file.findAtomCoveringAddress(*sect, offsetInSect, &offsetInTarget);
-    *addend = offsetInTarget;
-    return llvm::Error::success();
-  };
-
-  // Utility function for ArchHandler to find atom by its symbol index.
-  auto atomBySymbol = [&] (uint32_t symbolIndex, const lld::Atom **result)
-                           -> llvm::Error {
-    // Find symbol from index.
-    const lld::mach_o::normalized::Symbol *sym = nullptr;
-    uint32_t numStabs  = normalizedFile.stabsSymbols.size();
-    uint32_t numLocal  = normalizedFile.localSymbols.size();
-    uint32_t numGlobal = normalizedFile.globalSymbols.size();
-    uint32_t numUndef  = normalizedFile.undefinedSymbols.size();
-    assert(symbolIndex >= numStabs && "Searched for stab via atomBySymbol?");
-    if (symbolIndex < numStabs+numLocal) {
-      sym = &normalizedFile.localSymbols[symbolIndex-numStabs];
-    } else if (symbolIndex < numStabs+numLocal+numGlobal) {
-      sym = &normalizedFile.globalSymbols[symbolIndex-numStabs-numLocal];
-    } else if (symbolIndex < numStabs+numLocal+numGlobal+numUndef) {
-      sym = &normalizedFile.undefinedSymbols[symbolIndex-numStabs-numLocal-
-                                             numGlobal];
-    } else {
-      return llvm::make_error<GenericError>(Twine("symbol index (")
-                                     + Twine(symbolIndex) + ") out of range");
-    }
-
-    // Find atom from symbol.
-    if ((sym->type & N_TYPE) == N_SECT) {
-      if (sym->sect > normalizedFile.sections.size())
-        return llvm::make_error<GenericError>(Twine("symbol section index (")
-                                        + Twine(sym->sect) + ") out of range ");
-      const Section &symSection = normalizedFile.sections[sym->sect-1];
-      uint64_t targetOffsetInSect = sym->value - symSection.address;
-      MachODefinedAtom *target = file.findAtomCoveringAddress(symSection,
-                                                            targetOffsetInSect);
-      if (target) {
-        *result = target;
-        return llvm::Error::success();
-      }
-      return llvm::make_error<GenericError>("no atom found for defined symbol");
-    } else if ((sym->type & N_TYPE) == N_UNDF) {
-      const lld::Atom *target = file.findUndefAtom(sym->name);
-      if (target) {
-        *result = target;
-        return llvm::Error::success();
-      }
-      return llvm::make_error<GenericError>("no undefined atom found for sym");
-    } else {
-      // Search undefs
-      return llvm::make_error<GenericError>("no atom found for symbol");
-    }
-  };
-
-  const bool isBig = MachOLinkingContext::isBigEndian(normalizedFile.arch);
-  // Use old-school iterator so that paired relocations can be grouped.
-  for (auto it=section.relocations.begin(), e=section.relocations.end();
-                                                                it != e; ++it) {
-    const Relocation &reloc = *it;
-    // Find atom this relocation is in.
-    if (reloc.offset > section.content.size())
-      return llvm::make_error<GenericError>(
-                                    Twine("r_address (") + Twine(reloc.offset)
-                                    + ") is larger than section size ("
-                                    + Twine(section.content.size()) + ")");
-    uint32_t offsetInAtom;
-    MachODefinedAtom *inAtom = file.findAtomCoveringAddress(section,
-                                                            reloc.offset,
-                                                            &offsetInAtom);
-    assert(inAtom && "r_address in range, should have found atom");
-    uint64_t fixupAddress = section.address + reloc.offset;
-
-    const lld::Atom *target = nullptr;
-    Reference::Addend addend = 0;
-    Reference::KindValue kind;
-    if (handler.isPairedReloc(reloc)) {
-      // Handle paired relocations together.
-      const Relocation &reloc2 = *++it;
-      auto relocErr = handler.getPairReferenceInfo(
-          reloc, reloc2, inAtom, offsetInAtom, fixupAddress, isBig, scatterable,
-          atomByAddr, atomBySymbol, &kind, &target, &addend);
-      if (relocErr) {
-        return handleErrors(std::move(relocErr),
-                            [&](std::unique_ptr<GenericError> GE) {
-          return llvm::make_error<GenericError>(
-            Twine("bad relocation (") + GE->getMessage()
-             + ") in section "
-             + section.segmentName + "/" + section.sectionName
-             + " (r1_address=" + Twine::utohexstr(reloc.offset)
-             + ", r1_type=" + Twine(reloc.type)
-             + ", r1_extern=" + Twine(reloc.isExtern)
-             + ", r1_length=" + Twine((int)reloc.length)
-             + ", r1_pcrel=" + Twine(reloc.pcRel)
-             + (!reloc.scattered ? (Twine(", r1_symbolnum=")
-                                    + Twine(reloc.symbol))
-                                 : (Twine(", r1_scattered=1, r1_value=")
-                                    + Twine(reloc.value)))
-             + ")"
-             + ", (r2_address=" + Twine::utohexstr(reloc2.offset)
-             + ", r2_type=" + Twine(reloc2.type)
-             + ", r2_extern=" + Twine(reloc2.isExtern)
-             + ", r2_length=" + Twine((int)reloc2.length)
-             + ", r2_pcrel=" + Twine(reloc2.pcRel)
-             + (!reloc2.scattered ? (Twine(", r2_symbolnum=")
-                                     + Twine(reloc2.symbol))
-                                  : (Twine(", r2_scattered=1, r2_value=")
-                                     + Twine(reloc2.value)))
-             + ")" );
-          });
-      }
-    }
-    else {
-      // Use ArchHandler to convert relocation record into information
-      // needed to instantiate an lld::Reference object.
-      auto relocErr = handler.getReferenceInfo(
-          reloc, inAtom, offsetInAtom, fixupAddress, isBig, atomByAddr,
-          atomBySymbol, &kind, &target, &addend);
-      if (relocErr) {
-        return handleErrors(std::move(relocErr),
-                            [&](std::unique_ptr<GenericError> GE) {
-          return llvm::make_error<GenericError>(
-            Twine("bad relocation (") + GE->getMessage()
-             + ") in section "
-             + section.segmentName + "/" + section.sectionName
-             + " (r_address=" + Twine::utohexstr(reloc.offset)
-             + ", r_type=" + Twine(reloc.type)
-             + ", r_extern=" + Twine(reloc.isExtern)
-             + ", r_length=" + Twine((int)reloc.length)
-             + ", r_pcrel=" + Twine(reloc.pcRel)
-             + (!reloc.scattered ? (Twine(", r_symbolnum=") + Twine(reloc.symbol))
-                                 : (Twine(", r_scattered=1, r_value=")
-                                    + Twine(reloc.value)))
-             + ")" );
-          });
-      }
-    }
-    // Instantiate an lld::Reference object and add to its atom.
-    inAtom->addReference(Reference::KindNamespace::mach_o,
-                         handler.kindArch(),
-                         kind, offsetInAtom, target, addend);
-  }
-
-  return llvm::Error::success();
-}
-
-bool isDebugInfoSection(const Section &section) {
-  if ((section.attributes & S_ATTR_DEBUG) == 0)
-    return false;
-  return section.segmentName.equals("__DWARF");
-}
-
-static const Atom* findDefinedAtomByName(MachOFile &file, Twine name) {
-  std::string strName = name.str();
-  for (auto *atom : file.defined())
-    if (atom->name() == strName)
-      return atom;
-  return nullptr;
-}
-
-static StringRef copyDebugString(StringRef str, BumpPtrAllocator &alloc) {
-  char *strCopy = alloc.Allocate<char>(str.size() + 1);
-  memcpy(strCopy, str.data(), str.size());
-  strCopy[str.size()] = '\0';
-  return strCopy;
-}
-
-llvm::Error parseStabs(MachOFile &file,
-                       const NormalizedFile &normalizedFile,
-                       bool copyRefs) {
-
-  if (normalizedFile.stabsSymbols.empty())
-    return llvm::Error::success();
-
-  // FIXME: Kill this off when we can move to sane yaml parsing.
-  std::unique_ptr<BumpPtrAllocator> allocator;
-  if (copyRefs)
-    allocator = std::make_unique<BumpPtrAllocator>();
-
-  enum { start, inBeginEnd } state = start;
-
-  const Atom *currentAtom = nullptr;
-  uint64_t currentAtomAddress = 0;
-  StabsDebugInfo::StabsList stabsList;
-  for (const auto &stabSym : normalizedFile.stabsSymbols) {
-    Stab stab(nullptr, stabSym.type, stabSym.sect, stabSym.desc,
-              stabSym.value, stabSym.name);
-    switch (state) {
-    case start:
-      switch (static_cast<StabType>(stabSym.type)) {
-      case N_BNSYM:
-        state = inBeginEnd;
-        currentAtomAddress = stabSym.value;
-        Reference::Addend addend;
-        currentAtom = findAtomCoveringAddress(normalizedFile, file,
-                                              currentAtomAddress, addend);
-        if (addend != 0)
-          return llvm::make_error<GenericError>(
-                   "Non-zero addend for BNSYM '" + stabSym.name + "' in " +
-                   file.path());
-        if (currentAtom)
-          stab.atom = currentAtom;
-        else {
-          // FIXME: ld64 just issues a warning here - should we match that?
-          return llvm::make_error<GenericError>(
-                   "can't find atom for stabs BNSYM at " +
-                   Twine::utohexstr(stabSym.value) + " in " + file.path());
-        }
-        break;
-      case N_SO:
-      case N_OSO:
-        // Not associated with an atom, just copy.
-        if (copyRefs)
-          stab.str = copyDebugString(stabSym.name, *allocator);
-        else
-          stab.str = stabSym.name;
-        break;
-      case N_GSYM: {
-        auto colonIdx = stabSym.name.find(':');
-        if (colonIdx != StringRef::npos) {
-          StringRef name = stabSym.name.substr(0, colonIdx);
-          currentAtom = findDefinedAtomByName(file, "_" + name);
-          stab.atom = currentAtom;
-          if (copyRefs)
-            stab.str = copyDebugString(stabSym.name, *allocator);
-          else
-            stab.str = stabSym.name;
-        } else {
-          currentAtom = findDefinedAtomByName(file, stabSym.name);
-          stab.atom = currentAtom;
-          if (copyRefs)
-            stab.str = copyDebugString(stabSym.name, *allocator);
-          else
-            stab.str = stabSym.name;
-        }
-        if (stab.atom == nullptr)
-          return llvm::make_error<GenericError>(
-                   "can't find atom for N_GSYM stabs" + stabSym.name +
-                   " in " + file.path());
-        break;
-      }
-      case N_FUN:
-        return llvm::make_error<GenericError>(
-                 "old-style N_FUN stab '" + stabSym.name + "' unsupported");
-      default:
-        return llvm::make_error<GenericError>(
-                 "unrecognized stab symbol '" + stabSym.name + "'");
-      }
-      break;
-    case inBeginEnd:
-      stab.atom = currentAtom;
-      switch (static_cast<StabType>(stabSym.type)) {
-      case N_ENSYM:
-        state = start;
-        currentAtom = nullptr;
-        break;
-      case N_FUN:
-        // Just copy the string.
-        if (copyRefs)
-          stab.str = copyDebugString(stabSym.name, *allocator);
-        else
-          stab.str = stabSym.name;
-        break;
-      default:
-        return llvm::make_error<GenericError>(
-                 "unrecognized stab symbol '" + stabSym.name + "'");
-      }
-    }
-    llvm::dbgs() << "Adding to stabsList: " << stab << "\n";
-    stabsList.push_back(stab);
-  }
-
-  file.setDebugInfo(std::make_unique<StabsDebugInfo>(std::move(stabsList)));
-
-  // FIXME: Kill this off when we fix YAML memory ownership.
-  file.debugInfo()->setAllocator(std::move(allocator));
-
-  return llvm::Error::success();
-}
-
-static llvm::DataExtractor
-dataExtractorFromSection(const NormalizedFile &normalizedFile,
-                         const Section &S) {
-  const bool is64 = MachOLinkingContext::is64Bit(normalizedFile.arch);
-  const bool isBig = MachOLinkingContext::isBigEndian(normalizedFile.arch);
-  StringRef SecData(reinterpret_cast<const char*>(S.content.data()),
-                    S.content.size());
-  return llvm::DataExtractor(SecData, !isBig, is64 ? 8 : 4);
-}
-
-// FIXME: Cribbed from llvm-dwp -- should share "lightweight CU DIE
-//        inspection" code if possible.
-static uint64_t getCUAbbrevOffset(llvm::DataExtractor abbrevData,
-                                  uint64_t abbrCode) {
-  uint64_t offset = 0;
-  while (abbrevData.getULEB128(&offset) != abbrCode) {
-    // Tag
-    abbrevData.getULEB128(&offset);
-    // DW_CHILDREN
-    abbrevData.getU8(&offset);
-    // Attributes
-    while (abbrevData.getULEB128(&offset) | abbrevData.getULEB128(&offset))
-      ;
-  }
-  return offset;
-}
-
-// FIXME: Cribbed from llvm-dwp -- should share "lightweight CU DIE
-//        inspection" code if possible.
-static Expected<const char *>
-getIndexedString(const NormalizedFile &normalizedFile,
-                 llvm::dwarf::Form form, llvm::DataExtractor infoData,
-                 uint64_t &infoOffset, const Section &stringsSection) {
-  if (form == llvm::dwarf::DW_FORM_string)
-   return infoData.getCStr(&infoOffset);
-  if (form != llvm::dwarf::DW_FORM_strp)
-    return llvm::make_error<GenericError>(
-        "string field encoded without DW_FORM_strp");
-  uint64_t stringOffset = infoData.getU32(&infoOffset);
-  llvm::DataExtractor stringsData =
-    dataExtractorFromSection(normalizedFile, stringsSection);
-  return stringsData.getCStr(&stringOffset);
-}
-
-// FIXME: Cribbed from llvm-dwp -- should share "lightweight CU DIE
-//        inspection" code if possible.
-static llvm::Expected<TranslationUnitSource>
-readCompUnit(const NormalizedFile &normalizedFile,
-             const Section &info,
-             const Section &abbrev,
-             const Section &strings,
-             StringRef path) {
-  // FIXME: Cribbed from llvm-dwp -- should share "lightweight CU DIE
-  //        inspection" code if possible.
-  uint64_t offset = 0;
-  llvm::dwarf::DwarfFormat Format = llvm::dwarf::DwarfFormat::DWARF32;
-  auto infoData = dataExtractorFromSection(normalizedFile, info);
-  uint32_t length = infoData.getU32(&offset);
-  if (length == llvm::dwarf::DW_LENGTH_DWARF64) {
-    Format = llvm::dwarf::DwarfFormat::DWARF64;
-    infoData.getU64(&offset);
-  }
-  else if (length >= llvm::dwarf::DW_LENGTH_lo_reserved)
-    return llvm::make_error<GenericError>("Malformed DWARF in " + path);
-
-  uint16_t version = infoData.getU16(&offset);
-
-  if (version < 2 || version > 4)
-    return llvm::make_error<GenericError>("Unsupported DWARF version in " +
-                                          path);
-
-  infoData.getU32(&offset); // Abbrev offset (should be zero)
-  uint8_t addrSize = infoData.getU8(&offset);
-
-  uint32_t abbrCode = infoData.getULEB128(&offset);
-  auto abbrevData = dataExtractorFromSection(normalizedFile, abbrev);
-  uint64_t abbrevOffset = getCUAbbrevOffset(abbrevData, abbrCode);
-  uint64_t tag = abbrevData.getULEB128(&abbrevOffset);
-  if (tag != llvm::dwarf::DW_TAG_compile_unit)
-    return llvm::make_error<GenericError>("top level DIE is not a compile unit");
-  // DW_CHILDREN
-  abbrevData.getU8(&abbrevOffset);
-  uint32_t name;
-  llvm::dwarf::Form form;
-  llvm::dwarf::FormParams formParams = {version, addrSize, Format};
-  TranslationUnitSource tu;
-  while ((name = abbrevData.getULEB128(&abbrevOffset)) |
-         (form = static_cast<llvm::dwarf::Form>(
-             abbrevData.getULEB128(&abbrevOffset))) &&
-         (name != 0 || form != 0)) {
-    switch (name) {
-    case llvm::dwarf::DW_AT_name: {
-      if (auto eName = getIndexedString(normalizedFile, form, infoData, offset,
-                                        strings))
-          tu.name = *eName;
-      else
-        return eName.takeError();
-      break;
-    }
-    case llvm::dwarf::DW_AT_comp_dir: {
-      if (auto eName = getIndexedString(normalizedFile, form, infoData, offset,
-                                        strings))
-        tu.path = *eName;
-      else
-        return eName.takeError();
-      break;
-    }
-    default:
-      llvm::DWARFFormValue::skipValue(form, infoData, &offset, formParams);
-    }
-  }
-  return tu;
-}
-
-llvm::Error parseDebugInfo(MachOFile &file,
-                           const NormalizedFile &normalizedFile, bool copyRefs) {
-
-  // Find the interesting debug info sections.
-  const Section *debugInfo = nullptr;
-  const Section *debugAbbrev = nullptr;
-  const Section *debugStrings = nullptr;
-
-  for (auto &s : normalizedFile.sections) {
-    if (s.segmentName == "__DWARF") {
-      if (s.sectionName == "__debug_info")
-        debugInfo = &s;
-      else if (s.sectionName == "__debug_abbrev")
-        debugAbbrev = &s;
-      else if (s.sectionName == "__debug_str")
-        debugStrings = &s;
-    }
-  }
-
-  if (!debugInfo)
-    return parseStabs(file, normalizedFile, copyRefs);
-
-  if (debugInfo->content.size() == 0)
-    return llvm::Error::success();
-
-  if (debugInfo->content.size() < 12)
-    return llvm::make_error<GenericError>("Malformed __debug_info section in " +
-                                          file.path() + ": too small");
-
-  if (!debugAbbrev)
-    return llvm::make_error<GenericError>("Missing __dwarf_abbrev section in " +
-                                          file.path());
-
-  if (auto tuOrErr = readCompUnit(normalizedFile, *debugInfo, *debugAbbrev,
-                                  *debugStrings, file.path())) {
-    // FIXME: Kill of allocator and code under 'copyRefs' when we fix YAML
-    //        memory ownership.
-    std::unique_ptr<BumpPtrAllocator> allocator;
-    if (copyRefs) {
-      allocator = std::make_unique<BumpPtrAllocator>();
-      tuOrErr->name = copyDebugString(tuOrErr->name, *allocator);
-      tuOrErr->path = copyDebugString(tuOrErr->path, *allocator);
-    }
-    file.setDebugInfo(std::make_unique<DwarfDebugInfo>(std::move(*tuOrErr)));
-    if (copyRefs)
-      file.debugInfo()->setAllocator(std::move(allocator));
-  } else
-    return tuOrErr.takeError();
-
-  return llvm::Error::success();
-}
-
-static int64_t readSPtr(bool is64, bool isBig, const uint8_t *addr) {
-  if (is64)
-    return read64(addr, isBig);
-
-  int32_t res = read32(addr, isBig);
-  return res;
-}
-
-/// --- Augmentation String Processing ---
-
-struct CIEInfo {
-  bool _augmentationDataPresent = false;
-  bool _mayHaveEH = false;
-  uint32_t _offsetOfLSDA = ~0U;
-  uint32_t _offsetOfPersonality = ~0U;
-  uint32_t _offsetOfFDEPointerEncoding = ~0U;
-  uint32_t _augmentationDataLength = ~0U;
-};
-
-typedef llvm::DenseMap<const MachODefinedAtom*, CIEInfo> CIEInfoMap;
-
-static llvm::Error processAugmentationString(const uint8_t *augStr,
-                                             CIEInfo &cieInfo,
-                                             unsigned &len) {
-
-  if (augStr[0] == '\0') {
-    len = 1;
-    return llvm::Error::success();
-  }
-
-  if (augStr[0] != 'z')
-    return llvm::make_error<GenericError>("expected 'z' at start of "
-                                          "augmentation string");
-
-  cieInfo._augmentationDataPresent = true;
-  uint64_t idx = 1;
-
-  uint32_t offsetInAugmentationData = 0;
-  while (augStr[idx] != '\0') {
-    if (augStr[idx] == 'L') {
-      cieInfo._offsetOfLSDA = offsetInAugmentationData;
-      // This adds a single byte to the augmentation data.
-      ++offsetInAugmentationData;
-      ++idx;
-      continue;
-    }
-    if (augStr[idx] == 'P') {
-      cieInfo._offsetOfPersonality = offsetInAugmentationData;
-      // This adds a single byte to the augmentation data for the encoding,
-      // then a number of bytes for the pointer data.
-      // FIXME: We are assuming 4 is correct here for the pointer size as we
-      // always currently use delta32ToGOT.
-      offsetInAugmentationData += 5;
-      ++idx;
-      continue;
-    }
-    if (augStr[idx] == 'R') {
-      cieInfo._offsetOfFDEPointerEncoding = offsetInAugmentationData;
-      // This adds a single byte to the augmentation data.
-      ++offsetInAugmentationData;
-      ++idx;
-      continue;
-    }
-    if (augStr[idx] == 'e') {
-      if (augStr[idx + 1] != 'h')
-        return llvm::make_error<GenericError>("expected 'eh' in "
-                                              "augmentation string");
-      cieInfo._mayHaveEH = true;
-      idx += 2;
-      continue;
-    }
-    ++idx;
-  }
-
-  cieInfo._augmentationDataLength = offsetInAugmentationData;
-
-  len = idx + 1;
-  return llvm::Error::success();
-}
-
-static llvm::Error processCIE(const NormalizedFile &normalizedFile,
-                              MachOFile &file,
-                              mach_o::ArchHandler &handler,
-                              const Section *ehFrameSection,
-                              MachODefinedAtom *atom,
-                              uint64_t offset,
-                              CIEInfoMap &cieInfos) {
-  const bool isBig = MachOLinkingContext::isBigEndian(normalizedFile.arch);
-  const uint8_t *frameData = atom->rawContent().data();
-
-  CIEInfo cieInfo;
-
-  uint32_t size = read32(frameData, isBig);
-  uint64_t cieIDField = size == 0xffffffffU
-                          ? sizeof(uint32_t) + sizeof(uint64_t)
-                          : sizeof(uint32_t);
-  uint64_t versionField = cieIDField + sizeof(uint32_t);
-  uint64_t augmentationStringField = versionField + sizeof(uint8_t);
-
-  unsigned augmentationStringLength = 0;
-  if (auto err = processAugmentationString(frameData + augmentationStringField,
-                                           cieInfo, augmentationStringLength))
-    return err;
-
-  if (cieInfo._offsetOfPersonality != ~0U) {
-    // If we have augmentation data for the personality function, then we may
-    // need to implicitly generate its relocation.
-
-    // Parse the EH Data field which is pointer sized.
-    uint64_t EHDataField = augmentationStringField + augmentationStringLength;
-    const bool is64 = MachOLinkingContext::is64Bit(normalizedFile.arch);
-    unsigned EHDataFieldSize = (cieInfo._mayHaveEH ? (is64 ? 8 : 4) : 0);
-
-    // Parse Code Align Factor which is a ULEB128.
-    uint64_t CodeAlignField = EHDataField + EHDataFieldSize;
-    unsigned lengthFieldSize = 0;
-    llvm::decodeULEB128(frameData + CodeAlignField, &lengthFieldSize);
-
-    // Parse Data Align Factor which is a SLEB128.
-    uint64_t DataAlignField = CodeAlignField + lengthFieldSize;
-    llvm::decodeSLEB128(frameData + DataAlignField, &lengthFieldSize);
-
-    // Parse Return Address Register which is a byte.
-    uint64_t ReturnAddressField = DataAlignField + lengthFieldSize;
-
-    // Parse the augmentation length which is a ULEB128.
-    uint64_t AugmentationLengthField = ReturnAddressField + 1;
-    uint64_t AugmentationLength =
-      llvm::decodeULEB128(frameData + AugmentationLengthField,
-                          &lengthFieldSize);
-
-    if (AugmentationLength != cieInfo._augmentationDataLength)
-      return llvm::make_error<GenericError>("CIE augmentation data length "
-                                            "mismatch");
-
-    // Get the start address of the augmentation data.
-    uint64_t AugmentationDataField = AugmentationLengthField + lengthFieldSize;
-
-    // Parse the personality function from the augmentation data.
-    uint64_t PersonalityField =
-      AugmentationDataField + cieInfo._offsetOfPersonality;
-
-    // Parse the personality encoding.
-    // FIXME: Verify that this is a 32-bit pcrel offset.
-    uint64_t PersonalityFunctionField = PersonalityField + 1;
-
-    if (atom->begin() != atom->end()) {
-      // If we have an explicit relocation, then make sure it matches this
-      // offset as this is where we'd expect it to be applied to.
-      DefinedAtom::reference_iterator CurrentRef = atom->begin();
-      if (CurrentRef->offsetInAtom() != PersonalityFunctionField)
-        return llvm::make_error<GenericError>("CIE personality reloc at "
-                                              "wrong offset");
-
-      if (++CurrentRef != atom->end())
-        return llvm::make_error<GenericError>("CIE contains too many relocs");
-    } else {
-      // Implicitly generate the personality function reloc.  It's assumed to
-      // be a delta32 offset to a GOT entry.
-      // FIXME: Parse the encoding and check this.
-      int32_t funcDelta = read32(frameData + PersonalityFunctionField, isBig);
-      uint64_t funcAddress = ehFrameSection->address + offset +
-                             PersonalityFunctionField;
-      funcAddress += funcDelta;
-
-      const MachODefinedAtom *func = nullptr;
-      Reference::Addend addend;
-      func = findAtomCoveringAddress(normalizedFile, file, funcAddress,
-                                     addend);
-      atom->addReference(Reference::KindNamespace::mach_o, handler.kindArch(),
-                         handler.unwindRefToPersonalityFunctionKind(),
-                         PersonalityFunctionField, func, addend);
-    }
-  } else if (atom->begin() != atom->end()) {
-    // Otherwise, we expect there to be no relocations in this atom as the only
-    // relocation would have been to the personality function.
-    return llvm::make_error<GenericError>("unexpected relocation in CIE");
-  }
-
-
-  cieInfos[atom] = std::move(cieInfo);
-
-  return llvm::Error::success();
-}
-
-static llvm::Error processFDE(const NormalizedFile &normalizedFile,
-                              MachOFile &file,
-                              mach_o::ArchHandler &handler,
-                              const Section *ehFrameSection,
-                              MachODefinedAtom *atom,
-                              uint64_t offset,
-                              const CIEInfoMap &cieInfos) {
-
-  const bool isBig = MachOLinkingContext::isBigEndian(normalizedFile.arch);
-  const bool is64 = MachOLinkingContext::is64Bit(normalizedFile.arch);
-
-  // Compiler wasn't lazy and actually told us what it meant.
-  // Unfortunately, the compiler may not have generated references for all of
-  // [cie, func, lsda] and so we still need to parse the FDE and add references
-  // for any the compiler didn't generate.
-  if (atom->begin() != atom->end())
-    atom->sortReferences();
-
-  DefinedAtom::reference_iterator CurrentRef = atom->begin();
-
-  // This helper returns the reference (if one exists) at the offset we are
-  // currently processing.  It automatically increments the ref iterator if we
-  // do return a ref, and throws an error if we pass over a ref without
-  // comsuming it.
-  auto currentRefGetter = [&CurrentRef,
-                           &atom](uint64_t Offset)->const Reference* {
-    // If there are no more refs found, then we are done.
-    if (CurrentRef == atom->end())
-      return nullptr;
-
-    const Reference *Ref = *CurrentRef;
-
-    // If we haven't reached the offset for this reference, then return that
-    // we don't yet have a reference to process.
-    if (Offset < Ref->offsetInAtom())
-      return nullptr;
-
-    // If the offset is equal, then we want to process this ref.
-    if (Offset == Ref->offsetInAtom()) {
-      ++CurrentRef;
-      return Ref;
-    }
-
-    // The current ref is at an offset which is earlier than the current
-    // offset, then we failed to consume it when we should have.  In this case
-    // throw an error.
-    llvm::report_fatal_error("Skipped reference when processing FDE");
-  };
-
-  // Helper to either get the reference at this current location, and verify
-  // that it is of the expected type, or add a reference of that type.
-  // Returns the reference target.
-  auto verifyOrAddReference = [&](uint64_t targetAddress,
-                                  Reference::KindValue refKind,
-                                  uint64_t refAddress,
-                                  bool allowsAddend)->const Atom* {
-    if (auto *ref = currentRefGetter(refAddress)) {
-      // The compiler already emitted a relocation for the CIE ref.  This should
-      // have been converted to the correct type of reference in
-      // get[Pair]ReferenceInfo().
-      assert(ref->kindValue() == refKind &&
-             "Incorrect EHFrame reference kind");
-      return ref->target();
-    }
-    Reference::Addend addend;
-    auto *target = findAtomCoveringAddress(normalizedFile, file,
-                                           targetAddress, addend);
-    atom->addReference(Reference::KindNamespace::mach_o, handler.kindArch(),
-                       refKind, refAddress, target, addend);
-
-    if (!allowsAddend)
-      assert(!addend && "EHFrame reference cannot have addend");
-    return target;
-  };
-
-  const uint8_t *startFrameData = atom->rawContent().data();
-  const uint8_t *frameData = startFrameData;
-
-  uint32_t size = read32(frameData, isBig);
-  uint64_t cieFieldInFDE = size == 0xffffffffU
-    ? sizeof(uint32_t) + sizeof(uint64_t)
-    : sizeof(uint32_t);
-
-  // Linker needs to fixup a reference from the FDE to its parent CIE (a
-  // 32-bit byte offset backwards in the __eh_frame section).
-  uint32_t cieDelta = read32(frameData + cieFieldInFDE, isBig);
-  uint64_t cieAddress = ehFrameSection->address + offset + cieFieldInFDE;
-  cieAddress -= cieDelta;
-
-  auto *cieRefTarget = verifyOrAddReference(cieAddress,
-                                            handler.unwindRefToCIEKind(),
-                                            cieFieldInFDE, false);
-  const MachODefinedAtom *cie = dyn_cast<MachODefinedAtom>(cieRefTarget);
-  assert(cie && cie->contentType() == DefinedAtom::typeCFI &&
-         "FDE's CIE field does not point at the start of a CIE.");
-
-  const CIEInfo &cieInfo = cieInfos.find(cie)->second;
-
-  // Linker needs to fixup reference from the FDE to the function it's
-  // describing. FIXME: there are actually different ways to do this, and the
-  // particular method used is specified in the CIE's augmentation fields
-  // (hopefully)
-  uint64_t rangeFieldInFDE = cieFieldInFDE + sizeof(uint32_t);
-
-  int64_t functionFromFDE = readSPtr(is64, isBig,
-                                     frameData + rangeFieldInFDE);
-  uint64_t rangeStart = ehFrameSection->address + offset + rangeFieldInFDE;
-  rangeStart += functionFromFDE;
-
-  verifyOrAddReference(rangeStart,
-                       handler.unwindRefToFunctionKind(),
-                       rangeFieldInFDE, true);
-
-  // Handle the augmentation data if there is any.
-  if (cieInfo._augmentationDataPresent) {
-    // First process the augmentation data length field.
-    uint64_t augmentationDataLengthFieldInFDE =
-      rangeFieldInFDE + 2 * (is64 ? sizeof(uint64_t) : sizeof(uint32_t));
-    unsigned lengthFieldSize = 0;
-    uint64_t augmentationDataLength =
-      llvm::decodeULEB128(frameData + augmentationDataLengthFieldInFDE,
-                          &lengthFieldSize);
-
-    if (cieInfo._offsetOfLSDA != ~0U && augmentationDataLength > 0) {
-
-      // Look at the augmentation data field.
-      uint64_t augmentationDataFieldInFDE =
-        augmentationDataLengthFieldInFDE + lengthFieldSize;
-
-      int64_t lsdaFromFDE = readSPtr(is64, isBig,
-                                     frameData + augmentationDataFieldInFDE);
-      uint64_t lsdaStart =
-        ehFrameSection->address + offset + augmentationDataFieldInFDE +
-        lsdaFromFDE;
-
-      verifyOrAddReference(lsdaStart,
-                           handler.unwindRefToFunctionKind(),
-                           augmentationDataFieldInFDE, true);
-    }
-  }
-
-  return llvm::Error::success();
-}
-
-llvm::Error addEHFrameReferences(const NormalizedFile &normalizedFile,
-                                 MachOFile &file,
-                                 mach_o::ArchHandler &handler) {
-
-  const Section *ehFrameSection = nullptr;
-  for (auto &section : normalizedFile.sections)
-    if (section.segmentName == "__TEXT" &&
-        section.sectionName == "__eh_frame") {
-      ehFrameSection = &section;
-      break;
-    }
-
-  // No __eh_frame so nothing to do.
-  if (!ehFrameSection)
-    return llvm::Error::success();
-
-  llvm::Error ehFrameErr = llvm::Error::success();
-  CIEInfoMap cieInfos;
-
-  file.eachAtomInSection(*ehFrameSection,
-                         [&](MachODefinedAtom *atom, uint64_t offset) -> void {
-    assert(atom->contentType() == DefinedAtom::typeCFI);
-
-    // Bail out if we've encountered an error.
-    if (ehFrameErr)
-      return;
-
-    const bool isBig = MachOLinkingContext::isBigEndian(normalizedFile.arch);
-    if (ArchHandler::isDwarfCIE(isBig, atom))
-      ehFrameErr = processCIE(normalizedFile, file, handler, ehFrameSection,
-                              atom, offset, cieInfos);
-    else
-      ehFrameErr = processFDE(normalizedFile, file, handler, ehFrameSection,
-                              atom, offset, cieInfos);
-  });
-
-  return ehFrameErr;
-}
-
-llvm::Error parseObjCImageInfo(const Section &sect,
-                               const NormalizedFile &normalizedFile,
-                               MachOFile &file) {
-
-  //	struct objc_image_info  {
-  //		uint32_t	version;	// initially 0
-  //		uint32_t	flags;
-  //	};
-
-  ArrayRef<uint8_t> content = sect.content;
-  if (content.size() != 8)
-    return llvm::make_error<GenericError>(sect.segmentName + "/" +
-                                          sect.sectionName +
-                                          " in file " + file.path() +
-                                          " should be 8 bytes in size");
-
-  const bool isBig = MachOLinkingContext::isBigEndian(normalizedFile.arch);
-  uint32_t version = read32(content.data(), isBig);
-  if (version)
-    return llvm::make_error<GenericError>(sect.segmentName + "/" +
-                                          sect.sectionName +
-                                          " in file " + file.path() +
-                                          " should have version=0");
-
-  uint32_t flags = read32(content.data() + 4, isBig);
-  if (flags & (MachOLinkingContext::objc_supports_gc |
-               MachOLinkingContext::objc_gc_only))
-    return llvm::make_error<GenericError>(sect.segmentName + "/" +
-                                          sect.sectionName +
-                                          " in file " + file.path() +
-                                          " uses GC.  This is not supported");
-
-  if (flags & MachOLinkingContext::objc_retainReleaseForSimulator)
-    file.setObjcConstraint(MachOLinkingContext::objc_retainReleaseForSimulator);
-  else
-    file.setObjcConstraint(MachOLinkingContext::objc_retainRelease);
-
-  file.setSwiftVersion((flags >> 8) & 0xFF);
-
-  return llvm::Error::success();
-}
-
-/// Converts normalized mach-o file into an lld::File and lld::Atoms.
-llvm::Expected<std::unique_ptr<lld::File>>
-objectToAtoms(const NormalizedFile &normalizedFile, StringRef path,
-              bool copyRefs) {
-  auto file = std::make_unique<MachOFile>(path);
-  if (auto ec = normalizedObjectToAtoms(file.get(), normalizedFile, copyRefs))
-    return std::move(ec);
-  return std::unique_ptr<File>(std::move(file));
-}
-
-llvm::Expected<std::unique_ptr<lld::File>>
-dylibToAtoms(const NormalizedFile &normalizedFile, StringRef path,
-             bool copyRefs) {
-  // Instantiate SharedLibraryFile object.
-  auto file = std::make_unique<MachODylibFile>(path);
-  if (auto ec = normalizedDylibToAtoms(file.get(), normalizedFile, copyRefs))
-    return std::move(ec);
-  return std::unique_ptr<File>(std::move(file));
-}
-
-} // anonymous namespace
-
-namespace normalized {
-
-static bool isObjCImageInfo(const Section &sect) {
-  return (sect.segmentName == "__OBJC" && sect.sectionName == "__image_info") ||
-    (sect.segmentName == "__DATA" && sect.sectionName == "__objc_imageinfo");
-}
-
-llvm::Error
-normalizedObjectToAtoms(MachOFile *file,
-                        const NormalizedFile &normalizedFile,
-                        bool copyRefs) {
-  LLVM_DEBUG(llvm::dbgs() << "******** Normalizing file to atoms: "
-                          << file->path() << "\n");
-  bool scatterable = ((normalizedFile.flags & MH_SUBSECTIONS_VIA_SYMBOLS) != 0);
-
-  // Create atoms from each section.
-  for (auto &sect : normalizedFile.sections) {
-
-    // If this is a debug-info section parse it specially.
-    if (isDebugInfoSection(sect))
-      continue;
-
-    // If the file contains an objc_image_info struct, then we should parse the
-    // ObjC flags and Swift version.
-    if (isObjCImageInfo(sect)) {
-      if (auto ec = parseObjCImageInfo(sect, normalizedFile, *file))
-        return ec;
-      // We then skip adding atoms for this section as we use the ObjCPass to
-      // re-emit this data after it has been aggregated for all files.
-      continue;
-    }
-
-    bool customSectionName;
-    DefinedAtom::ContentType atomType = atomTypeFromSection(sect,
-                                                            customSectionName);
-    if (auto ec =  processSection(atomType, sect, customSectionName,
-                                  normalizedFile, *file, scatterable, copyRefs))
-      return ec;
-  }
-  // Create atoms from undefined symbols.
-  for (auto &sym : normalizedFile.undefinedSymbols) {
-    // Undefined symbols with n_value != 0 are actually tentative definitions.
-    if (sym.value == Hex64(0)) {
-      file->addUndefinedAtom(sym.name, copyRefs);
-    } else {
-      file->addTentativeDefAtom(sym.name, atomScope(sym.scope), sym.value,
-                                DefinedAtom::Alignment(1 << (sym.desc >> 8)),
-                                copyRefs);
-    }
-  }
-
-  // Convert mach-o relocations to References
-  std::unique_ptr<mach_o::ArchHandler> handler
-                                     = ArchHandler::create(normalizedFile.arch);
-  for (auto &sect : normalizedFile.sections) {
-    if (isDebugInfoSection(sect))
-      continue;
-    if (llvm::Error ec = convertRelocs(sect, normalizedFile, scatterable,
-                                       *file, *handler))
-      return ec;
-  }
-
-  // Add additional arch-specific References
-  file->eachDefinedAtom([&](MachODefinedAtom* atom) -> void {
-    handler->addAdditionalReferences(*atom);
-  });
-
-  // Each __eh_frame section needs references to both __text (the function we're
-  // providing unwind info for) and itself (FDE -> CIE). These aren't
-  // represented in the relocations on some architectures, so we have to add
-  // them back in manually there.
-  if (auto ec = addEHFrameReferences(normalizedFile, *file, *handler))
-    return ec;
-
-  // Process mach-o data-in-code regions array. That information is encoded in
-  // atoms as References at each transition point.
-  unsigned nextIndex = 0;
-  for (const DataInCode &entry : normalizedFile.dataInCode) {
-    ++nextIndex;
-    const Section* s = findSectionCoveringAddress(normalizedFile, entry.offset);
-    if (!s) {
-      return llvm::make_error<GenericError>(Twine("LC_DATA_IN_CODE address ("
-                                                  + Twine(entry.offset)
-                                                  + ") is not in any section"));
-    }
-    uint64_t offsetInSect = entry.offset - s->address;
-    uint32_t offsetInAtom;
-    MachODefinedAtom *atom = file->findAtomCoveringAddress(*s, offsetInSect,
-                                                           &offsetInAtom);
-    if (offsetInAtom + entry.length > atom->size()) {
-      return llvm::make_error<GenericError>(Twine("LC_DATA_IN_CODE entry "
-                                                  "(offset="
-                                                  + Twine(entry.offset)
-                                                  + ", length="
-                                                  + Twine(entry.length)
-                                                  + ") crosses atom boundary."));
-    }
-    // Add reference that marks start of data-in-code.
-    atom->addReference(Reference::KindNamespace::mach_o, handler->kindArch(),
-                       handler->dataInCodeTransitionStart(*atom),
-                       offsetInAtom, atom, entry.kind);
-
-    // Peek at next entry, if it starts where this one ends, skip ending ref.
-    if (nextIndex < normalizedFile.dataInCode.size()) {
-      const DataInCode &nextEntry = normalizedFile.dataInCode[nextIndex];
-      if (nextEntry.offset == (entry.offset + entry.length))
-        continue;
-    }
-
-    // If data goes to end of function, skip ending ref.
-    if ((offsetInAtom + entry.length) == atom->size())
-      continue;
-
-    // Add reference that marks end of data-in-code.
-    atom->addReference(Reference::KindNamespace::mach_o, handler->kindArch(),
-                       handler->dataInCodeTransitionEnd(*atom),
-                       offsetInAtom+entry.length, atom, 0);
-  }
-
-  // Cache some attributes on the file for use later.
-  file->setFlags(normalizedFile.flags);
-  file->setArch(normalizedFile.arch);
-  file->setOS(normalizedFile.os);
-  file->setMinVersion(normalizedFile.minOSverson);
-  file->setMinVersionLoadCommandKind(normalizedFile.minOSVersionKind);
-
-  // Sort references in each atom to their canonical order.
-  for (const DefinedAtom* defAtom : file->defined()) {
-    reinterpret_cast<const SimpleDefinedAtom*>(defAtom)->sortReferences();
-  }
-
-  if (auto err = parseDebugInfo(*file, normalizedFile, copyRefs))
-    return err;
-
-  return llvm::Error::success();
-}
-
-llvm::Error
-normalizedDylibToAtoms(MachODylibFile *file,
-                       const NormalizedFile &normalizedFile,
-                       bool copyRefs) {
-  file->setInstallName(normalizedFile.installName);
-  file->setCompatVersion(normalizedFile.compatVersion);
-  file->setCurrentVersion(normalizedFile.currentVersion);
-
-  // Tell MachODylibFile object about all symbols it exports.
-  if (!normalizedFile.exportInfo.empty()) {
-    // If exports trie exists, use it instead of traditional symbol table.
-    for (const Export &exp : normalizedFile.exportInfo) {
-      bool weakDef = (exp.flags & EXPORT_SYMBOL_FLAGS_WEAK_DEFINITION);
-      // StringRefs from export iterator are ephemeral, so force copy.
-      file->addExportedSymbol(exp.name, weakDef, true);
-    }
-  } else {
-    for (auto &sym : normalizedFile.globalSymbols) {
-      assert((sym.scope & N_EXT) && "only expect external symbols here");
-      bool weakDef = (sym.desc & N_WEAK_DEF);
-      file->addExportedSymbol(sym.name, weakDef, copyRefs);
-    }
-  }
-  // Tell MachODylibFile object about all dylibs it re-exports.
-  for (const DependentDylib &dep : normalizedFile.dependentDylibs) {
-    if (dep.kind == llvm::MachO::LC_REEXPORT_DYLIB)
-      file->addReExportedDylib(dep.path);
-  }
-  return llvm::Error::success();
-}
-
-void relocatableSectionInfoForContentType(DefinedAtom::ContentType atomType,
-                                          StringRef &segmentName,
-                                          StringRef &sectionName,
-                                          SectionType &sectionType,
-                                          SectionAttr &sectionAttrs,
-                                          bool &relocsToDefinedCanBeImplicit) {
-
-  for (const MachORelocatableSectionToAtomType *p = sectsToAtomType ;
-                                 p->atomType != DefinedAtom::typeUnknown; ++p) {
-    if (p->atomType != atomType)
-      continue;
-    // Wild carded entries are ignored for reverse lookups.
-    if (p->segmentName.empty() || p->sectionName.empty())
-      continue;
-    segmentName = p->segmentName;
-    sectionName = p->sectionName;
-    sectionType = p->sectionType;
-    sectionAttrs = 0;
-    relocsToDefinedCanBeImplicit = false;
-    if (atomType == DefinedAtom::typeCode)
-      sectionAttrs = S_ATTR_PURE_INSTRUCTIONS;
-    if (atomType == DefinedAtom::typeCFI)
-      relocsToDefinedCanBeImplicit = true;
-    return;
-  }
-  llvm_unreachable("content type not yet supported");
-}
-
-llvm::Expected<std::unique_ptr<lld::File>>
-normalizedToAtoms(const NormalizedFile &normalizedFile, StringRef path,
-                  bool copyRefs) {
-  switch (normalizedFile.fileType) {
-  case MH_DYLIB:
-  case MH_DYLIB_STUB:
-    return dylibToAtoms(normalizedFile, path, copyRefs);
-  case MH_OBJECT:
-    return objectToAtoms(normalizedFile, path, copyRefs);
-  default:
-    llvm_unreachable("unhandled MachO file type!");
-  }
-}
-
-} // namespace normalized
-} // namespace mach_o
-} // namespace lld
diff --git a/lld/lib/ReaderWriter/MachO/MachONormalizedFileYAML.cpp b/lld/lib/ReaderWriter/MachO/MachONormalizedFileYAML.cpp
deleted file mode 100644
index 3826e97d62b9..000000000000
--- a/lld/lib/ReaderWriter/MachO/MachONormalizedFileYAML.cpp
+++ /dev/null
@@ -1,840 +0,0 @@
-//===- lib/ReaderWriter/MachO/MachONormalizedFileYAML.cpp -----------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-///
-/// \file For mach-o object files, this implementation uses YAML I/O to
-/// provide the convert between YAML and the normalized mach-o (NM).
-///
-///                  +------------+         +------+
-///                  | normalized |   <->   | yaml |
-///                  +------------+         +------+
-
-#include "MachONormalizedFile.h"
-#include "lld/Common/LLVM.h"
-#include "lld/Core/Error.h"
-#include "lld/ReaderWriter/YamlContext.h"
-#include "llvm/ADT/SmallString.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/ADT/StringSwitch.h"
-#include "llvm/ADT/Twine.h"
-#include "llvm/BinaryFormat/MachO.h"
-#include "llvm/Support/Casting.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/Format.h"
-#include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/SourceMgr.h"
-#include "llvm/Support/YAMLTraits.h"
-#include "llvm/Support/raw_ostream.h"
-#include <system_error>
-
-using llvm::StringRef;
-using namespace llvm::yaml;
-using namespace llvm::MachO;
-using namespace lld::mach_o::normalized;
-using lld::YamlContext;
-
-LLVM_YAML_IS_SEQUENCE_VECTOR(Segment)
-LLVM_YAML_IS_SEQUENCE_VECTOR(DependentDylib)
-LLVM_YAML_IS_SEQUENCE_VECTOR(RebaseLocation)
-LLVM_YAML_IS_SEQUENCE_VECTOR(BindLocation)
-LLVM_YAML_IS_SEQUENCE_VECTOR(Export)
-LLVM_YAML_IS_SEQUENCE_VECTOR(DataInCode)
-
-
-// for compatibility with gcc-4.7 in C++11 mode, add extra namespace
-namespace llvm {
-namespace yaml {
-
-// A vector of Sections is a sequence.
-template<>
-struct SequenceTraits< std::vector<Section> > {
-  static size_t size(IO &io, std::vector<Section> &seq) {
-    return seq.size();
-  }
-  static Section& element(IO &io, std::vector<Section> &seq, size_t index) {
-    if ( index >= seq.size() )
-      seq.resize(index+1);
-    return seq[index];
-  }
-};
-
-template<>
-struct SequenceTraits< std::vector<Symbol> > {
-  static size_t size(IO &io, std::vector<Symbol> &seq) {
-    return seq.size();
-  }
-  static Symbol& element(IO &io, std::vector<Symbol> &seq, size_t index) {
-    if ( index >= seq.size() )
-      seq.resize(index+1);
-    return seq[index];
-  }
-};
-
-// A vector of Relocations is a sequence.
-template<>
-struct SequenceTraits< Relocations > {
-  static size_t size(IO &io, Relocations &seq) {
-    return seq.size();
-  }
-  static Relocation& element(IO &io, Relocations &seq, size_t index) {
-    if ( index >= seq.size() )
-      seq.resize(index+1);
-    return seq[index];
-  }
-};
-
-// The content for a section is represented as a flow sequence of hex bytes.
-template<>
-struct SequenceTraits< ContentBytes > {
-  static size_t size(IO &io, ContentBytes &seq) {
-    return seq.size();
-  }
-  static Hex8& element(IO &io, ContentBytes &seq, size_t index) {
-    if ( index >= seq.size() )
-      seq.resize(index+1);
-    return seq[index];
-  }
-  static const bool flow = true;
-};
-
-// The indirect symbols for a section is represented as a flow sequence
-// of numbers (symbol table indexes).
-template<>
-struct SequenceTraits< IndirectSymbols > {
-  static size_t size(IO &io, IndirectSymbols &seq) {
-    return seq.size();
-  }
-  static uint32_t& element(IO &io, IndirectSymbols &seq, size_t index) {
-    if ( index >= seq.size() )
-      seq.resize(index+1);
-    return seq[index];
-  }
-  static const bool flow = true;
-};
-
-template <>
-struct ScalarEnumerationTraits<lld::MachOLinkingContext::Arch> {
-  static void enumeration(IO &io, lld::MachOLinkingContext::Arch &value) {
-    io.enumCase(value, "unknown",lld::MachOLinkingContext::arch_unknown);
-    io.enumCase(value, "ppc",    lld::MachOLinkingContext::arch_ppc);
-    io.enumCase(value, "x86",    lld::MachOLinkingContext::arch_x86);
-    io.enumCase(value, "x86_64", lld::MachOLinkingContext::arch_x86_64);
-    io.enumCase(value, "armv6",  lld::MachOLinkingContext::arch_armv6);
-    io.enumCase(value, "armv7",  lld::MachOLinkingContext::arch_armv7);
-    io.enumCase(value, "armv7s", lld::MachOLinkingContext::arch_armv7s);
-    io.enumCase(value, "arm64",  lld::MachOLinkingContext::arch_arm64);
-  }
-};
-
-template <>
-struct ScalarEnumerationTraits<lld::MachOLinkingContext::OS> {
-  static void enumeration(IO &io, lld::MachOLinkingContext::OS &value) {
-    io.enumCase(value, "unknown",
-                          lld::MachOLinkingContext::OS::unknown);
-    io.enumCase(value, "Mac OS X",
-                          lld::MachOLinkingContext::OS::macOSX);
-    io.enumCase(value, "iOS",
-                          lld::MachOLinkingContext::OS::iOS);
-    io.enumCase(value, "iOS Simulator",
-                          lld::MachOLinkingContext::OS::iOS_simulator);
-  }
-};
-
-
-template <>
-struct ScalarEnumerationTraits<HeaderFileType> {
-  static void enumeration(IO &io, HeaderFileType &value) {
-    io.enumCase(value, "MH_OBJECT",   llvm::MachO::MH_OBJECT);
-    io.enumCase(value, "MH_DYLIB",    llvm::MachO::MH_DYLIB);
-    io.enumCase(value, "MH_EXECUTE",  llvm::MachO::MH_EXECUTE);
-    io.enumCase(value, "MH_BUNDLE",   llvm::MachO::MH_BUNDLE);
-  }
-};
-
-
-template <>
-struct ScalarBitSetTraits<FileFlags> {
-  static void bitset(IO &io, FileFlags &value) {
-    io.bitSetCase(value, "MH_TWOLEVEL",
-                          llvm::MachO::MH_TWOLEVEL);
-    io.bitSetCase(value, "MH_SUBSECTIONS_VIA_SYMBOLS",
-                          llvm::MachO::MH_SUBSECTIONS_VIA_SYMBOLS);
-  }
-};
-
-
-template <>
-struct ScalarEnumerationTraits<SectionType> {
-  static void enumeration(IO &io, SectionType &value) {
-    io.enumCase(value, "S_REGULAR",
-                        llvm::MachO::S_REGULAR);
-    io.enumCase(value, "S_ZEROFILL",
-                        llvm::MachO::S_ZEROFILL);
-    io.enumCase(value, "S_CSTRING_LITERALS",
-                        llvm::MachO::S_CSTRING_LITERALS);
-    io.enumCase(value, "S_4BYTE_LITERALS",
-                        llvm::MachO::S_4BYTE_LITERALS);
-    io.enumCase(value, "S_8BYTE_LITERALS",
-                        llvm::MachO::S_8BYTE_LITERALS);
-    io.enumCase(value, "S_LITERAL_POINTERS",
-                        llvm::MachO::S_LITERAL_POINTERS);
-    io.enumCase(value, "S_NON_LAZY_SYMBOL_POINTERS",
-                        llvm::MachO::S_NON_LAZY_SYMBOL_POINTERS);
-    io.enumCase(value, "S_LAZY_SYMBOL_POINTERS",
-                        llvm::MachO::S_LAZY_SYMBOL_POINTERS);
-    io.enumCase(value, "S_SYMBOL_STUBS",
-                        llvm::MachO::S_SYMBOL_STUBS);
-    io.enumCase(value, "S_MOD_INIT_FUNC_POINTERS",
-                        llvm::MachO::S_MOD_INIT_FUNC_POINTERS);
-    io.enumCase(value, "S_MOD_TERM_FUNC_POINTERS",
-                        llvm::MachO::S_MOD_TERM_FUNC_POINTERS);
-    io.enumCase(value, "S_COALESCED",
-                        llvm::MachO::S_COALESCED);
-    io.enumCase(value, "S_GB_ZEROFILL",
-                        llvm::MachO::S_GB_ZEROFILL);
-    io.enumCase(value, "S_INTERPOSING",
-                        llvm::MachO::S_INTERPOSING);
-    io.enumCase(value, "S_16BYTE_LITERALS",
-                        llvm::MachO::S_16BYTE_LITERALS);
-    io.enumCase(value, "S_DTRACE_DOF",
-                        llvm::MachO::S_DTRACE_DOF);
-    io.enumCase(value, "S_LAZY_DYLIB_SYMBOL_POINTERS",
-                        llvm::MachO::S_LAZY_DYLIB_SYMBOL_POINTERS);
-    io.enumCase(value, "S_THREAD_LOCAL_REGULAR",
-                        llvm::MachO::S_THREAD_LOCAL_REGULAR);
-    io.enumCase(value, "S_THREAD_LOCAL_ZEROFILL",
-                        llvm::MachO::S_THREAD_LOCAL_ZEROFILL);
-    io.enumCase(value, "S_THREAD_LOCAL_VARIABLES",
-                        llvm::MachO::S_THREAD_LOCAL_VARIABLES);
-    io.enumCase(value, "S_THREAD_LOCAL_VARIABLE_POINTERS",
-                        llvm::MachO::S_THREAD_LOCAL_VARIABLE_POINTERS);
-    io.enumCase(value, "S_THREAD_LOCAL_INIT_FUNCTION_POINTERS",
-                        llvm::MachO::S_THREAD_LOCAL_INIT_FUNCTION_POINTERS);
-  }
-};
-
-template <>
-struct ScalarBitSetTraits<SectionAttr> {
-  static void bitset(IO &io, SectionAttr &value) {
-    io.bitSetCase(value, "S_ATTR_PURE_INSTRUCTIONS",
-                          llvm::MachO::S_ATTR_PURE_INSTRUCTIONS);
-    io.bitSetCase(value, "S_ATTR_SOME_INSTRUCTIONS",
-                          llvm::MachO::S_ATTR_SOME_INSTRUCTIONS);
-    io.bitSetCase(value, "S_ATTR_NO_DEAD_STRIP",
-                          llvm::MachO::S_ATTR_NO_DEAD_STRIP);
-    io.bitSetCase(value, "S_ATTR_EXT_RELOC",
-                          llvm::MachO::S_ATTR_EXT_RELOC);
-    io.bitSetCase(value, "S_ATTR_LOC_RELOC",
-                          llvm::MachO::S_ATTR_LOC_RELOC);
-    io.bitSetCase(value, "S_ATTR_DEBUG",
-                         llvm::MachO::S_ATTR_DEBUG);
-  }
-};
-
-/// This is a custom formatter for SectionAlignment.  Values are
-/// the power to raise by, ie, the n in 2^n.
-template <> struct ScalarTraits<SectionAlignment> {
-  static void output(const SectionAlignment &value, void *ctxt,
-                     raw_ostream &out) {
-    out << llvm::format("%d", (uint32_t)value);
-  }
-
-  static StringRef input(StringRef scalar, void *ctxt,
-                         SectionAlignment &value) {
-    uint32_t alignment;
-    if (scalar.getAsInteger(0, alignment)) {
-      return "malformed alignment value";
-    }
-    if (!llvm::isPowerOf2_32(alignment))
-      return "alignment must be a power of 2";
-    value = alignment;
-    return StringRef(); // returning empty string means success
-  }
-
-  static QuotingType mustQuote(StringRef) { return QuotingType::None; }
-};
-
-template <>
-struct ScalarEnumerationTraits<NListType> {
-  static void enumeration(IO &io, NListType &value) {
-    io.enumCase(value, "N_UNDF",  llvm::MachO::N_UNDF);
-    io.enumCase(value, "N_ABS",   llvm::MachO::N_ABS);
-    io.enumCase(value, "N_SECT",  llvm::MachO::N_SECT);
-    io.enumCase(value, "N_PBUD",  llvm::MachO::N_PBUD);
-    io.enumCase(value, "N_INDR",  llvm::MachO::N_INDR);
-  }
-};
-
-template <>
-struct ScalarBitSetTraits<SymbolScope> {
-  static void bitset(IO &io, SymbolScope &value) {
-    io.bitSetCase(value, "N_EXT",   llvm::MachO::N_EXT);
-    io.bitSetCase(value, "N_PEXT",  llvm::MachO::N_PEXT);
-  }
-};
-
-template <>
-struct ScalarBitSetTraits<SymbolDesc> {
-  static void bitset(IO &io, SymbolDesc &value) {
-    io.bitSetCase(value, "N_NO_DEAD_STRIP",   llvm::MachO::N_NO_DEAD_STRIP);
-    io.bitSetCase(value, "N_WEAK_REF",        llvm::MachO::N_WEAK_REF);
-    io.bitSetCase(value, "N_WEAK_DEF",        llvm::MachO::N_WEAK_DEF);
-    io.bitSetCase(value, "N_ARM_THUMB_DEF",   llvm::MachO::N_ARM_THUMB_DEF);
-    io.bitSetCase(value, "N_SYMBOL_RESOLVER", llvm::MachO::N_SYMBOL_RESOLVER);
-  }
-};
-
-
-template <>
-struct MappingTraits<Section> {
-  struct NormalizedContentBytes;
-  static void mapping(IO &io, Section &sect) {
-    io.mapRequired("segment",         sect.segmentName);
-    io.mapRequired("section",         sect.sectionName);
-    io.mapRequired("type",            sect.type);
-    io.mapOptional("attributes",      sect.attributes);
-    io.mapOptional("alignment",       sect.alignment, (SectionAlignment)1);
-    io.mapRequired("address",         sect.address);
-    if (isZeroFillSection(sect.type)) {
-      // S_ZEROFILL sections use "size:" instead of "content:"
-      uint64_t size = sect.content.size();
-      io.mapOptional("size",          size);
-      if (!io.outputting()) {
-        uint8_t *bytes = nullptr;
-        sect.content = makeArrayRef(bytes, size);
-      }
-    } else {
-      MappingNormalization<NormalizedContent, ArrayRef<uint8_t>> content(
-        io, sect.content);
-      io.mapOptional("content",         content->_normalizedContent);
-    }
-    io.mapOptional("relocations",     sect.relocations);
-    io.mapOptional("indirect-syms",   sect.indirectSymbols);
-  }
-
-  struct NormalizedContent {
-    NormalizedContent(IO &io) : _io(io) {}
-    NormalizedContent(IO &io, ArrayRef<uint8_t> content) : _io(io) {
-      // When writing yaml, copy content byte array to Hex8 vector.
-      for (auto &c : content) {
-        _normalizedContent.push_back(c);
-      }
-    }
-    ArrayRef<uint8_t> denormalize(IO &io) {
-      // When reading yaml, allocate byte array owned by NormalizedFile and
-      // copy Hex8 vector to byte array.
-      YamlContext *info = reinterpret_cast<YamlContext *>(io.getContext());
-      assert(info != nullptr);
-      NormalizedFile *file = info->_normalizeMachOFile;
-      assert(file != nullptr);
-      size_t size = _normalizedContent.size();
-      if (!size)
-        return None;
-      uint8_t *bytes = file->ownedAllocations.Allocate<uint8_t>(size);
-      std::copy(_normalizedContent.begin(), _normalizedContent.end(), bytes);
-      return makeArrayRef(bytes, size);
-    }
-
-    IO                &_io;
-    ContentBytes       _normalizedContent;
-  };
-};
-
-
-template <>
-struct MappingTraits<Relocation> {
-  static void mapping(IO &io, Relocation &reloc) {
-    io.mapRequired("offset",    reloc.offset);
-    io.mapOptional("scattered", reloc.scattered, false);
-    io.mapRequired("type",      reloc.type);
-    io.mapRequired("length",    reloc.length);
-    io.mapRequired("pc-rel",    reloc.pcRel);
-    if ( !reloc.scattered )
-     io.mapRequired("extern",   reloc.isExtern);
-    if ( reloc.scattered )
-     io.mapRequired("value",    reloc.value);
-    if ( !reloc.scattered )
-     io.mapRequired("symbol",   reloc.symbol);
-  }
-};
-
-
-template <>
-struct ScalarEnumerationTraits<RelocationInfoType> {
-  static void enumeration(IO &io, RelocationInfoType &value) {
-    YamlContext *info = reinterpret_cast<YamlContext *>(io.getContext());
-    assert(info != nullptr);
-    NormalizedFile *file = info->_normalizeMachOFile;
-    assert(file != nullptr);
-    switch (file->arch) {
-    case lld::MachOLinkingContext::arch_x86_64:
-      io.enumCase(value, "X86_64_RELOC_UNSIGNED",
-                                  llvm::MachO::X86_64_RELOC_UNSIGNED);
-      io.enumCase(value, "X86_64_RELOC_SIGNED",
-                                  llvm::MachO::X86_64_RELOC_SIGNED);
-      io.enumCase(value, "X86_64_RELOC_BRANCH",
-                                  llvm::MachO::X86_64_RELOC_BRANCH);
-      io.enumCase(value, "X86_64_RELOC_GOT_LOAD",
-                                  llvm::MachO::X86_64_RELOC_GOT_LOAD);
-      io.enumCase(value, "X86_64_RELOC_GOT",
-                                  llvm::MachO::X86_64_RELOC_GOT);
-      io.enumCase(value, "X86_64_RELOC_SUBTRACTOR",
-                                  llvm::MachO::X86_64_RELOC_SUBTRACTOR);
-      io.enumCase(value, "X86_64_RELOC_SIGNED_1",
-                                  llvm::MachO::X86_64_RELOC_SIGNED_1);
-      io.enumCase(value, "X86_64_RELOC_SIGNED_2",
-                                  llvm::MachO::X86_64_RELOC_SIGNED_2);
-      io.enumCase(value, "X86_64_RELOC_SIGNED_4",
-                                  llvm::MachO::X86_64_RELOC_SIGNED_4);
-      io.enumCase(value, "X86_64_RELOC_TLV",
-                                  llvm::MachO::X86_64_RELOC_TLV);
-      break;
-    case lld::MachOLinkingContext::arch_x86:
-      io.enumCase(value, "GENERIC_RELOC_VANILLA",
-                                  llvm::MachO::GENERIC_RELOC_VANILLA);
-      io.enumCase(value, "GENERIC_RELOC_PAIR",
-                                  llvm::MachO::GENERIC_RELOC_PAIR);
-      io.enumCase(value, "GENERIC_RELOC_SECTDIFF",
-                                  llvm::MachO::GENERIC_RELOC_SECTDIFF);
-      io.enumCase(value, "GENERIC_RELOC_LOCAL_SECTDIFF",
-                                  llvm::MachO::GENERIC_RELOC_LOCAL_SECTDIFF);
-      io.enumCase(value, "GENERIC_RELOC_TLV",
-                                  llvm::MachO::GENERIC_RELOC_TLV);
-      break;
-    case lld::MachOLinkingContext::arch_armv6:
-    case lld::MachOLinkingContext::arch_armv7:
-    case lld::MachOLinkingContext::arch_armv7s:
-       io.enumCase(value, "ARM_RELOC_VANILLA",
-                                  llvm::MachO::ARM_RELOC_VANILLA);
-      io.enumCase(value, "ARM_RELOC_PAIR",
-                                  llvm::MachO::ARM_RELOC_PAIR);
-      io.enumCase(value, "ARM_RELOC_SECTDIFF",
-                                  llvm::MachO::ARM_RELOC_SECTDIFF);
-      io.enumCase(value, "ARM_RELOC_LOCAL_SECTDIFF",
-                                  llvm::MachO::ARM_RELOC_LOCAL_SECTDIFF);
-      io.enumCase(value, "ARM_RELOC_BR24",
-                                  llvm::MachO::ARM_RELOC_BR24);
-      io.enumCase(value, "ARM_THUMB_RELOC_BR22",
-                                  llvm::MachO::ARM_THUMB_RELOC_BR22);
-      io.enumCase(value, "ARM_RELOC_HALF",
-                                  llvm::MachO::ARM_RELOC_HALF);
-      io.enumCase(value, "ARM_RELOC_HALF_SECTDIFF",
-                                  llvm::MachO::ARM_RELOC_HALF_SECTDIFF);
-      break;
-    case lld::MachOLinkingContext::arch_arm64:
-      io.enumCase(value, "ARM64_RELOC_UNSIGNED",
-                                  llvm::MachO::ARM64_RELOC_UNSIGNED);
-      io.enumCase(value, "ARM64_RELOC_SUBTRACTOR",
-                                  llvm::MachO::ARM64_RELOC_SUBTRACTOR);
-      io.enumCase(value, "ARM64_RELOC_BRANCH26",
-                                  llvm::MachO::ARM64_RELOC_BRANCH26);
-      io.enumCase(value, "ARM64_RELOC_PAGE21",
-                                  llvm::MachO::ARM64_RELOC_PAGE21);
-      io.enumCase(value, "ARM64_RELOC_PAGEOFF12",
-                                  llvm::MachO::ARM64_RELOC_PAGEOFF12);
-      io.enumCase(value, "ARM64_RELOC_GOT_LOAD_PAGE21",
-                                  llvm::MachO::ARM64_RELOC_GOT_LOAD_PAGE21);
-      io.enumCase(value, "ARM64_RELOC_GOT_LOAD_PAGEOFF12",
-                                  llvm::MachO::ARM64_RELOC_GOT_LOAD_PAGEOFF12);
-      io.enumCase(value, "ARM64_RELOC_POINTER_TO_GOT",
-                                  llvm::MachO::ARM64_RELOC_POINTER_TO_GOT);
-      io.enumCase(value, "ARM64_RELOC_TLVP_LOAD_PAGE21",
-                                  llvm::MachO::ARM64_RELOC_TLVP_LOAD_PAGE21);
-      io.enumCase(value, "ARM64_RELOC_TLVP_LOAD_PAGEOFF12",
-                                  llvm::MachO::ARM64_RELOC_TLVP_LOAD_PAGEOFF12);
-      io.enumCase(value, "ARM64_RELOC_ADDEND",
-                                  llvm::MachO::ARM64_RELOC_ADDEND);
-      break;
-    default:
-      llvm_unreachable("unknown architecture");
-    }
- }
-};
-
-
-template <>
-struct MappingTraits<Symbol> {
-  static void mapping(IO &io, Symbol& sym) {
-    io.mapRequired("name",    sym.name);
-    io.mapRequired("type",    sym.type);
-    io.mapOptional("scope",   sym.scope, SymbolScope(0));
-    io.mapOptional("sect",    sym.sect, (uint8_t)0);
-    if (sym.type == llvm::MachO::N_UNDF) {
-      // In undef symbols, desc field contains alignment/ordinal info
-      // which is better represented as a hex vaule.
-      uint16_t t1 = sym.desc;
-      Hex16 t2 = t1;
-      io.mapOptional("desc",  t2, Hex16(0));
-      sym.desc = t2;
-    } else {
-      // In defined symbols, desc fit is a set of option bits.
-      io.mapOptional("desc",    sym.desc, SymbolDesc(0));
-    }
-    io.mapRequired("value",  sym.value);
-  }
-};
-
-// Custom mapping for VMProtect (e.g. "r-x").
-template <>
-struct ScalarTraits<VMProtect> {
-  static void output(const VMProtect &value, void*, raw_ostream &out) {
-    out << ( (value & llvm::MachO::VM_PROT_READ)    ? 'r' : '-');
-    out << ( (value & llvm::MachO::VM_PROT_WRITE)   ? 'w' : '-');
-    out << ( (value & llvm::MachO::VM_PROT_EXECUTE) ? 'x' : '-');
-  }
-  static StringRef input(StringRef scalar, void*, VMProtect &value) {
-    value = 0;
-    if (scalar.size() != 3)
-      return "segment access protection must be three chars (e.g. \"r-x\")";
-    switch (scalar[0]) {
-    case 'r':
-      value = llvm::MachO::VM_PROT_READ;
-      break;
-    case '-':
-      break;
-    default:
-      return "segment access protection first char must be 'r' or '-'";
-    }
-    switch (scalar[1]) {
-    case 'w':
-      value = value | llvm::MachO::VM_PROT_WRITE;
-      break;
-    case '-':
-      break;
-    default:
-      return "segment access protection second char must be 'w' or '-'";
-    }
-    switch (scalar[2]) {
-    case 'x':
-      value = value | llvm::MachO::VM_PROT_EXECUTE;
-      break;
-    case '-':
-      break;
-    default:
-      return "segment access protection third char must be 'x' or '-'";
-    }
-    // Return the empty string on success,
-    return StringRef();
-  }
-  static QuotingType mustQuote(StringRef) { return QuotingType::None; }
-};
-
-
-template <>
-struct MappingTraits<Segment> {
-  static void mapping(IO &io, Segment& seg) {
-    io.mapRequired("name",            seg.name);
-    io.mapRequired("address",         seg.address);
-    io.mapRequired("size",            seg.size);
-    io.mapRequired("init-access",     seg.init_access);
-    io.mapRequired("max-access",      seg.max_access);
-  }
-};
-
-template <>
-struct ScalarEnumerationTraits<LoadCommandType> {
-  static void enumeration(IO &io, LoadCommandType &value) {
-    io.enumCase(value, "LC_LOAD_DYLIB",
-                        llvm::MachO::LC_LOAD_DYLIB);
-    io.enumCase(value, "LC_LOAD_WEAK_DYLIB",
-                        llvm::MachO::LC_LOAD_WEAK_DYLIB);
-    io.enumCase(value, "LC_REEXPORT_DYLIB",
-                        llvm::MachO::LC_REEXPORT_DYLIB);
-    io.enumCase(value, "LC_LOAD_UPWARD_DYLIB",
-                        llvm::MachO::LC_LOAD_UPWARD_DYLIB);
-    io.enumCase(value, "LC_LAZY_LOAD_DYLIB",
-                        llvm::MachO::LC_LAZY_LOAD_DYLIB);
-    io.enumCase(value, "LC_VERSION_MIN_MACOSX",
-                        llvm::MachO::LC_VERSION_MIN_MACOSX);
-    io.enumCase(value, "LC_VERSION_MIN_IPHONEOS",
-                        llvm::MachO::LC_VERSION_MIN_IPHONEOS);
-    io.enumCase(value, "LC_VERSION_MIN_TVOS",
-                        llvm::MachO::LC_VERSION_MIN_TVOS);
-    io.enumCase(value, "LC_VERSION_MIN_WATCHOS",
-                        llvm::MachO::LC_VERSION_MIN_WATCHOS);
-  }
-};
-
-template <>
-struct MappingTraits<DependentDylib> {
-  static void mapping(IO &io, DependentDylib& dylib) {
-    io.mapRequired("path",            dylib.path);
-    io.mapOptional("kind",            dylib.kind,
-                                      llvm::MachO::LC_LOAD_DYLIB);
-    io.mapOptional("compat-version",  dylib.compatVersion,
-                                      PackedVersion(0x10000));
-    io.mapOptional("current-version", dylib.currentVersion,
-                                      PackedVersion(0x10000));
-  }
-};
-
-template <>
-struct ScalarEnumerationTraits<RebaseType> {
-  static void enumeration(IO &io, RebaseType &value) {
-    io.enumCase(value, "REBASE_TYPE_POINTER",
-                        llvm::MachO::REBASE_TYPE_POINTER);
-    io.enumCase(value, "REBASE_TYPE_TEXT_PCREL32",
-                        llvm::MachO::REBASE_TYPE_TEXT_PCREL32);
-    io.enumCase(value, "REBASE_TYPE_TEXT_ABSOLUTE32",
-                        llvm::MachO::REBASE_TYPE_TEXT_ABSOLUTE32);
-  }
-};
-
-
-template <>
-struct MappingTraits<RebaseLocation> {
-  static void mapping(IO &io, RebaseLocation& rebase) {
-    io.mapRequired("segment-index",   rebase.segIndex);
-    io.mapRequired("segment-offset",  rebase.segOffset);
-    io.mapOptional("kind",            rebase.kind,
-                                      llvm::MachO::REBASE_TYPE_POINTER);
-  }
-};
-
-
-
-template <>
-struct ScalarEnumerationTraits<BindType> {
-  static void enumeration(IO &io, BindType &value) {
-    io.enumCase(value, "BIND_TYPE_POINTER",
-                        llvm::MachO::BIND_TYPE_POINTER);
-    io.enumCase(value, "BIND_TYPE_TEXT_ABSOLUTE32",
-                        llvm::MachO::BIND_TYPE_TEXT_ABSOLUTE32);
-    io.enumCase(value, "BIND_TYPE_TEXT_PCREL32",
-                        llvm::MachO::BIND_TYPE_TEXT_PCREL32);
-  }
-};
-
-template <>
-struct MappingTraits<BindLocation> {
-  static void mapping(IO &io, BindLocation &bind) {
-    io.mapRequired("segment-index",   bind.segIndex);
-    io.mapRequired("segment-offset",  bind.segOffset);
-    io.mapOptional("kind",            bind.kind,
-                                      llvm::MachO::BIND_TYPE_POINTER);
-    io.mapOptional("can-be-null",     bind.canBeNull, false);
-    io.mapRequired("ordinal",         bind.ordinal);
-    io.mapRequired("symbol-name",     bind.symbolName);
-    io.mapOptional("addend",          bind.addend, Hex64(0));
-  }
-};
-
-
-template <>
-struct ScalarEnumerationTraits<ExportSymbolKind> {
-  static void enumeration(IO &io, ExportSymbolKind &value) {
-    io.enumCase(value, "EXPORT_SYMBOL_FLAGS_KIND_REGULAR",
-                        llvm::MachO::EXPORT_SYMBOL_FLAGS_KIND_REGULAR);
-    io.enumCase(value, "EXPORT_SYMBOL_FLAGS_KIND_THREAD_LOCAL",
-                        llvm::MachO::EXPORT_SYMBOL_FLAGS_KIND_THREAD_LOCAL);
-    io.enumCase(value, "EXPORT_SYMBOL_FLAGS_KIND_ABSOLUTE",
-                        llvm::MachO::EXPORT_SYMBOL_FLAGS_KIND_ABSOLUTE);
-  }
-};
-
-template <>
-struct ScalarBitSetTraits<ExportFlags> {
-  static void bitset(IO &io, ExportFlags &value) {
-    io.bitSetCase(value, "EXPORT_SYMBOL_FLAGS_WEAK_DEFINITION",
-                          llvm::MachO::EXPORT_SYMBOL_FLAGS_WEAK_DEFINITION);
-    io.bitSetCase(value, "EXPORT_SYMBOL_FLAGS_REEXPORT",
-                          llvm::MachO::EXPORT_SYMBOL_FLAGS_REEXPORT);
-    io.bitSetCase(value, "EXPORT_SYMBOL_FLAGS_STUB_AND_RESOLVER",
-                          llvm::MachO::EXPORT_SYMBOL_FLAGS_STUB_AND_RESOLVER);
-  }
-};
-
-
-template <>
-struct MappingTraits<Export> {
-  static void mapping(IO &io, Export &exp) {
-    io.mapRequired("name",         exp.name);
-    io.mapOptional("offset",       exp.offset);
-    io.mapOptional("kind",         exp.kind,
-                                llvm::MachO::EXPORT_SYMBOL_FLAGS_KIND_REGULAR);
-    if (!io.outputting() || exp.flags)
-      io.mapOptional("flags",      exp.flags);
-    io.mapOptional("other",        exp.otherOffset, Hex32(0));
-    io.mapOptional("other-name",   exp.otherName, StringRef());
-  }
-};
-
-template <>
-struct ScalarEnumerationTraits<DataRegionType> {
-  static void enumeration(IO &io, DataRegionType &value) {
-    io.enumCase(value, "DICE_KIND_DATA",
-                        llvm::MachO::DICE_KIND_DATA);
-    io.enumCase(value, "DICE_KIND_JUMP_TABLE8",
-                        llvm::MachO::DICE_KIND_JUMP_TABLE8);
-    io.enumCase(value, "DICE_KIND_JUMP_TABLE16",
-                        llvm::MachO::DICE_KIND_JUMP_TABLE16);
-    io.enumCase(value, "DICE_KIND_JUMP_TABLE32",
-                        llvm::MachO::DICE_KIND_JUMP_TABLE32);
-    io.enumCase(value, "DICE_KIND_ABS_JUMP_TABLE32",
-                        llvm::MachO::DICE_KIND_ABS_JUMP_TABLE32);
-  }
-};
-
-template <>
-struct MappingTraits<DataInCode> {
-  static void mapping(IO &io, DataInCode &entry) {
-    io.mapRequired("offset",       entry.offset);
-    io.mapRequired("length",       entry.length);
-    io.mapRequired("kind",         entry.kind);
-  }
-};
-
-template <>
-struct ScalarTraits<PackedVersion> {
-  static void output(const PackedVersion &value, void*, raw_ostream &out) {
-    out << llvm::format("%d.%d", (value >> 16), (value >> 8) & 0xFF);
-    if (value & 0xFF) {
-      out << llvm::format(".%d", (value & 0xFF));
-    }
-  }
-  static StringRef input(StringRef scalar, void*, PackedVersion &result) {
-    uint32_t value;
-    if (lld::MachOLinkingContext::parsePackedVersion(scalar, value))
-      return "malformed version number";
-    result = value;
-    // Return the empty string on success,
-    return StringRef();
-  }
-  static QuotingType mustQuote(StringRef) { return QuotingType::None; }
-};
-
-template <>
-struct MappingTraits<NormalizedFile> {
-  static void mapping(IO &io, NormalizedFile &file) {
-    io.mapRequired("arch",             file.arch);
-    io.mapRequired("file-type",        file.fileType);
-    io.mapOptional("flags",            file.flags);
-    io.mapOptional("dependents",       file.dependentDylibs);
-    io.mapOptional("install-name",     file.installName,    StringRef());
-    io.mapOptional("compat-version",   file.compatVersion,  PackedVersion(0x10000));
-    io.mapOptional("current-version",  file.currentVersion, PackedVersion(0x10000));
-    io.mapOptional("has-UUID",         file.hasUUID,        true);
-    io.mapOptional("rpaths",           file.rpaths);
-    io.mapOptional("entry-point",      file.entryAddress,   Hex64(0));
-    io.mapOptional("stack-size",       file.stackSize,      Hex64(0));
-    io.mapOptional("source-version",   file.sourceVersion,  Hex64(0));
-    io.mapOptional("OS",               file.os);
-    io.mapOptional("min-os-version",   file.minOSverson,    PackedVersion(0));
-    io.mapOptional("min-os-version-kind",   file.minOSVersionKind, (LoadCommandType)0);
-    io.mapOptional("sdk-version",      file.sdkVersion,     PackedVersion(0));
-    io.mapOptional("segments",         file.segments);
-    io.mapOptional("sections",         file.sections);
-    io.mapOptional("local-symbols",    file.localSymbols);
-    io.mapOptional("global-symbols",   file.globalSymbols);
-    io.mapOptional("undefined-symbols",file.undefinedSymbols);
-    io.mapOptional("page-size",        file.pageSize,       Hex32(4096));
-    io.mapOptional("rebasings",        file.rebasingInfo);
-    io.mapOptional("bindings",         file.bindingInfo);
-    io.mapOptional("weak-bindings",    file.weakBindingInfo);
-    io.mapOptional("lazy-bindings",    file.lazyBindingInfo);
-    io.mapOptional("exports",          file.exportInfo);
-    io.mapOptional("dataInCode",       file.dataInCode);
-  }
-  static std::string validate(IO &io, NormalizedFile &file) { return {}; }
-};
-
-} // namespace llvm
-} // namespace yaml
-
-
-namespace lld {
-namespace mach_o {
-
-/// Handles !mach-o tagged yaml documents.
-bool MachOYamlIOTaggedDocumentHandler::handledDocTag(llvm::yaml::IO &io,
-                                                 const lld::File *&file) const {
-  if (!io.mapTag("!mach-o"))
-    return false;
-  // Step 1: parse yaml into normalized mach-o struct.
-  NormalizedFile nf;
-  YamlContext *info = reinterpret_cast<YamlContext *>(io.getContext());
-  assert(info != nullptr);
-  assert(info->_normalizeMachOFile == nullptr);
-  info->_normalizeMachOFile = &nf;
-  MappingTraits<NormalizedFile>::mapping(io, nf);
-  // Step 2: parse normalized mach-o struct into atoms.
-  auto fileOrError = normalizedToAtoms(nf, info->_path, true);
-
-  // Check that we parsed successfully.
-  if (!fileOrError) {
-    std::string buffer;
-    llvm::raw_string_ostream stream(buffer);
-    handleAllErrors(fileOrError.takeError(),
-                    [&](const llvm::ErrorInfoBase &EI) {
-      EI.log(stream);
-      stream << "\n";
-    });
-    io.setError(stream.str());
-    return false;
-  }
-
-  if (nf.arch != _arch) {
-    io.setError(Twine("file is wrong architecture. Expected ("
-                      + MachOLinkingContext::nameFromArch(_arch)
-                      + ") found ("
-                      + MachOLinkingContext::nameFromArch(nf.arch)
-                      + ")"));
-    return false;
-  }
-  info->_normalizeMachOFile = nullptr;
-  file = fileOrError->release();
-  return true;
-}
-
-
-
-namespace normalized {
-
-/// Parses a yaml encoded mach-o file to produce an in-memory normalized view.
-llvm::Expected<std::unique_ptr<NormalizedFile>>
-readYaml(std::unique_ptr<MemoryBuffer> &mb) {
-  // Make empty NormalizedFile.
-  std::unique_ptr<NormalizedFile> f(new NormalizedFile());
-
-  // Create YAML Input parser.
-  YamlContext yamlContext;
-  yamlContext._normalizeMachOFile = f.get();
-  llvm::yaml::Input yin(mb->getBuffer(), &yamlContext);
-
-  // Fill NormalizedFile by parsing yaml.
-  yin >> *f;
-
-  // Return error if there were parsing problems.
-  if (auto ec = yin.error())
-    return llvm::make_error<GenericError>(Twine("YAML parsing error: ")
-                                          + ec.message());
-
-  // Hand ownership of instantiated NormalizedFile to caller.
-  return std::move(f);
-}
-
-
-/// Writes a yaml encoded mach-o files from an in-memory normalized view.
-std::error_code writeYaml(const NormalizedFile &file, raw_ostream &out) {
-  // YAML I/O is not const aware, so need to cast away ;-(
-  NormalizedFile *f = const_cast<NormalizedFile*>(&file);
-
-  // Create yaml Output writer, using yaml options for context.
-  YamlContext yamlContext;
-  yamlContext._normalizeMachOFile = f;
-  llvm::yaml::Output yout(out, &yamlContext);
-
-  // Stream out yaml.
-  yout << *f;
-
-  return std::error_code();
-}
-
-} // namespace normalized
-} // namespace mach_o
-} // namespace lld
diff --git a/lld/lib/ReaderWriter/MachO/MachOPasses.h b/lld/lib/ReaderWriter/MachO/MachOPasses.h
deleted file mode 100644
index 93cd3e4df281..000000000000
--- a/lld/lib/ReaderWriter/MachO/MachOPasses.h
+++ /dev/null
@@ -1,29 +0,0 @@
-//===- lib/ReaderWriter/MachO/MachOPasses.h -------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLD_READER_WRITER_MACHO_PASSES_H
-#define LLD_READER_WRITER_MACHO_PASSES_H
-
-#include "lld/Core/PassManager.h"
-#include "lld/ReaderWriter/MachOLinkingContext.h"
-
-namespace lld {
-namespace mach_o {
-
-void addLayoutPass(PassManager &pm, const MachOLinkingContext &ctx);
-void addStubsPass(PassManager &pm, const MachOLinkingContext &ctx);
-void addGOTPass(PassManager &pm, const MachOLinkingContext &ctx);
-void addTLVPass(PassManager &pm, const MachOLinkingContext &ctx);
-void addCompactUnwindPass(PassManager &pm, const MachOLinkingContext &ctx);
-void addObjCPass(PassManager &pm, const MachOLinkingContext &ctx);
-void addShimPass(PassManager &pm, const MachOLinkingContext &ctx);
-
-} // namespace mach_o
-} // namespace lld
-
-#endif // LLD_READER_WRITER_MACHO_PASSES_H
diff --git a/lld/lib/ReaderWriter/MachO/ObjCPass.cpp b/lld/lib/ReaderWriter/MachO/ObjCPass.cpp
deleted file mode 100644
index 02a95b5aa0c0..000000000000
--- a/lld/lib/ReaderWriter/MachO/ObjCPass.cpp
+++ /dev/null
@@ -1,131 +0,0 @@
-//===- lib/ReaderWriter/MachO/ObjCPass.cpp -------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-//===----------------------------------------------------------------------===//
-
-#include "ArchHandler.h"
-#include "File.h"
-#include "MachONormalizedFileBinaryUtils.h"
-#include "MachOPasses.h"
-#include "lld/Common/LLVM.h"
-#include "lld/Core/DefinedAtom.h"
-#include "lld/Core/File.h"
-#include "lld/Core/Reference.h"
-#include "lld/Core/Simple.h"
-#include "lld/ReaderWriter/MachOLinkingContext.h"
-#include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/STLExtras.h"
-
-namespace lld {
-namespace mach_o {
-
-///
-/// ObjC Image Info Atom created by the ObjC pass.
-///
-class ObjCImageInfoAtom : public SimpleDefinedAtom {
-public:
-  ObjCImageInfoAtom(const File &file, bool isBig,
-                    MachOLinkingContext::ObjCConstraint objCConstraint,
-                    uint32_t swiftVersion)
-      : SimpleDefinedAtom(file) {
-
-    Data.info.version = 0;
-
-    switch (objCConstraint) {
-    case MachOLinkingContext::objc_unknown:
-      llvm_unreachable("Shouldn't run the objc pass without a constraint");
-    case MachOLinkingContext::objc_supports_gc:
-    case MachOLinkingContext::objc_gc_only:
-      llvm_unreachable("GC is not supported");
-    case MachOLinkingContext::objc_retainReleaseForSimulator:
-      // The retain/release for simulator flag is already the correct
-      // encoded value for the data so just set it here.
-      Data.info.flags = (uint32_t)objCConstraint;
-      break;
-    case MachOLinkingContext::objc_retainRelease:
-      // We don't need to encode this flag, so just leave the flags as 0.
-      Data.info.flags = 0;
-      break;
-    }
-
-    Data.info.flags |= (swiftVersion << 8);
-
-    normalized::write32(Data.bytes + 4, Data.info.flags, isBig);
-  }
-
-  ~ObjCImageInfoAtom() override = default;
-
-  ContentType contentType() const override {
-    return DefinedAtom::typeObjCImageInfo;
-  }
-
-  Alignment alignment() const override {
-    return 4;
-  }
-
-  uint64_t size() const override {
-    return 8;
-  }
-
-  ContentPermissions permissions() const override {
-    return DefinedAtom::permR__;
-  }
-
-  ArrayRef<uint8_t> rawContent() const override {
-    return llvm::makeArrayRef(Data.bytes, size());
-  }
-
-private:
-
-  struct objc_image_info  {
-    uint32_t	version;
-    uint32_t	flags;
-  };
-
-  union {
-    objc_image_info info;
-    uint8_t bytes[8];
-  } Data;
-};
-
-class ObjCPass : public Pass {
-public:
-  ObjCPass(const MachOLinkingContext &context)
-      : _ctx(context),
-        _file(*_ctx.make_file<MachOFile>("<mach-o objc pass>")) {
-    _file.setOrdinal(_ctx.getNextOrdinalAndIncrement());
-  }
-
-  llvm::Error perform(SimpleFile &mergedFile) override {
-    // Add the image info.
-    mergedFile.addAtom(*getImageInfo());
-
-    return llvm::Error::success();
-  }
-
-private:
-
-  const DefinedAtom* getImageInfo() {
-    bool IsBig = MachOLinkingContext::isBigEndian(_ctx.arch());
-    return new (_file.allocator()) ObjCImageInfoAtom(_file, IsBig,
-                                                     _ctx.objcConstraint(),
-                                                     _ctx.swiftVersion());
-  }
-
-  const MachOLinkingContext   &_ctx;
-  MachOFile                   &_file;
-};
-
-
-
-void addObjCPass(PassManager &pm, const MachOLinkingContext &ctx) {
-  pm.add(std::make_unique<ObjCPass>(ctx));
-}
-
-} // end namespace mach_o
-} // end namespace lld
diff --git a/lld/lib/ReaderWriter/MachO/SectCreateFile.h b/lld/lib/ReaderWriter/MachO/SectCreateFile.h
deleted file mode 100644
index 7bb98e16695c..000000000000
--- a/lld/lib/ReaderWriter/MachO/SectCreateFile.h
+++ /dev/null
@@ -1,101 +0,0 @@
-//===---- lib/ReaderWriter/MachO/SectCreateFile.h ---------------*- c++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLD_READER_WRITER_MACHO_SECTCREATE_FILE_H
-#define LLD_READER_WRITER_MACHO_SECTCREATE_FILE_H
-
-#include "lld/Core/DefinedAtom.h"
-#include "lld/Core/Simple.h"
-#include "lld/ReaderWriter/MachOLinkingContext.h"
-
-namespace lld {
-namespace mach_o {
-
-//
-// A FlateNamespaceFile instance may be added as a resolution source of last
-// resort, depending on how -flat_namespace and -undefined are set.
-//
-class SectCreateFile : public File {
-public:
-  class SectCreateAtom : public SimpleDefinedAtom {
-  public:
-    SectCreateAtom(const File &file, StringRef segName, StringRef sectName,
-                   std::unique_ptr<MemoryBuffer> content)
-      : SimpleDefinedAtom(file),
-        _combinedName((segName + "/" + sectName).str()),
-        _content(std::move(content)) {}
-
-    ~SectCreateAtom() override = default;
-
-    uint64_t size() const override { return _content->getBufferSize(); }
-
-    Scope scope() const override { return scopeGlobal; }
-
-    ContentType contentType() const override { return typeSectCreate; }
-
-    SectionChoice sectionChoice() const override { return sectionCustomRequired; }
-
-    StringRef customSectionName() const override { return _combinedName; }
-
-    DeadStripKind deadStrip() const override { return deadStripNever; }
-
-    ArrayRef<uint8_t> rawContent() const override {
-      const uint8_t *data =
-        reinterpret_cast<const uint8_t*>(_content->getBufferStart());
-      return ArrayRef<uint8_t>(data, _content->getBufferSize());
-    }
-
-    StringRef segmentName() const { return _segName; }
-    StringRef sectionName() const { return _sectName; }
-
-  private:
-    std::string _combinedName;
-    StringRef _segName;
-    StringRef _sectName;
-    std::unique_ptr<MemoryBuffer> _content;
-  };
-
-  SectCreateFile() : File("sectcreate", kindSectCreateObject) {}
-
-  void addSection(StringRef seg, StringRef sect,
-                  std::unique_ptr<MemoryBuffer> content) {
-    _definedAtoms.push_back(
-      new (allocator()) SectCreateAtom(*this, seg, sect, std::move(content)));
-  }
-
-  const AtomRange<DefinedAtom> defined() const override {
-    return _definedAtoms;
-  }
-
-  const AtomRange<UndefinedAtom> undefined() const override {
-    return _noUndefinedAtoms;
-  }
-
-  const AtomRange<SharedLibraryAtom> sharedLibrary() const override {
-    return _noSharedLibraryAtoms;
-  }
-
-  const AtomRange<AbsoluteAtom> absolute() const override {
-    return _noAbsoluteAtoms;
-  }
-
-  void clearAtoms() override {
-    _definedAtoms.clear();
-    _noUndefinedAtoms.clear();
-    _noSharedLibraryAtoms.clear();
-    _noAbsoluteAtoms.clear();
-  }
-
-private:
-  AtomVector<DefinedAtom> _definedAtoms;
-};
-
-} // namespace mach_o
-} // namespace lld
-
-#endif // LLD_READER_WRITER_MACHO_SECTCREATE_FILE_H
diff --git a/lld/lib/ReaderWriter/MachO/ShimPass.cpp b/lld/lib/ReaderWriter/MachO/ShimPass.cpp
deleted file mode 100644
index 4c62ef9d330f..000000000000
--- a/lld/lib/ReaderWriter/MachO/ShimPass.cpp
+++ /dev/null
@@ -1,128 +0,0 @@
-//===- lib/ReaderWriter/MachO/ShimPass.cpp -------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This linker pass updates branch-sites whose target is a different mode
-// (thumb vs arm).
-//
-// Arm code has two instruction encodings thumb and arm.  When branching from
-// one code encoding to another, you need to use an instruction that switches
-// the instruction mode.  Usually the transition only happens at call sites, and
-// the linker can transform a BL instruction in BLX (or vice versa).  But if the
-// compiler did a tail call optimization and a function ends with a branch (not
-// branch and link), there is no pc-rel BX instruction.
-//
-// The ShimPass looks for pc-rel B instructions that will need to switch mode.
-// For those cases it synthesizes a shim which does the transition, then
-// modifies the original atom with the B instruction to target to the shim atom.
-//
-//===----------------------------------------------------------------------===//
-
-#include "ArchHandler.h"
-#include "File.h"
-#include "MachOPasses.h"
-#include "lld/Common/LLVM.h"
-#include "lld/Core/DefinedAtom.h"
-#include "lld/Core/File.h"
-#include "lld/Core/Reference.h"
-#include "lld/Core/Simple.h"
-#include "lld/ReaderWriter/MachOLinkingContext.h"
-#include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/STLExtras.h"
-
-namespace lld {
-namespace mach_o {
-
-class ShimPass : public Pass {
-public:
-  ShimPass(const MachOLinkingContext &context)
-      : _ctx(context), _archHandler(_ctx.archHandler()),
-        _stubInfo(_archHandler.stubInfo()),
-        _file(*_ctx.make_file<MachOFile>("<mach-o shim pass>")) {
-    _file.setOrdinal(_ctx.getNextOrdinalAndIncrement());
-  }
-
-  llvm::Error perform(SimpleFile &mergedFile) override {
-    // Scan all references in all atoms.
-    for (const DefinedAtom *atom : mergedFile.defined()) {
-      for (const Reference *ref : *atom) {
-        // Look at non-call branches.
-        if (!_archHandler.isNonCallBranch(*ref))
-          continue;
-        const Atom *target = ref->target();
-        assert(target != nullptr);
-        if (const lld::DefinedAtom *daTarget = dyn_cast<DefinedAtom>(target)) {
-          bool atomIsThumb = _archHandler.isThumbFunction(*atom);
-          bool targetIsThumb = _archHandler.isThumbFunction(*daTarget);
-          if (atomIsThumb != targetIsThumb)
-            updateBranchToUseShim(atomIsThumb, *daTarget, ref);
-        }
-      }
-    }
-    // Exit early if no shims needed.
-    if (_targetToShim.empty())
-      return llvm::Error::success();
-
-    // Sort shim atoms so the layout order is stable.
-    std::vector<const DefinedAtom *> shims;
-    shims.reserve(_targetToShim.size());
-    for (auto element : _targetToShim) {
-      shims.push_back(element.second);
-    }
-    std::sort(shims.begin(), shims.end(),
-              [](const DefinedAtom *l, const DefinedAtom *r) {
-                return (l->name() < r->name());
-              });
-
-    // Add all shims to merged file.
-    for (const DefinedAtom *shim : shims)
-      mergedFile.addAtom(*shim);
-
-    return llvm::Error::success();
-  }
-
-private:
-
-  void updateBranchToUseShim(bool thumbToArm, const DefinedAtom& target,
-                             const Reference *ref) {
-    // Make file-format specific stub and other support atoms.
-    const DefinedAtom *shim = this->getShim(thumbToArm, target);
-    assert(shim != nullptr);
-    // Switch branch site to target shim atom.
-    const_cast<Reference *>(ref)->setTarget(shim);
-  }
-
-  const DefinedAtom* getShim(bool thumbToArm, const DefinedAtom& target) {
-    auto pos = _targetToShim.find(&target);
-    if ( pos != _targetToShim.end() ) {
-      // Reuse an existing shim.
-      assert(pos->second != nullptr);
-      return pos->second;
-    } else {
-      // There is no existing shim, so create a new one.
-      const DefinedAtom *shim = _archHandler.createShim(_file, thumbToArm,
-                                                        target);
-       _targetToShim[&target] = shim;
-       return shim;
-    }
-  }
-
-  const MachOLinkingContext &_ctx;
-  mach_o::ArchHandler                            &_archHandler;
-  const ArchHandler::StubInfo                    &_stubInfo;
-  MachOFile                                      &_file;
-  llvm::DenseMap<const Atom*, const DefinedAtom*> _targetToShim;
-};
-
-
-
-void addShimPass(PassManager &pm, const MachOLinkingContext &ctx) {
-  pm.add(std::make_unique<ShimPass>(ctx));
-}
-
-} // end namespace mach_o
-} // end namespace lld
diff --git a/lld/lib/ReaderWriter/MachO/StubsPass.cpp b/lld/lib/ReaderWriter/MachO/StubsPass.cpp
deleted file mode 100644
index fbbd8b2c7584..000000000000
--- a/lld/lib/ReaderWriter/MachO/StubsPass.cpp
+++ /dev/null
@@ -1,377 +0,0 @@
-//===- lib/ReaderWriter/MachO/StubsPass.cpp ---------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This linker pass updates call-sites which have references to shared library
-// atoms to instead have a reference to a stub (PLT entry) for the specified
-// symbol.  Each file format defines a subclass of StubsPass which implements
-// the abstract methods for creating the file format specific StubAtoms.
-//
-//===----------------------------------------------------------------------===//
-
-#include "ArchHandler.h"
-#include "File.h"
-#include "MachOPasses.h"
-#include "lld/Common/LLVM.h"
-#include "lld/Core/DefinedAtom.h"
-#include "lld/Core/File.h"
-#include "lld/Core/Reference.h"
-#include "lld/Core/Simple.h"
-#include "lld/ReaderWriter/MachOLinkingContext.h"
-#include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/SmallVector.h"
-
-namespace lld {
-namespace mach_o {
-
-//
-//  Lazy Pointer Atom created by the stubs pass.
-//
-class LazyPointerAtom : public SimpleDefinedAtom {
-public:
-  LazyPointerAtom(const File &file, bool is64)
-    : SimpleDefinedAtom(file), _is64(is64) { }
-
-  ~LazyPointerAtom() override = default;
-
-  ContentType contentType() const override {
-    return DefinedAtom::typeLazyPointer;
-  }
-
-  Alignment alignment() const override {
-    return _is64 ? 8 : 4;
-  }
-
-  uint64_t size() const override {
-    return _is64 ? 8 : 4;
-  }
-
-  ContentPermissions permissions() const override {
-    return DefinedAtom::permRW_;
-  }
-
-  ArrayRef<uint8_t> rawContent() const override {
-    static const uint8_t zeros[] =
-        { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
-    return llvm::makeArrayRef(zeros, size());
-  }
-
-private:
-  const bool _is64;
-};
-
-//
-//  NonLazyPointer (GOT) Atom created by the stubs pass.
-//
-class NonLazyPointerAtom : public SimpleDefinedAtom {
-public:
-  NonLazyPointerAtom(const File &file, bool is64, ContentType contentType)
-    : SimpleDefinedAtom(file), _is64(is64), _contentType(contentType) { }
-
-  ~NonLazyPointerAtom() override = default;
-
-  ContentType contentType() const override {
-    return _contentType;
-  }
-
-  Alignment alignment() const override {
-    return _is64 ? 8 : 4;
-  }
-
-  uint64_t size() const override {
-    return _is64 ? 8 : 4;
-  }
-
-  ContentPermissions permissions() const override {
-    return DefinedAtom::permRW_;
-  }
-
-  ArrayRef<uint8_t> rawContent() const override {
-    static const uint8_t zeros[] =
-        { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
-    return llvm::makeArrayRef(zeros, size());
-  }
-
-private:
-  const bool _is64;
-  const ContentType _contentType;
-};
-
-//
-// Stub Atom created by the stubs pass.
-//
-class StubAtom : public SimpleDefinedAtom {
-public:
-  StubAtom(const File &file, const ArchHandler::StubInfo &stubInfo)
-      : SimpleDefinedAtom(file), _stubInfo(stubInfo){ }
-
-  ~StubAtom() override = default;
-
-  ContentType contentType() const override {
-    return DefinedAtom::typeStub;
-  }
-
-  Alignment alignment() const override {
-    return 1 << _stubInfo.codeAlignment;
-  }
-
-  uint64_t size() const override {
-    return _stubInfo.stubSize;
-  }
-
-  ContentPermissions permissions() const override {
-    return DefinedAtom::permR_X;
-  }
-
-  ArrayRef<uint8_t> rawContent() const override {
-    return llvm::makeArrayRef(_stubInfo.stubBytes, _stubInfo.stubSize);
-  }
-
-private:
-  const ArchHandler::StubInfo   &_stubInfo;
-};
-
-//
-// Stub Helper Atom created by the stubs pass.
-//
-class StubHelperAtom : public SimpleDefinedAtom {
-public:
-  StubHelperAtom(const File &file, const ArchHandler::StubInfo &stubInfo)
-      : SimpleDefinedAtom(file), _stubInfo(stubInfo) { }
-
-  ~StubHelperAtom() override = default;
-
-  ContentType contentType() const override {
-    return DefinedAtom::typeStubHelper;
-  }
-
-  Alignment alignment() const override {
-    return 1 << _stubInfo.codeAlignment;
-  }
-
-  uint64_t size() const override {
-    return _stubInfo.stubHelperSize;
-  }
-
-  ContentPermissions permissions() const override {
-    return DefinedAtom::permR_X;
-  }
-
-  ArrayRef<uint8_t> rawContent() const override {
-    return llvm::makeArrayRef(_stubInfo.stubHelperBytes,
-                              _stubInfo.stubHelperSize);
-  }
-
-private:
-  const ArchHandler::StubInfo   &_stubInfo;
-};
-
-//
-// Stub Helper Common Atom created by the stubs pass.
-//
-class StubHelperCommonAtom : public SimpleDefinedAtom {
-public:
-  StubHelperCommonAtom(const File &file, const ArchHandler::StubInfo &stubInfo)
-      : SimpleDefinedAtom(file), _stubInfo(stubInfo) { }
-
-  ~StubHelperCommonAtom() override = default;
-
-  ContentType contentType() const override {
-    return DefinedAtom::typeStubHelper;
-  }
-
-  Alignment alignment() const override {
-    return 1 << _stubInfo.stubHelperCommonAlignment;
-  }
-
-  uint64_t size() const override {
-    return _stubInfo.stubHelperCommonSize;
-  }
-
-  ContentPermissions permissions() const override {
-    return DefinedAtom::permR_X;
-  }
-
-  ArrayRef<uint8_t> rawContent() const override {
-    return llvm::makeArrayRef(_stubInfo.stubHelperCommonBytes,
-                        _stubInfo.stubHelperCommonSize);
-  }
-
-private:
-  const ArchHandler::StubInfo   &_stubInfo;
-};
-
-class StubsPass : public Pass {
-public:
-  StubsPass(const MachOLinkingContext &context)
-      : _ctx(context), _archHandler(_ctx.archHandler()),
-        _stubInfo(_archHandler.stubInfo()),
-        _file(*_ctx.make_file<MachOFile>("<mach-o Stubs pass>")) {
-    _file.setOrdinal(_ctx.getNextOrdinalAndIncrement());
-  }
-
-  llvm::Error perform(SimpleFile &mergedFile) override {
-    // Skip this pass if output format uses text relocations instead of stubs.
-    if (!this->noTextRelocs())
-      return llvm::Error::success();
-
-    // Scan all references in all atoms.
-    for (const DefinedAtom *atom : mergedFile.defined()) {
-      for (const Reference *ref : *atom) {
-        // Look at call-sites.
-        if (!this->isCallSite(*ref))
-          continue;
-        const Atom *target = ref->target();
-        assert(target != nullptr);
-        if (isa<SharedLibraryAtom>(target)) {
-          // Calls to shared libraries go through stubs.
-          _targetToUses[target].push_back(ref);
-          continue;
-        }
-        const DefinedAtom *defTarget = dyn_cast<DefinedAtom>(target);
-        if (defTarget && defTarget->interposable() != DefinedAtom::interposeNo){
-          // Calls to interposable functions in same linkage unit must also go
-          // through a stub.
-          assert(defTarget->scope() != DefinedAtom::scopeTranslationUnit);
-          _targetToUses[target].push_back(ref);
-        }
-      }
-    }
-
-    // Exit early if no stubs needed.
-    if (_targetToUses.empty())
-      return llvm::Error::success();
-
-    // First add help-common and GOT slots used by lazy binding.
-    SimpleDefinedAtom *helperCommonAtom =
-        new (_file.allocator()) StubHelperCommonAtom(_file, _stubInfo);
-    SimpleDefinedAtom *helperCacheNLPAtom =
-        new (_file.allocator()) NonLazyPointerAtom(_file, _ctx.is64Bit(),
-                                    _stubInfo.stubHelperImageCacheContentType);
-    SimpleDefinedAtom *helperBinderNLPAtom =
-        new (_file.allocator()) NonLazyPointerAtom(_file, _ctx.is64Bit(),
-                                    _stubInfo.stubHelperImageCacheContentType);
-    addReference(helperCommonAtom, _stubInfo.stubHelperCommonReferenceToCache,
-                 helperCacheNLPAtom);
-    addOptReference(
-        helperCommonAtom, _stubInfo.stubHelperCommonReferenceToCache,
-        _stubInfo.optStubHelperCommonReferenceToCache, helperCacheNLPAtom);
-    addReference(helperCommonAtom, _stubInfo.stubHelperCommonReferenceToBinder,
-                 helperBinderNLPAtom);
-    addOptReference(
-        helperCommonAtom, _stubInfo.stubHelperCommonReferenceToBinder,
-        _stubInfo.optStubHelperCommonReferenceToBinder, helperBinderNLPAtom);
-    mergedFile.addAtom(*helperCommonAtom);
-    mergedFile.addAtom(*helperBinderNLPAtom);
-    mergedFile.addAtom(*helperCacheNLPAtom);
-
-    // Add reference to dyld_stub_binder in libSystem.dylib
-    auto I = llvm::find_if(
-        mergedFile.sharedLibrary(), [&](const SharedLibraryAtom *atom) {
-          return atom->name().equals(_stubInfo.binderSymbolName);
-        });
-    assert(I != mergedFile.sharedLibrary().end() &&
-           "dyld_stub_binder not found");
-    addReference(helperBinderNLPAtom, _stubInfo.nonLazyPointerReferenceToBinder, *I);
-
-    // Sort targets by name, so stubs and lazy pointers are consistent
-    std::vector<const Atom *> targetsNeedingStubs;
-    for (auto it : _targetToUses)
-      targetsNeedingStubs.push_back(it.first);
-    std::sort(targetsNeedingStubs.begin(), targetsNeedingStubs.end(),
-              [](const Atom * left, const Atom * right) {
-      return (left->name().compare(right->name()) < 0);
-    });
-
-    // Make and append stubs, lazy pointers, and helpers in alphabetical order.
-    unsigned lazyOffset = 0;
-    for (const Atom *target : targetsNeedingStubs) {
-      auto *stub = new (_file.allocator()) StubAtom(_file, _stubInfo);
-      auto *lp =
-          new (_file.allocator()) LazyPointerAtom(_file, _ctx.is64Bit());
-      auto *helper = new (_file.allocator()) StubHelperAtom(_file, _stubInfo);
-
-      addReference(stub, _stubInfo.stubReferenceToLP, lp);
-      addOptReference(stub, _stubInfo.stubReferenceToLP,
-                      _stubInfo.optStubReferenceToLP, lp);
-      addReference(lp, _stubInfo.lazyPointerReferenceToHelper, helper);
-      addReference(lp, _stubInfo.lazyPointerReferenceToFinal, target);
-      addReference(helper, _stubInfo.stubHelperReferenceToImm, helper);
-      addReferenceAddend(helper, _stubInfo.stubHelperReferenceToImm, helper,
-                         lazyOffset);
-      addReference(helper, _stubInfo.stubHelperReferenceToHelperCommon,
-                   helperCommonAtom);
-
-      mergedFile.addAtom(*stub);
-      mergedFile.addAtom(*lp);
-      mergedFile.addAtom(*helper);
-
-      // Update each reference to use stub.
-      for (const Reference *ref : _targetToUses[target]) {
-        assert(ref->target() == target);
-        // Switch call site to reference stub atom instead.
-        const_cast<Reference *>(ref)->setTarget(stub);
-      }
-
-      // Calculate new offset
-      lazyOffset += target->name().size() + 12;
-    }
-
-    return llvm::Error::success();
-  }
-
-private:
-  bool noTextRelocs() {
-    return true;
-  }
-
-  bool isCallSite(const Reference &ref) {
-    return _archHandler.isCallSite(ref);
-  }
-
-  void addReference(SimpleDefinedAtom* atom,
-                    const ArchHandler::ReferenceInfo &refInfo,
-                    const lld::Atom* target) {
-    atom->addReference(Reference::KindNamespace::mach_o,
-                      refInfo.arch, refInfo.kind, refInfo.offset,
-                      target, refInfo.addend);
-  }
-
-  void addReferenceAddend(SimpleDefinedAtom *atom,
-                          const ArchHandler::ReferenceInfo &refInfo,
-                          const lld::Atom *target, uint64_t addend) {
-    atom->addReference(Reference::KindNamespace::mach_o, refInfo.arch,
-                       refInfo.kind, refInfo.offset, target, addend);
-  }
-
-   void addOptReference(SimpleDefinedAtom* atom,
-                    const ArchHandler::ReferenceInfo &refInfo,
-                    const ArchHandler::OptionalRefInfo &optRef,
-                    const lld::Atom* target) {
-      if (!optRef.used)
-        return;
-    atom->addReference(Reference::KindNamespace::mach_o,
-                      refInfo.arch, optRef.kind, optRef.offset,
-                      target, optRef.addend);
-  }
-
-  typedef llvm::DenseMap<const Atom*,
-                         llvm::SmallVector<const Reference *, 8>> TargetToUses;
-
-  const MachOLinkingContext &_ctx;
-  mach_o::ArchHandler                            &_archHandler;
-  const ArchHandler::StubInfo                    &_stubInfo;
-  MachOFile                                      &_file;
-  TargetToUses                                    _targetToUses;
-};
-
-void addStubsPass(PassManager &pm, const MachOLinkingContext &ctx) {
-  pm.add(std::unique_ptr<Pass>(new StubsPass(ctx)));
-}
-
-} // end namespace mach_o
-} // end namespace lld
diff --git a/lld/lib/ReaderWriter/MachO/TLVPass.cpp b/lld/lib/ReaderWriter/MachO/TLVPass.cpp
deleted file mode 100644
index e0a031cfb07b..000000000000
--- a/lld/lib/ReaderWriter/MachO/TLVPass.cpp
+++ /dev/null
@@ -1,140 +0,0 @@
-//===- lib/ReaderWriter/MachO/TLVPass.cpp -----------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-///
-/// \file
-/// This linker pass transforms all TLV references to real references.
-///
-//===----------------------------------------------------------------------===//
-
-#include "ArchHandler.h"
-#include "File.h"
-#include "MachOPasses.h"
-#include "lld/Core/Simple.h"
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/Support/Debug.h"
-
-namespace lld {
-namespace mach_o {
-
-//
-// TLVP Entry Atom created by the TLV pass.
-//
-class TLVPEntryAtom : public SimpleDefinedAtom {
-public:
-  TLVPEntryAtom(const File &file, bool is64, StringRef name)
-      : SimpleDefinedAtom(file), _is64(is64), _name(name) {}
-
-  ~TLVPEntryAtom() override = default;
-
-  ContentType contentType() const override {
-    return DefinedAtom::typeTLVInitializerPtr;
-  }
-
-  Alignment alignment() const override {
-    return _is64 ? 8 : 4;
-  }
-
-  uint64_t size() const override {
-    return _is64 ? 8 : 4;
-  }
-
-  ContentPermissions permissions() const override {
-    return DefinedAtom::permRW_;
-  }
-
-  ArrayRef<uint8_t> rawContent() const override {
-    static const uint8_t zeros[] =
-      { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
-    return llvm::makeArrayRef(zeros, size());
-  }
-
-  StringRef slotName() const {
-    return _name;
-  }
-
-private:
-  const bool _is64;
-  StringRef _name;
-};
-
-class TLVPass : public Pass {
-public:
-  TLVPass(const MachOLinkingContext &context)
-      : _ctx(context), _archHandler(_ctx.archHandler()),
-        _file(*_ctx.make_file<MachOFile>("<mach-o TLV pass>")) {
-    _file.setOrdinal(_ctx.getNextOrdinalAndIncrement());
-  }
-
-private:
-  llvm::Error perform(SimpleFile &mergedFile) override {
-    bool allowTLV = _ctx.minOS("10.7", "1.0");
-
-    for (const DefinedAtom *atom : mergedFile.defined()) {
-      for (const Reference *ref : *atom) {
-        if (!_archHandler.isTLVAccess(*ref))
-          continue;
-
-        if (!allowTLV)
-          return llvm::make_error<GenericError>(
-            "targeted OS version does not support use of thread local "
-            "variables in " + atom->name() + " for architecture " +
-            _ctx.archName());
-
-        const Atom *target = ref->target();
-        assert(target != nullptr);
-
-        const DefinedAtom *tlvpEntry = makeTLVPEntry(target);
-        const_cast<Reference*>(ref)->setTarget(tlvpEntry);
-        _archHandler.updateReferenceToTLV(ref);
-      }
-    }
-
-    std::vector<const TLVPEntryAtom*> entries;
-    entries.reserve(_targetToTLVP.size());
-    for (auto &it : _targetToTLVP)
-      entries.push_back(it.second);
-    std::sort(entries.begin(), entries.end(),
-              [](const TLVPEntryAtom *lhs, const TLVPEntryAtom *rhs) {
-                return (lhs->slotName().compare(rhs->slotName()) < 0);
-              });
-
-    for (const TLVPEntryAtom *slot : entries)
-      mergedFile.addAtom(*slot);
-
-    return llvm::Error::success();
-  }
-
-  const DefinedAtom *makeTLVPEntry(const Atom *target) {
-    auto pos = _targetToTLVP.find(target);
-
-    if (pos != _targetToTLVP.end())
-      return pos->second;
-
-    auto *tlvpEntry = new (_file.allocator())
-      TLVPEntryAtom(_file, _ctx.is64Bit(), target->name());
-    _targetToTLVP[target] = tlvpEntry;
-    const ArchHandler::ReferenceInfo &nlInfo =
-      _archHandler.stubInfo().nonLazyPointerReferenceToBinder;
-    tlvpEntry->addReference(Reference::KindNamespace::mach_o, nlInfo.arch,
-                            nlInfo.kind, 0, target, 0);
-    return tlvpEntry;
-  }
-
-  const MachOLinkingContext &_ctx;
-  mach_o::ArchHandler &_archHandler;
-  MachOFile           &_file;
-  llvm::DenseMap<const Atom*, const TLVPEntryAtom*> _targetToTLVP;
-};
-
-void addTLVPass(PassManager &pm, const MachOLinkingContext &ctx) {
-  assert(ctx.needsTLVPass());
-  pm.add(std::make_unique<TLVPass>(ctx));
-}
-
-} // end namespace mach_o
-} // end namespace lld
diff --git a/lld/lib/ReaderWriter/MachO/WriterMachO.cpp b/lld/lib/ReaderWriter/MachO/WriterMachO.cpp
deleted file mode 100644
index 60e0e9dd9a81..000000000000
--- a/lld/lib/ReaderWriter/MachO/WriterMachO.cpp
+++ /dev/null
@@ -1,70 +0,0 @@
-//===- lib/ReaderWriter/MachO/WriterMachO.cpp -----------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "ExecutableAtoms.h"
-#include "MachONormalizedFile.h"
-#include "lld/Core/File.h"
-#include "lld/Core/Writer.h"
-#include "lld/ReaderWriter/MachOLinkingContext.h"
-#include "llvm/BinaryFormat/MachO.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/FileOutputBuffer.h"
-#include "llvm/Support/raw_ostream.h"
-#include <system_error>
-
-using lld::mach_o::normalized::NormalizedFile;
-
-namespace lld {
-namespace mach_o {
-
-class MachOWriter : public Writer {
-public:
-  MachOWriter(const MachOLinkingContext &ctxt) : _ctx(ctxt) {}
-
-  llvm::Error writeFile(const lld::File &file, StringRef path) override {
-    // Construct empty normalized file from atoms.
-    llvm::Expected<std::unique_ptr<NormalizedFile>> nFile =
-        normalized::normalizedFromAtoms(file, _ctx);
-    if (auto ec = nFile.takeError())
-      return ec;
-
-    // For testing, write out yaml form of normalized file.
-    if (_ctx.printAtoms()) {
-      std::unique_ptr<Writer> yamlWriter = createWriterYAML(_ctx);
-      if (auto ec = yamlWriter->writeFile(file, "-"))
-        return ec;
-    }
-
-    // Write normalized file as mach-o binary.
-    return writeBinary(*nFile->get(), path);
-  }
-
-  void createImplicitFiles(std::vector<std::unique_ptr<File>> &r) override {
-    // When building main executables, add _main as required entry point.
-    if (_ctx.outputTypeHasEntry())
-      r.emplace_back(new CEntryFile(_ctx));
-    // If this can link with dylibs, need helper function (dyld_stub_binder).
-    if (_ctx.needsStubsPass())
-      r.emplace_back(new StubHelperFile(_ctx));
-    // Final linked images can access a symbol for their mach_header.
-    if (_ctx.outputMachOType() != llvm::MachO::MH_OBJECT)
-      r.emplace_back(new MachHeaderAliasFile(_ctx));
-  }
-private:
-  const MachOLinkingContext &_ctx;
- };
-
-
-} // namespace mach_o
-
-std::unique_ptr<Writer> createWriterMachO(const MachOLinkingContext &context) {
-  return std::unique_ptr<Writer>(new lld::mach_o::MachOWriter(context));
-}
-
-} // namespace lld
diff --git a/lld/lib/ReaderWriter/YAML/ReaderWriterYAML.cpp b/lld/lib/ReaderWriter/YAML/ReaderWriterYAML.cpp
deleted file mode 100644
index c0e6e0334fa6..000000000000
--- a/lld/lib/ReaderWriter/YAML/ReaderWriterYAML.cpp
+++ /dev/null
@@ -1,1403 +0,0 @@
-//===- lib/ReaderWriter/YAML/ReaderWriterYAML.cpp -------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "lld/Core/AbsoluteAtom.h"
-#include "lld/Core/ArchiveLibraryFile.h"
-#include "lld/Core/Atom.h"
-#include "lld/Core/DefinedAtom.h"
-#include "lld/Core/Error.h"
-#include "lld/Core/File.h"
-#include "lld/Core/LinkingContext.h"
-#include "lld/Core/Reader.h"
-#include "lld/Core/Reference.h"
-#include "lld/Core/SharedLibraryAtom.h"
-#include "lld/Core/Simple.h"
-#include "lld/Core/UndefinedAtom.h"
-#include "lld/Core/Writer.h"
-#include "lld/ReaderWriter/YamlContext.h"
-#include "llvm/ADT/ArrayRef.h"
-#include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/StringMap.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/ADT/Twine.h"
-#include "llvm/BinaryFormat/Magic.h"
-#include "llvm/Support/Allocator.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/Error.h"
-#include "llvm/Support/ErrorOr.h"
-#include "llvm/Support/FileSystem.h"
-#include "llvm/Support/Format.h"
-#include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/YAMLTraits.h"
-#include "llvm/Support/raw_ostream.h"
-#include <cassert>
-#include <cstdint>
-#include <cstring>
-#include <memory>
-#include <string>
-#include <system_error>
-#include <vector>
-
-using llvm::file_magic;
-using llvm::yaml::MappingTraits;
-using llvm::yaml::ScalarEnumerationTraits;
-using llvm::yaml::ScalarTraits;
-using llvm::yaml::IO;
-using llvm::yaml::SequenceTraits;
-using llvm::yaml::DocumentListTraits;
-
-using namespace lld;
-
-/// The conversion of Atoms to and from YAML uses LLVM's YAML I/O.  This
-/// file just defines template specializations on the lld types which control
-/// how the mapping is done to and from YAML.
-
-namespace {
-
-/// Used when writing yaml files.
-/// In most cases, atoms names are unambiguous, so references can just
-/// use the atom name as the target (e.g. target: foo).  But in a few
-/// cases that does not work, so ref-names are added.  These are labels
-/// used only in yaml.  The labels do not exist in the Atom model.
-///
-/// One need for ref-names are when atoms have no user supplied name
-/// (e.g. c-string literal).  Another case is when two object files with
-/// identically named static functions are merged (ld -r) into one object file.
-/// In that case referencing the function by name is ambiguous, so a unique
-/// ref-name is added.
-class RefNameBuilder {
-public:
-  RefNameBuilder(const lld::File &file)
-      : _collisionCount(0), _unnamedCounter(0) {
-    // visit all atoms
-    for (const lld::DefinedAtom *atom : file.defined()) {
-      // Build map of atoms names to detect duplicates
-      if (!atom->name().empty())
-        buildDuplicateNameMap(*atom);
-
-      // Find references to unnamed atoms and create ref-names for them.
-      for (const lld::Reference *ref : *atom) {
-        // create refname for any unnamed reference target
-        const lld::Atom *target = ref->target();
-        if ((target != nullptr) && target->name().empty()) {
-          std::string storage;
-          llvm::raw_string_ostream buffer(storage);
-          buffer << llvm::format("L%03d", _unnamedCounter++);
-          StringRef newName = copyString(buffer.str());
-          _refNames[target] = std::string(newName);
-          DEBUG_WITH_TYPE("WriterYAML",
-                          llvm::dbgs() << "unnamed atom: creating ref-name: '"
-                                       << newName << "' ("
-                                       << (const void *)newName.data() << ", "
-                                       << newName.size() << ")\n");
-        }
-      }
-    }
-    for (const lld::UndefinedAtom *undefAtom : file.undefined()) {
-      buildDuplicateNameMap(*undefAtom);
-    }
-    for (const lld::SharedLibraryAtom *shlibAtom : file.sharedLibrary()) {
-      buildDuplicateNameMap(*shlibAtom);
-    }
-    for (const lld::AbsoluteAtom *absAtom : file.absolute()) {
-      if (!absAtom->name().empty())
-        buildDuplicateNameMap(*absAtom);
-    }
-  }
-
-  void buildDuplicateNameMap(const lld::Atom &atom) {
-    assert(!atom.name().empty());
-    NameToAtom::iterator pos = _nameMap.find(atom.name());
-    if (pos != _nameMap.end()) {
-      // Found name collision, give each a unique ref-name.
-      std::string Storage;
-      llvm::raw_string_ostream buffer(Storage);
-      buffer << atom.name() << llvm::format(".%03d", ++_collisionCount);
-      StringRef newName = copyString(buffer.str());
-      _refNames[&atom] = std::string(newName);
-      DEBUG_WITH_TYPE("WriterYAML",
-                      llvm::dbgs() << "name collision: creating ref-name: '"
-                                   << newName << "' ("
-                                   << (const void *)newName.data()
-                                   << ", " << newName.size() << ")\n");
-      const lld::Atom *prevAtom = pos->second;
-      AtomToRefName::iterator pos2 = _refNames.find(prevAtom);
-      if (pos2 == _refNames.end()) {
-        // Only create ref-name for previous if none already created.
-        std::string Storage2;
-        llvm::raw_string_ostream buffer2(Storage2);
-        buffer2 << prevAtom->name() << llvm::format(".%03d", ++_collisionCount);
-        StringRef newName2 = copyString(buffer2.str());
-        _refNames[prevAtom] = std::string(newName2);
-        DEBUG_WITH_TYPE("WriterYAML",
-                        llvm::dbgs() << "name collision: creating ref-name: '"
-                                     << newName2 << "' ("
-                                     << (const void *)newName2.data() << ", "
-                                     << newName2.size() << ")\n");
-      }
-    } else {
-      // First time we've seen this name, just add it to map.
-      _nameMap[atom.name()] = &atom;
-      DEBUG_WITH_TYPE("WriterYAML", llvm::dbgs()
-                                        << "atom name seen for first time: '"
-                                        << atom.name() << "' ("
-                                        << (const void *)atom.name().data()
-                                        << ", " << atom.name().size() << ")\n");
-    }
-  }
-
-  bool hasRefName(const lld::Atom *atom) { return _refNames.count(atom); }
-
-  StringRef refName(const lld::Atom *atom) {
-    return _refNames.find(atom)->second;
-  }
-
-private:
-  typedef llvm::StringMap<const lld::Atom *> NameToAtom;
-  typedef llvm::DenseMap<const lld::Atom *, std::string> AtomToRefName;
-
-  // Allocate a new copy of this string in _storage, so the strings
-  // can be freed when RefNameBuilder is destroyed.
-  StringRef copyString(StringRef str) {
-    char *s = _storage.Allocate<char>(str.size());
-    memcpy(s, str.data(), str.size());
-    return StringRef(s, str.size());
-  }
-
-  unsigned int                         _collisionCount;
-  unsigned int                         _unnamedCounter;
-  NameToAtom                           _nameMap;
-  AtomToRefName                        _refNames;
-  llvm::BumpPtrAllocator               _storage;
-};
-
-/// Used when reading yaml files to find the target of a reference
-/// that could be a name or ref-name.
-class RefNameResolver {
-public:
-  RefNameResolver(const lld::File *file, IO &io);
-
-  const lld::Atom *lookup(StringRef name) const {
-    NameToAtom::const_iterator pos = _nameMap.find(name);
-    if (pos != _nameMap.end())
-      return pos->second;
-    _io.setError(Twine("no such atom name: ") + name);
-    return nullptr;
-  }
-
-private:
-  typedef llvm::StringMap<const lld::Atom *> NameToAtom;
-
-  void add(StringRef name, const lld::Atom *atom) {
-    if (_nameMap.count(name)) {
-      _io.setError(Twine("duplicate atom name: ") + name);
-    } else {
-      _nameMap[name] = atom;
-    }
-  }
-
-  IO &_io;
-  NameToAtom _nameMap;
-};
-
-/// Mapping of Atoms.
-template <typename T> class AtomList {
-  using Ty = std::vector<OwningAtomPtr<T>>;
-
-public:
-  typename Ty::iterator begin() { return _atoms.begin(); }
-  typename Ty::iterator end() { return _atoms.end(); }
-  Ty _atoms;
-};
-
-/// Mapping of kind: field in yaml files.
-enum FileKinds {
-  fileKindObjectAtoms, // atom based object file encoded in yaml
-  fileKindArchive,     // static archive library encoded in yaml
-  fileKindObjectMachO  // mach-o object files encoded in yaml
-};
-
-struct ArchMember {
-  FileKinds         _kind;
-  StringRef         _name;
-  const lld::File  *_content;
-};
-
-// The content bytes in a DefinedAtom are just uint8_t but we want
-// special formatting, so define a strong type.
-LLVM_YAML_STRONG_TYPEDEF(uint8_t, ImplicitHex8)
-
-// SharedLibraryAtoms have a bool canBeNull() method which we'd like to be
-// more readable than just true/false.
-LLVM_YAML_STRONG_TYPEDEF(bool, ShlibCanBeNull)
-
-// lld::Reference::Kind is a tuple of <namespace, arch, value>.
-// For yaml, we just want one string that encapsulates the tuple.
-struct RefKind {
-  Reference::KindNamespace  ns;
-  Reference::KindArch       arch;
-  Reference::KindValue      value;
-};
-
-} // end anonymous namespace
-
-LLVM_YAML_IS_SEQUENCE_VECTOR(ArchMember)
-LLVM_YAML_IS_SEQUENCE_VECTOR(const lld::Reference *)
-// Always write DefinedAtoms content bytes as a flow sequence.
-LLVM_YAML_IS_FLOW_SEQUENCE_VECTOR(ImplicitHex8)
-
-// for compatibility with gcc-4.7 in C++11 mode, add extra namespace
-namespace llvm {
-namespace yaml {
-
-// This is a custom formatter for RefKind
-template <> struct ScalarTraits<RefKind> {
-  static void output(const RefKind &kind, void *ctxt, raw_ostream &out) {
-    assert(ctxt != nullptr);
-    YamlContext *info = reinterpret_cast<YamlContext *>(ctxt);
-    assert(info->_registry);
-    StringRef str;
-    if (info->_registry->referenceKindToString(kind.ns, kind.arch, kind.value,
-                                               str))
-      out << str;
-    else
-      out << (int)(kind.ns) << "-" << (int)(kind.arch) << "-" << kind.value;
-  }
-
-  static StringRef input(StringRef scalar, void *ctxt, RefKind &kind) {
-    assert(ctxt != nullptr);
-    YamlContext *info = reinterpret_cast<YamlContext *>(ctxt);
-    assert(info->_registry);
-    if (info->_registry->referenceKindFromString(scalar, kind.ns, kind.arch,
-                                                 kind.value))
-      return StringRef();
-    return StringRef("unknown reference kind");
-  }
-
-  static QuotingType mustQuote(StringRef) { return QuotingType::None; }
-};
-
-template <> struct ScalarEnumerationTraits<lld::File::Kind> {
-  static void enumeration(IO &io, lld::File::Kind &value) {
-    io.enumCase(value, "error-object",   lld::File::kindErrorObject);
-    io.enumCase(value, "object",         lld::File::kindMachObject);
-    io.enumCase(value, "shared-library", lld::File::kindSharedLibrary);
-    io.enumCase(value, "static-library", lld::File::kindArchiveLibrary);
-  }
-};
-
-template <> struct ScalarEnumerationTraits<lld::Atom::Scope> {
-  static void enumeration(IO &io, lld::Atom::Scope &value) {
-    io.enumCase(value, "global", lld::Atom::scopeGlobal);
-    io.enumCase(value, "hidden", lld::Atom::scopeLinkageUnit);
-    io.enumCase(value, "static", lld::Atom::scopeTranslationUnit);
-  }
-};
-
-template <> struct ScalarEnumerationTraits<lld::DefinedAtom::SectionChoice> {
-  static void enumeration(IO &io, lld::DefinedAtom::SectionChoice &value) {
-    io.enumCase(value, "content", lld::DefinedAtom::sectionBasedOnContent);
-    io.enumCase(value, "custom",  lld::DefinedAtom::sectionCustomPreferred);
-    io.enumCase(value, "custom-required",
-                                 lld::DefinedAtom::sectionCustomRequired);
-  }
-};
-
-template <> struct ScalarEnumerationTraits<lld::DefinedAtom::Interposable> {
-  static void enumeration(IO &io, lld::DefinedAtom::Interposable &value) {
-    io.enumCase(value, "no",           DefinedAtom::interposeNo);
-    io.enumCase(value, "yes",          DefinedAtom::interposeYes);
-    io.enumCase(value, "yes-and-weak", DefinedAtom::interposeYesAndRuntimeWeak);
-  }
-};
-
-template <> struct ScalarEnumerationTraits<lld::DefinedAtom::Merge> {
-  static void enumeration(IO &io, lld::DefinedAtom::Merge &value) {
-    io.enumCase(value, "no",           lld::DefinedAtom::mergeNo);
-    io.enumCase(value, "as-tentative", lld::DefinedAtom::mergeAsTentative);
-    io.enumCase(value, "as-weak",      lld::DefinedAtom::mergeAsWeak);
-    io.enumCase(value, "as-addressed-weak",
-                                   lld::DefinedAtom::mergeAsWeakAndAddressUsed);
-    io.enumCase(value, "by-content",   lld::DefinedAtom::mergeByContent);
-    io.enumCase(value, "same-name-and-size",
-                lld::DefinedAtom::mergeSameNameAndSize);
-    io.enumCase(value, "largest", lld::DefinedAtom::mergeByLargestSection);
-  }
-};
-
-template <> struct ScalarEnumerationTraits<lld::DefinedAtom::DeadStripKind> {
-  static void enumeration(IO &io, lld::DefinedAtom::DeadStripKind &value) {
-    io.enumCase(value, "normal", lld::DefinedAtom::deadStripNormal);
-    io.enumCase(value, "never",  lld::DefinedAtom::deadStripNever);
-    io.enumCase(value, "always", lld::DefinedAtom::deadStripAlways);
-  }
-};
-
-template <> struct ScalarEnumerationTraits<lld::DefinedAtom::DynamicExport> {
-  static void enumeration(IO &io, lld::DefinedAtom::DynamicExport &value) {
-    io.enumCase(value, "normal", lld::DefinedAtom::dynamicExportNormal);
-    io.enumCase(value, "always", lld::DefinedAtom::dynamicExportAlways);
-  }
-};
-
-template <> struct ScalarEnumerationTraits<lld::DefinedAtom::CodeModel> {
-  static void enumeration(IO &io, lld::DefinedAtom::CodeModel &value) {
-    io.enumCase(value, "none", lld::DefinedAtom::codeNA);
-    io.enumCase(value, "mips-pic", lld::DefinedAtom::codeMipsPIC);
-    io.enumCase(value, "mips-micro", lld::DefinedAtom::codeMipsMicro);
-    io.enumCase(value, "mips-micro-pic", lld::DefinedAtom::codeMipsMicroPIC);
-    io.enumCase(value, "mips-16", lld::DefinedAtom::codeMips16);
-    io.enumCase(value, "arm-thumb", lld::DefinedAtom::codeARMThumb);
-    io.enumCase(value, "arm-a", lld::DefinedAtom::codeARM_a);
-    io.enumCase(value, "arm-d", lld::DefinedAtom::codeARM_d);
-    io.enumCase(value, "arm-t", lld::DefinedAtom::codeARM_t);
-  }
-};
-
-template <>
-struct ScalarEnumerationTraits<lld::DefinedAtom::ContentPermissions> {
-  static void enumeration(IO &io, lld::DefinedAtom::ContentPermissions &value) {
-    io.enumCase(value, "---",     lld::DefinedAtom::perm___);
-    io.enumCase(value, "r--",     lld::DefinedAtom::permR__);
-    io.enumCase(value, "r-x",     lld::DefinedAtom::permR_X);
-    io.enumCase(value, "rw-",     lld::DefinedAtom::permRW_);
-    io.enumCase(value, "rwx",     lld::DefinedAtom::permRWX);
-    io.enumCase(value, "rw-l",    lld::DefinedAtom::permRW_L);
-    io.enumCase(value, "unknown", lld::DefinedAtom::permUnknown);
-  }
-};
-
-template <> struct ScalarEnumerationTraits<lld::DefinedAtom::ContentType> {
-  static void enumeration(IO &io, lld::DefinedAtom::ContentType &value) {
-    io.enumCase(value, "unknown",         DefinedAtom::typeUnknown);
-    io.enumCase(value, "code",            DefinedAtom::typeCode);
-    io.enumCase(value, "stub",            DefinedAtom::typeStub);
-    io.enumCase(value, "constant",        DefinedAtom::typeConstant);
-    io.enumCase(value, "data",            DefinedAtom::typeData);
-    io.enumCase(value, "quick-data",      DefinedAtom::typeDataFast);
-    io.enumCase(value, "zero-fill",       DefinedAtom::typeZeroFill);
-    io.enumCase(value, "zero-fill-quick", DefinedAtom::typeZeroFillFast);
-    io.enumCase(value, "const-data",      DefinedAtom::typeConstData);
-    io.enumCase(value, "got",             DefinedAtom::typeGOT);
-    io.enumCase(value, "resolver",        DefinedAtom::typeResolver);
-    io.enumCase(value, "branch-island",   DefinedAtom::typeBranchIsland);
-    io.enumCase(value, "branch-shim",     DefinedAtom::typeBranchShim);
-    io.enumCase(value, "stub-helper",     DefinedAtom::typeStubHelper);
-    io.enumCase(value, "c-string",        DefinedAtom::typeCString);
-    io.enumCase(value, "utf16-string",    DefinedAtom::typeUTF16String);
-    io.enumCase(value, "unwind-cfi",      DefinedAtom::typeCFI);
-    io.enumCase(value, "unwind-lsda",     DefinedAtom::typeLSDA);
-    io.enumCase(value, "const-4-byte",    DefinedAtom::typeLiteral4);
-    io.enumCase(value, "const-8-byte",    DefinedAtom::typeLiteral8);
-    io.enumCase(value, "const-16-byte",   DefinedAtom::typeLiteral16);
-    io.enumCase(value, "lazy-pointer",    DefinedAtom::typeLazyPointer);
-    io.enumCase(value, "lazy-dylib-pointer",
-                                          DefinedAtom::typeLazyDylibPointer);
-    io.enumCase(value, "cfstring",        DefinedAtom::typeCFString);
-    io.enumCase(value, "initializer-pointer",
-                                          DefinedAtom::typeInitializerPtr);
-    io.enumCase(value, "terminator-pointer",
-                                          DefinedAtom::typeTerminatorPtr);
-    io.enumCase(value, "c-string-pointer",DefinedAtom::typeCStringPtr);
-    io.enumCase(value, "objc-class-pointer",
-                                          DefinedAtom::typeObjCClassPtr);
-    io.enumCase(value, "objc-category-list",
-                                          DefinedAtom::typeObjC2CategoryList);
-    io.enumCase(value, "objc-image-info",
-                                          DefinedAtom::typeObjCImageInfo);
-    io.enumCase(value, "objc-method-list",
-                                          DefinedAtom::typeObjCMethodList);
-    io.enumCase(value, "objc-class1",     DefinedAtom::typeObjC1Class);
-    io.enumCase(value, "dtraceDOF",       DefinedAtom::typeDTraceDOF);
-    io.enumCase(value, "interposing-tuples",
-                                          DefinedAtom::typeInterposingTuples);
-    io.enumCase(value, "lto-temp",        DefinedAtom::typeTempLTO);
-    io.enumCase(value, "compact-unwind",  DefinedAtom::typeCompactUnwindInfo);
-    io.enumCase(value, "unwind-info",     DefinedAtom::typeProcessedUnwindInfo);
-    io.enumCase(value, "tlv-thunk",       DefinedAtom::typeThunkTLV);
-    io.enumCase(value, "tlv-data",        DefinedAtom::typeTLVInitialData);
-    io.enumCase(value, "tlv-zero-fill",   DefinedAtom::typeTLVInitialZeroFill);
-    io.enumCase(value, "tlv-initializer-ptr",
-                                          DefinedAtom::typeTLVInitializerPtr);
-    io.enumCase(value, "mach_header",     DefinedAtom::typeMachHeader);
-    io.enumCase(value, "dso_handle",      DefinedAtom::typeDSOHandle);
-    io.enumCase(value, "sectcreate",      DefinedAtom::typeSectCreate);
-  }
-};
-
-template <> struct ScalarEnumerationTraits<lld::UndefinedAtom::CanBeNull> {
-  static void enumeration(IO &io, lld::UndefinedAtom::CanBeNull &value) {
-    io.enumCase(value, "never",       lld::UndefinedAtom::canBeNullNever);
-    io.enumCase(value, "at-runtime",  lld::UndefinedAtom::canBeNullAtRuntime);
-    io.enumCase(value, "at-buildtime",lld::UndefinedAtom::canBeNullAtBuildtime);
-  }
-};
-
-template <> struct ScalarEnumerationTraits<ShlibCanBeNull> {
-  static void enumeration(IO &io, ShlibCanBeNull &value) {
-    io.enumCase(value, "never",      false);
-    io.enumCase(value, "at-runtime", true);
-  }
-};
-
-template <>
-struct ScalarEnumerationTraits<lld::SharedLibraryAtom::Type> {
-  static void enumeration(IO &io, lld::SharedLibraryAtom::Type &value) {
-    io.enumCase(value, "code",    lld::SharedLibraryAtom::Type::Code);
-    io.enumCase(value, "data",    lld::SharedLibraryAtom::Type::Data);
-    io.enumCase(value, "unknown", lld::SharedLibraryAtom::Type::Unknown);
-  }
-};
-
-/// This is a custom formatter for lld::DefinedAtom::Alignment.  Values look
-/// like:
-///     8           # 8-byte aligned
-///     7 mod 16    # 16-byte aligned plus 7 bytes
-template <> struct ScalarTraits<lld::DefinedAtom::Alignment> {
-  static void output(const lld::DefinedAtom::Alignment &value, void *ctxt,
-                     raw_ostream &out) {
-    if (value.modulus == 0) {
-      out << llvm::format("%d", value.value);
-    } else {
-      out << llvm::format("%d mod %d", value.modulus, value.value);
-    }
-  }
-
-  static StringRef input(StringRef scalar, void *ctxt,
-                         lld::DefinedAtom::Alignment &value) {
-    value.modulus = 0;
-    size_t modStart = scalar.find("mod");
-    if (modStart != StringRef::npos) {
-      StringRef modStr = scalar.slice(0, modStart);
-      modStr = modStr.rtrim();
-      unsigned int modulus;
-      if (modStr.getAsInteger(0, modulus)) {
-        return "malformed alignment modulus";
-      }
-      value.modulus = modulus;
-      scalar = scalar.drop_front(modStart + 3);
-      scalar = scalar.ltrim();
-    }
-    unsigned int power;
-    if (scalar.getAsInteger(0, power)) {
-      return "malformed alignment power";
-    }
-    value.value = power;
-    if (value.modulus >= power) {
-      return "malformed alignment, modulus too large for power";
-    }
-    return StringRef(); // returning empty string means success
-  }
-
-  static QuotingType mustQuote(StringRef) { return QuotingType::None; }
-};
-
-template <> struct ScalarEnumerationTraits<FileKinds> {
-  static void enumeration(IO &io, FileKinds &value) {
-    io.enumCase(value, "object",        fileKindObjectAtoms);
-    io.enumCase(value, "archive",       fileKindArchive);
-    io.enumCase(value, "object-mach-o", fileKindObjectMachO);
-  }
-};
-
-template <> struct MappingTraits<ArchMember> {
-  static void mapping(IO &io, ArchMember &member) {
-    io.mapOptional("kind",    member._kind, fileKindObjectAtoms);
-    io.mapOptional("name",    member._name);
-    io.mapRequired("content", member._content);
-  }
-};
-
-// Declare that an AtomList is a yaml sequence.
-template <typename T> struct SequenceTraits<AtomList<T> > {
-  static size_t size(IO &io, AtomList<T> &seq) { return seq._atoms.size(); }
-  static T *&element(IO &io, AtomList<T> &seq, size_t index) {
-    if (index >= seq._atoms.size())
-      seq._atoms.resize(index + 1);
-    return seq._atoms[index].get();
-  }
-};
-
-// Declare that an AtomRange is a yaml sequence.
-template <typename T> struct SequenceTraits<File::AtomRange<T> > {
-  static size_t size(IO &io, File::AtomRange<T> &seq) { return seq.size(); }
-  static T *&element(IO &io, File::AtomRange<T> &seq, size_t index) {
-    assert(io.outputting() && "AtomRange only used when outputting");
-    assert(index < seq.size() && "Out of range access");
-    return seq[index].get();
-  }
-};
-
-// Used to allow DefinedAtom content bytes to be a flow sequence of
-// two-digit hex numbers without the leading 0x (e.g. FF, 04, 0A)
-template <> struct ScalarTraits<ImplicitHex8> {
-  static void output(const ImplicitHex8 &val, void *, raw_ostream &out) {
-    uint8_t num = val;
-    out << llvm::format("%02X", num);
-  }
-
-  static StringRef input(StringRef str, void *, ImplicitHex8 &val) {
-    unsigned long long n;
-    if (getAsUnsignedInteger(str, 16, n))
-      return "invalid two-digit-hex number";
-    if (n > 0xFF)
-      return "out of range two-digit-hex number";
-    val = n;
-    return StringRef(); // returning empty string means success
-  }
-
-  static QuotingType mustQuote(StringRef) { return QuotingType::None; }
-};
-
-// YAML conversion for std::vector<const lld::File*>
-template <> struct DocumentListTraits<std::vector<const lld::File *> > {
-  static size_t size(IO &io, std::vector<const lld::File *> &seq) {
-    return seq.size();
-  }
-  static const lld::File *&element(IO &io, std::vector<const lld::File *> &seq,
-                                   size_t index) {
-    if (index >= seq.size())
-      seq.resize(index + 1);
-    return seq[index];
-  }
-};
-
-// YAML conversion for const lld::File*
-template <> struct MappingTraits<const lld::File *> {
-  class NormArchiveFile : public lld::ArchiveLibraryFile {
-  public:
-    NormArchiveFile(IO &io) : ArchiveLibraryFile("") {}
-
-    NormArchiveFile(IO &io, const lld::File *file)
-        : ArchiveLibraryFile(file->path()), _path(file->path()) {
-      // If we want to support writing archives, this constructor would
-      // need to populate _members.
-    }
-
-    const lld::File *denormalize(IO &io) { return this; }
-
-    const AtomRange<lld::DefinedAtom> defined() const override {
-      return _noDefinedAtoms;
-    }
-
-    const AtomRange<lld::UndefinedAtom> undefined() const override {
-      return _noUndefinedAtoms;
-    }
-
-    const AtomRange<lld::SharedLibraryAtom> sharedLibrary() const override {
-      return _noSharedLibraryAtoms;
-    }
-
-    const AtomRange<lld::AbsoluteAtom> absolute() const override {
-      return _noAbsoluteAtoms;
-    }
-
-    void clearAtoms() override {
-      _noDefinedAtoms.clear();
-      _noUndefinedAtoms.clear();
-      _noSharedLibraryAtoms.clear();
-      _noAbsoluteAtoms.clear();
-    }
-
-    File *find(StringRef name) override {
-      for (const ArchMember &member : _members)
-        for (const lld::DefinedAtom *atom : member._content->defined())
-          if (name == atom->name())
-            return const_cast<File *>(member._content);
-      return nullptr;
-    }
-
-    std::error_code
-    parseAllMembers(std::vector<std::unique_ptr<File>> &result) override {
-      return std::error_code();
-    }
-
-    StringRef               _path;
-    std::vector<ArchMember> _members;
-  };
-
-  class NormalizedFile : public lld::File {
-  public:
-    NormalizedFile(IO &io)
-      : File("", kindNormalizedObject), _io(io), _rnb(nullptr),
-        _definedAtomsRef(_definedAtoms._atoms),
-        _undefinedAtomsRef(_undefinedAtoms._atoms),
-        _sharedLibraryAtomsRef(_sharedLibraryAtoms._atoms),
-        _absoluteAtomsRef(_absoluteAtoms._atoms) {}
-
-    NormalizedFile(IO &io, const lld::File *file)
-        : File(file->path(), kindNormalizedObject), _io(io),
-          _rnb(new RefNameBuilder(*file)), _path(file->path()),
-        _definedAtomsRef(file->defined()),
-        _undefinedAtomsRef(file->undefined()),
-        _sharedLibraryAtomsRef(file->sharedLibrary()),
-        _absoluteAtomsRef(file->absolute()) {
-    }
-
-    ~NormalizedFile() override {
-    }
-
-    const lld::File *denormalize(IO &io);
-
-    const AtomRange<lld::DefinedAtom> defined() const override {
-      return _definedAtomsRef;
-    }
-
-    const AtomRange<lld::UndefinedAtom> undefined() const override {
-      return _undefinedAtomsRef;
-    }
-
-    const AtomRange<lld::SharedLibraryAtom> sharedLibrary() const override {
-      return _sharedLibraryAtomsRef;
-    }
-
-    const AtomRange<lld::AbsoluteAtom> absolute() const override {
-      return _absoluteAtomsRef;
-    }
-
-    void clearAtoms() override {
-      _definedAtoms._atoms.clear();
-      _undefinedAtoms._atoms.clear();
-      _sharedLibraryAtoms._atoms.clear();
-      _absoluteAtoms._atoms.clear();
-    }
-
-    // Allocate a new copy of this string in _storage, so the strings
-    // can be freed when File is destroyed.
-    StringRef copyString(StringRef str) {
-      char *s = _storage.Allocate<char>(str.size());
-      memcpy(s, str.data(), str.size());
-      return StringRef(s, str.size());
-    }
-
-    IO                                  &_io;
-    std::unique_ptr<RefNameBuilder>      _rnb;
-    StringRef                            _path;
-    AtomList<lld::DefinedAtom>           _definedAtoms;
-    AtomList<lld::UndefinedAtom>         _undefinedAtoms;
-    AtomList<lld::SharedLibraryAtom>     _sharedLibraryAtoms;
-    AtomList<lld::AbsoluteAtom>          _absoluteAtoms;
-    AtomRange<lld::DefinedAtom>          _definedAtomsRef;
-    AtomRange<lld::UndefinedAtom>        _undefinedAtomsRef;
-    AtomRange<lld::SharedLibraryAtom>    _sharedLibraryAtomsRef;
-    AtomRange<lld::AbsoluteAtom>         _absoluteAtomsRef;
-    llvm::BumpPtrAllocator               _storage;
-  };
-
-  static void mapping(IO &io, const lld::File *&file) {
-    YamlContext *info = reinterpret_cast<YamlContext *>(io.getContext());
-    assert(info != nullptr);
-    // Let any register tag handler process this.
-    if (info->_registry && info->_registry->handleTaggedDoc(io, file))
-      return;
-    // If no registered handler claims this tag and there is no tag,
-    // grandfather in as "!native".
-    if (io.mapTag("!native", true) || io.mapTag("tag:yaml.org,2002:map"))
-      mappingAtoms(io, file);
-  }
-
-  static void mappingAtoms(IO &io, const lld::File *&file) {
-    YamlContext *info = reinterpret_cast<YamlContext *>(io.getContext());
-    MappingNormalizationHeap<NormalizedFile, const lld::File *>
-      keys(io, file, nullptr);
-    assert(info != nullptr);
-    info->_file = keys.operator->();
-
-    io.mapOptional("path",                 keys->_path);
-
-    if (io.outputting()) {
-      io.mapOptional("defined-atoms",        keys->_definedAtomsRef);
-      io.mapOptional("undefined-atoms",      keys->_undefinedAtomsRef);
-      io.mapOptional("shared-library-atoms", keys->_sharedLibraryAtomsRef);
-      io.mapOptional("absolute-atoms",       keys->_absoluteAtomsRef);
-    } else {
-      io.mapOptional("defined-atoms",        keys->_definedAtoms);
-      io.mapOptional("undefined-atoms",      keys->_undefinedAtoms);
-      io.mapOptional("shared-library-atoms", keys->_sharedLibraryAtoms);
-      io.mapOptional("absolute-atoms",       keys->_absoluteAtoms);
-    }
-  }
-
-  static void mappingArchive(IO &io, const lld::File *&file) {
-    YamlContext *info = reinterpret_cast<YamlContext *>(io.getContext());
-    MappingNormalizationHeap<NormArchiveFile, const lld::File *>
-      keys(io, file, &info->_file->allocator());
-
-    io.mapOptional("path",    keys->_path);
-    io.mapOptional("members", keys->_members);
-  }
-};
-
-// YAML conversion for const lld::Reference*
-template <> struct MappingTraits<const lld::Reference *> {
-  class NormalizedReference : public lld::Reference {
-  public:
-    NormalizedReference(IO &io)
-        : lld::Reference(lld::Reference::KindNamespace::all,
-                         lld::Reference::KindArch::all, 0),
-          _target(nullptr), _offset(0), _addend(0), _tag(0) {}
-
-    NormalizedReference(IO &io, const lld::Reference *ref)
-        : lld::Reference(ref->kindNamespace(), ref->kindArch(),
-                         ref->kindValue()),
-          _target(nullptr), _targetName(targetName(io, ref)),
-          _offset(ref->offsetInAtom()), _addend(ref->addend()),
-          _tag(ref->tag()) {
-      _mappedKind.ns = ref->kindNamespace();
-      _mappedKind.arch = ref->kindArch();
-      _mappedKind.value = ref->kindValue();
-    }
-
-    const lld::Reference *denormalize(IO &io) {
-      YamlContext *info = reinterpret_cast<YamlContext *>(io.getContext());
-      assert(info != nullptr);
-      typedef MappingTraits<const lld::File *>::NormalizedFile NormalizedFile;
-      NormalizedFile *f = reinterpret_cast<NormalizedFile *>(info->_file);
-      if (!_targetName.empty())
-        _targetName = f->copyString(_targetName);
-      DEBUG_WITH_TYPE("WriterYAML", llvm::dbgs()
-                                        << "created Reference to name: '"
-                                        << _targetName << "' ("
-                                        << (const void *)_targetName.data()
-                                        << ", " << _targetName.size() << ")\n");
-      setKindNamespace(_mappedKind.ns);
-      setKindArch(_mappedKind.arch);
-      setKindValue(_mappedKind.value);
-      return this;
-    }
-
-    void bind(const RefNameResolver &);
-    static StringRef targetName(IO &io, const lld::Reference *ref);
-
-    uint64_t offsetInAtom() const override { return _offset; }
-    const lld::Atom *target() const override { return _target; }
-    Addend addend() const override { return _addend; }
-    void setAddend(Addend a) override { _addend = a; }
-    void setTarget(const lld::Atom *a) override { _target = a; }
-
-    const lld::Atom *_target;
-    StringRef        _targetName;
-    uint32_t         _offset;
-    Addend           _addend;
-    RefKind          _mappedKind;
-    uint32_t         _tag;
-  };
-
-  static void mapping(IO &io, const lld::Reference *&ref) {
-    YamlContext *info = reinterpret_cast<YamlContext *>(io.getContext());
-    MappingNormalizationHeap<NormalizedReference, const lld::Reference *> keys(
-        io, ref, &info->_file->allocator());
-
-    io.mapRequired("kind",   keys->_mappedKind);
-    io.mapOptional("offset", keys->_offset);
-    io.mapOptional("target", keys->_targetName);
-    io.mapOptional("addend", keys->_addend, (lld::Reference::Addend)0);
-    io.mapOptional("tag",    keys->_tag, 0u);
-  }
-};
-
-// YAML conversion for const lld::DefinedAtom*
-template <> struct MappingTraits<const lld::DefinedAtom *> {
-
-  class NormalizedAtom : public lld::DefinedAtom {
-  public:
-    NormalizedAtom(IO &io)
-        : _file(fileFromContext(io)), _contentType(), _alignment(1) {
-      static uint32_t ordinalCounter = 1;
-      _ordinal = ordinalCounter++;
-    }
-
-    NormalizedAtom(IO &io, const lld::DefinedAtom *atom)
-        : _file(fileFromContext(io)), _name(atom->name()),
-          _scope(atom->scope()), _interpose(atom->interposable()),
-          _merge(atom->merge()), _contentType(atom->contentType()),
-          _alignment(atom->alignment()), _sectionChoice(atom->sectionChoice()),
-          _deadStrip(atom->deadStrip()), _dynamicExport(atom->dynamicExport()),
-          _codeModel(atom->codeModel()),
-          _permissions(atom->permissions()), _size(atom->size()),
-          _sectionName(atom->customSectionName()),
-          _sectionSize(atom->sectionSize()) {
-      for (const lld::Reference *r : *atom)
-        _references.push_back(r);
-      if (!atom->occupiesDiskSpace())
-        return;
-      ArrayRef<uint8_t> cont = atom->rawContent();
-      _content.reserve(cont.size());
-      for (uint8_t x : cont)
-        _content.push_back(x);
-    }
-
-    ~NormalizedAtom() override = default;
-
-    const lld::DefinedAtom *denormalize(IO &io) {
-      YamlContext *info = reinterpret_cast<YamlContext *>(io.getContext());
-      assert(info != nullptr);
-      typedef MappingTraits<const lld::File *>::NormalizedFile NormalizedFile;
-      NormalizedFile *f = reinterpret_cast<NormalizedFile *>(info->_file);
-      if (!_name.empty())
-        _name = f->copyString(_name);
-      if (!_refName.empty())
-        _refName = f->copyString(_refName);
-      if (!_sectionName.empty())
-        _sectionName = f->copyString(_sectionName);
-      DEBUG_WITH_TYPE("WriterYAML",
-                      llvm::dbgs() << "created DefinedAtom named: '" << _name
-                                   << "' (" << (const void *)_name.data()
-                                   << ", " << _name.size() << ")\n");
-      return this;
-    }
-
-    void bind(const RefNameResolver &);
-
-    // Extract current File object from YAML I/O parsing context
-    const lld::File &fileFromContext(IO &io) {
-      YamlContext *info = reinterpret_cast<YamlContext *>(io.getContext());
-      assert(info != nullptr);
-      assert(info->_file != nullptr);
-      return *info->_file;
-    }
-
-    const lld::File &file() const override { return _file; }
-    StringRef name() const override { return _name; }
-    uint64_t size() const override { return _size; }
-    Scope scope() const override { return _scope; }
-    Interposable interposable() const override { return _interpose; }
-    Merge merge() const override { return _merge; }
-    ContentType contentType() const override { return _contentType; }
-    Alignment alignment() const override { return _alignment; }
-    SectionChoice sectionChoice() const override { return _sectionChoice; }
-    StringRef customSectionName() const override { return _sectionName; }
-    uint64_t sectionSize() const override { return _sectionSize; }
-    DeadStripKind deadStrip() const override { return _deadStrip; }
-    DynamicExport dynamicExport() const override { return _dynamicExport; }
-    CodeModel codeModel() const override { return _codeModel; }
-    ContentPermissions permissions() const override { return _permissions; }
-    ArrayRef<uint8_t> rawContent() const override {
-      if (!occupiesDiskSpace())
-        return ArrayRef<uint8_t>();
-      return ArrayRef<uint8_t>(
-          reinterpret_cast<const uint8_t *>(_content.data()), _content.size());
-    }
-
-    uint64_t ordinal() const override { return _ordinal; }
-
-    reference_iterator begin() const override {
-      uintptr_t index = 0;
-      const void *it = reinterpret_cast<const void *>(index);
-      return reference_iterator(*this, it);
-    }
-    reference_iterator end() const override {
-      uintptr_t index = _references.size();
-      const void *it = reinterpret_cast<const void *>(index);
-      return reference_iterator(*this, it);
-    }
-    const lld::Reference *derefIterator(const void *it) const override {
-      uintptr_t index = reinterpret_cast<uintptr_t>(it);
-      assert(index < _references.size());
-      return _references[index];
-    }
-    void incrementIterator(const void *&it) const override {
-      uintptr_t index = reinterpret_cast<uintptr_t>(it);
-      ++index;
-      it = reinterpret_cast<const void *>(index);
-    }
-
-    void addReference(Reference::KindNamespace ns,
-                      Reference::KindArch arch,
-                      Reference::KindValue kindValue, uint64_t off,
-                      const Atom *target, Reference::Addend a) override {
-      assert(target && "trying to create reference to nothing");
-      auto node = new (file().allocator()) SimpleReference(ns, arch, kindValue,
-                                                           off, target, a);
-      _references.push_back(node);
-    }
-
-    const lld::File                    &_file;
-    StringRef                           _name;
-    StringRef                           _refName;
-    Scope                               _scope;
-    Interposable                        _interpose;
-    Merge                               _merge;
-    ContentType                         _contentType;
-    Alignment                           _alignment;
-    SectionChoice                       _sectionChoice;
-    DeadStripKind                       _deadStrip;
-    DynamicExport                       _dynamicExport;
-    CodeModel                           _codeModel;
-    ContentPermissions                  _permissions;
-    uint32_t                            _ordinal;
-    std::vector<ImplicitHex8>           _content;
-    uint64_t                            _size;
-    StringRef                           _sectionName;
-    uint64_t                            _sectionSize;
-    std::vector<const lld::Reference *> _references;
-  };
-
-  static void mapping(IO &io, const lld::DefinedAtom *&atom) {
-    YamlContext *info = reinterpret_cast<YamlContext *>(io.getContext());
-    MappingNormalizationHeap<NormalizedAtom, const lld::DefinedAtom *> keys(
-        io, atom, &info->_file->allocator());
-    if (io.outputting()) {
-      // If writing YAML, check if atom needs a ref-name.
-      typedef MappingTraits<const lld::File *>::NormalizedFile NormalizedFile;
-      assert(info != nullptr);
-      NormalizedFile *f = reinterpret_cast<NormalizedFile *>(info->_file);
-      assert(f);
-      assert(f->_rnb);
-      if (f->_rnb->hasRefName(atom)) {
-        keys->_refName = f->_rnb->refName(atom);
-      }
-    }
-
-    io.mapOptional("name",             keys->_name,    StringRef());
-    io.mapOptional("ref-name",         keys->_refName, StringRef());
-    io.mapOptional("scope",            keys->_scope,
-                                         DefinedAtom::scopeTranslationUnit);
-    io.mapOptional("type",             keys->_contentType,
-                                         DefinedAtom::typeCode);
-    io.mapOptional("content",          keys->_content);
-    io.mapOptional("size",             keys->_size, (uint64_t)keys->_content.size());
-    io.mapOptional("interposable",     keys->_interpose,
-                                         DefinedAtom::interposeNo);
-    io.mapOptional("merge",            keys->_merge, DefinedAtom::mergeNo);
-    io.mapOptional("alignment",        keys->_alignment,
-                                         DefinedAtom::Alignment(1));
-    io.mapOptional("section-choice",   keys->_sectionChoice,
-                                         DefinedAtom::sectionBasedOnContent);
-    io.mapOptional("section-name",     keys->_sectionName, StringRef());
-    io.mapOptional("section-size",     keys->_sectionSize, (uint64_t)0);
-    io.mapOptional("dead-strip",       keys->_deadStrip,
-                                         DefinedAtom::deadStripNormal);
-    io.mapOptional("dynamic-export",   keys->_dynamicExport,
-                                         DefinedAtom::dynamicExportNormal);
-    io.mapOptional("code-model",       keys->_codeModel, DefinedAtom::codeNA);
-    // default permissions based on content type
-    io.mapOptional("permissions",      keys->_permissions,
-                                         DefinedAtom::permissions(
-                                                          keys->_contentType));
-    io.mapOptional("references",       keys->_references);
-  }
-};
-
-template <> struct MappingTraits<lld::DefinedAtom *> {
-  static void mapping(IO &io, lld::DefinedAtom *&atom) {
-    const lld::DefinedAtom *atomPtr = atom;
-    MappingTraits<const lld::DefinedAtom *>::mapping(io, atomPtr);
-    atom = const_cast<lld::DefinedAtom *>(atomPtr);
-  }
-};
-
-// YAML conversion for const lld::UndefinedAtom*
-template <> struct MappingTraits<const lld::UndefinedAtom *> {
-  class NormalizedAtom : public lld::UndefinedAtom {
-  public:
-    NormalizedAtom(IO &io)
-        : _file(fileFromContext(io)), _canBeNull(canBeNullNever) {}
-
-    NormalizedAtom(IO &io, const lld::UndefinedAtom *atom)
-        : _file(fileFromContext(io)), _name(atom->name()),
-          _canBeNull(atom->canBeNull()) {}
-
-    ~NormalizedAtom() override = default;
-
-    const lld::UndefinedAtom *denormalize(IO &io) {
-      YamlContext *info = reinterpret_cast<YamlContext *>(io.getContext());
-      assert(info != nullptr);
-      typedef MappingTraits<const lld::File *>::NormalizedFile NormalizedFile;
-      NormalizedFile *f = reinterpret_cast<NormalizedFile *>(info->_file);
-      if (!_name.empty())
-        _name = f->copyString(_name);
-
-      DEBUG_WITH_TYPE("WriterYAML",
-                      llvm::dbgs() << "created UndefinedAtom named: '" << _name
-                      << "' (" << (const void *)_name.data() << ", "
-                      << _name.size() << ")\n");
-      return this;
-    }
-
-    // Extract current File object from YAML I/O parsing context
-    const lld::File &fileFromContext(IO &io) {
-      YamlContext *info = reinterpret_cast<YamlContext *>(io.getContext());
-      assert(info != nullptr);
-      assert(info->_file != nullptr);
-      return *info->_file;
-    }
-
-    const lld::File &file() const override { return _file; }
-    StringRef name() const override { return _name; }
-    CanBeNull canBeNull() const override { return _canBeNull; }
-
-    const lld::File     &_file;
-    StringRef            _name;
-    CanBeNull            _canBeNull;
-  };
-
-  static void mapping(IO &io, const lld::UndefinedAtom *&atom) {
-    YamlContext *info = reinterpret_cast<YamlContext *>(io.getContext());
-    MappingNormalizationHeap<NormalizedAtom, const lld::UndefinedAtom *> keys(
-        io, atom, &info->_file->allocator());
-
-    io.mapRequired("name",        keys->_name);
-    io.mapOptional("can-be-null", keys->_canBeNull,
-                                  lld::UndefinedAtom::canBeNullNever);
-  }
-};
-
-template <> struct MappingTraits<lld::UndefinedAtom *> {
-  static void mapping(IO &io, lld::UndefinedAtom *&atom) {
-    const lld::UndefinedAtom *atomPtr = atom;
-    MappingTraits<const lld::UndefinedAtom *>::mapping(io, atomPtr);
-    atom = const_cast<lld::UndefinedAtom *>(atomPtr);
-  }
-};
-
-// YAML conversion for const lld::SharedLibraryAtom*
-template <> struct MappingTraits<const lld::SharedLibraryAtom *> {
-  class NormalizedAtom : public lld::SharedLibraryAtom {
-  public:
-    NormalizedAtom(IO &io)
-        : _file(fileFromContext(io)), _canBeNull(false),
-          _type(Type::Unknown), _size(0) {}
-
-    NormalizedAtom(IO &io, const lld::SharedLibraryAtom *atom)
-        : _file(fileFromContext(io)), _name(atom->name()),
-          _loadName(atom->loadName()), _canBeNull(atom->canBeNullAtRuntime()),
-          _type(atom->type()), _size(atom->size()) {}
-
-    ~NormalizedAtom() override = default;
-
-    const lld::SharedLibraryAtom *denormalize(IO &io) {
-      YamlContext *info = reinterpret_cast<YamlContext *>(io.getContext());
-      assert(info != nullptr);
-      typedef MappingTraits<const lld::File *>::NormalizedFile NormalizedFile;
-      NormalizedFile *f = reinterpret_cast<NormalizedFile *>(info->_file);
-      if (!_name.empty())
-        _name = f->copyString(_name);
-      if (!_loadName.empty())
-        _loadName = f->copyString(_loadName);
-
-      DEBUG_WITH_TYPE("WriterYAML",
-                      llvm::dbgs() << "created SharedLibraryAtom named: '"
-                                   << _name << "' ("
-                                   << (const void *)_name.data()
-                                   << ", " << _name.size() << ")\n");
-      return this;
-    }
-
-    // Extract current File object from YAML I/O parsing context
-    const lld::File &fileFromContext(IO &io) {
-      YamlContext *info = reinterpret_cast<YamlContext *>(io.getContext());
-      assert(info != nullptr);
-      assert(info->_file != nullptr);
-      return *info->_file;
-    }
-
-    const lld::File &file() const override { return _file; }
-    StringRef name() const override { return _name; }
-    StringRef loadName() const override { return _loadName; }
-    bool canBeNullAtRuntime() const override { return _canBeNull; }
-    Type type() const override { return _type; }
-    uint64_t size() const override { return _size; }
-
-    const lld::File &_file;
-    StringRef        _name;
-    StringRef        _loadName;
-    ShlibCanBeNull   _canBeNull;
-    Type             _type;
-    uint64_t         _size;
-  };
-
-  static void mapping(IO &io, const lld::SharedLibraryAtom *&atom) {
-
-    YamlContext *info = reinterpret_cast<YamlContext *>(io.getContext());
-    MappingNormalizationHeap<NormalizedAtom, const lld::SharedLibraryAtom *>
-    keys(io, atom, &info->_file->allocator());
-
-    io.mapRequired("name",        keys->_name);
-    io.mapOptional("load-name",   keys->_loadName);
-    io.mapOptional("can-be-null", keys->_canBeNull, (ShlibCanBeNull) false);
-    io.mapOptional("type",        keys->_type, SharedLibraryAtom::Type::Code);
-    io.mapOptional("size",        keys->_size, uint64_t(0));
-  }
-};
-
-template <> struct MappingTraits<lld::SharedLibraryAtom *> {
-  static void mapping(IO &io, lld::SharedLibraryAtom *&atom) {
-    const lld::SharedLibraryAtom *atomPtr = atom;
-    MappingTraits<const lld::SharedLibraryAtom *>::mapping(io, atomPtr);
-    atom = const_cast<lld::SharedLibraryAtom *>(atomPtr);
-  }
-};
-
-// YAML conversion for const lld::AbsoluteAtom*
-template <> struct MappingTraits<const lld::AbsoluteAtom *> {
-  class NormalizedAtom : public lld::AbsoluteAtom {
-  public:
-    NormalizedAtom(IO &io)
-        : _file(fileFromContext(io)), _scope(), _value(0) {}
-
-    NormalizedAtom(IO &io, const lld::AbsoluteAtom *atom)
-        : _file(fileFromContext(io)), _name(atom->name()),
-          _scope(atom->scope()), _value(atom->value()) {}
-
-    ~NormalizedAtom() override = default;
-
-    const lld::AbsoluteAtom *denormalize(IO &io) {
-      YamlContext *info = reinterpret_cast<YamlContext *>(io.getContext());
-      assert(info != nullptr);
-      typedef MappingTraits<const lld::File *>::NormalizedFile NormalizedFile;
-      NormalizedFile *f = reinterpret_cast<NormalizedFile *>(info->_file);
-      if (!_name.empty())
-        _name = f->copyString(_name);
-
-      DEBUG_WITH_TYPE("WriterYAML",
-                      llvm::dbgs() << "created AbsoluteAtom named: '" << _name
-                                   << "' (" << (const void *)_name.data()
-                                   << ", " << _name.size() << ")\n");
-      return this;
-    }
-
-    // Extract current File object from YAML I/O parsing context
-    const lld::File &fileFromContext(IO &io) {
-      YamlContext *info = reinterpret_cast<YamlContext *>(io.getContext());
-      assert(info != nullptr);
-      assert(info->_file != nullptr);
-      return *info->_file;
-    }
-
-    const lld::File &file() const override { return _file; }
-    StringRef name() const override { return _name; }
-    uint64_t value() const override { return _value; }
-    Scope scope() const override { return _scope; }
-
-    const lld::File &_file;
-    StringRef        _name;
-    StringRef        _refName;
-    Scope            _scope;
-    Hex64            _value;
-  };
-
-  static void mapping(IO &io, const lld::AbsoluteAtom *&atom) {
-    YamlContext *info = reinterpret_cast<YamlContext *>(io.getContext());
-    MappingNormalizationHeap<NormalizedAtom, const lld::AbsoluteAtom *> keys(
-        io, atom, &info->_file->allocator());
-
-    if (io.outputting()) {
-      typedef MappingTraits<const lld::File *>::NormalizedFile NormalizedFile;
-      YamlContext *info = reinterpret_cast<YamlContext *>(io.getContext());
-      assert(info != nullptr);
-      NormalizedFile *f = reinterpret_cast<NormalizedFile *>(info->_file);
-      assert(f);
-      assert(f->_rnb);
-      if (f->_rnb->hasRefName(atom)) {
-        keys->_refName = f->_rnb->refName(atom);
-      }
-    }
-
-    io.mapRequired("name",     keys->_name);
-    io.mapOptional("ref-name", keys->_refName, StringRef());
-    io.mapOptional("scope",    keys->_scope);
-    io.mapRequired("value",    keys->_value);
-  }
-};
-
-template <> struct MappingTraits<lld::AbsoluteAtom *> {
-  static void mapping(IO &io, lld::AbsoluteAtom *&atom) {
-    const lld::AbsoluteAtom *atomPtr = atom;
-    MappingTraits<const lld::AbsoluteAtom *>::mapping(io, atomPtr);
-    atom = const_cast<lld::AbsoluteAtom *>(atomPtr);
-  }
-};
-
-} // end namespace llvm
-} // end namespace yaml
-
-RefNameResolver::RefNameResolver(const lld::File *file, IO &io) : _io(io) {
-  typedef MappingTraits<const lld::DefinedAtom *>::NormalizedAtom
-  NormalizedAtom;
-  for (const lld::DefinedAtom *a : file->defined()) {
-    const auto *na = (const NormalizedAtom *)a;
-    if (!na->_refName.empty())
-      add(na->_refName, a);
-    else if (!na->_name.empty())
-      add(na->_name, a);
-  }
-
-  for (const lld::UndefinedAtom *a : file->undefined())
-    add(a->name(), a);
-
-  for (const lld::SharedLibraryAtom *a : file->sharedLibrary())
-    add(a->name(), a);
-
-  typedef MappingTraits<const lld::AbsoluteAtom *>::NormalizedAtom NormAbsAtom;
-  for (const lld::AbsoluteAtom *a : file->absolute()) {
-    const auto *na = (const NormAbsAtom *)a;
-    if (na->_refName.empty())
-      add(na->_name, a);
-    else
-      add(na->_refName, a);
-  }
-}
-
-inline const lld::File *
-MappingTraits<const lld::File *>::NormalizedFile::denormalize(IO &io) {
-  typedef MappingTraits<const lld::DefinedAtom *>::NormalizedAtom
-  NormalizedAtom;
-
-  RefNameResolver nameResolver(this, io);
-  // Now that all atoms are parsed, references can be bound.
-  for (const lld::DefinedAtom *a : this->defined()) {
-    auto *normAtom = (NormalizedAtom *)const_cast<DefinedAtom *>(a);
-    normAtom->bind(nameResolver);
-  }
-
-  return this;
-}
-
-inline void MappingTraits<const lld::DefinedAtom *>::NormalizedAtom::bind(
-    const RefNameResolver &resolver) {
-  typedef MappingTraits<const lld::Reference *>::NormalizedReference
-  NormalizedReference;
-  for (const lld::Reference *ref : _references) {
-    auto *normRef = (NormalizedReference *)const_cast<Reference *>(ref);
-    normRef->bind(resolver);
-  }
-}
-
-inline void MappingTraits<const lld::Reference *>::NormalizedReference::bind(
-    const RefNameResolver &resolver) {
-  _target = resolver.lookup(_targetName);
-}
-
-inline StringRef
-MappingTraits<const lld::Reference *>::NormalizedReference::targetName(
-    IO &io, const lld::Reference *ref) {
-  if (ref->target() == nullptr)
-    return StringRef();
-  YamlContext *info = reinterpret_cast<YamlContext *>(io.getContext());
-  assert(info != nullptr);
-  typedef MappingTraits<const lld::File *>::NormalizedFile NormalizedFile;
-  NormalizedFile *f = reinterpret_cast<NormalizedFile *>(info->_file);
-  RefNameBuilder &rnb = *f->_rnb;
-  if (rnb.hasRefName(ref->target()))
-    return rnb.refName(ref->target());
-  return ref->target()->name();
-}
-
-namespace lld {
-namespace yaml {
-
-class Writer : public lld::Writer {
-public:
-  Writer(const LinkingContext &context) : _ctx(context) {}
-
-  llvm::Error writeFile(const lld::File &file, StringRef outPath) override {
-    // Create stream to path.
-    std::error_code ec;
-    llvm::raw_fd_ostream out(outPath, ec, llvm::sys::fs::OF_TextWithCRLF);
-    if (ec)
-      return llvm::errorCodeToError(ec);
-
-    // Create yaml Output writer, using yaml options for context.
-    YamlContext yamlContext;
-    yamlContext._ctx = &_ctx;
-    yamlContext._registry = &_ctx.registry();
-    llvm::yaml::Output yout(out, &yamlContext);
-
-    // Write yaml output.
-    const lld::File *fileRef = &file;
-    yout << fileRef;
-
-    return llvm::Error::success();
-  }
-
-private:
-  const LinkingContext &_ctx;
-};
-
-} // end namespace yaml
-
-namespace {
-
-/// Handles !native tagged yaml documents.
-class NativeYamlIOTaggedDocumentHandler : public YamlIOTaggedDocumentHandler {
-  bool handledDocTag(llvm::yaml::IO &io, const lld::File *&file) const override {
-    if (io.mapTag("!native")) {
-      MappingTraits<const lld::File *>::mappingAtoms(io, file);
-      return true;
-    }
-    return false;
-  }
-};
-
-/// Handles !archive tagged yaml documents.
-class ArchiveYamlIOTaggedDocumentHandler : public YamlIOTaggedDocumentHandler {
-  bool handledDocTag(llvm::yaml::IO &io, const lld::File *&file) const override {
-    if (io.mapTag("!archive")) {
-      MappingTraits<const lld::File *>::mappingArchive(io, file);
-      return true;
-    }
-    return false;
-  }
-};
-
-class YAMLReader : public Reader {
-public:
-  YAMLReader(const Registry &registry) : _registry(registry) {}
-
-  bool canParse(file_magic magic, MemoryBufferRef mb) const override {
-    StringRef name = mb.getBufferIdentifier();
-    return name.endswith(".objtxt") || name.endswith(".yaml");
-  }
-
-  ErrorOr<std::unique_ptr<File>>
-  loadFile(std::unique_ptr<MemoryBuffer> mb,
-           const class Registry &) const override {
-    // Create YAML Input Reader.
-    YamlContext yamlContext;
-    yamlContext._registry = &_registry;
-    yamlContext._path = mb->getBufferIdentifier();
-    llvm::yaml::Input yin(mb->getBuffer(), &yamlContext);
-
-    // Fill vector with File objects created by parsing yaml.
-    std::vector<const lld::File *> createdFiles;
-    yin >> createdFiles;
-    assert(createdFiles.size() == 1);
-
-    // Error out now if there were parsing errors.
-    if (yin.error())
-      return make_error_code(lld::YamlReaderError::illegal_value);
-
-    std::shared_ptr<MemoryBuffer> smb(mb.release());
-    const File *file = createdFiles[0];
-    // Note: loadFile() should return vector of *const* File
-    File *f = const_cast<File *>(file);
-    f->setLastError(std::error_code());
-    f->setSharedMemoryBuffer(smb);
-    return std::unique_ptr<File>(f);
-  }
-
-private:
-  const Registry &_registry;
-};
-
-} // end anonymous namespace
-
-void Registry::addSupportYamlFiles() {
-  add(std::unique_ptr<Reader>(new YAMLReader(*this)));
-  add(std::unique_ptr<YamlIOTaggedDocumentHandler>(
-                                    new NativeYamlIOTaggedDocumentHandler()));
-  add(std::unique_ptr<YamlIOTaggedDocumentHandler>(
-                                    new ArchiveYamlIOTaggedDocumentHandler()));
-}
-
-std::unique_ptr<Writer> createWriterYAML(const LinkingContext &context) {
-  return std::unique_ptr<Writer>(new lld::yaml::Writer(context));
-}
-
-} // end namespace lld
diff --git a/lld/tools/lld/lld.cpp b/lld/tools/lld/lld.cpp
index bfd4370d3f2d..cad97f2153c2 100644
--- a/lld/tools/lld/lld.cpp
+++ b/lld/tools/lld/lld.cpp
@@ -48,11 +48,10 @@ using namespace llvm::sys;
 
 enum Flavor {
   Invalid,
-  Gnu,       // -flavor gnu
-  WinLink,   // -flavor link
-  Darwin,    // -flavor darwin
-  DarwinOld, // -flavor darwinold
-  Wasm,      // -flavor wasm
+  Gnu,     // -flavor gnu
+  WinLink, // -flavor link
+  Darwin,  // -flavor darwin
+  Wasm,    // -flavor wasm
 };
 
 [[noreturn]] static void die(const Twine &s) {
@@ -65,9 +64,7 @@ static Flavor getFlavor(StringRef s) {
       .CasesLower("ld", "ld.lld", "gnu", Gnu)
       .CasesLower("wasm", "ld-wasm", Wasm)
       .CaseLower("link", WinLink)
-      .CasesLower("ld64", "ld64.lld", "darwin", "darwinnew",
-                  "ld64.lld.darwinnew", Darwin)
-      .CasesLower("darwinold", "ld64.lld.darwinold", DarwinOld)
+      .CasesLower("ld64", "ld64.lld", "darwin", Darwin)
       .Default(Invalid);
 }
 
@@ -151,8 +148,6 @@ static int lldMain(int argc, const char **argv, llvm::raw_ostream &stdoutOS,
     return !coff::link(args, exitEarly, stdoutOS, stderrOS);
   case Darwin:
     return !macho::link(args, exitEarly, stdoutOS, stderrOS);
-  case DarwinOld:
-    return !mach_o::link(args, exitEarly, stdoutOS, stderrOS);
   case Wasm:
     return !lld::wasm::link(args, exitEarly, stdoutOS, stderrOS);
   default:
diff --git a/lldb/bindings/interface/SBData.i b/lldb/bindings/interface/SBData.i
index a1fb4472cd23..1f2f9fbf05e2 100644
--- a/lldb/bindings/interface/SBData.i
+++ b/lldb/bindings/interface/SBData.i
@@ -96,6 +96,10 @@ public:
     void
     SetData (lldb::SBError& error, const void *buf, size_t size, lldb::ByteOrder endian, uint8_t addr_size);
 
+    void
+    SetDataWithOwnership (lldb::SBError& error, const void *buf, size_t size,
+                          lldb::ByteOrder endian, uint8_t addr_size);
+
     bool
     Append (const SBData& rhs);
 
diff --git a/lldb/bindings/lua/lua-swigsafecast.swig b/lldb/bindings/lua/lua-swigsafecast.swig
index 0b67c41434e9..35cb5e22a4c5 100644
--- a/lldb/bindings/lua/lua-swigsafecast.swig
+++ b/lldb/bindings/lua/lua-swigsafecast.swig
@@ -1,27 +1,20 @@
-template <typename SBClass>
-void
-PushSBClass (lua_State* L, SBClass* obj);
+template <typename SBClass> void PushSBClass(lua_State *L, SBClass *obj);
 
-void
-PushSBClass (lua_State* L, lldb::SBFrame* frame_sb)
-{
-   SWIG_NewPointerObj(L, frame_sb, SWIGTYPE_p_lldb__SBFrame, 0);
+void PushSBClass(lua_State *L, lldb::SBFrame *frame_sb) {
+  SWIG_NewPointerObj(L, frame_sb, SWIGTYPE_p_lldb__SBFrame, 0);
 }
 
-void
-PushSBClass (lua_State* L, lldb::SBBreakpointLocation* breakpoint_location_sb)
-{
-   SWIG_NewPointerObj(L, breakpoint_location_sb, SWIGTYPE_p_lldb__SBBreakpointLocation, 0);
+void PushSBClass(lua_State *L,
+                 lldb::SBBreakpointLocation *breakpoint_location_sb) {
+  SWIG_NewPointerObj(L, breakpoint_location_sb,
+                     SWIGTYPE_p_lldb__SBBreakpointLocation, 0);
 }
 
-void
-PushSBClass (lua_State* L, lldb::SBWatchpoint* watchpoint_sb)
-{
-   SWIG_NewPointerObj(L, watchpoint_sb, SWIGTYPE_p_lldb__SBWatchpoint, 0);
+void PushSBClass(lua_State *L, lldb::SBWatchpoint *watchpoint_sb) {
+  SWIG_NewPointerObj(L, watchpoint_sb, SWIGTYPE_p_lldb__SBWatchpoint, 0);
 }
 
-void
-PushSBClass (lua_State* L, lldb::SBStructuredData* structured_data_sb)
-{
-   SWIG_NewPointerObj(L, structured_data_sb, SWIGTYPE_p_lldb__SBStructuredData, 0);
+void PushSBClass(lua_State *L, lldb::SBStructuredData *structured_data_sb) {
+  SWIG_NewPointerObj(L, structured_data_sb, SWIGTYPE_p_lldb__SBStructuredData,
+                     0);
 }
diff --git a/lldb/bindings/lua/lua-typemaps.swig b/lldb/bindings/lua/lua-typemaps.swig
index e3b3f5718d15..15a18deaa3a4 100644
--- a/lldb/bindings/lua/lua-typemaps.swig
+++ b/lldb/bindings/lua/lua-typemaps.swig
@@ -76,11 +76,11 @@ LLDB_NUMBER_TYPEMAP(enum SWIGTYPE);
 
 // typemap for a char buffer
 %typemap(in) (char *dst, size_t dst_len) {
-   $2 = luaL_checkinteger(L, $input);
-   if ($2 <= 0) {
-      return luaL_error(L, "Positive integer expected");
-   }
-   $1 = (char *) malloc($2);
+  $2 = luaL_checkinteger(L, $input);
+  if ($2 <= 0) {
+    return luaL_error(L, "Positive integer expected");
+  }
+  $1 = (char *)malloc($2);
 }
 
 // SBProcess::ReadCStringFromMemory() uses a void*, but needs to be treated
@@ -92,14 +92,14 @@ LLDB_NUMBER_TYPEMAP(enum SWIGTYPE);
 
 // Return the char buffer.  Discarding any previous return result
 %typemap(argout) (char *dst, size_t dst_len) {
-   lua_pop(L, 1); // Blow away the previous result
-   if ($result == 0) {
-      lua_pushliteral(L, "");
-   } else {
-      lua_pushlstring(L, (const char *)$1, $result);
-   }
-   free($1);
-   // SWIG_arg was already incremented
+  lua_pop(L, 1); // Blow away the previous result
+  if ($result == 0) {
+    lua_pushliteral(L, "");
+  } else {
+    lua_pushlstring(L, (const char *)$1, $result);
+  }
+  free($1);
+  // SWIG_arg was already incremented
 }
 
 // SBProcess::ReadCStringFromMemory() uses a void*, but needs to be treated
@@ -114,18 +114,18 @@ LLDB_NUMBER_TYPEMAP(enum SWIGTYPE);
 // Typemap for handling a snprintf-like API like SBThread::GetStopDescription.
 
 %typemap(in) (char *dst_or_null, size_t dst_len) {
-   $2 = luaL_checkinteger(L, $input);
-   if ($2 <= 0) {
-      return luaL_error(L, "Positive integer expected");
-   }
-   $1 = (char *)malloc($2);
+  $2 = luaL_checkinteger(L, $input);
+  if ($2 <= 0) {
+    return luaL_error(L, "Positive integer expected");
+  }
+  $1 = (char *)malloc($2);
 }
 
 %typemap(argout) (char *dst_or_null, size_t dst_len) {
-   lua_pop(L, 1); // Blow away the previous result
-   lua_pushlstring(L, (const char *)$1, $result);
-   free($1);
-   // SWIG_arg was already incremented
+  lua_pop(L, 1); // Blow away the previous result
+  lua_pushlstring(L, (const char *)$1, $result);
+  free($1);
+  // SWIG_arg was already incremented
 }
 
 //===----------------------------------------------------------------------===//
@@ -133,22 +133,22 @@ LLDB_NUMBER_TYPEMAP(enum SWIGTYPE);
 // Typemap for handling SBModule::GetVersion
 
 %typemap(in) (uint32_t *versions, uint32_t num_versions) {
-   $2 = 99;
-   $1 = (uint32_t *)malloc(sizeof(uint32_t) * $2);
+  $2 = 99;
+  $1 = (uint32_t *)malloc(sizeof(uint32_t) * $2);
 }
 
 %typemap(argout) (uint32_t *versions, uint32_t num_versions) {
-   uint32_t count = result;
-   if (count >= $2)
-      count = $2;
-   lua_newtable(L);
-   int i = 0;
-   while (i++ < count) {
-      lua_pushinteger(L, $1[i - 1]);
-      lua_seti(L, -2, i);
-   }
-   SWIG_arg++;
-   free($1);
+  uint32_t count = result;
+  if (count >= $2)
+    count = $2;
+  lua_newtable(L);
+  int i = 0;
+  while (i++ < count) {
+    lua_pushinteger(L, $1[i - 1]);
+    lua_seti(L, -2, i);
+  }
+  SWIG_arg++;
+  free($1);
 }
 
 //===----------------------------------------------------------------------===//
@@ -156,15 +156,15 @@ LLDB_NUMBER_TYPEMAP(enum SWIGTYPE);
 // Typemap for handling SBDebugger::SetLoggingCallback
 
 %typemap(in) (lldb::LogOutputCallback log_callback, void *baton) {
-   $1 = LLDBSwigLuaCallLuaLogOutputCallback;
-   $2 = (void *)L;
+  $1 = LLDBSwigLuaCallLuaLogOutputCallback;
+  $2 = (void *)L;
 
-   luaL_checktype(L, 2, LUA_TFUNCTION);
-   lua_settop(L, 2);
+  luaL_checktype(L, 2, LUA_TFUNCTION);
+  lua_settop(L, 2);
 
-   lua_pushlightuserdata(L, (void *)&LLDBSwigLuaCallLuaLogOutputCallback);
-   lua_insert(L, 2);
-   lua_settable(L, LUA_REGISTRYINDEX);
+  lua_pushlightuserdata(L, (void *)&LLDBSwigLuaCallLuaLogOutputCallback);
+  lua_insert(L, 2);
+  lua_settable(L, LUA_REGISTRYINDEX);
 }
 
 //===----------------------------------------------------------------------===//
@@ -172,20 +172,20 @@ LLDB_NUMBER_TYPEMAP(enum SWIGTYPE);
 // Typemap for handling SBEvent::SBEvent(uint32_t event, const char *cstr, uint32_t cstr_len)
 
 %typemap(in) (const char *cstr, uint32_t cstr_len) {
-   $1 = (char *)luaL_checklstring(L, $input, (size_t *)&$2);
+  $1 = (char *)luaL_checklstring(L, $input, (size_t *)&$2);
 }
 
 // Typemap for handling SBProcess::PutSTDIN
 
 %typemap(in) (const char *src, size_t src_len) {
-   $1 = (char *)luaL_checklstring(L, $input, &$2);
+  $1 = (char *)luaL_checklstring(L, $input, &$2);
 }
 
 // Typemap for handling SBProcess::WriteMemory, SBTarget::GetInstructions...
 
 %typemap(in) (const void *buf, size_t size),
              (const void *data, size_t data_len) {
-   $1 = (void *)luaL_checklstring(L, $input, &$2);
+  $1 = (void *)luaL_checklstring(L, $input, &$2);
 }
 
 //===----------------------------------------------------------------------===//
@@ -195,35 +195,35 @@ LLDB_NUMBER_TYPEMAP(enum SWIGTYPE);
 // It should accept a Lua table of strings, for stuff like "argv" and "envp".
 
 %typemap(in) char ** {
-   if (lua_istable(L, $input)) {
-      size_t size = lua_rawlen(L, $input);
-      $1 = (char **)malloc((size + 1) * sizeof(char *));
-      int i = 0, j = 0;
-      while (i++ < size) {
-         lua_rawgeti(L, $input, i);
-         if (!lua_isstring(L, -1)) {
-            // if current element cannot be converted to string, raise an error
-            lua_pop(L, 1);
-            return luaL_error(L, "List should only contain strings");
-         }
-         $1[j++] = (char *)lua_tostring(L, -1);
-         lua_pop(L, 1);
+  if (lua_istable(L, $input)) {
+    size_t size = lua_rawlen(L, $input);
+    $1 = (char **)malloc((size + 1) * sizeof(char *));
+    int i = 0, j = 0;
+    while (i++ < size) {
+      lua_rawgeti(L, $input, i);
+      if (!lua_isstring(L, -1)) {
+        // if current element cannot be converted to string, raise an error
+        lua_pop(L, 1);
+        return luaL_error(L, "List should only contain strings");
       }
-      $1[j] = 0;
-   } else if (lua_isnil(L, $input)) {
-      // "nil" is also acceptable, equivalent as an empty table
-      $1 = NULL;
-   } else {
-      return luaL_error(L, "A list of strings expected");
-   }
+      $1[j++] = (char *)lua_tostring(L, -1);
+      lua_pop(L, 1);
+    }
+    $1[j] = 0;
+  } else if (lua_isnil(L, $input)) {
+    // "nil" is also acceptable, equivalent as an empty table
+    $1 = NULL;
+  } else {
+    return luaL_error(L, "A list of strings expected");
+  }
 }
 
 %typemap(freearg) char ** {
-   free((char *) $1);
+  free((char *) $1);
 }
 
 %typecheck(SWIG_TYPECHECK_STRING_ARRAY) char ** {
-   $1 = (lua_istable(L, $input) || lua_isnil(L, $input));
+  $1 = (lua_istable(L, $input) || lua_isnil(L, $input));
 }
 
 //===----------------------------------------------------------------------===//
@@ -231,30 +231,30 @@ LLDB_NUMBER_TYPEMAP(enum SWIGTYPE);
 // Typemap for file handles (e.g. used in SBDebugger::SetOutputFile)
 
 %typemap(in) lldb::FileSP {
-   luaL_Stream *p = (luaL_Stream *)luaL_checkudata(L, $input, LUA_FILEHANDLE);
-   lldb::FileSP file_sp;
-   file_sp = std::make_shared<lldb_private::NativeFile>(p->f, false);
-   if (!file_sp->IsValid())
-      return luaL_error(L, "Invalid file");
-   $1 = file_sp;
+  luaL_Stream *p = (luaL_Stream *)luaL_checkudata(L, $input, LUA_FILEHANDLE);
+  lldb::FileSP file_sp;
+  file_sp = std::make_shared<lldb_private::NativeFile>(p->f, false);
+  if (!file_sp->IsValid())
+    return luaL_error(L, "Invalid file");
+  $1 = file_sp;
 }
 
 %typecheck(SWIG_TYPECHECK_POINTER) lldb::FileSP {
-   $1 = (lua_isuserdata(L, $input)) &&
-        (luaL_testudata(L, $input, LUA_FILEHANDLE) != nullptr);
+  $1 = (lua_isuserdata(L, $input)) &&
+       (luaL_testudata(L, $input, LUA_FILEHANDLE) != nullptr);
 }
 
 // Typemap for file handles (e.g. used in SBDebugger::GetOutputFileHandle)
 
 %typemap(out) lldb::FileSP {
-   lldb::FileSP &sp = $1;
-   if (sp && sp->IsValid()) {
-      luaL_Stream *p = (luaL_Stream *)lua_newuserdata(L, sizeof(luaL_Stream));
-      p->closef = &LLDBSwigLuaCloseFileHandle;
-      p->f = sp->GetStream();
-      luaL_setmetatable(L, LUA_FILEHANDLE);
-      SWIG_arg++;
-   }
+  lldb::FileSP &sp = $1;
+  if (sp && sp->IsValid()) {
+    luaL_Stream *p = (luaL_Stream *)lua_newuserdata(L, sizeof(luaL_Stream));
+    p->closef = &LLDBSwigLuaCloseFileHandle;
+    p->f = sp->GetStream();
+    luaL_setmetatable(L, LUA_FILEHANDLE);
+    SWIG_arg++;
+  }
 }
 
 //===----------------------------------------------------------------------===//
@@ -266,29 +266,29 @@ LLDB_NUMBER_TYPEMAP(enum SWIGTYPE);
              (int64_t* array, size_t array_len),
              (int32_t* array, size_t array_len),
              (double* array, size_t array_len) {
-   if (lua_istable(L, $input)) {
-      // It should accept a table of numbers.
-      $2 = lua_rawlen(L, $input);
-      $1 = ($1_ltype)malloc(($2) * sizeof($*1_type));
-      int i = 0, j = 0;
-      while (i++ < $2) {
-         lua_rawgeti(L, $input, i);
-         if (!lua_isnumber(L, -1)) {
-            // if current element cannot be converted to number, raise an error
-            lua_pop(L, 1);
-            return luaL_error(L, "List should only contain numbers");
-         }
-         $1[j++] = ($*1_ltype)lua_tonumber(L, -1);
-         lua_pop(L, 1);
+  if (lua_istable(L, $input)) {
+    // It should accept a table of numbers.
+    $2 = lua_rawlen(L, $input);
+    $1 = ($1_ltype)malloc(($2) * sizeof($*1_type));
+    int i = 0, j = 0;
+    while (i++ < $2) {
+      lua_rawgeti(L, $input, i);
+      if (!lua_isnumber(L, -1)) {
+        // if current element cannot be converted to number, raise an error
+        lua_pop(L, 1);
+        return luaL_error(L, "List should only contain numbers");
       }
-   } else if (lua_isnil(L, $input)) {
-      // "nil" is also acceptable, equivalent as an empty table
-      $1 = NULL;
-      $2 = 0;
-   } else {
-      // else raise an error
-      return luaL_error(L, "A list of numbers expected.");
-   }
+      $1[j++] = ($*1_ltype) lua_tonumber(L, -1);
+      lua_pop(L, 1);
+    }
+  } else if (lua_isnil(L, $input)) {
+    // "nil" is also acceptable, equivalent as an empty table
+    $1 = NULL;
+    $2 = 0;
+  } else {
+    // else raise an error
+    return luaL_error(L, "A list of numbers expected.");
+  }
 }
 
 %typemap(freearg) (uint64_t* array, size_t array_len),
@@ -296,7 +296,7 @@ LLDB_NUMBER_TYPEMAP(enum SWIGTYPE);
              (int64_t* array, size_t array_len),
              (int32_t* array, size_t array_len),
              (double* array, size_t array_len) {
-   free($1);
+  free($1);
 }
 
 //===----------------------------------------------------------------------===//
@@ -304,13 +304,12 @@ LLDB_NUMBER_TYPEMAP(enum SWIGTYPE);
 // Typemap for SBCommandReturnObject::PutCString
 
 %typemap(in) (const char *string, int len) {
-   if (lua_isnil(L, $input)) {
-      $1 = NULL;
-      $2 = 0;
-   }
-   else {
-      $1 = (char *)luaL_checklstring(L, $input, (size_t *)&$2);
-   }
+  if (lua_isnil(L, $input)) {
+    $1 = NULL;
+    $2 = 0;
+  } else {
+    $1 = (char *)luaL_checklstring(L, $input, (size_t *)&$2);
+  }
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/lldb/bindings/lua/lua-wrapper.swig b/lldb/bindings/lua/lua-wrapper.swig
index c51911bb6bf7..a36e69a6fc93 100644
--- a/lldb/bindings/lua/lua-wrapper.swig
+++ b/lldb/bindings/lua/lua-wrapper.swig
@@ -1,120 +1,87 @@
 %header %{
 
-template <typename T>
-void
-PushSBClass(lua_State* L, T* obj);
-
-%}
-
-%runtime %{
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-void LLDBSwigLuaCallLuaLogOutputCallback(const char *str, void *baton);
-int LLDBSwigLuaCloseFileHandle(lua_State *L);
-
-#ifdef __cplusplus
-}
-#endif
-%}
-
-%wrapper %{
+template <typename T> void PushSBClass(lua_State * L, T * obj);
 
 // This function is called from Lua::CallBreakpointCallback
-SWIGEXPORT llvm::Expected<bool>
-LLDBSwigLuaBreakpointCallbackFunction
-(
-   lua_State *L,
-   lldb::StackFrameSP stop_frame_sp,
-   lldb::BreakpointLocationSP bp_loc_sp,
-   StructuredDataImpl *extra_args_impl
-)
-{
-   lldb::SBFrame sb_frame(stop_frame_sp);
-   lldb::SBBreakpointLocation sb_bp_loc(bp_loc_sp);
-   int nargs = 2;
-
-   llvm::Optional<lldb::SBStructuredData> extra_args;
-   if (extra_args_impl)
-      extra_args = lldb::SBStructuredData(extra_args_impl);
-
-   // Push the Lua wrappers
-   PushSBClass(L, &sb_frame);
-   PushSBClass(L, &sb_bp_loc);
-
-   if (extra_args.hasValue()) {
-      PushSBClass(L, extra_args.getPointer());
-      nargs++;
-   }
-
-   // Call into the Lua callback passing 'sb_frame' and 'sb_bp_loc'.
-   // Expects a boolean return.
-   if (lua_pcall(L, nargs, 1, 0) != LUA_OK) {
-      llvm::Error E = llvm::make_error<llvm::StringError>(
-            llvm::formatv("{0}\n", lua_tostring(L, -1)),
-            llvm::inconvertibleErrorCode());
-      // Pop error message from the stack.
-      lua_pop(L, 1);
-      return std::move(E);
-   }
-
-   // Boolean return from the callback
-   bool stop = lua_toboolean(L, -1);
-   lua_pop(L, 1);
-
-   return stop;
+llvm::Expected<bool> lldb_private::LLDBSwigLuaBreakpointCallbackFunction(
+    lua_State * L, lldb::StackFrameSP stop_frame_sp,
+    lldb::BreakpointLocationSP bp_loc_sp,
+    const StructuredDataImpl &extra_args_impl) {
+  lldb::SBFrame sb_frame(stop_frame_sp);
+  lldb::SBBreakpointLocation sb_bp_loc(bp_loc_sp);
+  int nargs = 2;
+
+  lldb::SBStructuredData extra_args(extra_args_impl);
+
+  // Push the Lua wrappers
+  PushSBClass(L, &sb_frame);
+  PushSBClass(L, &sb_bp_loc);
+
+  if (extra_args.IsValid()) {
+    PushSBClass(L, &extra_args);
+    nargs++;
+  }
+
+  // Call into the Lua callback passing 'sb_frame' and 'sb_bp_loc'.
+  // Expects a boolean return.
+  if (lua_pcall(L, nargs, 1, 0) != LUA_OK) {
+    llvm::Error E = llvm::make_error<llvm::StringError>(
+        llvm::formatv("{0}\n", lua_tostring(L, -1)),
+        llvm::inconvertibleErrorCode());
+    // Pop error message from the stack.
+    lua_pop(L, 1);
+    return std::move(E);
+  }
+
+  // Boolean return from the callback
+  bool stop = lua_toboolean(L, -1);
+  lua_pop(L, 1);
+
+  return stop;
 }
 
 // This function is called from Lua::CallWatchpointCallback
-SWIGEXPORT llvm::Expected<bool>
-LLDBSwigLuaWatchpointCallbackFunction
-(
-   lua_State *L,
-   lldb::StackFrameSP stop_frame_sp,
-   lldb::WatchpointSP wp_sp
-)
-{
-   lldb::SBFrame sb_frame(stop_frame_sp);
-   lldb::SBWatchpoint sb_wp(wp_sp);
-   int nargs = 2;
-
-   // Push the Lua wrappers
-   PushSBClass(L, &sb_frame);
-   PushSBClass(L, &sb_wp);
-
-   // Call into the Lua callback passing 'sb_frame' and 'sb_wp'.
-   // Expects a boolean return.
-   if (lua_pcall(L, nargs, 1, 0) != LUA_OK) {
-      llvm::Error E = llvm::make_error<llvm::StringError>(
-            llvm::formatv("{0}\n", lua_tostring(L, -1)),
-            llvm::inconvertibleErrorCode());
-      // Pop error message from the stack.
-      lua_pop(L, 1);
-      return std::move(E);
-   }
-
-   // Boolean return from the callback
-   bool stop = lua_toboolean(L, -1);
-   lua_pop(L, 1);
-
-   return stop;
+llvm::Expected<bool> lldb_private::LLDBSwigLuaWatchpointCallbackFunction(
+    lua_State * L, lldb::StackFrameSP stop_frame_sp, lldb::WatchpointSP wp_sp) {
+  lldb::SBFrame sb_frame(stop_frame_sp);
+  lldb::SBWatchpoint sb_wp(wp_sp);
+  int nargs = 2;
+
+  // Push the Lua wrappers
+  PushSBClass(L, &sb_frame);
+  PushSBClass(L, &sb_wp);
+
+  // Call into the Lua callback passing 'sb_frame' and 'sb_wp'.
+  // Expects a boolean return.
+  if (lua_pcall(L, nargs, 1, 0) != LUA_OK) {
+    llvm::Error E = llvm::make_error<llvm::StringError>(
+        llvm::formatv("{0}\n", lua_tostring(L, -1)),
+        llvm::inconvertibleErrorCode());
+    // Pop error message from the stack.
+    lua_pop(L, 1);
+    return std::move(E);
+  }
+
+  // Boolean return from the callback
+  bool stop = lua_toboolean(L, -1);
+  lua_pop(L, 1);
+
+  return stop;
 }
 
-SWIGEXPORT void
-LLDBSwigLuaCallLuaLogOutputCallback(const char *str, void *baton) {
-   lua_State *L = (lua_State *)baton;
+static void LLDBSwigLuaCallLuaLogOutputCallback(const char *str, void *baton) {
+  lua_State *L = (lua_State *)baton;
 
-   lua_pushlightuserdata(L, (void *)&LLDBSwigLuaCallLuaLogOutputCallback);
-   lua_gettable(L, LUA_REGISTRYINDEX);
+  lua_pushlightuserdata(L, (void *)&LLDBSwigLuaCallLuaLogOutputCallback);
+  lua_gettable(L, LUA_REGISTRYINDEX);
 
-   // FIXME: There's no way to report errors back to the user
-   lua_pushstring(L, str);
-   lua_pcall(L, 1, 0, 0);
+  // FIXME: There's no way to report errors back to the user
+  lua_pushstring(L, str);
+  lua_pcall(L, 1, 0, 0);
 }
 
-int LLDBSwigLuaCloseFileHandle(lua_State *L) {
-   return luaL_error(L, "You cannot close a file handle used by lldb.");
+static int LLDBSwigLuaCloseFileHandle(lua_State * L) {
+  return luaL_error(L, "You cannot close a file handle used by lldb.");
 }
 
 %}
diff --git a/lldb/bindings/lua/lua.swig b/lldb/bindings/lua/lua.swig
index 21fa44c8b4d8..92f85fee22fc 100644
--- a/lldb/bindings/lua/lua.swig
+++ b/lldb/bindings/lua/lua.swig
@@ -17,6 +17,7 @@
 #include "llvm/Support/Error.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "../bindings/lua/lua-swigsafecast.swig"
+#include "../source/Plugins/ScriptInterpreter/Lua/SWIGLuaBridge.h"
 
 // required headers for typemaps
 #include "lldb/Host/File.h"
diff --git a/lldb/bindings/python/python-swigsafecast.swig b/lldb/bindings/python/python-swigsafecast.swig
index fdd3b4e62c10..7d639e664f53 100644
--- a/lldb/bindings/python/python-swigsafecast.swig
+++ b/lldb/bindings/python/python-swigsafecast.swig
@@ -5,55 +5,11 @@ PyObject *SBTypeToSWIGWrapper(lldb::SBEvent &event_sb) {
   return SWIG_NewPointerObj(&event_sb, SWIGTYPE_p_lldb__SBEvent, 0);
 }
 
-PyObject *SBTypeToSWIGWrapper(lldb::SBThread &thread_sb) {
-  return SWIG_NewPointerObj(&thread_sb, SWIGTYPE_p_lldb__SBThread, 0);
-}
-
-PyObject *SBTypeToSWIGWrapper(lldb::SBFrame &frame_sb) {
-  return SWIG_NewPointerObj(&frame_sb, SWIGTYPE_p_lldb__SBFrame, 0);
-}
-
-PyObject *SBTypeToSWIGWrapper(lldb::SBDebugger &debugger_sb) {
-  return SWIG_NewPointerObj(&debugger_sb, SWIGTYPE_p_lldb__SBDebugger, 0);
-}
-
-PyObject *SBTypeToSWIGWrapper(lldb::SBWatchpoint &watchpoint_sb) {
-  return SWIG_NewPointerObj(&watchpoint_sb, SWIGTYPE_p_lldb__SBWatchpoint, 0);
-}
-
-PyObject *
-SBTypeToSWIGWrapper(lldb::SBBreakpointLocation &breakpoint_location_sb) {
-  return SWIG_NewPointerObj(&breakpoint_location_sb,
-                            SWIGTYPE_p_lldb__SBBreakpointLocation, 0);
-}
-
 PyObject *SBTypeToSWIGWrapper(lldb::SBCommandReturnObject &cmd_ret_obj_sb) {
   return SWIG_NewPointerObj(&cmd_ret_obj_sb,
                             SWIGTYPE_p_lldb__SBCommandReturnObject, 0);
 }
 
-PyObject *SBTypeToSWIGWrapper(lldb::SBExecutionContext &ctx_sb) {
-  return SWIG_NewPointerObj(&ctx_sb, SWIGTYPE_p_lldb__SBExecutionContext, 0);
-}
-
-PyObject *SBTypeToSWIGWrapper(lldb::SBTypeSummaryOptions &summary_options_sb) {
-  return SWIG_NewPointerObj(&summary_options_sb,
-                            SWIGTYPE_p_lldb__SBTypeSummaryOptions, 0);
-}
-
-PyObject *SBTypeToSWIGWrapper(lldb::SBStructuredData &structured_data_sb) {
-  return SWIG_NewPointerObj(&structured_data_sb,
-                            SWIGTYPE_p_lldb__SBStructuredData, 0);
-}
-
-PyObject *SBTypeToSWIGWrapper(lldb::SBSymbolContext &sym_ctx_sb) {
-  return SWIG_NewPointerObj(&sym_ctx_sb, SWIGTYPE_p_lldb__SBSymbolContext, 0);
-}
-
-PyObject *SBTypeToSWIGWrapper(lldb::SBStream &stream_sb) {
-  return SWIG_NewPointerObj(&stream_sb, SWIGTYPE_p_lldb__SBStream, 0);
-}
-
 PythonObject ToSWIGHelper(void *obj, swig_type_info *info) {
   return {PyRefType::Owned, SWIG_NewPointerObj(obj, info, SWIG_POINTER_OWN)};
 }
@@ -86,5 +42,57 @@ PythonObject ToSWIGWrapper(lldb::BreakpointSP breakpoint_sp) {
                       SWIGTYPE_p_lldb__SBBreakpoint);
 }
 
+PythonObject ToSWIGWrapper(std::unique_ptr<lldb::SBStream> stream_sb) {
+  return ToSWIGHelper(stream_sb.release(), SWIGTYPE_p_lldb__SBStream);
+}
+
+PythonObject ToSWIGWrapper(std::unique_ptr<lldb::SBStructuredData> data_sb) {
+  return ToSWIGHelper(data_sb.release(), SWIGTYPE_p_lldb__SBStructuredData);
+}
+
+PythonObject ToSWIGWrapper(const StructuredDataImpl &data_impl) {
+  return ToSWIGWrapper(std::make_unique<lldb::SBStructuredData>(data_impl));
+}
+
+PythonObject ToSWIGWrapper(lldb::ThreadSP thread_sp) {
+  return ToSWIGHelper(new lldb::SBThread(std::move(thread_sp)),
+                      SWIGTYPE_p_lldb__SBThread);
+}
+
+PythonObject ToSWIGWrapper(lldb::StackFrameSP frame_sp) {
+  return ToSWIGHelper(new lldb::SBFrame(std::move(frame_sp)),
+                      SWIGTYPE_p_lldb__SBFrame);
+}
+
+PythonObject ToSWIGWrapper(lldb::DebuggerSP debugger_sp) {
+  return ToSWIGHelper(new lldb::SBDebugger(std::move(debugger_sp)),
+                      SWIGTYPE_p_lldb__SBDebugger);
+}
+
+PythonObject ToSWIGWrapper(lldb::WatchpointSP watchpoint_sp) {
+  return ToSWIGHelper(new lldb::SBWatchpoint(std::move(watchpoint_sp)),
+                      SWIGTYPE_p_lldb__SBWatchpoint);
+}
+
+PythonObject ToSWIGWrapper(lldb::BreakpointLocationSP bp_loc_sp) {
+  return ToSWIGHelper(new lldb::SBBreakpointLocation(std::move(bp_loc_sp)),
+                      SWIGTYPE_p_lldb__SBBreakpointLocation);
+}
+
+PythonObject ToSWIGWrapper(lldb::ExecutionContextRefSP ctx_sp) {
+  return ToSWIGHelper(new lldb::SBExecutionContext(std::move(ctx_sp)),
+                      SWIGTYPE_p_lldb__SBExecutionContext);
+}
+
+PythonObject ToSWIGWrapper(const TypeSummaryOptions &summary_options) {
+  return ToSWIGHelper(new lldb::SBTypeSummaryOptions(summary_options),
+                      SWIGTYPE_p_lldb__SBTypeSummaryOptions);
+}
+
+PythonObject ToSWIGWrapper(const SymbolContext &sym_ctx) {
+  return ToSWIGHelper(new lldb::SBSymbolContext(sym_ctx),
+                      SWIGTYPE_p_lldb__SBSymbolContext);
+}
+
 } // namespace python
 } // namespace lldb_private
diff --git a/lldb/bindings/python/python-typemaps.swig b/lldb/bindings/python/python-typemaps.swig
index b1ace4ff3b1e..bf3de66b91bf 100644
--- a/lldb/bindings/python/python-typemaps.swig
+++ b/lldb/bindings/python/python-typemaps.swig
@@ -12,22 +12,22 @@
     PythonList list(PyRefType::Borrowed, $input);
     int size = list.GetSize();
     int i = 0;
-    $1 = (char**)malloc((size+1)*sizeof(char*));
+    $1 = (char **)malloc((size + 1) * sizeof(char *));
     for (i = 0; i < size; i++) {
       PythonString py_str = list.GetItemAtIndex(i).AsType<PythonString>();
       if (!py_str.IsAllocated()) {
-        PyErr_SetString(PyExc_TypeError,"list must contain strings");
+        PyErr_SetString(PyExc_TypeError, "list must contain strings");
         free($1);
         return nullptr;
       }
 
-      $1[i] = const_cast<char*>(py_str.GetString().data());
+      $1[i] = const_cast<char *>(py_str.GetString().data());
     }
     $1[i] = 0;
   } else if ($input == Py_None) {
-    $1 =  NULL;
+    $1 = NULL;
   } else {
-    PyErr_SetString(PyExc_TypeError,"not a list");
+    PyErr_SetString(PyExc_TypeError, "not a list");
     return NULL;
   }
 }
@@ -41,12 +41,12 @@
     int i = 0;
     for (i = 0; i < size; i++) {
       PythonString s = list.GetItemAtIndex(i).AsType<PythonString>();
-      if (!s.IsAllocated()) { $1 = 0; }
+      if (!s.IsAllocated()) {
+        $1 = 0;
+      }
     }
-  }
-  else
-  {
-    $1 = ( ($input == Py_None) ? 1 : 0);
+  } else {
+    $1 = (($input == Py_None) ? 1 : 0);
   }
 }
 
@@ -58,7 +58,8 @@
   int len;
   int i;
   len = 0;
-  while ($1[len]) len++;
+  while ($1[len])
+    len++;
   PythonList list(len);
   for (i = 0; i < len; i++)
     list.SetItemAtIndex(i, PythonString($1[i]));
@@ -76,7 +77,7 @@
 %typemap(in) lldb::StateType {
   PythonObject obj = Retain<PythonObject>($input);
   unsigned long long state_type_value =
-    unwrapOrSetPythonException(As<unsigned long long>(obj));
+      unwrapOrSetPythonException(As<unsigned long long>(obj));
   if (PyErr_Occurred())
     return nullptr;
   if (state_type_value > lldb::StateType::kLastStateType) {
@@ -90,16 +91,16 @@
 
 // typemap for a char buffer
 %typemap(in) (char *dst, size_t dst_len) {
-   if (!PyInt_Check($input)) {
-       PyErr_SetString(PyExc_ValueError, "Expecting an integer");
-       return NULL;
-   }
-   $2 = PyInt_AsLong($input);
-   if ($2 <= 0) {
-       PyErr_SetString(PyExc_ValueError, "Positive integer expected");
-       return NULL;
-   }
-   $1 = (char *) malloc($2);
+  if (!PyInt_Check($input)) {
+    PyErr_SetString(PyExc_ValueError, "Expecting an integer");
+    return NULL;
+  }
+  $2 = PyInt_AsLong($input);
+  if ($2 <= 0) {
+    PyErr_SetString(PyExc_ValueError, "Positive integer expected");
+    return NULL;
+  }
+  $1 = (char *)malloc($2);
 }
 // SBProcess::ReadCStringFromMemory() uses a void*, but needs to be treated
 // as char data instead of byte data.
@@ -107,17 +108,17 @@
 
 // Return the char buffer.  Discarding any previous return result
 %typemap(argout) (char *dst, size_t dst_len) {
-   Py_XDECREF($result);   /* Blow away any previous result */
-   if (result == 0) {
-      PythonString string("");
-      $result = string.release();
-      Py_INCREF($result);
-   } else {
-      llvm::StringRef ref(static_cast<const char*>($1), result);
-      PythonString string(ref);
-      $result = string.release();
-   }
-   free($1);
+  Py_XDECREF($result); /* Blow away any previous result */
+  if (result == 0) {
+    PythonString string("");
+    $result = string.release();
+    Py_INCREF($result);
+  } else {
+    llvm::StringRef ref(static_cast<const char *>($1), result);
+    PythonString string(ref);
+    $result = string.release();
+  }
+  free($1);
 }
 // SBProcess::ReadCStringFromMemory() uses a void*, but needs to be treated
 // as char data instead of byte data.
@@ -126,23 +127,23 @@
 
 // typemap for handling an snprintf-like API like SBThread::GetStopDescription.
 %typemap(in) (char *dst_or_null, size_t dst_len) {
-   if (!PyInt_Check($input)) {
-       PyErr_SetString(PyExc_ValueError, "Expecting an integer");
-       return NULL;
-   }
-   $2 = PyInt_AsLong($input);
-   if ($2 <= 0) {
-       PyErr_SetString(PyExc_ValueError, "Positive integer expected");
-       return NULL;
-   }
-   $1 = (char *) malloc($2);
+  if (!PyInt_Check($input)) {
+    PyErr_SetString(PyExc_ValueError, "Expecting an integer");
+    return NULL;
+  }
+  $2 = PyInt_AsLong($input);
+  if ($2 <= 0) {
+    PyErr_SetString(PyExc_ValueError, "Positive integer expected");
+    return NULL;
+  }
+  $1 = (char *)malloc($2);
 }
 %typemap(argout) (char *dst_or_null, size_t dst_len) {
-   Py_XDECREF($result);   /* Blow away any previous result */
-   llvm::StringRef ref($1);
-   PythonString string(ref);
-   $result = string.release();
-   free($1);
+  Py_XDECREF($result); /* Blow away any previous result */
+  llvm::StringRef ref($1);
+  PythonString string(ref);
+  $result = string.release();
+  free($1);
 }
 
 
@@ -151,80 +152,74 @@
 // Ditto for SBProcess::PutSTDIN(const char *src, size_t src_len).
 %typemap(in) (const char *cstr, uint32_t cstr_len),
              (const char *src, size_t src_len) {
-   if (PythonString::Check($input)) {
-      PythonString str(PyRefType::Borrowed, $input);
-      $1 = (char*)str.GetString().data();
-      $2 = str.GetSize();
-   }
-   else if(PythonByteArray::Check($input)) {
-      PythonByteArray bytearray(PyRefType::Borrowed, $input);
-      $1 = (char*)bytearray.GetBytes().data();
-      $2 = bytearray.GetSize();
-   }
-   else if (PythonBytes::Check($input)) {
-      PythonBytes bytes(PyRefType::Borrowed, $input);
-      $1 = (char*)bytes.GetBytes().data();
-      $2 = bytes.GetSize();
-   }
-   else {
-      PyErr_SetString(PyExc_ValueError, "Expecting a string");
-      return NULL;
-   }
+  if (PythonString::Check($input)) {
+    PythonString str(PyRefType::Borrowed, $input);
+    $1 = (char *)str.GetString().data();
+    $2 = str.GetSize();
+  } else if (PythonByteArray::Check($input)) {
+    PythonByteArray bytearray(PyRefType::Borrowed, $input);
+    $1 = (char *)bytearray.GetBytes().data();
+    $2 = bytearray.GetSize();
+  } else if (PythonBytes::Check($input)) {
+    PythonBytes bytes(PyRefType::Borrowed, $input);
+    $1 = (char *)bytes.GetBytes().data();
+    $2 = bytes.GetSize();
+  } else {
+    PyErr_SetString(PyExc_ValueError, "Expecting a string");
+    return NULL;
+  }
 }
 // For SBProcess::WriteMemory, SBTarget::GetInstructions and SBDebugger::DispatchInput.
 %typemap(in) (const void *buf, size_t size),
              (const void *data, size_t data_len) {
-   if (PythonString::Check($input)) {
-      PythonString str(PyRefType::Borrowed, $input);
-      $1 = (void*)str.GetString().data();
-      $2 = str.GetSize();
-   }
-   else if(PythonByteArray::Check($input)) {
-      PythonByteArray bytearray(PyRefType::Borrowed, $input);
-      $1 = (void*)bytearray.GetBytes().data();
-      $2 = bytearray.GetSize();
-   }
-   else if (PythonBytes::Check($input)) {
-      PythonBytes bytes(PyRefType::Borrowed, $input);
-      $1 = (void*)bytes.GetBytes().data();
-      $2 = bytes.GetSize();
-   }
-   else {
-      PyErr_SetString(PyExc_ValueError, "Expecting a buffer");
-      return NULL;
-   }
+  if (PythonString::Check($input)) {
+    PythonString str(PyRefType::Borrowed, $input);
+    $1 = (void *)str.GetString().data();
+    $2 = str.GetSize();
+  } else if (PythonByteArray::Check($input)) {
+    PythonByteArray bytearray(PyRefType::Borrowed, $input);
+    $1 = (void *)bytearray.GetBytes().data();
+    $2 = bytearray.GetSize();
+  } else if (PythonBytes::Check($input)) {
+    PythonBytes bytes(PyRefType::Borrowed, $input);
+    $1 = (void *)bytes.GetBytes().data();
+    $2 = bytes.GetSize();
+  } else {
+    PyErr_SetString(PyExc_ValueError, "Expecting a buffer");
+    return NULL;
+  }
 }
 
 // typemap for an incoming buffer
 // See also SBProcess::ReadMemory.
 %typemap(in) (void *buf, size_t size) {
-   if (PyInt_Check($input)) {
-      $2 = PyInt_AsLong($input);
-   } else if (PyLong_Check($input)) {
-      $2 = PyLong_AsLong($input);
-   } else {
-      PyErr_SetString(PyExc_ValueError, "Expecting an integer or long object");
-      return NULL;
-   }
-   if ($2 <= 0) {
-       PyErr_SetString(PyExc_ValueError, "Positive integer expected");
-       return NULL;
-   }
-   $1 = (void *) malloc($2);
+  if (PyInt_Check($input)) {
+    $2 = PyInt_AsLong($input);
+  } else if (PyLong_Check($input)) {
+    $2 = PyLong_AsLong($input);
+  } else {
+    PyErr_SetString(PyExc_ValueError, "Expecting an integer or long object");
+    return NULL;
+  }
+  if ($2 <= 0) {
+    PyErr_SetString(PyExc_ValueError, "Positive integer expected");
+    return NULL;
+  }
+  $1 = (void *)malloc($2);
 }
 
 // Return the buffer.  Discarding any previous return result
 // See also SBProcess::ReadMemory.
 %typemap(argout) (void *buf, size_t size) {
-   Py_XDECREF($result);   /* Blow away any previous result */
-   if (result == 0) {
-      $result = Py_None;
-      Py_INCREF($result);
-   } else {
-      PythonBytes bytes(static_cast<const uint8_t*>($1), result);
-      $result = bytes.release();
-   }
-   free($1);
+  Py_XDECREF($result); /* Blow away any previous result */
+  if (result == 0) {
+    $result = Py_None;
+    Py_INCREF($result);
+  } else {
+    PythonBytes bytes(static_cast<const uint8_t *>($1), result);
+    $result = bytes.release();
+  }
+  free($1);
 }
 
 %{
@@ -250,19 +245,18 @@ template <> int32_t PyLongAsT<int32_t>(PyObject *obj) {
   return static_cast<int32_t>(PyLong_AsLong(obj));
 }
 
-template <class T>
-bool SetNumberFromPyObject(T &number, PyObject *obj) {
+template <class T> bool SetNumberFromPyObject(T &number, PyObject *obj) {
   if (PyInt_Check(obj))
     number = static_cast<T>(PyInt_AsLong(obj));
   else if (PyLong_Check(obj))
     number = PyLongAsT<T>(obj);
-  else return false;
+  else
+    return false;
 
   return true;
 }
 
-template <>
-bool SetNumberFromPyObject<double>(double &number, PyObject *obj) {
+template <> bool SetNumberFromPyObject<double>(double &number, PyObject *obj) {
   if (PyFloat_Check(obj)) {
     number = PyFloat_AsDouble(obj);
     return true;
@@ -287,11 +281,11 @@ bool SetNumberFromPyObject<double>(double &number, PyObject *obj) {
     int size = PyList_Size($input);
     int i = 0;
     $2 = size;
-    $1 = ($1_type) malloc(size * sizeof($*1_type));
+    $1 = ($1_type)malloc(size * sizeof($*1_type));
     for (i = 0; i < size; i++) {
-      PyObject *o = PyList_GetItem($input,i);
+      PyObject *o = PyList_GetItem($input, i);
       if (!SetNumberFromPyObject($1[i], o)) {
-        PyErr_SetString(PyExc_TypeError,"list must contain numbers");
+        PyErr_SetString(PyExc_TypeError, "list must contain numbers");
         free($1);
         return NULL;
       }
@@ -302,10 +296,10 @@ bool SetNumberFromPyObject<double>(double &number, PyObject *obj) {
       }
     }
   } else if ($input == Py_None) {
-    $1 =  NULL;
+    $1 = NULL;
     $2 = 0;
   } else {
-    PyErr_SetString(PyExc_TypeError,"not a list");
+    PyErr_SetString(PyExc_TypeError, "not a list");
     return NULL;
   }
 }
@@ -322,43 +316,42 @@ bool SetNumberFromPyObject<double>(double &number, PyObject *obj) {
 // to the more Pythonic style where a list is returned and no previous allocation
 // is necessary - this will break if more than 50 versions are ever returned
 %typemap(typecheck) (uint32_t *versions, uint32_t num_versions) {
-    $1 = ($input == Py_None ? 1 : 0);
+  $1 = ($input == Py_None ? 1 : 0);
 }
 
 %typemap(in, numinputs=0) (uint32_t *versions) {
-    $1 = (uint32_t*)malloc(sizeof(uint32_t) * 50);
+  $1 = (uint32_t *)malloc(sizeof(uint32_t) * 50);
 }
 
 %typemap(in, numinputs=0) (uint32_t num_versions) {
-    $1 = 50;
+  $1 = 50;
 }
 
 %typemap(argout) (uint32_t *versions, uint32_t num_versions) {
-    uint32_t count = result;
-    if (count >= $2)
-        count = $2;
-    PyObject* list = PyList_New(count);
-    for (uint32_t j = 0; j < count; j++)
-    {
-        PyObject* item = PyInt_FromLong($1[j]);
-        int ok = PyList_SetItem(list,j,item);
-        if (ok != 0)
-        {
-            $result = Py_None;
-            break;
-        }
+  uint32_t count = result;
+  if (count >= $2)
+    count = $2;
+  PyObject *list = PyList_New(count);
+  for (uint32_t j = 0; j < count; j++) {
+    PyObject *item = PyInt_FromLong($1[j]);
+    int ok = PyList_SetItem(list, j, item);
+    if (ok != 0) {
+      $result = Py_None;
+      break;
     }
-    $result = list;
+  }
+  $result = list;
 }
 
 %typemap(freearg) (uint32_t *versions) {
-    free($1);
+  free($1);
 }
 
 
 // For Log::LogOutputCallback
 %typemap(in) (lldb::LogOutputCallback log_callback, void *baton) {
-  if (!($input == Py_None || PyCallable_Check(reinterpret_cast<PyObject*>($input)))) {
+  if (!($input == Py_None ||
+        PyCallable_Check(reinterpret_cast<PyObject *>($input)))) {
     PyErr_SetString(PyExc_TypeError, "Need a callable object or None!");
     return NULL;
   }
@@ -376,7 +369,7 @@ bool SetNumberFromPyObject<double>(double &number, PyObject *obj) {
 
 %typemap(typecheck) (lldb::LogOutputCallback log_callback, void *baton) {
   $1 = $input == Py_None;
-  $1 = $1 || PyCallable_Check(reinterpret_cast<PyObject*>($input));
+  $1 = $1 || PyCallable_Check(reinterpret_cast<PyObject *>($input));
 }
 
 
@@ -398,7 +391,8 @@ bool SetNumberFromPyObject<double>(double &number, PyObject *obj) {
     PyErr_SetString(PyExc_TypeError, "not a file");
     return nullptr;
   }
-  auto sp = unwrapOrSetPythonException(py_file.ConvertToFileForcingUseOfScriptingIOMethods());
+  auto sp = unwrapOrSetPythonException(
+      py_file.ConvertToFileForcingUseOfScriptingIOMethods());
   if (!sp)
     return nullptr;
   $1 = sp;
@@ -410,7 +404,8 @@ bool SetNumberFromPyObject<double>(double &number, PyObject *obj) {
     PyErr_SetString(PyExc_TypeError, "not a file");
     return nullptr;
   }
-  auto sp = unwrapOrSetPythonException(py_file.ConvertToFile(/*borrowed=*/true));
+  auto sp =
+      unwrapOrSetPythonException(py_file.ConvertToFile(/*borrowed=*/true));
   if (!sp)
     return nullptr;
   $1 = sp;
@@ -422,7 +417,8 @@ bool SetNumberFromPyObject<double>(double &number, PyObject *obj) {
     PyErr_SetString(PyExc_TypeError, "not a file");
     return nullptr;
   }
-  auto sp = unwrapOrSetPythonException(py_file.ConvertToFileForcingUseOfScriptingIOMethods(/*borrowed=*/true));
+  auto sp = unwrapOrSetPythonException(
+      py_file.ConvertToFileForcingUseOfScriptingIOMethods(/*borrowed=*/true));
   if (!sp)
     return nullptr;
   $1 = sp;
@@ -446,40 +442,34 @@ bool SetNumberFromPyObject<double>(double &number, PyObject *obj) {
       return nullptr;
     $result = pyfile.release();
   }
-  if (!$result)
-  {
-      $result = Py_None;
-      Py_INCREF(Py_None);
+  if (!$result) {
+    $result = Py_None;
+    Py_INCREF(Py_None);
   }
 }
 
 %typemap(in) (const char* string, int len) {
-    if ($input == Py_None)
-    {
-        $1 = NULL;
-        $2 = 0;
-    }
-    else if (PythonString::Check($input))
-    {
-        PythonString py_str(PyRefType::Borrowed, $input);
-        llvm::StringRef str = py_str.GetString();
-        $1 = const_cast<char*>(str.data());
-        $2 = str.size();
-        // In Python 2, if $input is a PyUnicode object then this
-        // will trigger a Unicode -> String conversion, in which
-        // case the `PythonString` will now own the PyString.  Thus
-        // if it goes out of scope, the data will be deleted.  The
-        // only way to avoid this is to leak the Python object in
-        // that case.  Note that if there was no conversion, then
-        // releasing the string will not leak anything, since we
-        // created this as a borrowed reference.
-        py_str.release();
-    }
-    else
-    {
-        PyErr_SetString(PyExc_TypeError,"not a string-like object");
-        return NULL;
-    }
+  if ($input == Py_None) {
+    $1 = NULL;
+    $2 = 0;
+  } else if (PythonString::Check($input)) {
+    PythonString py_str(PyRefType::Borrowed, $input);
+    llvm::StringRef str = py_str.GetString();
+    $1 = const_cast<char *>(str.data());
+    $2 = str.size();
+    // In Python 2, if $input is a PyUnicode object then this
+    // will trigger a Unicode -> String conversion, in which
+    // case the `PythonString` will now own the PyString.  Thus
+    // if it goes out of scope, the data will be deleted.  The
+    // only way to avoid this is to leak the Python object in
+    // that case.  Note that if there was no conversion, then
+    // releasing the string will not leak anything, since we
+    // created this as a borrowed reference.
+    py_str.release();
+  } else {
+    PyErr_SetString(PyExc_TypeError, "not a string-like object");
+    return NULL;
+  }
 }
 
 // These two pybuffer macros are copied out of swig/Lib/python/pybuffer.i,
@@ -491,7 +481,9 @@ bool SetNumberFromPyObject<double>(double &number, PyObject *obj) {
 
 %define %pybuffer_mutable_binary(TYPEMAP, SIZE)
 %typemap(in) (TYPEMAP, SIZE) (Py_buffer_RAII view) {
-  int res; Py_ssize_t size = 0; void *buf = 0;
+  int res;
+  Py_ssize_t size = 0;
+  void *buf = 0;
   res = PyObject_GetBuffer($input, &view.buffer, PyBUF_WRITABLE);
   if (res < 0) {
     PyErr_Clear();
@@ -499,14 +491,16 @@ bool SetNumberFromPyObject<double>(double &number, PyObject *obj) {
   }
   size = view.buffer.len;
   buf = view.buffer.buf;
-  $1 = ($1_ltype) buf;
-  $2 = ($2_ltype) (size/sizeof($*1_type));
+  $1 = ($1_ltype)buf;
+  $2 = ($2_ltype)(size / sizeof($*1_type));
 }
 %enddef
 
 %define %pybuffer_binary(TYPEMAP, SIZE)
 %typemap(in) (TYPEMAP, SIZE) (Py_buffer_RAII view) {
-  int res; Py_ssize_t size = 0; const void *buf = 0;
+  int res;
+  Py_ssize_t size = 0;
+  const void *buf = 0;
   res = PyObject_GetBuffer($input, &view.buffer, PyBUF_CONTIG_RO);
   if (res < 0) {
     PyErr_Clear();
@@ -514,8 +508,8 @@ bool SetNumberFromPyObject<double>(double &number, PyObject *obj) {
   }
   size = view.buffer.len;
   buf = view.buffer.buf;
-  $1 = ($1_ltype) buf;
-  $2 = ($2_ltype) (size / sizeof($*1_type));
+  $1 = ($1_ltype)buf;
+  $2 = ($2_ltype)(size / sizeof($*1_type));
 }
 %enddef
 
diff --git a/lldb/bindings/python/python-wrapper.swig b/lldb/bindings/python/python-wrapper.swig
index 079f8d12dafa..a2c1f756a0a2 100644
--- a/lldb/bindings/python/python-wrapper.swig
+++ b/lldb/bindings/python/python-wrapper.swig
@@ -1,72 +1,55 @@
 %header %{
 
-class PyErr_Cleaner
-{
+class PyErr_Cleaner {
 public:
-    PyErr_Cleaner(bool print=false) :
-    m_print(print)
-    {
-    }
+  PyErr_Cleaner(bool print = false) : m_print(print) {}
 
-    ~PyErr_Cleaner()
-    {
-        if (PyErr_Occurred())
-        {
-            if(m_print && !PyErr_ExceptionMatches(PyExc_SystemExit))
-                PyErr_Print();
-            PyErr_Clear();
-        }
+  ~PyErr_Cleaner() {
+    if (PyErr_Occurred()) {
+      if (m_print && !PyErr_ExceptionMatches(PyExc_SystemExit))
+        PyErr_Print();
+      PyErr_Clear();
     }
+  }
 
 private:
-    bool m_print;
+  bool m_print;
 };
 
-llvm::Expected<bool>
-lldb_private::LLDBSwigPythonBreakpointCallbackFunction
-(
-    const char *python_function_name,
-    const char *session_dictionary_name,
-    const lldb::StackFrameSP& frame_sp,
-    const lldb::BreakpointLocationSP& bp_loc_sp,
-    lldb_private::StructuredDataImpl *args_impl
-)
-{
-    using namespace llvm;
-
-    lldb::SBFrame sb_frame (frame_sp);
-    lldb::SBBreakpointLocation sb_bp_loc(bp_loc_sp);
-
-    PyErr_Cleaner py_err_cleaner(true);
-    auto dict = PythonModule::MainModule().ResolveName<PythonDictionary>(session_dictionary_name);
-    auto pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(python_function_name, dict);
-
-    unsigned max_positional_args;
-    if (auto arg_info = pfunc.GetArgInfo())
-        max_positional_args = arg_info.get().max_positional_args;
-    else
-        return arg_info.takeError();
-
-    PythonObject frame_arg(PyRefType::Owned, SBTypeToSWIGWrapper(sb_frame));
-    PythonObject bp_loc_arg(PyRefType::Owned, SBTypeToSWIGWrapper(sb_bp_loc));
-
-    auto result = [&] () -> Expected<PythonObject> {
-        // If the called function doesn't take extra_args, drop them here:
-        if (max_positional_args < 4) {
-            return pfunc.Call(frame_arg, bp_loc_arg, dict);
-        } else {
-            // FIXME: SBStructuredData leaked here
-            lldb::SBStructuredData *args_value = new lldb::SBStructuredData(args_impl);
-            PythonObject args_arg(PyRefType::Owned, SBTypeToSWIGWrapper(*args_value));
-            return pfunc.Call(frame_arg, bp_loc_arg, args_arg, dict);
-        }
-    } ();
-
-    if (!result)
-        return result.takeError();
-
-    // Only False counts as false!
-    return result.get().get() != Py_False;
+llvm::Expected<bool> lldb_private::LLDBSwigPythonBreakpointCallbackFunction(
+    const char *python_function_name, const char *session_dictionary_name,
+    const lldb::StackFrameSP &frame_sp,
+    const lldb::BreakpointLocationSP &bp_loc_sp,
+    const lldb_private::StructuredDataImpl &args_impl) {
+  using namespace llvm;
+
+  lldb::SBBreakpointLocation sb_bp_loc(bp_loc_sp);
+
+  PyErr_Cleaner py_err_cleaner(true);
+  auto dict = PythonModule::MainModule().ResolveName<PythonDictionary>(
+      session_dictionary_name);
+  auto pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(
+      python_function_name, dict);
+
+  unsigned max_positional_args;
+  if (auto arg_info = pfunc.GetArgInfo())
+    max_positional_args = arg_info.get().max_positional_args;
+  else
+    return arg_info.takeError();
+
+  PythonObject frame_arg = ToSWIGWrapper(frame_sp);
+  PythonObject bp_loc_arg = ToSWIGWrapper(bp_loc_sp);
+
+  auto result =
+      max_positional_args < 4
+          ? pfunc.Call(frame_arg, bp_loc_arg, dict)
+          : pfunc.Call(frame_arg, bp_loc_arg, ToSWIGWrapper(args_impl), dict);
+
+  if (!result)
+    return result.takeError();
+
+  // Only False counts as false!
+  return result.get().get() != Py_False;
 }
 
 // resolve a dotted Python name in the form
@@ -74,1192 +57,1036 @@ lldb_private::LLDBSwigPythonBreakpointCallbackFunction
 // if pmodule is NULL, the __main__ module will be used
 // as the starting point for the search
 
+// This function is called by
+// lldb_private::ScriptInterpreterPython::BreakpointCallbackFunction(...) and is
+// used when a script command is attached to a breakpoint for execution.
 
-// This function is called by lldb_private::ScriptInterpreterPython::BreakpointCallbackFunction(...)
-// and is used when a script command is attached to a breakpoint for execution.
-
-// This function is called by lldb_private::ScriptInterpreterPython::WatchpointCallbackFunction(...)
-// and is used when a script command is attached to a watchpoint for execution.
+// This function is called by
+// lldb_private::ScriptInterpreterPython::WatchpointCallbackFunction(...) and is
+// used when a script command is attached to a watchpoint for execution.
 
-bool
-lldb_private::LLDBSwigPythonWatchpointCallbackFunction
-(
-    const char *python_function_name,
-    const char *session_dictionary_name,
-    const lldb::StackFrameSP& frame_sp,
-    const lldb::WatchpointSP& wp_sp
-)
-{
-    lldb::SBFrame sb_frame (frame_sp);
-    lldb::SBWatchpoint sb_wp(wp_sp);
+bool lldb_private::LLDBSwigPythonWatchpointCallbackFunction(
+    const char *python_function_name, const char *session_dictionary_name,
+    const lldb::StackFrameSP &frame_sp, const lldb::WatchpointSP &wp_sp) {
 
-    bool stop_at_watchpoint = true;
+  bool stop_at_watchpoint = true;
 
-    PyErr_Cleaner py_err_cleaner(true);
+  PyErr_Cleaner py_err_cleaner(true);
 
-    auto dict = PythonModule::MainModule().ResolveName<PythonDictionary>(session_dictionary_name);
-    auto pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(python_function_name, dict);
+  auto dict = PythonModule::MainModule().ResolveName<PythonDictionary>(
+      session_dictionary_name);
+  auto pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(
+      python_function_name, dict);
 
-    if (!pfunc.IsAllocated())
-        return stop_at_watchpoint;
+  if (!pfunc.IsAllocated())
+    return stop_at_watchpoint;
 
-    PythonObject frame_arg(PyRefType::Owned, SBTypeToSWIGWrapper(sb_frame));
-    PythonObject wp_arg(PyRefType::Owned, SBTypeToSWIGWrapper(sb_wp));
-    PythonObject result = pfunc(frame_arg, wp_arg, dict);
+  PythonObject result =
+      pfunc(ToSWIGWrapper(frame_sp), ToSWIGWrapper(wp_sp), dict);
 
-    if (result.get() == Py_False)
-        stop_at_watchpoint = false;
+  if (result.get() == Py_False)
+    stop_at_watchpoint = false;
 
-    return stop_at_watchpoint;
+  return stop_at_watchpoint;
 }
 
-bool
-lldb_private::LLDBSwigPythonCallTypeScript
-(
-    const char *python_function_name,
-    const void *session_dictionary,
-    const lldb::ValueObjectSP& valobj_sp,
-    void** pyfunct_wrapper,
-    const lldb::TypeSummaryOptionsSP& options_sp,
-    std::string& retval
-)
-{
-    lldb::SBTypeSummaryOptions sb_options(options_sp.get());
-
-    retval.clear();
-
-    if (!python_function_name || !session_dictionary)
-        return false;
-
-    PyObject *pfunc_impl = nullptr;
-
-    if (pyfunct_wrapper && *pyfunct_wrapper && PyFunction_Check (*pyfunct_wrapper))
-    {
-        pfunc_impl = (PyObject*)(*pyfunct_wrapper);
-        if (pfunc_impl->ob_refcnt == 1)
-        {
-            Py_XDECREF(pfunc_impl);
-            pfunc_impl = NULL;
-        }
-    }
-
-    PyObject *py_dict = (PyObject*)session_dictionary;
-    if (!PythonDictionary::Check(py_dict))
-        return true;
-
-    PythonDictionary dict(PyRefType::Borrowed, py_dict);
+bool lldb_private::LLDBSwigPythonCallTypeScript(
+    const char *python_function_name, const void *session_dictionary,
+    const lldb::ValueObjectSP &valobj_sp, void **pyfunct_wrapper,
+    const lldb::TypeSummaryOptionsSP &options_sp, std::string &retval) {
 
-    PyErr_Cleaner pyerr_cleanup(true);  // show Python errors
+  retval.clear();
 
-    PythonCallable pfunc(PyRefType::Borrowed, pfunc_impl);
+  if (!python_function_name || !session_dictionary)
+    return false;
 
-    if (!pfunc.IsAllocated())
-    {
-        pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(python_function_name, dict);
-        if (!pfunc.IsAllocated())
-            return false;
-
-        if (pyfunct_wrapper)
-        {
-            *pyfunct_wrapper = pfunc.get();
-            Py_XINCREF(pfunc.get());
-        }
-    }
+  PyObject *pfunc_impl = nullptr;
 
-    PythonObject result;
-    auto argc = pfunc.GetArgInfo();
-    if (!argc) {
-        llvm::consumeError(argc.takeError());
-        return false;
+  if (pyfunct_wrapper && *pyfunct_wrapper &&
+      PyFunction_Check(*pyfunct_wrapper)) {
+    pfunc_impl = (PyObject *)(*pyfunct_wrapper);
+    if (pfunc_impl->ob_refcnt == 1) {
+      Py_XDECREF(pfunc_impl);
+      pfunc_impl = NULL;
     }
+  }
 
-    PythonObject value_arg = ToSWIGWrapper(valobj_sp);
-    PythonObject options_arg(PyRefType::Owned, SBTypeToSWIGWrapper(sb_options));
-
-    if (argc.get().max_positional_args < 3)
-        result = pfunc(value_arg,dict);
-    else
-        result = pfunc(value_arg,dict,options_arg);
-
-    retval = result.Str().GetString().str();
-
+  PyObject *py_dict = (PyObject *)session_dictionary;
+  if (!PythonDictionary::Check(py_dict))
     return true;
-}
 
-void*
-lldb_private::LLDBSwigPythonCreateSyntheticProvider
-(
-    const char *python_class_name,
-    const char *session_dictionary_name,
-    const lldb::ValueObjectSP& valobj_sp
-)
-{
-    if (python_class_name == NULL || python_class_name[0] == '\0' || !session_dictionary_name)
-        Py_RETURN_NONE;
+  PythonDictionary dict(PyRefType::Borrowed, py_dict);
 
-    PyErr_Cleaner py_err_cleaner(true);
+  PyErr_Cleaner pyerr_cleanup(true); // show Python errors
 
-    auto dict = PythonModule::MainModule().ResolveName<PythonDictionary>(session_dictionary_name);
-    auto pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(python_class_name,dict);
+  PythonCallable pfunc(PyRefType::Borrowed, pfunc_impl);
 
+  if (!pfunc.IsAllocated()) {
+    pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(
+        python_function_name, dict);
     if (!pfunc.IsAllocated())
-        Py_RETURN_NONE;
-
-    auto sb_value = std::make_unique<lldb::SBValue>(valobj_sp);
-    sb_value->SetPreferSyntheticValue(false);
+      return false;
 
-    PythonObject val_arg = ToSWIGWrapper(std::move(sb_value));
-    if (!val_arg.IsAllocated())
-        Py_RETURN_NONE;
+    if (pyfunct_wrapper) {
+      *pyfunct_wrapper = pfunc.get();
+      Py_XINCREF(pfunc.get());
+    }
+  }
 
-    PythonObject result = pfunc(val_arg, dict);
+  PythonObject result;
+  auto argc = pfunc.GetArgInfo();
+  if (!argc) {
+    llvm::consumeError(argc.takeError());
+    return false;
+  }
 
-    if (result.IsAllocated())
-        return result.release();
+  PythonObject value_arg = ToSWIGWrapper(valobj_sp);
 
-    Py_RETURN_NONE;
-}
+  if (argc.get().max_positional_args < 3)
+    result = pfunc(value_arg, dict);
+  else
+    result = pfunc(value_arg, dict, ToSWIGWrapper(*options_sp));
 
-void*
-lldb_private::LLDBSwigPythonCreateCommandObject
-(
-    const char *python_class_name,
-    const char *session_dictionary_name,
-    const lldb::DebuggerSP debugger_sp
-)
-{
-    if (python_class_name == NULL || python_class_name[0] == '\0' || !session_dictionary_name)
-        Py_RETURN_NONE;
+  retval = result.Str().GetString().str();
 
-    PyErr_Cleaner py_err_cleaner(true);
-    auto dict = PythonModule::MainModule().ResolveName<PythonDictionary>(session_dictionary_name);
-    auto pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(python_class_name, dict);
+  return true;
+}
 
-    if (!pfunc.IsAllocated())
-        return nullptr;
+void *lldb_private::LLDBSwigPythonCreateSyntheticProvider(
+    const char *python_class_name, const char *session_dictionary_name,
+    const lldb::ValueObjectSP &valobj_sp) {
+  if (python_class_name == NULL || python_class_name[0] == '\0' ||
+      !session_dictionary_name)
+    Py_RETURN_NONE;
 
-    lldb::SBDebugger debugger_sb(debugger_sp);
-    PythonObject debugger_arg(PyRefType::Owned, SBTypeToSWIGWrapper(debugger_sb));
-    PythonObject result = pfunc(debugger_arg, dict);
+  PyErr_Cleaner py_err_cleaner(true);
 
-    if (result.IsAllocated())
-        return result.release();
+  auto dict = PythonModule::MainModule().ResolveName<PythonDictionary>(
+      session_dictionary_name);
+  auto pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(
+      python_class_name, dict);
 
+  if (!pfunc.IsAllocated())
     Py_RETURN_NONE;
-}
-
-void*
-lldb_private::LLDBSwigPythonCreateScriptedProcess
-(
-    const char *python_class_name,
-    const char *session_dictionary_name,
-    const lldb::TargetSP& target_sp,
-    lldb_private::StructuredDataImpl *args_impl,
-    std::string &error_string
-)
-{
-    if (python_class_name == NULL || python_class_name[0] == '\0' || !session_dictionary_name)
-        Py_RETURN_NONE;
 
-    PyErr_Cleaner py_err_cleaner(true);
+  auto sb_value = std::make_unique<lldb::SBValue>(valobj_sp);
+  sb_value->SetPreferSyntheticValue(false);
 
-    auto dict = PythonModule::MainModule().ResolveName<PythonDictionary>(session_dictionary_name);
-    auto pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(python_class_name, dict);
-
-    if (!pfunc.IsAllocated()) {
-        error_string.append("could not find script class: ");
-        error_string.append(python_class_name);
-        return nullptr;
-    }
+  PythonObject val_arg = ToSWIGWrapper(std::move(sb_value));
+  if (!val_arg.IsAllocated())
+    Py_RETURN_NONE;
 
-    PythonObject target_arg = ToSWIGWrapper(target_sp);
-
-    llvm::Expected<PythonCallable::ArgInfo> arg_info = pfunc.GetArgInfo();
-    if (!arg_info) {
-        llvm::handleAllErrors(
-            arg_info.takeError(),
-            [&](PythonException &E) {
-                error_string.append(E.ReadBacktrace());
-            },
-            [&](const llvm::ErrorInfoBase &E) {
-                error_string.append(E.message());
-            });
-        Py_RETURN_NONE;
-    }
+  PythonObject result = pfunc(val_arg, dict);
 
-    PythonObject result = {};
-    if (arg_info.get().max_positional_args == 2) {
-        // FIXME: SBStructuredData leaked here
-        PythonObject args_arg(PyRefType::Owned, SBTypeToSWIGWrapper(*new lldb::SBStructuredData(args_impl)));
-        result = pfunc(target_arg, args_arg);
-    } else {
-        error_string.assign("wrong number of arguments in __init__, should be 2 (not including self)");
-        Py_RETURN_NONE;
-    }
+  if (result.IsAllocated())
+    return result.release();
 
-    if (result.IsAllocated())
-        return result.release();
-    Py_RETURN_NONE;
+  Py_RETURN_NONE;
 }
 
-void*
-lldb_private::LLDBSwigPythonCreateScriptedThread
-(
-    const char *python_class_name,
-    const char *session_dictionary_name,
-    const lldb::ProcessSP& process_sp,
-    lldb_private::StructuredDataImpl *args_impl,
-    std::string &error_string
-)
-{
-    if (python_class_name == NULL || python_class_name[0] == '\0' || !session_dictionary_name)
-        Py_RETURN_NONE;
+void *lldb_private::LLDBSwigPythonCreateCommandObject(
+    const char *python_class_name, const char *session_dictionary_name,
+    lldb::DebuggerSP debugger_sp) {
+  if (python_class_name == NULL || python_class_name[0] == '\0' ||
+      !session_dictionary_name)
+    Py_RETURN_NONE;
 
-    PyErr_Cleaner py_err_cleaner(true);
+  PyErr_Cleaner py_err_cleaner(true);
+  auto dict = PythonModule::MainModule().ResolveName<PythonDictionary>(
+      session_dictionary_name);
+  auto pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(
+      python_class_name, dict);
 
-    auto dict = PythonModule::MainModule().ResolveName<PythonDictionary>(session_dictionary_name);
-    auto pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(python_class_name, dict);
+  if (!pfunc.IsAllocated())
+    return nullptr;
 
-    if (!pfunc.IsAllocated()) {
-        error_string.append("could not find script class: ");
-        error_string.append(python_class_name);
-        return nullptr;
-    }
+  PythonObject result = pfunc(ToSWIGWrapper(std::move(debugger_sp)), dict);
 
-    llvm::Expected<PythonCallable::ArgInfo> arg_info = pfunc.GetArgInfo();
-    if (!arg_info) {
-        llvm::handleAllErrors(
-            arg_info.takeError(),
-            [&](PythonException &E) {
-                error_string.append(E.ReadBacktrace());
-            },
-            [&](const llvm::ErrorInfoBase &E) {
-                error_string.append(E.message());
-            });
-        Py_RETURN_NONE;
-    }
+  if (result.IsAllocated())
+    return result.release();
 
-    PythonObject result = {};
-    if (arg_info.get().max_positional_args == 2) {
-        // FIXME: SBStructuredData leaked here
-        PythonObject args_arg(PyRefType::Owned, SBTypeToSWIGWrapper(*new lldb::SBStructuredData(args_impl)));
-        result = pfunc(ToSWIGWrapper(process_sp), args_arg);
-    } else {
-        error_string.assign("wrong number of arguments in __init__, should be 2 (not including self)");
-        Py_RETURN_NONE;
-    }
+  Py_RETURN_NONE;
+}
 
-    if (result.IsAllocated())
-        return result.release();
+void *lldb_private::LLDBSwigPythonCreateScriptedProcess(
+    const char *python_class_name, const char *session_dictionary_name,
+    const lldb::TargetSP &target_sp,
+    const lldb_private::StructuredDataImpl &args_impl,
+    std::string &error_string) {
+  if (python_class_name == NULL || python_class_name[0] == '\0' ||
+      !session_dictionary_name)
     Py_RETURN_NONE;
-}
 
-void*
-lldb_private::LLDBSwigPythonCreateScriptedThreadPlan
-(
-    const char *python_class_name,
-    const char *session_dictionary_name,
-    lldb_private::StructuredDataImpl *args_impl,
-    std::string &error_string,
-    const lldb::ThreadPlanSP& thread_plan_sp
-)
-{
-    if (python_class_name == NULL || python_class_name[0] == '\0' || !session_dictionary_name)
-        Py_RETURN_NONE;
+  PyErr_Cleaner py_err_cleaner(true);
+
+  auto dict = PythonModule::MainModule().ResolveName<PythonDictionary>(
+      session_dictionary_name);
+  auto pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(
+      python_class_name, dict);
+
+  if (!pfunc.IsAllocated()) {
+    error_string.append("could not find script class: ");
+    error_string.append(python_class_name);
+    return nullptr;
+  }
+
+  PythonObject target_arg = ToSWIGWrapper(target_sp);
+
+  llvm::Expected<PythonCallable::ArgInfo> arg_info = pfunc.GetArgInfo();
+  if (!arg_info) {
+    llvm::handleAllErrors(
+        arg_info.takeError(),
+        [&](PythonException &E) { error_string.append(E.ReadBacktrace()); },
+        [&](const llvm::ErrorInfoBase &E) {
+          error_string.append(E.message());
+        });
+    Py_RETURN_NONE;
+  }
+
+  PythonObject result = {};
+  if (arg_info.get().max_positional_args == 2) {
+    result = pfunc(target_arg, ToSWIGWrapper(args_impl));
+  } else {
+    error_string.assign("wrong number of arguments in __init__, should be 2 "
+                        "(not including self)");
+    Py_RETURN_NONE;
+  }
 
+  if (result.IsAllocated())
+    return result.release();
+  Py_RETURN_NONE;
+}
 
-    PyErr_Cleaner py_err_cleaner(true);
+void *lldb_private::LLDBSwigPythonCreateScriptedThread(
+    const char *python_class_name, const char *session_dictionary_name,
+    const lldb::ProcessSP &process_sp, const StructuredDataImpl &args_impl,
+    std::string &error_string) {
+  if (python_class_name == NULL || python_class_name[0] == '\0' ||
+      !session_dictionary_name)
+    Py_RETURN_NONE;
 
-    auto dict = PythonModule::MainModule().ResolveName<PythonDictionary>(session_dictionary_name);
-    auto pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(python_class_name, dict);
+  PyErr_Cleaner py_err_cleaner(true);
+
+  auto dict = PythonModule::MainModule().ResolveName<PythonDictionary>(
+      session_dictionary_name);
+  auto pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(
+      python_class_name, dict);
+
+  if (!pfunc.IsAllocated()) {
+    error_string.append("could not find script class: ");
+    error_string.append(python_class_name);
+    return nullptr;
+  }
+
+  llvm::Expected<PythonCallable::ArgInfo> arg_info = pfunc.GetArgInfo();
+  if (!arg_info) {
+    llvm::handleAllErrors(
+        arg_info.takeError(),
+        [&](PythonException &E) { error_string.append(E.ReadBacktrace()); },
+        [&](const llvm::ErrorInfoBase &E) {
+          error_string.append(E.message());
+        });
+    Py_RETURN_NONE;
+  }
+
+  PythonObject result = {};
+  if (arg_info.get().max_positional_args == 2) {
+    result = pfunc(ToSWIGWrapper(process_sp), ToSWIGWrapper(args_impl));
+  } else {
+    error_string.assign("wrong number of arguments in __init__, should be 2 "
+                        "(not including self)");
+    Py_RETURN_NONE;
+  }
 
-    if (!pfunc.IsAllocated()) {
-        error_string.append("could not find script class: ");
-        error_string.append(python_class_name);
-        return nullptr;
-    }
+  if (result.IsAllocated())
+    return result.release();
+  Py_RETURN_NONE;
+}
 
-    PythonObject tp_arg = ToSWIGWrapper(thread_plan_sp);
-
-    llvm::Expected<PythonCallable::ArgInfo> arg_info = pfunc.GetArgInfo();
-    if (!arg_info) {
-        llvm::handleAllErrors(
-            arg_info.takeError(),
-            [&](PythonException &E) {
-                error_string.append(E.ReadBacktrace());
-            },
-            [&](const llvm::ErrorInfoBase &E) {
-                error_string.append(E.message());
-            });
-        Py_RETURN_NONE;
-    }
+void *lldb_private::LLDBSwigPythonCreateScriptedThreadPlan(
+    const char *python_class_name, const char *session_dictionary_name,
+    const lldb_private::StructuredDataImpl &args_impl,
+    std::string &error_string, const lldb::ThreadPlanSP &thread_plan_sp) {
+  if (python_class_name == NULL || python_class_name[0] == '\0' ||
+      !session_dictionary_name)
+    Py_RETURN_NONE;
 
-    PythonObject result = {};
-    if (arg_info.get().max_positional_args == 2) {
-        if (args_impl != nullptr) {
-           error_string.assign("args passed, but __init__ does not take an args dictionary");
-           Py_RETURN_NONE;
-        }
-        result = pfunc(tp_arg, dict);
-    } else if (arg_info.get().max_positional_args >= 3) {
-        // FIXME: SBStructuredData leaked here
-        PythonObject args_arg(PyRefType::Owned, SBTypeToSWIGWrapper(*new lldb::SBStructuredData(args_impl)));
-        result = pfunc(tp_arg, args_arg, dict);
-    } else {
-        error_string.assign("wrong number of arguments in __init__, should be 2 or 3 (not including self)");
-        Py_RETURN_NONE;
+  PyErr_Cleaner py_err_cleaner(true);
+
+  auto dict = PythonModule::MainModule().ResolveName<PythonDictionary>(
+      session_dictionary_name);
+  auto pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(
+      python_class_name, dict);
+
+  if (!pfunc.IsAllocated()) {
+    error_string.append("could not find script class: ");
+    error_string.append(python_class_name);
+    return nullptr;
+  }
+
+  PythonObject tp_arg = ToSWIGWrapper(thread_plan_sp);
+
+  llvm::Expected<PythonCallable::ArgInfo> arg_info = pfunc.GetArgInfo();
+  if (!arg_info) {
+    llvm::handleAllErrors(
+        arg_info.takeError(),
+        [&](PythonException &E) { error_string.append(E.ReadBacktrace()); },
+        [&](const llvm::ErrorInfoBase &E) {
+          error_string.append(E.message());
+        });
+    Py_RETURN_NONE;
+  }
+
+  PythonObject result = {};
+  auto args_sb = std::make_unique<lldb::SBStructuredData>(args_impl);
+  if (arg_info.get().max_positional_args == 2) {
+    if (args_sb->IsValid()) {
+      error_string.assign(
+          "args passed, but __init__ does not take an args dictionary");
+      Py_RETURN_NONE;
     }
+    result = pfunc(tp_arg, dict);
+  } else if (arg_info.get().max_positional_args >= 3) {
+    result = pfunc(tp_arg, ToSWIGWrapper(std::move(args_sb)), dict);
+  } else {
+    error_string.assign("wrong number of arguments in __init__, should be 2 or "
+                        "3 (not including self)");
+    Py_RETURN_NONE;
+  }
 
-    // FIXME: At this point we should check that the class we found supports all the methods
-    // that we need.
+  // FIXME: At this point we should check that the class we found supports all
+  // the methods that we need.
 
-    if (result.IsAllocated())
-        return result.release();
-    Py_RETURN_NONE;
+  if (result.IsAllocated())
+    return result.release();
+  Py_RETURN_NONE;
 }
 
-bool
-lldb_private::LLDBSWIGPythonCallThreadPlan
-(
-    void *implementor,
-    const char *method_name,
-    lldb_private::Event *event,
-    bool &got_error
-)
-{
-    got_error = false;
-
-    PyErr_Cleaner py_err_cleaner(false);
-    PythonObject self(PyRefType::Borrowed, static_cast<PyObject*>(implementor));
-    auto pfunc = self.ResolveName<PythonCallable>(method_name);
+bool lldb_private::LLDBSWIGPythonCallThreadPlan(
+    void *implementor, const char *method_name, lldb_private::Event *event,
+    bool &got_error) {
+  got_error = false;
 
-    if (!pfunc.IsAllocated())
-        return false;
-
-    PythonObject result;
-    if (event != nullptr)
-    {
-        lldb::SBEvent sb_event(event);
-        PythonObject event_arg(PyRefType::Owned, SBTypeToSWIGWrapper(sb_event));
-        result = pfunc(event_arg);
-    }
-    else
-        result = pfunc();
+  PyErr_Cleaner py_err_cleaner(false);
+  PythonObject self(PyRefType::Borrowed, static_cast<PyObject *>(implementor));
+  auto pfunc = self.ResolveName<PythonCallable>(method_name);
 
-    if (PyErr_Occurred())
-    {
-        got_error = true;
-        printf ("Return value was neither false nor true for call to %s.\n", method_name);
-        PyErr_Print();
-        return false;
-    }
+  if (!pfunc.IsAllocated())
+    return false;
 
-    if (result.get() == Py_True)
-        return true;
-    else if (result.get() == Py_False)
-        return false;
+  PythonObject result;
+  if (event != nullptr) {
+    lldb::SBEvent sb_event(event);
+    PythonObject event_arg(PyRefType::Owned, SBTypeToSWIGWrapper(sb_event));
+    result = pfunc(event_arg);
+  } else
+    result = pfunc();
 
-    // Somebody returned the wrong thing...
+  if (PyErr_Occurred()) {
     got_error = true;
-    printf ("Wrong return value type for call to %s.\n", method_name);
+    printf("Return value was neither false nor true for call to %s.\n",
+           method_name);
+    PyErr_Print();
     return false;
+  }
+
+  if (result.get() == Py_True)
+    return true;
+  else if (result.get() == Py_False)
+    return false;
+
+  // Somebody returned the wrong thing...
+  got_error = true;
+  printf("Wrong return value type for call to %s.\n", method_name);
+  return false;
 }
 
 void *lldb_private::LLDBSwigPythonCreateScriptedBreakpointResolver(
     const char *python_class_name, const char *session_dictionary_name,
-    lldb_private::StructuredDataImpl *args_impl,
+    const StructuredDataImpl &args_impl,
     const lldb::BreakpointSP &breakpoint_sp) {
 
-    if (python_class_name == NULL || python_class_name[0] == '\0' || !session_dictionary_name)
-        Py_RETURN_NONE;
+  if (python_class_name == NULL || python_class_name[0] == '\0' ||
+      !session_dictionary_name)
+    Py_RETURN_NONE;
 
-    PyErr_Cleaner py_err_cleaner(true);
+  PyErr_Cleaner py_err_cleaner(true);
 
-    auto dict = PythonModule::MainModule().ResolveName<PythonDictionary>(session_dictionary_name);
-    auto pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(python_class_name, dict);
+  auto dict = PythonModule::MainModule().ResolveName<PythonDictionary>(
+      session_dictionary_name);
+  auto pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(
+      python_class_name, dict);
 
-    if (!pfunc.IsAllocated())
-        return nullptr;
-
-    // FIXME: SBStructuredData leaked here
-    lldb::SBStructuredData *args_value = new lldb::SBStructuredData(args_impl);
-    PythonObject args_arg(PyRefType::Owned, SBTypeToSWIGWrapper(*args_value));
-
-    PythonObject result = pfunc(ToSWIGWrapper(breakpoint_sp), args_arg, dict);
-    // FIXME: At this point we should check that the class we found supports all the methods
-    // that we need.
-
-    if (result.IsAllocated())
-    {
-        // Check that __callback__ is defined:
-        auto callback_func = result.ResolveName<PythonCallable>("__callback__");
-        if (callback_func.IsAllocated())
-            return result.release();
-        else
-            result.release();
-    }
-    Py_RETURN_NONE;
+  if (!pfunc.IsAllocated())
+    return nullptr;
+
+  PythonObject result =
+      pfunc(ToSWIGWrapper(breakpoint_sp), ToSWIGWrapper(args_impl), dict);
+  // FIXME: At this point we should check that the class we found supports all
+  // the methods that we need.
+
+  if (result.IsAllocated()) {
+    // Check that __callback__ is defined:
+    auto callback_func = result.ResolveName<PythonCallable>("__callback__");
+    if (callback_func.IsAllocated())
+      return result.release();
+    else
+      result.release();
+  }
+  Py_RETURN_NONE;
 }
 
-unsigned int
-lldb_private::LLDBSwigPythonCallBreakpointResolver
-(
-    void *implementor,
-    const char *method_name,
-    lldb_private::SymbolContext *sym_ctx
-)
-{
-    PyErr_Cleaner py_err_cleaner(false);
-    PythonObject self(PyRefType::Borrowed, static_cast<PyObject*>(implementor));
-    auto pfunc = self.ResolveName<PythonCallable>(method_name);
+unsigned int lldb_private::LLDBSwigPythonCallBreakpointResolver(
+    void *implementor, const char *method_name,
+    lldb_private::SymbolContext *sym_ctx) {
+  PyErr_Cleaner py_err_cleaner(false);
+  PythonObject self(PyRefType::Borrowed, static_cast<PyObject *>(implementor));
+  auto pfunc = self.ResolveName<PythonCallable>(method_name);
 
-    if (!pfunc.IsAllocated())
-        return 0;
-
-    PythonObject result;
-    if (sym_ctx != nullptr) {
-      lldb::SBSymbolContext sb_sym_ctx(sym_ctx);
-      PythonObject sym_ctx_arg(PyRefType::Owned, SBTypeToSWIGWrapper(sb_sym_ctx));
-      result = pfunc(sym_ctx_arg);
-    } else
-      result = pfunc();
-
-    if (PyErr_Occurred())
-    {
-        PyErr_Print();
-        PyErr_Clear();
-        return 0;
-    }
+  if (!pfunc.IsAllocated())
+    return 0;
 
-    // The callback will return a bool, but we're need to also return ints
-    // so we're squirrelling the bool through as an int...  And if you return
-    // nothing, we'll continue.
-    if (strcmp(method_name, "__callback__") == 0) {
-        if (result.get() == Py_False)
-          return 0;
-        else
-          return 1;
-    }
+  PythonObject result = sym_ctx ? pfunc(ToSWIGWrapper(*sym_ctx)) : pfunc();
 
-    long long ret_val = unwrapOrSetPythonException(As<long long>(result));
+  if (PyErr_Occurred()) {
+    PyErr_Print();
+    PyErr_Clear();
+    return 0;
+  }
 
-    if (PyErr_Occurred()) {
-        PyErr_Print();
-        PyErr_Clear();
-        return 0;
-    }
+  // The callback will return a bool, but we're need to also return ints
+  // so we're squirrelling the bool through as an int...  And if you return
+  // nothing, we'll continue.
+  if (strcmp(method_name, "__callback__") == 0) {
+    if (result.get() == Py_False)
+      return 0;
+    else
+      return 1;
+  }
 
-    return ret_val;
-}
+  long long ret_val = unwrapOrSetPythonException(As<long long>(result));
 
-void *
-lldb_private::LLDBSwigPythonCreateScriptedStopHook
-(
-    lldb::TargetSP target_sp,
-    const char *python_class_name,
-    const char *session_dictionary_name,
-    lldb_private::StructuredDataImpl *args_impl,
-    Status &error
-)
-{
-    if (python_class_name == NULL || python_class_name[0] == '\0') {
-        error.SetErrorString("Empty class name.");
-        Py_RETURN_NONE;
-    }
-    if (!session_dictionary_name) {
-      error.SetErrorString("No session dictionary");
-      Py_RETURN_NONE;
-    }
-    
-    PyErr_Cleaner py_err_cleaner(true);
-
-    auto dict = 
-        PythonModule::MainModule().ResolveName<PythonDictionary>(
-            session_dictionary_name);
-    auto pfunc = 
-        PythonObject::ResolveNameWithDictionary<PythonCallable>(
-            python_class_name, dict);
-
-    if (!pfunc.IsAllocated()) {
-        error.SetErrorStringWithFormat("Could not find class: %s.", 
-                                       python_class_name);
-        return nullptr;
-    }
+  if (PyErr_Occurred()) {
+    PyErr_Print();
+    PyErr_Clear();
+    return 0;
+  }
+
+  return ret_val;
+}
 
-    // FIXME: SBStructuredData leaked here
-    lldb::SBStructuredData *args_value = new lldb::SBStructuredData(args_impl);
-    PythonObject args_arg(PyRefType::Owned, SBTypeToSWIGWrapper(*args_value));
-
-    PythonObject result = pfunc(ToSWIGWrapper(target_sp), args_arg, dict);
-
-    if (result.IsAllocated())
-    {
-        // Check that the handle_stop callback is defined:
-        auto callback_func = result.ResolveName<PythonCallable>("handle_stop");
-        if (callback_func.IsAllocated()) {
-          if (auto args_info = callback_func.GetArgInfo()) {
-            size_t num_args = (*args_info).max_positional_args; 
-            if (num_args != 2) {
-              error.SetErrorStringWithFormat("Wrong number of args for "
+void *lldb_private::LLDBSwigPythonCreateScriptedStopHook(
+    lldb::TargetSP target_sp, const char *python_class_name,
+    const char *session_dictionary_name, const StructuredDataImpl &args_impl,
+    Status &error) {
+  if (python_class_name == NULL || python_class_name[0] == '\0') {
+    error.SetErrorString("Empty class name.");
+    Py_RETURN_NONE;
+  }
+  if (!session_dictionary_name) {
+    error.SetErrorString("No session dictionary");
+    Py_RETURN_NONE;
+  }
+
+  PyErr_Cleaner py_err_cleaner(true);
+
+  auto dict = PythonModule::MainModule().ResolveName<PythonDictionary>(
+      session_dictionary_name);
+  auto pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(
+      python_class_name, dict);
+
+  if (!pfunc.IsAllocated()) {
+    error.SetErrorStringWithFormat("Could not find class: %s.",
+                                   python_class_name);
+    return nullptr;
+  }
+
+  PythonObject result =
+      pfunc(ToSWIGWrapper(target_sp), ToSWIGWrapper(args_impl), dict);
+
+  if (result.IsAllocated()) {
+    // Check that the handle_stop callback is defined:
+    auto callback_func = result.ResolveName<PythonCallable>("handle_stop");
+    if (callback_func.IsAllocated()) {
+      if (auto args_info = callback_func.GetArgInfo()) {
+        size_t num_args = (*args_info).max_positional_args;
+        if (num_args != 2) {
+          error.SetErrorStringWithFormat(
+              "Wrong number of args for "
               "handle_stop callback, should be 2 (excluding self), got: %zu",
               num_args);
-              Py_RETURN_NONE;
-            } else
-              return result.release();
-          } else {
-            error.SetErrorString("Couldn't get num arguments for handle_stop "
-                                 "callback.");
-            Py_RETURN_NONE;
-          }
+          Py_RETURN_NONE;
+        } else
           return result.release();
-        }
-        else {
-          error.SetErrorStringWithFormat("Class \"%s\" is missing the required "
-                                         "handle_stop callback.",
-                                         python_class_name);
-          result.release();
-        }
+      } else {
+        error.SetErrorString("Couldn't get num arguments for handle_stop "
+                             "callback.");
+        Py_RETURN_NONE;
+      }
+      return result.release();
+    } else {
+      error.SetErrorStringWithFormat("Class \"%s\" is missing the required "
+                                     "handle_stop callback.",
+                                     python_class_name);
+      result.release();
     }
-    Py_RETURN_NONE;
+  }
+  Py_RETURN_NONE;
 }
 
-bool
-lldb_private::LLDBSwigPythonStopHookCallHandleStop
-(
-    void *implementor,
-    lldb::ExecutionContextRefSP exc_ctx_sp,
-    lldb::StreamSP stream
-)
-{
-    // handle_stop will return a bool with the meaning "should_stop"...
-    // If you return nothing we'll assume we are going to stop.
-    // Also any errors should return true, since we should stop on error.
-
-    PyErr_Cleaner py_err_cleaner(false);
-    PythonObject self(PyRefType::Borrowed, static_cast<PyObject*>(implementor));
-    auto pfunc = self.ResolveName<PythonCallable>("handle_stop");
+bool lldb_private::LLDBSwigPythonStopHookCallHandleStop(
+    void *implementor, lldb::ExecutionContextRefSP exc_ctx_sp,
+    lldb::StreamSP stream) {
+  // handle_stop will return a bool with the meaning "should_stop"...
+  // If you return nothing we'll assume we are going to stop.
+  // Also any errors should return true, since we should stop on error.
 
-    if (!pfunc.IsAllocated())
-        return true;
-
-    PythonObject result;
-    lldb::SBExecutionContext sb_exc_ctx(exc_ctx_sp);
-    PythonObject exc_ctx_arg(PyRefType::Owned, SBTypeToSWIGWrapper(sb_exc_ctx));
-    lldb::SBStream sb_stream;
-    PythonObject sb_stream_arg(PyRefType::Owned, 
-                               SBTypeToSWIGWrapper(sb_stream));
-    result = pfunc(exc_ctx_arg, sb_stream_arg);
-
-    if (PyErr_Occurred())
-    {
-        stream->PutCString("Python error occurred handling stop-hook.");
-        PyErr_Print();
-        PyErr_Clear();
-        return true;
-    }
-    
-    // Now add the result to the output stream.  SBStream only
-    // makes an internally help StreamString which I can't interpose, so I 
-    // have to copy it over here.
-    stream->PutCString(sb_stream.GetData());
-    
-    if (result.get() == Py_False)
-      return false;
-    else
-      return true;
-}
+  PyErr_Cleaner py_err_cleaner(false);
+  PythonObject self(PyRefType::Borrowed, static_cast<PyObject *>(implementor));
+  auto pfunc = self.ResolveName<PythonCallable>("handle_stop");
 
-// wrapper that calls an optional instance member of an object taking no arguments
-static PyObject*
-LLDBSwigPython_CallOptionalMember
-(
-    PyObject* implementor,
-    char* callee_name,
-    PyObject* ret_if_not_found = Py_None,
-    bool* was_found = NULL
-)
-{
-    PyErr_Cleaner py_err_cleaner(false);
-
-    PythonObject self(PyRefType::Borrowed, static_cast<PyObject*>(implementor));
-    auto pfunc = self.ResolveName<PythonCallable>(callee_name);
+  if (!pfunc.IsAllocated())
+    return true;
 
-    if (!pfunc.IsAllocated())
-    {
-        if (was_found)
-            *was_found = false;
-        Py_XINCREF(ret_if_not_found);
-        return ret_if_not_found;
-    }
+  auto *sb_stream = new lldb::SBStream();
+  PythonObject sb_stream_arg =
+      ToSWIGWrapper(std::unique_ptr<lldb::SBStream>(sb_stream));
+  PythonObject result =
+      pfunc(ToSWIGWrapper(std::move(exc_ctx_sp)), sb_stream_arg);
 
-    if (was_found)
-        *was_found = true;
+  if (PyErr_Occurred()) {
+    stream->PutCString("Python error occurred handling stop-hook.");
+    PyErr_Print();
+    PyErr_Clear();
+    return true;
+  }
 
-    PythonObject result = pfunc();
-    return result.release();
-}
+  // Now add the result to the output stream.  SBStream only
+  // makes an internally help StreamString which I can't interpose, so I
+  // have to copy it over here.
+  stream->PutCString(sb_stream->GetData());
 
-size_t
-lldb_private::LLDBSwigPython_CalculateNumChildren
-(
-    PyObject *implementor,
-    uint32_t max
-)
-{
-    PythonObject self(PyRefType::Borrowed, implementor);
-    auto pfunc = self.ResolveName<PythonCallable>("num_children");
+  if (result.get() == Py_False)
+    return false;
+  else
+    return true;
+}
 
-    if (!pfunc.IsAllocated())
-        return 0;
+// wrapper that calls an optional instance member of an object taking no
+// arguments
+static PyObject *LLDBSwigPython_CallOptionalMember(
+    PyObject * implementor, char *callee_name,
+    PyObject *ret_if_not_found = Py_None, bool *was_found = NULL) {
+  PyErr_Cleaner py_err_cleaner(false);
 
-    auto arg_info = pfunc.GetArgInfo();
-    if (!arg_info) {
-        llvm::consumeError(arg_info.takeError());
-        return 0;
-    }
+  PythonObject self(PyRefType::Borrowed, static_cast<PyObject *>(implementor));
+  auto pfunc = self.ResolveName<PythonCallable>(callee_name);
 
-    size_t ret_val;
-    if (arg_info.get().max_positional_args < 1)
-        ret_val = unwrapOrSetPythonException(As<long long>(pfunc.Call()));
-    else
-        ret_val = unwrapOrSetPythonException(As<long long>(pfunc.Call(PythonInteger(max))));
+  if (!pfunc.IsAllocated()) {
+    if (was_found)
+      *was_found = false;
+    Py_XINCREF(ret_if_not_found);
+    return ret_if_not_found;
+  }
 
-    if (PyErr_Occurred())
-    {
-        PyErr_Print();
-        PyErr_Clear();
-        return 0;
-    }
+  if (was_found)
+    *was_found = true;
 
-    if (arg_info.get().max_positional_args < 1)
-        ret_val = std::min(ret_val, static_cast<size_t>(max));
+  PythonObject result = pfunc();
+  return result.release();
+}
 
-    return ret_val;
+size_t lldb_private::LLDBSwigPython_CalculateNumChildren(PyObject * implementor,
+                                                         uint32_t max) {
+  PythonObject self(PyRefType::Borrowed, implementor);
+  auto pfunc = self.ResolveName<PythonCallable>("num_children");
+
+  if (!pfunc.IsAllocated())
+    return 0;
+
+  auto arg_info = pfunc.GetArgInfo();
+  if (!arg_info) {
+    llvm::consumeError(arg_info.takeError());
+    return 0;
+  }
+
+  size_t ret_val;
+  if (arg_info.get().max_positional_args < 1)
+    ret_val = unwrapOrSetPythonException(As<long long>(pfunc.Call()));
+  else
+    ret_val = unwrapOrSetPythonException(
+        As<long long>(pfunc.Call(PythonInteger(max))));
+
+  if (PyErr_Occurred()) {
+    PyErr_Print();
+    PyErr_Clear();
+    return 0;
+  }
+
+  if (arg_info.get().max_positional_args < 1)
+    ret_val = std::min(ret_val, static_cast<size_t>(max));
+
+  return ret_val;
 }
 
-PyObject*
-lldb_private::LLDBSwigPython_GetChildAtIndex
-(
-    PyObject *implementor,
-    uint32_t idx
-)
-{
-    PyErr_Cleaner py_err_cleaner(true);
+PyObject *lldb_private::LLDBSwigPython_GetChildAtIndex(PyObject * implementor,
+                                                       uint32_t idx) {
+  PyErr_Cleaner py_err_cleaner(true);
 
-    PythonObject self(PyRefType::Borrowed, implementor);
-    auto pfunc = self.ResolveName<PythonCallable>("get_child_at_index");
+  PythonObject self(PyRefType::Borrowed, implementor);
+  auto pfunc = self.ResolveName<PythonCallable>("get_child_at_index");
 
-    if (!pfunc.IsAllocated())
-        return nullptr;
+  if (!pfunc.IsAllocated())
+    return nullptr;
 
-    PythonObject result = pfunc(PythonInteger(idx));
+  PythonObject result = pfunc(PythonInteger(idx));
 
-    if (!result.IsAllocated())
-        return nullptr;
+  if (!result.IsAllocated())
+    return nullptr;
 
-    lldb::SBValue* sbvalue_ptr = nullptr;
-    if (SWIG_ConvertPtr(result.get(), (void**)&sbvalue_ptr, SWIGTYPE_p_lldb__SBValue, 0) == -1)
-        return nullptr;
+  lldb::SBValue *sbvalue_ptr = nullptr;
+  if (SWIG_ConvertPtr(result.get(), (void **)&sbvalue_ptr,
+                      SWIGTYPE_p_lldb__SBValue, 0) == -1)
+    return nullptr;
 
-    if (sbvalue_ptr == nullptr)
-        return nullptr;
+  if (sbvalue_ptr == nullptr)
+    return nullptr;
 
-    return result.release();
+  return result.release();
 }
 
-int
-lldb_private::LLDBSwigPython_GetIndexOfChildWithName
-(
-    PyObject *implementor,
-    const char* child_name
-)
-{
-    PyErr_Cleaner py_err_cleaner(true);
+int lldb_private::LLDBSwigPython_GetIndexOfChildWithName(
+    PyObject * implementor, const char *child_name) {
+  PyErr_Cleaner py_err_cleaner(true);
 
-    PythonObject self(PyRefType::Borrowed, implementor);
-    auto pfunc = self.ResolveName<PythonCallable>("get_child_index");
+  PythonObject self(PyRefType::Borrowed, implementor);
+  auto pfunc = self.ResolveName<PythonCallable>("get_child_index");
 
-    if (!pfunc.IsAllocated())
-        return UINT32_MAX;
+  if (!pfunc.IsAllocated())
+    return UINT32_MAX;
 
-    llvm::Expected<PythonObject> result = pfunc.Call(PythonString(child_name));
+  llvm::Expected<PythonObject> result = pfunc.Call(PythonString(child_name));
 
-    long long retval = unwrapOrSetPythonException(As<long long>(std::move(result)));
+  long long retval =
+      unwrapOrSetPythonException(As<long long>(std::move(result)));
 
-    if (PyErr_Occurred()) { 
-        PyErr_Clear(); // FIXME print this? do something else
-        return UINT32_MAX;
-    }
+  if (PyErr_Occurred()) {
+    PyErr_Clear(); // FIXME print this? do something else
+    return UINT32_MAX;
+  }
 
-    if (retval >= 0)
-        return (uint32_t)retval;
+  if (retval >= 0)
+    return (uint32_t)retval;
 
-    return UINT32_MAX;
+  return UINT32_MAX;
 }
 
-bool
-lldb_private::LLDBSwigPython_UpdateSynthProviderInstance
-(
-    PyObject *implementor
-)
-{
-    bool ret_val = false;
+bool lldb_private::LLDBSwigPython_UpdateSynthProviderInstance(PyObject *
+                                                              implementor) {
+  bool ret_val = false;
 
-    static char callee_name[] = "update";
+  static char callee_name[] = "update";
 
-    PyObject* py_return = LLDBSwigPython_CallOptionalMember(implementor,callee_name);
+  PyObject *py_return =
+      LLDBSwigPython_CallOptionalMember(implementor, callee_name);
 
-    if (py_return == Py_True)
-        ret_val = true;
+  if (py_return == Py_True)
+    ret_val = true;
 
-    Py_XDECREF(py_return);
+  Py_XDECREF(py_return);
 
-    return ret_val;
+  return ret_val;
 }
 
-bool
-lldb_private::LLDBSwigPython_MightHaveChildrenSynthProviderInstance
-(
-    PyObject *implementor
-)
-{
-    bool ret_val = false;
+bool lldb_private::LLDBSwigPython_MightHaveChildrenSynthProviderInstance(
+    PyObject * implementor) {
+  bool ret_val = false;
 
-    static char callee_name[] = "has_children";
+  static char callee_name[] = "has_children";
 
-    PyObject* py_return = LLDBSwigPython_CallOptionalMember(implementor,callee_name, Py_True);
+  PyObject *py_return =
+      LLDBSwigPython_CallOptionalMember(implementor, callee_name, Py_True);
 
-    if (py_return == Py_True)
-        ret_val = true;
+  if (py_return == Py_True)
+    ret_val = true;
 
-    Py_XDECREF(py_return);
+  Py_XDECREF(py_return);
 
-    return ret_val;
+  return ret_val;
 }
 
-PyObject*
-lldb_private::LLDBSwigPython_GetValueSynthProviderInstance
-(
-    PyObject *implementor
-)
-{
-    PyObject* ret_val = nullptr;
+PyObject *lldb_private::LLDBSwigPython_GetValueSynthProviderInstance(
+    PyObject * implementor) {
+  PyObject *ret_val = nullptr;
 
-    static char callee_name[] = "get_value";
+  static char callee_name[] = "get_value";
 
-    PyObject* py_return = LLDBSwigPython_CallOptionalMember(implementor,callee_name, Py_None);
+  PyObject *py_return =
+      LLDBSwigPython_CallOptionalMember(implementor, callee_name, Py_None);
 
-    if (py_return == Py_None || py_return == nullptr)
-        ret_val = nullptr;
+  if (py_return == Py_None || py_return == nullptr)
+    ret_val = nullptr;
 
-    lldb::SBValue* sbvalue_ptr = NULL;
+  lldb::SBValue *sbvalue_ptr = NULL;
 
-    if (SWIG_ConvertPtr(py_return, (void**)&sbvalue_ptr, SWIGTYPE_p_lldb__SBValue, 0) == -1)
-        ret_val = nullptr;
-    else if (sbvalue_ptr == NULL)
-        ret_val = nullptr;
-    else
-        ret_val = py_return;
+  if (SWIG_ConvertPtr(py_return, (void **)&sbvalue_ptr,
+                      SWIGTYPE_p_lldb__SBValue, 0) == -1)
+    ret_val = nullptr;
+  else if (sbvalue_ptr == NULL)
+    ret_val = nullptr;
+  else
+    ret_val = py_return;
 
-    Py_XDECREF(py_return);
-    return ret_val;
+  Py_XDECREF(py_return);
+  return ret_val;
 }
 
-void*
-lldb_private::LLDBSWIGPython_CastPyObjectToSBData
-(
-    PyObject* data
-)
-{
-    lldb::SBData* sb_ptr = nullptr;
+void *lldb_private::LLDBSWIGPython_CastPyObjectToSBData(PyObject * data) {
+  lldb::SBData *sb_ptr = nullptr;
 
-    int valid_cast = SWIG_ConvertPtr(data, (void**)&sb_ptr, SWIGTYPE_p_lldb__SBData, 0);
+  int valid_cast =
+      SWIG_ConvertPtr(data, (void **)&sb_ptr, SWIGTYPE_p_lldb__SBData, 0);
 
-    if (valid_cast == -1)
-        return NULL;
+  if (valid_cast == -1)
+    return NULL;
 
-    return sb_ptr;
+  return sb_ptr;
 }
 
+void *lldb_private::LLDBSWIGPython_CastPyObjectToSBError(PyObject * data) {
+  lldb::SBError *sb_ptr = nullptr;
 
-void*
-lldb_private::LLDBSWIGPython_CastPyObjectToSBError
-(
-    PyObject* data
-)
-{
-    lldb::SBError* sb_ptr = nullptr;
-
-    int valid_cast = SWIG_ConvertPtr(data, (void**)&sb_ptr, SWIGTYPE_p_lldb__SBError, 0);
+  int valid_cast =
+      SWIG_ConvertPtr(data, (void **)&sb_ptr, SWIGTYPE_p_lldb__SBError, 0);
 
-    if (valid_cast == -1)
-        return NULL;
+  if (valid_cast == -1)
+    return NULL;
 
-    return sb_ptr;
+  return sb_ptr;
 }
 
+void *lldb_private::LLDBSWIGPython_CastPyObjectToSBValue(PyObject * data) {
+  lldb::SBValue *sb_ptr = NULL;
 
-void*
-lldb_private::LLDBSWIGPython_CastPyObjectToSBValue
-(
-    PyObject* data
-)
-{
-    lldb::SBValue* sb_ptr = NULL;
+  int valid_cast =
+      SWIG_ConvertPtr(data, (void **)&sb_ptr, SWIGTYPE_p_lldb__SBValue, 0);
 
-    int valid_cast = SWIG_ConvertPtr(data, (void**)&sb_ptr, SWIGTYPE_p_lldb__SBValue, 0);
+  if (valid_cast == -1)
+    return NULL;
 
-    if (valid_cast == -1)
-        return NULL;
-
-    return sb_ptr;
+  return sb_ptr;
 }
 
-void*
-lldb_private::LLDBSWIGPython_CastPyObjectToSBMemoryRegionInfo
-(
-    PyObject* data
-)
-{
-    lldb::SBMemoryRegionInfo* sb_ptr = NULL;
+void *lldb_private::LLDBSWIGPython_CastPyObjectToSBMemoryRegionInfo(PyObject *
+                                                                    data) {
+  lldb::SBMemoryRegionInfo *sb_ptr = NULL;
 
-    int valid_cast = SWIG_ConvertPtr(data, (void**)&sb_ptr, SWIGTYPE_p_lldb__SBMemoryRegionInfo, 0);
+  int valid_cast = SWIG_ConvertPtr(data, (void **)&sb_ptr,
+                                   SWIGTYPE_p_lldb__SBMemoryRegionInfo, 0);
 
-    if (valid_cast == -1)
-        return NULL;
+  if (valid_cast == -1)
+    return NULL;
 
-    return sb_ptr;
+  return sb_ptr;
 }
 
-bool
-lldb_private::LLDBSwigPythonCallCommand
-(
-    const char *python_function_name,
-    const char *session_dictionary_name,
-    lldb::DebuggerSP& debugger,
-    const char* args,
-    lldb_private::CommandReturnObject& cmd_retobj,
-    lldb::ExecutionContextRefSP exe_ctx_ref_sp
-)
-{
-    lldb::SBCommandReturnObject cmd_retobj_sb(cmd_retobj);
-    lldb::SBDebugger debugger_sb(debugger);
-    lldb::SBExecutionContext exe_ctx_sb(exe_ctx_ref_sp);
-
-    PyErr_Cleaner py_err_cleaner(true);
-    auto dict = PythonModule::MainModule().ResolveName<PythonDictionary>(session_dictionary_name);
-    auto pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(python_function_name, dict);
+bool lldb_private::LLDBSwigPythonCallCommand(
+    const char *python_function_name, const char *session_dictionary_name,
+    lldb::DebuggerSP debugger, const char *args,
+    lldb_private::CommandReturnObject &cmd_retobj,
+    lldb::ExecutionContextRefSP exe_ctx_ref_sp) {
+  lldb::SBCommandReturnObject cmd_retobj_sb(cmd_retobj);
+
+  PyErr_Cleaner py_err_cleaner(true);
+  auto dict = PythonModule::MainModule().ResolveName<PythonDictionary>(
+      session_dictionary_name);
+  auto pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(
+      python_function_name, dict);
+
+  if (!pfunc.IsAllocated())
+    return false;
 
-    if (!pfunc.IsAllocated())
-        return false;
+  auto argc = pfunc.GetArgInfo();
+  if (!argc) {
+    llvm::consumeError(argc.takeError());
+    return false;
+  }
+  PythonObject debugger_arg = ToSWIGWrapper(std::move(debugger));
+  PythonObject cmd_retobj_arg(PyRefType::Owned,
+                              SBTypeToSWIGWrapper(cmd_retobj_sb));
+
+  if (argc.get().max_positional_args < 5u)
+    pfunc(debugger_arg, PythonString(args), cmd_retobj_arg, dict);
+  else
+    pfunc(debugger_arg, PythonString(args),
+          ToSWIGWrapper(std::move(exe_ctx_ref_sp)), cmd_retobj_arg, dict);
+
+  return true;
+}
 
-    auto argc = pfunc.GetArgInfo();
-    if (!argc) {
-        llvm::consumeError(argc.takeError());
-        return false;
-    }
-    PythonObject debugger_arg(PyRefType::Owned, SBTypeToSWIGWrapper(debugger_sb));
-    PythonObject exe_ctx_arg(PyRefType::Owned, SBTypeToSWIGWrapper(exe_ctx_sb));
-    PythonObject cmd_retobj_arg(PyRefType::Owned, SBTypeToSWIGWrapper(cmd_retobj_sb));
+bool lldb_private::LLDBSwigPythonCallCommandObject(
+    PyObject *implementor, lldb::DebuggerSP debugger, const char *args,
+    lldb_private::CommandReturnObject &cmd_retobj,
+    lldb::ExecutionContextRefSP exe_ctx_ref_sp) {
+  lldb::SBCommandReturnObject cmd_retobj_sb(cmd_retobj);
 
-    if (argc.get().max_positional_args < 5u)
-        pfunc(debugger_arg, PythonString(args), cmd_retobj_arg, dict);
-    else
-        pfunc(debugger_arg, PythonString(args), exe_ctx_arg, cmd_retobj_arg, dict);
+  PyErr_Cleaner py_err_cleaner(true);
 
-    return true;
-}
+  PythonObject self(PyRefType::Borrowed, implementor);
+  auto pfunc = self.ResolveName<PythonCallable>("__call__");
 
-bool
-lldb_private::LLDBSwigPythonCallCommandObject
-(
-    PyObject *implementor,
-    lldb::DebuggerSP& debugger,
-    const char* args,
-    lldb_private::CommandReturnObject& cmd_retobj,
-    lldb::ExecutionContextRefSP exe_ctx_ref_sp
-)
-{
-    lldb::SBCommandReturnObject cmd_retobj_sb(cmd_retobj);
-    lldb::SBDebugger debugger_sb(debugger);
-    lldb::SBExecutionContext exe_ctx_sb(exe_ctx_ref_sp);
-
-    PyErr_Cleaner py_err_cleaner(true);
-
-    PythonObject self(PyRefType::Borrowed, implementor);
-    auto pfunc = self.ResolveName<PythonCallable>("__call__");
-
-    if (!pfunc.IsAllocated())
-        return false;
+  if (!pfunc.IsAllocated())
+    return false;
 
-    PythonObject debugger_arg(PyRefType::Owned, SBTypeToSWIGWrapper(debugger_sb));
-    PythonObject exe_ctx_arg(PyRefType::Owned, SBTypeToSWIGWrapper(exe_ctx_sb));
-    PythonObject cmd_retobj_arg(PyRefType::Owned, SBTypeToSWIGWrapper(cmd_retobj_sb));
+  PythonObject cmd_retobj_arg(PyRefType::Owned,
+                              SBTypeToSWIGWrapper(cmd_retobj_sb));
 
-    pfunc(debugger_arg, PythonString(args), exe_ctx_arg, cmd_retobj_arg);
+  pfunc(ToSWIGWrapper(std::move(debugger)), PythonString(args),
+        ToSWIGWrapper(exe_ctx_ref_sp), cmd_retobj_arg);
 
-    return true;
+  return true;
 }
 
-void*
-lldb_private::LLDBSWIGPythonCreateOSPlugin
-(
-    const char *python_class_name,
-    const char *session_dictionary_name,
-    const lldb::ProcessSP& process_sp
-)
-{
-    if (python_class_name == NULL || python_class_name[0] == '\0' || !session_dictionary_name)
-        Py_RETURN_NONE;
+void *lldb_private::LLDBSWIGPythonCreateOSPlugin(
+    const char *python_class_name, const char *session_dictionary_name,
+    const lldb::ProcessSP &process_sp) {
+  if (python_class_name == NULL || python_class_name[0] == '\0' ||
+      !session_dictionary_name)
+    Py_RETURN_NONE;
 
-    PyErr_Cleaner py_err_cleaner(true);
+  PyErr_Cleaner py_err_cleaner(true);
 
-    auto dict = PythonModule::MainModule().ResolveName<PythonDictionary>(session_dictionary_name);
-    auto pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(python_class_name, dict);
+  auto dict = PythonModule::MainModule().ResolveName<PythonDictionary>(
+      session_dictionary_name);
+  auto pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(
+      python_class_name, dict);
 
-    if (!pfunc.IsAllocated())
-        Py_RETURN_NONE;
+  if (!pfunc.IsAllocated())
+    Py_RETURN_NONE;
 
-    auto result = pfunc(ToSWIGWrapper(process_sp));
+  auto result = pfunc(ToSWIGWrapper(process_sp));
 
-    if (result.IsAllocated())
-        return result.release();
+  if (result.IsAllocated())
+    return result.release();
 
-    Py_RETURN_NONE;
+  Py_RETURN_NONE;
 }
 
-void*
-lldb_private::LLDBSWIGPython_CreateFrameRecognizer
-(
-    const char *python_class_name,
-    const char *session_dictionary_name
-)
-{
-    if (python_class_name == NULL || python_class_name[0] == '\0' || !session_dictionary_name)
-        Py_RETURN_NONE;
+void *lldb_private::LLDBSWIGPython_CreateFrameRecognizer(
+    const char *python_class_name, const char *session_dictionary_name) {
+  if (python_class_name == NULL || python_class_name[0] == '\0' ||
+      !session_dictionary_name)
+    Py_RETURN_NONE;
 
-    PyErr_Cleaner py_err_cleaner(true);
+  PyErr_Cleaner py_err_cleaner(true);
 
-    auto dict = PythonModule::MainModule().ResolveName<PythonDictionary>(session_dictionary_name);
-    auto pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(python_class_name, dict);
+  auto dict = PythonModule::MainModule().ResolveName<PythonDictionary>(
+      session_dictionary_name);
+  auto pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(
+      python_class_name, dict);
 
-    if (!pfunc.IsAllocated())
-        Py_RETURN_NONE;
+  if (!pfunc.IsAllocated())
+    Py_RETURN_NONE;
 
-    auto result = pfunc();
+  auto result = pfunc();
 
-    if (result.IsAllocated())
-        return result.release();
+  if (result.IsAllocated())
+    return result.release();
 
-    Py_RETURN_NONE;
+  Py_RETURN_NONE;
 }
 
-PyObject*
-lldb_private::LLDBSwigPython_GetRecognizedArguments
-(
-    PyObject *implementor,
-    const lldb::StackFrameSP& frame_sp
-)
-{
-    static char callee_name[] = "get_recognized_arguments";
-
-    lldb::SBFrame frame_sb(frame_sp);
-    PyObject *arg = SBTypeToSWIGWrapper(frame_sb);
-
-    PythonString str(callee_name);
-    PyObject* result = PyObject_CallMethodObjArgs(implementor, str.get(), arg,
-                                                  NULL);
-    return result;
+PyObject *lldb_private::LLDBSwigPython_GetRecognizedArguments(
+    PyObject * implementor, const lldb::StackFrameSP &frame_sp) {
+  static char callee_name[] = "get_recognized_arguments";
+
+  PythonObject arg = ToSWIGWrapper(frame_sp);
+
+  PythonString str(callee_name);
+  PyObject *result =
+      PyObject_CallMethodObjArgs(implementor, str.get(), arg.get(), NULL);
+  return result;
 }
 
-void*
-lldb_private::LLDBSWIGPython_GetDynamicSetting (void* module, const char* setting, const lldb::TargetSP& target_sp)
-{
-    if (!module || !setting)
-        Py_RETURN_NONE;
+void *lldb_private::LLDBSWIGPython_GetDynamicSetting(
+    void *module, const char *setting, const lldb::TargetSP &target_sp) {
+  if (!module || !setting)
+    Py_RETURN_NONE;
 
-    PyErr_Cleaner py_err_cleaner(true);
-    PythonObject py_module(PyRefType::Borrowed, (PyObject *)module);
-    auto pfunc = py_module.ResolveName<PythonCallable>("get_dynamic_setting");
+  PyErr_Cleaner py_err_cleaner(true);
+  PythonObject py_module(PyRefType::Borrowed, (PyObject *)module);
+  auto pfunc = py_module.ResolveName<PythonCallable>("get_dynamic_setting");
 
-    if (!pfunc.IsAllocated())
-        Py_RETURN_NONE;
+  if (!pfunc.IsAllocated())
+    Py_RETURN_NONE;
 
-    auto result = pfunc(ToSWIGWrapper(target_sp), PythonString(setting));
+  auto result = pfunc(ToSWIGWrapper(target_sp), PythonString(setting));
 
-    return result.release();
+  return result.release();
 }
 
 bool lldb_private::LLDBSWIGPythonRunScriptKeywordProcess(
     const char *python_function_name, const char *session_dictionary_name,
     const lldb::ProcessSP &process, std::string &output) {
 
-    if (python_function_name == NULL || python_function_name[0] == '\0' || !session_dictionary_name)
-        return false;
+  if (python_function_name == NULL || python_function_name[0] == '\0' ||
+      !session_dictionary_name)
+    return false;
 
-    PyErr_Cleaner py_err_cleaner(true);
+  PyErr_Cleaner py_err_cleaner(true);
 
-    auto dict = PythonModule::MainModule().ResolveName<PythonDictionary>(session_dictionary_name);
-    auto pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(python_function_name, dict);
+  auto dict = PythonModule::MainModule().ResolveName<PythonDictionary>(
+      session_dictionary_name);
+  auto pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(
+      python_function_name, dict);
 
-    if (!pfunc.IsAllocated())
-        return false;
+  if (!pfunc.IsAllocated())
+    return false;
 
-    auto result = pfunc(ToSWIGWrapper(process), dict);
+  auto result = pfunc(ToSWIGWrapper(process), dict);
 
-    output = result.Str().GetString().str();
+  output = result.Str().GetString().str();
 
-    return true;
+  return true;
 }
 
-bool
-lldb_private::LLDBSWIGPythonRunScriptKeywordThread
-(const char* python_function_name,
-const char* session_dictionary_name,
-lldb::ThreadSP& thread,
-std::string& output)
-
-{
-    if (python_function_name == NULL || python_function_name[0] == '\0' || !session_dictionary_name)
-        return false;
-
-    PyErr_Cleaner py_err_cleaner(true);
+llvm::Optional<std::string> lldb_private::LLDBSWIGPythonRunScriptKeywordThread(
+    const char *python_function_name, const char *session_dictionary_name,
+    lldb::ThreadSP thread) {
+  if (python_function_name == NULL || python_function_name[0] == '\0' ||
+      !session_dictionary_name)
+    return llvm::None;
 
-    auto dict = PythonModule::MainModule().ResolveName<PythonDictionary>(session_dictionary_name);
-    auto pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(python_function_name, dict);
+  PyErr_Cleaner py_err_cleaner(true);
 
-    if (!pfunc.IsAllocated())
-        return false;
+  auto dict = PythonModule::MainModule().ResolveName<PythonDictionary>(
+      session_dictionary_name);
+  auto pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(
+      python_function_name, dict);
 
-    lldb::SBThread thread_sb(thread);
-    PythonObject thread_arg(PyRefType::Owned, SBTypeToSWIGWrapper(thread_sb));
-    auto result = pfunc(thread_arg, dict);
+  if (!pfunc.IsAllocated())
+    return llvm::None;
 
-    output = result.Str().GetString().str();
+  auto result = pfunc(ToSWIGWrapper(std::move(thread)), dict);
 
-    return true;
+  return result.Str().GetString().str();
 }
 
 bool lldb_private::LLDBSWIGPythonRunScriptKeywordTarget(
     const char *python_function_name, const char *session_dictionary_name,
     const lldb::TargetSP &target, std::string &output) {
 
-    if (python_function_name == NULL || python_function_name[0] == '\0' || !session_dictionary_name)
-        return false;
+  if (python_function_name == NULL || python_function_name[0] == '\0' ||
+      !session_dictionary_name)
+    return false;
 
-    PyErr_Cleaner py_err_cleaner(true);
+  PyErr_Cleaner py_err_cleaner(true);
 
-    auto dict = PythonModule::MainModule().ResolveName<PythonDictionary>(session_dictionary_name);
-    auto pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(python_function_name,dict);
+  auto dict = PythonModule::MainModule().ResolveName<PythonDictionary>(
+      session_dictionary_name);
+  auto pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(
+      python_function_name, dict);
 
-    if (!pfunc.IsAllocated())
-        return false;
+  if (!pfunc.IsAllocated())
+    return false;
 
-    auto result = pfunc(ToSWIGWrapper(target), dict);
+  auto result = pfunc(ToSWIGWrapper(target), dict);
 
-    output = result.Str().GetString().str();
+  output = result.Str().GetString().str();
 
-    return true;
+  return true;
 }
 
-bool
-lldb_private::LLDBSWIGPythonRunScriptKeywordFrame
-(const char* python_function_name,
-const char* session_dictionary_name,
-lldb::StackFrameSP& frame,
-std::string& output)
-
-{
-    if (python_function_name == NULL || python_function_name[0] == '\0' || !session_dictionary_name)
-        return false;
-
-    PyErr_Cleaner py_err_cleaner(true);
+llvm::Optional<std::string> lldb_private::LLDBSWIGPythonRunScriptKeywordFrame(
+    const char *python_function_name, const char *session_dictionary_name,
+    lldb::StackFrameSP frame) {
+  if (python_function_name == NULL || python_function_name[0] == '\0' ||
+      !session_dictionary_name)
+    return llvm::None;
 
-    auto dict = PythonModule::MainModule().ResolveName<PythonDictionary>(session_dictionary_name);
-    auto pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(python_function_name,dict);
+  PyErr_Cleaner py_err_cleaner(true);
 
-    if (!pfunc.IsAllocated())
-        return false;
+  auto dict = PythonModule::MainModule().ResolveName<PythonDictionary>(
+      session_dictionary_name);
+  auto pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(
+      python_function_name, dict);
 
-    lldb::SBFrame frame_sb(frame);
-    PythonObject frame_arg(PyRefType::Owned, SBTypeToSWIGWrapper(frame_sb));
-    auto result = pfunc(frame_arg, dict);
+  if (!pfunc.IsAllocated())
+    return llvm::None;
 
-    output = result.Str().GetString().str();
+  auto result = pfunc(ToSWIGWrapper(std::move(frame)), dict);
 
-    return true;
+  return result.Str().GetString().str();
 }
 
 bool lldb_private::LLDBSWIGPythonRunScriptKeywordValue(
     const char *python_function_name, const char *session_dictionary_name,
     const lldb::ValueObjectSP &value, std::string &output) {
 
-    if (python_function_name == NULL || python_function_name[0] == '\0' || !session_dictionary_name)
-        return false;
+  if (python_function_name == NULL || python_function_name[0] == '\0' ||
+      !session_dictionary_name)
+    return false;
 
-    PyErr_Cleaner py_err_cleaner(true);
+  PyErr_Cleaner py_err_cleaner(true);
 
-    auto dict = PythonModule::MainModule().ResolveName<PythonDictionary>(session_dictionary_name);
-    auto pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(python_function_name, dict);
+  auto dict = PythonModule::MainModule().ResolveName<PythonDictionary>(
+      session_dictionary_name);
+  auto pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(
+      python_function_name, dict);
 
-    if (!pfunc.IsAllocated())
-        return false;
+  if (!pfunc.IsAllocated())
+    return false;
 
-    auto result = pfunc(ToSWIGWrapper(value), dict);
+  auto result = pfunc(ToSWIGWrapper(value), dict);
 
-    output = result.Str().GetString().str();
+  output = result.Str().GetString().str();
 
-    return true;
+  return true;
 }
 
-bool
-lldb_private::LLDBSwigPythonCallModuleInit
-(
-    const char *python_module_name,
-    const char *session_dictionary_name,
-    lldb::DebuggerSP& debugger
-)
-{
-    std::string python_function_name_string = python_module_name;
-    python_function_name_string += ".__lldb_init_module";
-    const char* python_function_name = python_function_name_string.c_str();
-
-    PyErr_Cleaner py_err_cleaner(true);
-
-    auto dict = PythonModule::MainModule().ResolveName<PythonDictionary>(session_dictionary_name);
-    auto pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(python_function_name, dict);
-
-    // This method is optional and need not exist.  So if we don't find it,
-    // it's actually a success, not a failure.
-    if (!pfunc.IsAllocated())
-        return true;
+bool lldb_private::LLDBSwigPythonCallModuleInit(
+    const char *python_module_name, const char *session_dictionary_name,
+    lldb::DebuggerSP debugger) {
+  std::string python_function_name_string = python_module_name;
+  python_function_name_string += ".__lldb_init_module";
+  const char *python_function_name = python_function_name_string.c_str();
+
+  PyErr_Cleaner py_err_cleaner(true);
 
-    lldb::SBDebugger debugger_sb(debugger);
-    PythonObject debugger_arg(PyRefType::Owned, SBTypeToSWIGWrapper(debugger_sb));
-    pfunc(debugger_arg, dict);
+  auto dict = PythonModule::MainModule().ResolveName<PythonDictionary>(
+      session_dictionary_name);
+  auto pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(
+      python_function_name, dict);
 
+  // This method is optional and need not exist.  So if we don't find it,
+  // it's actually a success, not a failure.
+  if (!pfunc.IsAllocated())
     return true;
+
+  pfunc(ToSWIGWrapper(std::move(debugger)), dict);
+
+  return true;
 }
 
-lldb::ValueObjectSP
-lldb_private::LLDBSWIGPython_GetValueObjectSPFromSBValue (void* data)
-{
-    lldb::ValueObjectSP valobj_sp;
-    if (data)
-    {
-        lldb::SBValue* sb_ptr = (lldb::SBValue *)data;
-        valobj_sp = sb_ptr->GetSP();
-    }
-    return valobj_sp;
+lldb::ValueObjectSP lldb_private::LLDBSWIGPython_GetValueObjectSPFromSBValue(
+    void *data) {
+  lldb::ValueObjectSP valobj_sp;
+  if (data) {
+    lldb::SBValue *sb_ptr = (lldb::SBValue *)data;
+    valobj_sp = sb_ptr->GetSP();
+  }
+  return valobj_sp;
 }
 
 // For the LogOutputCallback functions
-static void LLDBSwigPythonCallPythonLogOutputCallback(const char *str, void *baton) {
-    if (baton != Py_None) {
-      SWIG_PYTHON_THREAD_BEGIN_BLOCK;
-      PyObject *result = PyObject_CallFunction(reinterpret_cast<PyObject*>(baton), const_cast<char*>("s"), str);
-	  Py_XDECREF(result);
-      SWIG_PYTHON_THREAD_END_BLOCK;
-    }
+static void LLDBSwigPythonCallPythonLogOutputCallback(const char *str,
+                                                      void *baton) {
+  if (baton != Py_None) {
+    SWIG_PYTHON_THREAD_BEGIN_BLOCK;
+    PyObject *result = PyObject_CallFunction(
+        reinterpret_cast<PyObject *>(baton), const_cast<char *>("s"), str);
+    Py_XDECREF(result);
+    SWIG_PYTHON_THREAD_END_BLOCK;
+  }
 }
 %}
diff --git a/lldb/include/lldb/API/SBData.h b/lldb/include/lldb/API/SBData.h
index 85c8110e181c..89a699f2f713 100644
--- a/lldb/include/lldb/API/SBData.h
+++ b/lldb/include/lldb/API/SBData.h
@@ -83,6 +83,9 @@ public:
   void SetData(lldb::SBError &error, const void *buf, size_t size,
                lldb::ByteOrder endian, uint8_t addr_size);
 
+  void SetDataWithOwnership(lldb::SBError &error, const void *buf, size_t size,
+                            lldb::ByteOrder endian, uint8_t addr_size);
+
   // see SetData() for why we don't have Append(const void* buf, size_t size)
   bool Append(const SBData &rhs);
 
diff --git a/lldb/include/lldb/API/SBStructuredData.h b/lldb/include/lldb/API/SBStructuredData.h
index 07075abbf1d0..533dcc8fc07c 100644
--- a/lldb/include/lldb/API/SBStructuredData.h
+++ b/lldb/include/lldb/API/SBStructuredData.h
@@ -22,7 +22,7 @@ public:
 
   SBStructuredData(const lldb::EventSP &event_sp);
 
-  SBStructuredData(lldb_private::StructuredDataImpl *impl);
+  SBStructuredData(const lldb_private::StructuredDataImpl &impl);
 
   ~SBStructuredData();
 
diff --git a/lldb/include/lldb/API/SBSymbolContext.h b/lldb/include/lldb/API/SBSymbolContext.h
index 16ad29ea8730..b4c5921d81a9 100644
--- a/lldb/include/lldb/API/SBSymbolContext.h
+++ b/lldb/include/lldb/API/SBSymbolContext.h
@@ -25,7 +25,7 @@ public:
 
   SBSymbolContext(const lldb::SBSymbolContext &rhs);
 
-  SBSymbolContext(const lldb_private::SymbolContext *sc_ptr);
+  SBSymbolContext(const lldb_private::SymbolContext &sc_ptr);
 
   ~SBSymbolContext();
 
@@ -72,8 +72,6 @@ protected:
 
   lldb_private::SymbolContext *get() const;
 
-  void SetSymbolContext(const lldb_private::SymbolContext *sc_ptr);
-
 private:
   std::unique_ptr<lldb_private::SymbolContext> m_opaque_up;
 };
diff --git a/lldb/include/lldb/API/SBTypeSummary.h b/lldb/include/lldb/API/SBTypeSummary.h
index 929bfb6124b2..e9963682f7ab 100644
--- a/lldb/include/lldb/API/SBTypeSummary.h
+++ b/lldb/include/lldb/API/SBTypeSummary.h
@@ -19,7 +19,7 @@ public:
 
   SBTypeSummaryOptions(const lldb::SBTypeSummaryOptions &rhs);
 
-  SBTypeSummaryOptions(const lldb_private::TypeSummaryOptions *lldb_object_ptr);
+  SBTypeSummaryOptions(const lldb_private::TypeSummaryOptions &lldb_object);
 
   ~SBTypeSummaryOptions();
 
@@ -48,8 +48,6 @@ protected:
 
   const lldb_private::TypeSummaryOptions &ref() const;
 
-  void SetOptions(const lldb_private::TypeSummaryOptions *lldb_object_ptr);
-
 private:
   std::unique_ptr<lldb_private::TypeSummaryOptions> m_opaque_up;
 };
diff --git a/lldb/include/lldb/Breakpoint/BreakpointResolverScripted.h b/lldb/include/lldb/Breakpoint/BreakpointResolverScripted.h
index 26fd6f2f04d7..cecd0f92a21a 100644
--- a/lldb/include/lldb/Breakpoint/BreakpointResolverScripted.h
+++ b/lldb/include/lldb/Breakpoint/BreakpointResolverScripted.h
@@ -9,10 +9,10 @@
 #ifndef LLDB_BREAKPOINT_BREAKPOINTRESOLVERSCRIPTED_H
 #define LLDB_BREAKPOINT_BREAKPOINTRESOLVERSCRIPTED_H
 
-#include "lldb/lldb-forward.h"
 #include "lldb/Breakpoint/BreakpointResolver.h"
 #include "lldb/Core/ModuleSpec.h"
-
+#include "lldb/Core/StructuredDataImpl.h"
+#include "lldb/lldb-forward.h"
 
 namespace lldb_private {
 
@@ -26,7 +26,7 @@ public:
   BreakpointResolverScripted(const lldb::BreakpointSP &bkpt,
                              const llvm::StringRef class_name,
                              lldb::SearchDepth depth,
-                             StructuredDataImpl *args_data);
+                             const StructuredDataImpl &args_data);
 
   ~BreakpointResolverScripted() override = default;
 
@@ -64,10 +64,7 @@ private:
   
   std::string m_class_name;
   lldb::SearchDepth m_depth;
-  StructuredDataImpl *m_args_ptr; // We own this, but the implementation
-                                  // has to manage the UP (since that is
-                                  // how it gets stored in the
-                                  // SBStructuredData).
+  StructuredDataImpl m_args;
   StructuredData::GenericSP m_implementation_sp;
 
   BreakpointResolverScripted(const BreakpointResolverScripted &) = delete;
diff --git a/lldb/include/lldb/Core/DataFileCache.h b/lldb/include/lldb/Core/DataFileCache.h
new file mode 100644
index 000000000000..3016c531f674
--- /dev/null
+++ b/lldb/include/lldb/Core/DataFileCache.h
@@ -0,0 +1,216 @@
+//===-- DataFileCache.h -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_CORE_DATAFILECACHE_H
+#define LLDB_CORE_DATAFILECACHE_H
+
+#include "lldb/Utility/FileSpec.h"
+#include "lldb/Utility/Status.h"
+#include "lldb/Utility/UUID.h"
+#include "lldb/lldb-forward.h"
+#include "llvm/Support/Caching.h"
+#include <mutex>
+
+namespace lldb_private {
+
+/// This class enables data to be cached into a directory using the llvm
+/// caching code. Data can be stored and accessed using a unique string key.
+/// The data will be stored in the directory that is specified in the
+/// DataFileCache constructor. The data will be stored in files that start with
+/// "llvmcache-<key>" where <key> is the key name specified when getting to
+/// setting cached data.
+///
+/// Sample code for how to use the cache:
+///
+///   DataFileCache cache("/tmp/lldb-test-cache");
+///   StringRef key("Key1");
+///   auto mem_buffer_up = cache.GetCachedData(key);
+///   if (mem_buffer_up) {
+///     printf("cached data:\n%s", mem_buffer_up->getBufferStart());
+///   } else {
+///     std::vector<uint8_t> data = { 'h', 'e', 'l', 'l', 'o', '\n' };
+///     cache.SetCachedData(key, data);
+///   }
+
+class DataFileCache {
+public:
+  /// Create a data file cache in the directory path that is specified.
+  ///
+  /// Data will be cached in files created in this directory when clients call
+  /// DataFileCache::SetCacheData.
+  DataFileCache(llvm::StringRef path);
+
+  /// Get cached data from the cache directory for the specified key.
+  ///
+  /// Keys must be unique for any given data. This function attempts to see if
+  /// the data is available for the specified key and will return a valid memory
+  /// buffer is data is available.
+  ///
+  /// \param key
+  ///   The unique string key that identifies data being cached.
+  ///
+  /// \return
+  ///   A valid unique pointer to a memory buffer if the data is available, or
+  ///   a unique pointer that contains NULL if the data is not available.
+  std::unique_ptr<llvm::MemoryBuffer> GetCachedData(llvm::StringRef key);
+
+  /// Set cached data for the specified key.
+  ///
+  /// Setting the cached data will save a file in the cache directory to contain
+  /// the specified data.
+  ///
+  /// \param key
+  ///   The unique string key that identifies data being cached.
+  ///
+  /// \return
+  ///   True if the data was successfully cached, false otherwise.
+  bool SetCachedData(llvm::StringRef key, llvm::ArrayRef<uint8_t> data);
+
+  /// Remove the cache file associated with the key.
+  Status RemoveCacheFile(llvm::StringRef key);
+
+private:
+  /// Return the cache file that is associated with the key.
+  FileSpec GetCacheFilePath(llvm::StringRef key);
+
+  llvm::FileCache m_cache_callback;
+  FileSpec m_cache_dir;
+  std::mutex m_mutex;
+  std::unique_ptr<llvm::MemoryBuffer> m_mem_buff_up;
+  bool m_take_ownership = false;
+};
+
+/// A signature for a given file on disk.
+///
+/// Any files that are cached in the LLDB index cached need some data that
+/// uniquely identifies a file on disk and this information should be written
+/// into each cache file so we can validate if the cache file still matches
+/// the file we are trying to load cached data for. Objects can fill out this
+/// signature and then encode and decode them to validate the signatures
+/// match. If they do not match, the cache file on disk should be removed as
+/// it is out of date.
+struct CacheSignature {
+  /// UUID of object file or module.
+  llvm::Optional<UUID> m_uuid = llvm::None;
+  /// Modification time of file on disk.
+  llvm::Optional<std::time_t> m_mod_time = llvm::None;
+  /// If this describes a .o file with a BSD archive, the BSD archive's
+  /// modification time will be in m_mod_time, and the .o file's modification
+  /// time will be in this m_obj_mod_time.
+  llvm::Optional<std::time_t> m_obj_mod_time = llvm::None;
+
+  CacheSignature() = default;
+
+  /// Create a signature from a module.
+  CacheSignature(lldb_private::Module *module);
+
+  /// Create a signature from an object file.
+  CacheSignature(lldb_private::ObjectFile *objfile);
+
+  void Clear() {
+    m_uuid = llvm::None;
+    m_mod_time = llvm::None;
+    m_obj_mod_time = llvm::None;
+  }
+
+  /// Return true if any of the signature member variables have valid values.
+  bool IsValid() const {
+    return m_uuid.hasValue() || m_mod_time.hasValue() ||
+           m_obj_mod_time.hasValue();
+  }
+
+  /// Check if two signatures are the same.
+  bool operator!=(const CacheSignature &rhs) {
+    if (m_uuid != rhs.m_uuid)
+      return true;
+    if (m_mod_time != rhs.m_mod_time)
+      return true;
+    if (m_obj_mod_time != rhs.m_obj_mod_time)
+      return true;
+    return false;
+  }
+  /// Encode this object into a data encoder object.
+  ///
+  /// This allows this object to be serialized to disk. The CacheSignature
+  /// object must have at least one member variable that has a value in order to
+  /// be serialized so that we can match this data to when the cached file is
+  /// loaded at a later time.
+  ///
+  /// \param encoder
+  ///   A data encoder object that serialized bytes will be encoded into.
+  ///
+  /// \return
+  ///   True if a signature was encoded, and false if there were no member
+  ///   variables that had value. False indicates this data should not be
+  ///   cached to disk because we were unable to encode a valid signature.
+  bool Encode(DataEncoder &encoder);
+
+  /// Decode a serialized version of this object from data.
+  ///
+  /// \param data
+  ///   The decoder object that references the serialized data.
+  ///
+  /// \param offset_ptr
+  ///   A pointer that contains the offset from which the data will be decoded
+  ///   from that gets updated as data gets decoded.
+  ///
+  /// \return
+  ///   True if the signature was successfully decoded, false otherwise.
+  bool Decode(const DataExtractor &data, lldb::offset_t *offset_ptr);
+};
+
+/// Many cache files require string tables to store data efficiently. This
+/// class helps create string tables.
+class ConstStringTable {
+public:
+  ConstStringTable() = default;
+  /// Add a string into the string table.
+  ///
+  /// Add a string to the string table will only add the same string one time
+  /// and will return the offset in the string table buffer to that string.
+  /// String tables are easy to build with ConstString objects since most LLDB
+  /// classes for symbol or debug info use them already and they provide
+  /// permanent storage for the string.
+  ///
+  /// \param s
+  ///   The string to insert into the string table.
+  ///
+  /// \return
+  ///   The byte offset from the start of the string table for the inserted
+  ///   string. Duplicate strings that get inserted will return the same
+  ///   byte offset.
+  uint32_t Add(ConstString s);
+
+  bool Encode(DataEncoder &encoder);
+
+private:
+  std::vector<ConstString> m_strings;
+  std::map<ConstString, uint32_t> m_string_to_offset;
+  /// Skip one byte to start the string table off with an empty string.
+  uint32_t m_next_offset = 1;
+};
+
+/// Many cache files require string tables to store data efficiently. This
+/// class helps give out strings from a string table that was read from a
+/// cache file.
+class StringTableReader {
+public:
+  StringTableReader() = default;
+
+  llvm::StringRef Get(uint32_t offset) const;
+
+  bool Decode(const DataExtractor &data, lldb::offset_t *offset_ptr);
+
+protected:
+  /// All of the strings in the string table are contained in m_data.
+  llvm::StringRef m_data;
+};
+
+} // namespace lldb_private
+
+#endif // LLDB_CORE_DATAFILECACHE_H
diff --git a/lldb/include/lldb/Core/Mangled.h b/lldb/include/lldb/Core/Mangled.h
index c0542157f85d..6c92591a0881 100644
--- a/lldb/include/lldb/Core/Mangled.h
+++ b/lldb/include/lldb/Core/Mangled.h
@@ -12,9 +12,8 @@
 
 #include "lldb/lldb-enumerations.h"
 #include "lldb/lldb-forward.h"
-
+#include "lldb/lldb-types.h"
 #include "lldb/Utility/ConstString.h"
-
 #include "llvm/ADT/StringRef.h"
 
 #include <cstddef>
@@ -64,6 +63,15 @@ public:
 
   explicit Mangled(llvm::StringRef name);
 
+  bool operator==(const Mangled &rhs) const {
+    return m_mangled == rhs.m_mangled &&
+           GetDemangledName() == rhs.GetDemangledName();
+  }
+
+  bool operator!=(const Mangled &rhs) const {
+    return !(*this == rhs);
+  }
+
   /// Convert to pointer operator.
   ///
   /// This allows code to check a Mangled object to see if it contains a valid
@@ -270,6 +278,35 @@ public:
   ///     for s, otherwise the enumerator for the mangling scheme detected.
   static Mangled::ManglingScheme GetManglingScheme(llvm::StringRef const name);
 
+  /// Decode a serialized version of this object from data.
+  ///
+  /// \param data
+  ///   The decoder object that references the serialized data.
+  ///
+  /// \param offset_ptr
+  ///   A pointer that contains the offset from which the data will be decoded
+  ///   from that gets updated as data gets decoded.
+  ///
+  /// \param strtab
+  ///   All strings in cache files are put into string tables for efficiency
+  ///   and cache file size reduction. Strings are stored as uint32_t string
+  ///   table offsets in the cache data.
+  bool Decode(const DataExtractor &data, lldb::offset_t *offset_ptr,
+              const StringTableReader &strtab);
+
+  /// Encode this object into a data encoder object.
+  ///
+  /// This allows this object to be serialized to disk.
+  ///
+  /// \param encoder
+  ///   A data encoder object that serialized bytes will be encoded into.
+  ///
+  /// \param strtab
+  ///   All strings in cache files are put into string tables for efficiency
+  ///   and cache file size reduction. Strings are stored as uint32_t string
+  ///   table offsets in the cache data.
+  void Encode(DataEncoder &encoder, ConstStringTable &strtab) const;
+
 private:
   /// Mangled member variables.
   ConstString m_mangled;           ///< The mangled version of the name
diff --git a/lldb/include/lldb/Core/Module.h b/lldb/include/lldb/Core/Module.h
index b80f4fd9b85a..6b1e4f8f3704 100644
--- a/lldb/include/lldb/Core/Module.h
+++ b/lldb/include/lldb/Core/Module.h
@@ -21,8 +21,8 @@
 #include "lldb/Utility/ConstString.h"
 #include "lldb/Utility/FileSpec.h"
 #include "lldb/Utility/Status.h"
-#include "lldb/Utility/XcodeSDK.h"
 #include "lldb/Utility/UUID.h"
+#include "lldb/Utility/XcodeSDK.h"
 #include "lldb/lldb-defines.h"
 #include "lldb/lldb-enumerations.h"
 #include "lldb/lldb-forward.h"
@@ -128,7 +128,7 @@ public:
   Module(const ModuleSpec &module_spec);
 
   template <typename ObjFilePlugin, typename... Args>
-  static lldb::ModuleSP CreateModuleFromObjectFile(Args &&... args) {
+  static lldb::ModuleSP CreateModuleFromObjectFile(Args &&...args) {
     // Must create a module and place it into a shared pointer before we can
     // create an object file since it has a std::weak_ptr back to the module,
     // so we need to control the creation carefully in this static function
@@ -255,8 +255,7 @@ public:
   ///     Returns a valid symbol pointer if a symbol was found,
   ///     nullptr otherwise.
   const Symbol *FindFirstSymbolWithNameAndType(
-      ConstString name,
-      lldb::SymbolType symbol_type = lldb::eSymbolTypeAny);
+      ConstString name, lldb::SymbolType symbol_type = lldb::eSymbolTypeAny);
 
   void FindSymbolsWithNameAndType(ConstString name,
                                   lldb::SymbolType symbol_type,
@@ -439,12 +438,13 @@ public:
   ///
   /// \param searched_symbol_files
   ///     Prevents one file from being visited multiple times.
-  void FindTypes(llvm::ArrayRef<CompilerContext> pattern, LanguageSet languages,
-                 llvm::DenseSet<lldb_private::SymbolFile *> &searched_symbol_files,
-                 TypeMap &types);
+  void
+  FindTypes(llvm::ArrayRef<CompilerContext> pattern, LanguageSet languages,
+            llvm::DenseSet<lldb_private::SymbolFile *> &searched_symbol_files,
+            TypeMap &types);
 
-  lldb::TypeSP FindFirstType(const SymbolContext &sc,
-                             ConstString type_name, bool exact_match);
+  lldb::TypeSP FindFirstType(const SymbolContext &sc, ConstString type_name,
+                             bool exact_match);
 
   /// Find types by name that are in a namespace. This function is used by the
   /// expression parser when searches need to happen in an exact namespace
@@ -788,8 +788,7 @@ public:
       const FileSpec &file_spec, uint32_t line, bool check_inlines,
       lldb::SymbolContextItem resolve_scope, SymbolContextList &sc_list);
 
-  void SetFileSpecAndObjectName(const FileSpec &file,
-                                ConstString object_name);
+  void SetFileSpecAndObjectName(const FileSpec &file, ConstString object_name);
 
   bool GetIsDynamicLinkEditor();
 
@@ -876,7 +875,7 @@ public:
   /// The value is returned as a reference to allow it to be updated by the
   /// ElapsedTime RAII object.
   StatsDuration &GetSymtabParseTime() { return m_symtab_parse_time; }
-  
+
   /// Accessor for the symbol table index time metric.
   ///
   /// The value is returned as a reference to allow it to be updated by the
@@ -946,30 +945,67 @@ public:
     bool m_match_name_after_lookup = false;
   };
 
+  /// Get a unique hash for this module.
+  ///
+  /// The hash should be enough to identify the file on disk and the
+  /// architecture of the file. If the module represents an object inside of a
+  /// file, then the hash should include the object name and object offset to
+  /// ensure a unique hash. Some examples:
+  /// - just a regular object file (mach-o, elf, coff, etc) should create a hash
+  /// - a universal mach-o file that contains to multiple architectures,
+  ///   each architecture slice should have a unique hash even though they come
+  ///   from the same file
+  /// - a .o file inside of a BSD archive. Each .o file will have an object name
+  ///   and object offset that should produce a unique hash. The object offset
+  ///   is needed as BSD archive files can contain multiple .o files that have
+  ///   the same name.
+  uint32_t Hash();
+
+  /// Get a unique cache key for the current module.
+  ///
+  /// The cache key must be unique for a file on disk and not change if the file
+  /// is updated. This allows cache data to use this key as a prefix and as
+  /// files are modified in disk, we will overwrite the cache files. If one file
+  /// can contain multiple files, like a universal mach-o file or like a BSD
+  /// archive, the cache key must contain enough information to differentiate
+  /// these different files.
+  std::string GetCacheKey();
+
+  /// Get the global index file cache.
+  ///
+  /// LLDB can cache data for a module between runs. This cache directory can be
+  /// used to stored data that previously was manually created each time you debug.
+  /// Examples include debug information indexes, symbol tables, symbol table
+  /// indexes, and more.
+  ///
+  /// \returns
+  ///   If caching is enabled in the lldb settings, return a pointer to the data
+  ///   file cache. If caching is not enabled, return NULL.
+  static DataFileCache *GetIndexCache();
 protected:
   // Member Variables
   mutable std::recursive_mutex m_mutex; ///< A mutex to keep this object happy
-                                        ///in multi-threaded environments.
+                                        /// in multi-threaded environments.
 
   /// The modification time for this module when it was created.
   llvm::sys::TimePoint<> m_mod_time;
 
-  ArchSpec m_arch;      ///< The architecture for this module.
+  ArchSpec m_arch; ///< The architecture for this module.
   UUID m_uuid; ///< Each module is assumed to have a unique identifier to help
-               ///match it up to debug symbols.
+               /// match it up to debug symbols.
   FileSpec m_file; ///< The file representation on disk for this module (if
-                   ///there is one).
+                   /// there is one).
   FileSpec m_platform_file; ///< The path to the module on the platform on which
-                            ///it is being debugged
+                            /// it is being debugged
   FileSpec m_remote_install_file; ///< If set when debugging on remote
-                                  ///platforms, this module will be installed at
-                                  ///this location
+                                  /// platforms, this module will be installed
+                                  /// at this location
   FileSpec m_symfile_spec;   ///< If this path is valid, then this is the file
-                             ///that _will_ be used as the symbol file for this
-                             ///module
+                             /// that _will_ be used as the symbol file for this
+                             /// module
   ConstString m_object_name; ///< The name an object within this module that is
-                             ///selected, or empty of the module is represented
-                             ///by \a m_file.
+                             /// selected, or empty of the module is represented
+                             /// by \a m_file.
   uint64_t m_object_offset = 0;
   llvm::sys::TimePoint<> m_object_mod_time;
 
@@ -979,8 +1015,8 @@ protected:
   lldb::DataBufferSP m_data_sp;
 
   lldb::ObjectFileSP m_objfile_sp; ///< A shared pointer to the object file
-                                   ///parser for this module as it may or may
-                                   ///not be shared with the SymbolFile
+                                   /// parser for this module as it may or may
+                                   /// not be shared with the SymbolFile
   llvm::Optional<UnwindTable> m_unwind_table; ///< Table of FuncUnwinders
                                               /// objects created for this
                                               /// Module's functions
@@ -988,11 +1024,11 @@ protected:
       m_symfile_up; ///< A pointer to the symbol vendor for this module.
   std::vector<lldb::SymbolVendorUP>
       m_old_symfiles; ///< If anyone calls Module::SetSymbolFileFileSpec() and
-                      ///changes the symbol file,
+                      /// changes the symbol file,
   ///< we need to keep all old symbol files around in case anyone has type
-  ///references to them
-  TypeSystemMap m_type_system_map;   ///< A map of any type systems associated
-                                     ///with this module
+  /// references to them
+  TypeSystemMap m_type_system_map; ///< A map of any type systems associated
+                                   /// with this module
   /// Module specific source remappings for when you have debug info for a
   /// module that doesn't match where the sources currently are.
   PathMappingList m_source_mappings =
diff --git a/lldb/include/lldb/Core/ModuleList.h b/lldb/include/lldb/Core/ModuleList.h
index 6ca5813d9662..01b9991129eb 100644
--- a/lldb/include/lldb/Core/ModuleList.h
+++ b/lldb/include/lldb/Core/ModuleList.h
@@ -60,6 +60,13 @@ public:
   bool SetClangModulesCachePath(const FileSpec &path);
   bool GetEnableExternalLookup() const;
   bool SetEnableExternalLookup(bool new_value);
+  bool GetEnableLLDBIndexCache() const;
+  bool SetEnableLLDBIndexCache(bool new_value);
+  uint64_t GetLLDBIndexCacheMaxByteSize();
+  uint64_t GetLLDBIndexCacheMaxPercent();
+  uint64_t GetLLDBIndexCacheExpirationDays();
+  FileSpec GetLLDBIndexCachePath() const;
+  bool SetLLDBIndexCachePath(const FileSpec &path);
 
   PathMappingList GetSymlinkMappings() const;
 };
diff --git a/lldb/include/lldb/Core/PluginManager.h b/lldb/include/lldb/Core/PluginManager.h
index 7dc99bf3e755..1ab9d26d3af4 100644
--- a/lldb/include/lldb/Core/PluginManager.h
+++ b/lldb/include/lldb/Core/PluginManager.h
@@ -449,6 +449,8 @@ public:
 
   static REPLCreateInstance GetREPLCreateCallbackAtIndex(uint32_t idx);
 
+  static LanguageSet GetREPLSupportedLanguagesAtIndex(uint32_t idx);
+
   static LanguageSet GetREPLAllTypeSystemSupportedLanguages();
 
   // Some plug-ins might register a DebuggerInitializeCallback callback when
diff --git a/lldb/include/lldb/Core/StructuredDataImpl.h b/lldb/include/lldb/Core/StructuredDataImpl.h
index d6f64451e5c2..8930ebff8166 100644
--- a/lldb/include/lldb/Core/StructuredDataImpl.h
+++ b/lldb/include/lldb/Core/StructuredDataImpl.h
@@ -29,6 +29,9 @@ public:
 
   StructuredDataImpl(const StructuredDataImpl &rhs) = default;
 
+  StructuredDataImpl(StructuredData::ObjectSP obj)
+      : m_data_sp(std::move(obj)) {}
+
   StructuredDataImpl(const lldb::EventSP &event_sp)
       : m_plugin_wp(
             EventDataStructuredData::GetPluginFromEvent(event_sp.get())),
diff --git a/lldb/include/lldb/Core/ValueObject.h b/lldb/include/lldb/Core/ValueObject.h
index 5f1cbc65b320..192149f05436 100644
--- a/lldb/include/lldb/Core/ValueObject.h
+++ b/lldb/include/lldb/Core/ValueObject.h
@@ -1000,7 +1000,7 @@ protected:
   void SetPreferredDisplayLanguageIfNeeded(lldb::LanguageType);
 
 protected:
-  virtual void DoUpdateChildrenAddressType(ValueObject &valobj) { return; };
+  virtual void DoUpdateChildrenAddressType(ValueObject &valobj){};
 
 private:
   virtual CompilerType MaybeCalculateCompleteType();
diff --git a/lldb/include/lldb/Expression/UserExpression.h b/lldb/include/lldb/Expression/UserExpression.h
index 8236c417f73a..3874a60e06f0 100644
--- a/lldb/include/lldb/Expression/UserExpression.h
+++ b/lldb/include/lldb/Expression/UserExpression.h
@@ -266,10 +266,8 @@ public:
       0x1001; ///< ValueObject::GetError() returns this if there is no result
               /// from the expression.
 
-  const char *GetFixedText() {
-    if (m_fixed_text.empty())
-      return nullptr;
-    return m_fixed_text.c_str();
+  llvm::StringRef GetFixedText() {
+    return m_fixed_text;
   }
 
 protected:
diff --git a/lldb/include/lldb/Host/Config.h.cmake b/lldb/include/lldb/Host/Config.h.cmake
index 777a6d1be541..ffe919aa9956 100644
--- a/lldb/include/lldb/Host/Config.h.cmake
+++ b/lldb/include/lldb/Host/Config.h.cmake
@@ -45,6 +45,8 @@
 
 #cmakedefine01 LLDB_ENABLE_PYTHON
 
+#cmakedefine01 LLDB_ENABLE_FBSDVMCORE
+
 #cmakedefine01 LLDB_EMBED_PYTHON_HOME
 
 #cmakedefine LLDB_PYTHON_HOME R"(${LLDB_PYTHON_HOME})"
diff --git a/lldb/include/lldb/Host/FileSystem.h b/lldb/include/lldb/Host/FileSystem.h
index 93563d4d26e3..54f7ac5ddb58 100644
--- a/lldb/include/lldb/Host/FileSystem.h
+++ b/lldb/include/lldb/Host/FileSystem.h
@@ -142,6 +142,14 @@ public:
   void Resolve(FileSpec &file_spec);
   /// \}
 
+  /// Remove a single file.
+  ///
+  /// The path must specify a file and not a directory.
+  /// \{
+  Status RemoveFile(const FileSpec &file_spec);
+  Status RemoveFile(const llvm::Twine &path);
+  /// \}
+
   //// Create memory buffer from path.
   /// \{
   std::shared_ptr<DataBufferLLVM> CreateDataBuffer(const llvm::Twine &path,
diff --git a/lldb/include/lldb/Interpreter/CommandReturnObject.h b/lldb/include/lldb/Interpreter/CommandReturnObject.h
index 0c995b73c463..72518a902144 100644
--- a/lldb/include/lldb/Interpreter/CommandReturnObject.h
+++ b/lldb/include/lldb/Interpreter/CommandReturnObject.h
@@ -132,15 +132,15 @@ public:
 
   void SetError(const Status &error, const char *fallback_error_cstr = nullptr);
 
-  lldb::ReturnStatus GetStatus();
+  lldb::ReturnStatus GetStatus() const;
 
   void SetStatus(lldb::ReturnStatus status);
 
-  bool Succeeded();
+  bool Succeeded() const;
 
-  bool HasResult();
+  bool HasResult() const;
 
-  bool GetDidChangeProcessState();
+  bool GetDidChangeProcessState() const;
 
   void SetDidChangeProcessState(bool b);
 
diff --git a/lldb/include/lldb/Interpreter/ScriptInterpreter.h b/lldb/include/lldb/Interpreter/ScriptInterpreter.h
index 2b96021fffc9..83f784bde712 100644
--- a/lldb/include/lldb/Interpreter/ScriptInterpreter.h
+++ b/lldb/include/lldb/Interpreter/ScriptInterpreter.h
@@ -274,7 +274,7 @@ public:
 
   virtual StructuredData::ObjectSP
   CreateScriptedThreadPlan(const char *class_name,
-                           StructuredDataImpl *args_data,
+                           const StructuredDataImpl &args_data,
                            std::string &error_str,
                            lldb::ThreadPlanSP thread_plan_sp) {
     return StructuredData::ObjectSP();
@@ -310,7 +310,7 @@ public:
 
   virtual StructuredData::GenericSP
   CreateScriptedBreakpointResolver(const char *class_name,
-                                   StructuredDataImpl *args_data,
+                                   const StructuredDataImpl &args_data,
                                    lldb::BreakpointSP &bkpt_sp) {
     return StructuredData::GenericSP();
   }
@@ -330,7 +330,7 @@ public:
 
   virtual StructuredData::GenericSP
   CreateScriptedStopHook(lldb::TargetSP target_sp, const char *class_name,
-                         StructuredDataImpl *args_data, Status &error) {
+                         const StructuredDataImpl &args_data, Status &error) {
     error.SetErrorString("Creating scripted stop-hooks with the current "
                          "script interpreter is not supported.");
     return StructuredData::GenericSP();
diff --git a/lldb/include/lldb/Symbol/ObjectFile.h b/lldb/include/lldb/Symbol/ObjectFile.h
index 0a8b38b2c642..8bf047cd4514 100644
--- a/lldb/include/lldb/Symbol/ObjectFile.h
+++ b/lldb/include/lldb/Symbol/ObjectFile.h
@@ -19,6 +19,7 @@
 #include "lldb/Utility/FileSpec.h"
 #include "lldb/Utility/UUID.h"
 #include "lldb/lldb-private.h"
+#include "llvm/ADT/Optional.h"
 #include "llvm/Support/Threading.h"
 #include "llvm/Support/VersionTuple.h"
 
@@ -526,11 +527,15 @@ public:
   /// binary is exactly which removes ambiguity when there are multiple
   /// binaries present in the captured memory pages.
   ///
-  /// \param[out] address
-  ///   If the address of the binary is specified, this will be set.
-  ///   This is an address is the virtual address space of the core file
-  ///   memory segments; it is not an offset into the object file.
-  ///   If no address is available, will be set to LLDB_INVALID_ADDRESS.
+  /// \param[out] value
+  ///   The address or offset (slide) where the binary is loaded in memory.
+  ///   LLDB_INVALID_ADDRESS for unspecified.  If an offset is given,
+  ///   this offset should be added to the binary's file address to get
+  ///   the load address.
+  ///
+  /// \param[out] value_is_offset
+  ///   Specifies if \b value is a load address, or an offset to calculate
+  ///   the load address.
   ///
   /// \param[out] uuid
   ///   If the uuid of the binary is specified, this will be set.
@@ -542,9 +547,11 @@ public:
   ///
   /// \return
   ///   Returns true if either address or uuid has been set.
-  virtual bool GetCorefileMainBinaryInfo(lldb::addr_t &address, UUID &uuid,
+  virtual bool GetCorefileMainBinaryInfo(lldb::addr_t &value,
+                                         bool &value_is_offset, UUID &uuid,
                                          ObjectFile::BinaryType &type) {
-    address = LLDB_INVALID_ADDRESS;
+    value = LLDB_INVALID_ADDRESS;
+    value_is_offset = false;
     uuid.Clear();
     return false;
   }
@@ -707,6 +714,15 @@ public:
     return false;
   }
 
+  /// Get a hash that can be used for caching object file releated information.
+  ///
+  /// Data for object files can be cached between runs of debug sessions and
+  /// a module can end up using a main file and a symbol file, both of which
+  /// can be object files. So we need a unique hash that identifies an object
+  /// file when storing cached data.
+  uint32_t GetCacheHash();
+
+
 protected:
   // Member variables.
   FileSpec m_file;
@@ -729,6 +745,7 @@ protected:
   /// need to use a std::unique_ptr to a llvm::once_flag so if we clear the
   /// symbol table, we can have a new once flag to use when it is created again.
   std::unique_ptr<llvm::once_flag> m_symtab_once_up;
+  llvm::Optional<uint32_t> m_cache_hash;
 
   /// Sets the architecture for a module.  At present the architecture can
   /// only be set if it is invalid.  It is not allowed to switch from one
diff --git a/lldb/include/lldb/Symbol/Symbol.h b/lldb/include/lldb/Symbol/Symbol.h
index cb2385468702..23a68090ff4d 100644
--- a/lldb/include/lldb/Symbol/Symbol.h
+++ b/lldb/include/lldb/Symbol/Symbol.h
@@ -235,6 +235,46 @@ public:
     return "___lldb_unnamed_symbol";
   }
 
+  /// Decode a serialized version of this object from data.
+  ///
+  /// \param data
+  ///   The decoder object that references the serialized data.
+  ///
+  /// \param offset_ptr
+  ///   A pointer that contains the offset from which the data will be decoded
+  ///   from that gets updated as data gets decoded.
+  ///
+  /// \param section_list
+  ///   A section list that allows lldb_private::Address objects to be filled
+  ///   in. The address information for symbols are serilized as file addresses
+  ///   and must be converted into Address objects with the right section and
+  ///   offset.
+  ///
+  /// \param strtab
+  ///   All strings in cache files are put into string tables for efficiency
+  ///   and cache file size reduction. Strings are stored as uint32_t string
+  ///   table offsets in the cache data.
+  ///
+  /// \return
+  ///   True if the symbol is successfully decoded, false otherwise.
+  bool Decode(const DataExtractor &data, lldb::offset_t *offset_ptr,
+              const SectionList *section_list, const StringTableReader &strtab);
+
+  /// Encode this object into a data encoder object.
+  ///
+  /// This allows this object to be serialized to disk.
+  ///
+  /// \param encoder
+  ///   A data encoder object that serialized bytes will be encoded into.
+  ///
+  /// \param strtab
+  ///   All strings in cache files are put into string tables for efficiency
+  ///   and cache file size reduction. Strings are stored as uint32_t string
+  ///   table offsets in the cache data.
+  void Encode(DataEncoder &encoder, ConstStringTable &strtab) const;
+
+  bool operator==(const Symbol &rhs) const;
+
 protected:
   // This is the internal guts of ResolveReExportedSymbol, it assumes
   // reexport_name is not null, and that module_spec is valid.  We track the
diff --git a/lldb/include/lldb/Symbol/SymbolFile.h b/lldb/include/lldb/Symbol/SymbolFile.h
index 9ab63cac56dd..7c0365483c12 100644
--- a/lldb/include/lldb/Symbol/SymbolFile.h
+++ b/lldb/include/lldb/Symbol/SymbolFile.h
@@ -181,7 +181,6 @@ public:
   virtual size_t ParseVariablesForContext(const SymbolContext &sc) = 0;
   virtual Type *ResolveTypeUID(lldb::user_id_t type_uid) = 0;
 
-
   /// The characteristics of an array type.
   struct ArrayInfo {
     int64_t first_index = 0;
@@ -318,19 +317,14 @@ public:
   ///
   /// \returns 0.0 if no information has been parsed or if there is
   /// no computational cost to parsing the debug information.
-  virtual StatsDuration GetDebugInfoParseTime() {
-    return StatsDuration(0.0);
-  }
+  virtual StatsDuration GetDebugInfoParseTime() { return StatsDuration(0.0); }
 
   /// Return the time it took to index the debug information in the object
   /// file.
   ///
   /// \returns 0.0 if the file doesn't need to be indexed or if it
   /// hasn't been indexed yet, or a valid duration if it has.
-  virtual StatsDuration GetDebugInfoIndexTime() {
-    return StatsDuration(0.0);
-  }
-
+  virtual StatsDuration GetDebugInfoIndexTime() { return StatsDuration(0.0); }
 
 protected:
   void AssertModuleLock();
diff --git a/lldb/include/lldb/Symbol/Symtab.h b/lldb/include/lldb/Symbol/Symtab.h
index e5d21c1bf4b3..fe0a82306c4f 100644
--- a/lldb/include/lldb/Symbol/Symtab.h
+++ b/lldb/include/lldb/Symbol/Symtab.h
@@ -131,7 +131,86 @@ public:
                               bool add_demangled, bool add_mangled,
                               NameToIndexMap &name_to_index_map) const;
 
-  ObjectFile *GetObjectFile() { return m_objfile; }
+  ObjectFile *GetObjectFile() const { return m_objfile; }
+
+  /// Decode a serialized version of this object from data.
+  ///
+  /// \param data
+  ///   The decoder object that references the serialized data.
+  ///
+  /// \param offset_ptr
+  ///   A pointer that contains the offset from which the data will be decoded
+  ///   from that gets updated as data gets decoded.
+  ///
+  /// \param[out] uuid_mismatch
+  ///   Set to true if a cache file exists but the UUID didn't match, false
+  ///   otherwise.
+  ///
+  /// \return
+  ///   True if the symbol table is successfully decoded and can be used,
+  ///   false otherwise.
+  bool Decode(const DataExtractor &data, lldb::offset_t *offset_ptr,
+              bool &uuid_mismatch);
+
+  /// Encode this object into a data encoder object.
+  ///
+  /// This allows this object to be serialized to disk. The object file must
+  /// have a valid Signature in order to be serialized as it is used to make
+  /// sure the cached information matches when cached data is loaded at a later
+  /// time. If the object file doesn't have a valid signature false will be
+  /// returned and it will indicate we should not cache this data.
+  ///
+  /// \param encoder
+  ///   A data encoder object that serialized bytes will be encoded into.
+  ///
+  /// \return
+  ///   True if the symbol table's object file can generate a valid signature
+  ///   and all data for the symbol table was encoded, false otherwise.
+  bool Encode(DataEncoder &encoder) const;
+
+  /// Get the cache key string for this symbol table.
+  ///
+  /// The cache key must start with the module's cache key and is followed
+  /// by information that indicates this key is for caching the symbol table
+  /// contents and should also include the has of the object file. A module can
+  /// be represented by an ObjectFile object for the main executable, but can
+  /// also have a symbol file that is from the same or a different object file.
+  /// This means we might have two symbol tables cached in the index cache, one
+  /// for the main executable and one for the symbol file.
+  ///
+  /// \return
+  ///   The unique cache key used to save and retrieve data from the index cache.
+  std::string GetCacheKey();
+
+  /// Save the symbol table data out into a cache.
+  ///
+  /// The symbol table will only be saved to a cache file if caching is enabled.
+  ///
+  /// We cache the contents of the symbol table since symbol tables in LLDB take
+  /// some time to initialize. This is due to the many sources for data that are
+  /// used to create a symbol table:
+  /// - standard symbol table
+  /// - dynamic symbol table (ELF)
+  /// - compressed debug info sections
+  /// - unwind information
+  /// - function pointers found in runtimes for global constructor/destructors
+  /// - other sources.
+  /// All of the above sources are combined and one symbol table results after
+  /// all sources have been considered.
+  void SaveToCache();
+
+  /// Load the symbol table from the index cache.
+  ///
+  /// Quickly load the finalized symbol table from the index cache. This saves
+  /// time when the debugger starts up. The index cache file for the symbol
+  /// table has the modification time set to the same time as the main module.
+  /// If the cache file exists and the modification times match, we will load
+  /// the symbol table from the serlized cache file.
+  ///
+  /// \return
+  ///   True if the symbol table was successfully loaded from the index cache,
+  ///   false if the symbol table wasn't cached or was out of date.
+  bool LoadFromCache();
 
 protected:
   typedef std::vector<Symbol> collection;
diff --git a/lldb/include/lldb/Symbol/Type.h b/lldb/include/lldb/Symbol/Type.h
index 9e671a565dd1..7754e0299096 100644
--- a/lldb/include/lldb/Symbol/Type.h
+++ b/lldb/include/lldb/Symbol/Type.h
@@ -66,24 +66,30 @@ protected:
 class Type : public std::enable_shared_from_this<Type>, public UserID {
 public:
   enum EncodingDataType {
+    /// Invalid encoding.
     eEncodingInvalid,
-    eEncodingIsUID,      ///< This type is the type whose UID is m_encoding_uid
-    eEncodingIsConstUID, ///< This type is the type whose UID is m_encoding_uid
-                         /// with the const qualifier added
-    eEncodingIsRestrictUID, ///< This type is the type whose UID is
-                            /// m_encoding_uid with the restrict qualifier added
-    eEncodingIsVolatileUID, ///< This type is the type whose UID is
-                            /// m_encoding_uid with the volatile qualifier added
-    eEncodingIsTypedefUID,  ///< This type is pointer to a type whose UID is
-                            /// m_encoding_uid
-    eEncodingIsPointerUID,  ///< This type is pointer to a type whose UID is
-                            /// m_encoding_uid
-    eEncodingIsLValueReferenceUID, ///< This type is L value reference to a type
-                                   /// whose UID is m_encoding_uid
-    eEncodingIsRValueReferenceUID, ///< This type is R value reference to a type
-                                   /// whose UID is m_encoding_uid,
-    eEncodingIsAtomicUID,          ///< This type is the type whose UID is
-                                   /// m_encoding_uid as an atomic type.
+    /// This type is the type whose UID is m_encoding_uid.
+    eEncodingIsUID,
+    /// This type is the type whose UID is m_encoding_uid with the const
+    /// qualifier added.
+    eEncodingIsConstUID,
+    /// This type is the type whose UID is m_encoding_uid with the restrict
+    /// qualifier added.
+    eEncodingIsRestrictUID,
+    /// This type is the type whose UID is m_encoding_uid with the volatile
+    /// qualifier added.
+    eEncodingIsVolatileUID,
+    /// This type is alias to a type whose UID is m_encoding_uid.
+    eEncodingIsTypedefUID,
+    /// This type is pointer to a type whose UID is m_encoding_uid.
+    eEncodingIsPointerUID,
+    /// This type is L value reference to a type whose UID is m_encoding_uid.
+    eEncodingIsLValueReferenceUID,
+    /// This type is R value reference to a type whose UID is m_encoding_uid.
+    eEncodingIsRValueReferenceUID,
+    /// This type is the type whose UID is m_encoding_uid as an atomic type.
+    eEncodingIsAtomicUID,
+    /// This type is the synthetic type whose UID is m_encoding_uid.
     eEncodingIsSyntheticUID
   };
 
@@ -197,7 +203,7 @@ public:
 
   // From a fully qualified typename, split the type into the type basename and
   // the remaining type scope (namespaces/classes).
-  static bool GetTypeScopeAndBasename(const llvm::StringRef& name,
+  static bool GetTypeScopeAndBasename(llvm::StringRef name,
                                       llvm::StringRef &scope,
                                       llvm::StringRef &basename,
                                       lldb::TypeClass &type_class);
@@ -473,8 +479,8 @@ class TypeEnumMemberImpl {
 public:
   TypeEnumMemberImpl() : m_integer_type_sp(), m_name("<invalid>"), m_value() {}
 
-  TypeEnumMemberImpl(const lldb::TypeImplSP &integer_type_sp,
-                     ConstString name, const llvm::APSInt &value);
+  TypeEnumMemberImpl(const lldb::TypeImplSP &integer_type_sp, ConstString name,
+                     const llvm::APSInt &value);
 
   TypeEnumMemberImpl(const TypeEnumMemberImpl &rhs) = default;
 
diff --git a/lldb/include/lldb/Target/LanguageRuntime.h b/lldb/include/lldb/Target/LanguageRuntime.h
index 2f95c2643318..ba96d080f908 100644
--- a/lldb/include/lldb/Target/LanguageRuntime.h
+++ b/lldb/include/lldb/Target/LanguageRuntime.h
@@ -141,7 +141,7 @@ public:
     return false;
   }
 
-  virtual void SymbolsDidLoad(const ModuleList &module_list) { return; }
+  virtual void SymbolsDidLoad(const ModuleList &module_list) {}
 
   virtual lldb::ThreadPlanSP GetStepThroughTrampolinePlan(Thread &thread,
                                                           bool stop_others) = 0;
diff --git a/lldb/include/lldb/Target/Target.h b/lldb/include/lldb/Target/Target.h
index 7e8e1373a506..b85839c15b2f 100644
--- a/lldb/include/lldb/Target/Target.h
+++ b/lldb/include/lldb/Target/Target.h
@@ -21,6 +21,7 @@
 #include "lldb/Core/Architecture.h"
 #include "lldb/Core/Disassembler.h"
 #include "lldb/Core/ModuleList.h"
+#include "lldb/Core/StructuredDataImpl.h"
 #include "lldb/Core/UserSettingsController.h"
 #include "lldb/Expression/Expression.h"
 #include "lldb/Host/ProcessLaunchInfo.h"
@@ -1309,8 +1310,7 @@ public:
     std::string m_class_name;
     /// This holds the dictionary of keys & values that can be used to
     /// parametrize any given callback's behavior.
-    StructuredDataImpl *m_extra_args; // We own this structured data,
-                                      // but the SD itself manages the UP.
+    StructuredDataImpl m_extra_args;
     /// This holds the python callback object.
     StructuredData::GenericSP m_implementation_sp;
 
diff --git a/lldb/include/lldb/Target/ThreadPlanPython.h b/lldb/include/lldb/Target/ThreadPlanPython.h
index 7b37b2b9ce5a..f148f88d4c46 100644
--- a/lldb/include/lldb/Target/ThreadPlanPython.h
+++ b/lldb/include/lldb/Target/ThreadPlanPython.h
@@ -12,8 +12,7 @@
 
 #include <string>
 
-#include "lldb/lldb-forward.h"
-
+#include "lldb/Core/StructuredDataImpl.h"
 #include "lldb/Target/Process.h"
 #include "lldb/Target/StopInfo.h"
 #include "lldb/Target/Target.h"
@@ -22,6 +21,7 @@
 #include "lldb/Target/ThreadPlanTracer.h"
 #include "lldb/Utility/StructuredData.h"
 #include "lldb/Utility/UserID.h"
+#include "lldb/lldb-forward.h"
 #include "lldb/lldb-private.h"
 
 namespace lldb_private {
@@ -31,9 +31,8 @@ namespace lldb_private {
 
 class ThreadPlanPython : public ThreadPlan {
 public:
-  ThreadPlanPython(Thread &thread, const char *class_name, 
-                   StructuredDataImpl *args_data);
-  ~ThreadPlanPython() override;
+  ThreadPlanPython(Thread &thread, const char *class_name,
+                   const StructuredDataImpl &args_data);
 
   void GetDescription(Stream *s, lldb::DescriptionLevel level) override;
 
@@ -62,10 +61,7 @@ protected:
 
 private:
   std::string m_class_name;
-  StructuredDataImpl *m_args_data; // We own this, but the implementation
-                                   // has to manage the UP (since that is
-                                   // how it gets stored in the
-                                   // SBStructuredData).
+  StructuredDataImpl m_args_data;
   std::string m_error_str;
   StructuredData::ObjectSP m_implementation_sp;
   bool m_did_push;
diff --git a/lldb/include/lldb/Utility/DataEncoder.h b/lldb/include/lldb/Utility/DataEncoder.h
index b944c09d5c47..7e1dec6992b2 100644
--- a/lldb/include/lldb/Utility/DataEncoder.h
+++ b/lldb/include/lldb/Utility/DataEncoder.h
@@ -16,6 +16,9 @@
 #include "lldb/lldb-forward.h"
 #include "lldb/lldb-types.h"
 
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/StringRef.h"
+
 #include <cstddef>
 #include <cstdint>
 
@@ -26,21 +29,34 @@ namespace lldb_private {
 /// An binary data encoding class.
 ///
 /// DataEncoder is a class that can encode binary data (swapping if needed) to
-/// a data buffer. The data buffer can be caller owned, or can be shared data
-/// that can be shared between multiple DataEncoder or DataEncoder instances.
+/// a data buffer. The DataEncoder can be constructed with data that will be
+/// copied into the internally owned buffer. This allows data to be modified
+/// in the internal buffer. The DataEncoder object can also be constructed with
+/// just a byte order and address size and data can be appended to the
+/// internally owned buffer.
+///
+/// Clients can get a shared pointer to the data buffer when done modifying or
+/// creating the data to keep the data around after the lifetime of a
+/// DataEncoder object. \see GetDataBuffer
 ///
-/// \see DataBuffer
+/// Client can get a reference to the object owned data as an array by calling
+/// the GetData method. \see GetData
 class DataEncoder {
 public:
   /// Default constructor.
   ///
-  /// Initialize all members to a default empty state.
+  /// Initialize all members to a default empty state and create a empty memory
+  /// buffer that can be appended to. The ByteOrder and address size will be set
+  /// to match the current host system.
   DataEncoder();
 
-  /// Construct with a buffer that is owned by the caller.
+  /// Construct an encoder that copies the specified data into the object owned
+  /// data buffer.
   ///
-  /// This constructor allows us to use data that is owned by the caller. The
-  /// data must stay around as long as this object is valid.
+  /// This constructor is designed to be used when you have a data buffer and
+  /// want to modify values within the buffer. A copy of the data will be made
+  /// in the internally owned buffer and that data can be fixed up and appended
+  /// to.
   ///
   /// \param[in] data
   ///     A pointer to caller owned data.
@@ -49,54 +65,37 @@ public:
   ///     The length in bytes of \a data.
   ///
   /// \param[in] byte_order
-  ///     A byte order of the data that we are extracting from.
+  ///     A byte order for the data that will be encoded.
   ///
   /// \param[in] addr_size
-  ///     A new address byte size value.
-  DataEncoder(void *data, uint32_t data_length, lldb::ByteOrder byte_order,
-              uint8_t addr_size);
+  ///     A size of an address in bytes. \see PutAddress, AppendAddress
+  DataEncoder(const void *data, uint32_t data_length,
+              lldb::ByteOrder byte_order, uint8_t addr_size);
 
-  /// Construct with shared data.
-  ///
-  /// Copies the data shared pointer which adds a reference to the contained
-  /// in \a data_sp. The shared data reference is reference counted to ensure
-  /// the data lives as long as anyone still has a valid shared pointer to the
-  /// data in \a data_sp.
+  /// Construct an encoder that owns a heap based memory buffer.
   ///
-  /// \param[in] data_sp
-  ///     A shared pointer to data.
+  /// This allows clients to create binary data from scratch by appending values
+  /// with the methods that start with "Append".
   ///
   /// \param[in] byte_order
-  ///     A byte order of the data that we are extracting from.
+  ///     A byte order for the data that will be encoded.
   ///
   /// \param[in] addr_size
-  ///     A new address byte size value.
-  DataEncoder(const lldb::DataBufferSP &data_sp, lldb::ByteOrder byte_order,
-              uint8_t addr_size);
+  ///     A size of an address in bytes. \see PutAddress, AppendAddress
+  DataEncoder(lldb::ByteOrder byte_order, uint8_t addr_size);
 
-  /// Destructor
-  ///
-  /// If this object contains a valid shared data reference, the reference
-  /// count on the data will be decremented, and if zero, the data will be
-  /// freed.
   ~DataEncoder();
 
-  /// Clears the object state.
-  ///
-  /// Clears the object contents back to a default invalid state, and release
-  /// any references to shared data that this object may contain.
-  void Clear();
-
   /// Encode an unsigned integer of size \a byte_size to \a offset.
   ///
   /// Encode a single integer value at \a offset and return the offset that
   /// follows the newly encoded integer when the data is successfully encoded
-  /// into the existing data. There must be enough room in the data, else
-  /// UINT32_MAX will be returned to indicate that encoding failed.
+  /// into the existing data. There must be enough room in the existing data,
+  /// else UINT32_MAX will be returned to indicate that encoding failed.
   ///
   /// \param[in] offset
-  ///     The offset within the contained data at which to put the
-  ///     encoded integer.
+  ///     The offset within the contained data at which to put the encoded
+  ///     integer.
   ///
   /// \param[in] byte_size
   ///     The size in byte of the integer to encode.
@@ -111,6 +110,72 @@ public:
   ///     was successfully encoded, UINT32_MAX if the encoding failed.
   uint32_t PutUnsigned(uint32_t offset, uint32_t byte_size, uint64_t value);
 
+  /// Encode an unsigned integer at offset \a offset.
+  ///
+  /// Encode a single unsigned integer value at \a offset and return the offset
+  /// that follows the newly encoded integer when the data is successfully
+  /// encoded into the existing data. There must be enough room in the data,
+  /// else UINT32_MAX will be returned to indicate that encoding failed.
+  ///
+  /// \param[in] offset
+  ///     The offset within the contained data at which to put the encoded
+  ///     integer.
+  ///
+  /// \param[in] value
+  ///     The integer value to write.
+  ///
+  /// \return
+  ///     The next offset in the bytes of this data if the integer was
+  ///     successfully encoded, UINT32_MAX if the encoding failed.
+  uint32_t PutU8(uint32_t offset, uint8_t value);
+  uint32_t PutU16(uint32_t offset, uint16_t value);
+  uint32_t PutU32(uint32_t offset, uint32_t value);
+  uint32_t PutU64(uint32_t offset, uint64_t value);
+
+  /// Append a unsigned integer to the end of the owned data.
+  ///
+  /// \param value
+  ///   A unsigned integer value to append.
+  void AppendU8(uint8_t value);
+  void AppendU16(uint16_t value);
+  void AppendU32(uint32_t value);
+  void AppendU64(uint64_t value);
+
+  /// Append an address sized integer to the end of the owned data.
+  ///
+  /// \param addr
+  ///    A unsigned integer address value to append. The size of the address
+  ///    will be determined by the address size specified in the constructor.
+  void AppendAddress(lldb::addr_t addr);
+
+  /// Append a bytes to the end of the owned data.
+  ///
+  /// Append the bytes contained in the string reference. This function will
+  /// not append a NULL termination character for a C string. Use the
+  /// AppendCString function for this purpose.
+  ///
+  /// \param data
+  ///     A string reference that contains bytes to append.
+  void AppendData(llvm::StringRef data);
+
+  /// Append a bytes to the end of the owned data.
+  ///
+  /// Append the bytes contained in the array reference.
+  ///
+  /// \param data
+  ///     A array reference that contains bytes to append.
+  void AppendData(llvm::ArrayRef<uint8_t> data);
+
+  /// Append a C string to the end of the owned data.
+  ///
+  /// Append the bytes contained in the string reference along with an extra
+  /// NULL termination character if the StringRef bytes doesn't include one as
+  /// the last byte.
+  ///
+  /// \param data
+  ///     A string reference that contains bytes to append.
+  void AppendCString(llvm::StringRef data);
+
   /// Encode an arbitrary number of bytes.
   ///
   /// \param[in] offset
@@ -131,11 +196,10 @@ public:
   /// Encode an address in the existing buffer at \a offset bytes into the
   /// buffer.
   ///
-  /// Encode a single address (honoring the m_addr_size member) to the data
-  /// and return the next offset where subsequent data would go. pointed to by
-  /// \a offset_ptr. The size of the extracted address comes from the \a
-  /// m_addr_size member variable and should be set correctly prior to
-  /// extracting any address values.
+  /// Encode a single address to the data and return the next offset where
+  /// subsequent data would go. The size of the address comes from the \a
+  /// m_addr_size member variable and should be set correctly prior to encoding
+  /// any address values.
   ///
   /// \param[in] offset
   ///     The offset where to encode the address.
@@ -150,7 +214,10 @@ public:
 
   /// Put a C string to \a offset.
   ///
-  /// Encodes a C string into the existing data including the terminating
+  /// Encodes a C string into the existing data including the terminating. If
+  /// there is not enough room in the buffer to fit the entire C string and the
+  /// NULL terminator in the existing buffer bounds, then this function will
+  /// fail.
   ///
   /// \param[in] offset
   ///     The offset where to encode the string.
@@ -159,18 +226,43 @@ public:
   ///     The string to encode.
   ///
   /// \return
-  ///     A pointer to the C string value in the data. If the offset
-  ///     pointed to by \a offset_ptr is out of bounds, or if the
-  ///     offset plus the length of the C string is out of bounds,
-  ///     NULL will be returned.
+  ///     The next valid offset within data if the put operation was successful,
+  ///     else UINT32_MAX to indicate the put failed.
   uint32_t PutCString(uint32_t offset, const char *cstr);
 
-private:
-  uint32_t PutU8(uint32_t offset, uint8_t value);
-  uint32_t PutU16(uint32_t offset, uint16_t value);
-  uint32_t PutU32(uint32_t offset, uint32_t value);
-  uint32_t PutU64(uint32_t offset, uint64_t value);
+  /// Get a shared copy of the heap based memory buffer owned by this object.
+  ///
+  /// This allows a data encoder to be used to create a data buffer that can
+  /// be extracted and used elsewhere after this object is destroyed.
+  ///
+  /// \return
+  ///     A shared pointer to the DataBufferHeap that contains the data that was
+  ///     encoded into this object.
+  std::shared_ptr<lldb_private::DataBufferHeap> GetDataBuffer() {
+    return m_data_sp;
+  }
+
+  /// Get a access to the bytes that this references.
+  ///
+  /// This value will always return the data that this object references even if
+  /// the object was constructed with caller owned data.
+  ///
+  /// \return
+  ///     A array reference to the data that this object references.
+  llvm::ArrayRef<uint8_t> GetData() const;
+
+  /// Get the number of bytes contained in this object.
+  ///
+  /// \return
+  ///     The total number of bytes of data this object refers to.
+  size_t GetByteSize() const;
+
+  lldb::ByteOrder GetByteOrder() const { return m_byte_order; }
+
+  /// The address size to use when encoding pointers or addresses.
+  uint8_t GetAddressByteSize() const { return m_addr_size; }
 
+private:
   uint32_t BytesLeft(uint32_t offset) const {
     const uint32_t size = GetByteSize();
     if (size > offset)
@@ -187,31 +279,6 @@ private:
     return length <= BytesLeft(offset);
   }
 
-  /// Adopt a subset of shared data in \a data_sp.
-  ///
-  /// Copies the data shared pointer which adds a reference to the contained
-  /// in \a data_sp. The shared data reference is reference counted to ensure
-  /// the data lives as long as anyone still has a valid shared pointer to the
-  /// data in \a data_sp. The byte order and address byte size settings remain
-  /// the same. If \a offset is not a valid offset in \a data_sp, then no
-  /// reference to the shared data will be added. If there are not \a length
-  /// bytes available in \a data starting at \a offset, the length will be
-  /// truncated to contains as many bytes as possible.
-  ///
-  /// \param[in] data_sp
-  ///     A shared pointer to data.
-  ///
-  /// \param[in] offset
-  ///     The offset into \a data_sp at which the subset starts.
-  ///
-  /// \param[in] length
-  ///     The length in bytes of the subset of \a data_sp.
-  ///
-  /// \return
-  ///     The number of bytes that this object now contains.
-  uint32_t SetData(const lldb::DataBufferSP &data_sp, uint32_t offset = 0,
-                   uint32_t length = UINT32_MAX);
-
   /// Test the validity of \a offset.
   ///
   /// \return
@@ -219,28 +286,14 @@ private:
   ///     object, \b false otherwise.
   bool ValidOffset(uint32_t offset) const { return offset < GetByteSize(); }
 
-  /// Get the number of bytes contained in this object.
-  ///
-  /// \return
-  ///     The total number of bytes of data this object refers to.
-  size_t GetByteSize() const { return m_end - m_start; }
-
-  /// A pointer to the first byte of data.
-  uint8_t *m_start = nullptr;
-
-  /// A pointer to the byte that is past the end of the data.
-  uint8_t *m_end = nullptr;
-
-  /// The byte order of the data we are extracting from.
-  lldb::ByteOrder m_byte_order;
+  /// The shared pointer to data that can grow as data is added
+  std::shared_ptr<lldb_private::DataBufferHeap> m_data_sp;
 
-  /// The address size to use when extracting pointers or
-  /// addresses
-  uint8_t m_addr_size;
+  /// The byte order of the data we are encoding to.
+  const lldb::ByteOrder m_byte_order;
 
-  /// The shared pointer to data that can
-  /// be shared among multiple instances
-  mutable lldb::DataBufferSP m_data_sp;
+  /// The address size to use when encoding pointers or addresses.
+  const uint8_t m_addr_size;
 
   DataEncoder(const DataEncoder &) = delete;
   const DataEncoder &operator=(const DataEncoder &) = delete;
diff --git a/lldb/include/lldb/Utility/RangeMap.h b/lldb/include/lldb/Utility/RangeMap.h
index 118fdfd85fa9..422f90d807a7 100644
--- a/lldb/include/lldb/Utility/RangeMap.h
+++ b/lldb/include/lldb/Utility/RangeMap.h
@@ -360,7 +360,6 @@ protected:
           m_entries.erase(next);
       }
     }
-    return;
   }
 
   Collection m_entries;
diff --git a/lldb/include/lldb/Utility/Reproducer.h b/lldb/include/lldb/Utility/Reproducer.h
index 4659254e57d6..35043d688511 100644
--- a/lldb/include/lldb/Utility/Reproducer.h
+++ b/lldb/include/lldb/Utility/Reproducer.h
@@ -29,8 +29,6 @@ class Reproducer;
 
 enum class ReproducerMode {
   Capture,
-  Replay,
-  PassiveReplay,
   Off,
 };
 
@@ -179,15 +177,12 @@ public:
 
   const FileSpec &GetRoot() const { return m_root; }
 
-  bool IsPassiveReplay() const { return m_passive_replay; }
-
 private:
   bool HasFile(llvm::StringRef file);
 
   FileSpec m_root;
   std::vector<std::string> m_files;
   bool m_loaded;
-  bool m_passive_replay;
 };
 
 /// The reproducer enables clients to obtain access to the Generator and
@@ -212,11 +207,9 @@ public:
   FileSpec GetReproducerPath() const;
 
   bool IsCapturing() { return static_cast<bool>(m_generator); };
-  bool IsReplaying() { return static_cast<bool>(m_loader); };
 
 protected:
   llvm::Error SetCapture(llvm::Optional<FileSpec> root);
-  llvm::Error SetReplay(llvm::Optional<FileSpec> root, bool passive = false);
 
 private:
   static llvm::Optional<Reproducer> &InstanceImpl();
diff --git a/lldb/include/lldb/Version/Version.h b/lldb/include/lldb/Version/Version.h
new file mode 100644
index 000000000000..15b613560187
--- /dev/null
+++ b/lldb/include/lldb/Version/Version.h
@@ -0,0 +1,23 @@
+//===-- Version.h -----------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_VERSION_VERSION_H
+#define LLDB_VERSION_VERSION_H
+
+#include <string>
+
+namespace lldb_private {
+
+/// Retrieves a string representing the complete LLDB version, which includes
+/// the lldb version number, as well as embedded compiler versions and the
+/// vendor tag.
+const char *GetVersion();
+
+} // namespace lldb_private
+
+#endif // LLDB_VERSION_VERSION_H
diff --git a/lldb/include/lldb/Version/Version.inc.in b/lldb/include/lldb/Version/Version.inc.in
new file mode 100644
index 000000000000..5098bb1c2be5
--- /dev/null
+++ b/lldb/include/lldb/Version/Version.inc.in
@@ -0,0 +1,6 @@
+#define LLDB_VERSION @LLDB_VERSION@
+#define LLDB_VERSION_STRING "@LLDB_VERSION@"
+#define LLDB_VERSION_MAJOR @LLDB_VERSION_MAJOR@
+#define LLDB_VERSION_MINOR @LLDB_VERSION_MINOR@
+#define LLDB_VERSION_PATCHLEVEL @LLDB_VERSION_PATCHLEVEL@
+#cmakedefine LLDB_FULL_VERSION_STRING "@LLDB_FULL_VERSION_STRING@"
diff --git a/lldb/include/lldb/lldb-forward.h b/lldb/include/lldb/lldb-forward.h
index 482da17bfacb..6f8c52ff3749 100644
--- a/lldb/include/lldb/lldb-forward.h
+++ b/lldb/include/lldb/lldb-forward.h
@@ -46,6 +46,7 @@ class BroadcastEventSpec;
 class Broadcaster;
 class BroadcasterManager;
 class CXXSyntheticChildren;
+struct CacheSignature;
 class CallFrameInfo;
 class CommandInterpreter;
 class CommandInterpreterRunOptions;
@@ -61,12 +62,15 @@ class CompilerType;
 class Connection;
 class ConnectionFileDescriptor;
 class ConstString;
+class ConstStringTable;
 class DWARFCallFrameInfo;
 class DWARFDataExtractor;
 class DWARFExpression;
 class DataBuffer;
+class DataBufferHeap;
 class DataEncoder;
 class DataExtractor;
+class DataFileCache;
 class Debugger;
 class Declaration;
 class DiagnosticManager;
@@ -198,6 +202,7 @@ class Stream;
 class StreamFile;
 class StreamString;
 class StringList;
+class StringTableReader;
 class StructuredDataImpl;
 class StructuredDataPlugin;
 class Symbol;
diff --git a/lldb/include/lldb/lldb-private.h b/lldb/include/lldb/lldb-private.h
index d65773aecd6d..ac07047fb4ed 100644
--- a/lldb/include/lldb/lldb-private.h
+++ b/lldb/include/lldb/lldb-private.h
@@ -17,12 +17,6 @@
 #include "lldb/lldb-private-types.h"
 #include "lldb/lldb-public.h"
 
-namespace lldb_private {
-
-const char *GetVersion();
-
-} // namespace lldb_private
-
 #endif // defined(__cplusplus)
 
 #endif // LLDB_LLDB_PRIVATE_H
diff --git a/lldb/source/API/SBData.cpp b/lldb/source/API/SBData.cpp
index daf313ad55c9..9fc590578bce 100644
--- a/lldb/source/API/SBData.cpp
+++ b/lldb/source/API/SBData.cpp
@@ -374,6 +374,25 @@ void SBData::SetData(lldb::SBError &error, const void *buf, size_t size,
   }
 }
 
+void SBData::SetDataWithOwnership(lldb::SBError &error, const void *buf,
+                                  size_t size, lldb::ByteOrder endian,
+                                  uint8_t addr_size) {
+  LLDB_RECORD_DUMMY(
+      void, SBData, SetData,
+      (lldb::SBError &, const void *, size_t, lldb::ByteOrder, uint8_t, bool),
+      error, buf, size, endian, addr_size, copy);
+
+  lldb::DataBufferSP buffer_sp = std::make_shared<DataBufferHeap>(buf, size);
+
+  if (!m_opaque_sp.get())
+    m_opaque_sp = std::make_shared<DataExtractor>(buf, size, endian, addr_size);
+  else {
+    m_opaque_sp->SetData(buffer_sp);
+    m_opaque_sp->SetByteOrder(endian);
+    m_opaque_sp->SetAddressByteSize(addr_size);
+  }
+}
+
 bool SBData::Append(const SBData &rhs) {
   LLDB_RECORD_METHOD(bool, SBData, Append, (const lldb::SBData &), rhs);
 
diff --git a/lldb/source/API/SBDebugger.cpp b/lldb/source/API/SBDebugger.cpp
index 844b91de4cd0..fa5dcb57de7e 100644
--- a/lldb/source/API/SBDebugger.cpp
+++ b/lldb/source/API/SBDebugger.cpp
@@ -11,8 +11,6 @@
 
 #include "lldb/API/SBDebugger.h"
 
-#include "lldb/lldb-private.h"
-
 #include "lldb/API/SBBroadcaster.h"
 #include "lldb/API/SBCommandInterpreter.h"
 #include "lldb/API/SBCommandInterpreterRunOptions.h"
@@ -52,6 +50,7 @@
 #include "lldb/Target/TargetList.h"
 #include "lldb/Utility/Args.h"
 #include "lldb/Utility/State.h"
+#include "lldb/Version/Version.h"
 
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringRef.h"
@@ -750,6 +749,9 @@ SBStructuredData SBDebugger::GetBuildConfiguration() {
   AddBoolConfigEntry(
       *config_up, "lua", LLDB_ENABLE_LUA,
       "A boolean value that indicates if lua support is enabled in LLDB");
+  AddBoolConfigEntry(*config_up, "fbsdvmcore", LLDB_ENABLE_FBSDVMCORE,
+                     "A boolean value that indicates if fbsdvmcore support is "
+                     "enabled in LLDB");
   AddLLVMTargets(*config_up);
 
   SBStructuredData data;
diff --git a/lldb/source/API/SBFrame.cpp b/lldb/source/API/SBFrame.cpp
index 7107768ba884..c6bc3288c4b2 100644
--- a/lldb/source/API/SBFrame.cpp
+++ b/lldb/source/API/SBFrame.cpp
@@ -119,15 +119,13 @@ SBSymbolContext SBFrame::GetSymbolContext(uint32_t resolve_scope) const {
   std::unique_lock<std::recursive_mutex> lock;
   ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
   SymbolContextItem scope = static_cast<SymbolContextItem>(resolve_scope);
-  StackFrame *frame = nullptr;
   Target *target = exe_ctx.GetTargetPtr();
   Process *process = exe_ctx.GetProcessPtr();
   if (target && process) {
     Process::StopLocker stop_locker;
     if (stop_locker.TryLock(&process->GetRunLock())) {
-      frame = exe_ctx.GetFramePtr();
-      if (frame)
-        sb_sym_ctx.SetSymbolContext(&frame->GetSymbolContext(scope));
+      if (StackFrame *frame = exe_ctx.GetFramePtr())
+        sb_sym_ctx = frame->GetSymbolContext(scope);
     }
   }
 
diff --git a/lldb/source/API/SBReproducer.cpp b/lldb/source/API/SBReproducer.cpp
index 68e632da1bde..c9c9a03c694a 100644
--- a/lldb/source/API/SBReproducer.cpp
+++ b/lldb/source/API/SBReproducer.cpp
@@ -23,7 +23,7 @@
 #include "lldb/API/SBHostOS.h"
 #include "lldb/API/SBReproducer.h"
 #include "lldb/Host/FileSystem.h"
-#include "lldb/lldb-private.h"
+#include "lldb/Version/Version.h"
 
 using namespace lldb;
 using namespace lldb_private;
@@ -165,111 +165,24 @@ const char *SBReproducer::Capture(const char *path) {
 }
 
 const char *SBReproducer::PassiveReplay(const char *path) {
-  static std::string error;
-  if (auto e = Reproducer::Initialize(ReproducerMode::PassiveReplay,
-                                      FileSpec(path))) {
-    error = llvm::toString(std::move(e));
-    return error.c_str();
-  }
-
-  if (auto *l = lldb_private::repro::Reproducer::Instance().GetLoader()) {
-    FileSpec file = l->GetFile<SBProvider::Info>();
-    auto error_or_file = llvm::MemoryBuffer::getFile(file.GetPath());
-    if (!error_or_file) {
-      error =
-          "unable to read SB API data: " + error_or_file.getError().message();
-      return error.c_str();
-    }
-    static ReplayData r(std::move(*error_or_file));
-    InstrumentationData::Initialize(r.GetDeserializer(), r.GetRegistry());
-  }
-
-  return nullptr;
+  return "Reproducer replay has been removed";
 }
 
 const char *SBReproducer::Replay(const char *path) {
-  SBReplayOptions options;
-  return SBReproducer::Replay(path, options);
+  return "Reproducer replay has been removed";
 }
 
 const char *SBReproducer::Replay(const char *path, bool skip_version_check) {
-  SBReplayOptions options;
-  options.SetCheckVersion(!skip_version_check);
-  return SBReproducer::Replay(path, options);
+  return Replay(path);
 }
 
 const char *SBReproducer::Replay(const char *path,
                                  const SBReplayOptions &options) {
-  static std::string error;
-  if (auto e = Reproducer::Initialize(ReproducerMode::Replay, FileSpec(path))) {
-    error = llvm::toString(std::move(e));
-    return error.c_str();
-  }
-
-  repro::Loader *loader = repro::Reproducer::Instance().GetLoader();
-  if (!loader) {
-    error = "unable to get replay loader.";
-    return error.c_str();
-  }
-
-  if (options.GetCheckVersion()) {
-    llvm::Expected<std::string> version = loader->LoadBuffer<VersionProvider>();
-    if (!version) {
-      error = llvm::toString(version.takeError());
-      return error.c_str();
-    }
-    if (lldb_private::GetVersion() != llvm::StringRef(*version).rtrim()) {
-      error = "reproducer capture and replay version don't match:\n";
-      error.append("reproducer captured with:\n");
-      error.append(*version);
-      error.append("reproducer replayed with:\n");
-      error.append(lldb_private::GetVersion());
-      return error.c_str();
-    }
-  }
-
-  if (options.GetVerify()) {
-    bool verification_failed = false;
-    llvm::raw_string_ostream os(error);
-    auto error_callback = [&](llvm::StringRef error) {
-      verification_failed = true;
-      os << "\nerror: " << error;
-    };
-
-    auto warning_callback = [&](llvm::StringRef warning) {
-      verification_failed = true;
-      os << "\nwarning: " << warning;
-    };
-
-    auto note_callback = [&](llvm::StringRef warning) {};
-
-    Verifier verifier(loader);
-    verifier.Verify(error_callback, warning_callback, note_callback);
-
-    if (verification_failed) {
-      os.flush();
-      return error.c_str();
-    }
-  }
-
-  FileSpec file = loader->GetFile<SBProvider::Info>();
-  if (!file) {
-    error = "unable to get replay data from reproducer.";
-    return error.c_str();
-  }
-
-  SBRegistry registry;
-  registry.Replay(file);
-
-  return nullptr;
+  return Replay(path);
 }
 
 const char *SBReproducer::Finalize(const char *path) {
   static std::string error;
-  if (auto e = Reproducer::Initialize(ReproducerMode::Replay, FileSpec(path))) {
-    error = llvm::toString(std::move(e));
-    return error.c_str();
-  }
 
   repro::Loader *loader = repro::Reproducer::Instance().GetLoader();
   if (!loader) {
diff --git a/lldb/source/API/SBStructuredData.cpp b/lldb/source/API/SBStructuredData.cpp
index 97a9eadcaf07..e99c6194c516 100644
--- a/lldb/source/API/SBStructuredData.cpp
+++ b/lldb/source/API/SBStructuredData.cpp
@@ -39,10 +39,10 @@ SBStructuredData::SBStructuredData(const lldb::EventSP &event_sp)
   LLDB_RECORD_CONSTRUCTOR(SBStructuredData, (const lldb::EventSP &), event_sp);
 }
 
-SBStructuredData::SBStructuredData(lldb_private::StructuredDataImpl *impl)
-    : m_impl_up(impl ? impl : new StructuredDataImpl()) {
+SBStructuredData::SBStructuredData(const lldb_private::StructuredDataImpl &impl)
+    : m_impl_up(new StructuredDataImpl(impl)) {
   LLDB_RECORD_CONSTRUCTOR(SBStructuredData,
-                          (lldb_private::StructuredDataImpl *), impl);
+                          (const lldb_private::StructuredDataImpl &), impl);
 }
 
 SBStructuredData::~SBStructuredData() = default;
@@ -210,7 +210,7 @@ template <> void RegisterMethods<SBStructuredData>(Registry &R) {
   LLDB_REGISTER_CONSTRUCTOR(SBStructuredData, (const lldb::SBStructuredData &));
   LLDB_REGISTER_CONSTRUCTOR(SBStructuredData, (const lldb::EventSP &));
   LLDB_REGISTER_CONSTRUCTOR(SBStructuredData,
-                            (lldb_private::StructuredDataImpl *));
+                            (const lldb_private::StructuredDataImpl &));
   LLDB_REGISTER_METHOD(
       lldb::SBStructuredData &,
       SBStructuredData, operator=,(const lldb::SBStructuredData &));
diff --git a/lldb/source/API/SBSymbolContext.cpp b/lldb/source/API/SBSymbolContext.cpp
index 488d49884903..89fe051658ff 100644
--- a/lldb/source/API/SBSymbolContext.cpp
+++ b/lldb/source/API/SBSymbolContext.cpp
@@ -22,12 +22,10 @@ SBSymbolContext::SBSymbolContext() : m_opaque_up() {
   LLDB_RECORD_CONSTRUCTOR_NO_ARGS(SBSymbolContext);
 }
 
-SBSymbolContext::SBSymbolContext(const SymbolContext *sc_ptr) : m_opaque_up() {
+SBSymbolContext::SBSymbolContext(const SymbolContext &sc)
+    : m_opaque_up(std::make_unique<SymbolContext>(sc)) {
   LLDB_RECORD_CONSTRUCTOR(SBSymbolContext,
-                          (const lldb_private::SymbolContext *), sc_ptr);
-
-  if (sc_ptr)
-    m_opaque_up = std::make_unique<SymbolContext>(*sc_ptr);
+                          (const lldb_private::SymbolContext &), sc);
 }
 
 SBSymbolContext::SBSymbolContext(const SBSymbolContext &rhs) : m_opaque_up() {
@@ -49,13 +47,6 @@ const SBSymbolContext &SBSymbolContext::operator=(const SBSymbolContext &rhs) {
   return LLDB_RECORD_RESULT(*this);
 }
 
-void SBSymbolContext::SetSymbolContext(const SymbolContext *sc_ptr) {
-  if (sc_ptr)
-    m_opaque_up = std::make_unique<SymbolContext>(*sc_ptr);
-  else
-    m_opaque_up->Clear(true);
-}
-
 bool SBSymbolContext::IsValid() const {
   LLDB_RECORD_METHOD_CONST_NO_ARGS(bool, SBSymbolContext, IsValid);
   return this->operator bool();
@@ -237,7 +228,7 @@ template <>
 void RegisterMethods<SBSymbolContext>(Registry &R) {
   LLDB_REGISTER_CONSTRUCTOR(SBSymbolContext, ());
   LLDB_REGISTER_CONSTRUCTOR(SBSymbolContext,
-                            (const lldb_private::SymbolContext *));
+                            (const lldb_private::SymbolContext &));
   LLDB_REGISTER_CONSTRUCTOR(SBSymbolContext, (const lldb::SBSymbolContext &));
   LLDB_REGISTER_METHOD(
       const lldb::SBSymbolContext &,
diff --git a/lldb/source/API/SBSymbolContextList.cpp b/lldb/source/API/SBSymbolContextList.cpp
index 9db84dc1bf4b..70a8bbe6694c 100644
--- a/lldb/source/API/SBSymbolContextList.cpp
+++ b/lldb/source/API/SBSymbolContextList.cpp
@@ -56,9 +56,8 @@ SBSymbolContext SBSymbolContextList::GetContextAtIndex(uint32_t idx) {
   SBSymbolContext sb_sc;
   if (m_opaque_up) {
     SymbolContext sc;
-    if (m_opaque_up->GetContextAtIndex(idx, sc)) {
-      sb_sc.SetSymbolContext(&sc);
-    }
+    if (m_opaque_up->GetContextAtIndex(idx, sc))
+      sb_sc = sc;
   }
   return LLDB_RECORD_RESULT(sb_sc);
 }
diff --git a/lldb/source/API/SBThreadPlan.cpp b/lldb/source/API/SBThreadPlan.cpp
index 9af673b0f3a9..99ecb321595f 100644
--- a/lldb/source/API/SBThreadPlan.cpp
+++ b/lldb/source/API/SBThreadPlan.cpp
@@ -69,8 +69,8 @@ SBThreadPlan::SBThreadPlan(lldb::SBThread &sb_thread, const char *class_name) {
 
   Thread *thread = sb_thread.get();
   if (thread)
-    m_opaque_wp =
-        std::make_shared<ThreadPlanPython>(*thread, class_name, nullptr);
+    m_opaque_wp = std::make_shared<ThreadPlanPython>(*thread, class_name,
+                                                     StructuredDataImpl());
 }
 
 SBThreadPlan::SBThreadPlan(lldb::SBThread &sb_thread, const char *class_name,
@@ -82,7 +82,7 @@ SBThreadPlan::SBThreadPlan(lldb::SBThread &sb_thread, const char *class_name,
   Thread *thread = sb_thread.get();
   if (thread)
     m_opaque_wp = std::make_shared<ThreadPlanPython>(*thread, class_name,
-                                                     args_data.m_impl_up.get());
+                                                     *args_data.m_impl_up);
 }
 
 // Assignment operator
diff --git a/lldb/source/API/SBTypeSummary.cpp b/lldb/source/API/SBTypeSummary.cpp
index 3800ae940c70..2d7f8ef340c9 100644
--- a/lldb/source/API/SBTypeSummary.cpp
+++ b/lldb/source/API/SBTypeSummary.cpp
@@ -100,20 +100,11 @@ const lldb_private::TypeSummaryOptions &SBTypeSummaryOptions::ref() const {
 }
 
 SBTypeSummaryOptions::SBTypeSummaryOptions(
-    const lldb_private::TypeSummaryOptions *lldb_object_ptr) {
+    const lldb_private::TypeSummaryOptions &lldb_object)
+    : m_opaque_up(std::make_unique<TypeSummaryOptions>(lldb_object)) {
   LLDB_RECORD_CONSTRUCTOR(SBTypeSummaryOptions,
-                          (const lldb_private::TypeSummaryOptions *),
-                          lldb_object_ptr);
-
-  SetOptions(lldb_object_ptr);
-}
-
-void SBTypeSummaryOptions::SetOptions(
-    const lldb_private::TypeSummaryOptions *lldb_object_ptr) {
-  if (lldb_object_ptr)
-    m_opaque_up = std::make_unique<TypeSummaryOptions>(*lldb_object_ptr);
-  else
-    m_opaque_up = std::make_unique<TypeSummaryOptions>();
+                          (const lldb_private::TypeSummaryOptions &),
+                          lldb_object);
 }
 
 SBTypeSummary::SBTypeSummary() : m_opaque_sp() {
@@ -175,7 +166,7 @@ SBTypeSummary SBTypeSummary::CreateWithCallback(FormatCallback cb,
              const TypeSummaryOptions &opt) -> bool {
           SBStream stream;
           SBValue sb_value(valobj.GetSP());
-          SBTypeSummaryOptions options(&opt);
+          SBTypeSummaryOptions options(opt);
           if (!cb(sb_value, options, stream))
             return false;
           stm.Write(stream.GetData(), stream.GetSize());
@@ -492,7 +483,7 @@ void RegisterMethods<SBTypeSummaryOptions>(Registry &R) {
   LLDB_REGISTER_METHOD(void, SBTypeSummaryOptions, SetCapping,
                        (lldb::TypeSummaryCapping));
   LLDB_REGISTER_CONSTRUCTOR(SBTypeSummaryOptions,
-                            (const lldb_private::TypeSummaryOptions *));
+                            (const lldb_private::TypeSummaryOptions &));
 }
 
 template <>
diff --git a/lldb/source/API/SystemInitializerFull.cpp b/lldb/source/API/SystemInitializerFull.cpp
index cc6cb6925bd0..b01cb2ff545b 100644
--- a/lldb/source/API/SystemInitializerFull.cpp
+++ b/lldb/source/API/SystemInitializerFull.cpp
@@ -50,15 +50,8 @@ SystemInitializerFull::~SystemInitializerFull() = default;
 
 llvm::Error SystemInitializerFull::Initialize() {
   llvm::Error error = SystemInitializerCommon::Initialize();
-  if (error) {
-    // During active replay, the ::Initialize call is replayed like any other
-    // SB API call and the return value is ignored. Since we can't intercept
-    // this, we terminate here before the uninitialized debugger inevitably
-    // crashes.
-    if (repro::Reproducer::Instance().IsReplaying())
-      llvm::report_fatal_error(std::move(error));
+  if (error)
     return error;
-  }
 
   // Initialize LLVM and Clang
   llvm::InitializeAllTargets();
diff --git a/lldb/source/Breakpoint/BreakpointResolverScripted.cpp b/lldb/source/Breakpoint/BreakpointResolverScripted.cpp
index 92297fbc7c4b..308c3b987f58 100644
--- a/lldb/source/Breakpoint/BreakpointResolverScripted.cpp
+++ b/lldb/source/Breakpoint/BreakpointResolverScripted.cpp
@@ -27,10 +27,9 @@ using namespace lldb_private;
 // BreakpointResolverScripted:
 BreakpointResolverScripted::BreakpointResolverScripted(
     const BreakpointSP &bkpt, const llvm::StringRef class_name,
-    lldb::SearchDepth depth, StructuredDataImpl *args_data)
+    lldb::SearchDepth depth, const StructuredDataImpl &args_data)
     : BreakpointResolver(bkpt, BreakpointResolver::PythonResolver),
-      m_class_name(std::string(class_name)), m_depth(depth),
-      m_args_ptr(args_data) {
+      m_class_name(std::string(class_name)), m_depth(depth), m_args(args_data) {
   CreateImplementationIfNeeded(bkpt);
 }
 
@@ -52,7 +51,7 @@ void BreakpointResolverScripted::CreateImplementationIfNeeded(
     return;
 
   m_implementation_sp = script_interp->CreateScriptedBreakpointResolver(
-      m_class_name.c_str(), m_args_ptr, breakpoint_sp);
+      m_class_name.c_str(), m_args, breakpoint_sp);
 }
 
 void BreakpointResolverScripted::NotifyBreakpointSet() {
@@ -75,14 +74,12 @@ BreakpointResolverScripted::CreateFromStructuredData(
   // The Python function will actually provide the search depth, this is a
   // placeholder.
   lldb::SearchDepth depth = lldb::eSearchDepthTarget;
-  
-  StructuredDataImpl *args_data_impl = new StructuredDataImpl();
+
+  StructuredDataImpl args_data_impl;
   StructuredData::Dictionary *args_dict = nullptr;
-  success = options_dict.GetValueForKeyAsDictionary(
-    GetKey(OptionNames::ScriptArgs), args_dict);
-  if (success) {
-      args_data_impl->SetObjectSP(args_dict->shared_from_this());
-  }
+  if (options_dict.GetValueForKeyAsDictionary(GetKey(OptionNames::ScriptArgs),
+                                              args_dict))
+    args_data_impl.SetObjectSP(args_dict->shared_from_this());
   return new BreakpointResolverScripted(bkpt, class_name, depth, 
                                         args_data_impl);
 }
@@ -94,9 +91,9 @@ BreakpointResolverScripted::SerializeToStructuredData() {
 
   options_dict_sp->AddStringItem(GetKey(OptionNames::PythonClassName),
                                    m_class_name);
-  if (m_args_ptr->IsValid())
-      options_dict_sp->AddItem(GetKey(OptionNames::ScriptArgs),
-          m_args_ptr->GetObjectSP());
+  if (m_args.IsValid())
+    options_dict_sp->AddItem(GetKey(OptionNames::ScriptArgs),
+                             m_args.GetObjectSP());
 
   return WrapOptionsDict(options_dict_sp);
 }
@@ -151,10 +148,6 @@ void BreakpointResolverScripted::Dump(Stream *s) const {}
 
 lldb::BreakpointResolverSP
 BreakpointResolverScripted::CopyForBreakpoint(BreakpointSP &breakpoint) {
-  // FIXME: Have to make a copy of the arguments from the m_args_ptr and then
-  // pass that to the new resolver.
-  lldb::BreakpointResolverSP ret_sp(
-      new BreakpointResolverScripted(breakpoint, m_class_name, m_depth, 
-                                     nullptr));
-  return ret_sp;
+  return std::make_shared<BreakpointResolverScripted>(breakpoint, m_class_name,
+                                                      m_depth, m_args);
 }
diff --git a/lldb/source/Commands/CommandObjectMemory.cpp b/lldb/source/Commands/CommandObjectMemory.cpp
index 094ce6f8558f..9df42f36fafd 100644
--- a/lldb/source/Commands/CommandObjectMemory.cpp
+++ b/lldb/source/Commands/CommandObjectMemory.cpp
@@ -716,7 +716,7 @@ protected:
         if (item_byte_size == read) {
           result.AppendWarningWithFormat(
               "unable to find a NULL terminated string at 0x%" PRIx64
-              ".Consider increasing the maximum read length.\n",
+              ". Consider increasing the maximum read length.\n",
               data_addr);
           --read;
           break_on_no_NULL = true;
diff --git a/lldb/source/Commands/CommandObjectReproducer.cpp b/lldb/source/Commands/CommandObjectReproducer.cpp
index 01f9dc64e6f0..4db3e070df3c 100644
--- a/lldb/source/Commands/CommandObjectReproducer.cpp
+++ b/lldb/source/Commands/CommandObjectReproducer.cpp
@@ -195,10 +195,6 @@ protected:
         SetError(result, std::move(e));
         return result.Succeeded();
       }
-    } else if (r.IsReplaying()) {
-      // Make this operation a NO-OP in replay mode.
-      result.SetStatus(eReturnStatusSuccessFinishNoResult);
-      return result.Succeeded();
     } else {
       result.AppendErrorWithFormat("Unable to get the reproducer generator");
       return false;
@@ -276,7 +272,7 @@ protected:
 
     auto &r = Reproducer::Instance();
 
-    if (!r.IsCapturing() && !r.IsReplaying()) {
+    if (!r.IsCapturing()) {
       result.AppendError(
           "forcing a crash is only supported when capturing a reproducer.");
       result.SetStatus(eReturnStatusSuccessFinishNoResult);
@@ -326,15 +322,10 @@ protected:
     auto &r = Reproducer::Instance();
     if (r.IsCapturing()) {
       result.GetOutputStream() << "Reproducer is in capture mode.\n";
-    } else if (r.IsReplaying()) {
-      result.GetOutputStream() << "Reproducer is in replay mode.\n";
-    } else {
-      result.GetOutputStream() << "Reproducer is off.\n";
-    }
-
-    if (r.IsCapturing() || r.IsReplaying()) {
       result.GetOutputStream()
           << "Path: " << r.GetReproducerPath().GetPath() << '\n';
+    } else {
+      result.GetOutputStream() << "Reproducer is off.\n";
     }
 
     // Auto generate is hidden unless enabled because this is mostly for
diff --git a/lldb/source/Commands/CommandObjectVersion.cpp b/lldb/source/Commands/CommandObjectVersion.cpp
index 20c2d25b745c..e15faba5661e 100644
--- a/lldb/source/Commands/CommandObjectVersion.cpp
+++ b/lldb/source/Commands/CommandObjectVersion.cpp
@@ -9,7 +9,7 @@
 #include "CommandObjectVersion.h"
 
 #include "lldb/Interpreter/CommandReturnObject.h"
-#include "lldb/lldb-private.h"
+#include "lldb/Version/Version.h"
 
 using namespace lldb;
 using namespace lldb_private;
diff --git a/lldb/source/Core/CoreProperties.td b/lldb/source/Core/CoreProperties.td
index e48879660403..038ed00905f1 100644
--- a/lldb/source/Core/CoreProperties.td
+++ b/lldb/source/Core/CoreProperties.td
@@ -13,6 +13,26 @@ let Definition = "modulelist" in {
     Global,
     DefaultStringValue<"">,
     Desc<"Debug info path which should be resolved while parsing, relative to the host filesystem.">;
+  def EnableLLDBIndexCache: Property<"enable-lldb-index-cache", "Boolean">,
+    Global,
+    DefaultFalse,
+    Desc<"Enable caching for debug sessions in LLDB. LLDB can cache data for each module for improved performance in subsequent debug sessions.">;
+  def LLDBIndexCachePath: Property<"lldb-index-cache-path", "FileSpec">,
+    Global,
+    DefaultStringValue<"">,
+    Desc<"The path to the LLDB index cache directory.">;
+  def LLDBIndexCacheMaxByteSize: Property<"lldb-index-cache-max-byte-size", "UInt64">,
+    Global,
+    DefaultUnsignedValue<0>,
+    Desc<"The maximum size for the LLDB index cache directory in bytes. A value over the amount of available space on the disk will be reduced to the amount of available space. A value of 0 disables the absolute size-based pruning.">;
+  def LLDBIndexCacheMaxPercent: Property<"lldb-index-cache-max-percent", "UInt64">,
+    Global,
+    DefaultUnsignedValue<0>,
+    Desc<"The maximum size for the cache directory in terms of percentage of the available space on the disk. Set to 100 to indicate no limit, 50 to indicate that the cache size will not be left over half the available disk space. A value over 100 will be reduced to 100. A value of 0 disables the percentage size-based pruning.">;
+  def LLDBIndexCacheExpirationDays: Property<"lldb-index-cache-expiration-days", "UInt64">,
+    Global,
+    DefaultUnsignedValue<7>,
+    Desc<"The expiration time in days for a file. When a file hasn't been accessed for the specified amount of days, it is removed from the cache. A value of 0 disables the expiration-based pruning.">;
 }
 
 let Definition = "debugger" in {
diff --git a/lldb/source/Core/DataFileCache.cpp b/lldb/source/Core/DataFileCache.cpp
new file mode 100644
index 000000000000..3f52b925ef46
--- /dev/null
+++ b/lldb/source/Core/DataFileCache.cpp
@@ -0,0 +1,307 @@
+//===-- DataFileCache.cpp -------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "lldb/Core/DataFileCache.h"
+#include "lldb/Core/Module.h"
+#include "lldb/Core/ModuleList.h"
+#include "lldb/Host/FileSystem.h"
+#include "lldb/Symbol/ObjectFile.h"
+#include "lldb/Utility/DataEncoder.h"
+#include "lldb/Utility/Log.h"
+#include "lldb/Utility/Logging.h"
+#include "llvm/Support/CachePruning.h"
+#include "llvm/Support/MemoryBuffer.h"
+
+using namespace llvm;
+using namespace lldb_private;
+
+DataFileCache::DataFileCache(StringRef path) {
+  m_cache_dir.SetPath(path);
+
+  // Prune the cache based off of the LLDB settings each time we create a cache
+  // object.
+  ModuleListProperties &properties =
+      ModuleList::GetGlobalModuleListProperties();
+  CachePruningPolicy policy;
+  // Only scan once an hour. If we have lots of debug sessions we don't want
+  // to scan this directory too often. A timestamp file is written to the
+  // directory to ensure different processes don't scan the directory too often.
+  // This setting doesn't mean that a thread will continually scan the cache
+  // directory within this process.
+  policy.Interval = std::chrono::hours(1);
+  // Get the user settings for pruning.
+  policy.MaxSizeBytes = properties.GetLLDBIndexCacheMaxByteSize();
+  policy.MaxSizePercentageOfAvailableSpace =
+      properties.GetLLDBIndexCacheMaxPercent();
+  policy.Expiration =
+      std::chrono::hours(properties.GetLLDBIndexCacheExpirationDays() * 24);
+  pruneCache(path, policy);
+
+  // This lambda will get called when the data is gotten from the cache and
+  // also after the data was set for a given key. We only need to take
+  // ownership of the data if we are geting the data, so we use the
+  // m_take_ownership member variable to indicate if we need to take
+  // ownership.
+
+  auto add_buffer = [this](unsigned task, std::unique_ptr<llvm::MemoryBuffer> m) {
+    if (m_take_ownership)
+      m_mem_buff_up = std::move(m);
+  };
+  Expected<FileCache> cache_or_err =
+      llvm::localCache("LLDBModuleCache", "lldb-module", path, add_buffer);
+  if (cache_or_err)
+    m_cache_callback = std::move(*cache_or_err);
+  else {
+    Log *log(lldb_private::GetLogIfAnyCategoriesSet(LIBLLDB_LOG_MODULES));
+    LLDB_LOG_ERROR(log, cache_or_err.takeError(),
+                   "failed to create lldb index cache directory: {0}");
+  }
+}
+
+std::unique_ptr<llvm::MemoryBuffer>
+DataFileCache::GetCachedData(StringRef key) {
+  std::lock_guard<std::mutex> guard(m_mutex);
+
+  const unsigned task = 1;
+  m_take_ownership = true;
+  // If we call the "m_cache_callback" function and the data is cached, it will
+  // call the "add_buffer" lambda function from the constructor which will in
+  // turn take ownership of the member buffer that is passed to the callback and
+  // put it into a member variable.
+  Expected<AddStreamFn> add_stream_or_err = m_cache_callback(task, key);
+  m_take_ownership = false;
+  // At this point we either already called the "add_buffer" lambda with
+  // the data or we haven't. We can tell if we got the cached data by checking
+  // the add_stream function pointer value below.
+  if (add_stream_or_err) {
+    AddStreamFn &add_stream = *add_stream_or_err;
+    // If the "add_stream" is nullptr, then the data was cached and we already
+    // called the "add_buffer" lambda. If it is valid, then if we were to call
+    // the add_stream function it would cause a cache file to get generated
+    // and we would be expected to fill in the data. In this function we only
+    // want to check if the data was cached, so we don't want to call
+    // "add_stream" in this function.
+    if (!add_stream)
+      return std::move(m_mem_buff_up);
+  } else {
+    Log *log(lldb_private::GetLogIfAnyCategoriesSet(LIBLLDB_LOG_MODULES));
+    LLDB_LOG_ERROR(log, add_stream_or_err.takeError(),
+                   "failed to get the cache add stream callback for key: {0}");
+  }
+  // Data was not cached.
+  return std::unique_ptr<llvm::MemoryBuffer>();
+}
+
+bool DataFileCache::SetCachedData(StringRef key, llvm::ArrayRef<uint8_t> data) {
+  std::lock_guard<std::mutex> guard(m_mutex);
+  const unsigned task = 2;
+  // If we call this function and the data is cached, it will call the
+  // add_buffer lambda function from the constructor which will ignore the
+  // data.
+  Expected<AddStreamFn> add_stream_or_err = m_cache_callback(task, key);
+  // If we reach this code then we either already called the callback with
+  // the data or we haven't. We can tell if we had the cached data by checking
+  // the CacheAddStream function pointer value below.
+  if (add_stream_or_err) {
+    AddStreamFn &add_stream = *add_stream_or_err;
+    // If the "add_stream" is nullptr, then the data was cached. If it is
+    // valid, then if we call the add_stream function with a task it will
+    // cause the file to get generated, but we only want to check if the data
+    // is cached here, so we don't want to call it here. Note that the
+    // add_buffer will also get called in this case after the data has been
+    // provided, but we won't take ownership of the memory buffer as we just
+    // want to write the data.
+    if (add_stream) {
+      Expected<std::unique_ptr<CachedFileStream>> file_or_err =
+          add_stream(task);
+      if (file_or_err) {
+        CachedFileStream *cfs = file_or_err->get();
+        cfs->OS->write((const char *)data.data(), data.size());
+        return true;
+      } else {
+        Log *log(lldb_private::GetLogIfAnyCategoriesSet(LIBLLDB_LOG_MODULES));
+        LLDB_LOG_ERROR(log, file_or_err.takeError(),
+                       "failed to get the cache file stream for key: {0}");
+      }
+    }
+  } else {
+    Log *log(lldb_private::GetLogIfAnyCategoriesSet(LIBLLDB_LOG_MODULES));
+    LLDB_LOG_ERROR(log, add_stream_or_err.takeError(),
+                   "failed to get the cache add stream callback for key: {0}");
+  }
+  return false;
+}
+
+FileSpec DataFileCache::GetCacheFilePath(llvm::StringRef key) {
+  FileSpec cache_file(m_cache_dir);
+  std::string filename("llvmcache-");
+  filename += key.str();
+  cache_file.AppendPathComponent(filename);
+  return cache_file;
+}
+
+Status DataFileCache::RemoveCacheFile(llvm::StringRef key) {
+  FileSpec cache_file = GetCacheFilePath(key);
+  FileSystem &fs = FileSystem::Instance();
+  if (!fs.Exists(cache_file))
+    return Status();
+  return fs.RemoveFile(cache_file);
+}
+
+CacheSignature::CacheSignature(lldb_private::Module *module) {
+  Clear();
+  UUID uuid = module->GetUUID();
+  if (uuid.IsValid())
+    m_uuid = uuid;
+
+  std::time_t mod_time = 0;
+  mod_time = llvm::sys::toTimeT(module->GetModificationTime());
+  if (mod_time != 0)
+    m_mod_time = mod_time;
+
+  mod_time = llvm::sys::toTimeT(module->GetObjectModificationTime());
+  if (mod_time != 0)
+    m_obj_mod_time = mod_time;
+}
+
+CacheSignature::CacheSignature(lldb_private::ObjectFile *objfile) {
+  Clear();
+  UUID uuid = objfile->GetUUID();
+  if (uuid.IsValid())
+    m_uuid = uuid;
+
+  std::time_t mod_time = 0;
+  // Grab the modification time of the object file's file. It isn't always the
+  // same as the module's file when you have a executable file as the main
+  // executable, and you have a object file for a symbol file.
+  FileSystem &fs = FileSystem::Instance();
+  mod_time = llvm::sys::toTimeT(fs.GetModificationTime(objfile->GetFileSpec()));
+  if (mod_time != 0)
+    m_mod_time = mod_time;
+
+  mod_time =
+      llvm::sys::toTimeT(objfile->GetModule()->GetObjectModificationTime());
+  if (mod_time != 0)
+    m_obj_mod_time = mod_time;
+}
+
+enum SignatureEncoding {
+  eSignatureUUID = 1u,
+  eSignatureModTime = 2u,
+  eSignatureObjectModTime = 3u,
+  eSignatureEnd = 255u,
+};
+
+bool CacheSignature::Encode(DataEncoder &encoder) {
+  if (!IsValid())
+    return false; // Invalid signature, return false!
+
+  if (m_uuid.hasValue()) {
+    llvm::ArrayRef<uint8_t> uuid_bytes = m_uuid->GetBytes();
+    encoder.AppendU8(eSignatureUUID);
+    encoder.AppendU8(uuid_bytes.size());
+    encoder.AppendData(uuid_bytes);
+  }
+  if (m_mod_time.hasValue()) {
+    encoder.AppendU8(eSignatureModTime);
+    encoder.AppendU32(*m_mod_time);
+  }
+  if (m_obj_mod_time.hasValue()) {
+    encoder.AppendU8(eSignatureObjectModTime);
+    encoder.AppendU32(*m_obj_mod_time);
+  }
+  encoder.AppendU8(eSignatureEnd);
+  return true;
+}
+
+bool CacheSignature::Decode(const DataExtractor &data,
+                            lldb::offset_t *offset_ptr) {
+  Clear();
+  while (uint8_t sig_encoding = data.GetU8(offset_ptr)) {
+    switch (sig_encoding) {
+    case eSignatureUUID: {
+      const uint8_t length = data.GetU8(offset_ptr);
+      const uint8_t *bytes = (const uint8_t *)data.GetData(offset_ptr, length);
+      if (bytes != nullptr && length > 0)
+        m_uuid = UUID::fromData(llvm::ArrayRef<uint8_t>(bytes, length));
+    } break;
+    case eSignatureModTime: {
+      uint32_t mod_time = data.GetU32(offset_ptr);
+      if (mod_time > 0)
+        m_mod_time = mod_time;
+    } break;
+    case eSignatureObjectModTime: {
+      uint32_t mod_time = data.GetU32(offset_ptr);
+      if (mod_time > 0)
+        m_mod_time = mod_time;
+    } break;
+    case eSignatureEnd:
+      return true;
+    default:
+      break;
+    }
+  }
+  return false;
+}
+
+uint32_t ConstStringTable::Add(ConstString s) {
+  auto pos = m_string_to_offset.find(s);
+  if (pos != m_string_to_offset.end())
+    return pos->second;
+  const uint32_t offset = m_next_offset;
+  m_strings.push_back(s);
+  m_string_to_offset[s] = offset;
+  m_next_offset += s.GetLength() + 1;
+  return offset;
+}
+
+static const llvm::StringRef kStringTableIdentifier("STAB");
+
+bool ConstStringTable::Encode(DataEncoder &encoder) {
+  // Write an 4 character code into the stream. This will help us when decoding
+  // to make sure we find this identifier when decoding the string table to make
+  // sure we have the rigth data. It also helps to identify the string table
+  // when dumping the hex bytes in a cache file.
+  encoder.AppendData(kStringTableIdentifier);
+  size_t length_offset = encoder.GetByteSize();
+  encoder.AppendU32(0); // Total length of all strings which will be fixed up.
+  size_t strtab_offset = encoder.GetByteSize();
+  encoder.AppendU8(0); // Start the string table with with an empty string.
+  for (auto s: m_strings) {
+    // Make sure all of the offsets match up with what we handed out!
+    assert(m_string_to_offset.find(s)->second ==
+           encoder.GetByteSize() - strtab_offset);
+    // Append the C string into the encoder
+    encoder.AppendCString(s.GetStringRef());
+  }
+  // Fixup the string table length.
+  encoder.PutU32(length_offset, encoder.GetByteSize() - strtab_offset);
+  return true;
+}
+
+bool StringTableReader::Decode(const DataExtractor &data,
+                               lldb::offset_t *offset_ptr) {
+  llvm::StringRef identifier((const char *)data.GetData(offset_ptr, 4), 4);
+  if (identifier != kStringTableIdentifier)
+    return false;
+  const uint32_t length = data.GetU32(offset_ptr);
+  // We always have at least one byte for the empty string at offset zero.
+  if (length == 0)
+    return false;
+  const char *bytes = (const char *)data.GetData(offset_ptr, length);
+  if (bytes == nullptr)
+    return false;
+  m_data = StringRef(bytes, length);
+  return true;
+}
+
+StringRef StringTableReader::Get(uint32_t offset) const {
+  if (offset >= m_data.size())
+    return StringRef();
+  return StringRef(m_data.data() + offset);
+}
diff --git a/lldb/source/Core/IOHandlerCursesGUI.cpp b/lldb/source/Core/IOHandlerCursesGUI.cpp
index 9122117ef5ff..60207f75b7df 100644
--- a/lldb/source/Core/IOHandlerCursesGUI.cpp
+++ b/lldb/source/Core/IOHandlerCursesGUI.cpp
@@ -1034,7 +1034,7 @@ public:
   // navigates to the next or the previous field. This is particularly useful to
   // do in-field validation and error setting. Fields with internal navigation
   // should call this method on their fields.
-  virtual void FieldDelegateExitCallback() { return; }
+  virtual void FieldDelegateExitCallback() {}
 
   // Fields may have internal navigation, for instance, a List Field have
   // multiple internal elements, which needs to be navigated. To allow for this
@@ -1055,10 +1055,10 @@ public:
   virtual bool FieldDelegateOnLastOrOnlyElement() { return true; }
 
   // Select the first element in the field if multiple elements exists.
-  virtual void FieldDelegateSelectFirstElement() { return; }
+  virtual void FieldDelegateSelectFirstElement() {}
 
   // Select the last element in the field if multiple elements exists.
-  virtual void FieldDelegateSelectLastElement() { return; }
+  virtual void FieldDelegateSelectLastElement() {}
 
   // Returns true if the field has an error, false otherwise.
   virtual bool FieldDelegateHasError() { return false; }
@@ -2000,7 +2000,6 @@ public:
 
   void FieldDelegateSelectLastElement() override {
     m_selection_type = SelectionType::NewButton;
-    return;
   }
 
   int GetNumberOfFields() { return m_fields.size(); }
@@ -2292,7 +2291,7 @@ public:
 
   virtual std::string GetName() = 0;
 
-  virtual void UpdateFieldsVisibility() { return; }
+  virtual void UpdateFieldsVisibility() {}
 
   FieldDelegate *GetField(uint32_t field_index) {
     if (field_index < m_fields.size())
@@ -3765,13 +3764,11 @@ public:
   void SelectNext() {
     if (m_selected_match != m_delegate_sp->GetNumberOfMatches() - 1)
       m_selected_match++;
-    return;
   }
 
   void SelectPrevious() {
     if (m_selected_match != 0)
       m_selected_match--;
-    return;
   }
 
   void ExecuteCallback(Window &window) {
@@ -4608,9 +4605,7 @@ public:
   virtual void TreeDelegateDrawTreeItem(TreeItem &item, Window &window) = 0;
   virtual void TreeDelegateGenerateChildren(TreeItem &item) = 0;
   virtual void TreeDelegateUpdateSelection(TreeItem &root, int &selection_index,
-                                           TreeItem *&selected_item) {
-    return;
-  }
+                                           TreeItem *&selected_item) {}
   // This is invoked when a tree item is selected. If true is returned, the
   // views are updated.
   virtual bool TreeDelegateItemSelected(TreeItem &item) = 0;
diff --git a/lldb/source/Core/Mangled.cpp b/lldb/source/Core/Mangled.cpp
index 20f4dbdb419f..c8aacdefefa2 100644
--- a/lldb/source/Core/Mangled.cpp
+++ b/lldb/source/Core/Mangled.cpp
@@ -8,9 +8,11 @@
 
 #include "lldb/Core/Mangled.h"
 
+#include "lldb/Core/DataFileCache.h"
 #include "lldb/Core/RichManglingContext.h"
 #include "lldb/Target/Language.h"
 #include "lldb/Utility/ConstString.h"
+#include "lldb/Utility/DataEncoder.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/Logging.h"
 #include "lldb/Utility/RegularExpression.h"
@@ -411,3 +413,111 @@ Stream &operator<<(Stream &s, const Mangled &obj) {
     s << ", demangled = <error>";
   return s;
 }
+
+// When encoding Mangled objects we can get away with encoding as little
+// information as is required. The enumeration below helps us to efficiently
+// encode Mangled objects.
+enum MangledEncoding {
+  /// If the Mangled object has neither a mangled name or demangled name we can
+  /// encode the object with one zero byte using the Empty enumeration.
+  Empty = 0u,
+  /// If the Mangled object has only a demangled name and no mangled named, we
+  /// can encode only the demangled name.
+  DemangledOnly = 1u,
+  /// If the mangle name can calculate the demangled name (it is the
+  /// mangled/demangled counterpart), then we only need to encode the mangled
+  /// name as the demangled name can be recomputed.
+  MangledOnly = 2u,
+  /// If we have a Mangled object with two different names that are not related
+  /// then we need to save both strings. This can happen if we have a name that
+  /// isn't a true mangled name, but we want to be able to lookup a symbol by
+  /// name and type in the symbol table. We do this for Objective C symbols like
+  /// "OBJC_CLASS_$_NSValue" where the mangled named will be set to
+  /// "OBJC_CLASS_$_NSValue" and the demangled name will be manually set to
+  /// "NSValue". If we tried to demangled the name "OBJC_CLASS_$_NSValue" it
+  /// would fail, but in these cases we want these unrelated names to be
+  /// preserved.
+  MangledAndDemangled = 3u
+};
+
+bool Mangled::Decode(const DataExtractor &data, lldb::offset_t *offset_ptr,
+                     const StringTableReader &strtab) {
+  m_mangled.Clear();
+  m_demangled.Clear();
+  MangledEncoding encoding = (MangledEncoding)data.GetU8(offset_ptr);
+  switch (encoding) {
+    case Empty:
+      return true;
+
+    case DemangledOnly:
+      m_demangled.SetString(strtab.Get(data.GetU32(offset_ptr)));
+      return true;
+
+    case MangledOnly:
+      m_mangled.SetString(strtab.Get(data.GetU32(offset_ptr)));
+      return true;
+
+    case MangledAndDemangled:
+      m_mangled.SetString(strtab.Get(data.GetU32(offset_ptr)));
+      m_demangled.SetString(strtab.Get(data.GetU32(offset_ptr)));
+      return true;
+  }
+  return false;
+}
+/// The encoding format for the Mangled object is as follows:
+///
+/// uint8_t encoding;
+/// char str1[]; (only if DemangledOnly, MangledOnly)
+/// char str2[]; (only if MangledAndDemangled)
+///
+/// The strings are stored as NULL terminated UTF8 strings and str1 and str2
+/// are only saved if we need them based on the encoding.
+///
+/// Some mangled names have a mangled name that can be demangled by the built
+/// in demanglers. These kinds of mangled objects know when the mangled and
+/// demangled names are the counterparts for each other. This is done because
+/// demangling is very expensive and avoiding demangling the same name twice
+/// saves us a lot of compute time. For these kinds of names we only need to
+/// save the mangled name and have the encoding set to "MangledOnly".
+///
+/// If a mangled obejct has only a demangled name, then we save only that string
+/// and have the encoding set to "DemangledOnly".
+///
+/// Some mangled objects have both mangled and demangled names, but the
+/// demangled name can not be computed from the mangled name. This is often used
+/// for runtime named, like Objective C runtime V2 and V3 names. Both these
+/// names must be saved and the encoding is set to "MangledAndDemangled".
+///
+/// For a Mangled object with no names, we only need to set the encoding to
+/// "Empty" and not store any string values.
+void Mangled::Encode(DataEncoder &file, ConstStringTable &strtab) const {
+  MangledEncoding encoding = Empty;
+  if (m_mangled) {
+    encoding = MangledOnly;
+    if (m_demangled) {
+      // We have both mangled and demangled names. If the demangled name is the
+      // counterpart of the mangled name, then we only need to save the mangled
+      // named. If they are different, we need to save both.
+      ConstString s;
+      if (!(m_mangled.GetMangledCounterpart(s) && s == m_demangled))
+        encoding = MangledAndDemangled;
+    }
+  } else if (m_demangled) {
+    encoding = DemangledOnly;
+  }
+  file.AppendU8(encoding);
+  switch (encoding) {
+    case Empty:
+      break;
+    case DemangledOnly:
+      file.AppendU32(strtab.Add(m_demangled));
+      break;
+    case MangledOnly:
+      file.AppendU32(strtab.Add(m_mangled));
+      break;
+    case MangledAndDemangled:
+      file.AppendU32(strtab.Add(m_mangled));
+      file.AppendU32(strtab.Add(m_demangled));
+      break;
+  }
+}
diff --git a/lldb/source/Core/Module.cpp b/lldb/source/Core/Module.cpp
index cbecbb9aa5fe..feecf984f7bf 100644
--- a/lldb/source/Core/Module.cpp
+++ b/lldb/source/Core/Module.cpp
@@ -10,6 +10,7 @@
 
 #include "lldb/Core/AddressRange.h"
 #include "lldb/Core/AddressResolverFileLine.h"
+#include "lldb/Core/DataFileCache.h"
 #include "lldb/Core/Debugger.h"
 #include "lldb/Core/FileSpecList.h"
 #include "lldb/Core/Mangled.h"
@@ -55,7 +56,10 @@
 
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/Compiler.h"
+#include "llvm/Support/DJB.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/JSON.h"
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -233,8 +237,8 @@ Module::Module(const ModuleSpec &module_spec)
 Module::Module(const FileSpec &file_spec, const ArchSpec &arch,
                const ConstString *object_name, lldb::offset_t object_offset,
                const llvm::sys::TimePoint<> &object_mod_time)
-    : m_mod_time(FileSystem::Instance().GetModificationTime(file_spec)), m_arch(arch),
-      m_file(file_spec), m_object_offset(object_offset),
+    : m_mod_time(FileSystem::Instance().GetModificationTime(file_spec)),
+      m_arch(arch), m_file(file_spec), m_object_offset(object_offset),
       m_object_mod_time(object_mod_time), m_file_has_changed(false),
       m_first_file_changed_log(false) {
   // Scope for locker below...
@@ -559,9 +563,8 @@ uint32_t Module::ResolveSymbolContextForAddress(
             // that the symbol has been resolved.
             if (so_addr.GetOffset() ==
                     addr_range.GetBaseAddress().GetOffset() ||
-                so_addr.GetOffset() ==
-                    addr_range.GetBaseAddress().GetOffset() +
-                        addr_range.GetByteSize()) {
+                so_addr.GetOffset() == addr_range.GetBaseAddress().GetOffset() +
+                                           addr_range.GetByteSize()) {
               resolved_flags |= flags;
             }
           } else {
@@ -765,8 +768,7 @@ void Module::LookupInfo::Prune(SymbolContextList &sc_list,
       // pull anything out
       ConstString mangled_name(sc.GetFunctionName(Mangled::ePreferMangled));
       ConstString full_name(sc.GetFunctionName());
-      if (mangled_name != m_name && full_name != m_name)
-      {
+      if (mangled_name != m_name && full_name != m_name) {
         CPlusPlusLanguage::MethodName cpp_method(full_name);
         if (cpp_method.IsValid()) {
           if (cpp_method.GetContext().empty()) {
@@ -940,7 +942,6 @@ void Module::FindTypes_Impl(
     size_t max_matches,
     llvm::DenseSet<lldb_private::SymbolFile *> &searched_symbol_files,
     TypeMap &types) {
-  LLDB_SCOPED_TIMER();
   if (SymbolFile *symbols = GetSymbolFile())
     symbols->FindTypes(name, parent_decl_ctx, max_matches,
                        searched_symbol_files, types);
@@ -960,8 +961,8 @@ void Module::FindTypesInNamespace(ConstString type_name,
   }
 }
 
-lldb::TypeSP Module::FindFirstType(const SymbolContext &sc,
-                                   ConstString name, bool exact_match) {
+lldb::TypeSP Module::FindFirstType(const SymbolContext &sc, ConstString name,
+                                   bool exact_match) {
   TypeList type_list;
   llvm::DenseSet<lldb_private::SymbolFile *> searched_symbol_files;
   FindTypes(name, exact_match, 1, searched_symbol_files, type_list);
@@ -1332,9 +1333,8 @@ void Module::SymbolIndicesToSymbolContextList(
   }
 }
 
-void Module::FindFunctionSymbols(ConstString name,
-                                   uint32_t name_type_mask,
-                                   SymbolContextList &sc_list) {
+void Module::FindFunctionSymbols(ConstString name, uint32_t name_type_mask,
+                                 SymbolContextList &sc_list) {
   LLDB_SCOPED_TIMERF("Module::FindSymbolsFunctions (name = %s, mask = 0x%8.8x)",
                      name.AsCString(), name_type_mask);
   if (Symtab *symtab = GetSymtab())
@@ -1342,13 +1342,10 @@ void Module::FindFunctionSymbols(ConstString name,
 }
 
 void Module::FindSymbolsWithNameAndType(ConstString name,
-                                          SymbolType symbol_type,
-                                          SymbolContextList &sc_list) {
+                                        SymbolType symbol_type,
+                                        SymbolContextList &sc_list) {
   // No need to protect this call using m_mutex all other method calls are
   // already thread safe.
-  LLDB_SCOPED_TIMERF(
-      "Module::FindSymbolsWithNameAndType (name = %s, type = %i)",
-      name.AsCString(), symbol_type);
   if (Symtab *symtab = GetSymtab()) {
     std::vector<uint32_t> symbol_indexes;
     symtab->FindAllSymbolsWithNameAndType(name, symbol_type, symbol_indexes);
@@ -1608,14 +1605,16 @@ bool Module::FindSourceFile(const FileSpec &orig_spec,
   return false;
 }
 
-llvm::Optional<std::string> Module::RemapSourceFile(llvm::StringRef path) const {
+llvm::Optional<std::string>
+Module::RemapSourceFile(llvm::StringRef path) const {
   std::lock_guard<std::recursive_mutex> guard(m_mutex);
   if (auto remapped = m_source_mappings.RemapPath(path))
     return remapped->GetPath();
   return {};
 }
 
-void Module::RegisterXcodeSDK(llvm::StringRef sdk_name, llvm::StringRef sysroot) {
+void Module::RegisterXcodeSDK(llvm::StringRef sdk_name,
+                              llvm::StringRef sysroot) {
   XcodeSDK sdk(sdk_name.str());
   llvm::StringRef sdk_path(HostInfo::GetXcodeSDKPath(sdk));
   if (sdk_path.empty())
@@ -1661,3 +1660,36 @@ bool Module::GetIsDynamicLinkEditor() {
 
   return false;
 }
+
+uint32_t Module::Hash() {
+  std::string identifier;
+  llvm::raw_string_ostream id_strm(identifier);
+  id_strm << m_arch.GetTriple().str() << '-' << m_file.GetPath();
+  if (m_object_name)
+    id_strm << '(' << m_object_name.GetStringRef() << ')';
+  if (m_object_offset > 0)
+    id_strm << m_object_offset;
+  const auto mtime = llvm::sys::toTimeT(m_object_mod_time);
+  if (mtime > 0)
+    id_strm << mtime;
+  return llvm::djbHash(id_strm.str());
+}
+
+std::string Module::GetCacheKey() {
+  std::string key;
+  llvm::raw_string_ostream strm(key);
+  strm << m_arch.GetTriple().str() << '-' << m_file.GetFilename();
+  if (m_object_name)
+    strm << '(' << m_object_name.GetStringRef() << ')';
+  strm << '-' << llvm::format_hex(Hash(), 10);
+  return strm.str();
+}
+
+DataFileCache *Module::GetIndexCache() {
+  if (!ModuleList::GetGlobalModuleListProperties().GetEnableLLDBIndexCache())
+    return nullptr;
+  // NOTE: intentional leak so we don't crash if global destructor chain gets
+  // called as other threads still use the result of this function
+  static DataFileCache *g_data_file_cache = new DataFileCache(ModuleList::GetGlobalModuleListProperties().GetLLDBIndexCachePath().GetPath());
+  return g_data_file_cache;
+}
diff --git a/lldb/source/Core/ModuleList.cpp b/lldb/source/Core/ModuleList.cpp
index 9176c9dbb357..48412137546d 100644
--- a/lldb/source/Core/ModuleList.cpp
+++ b/lldb/source/Core/ModuleList.cpp
@@ -85,6 +85,14 @@ ModuleListProperties::ModuleListProperties() {
   if (clang::driver::Driver::getDefaultModuleCachePath(path)) {
     lldbassert(SetClangModulesCachePath(FileSpec(path)));
   }
+
+  path.clear();
+  if (llvm::sys::path::cache_directory(path)) {
+    llvm::sys::path::append(path, "lldb");
+    llvm::sys::path::append(path, "IndexCache");
+    lldbassert(SetLLDBIndexCachePath(FileSpec(path)));
+  }
+
 }
 
 bool ModuleListProperties::GetEnableExternalLookup() const {
@@ -110,6 +118,47 @@ bool ModuleListProperties::SetClangModulesCachePath(const FileSpec &path) {
       nullptr, ePropertyClangModulesCachePath, path);
 }
 
+FileSpec ModuleListProperties::GetLLDBIndexCachePath() const {
+  return m_collection_sp
+      ->GetPropertyAtIndexAsOptionValueFileSpec(nullptr, false,
+                                                ePropertyLLDBIndexCachePath)
+      ->GetCurrentValue();
+}
+
+bool ModuleListProperties::SetLLDBIndexCachePath(const FileSpec &path) {
+  return m_collection_sp->SetPropertyAtIndexAsFileSpec(
+      nullptr, ePropertyLLDBIndexCachePath, path);
+}
+
+bool ModuleListProperties::GetEnableLLDBIndexCache() const {
+  const uint32_t idx = ePropertyEnableLLDBIndexCache;
+  return m_collection_sp->GetPropertyAtIndexAsBoolean(
+      nullptr, idx, g_modulelist_properties[idx].default_uint_value != 0);
+}
+
+bool ModuleListProperties::SetEnableLLDBIndexCache(bool new_value) {
+  return m_collection_sp->SetPropertyAtIndexAsBoolean(
+      nullptr, ePropertyEnableLLDBIndexCache, new_value);
+}
+
+uint64_t ModuleListProperties::GetLLDBIndexCacheMaxByteSize() {
+  const uint32_t idx = ePropertyLLDBIndexCacheMaxByteSize;
+  return m_collection_sp->GetPropertyAtIndexAsUInt64(
+      nullptr, idx, g_modulelist_properties[idx].default_uint_value);
+}
+
+uint64_t ModuleListProperties::GetLLDBIndexCacheMaxPercent() {
+  const uint32_t idx = ePropertyLLDBIndexCacheMaxPercent;
+  return m_collection_sp->GetPropertyAtIndexAsUInt64(
+      nullptr, idx, g_modulelist_properties[idx].default_uint_value);
+}
+
+uint64_t ModuleListProperties::GetLLDBIndexCacheExpirationDays() {
+  const uint32_t idx = ePropertyLLDBIndexCacheExpirationDays;
+  return m_collection_sp->GetPropertyAtIndexAsUInt64(
+      nullptr, idx, g_modulelist_properties[idx].default_uint_value);
+}
+
 void ModuleListProperties::UpdateSymlinkMappings() {
   FileSpecList list = m_collection_sp
                           ->GetPropertyAtIndexAsOptionValueFileSpecList(
diff --git a/lldb/source/Core/PluginManager.cpp b/lldb/source/Core/PluginManager.cpp
index 801591129244..37050494aa2e 100644
--- a/lldb/source/Core/PluginManager.cpp
+++ b/lldb/source/Core/PluginManager.cpp
@@ -1345,6 +1345,12 @@ REPLCreateInstance PluginManager::GetREPLCreateCallbackAtIndex(uint32_t idx) {
   return GetREPLInstances().GetCallbackAtIndex(idx);
 }
 
+LanguageSet PluginManager::GetREPLSupportedLanguagesAtIndex(uint32_t idx) {
+  const auto &instances = GetREPLInstances().GetInstances();
+  return idx < instances.size() ? instances[idx].supported_languages
+                                : LanguageSet();
+}
+
 LanguageSet PluginManager::GetREPLAllTypeSystemSupportedLanguages() {
   const auto &instances = GetREPLInstances().GetInstances();
   LanguageSet all;
diff --git a/lldb/source/DataFormatters/CXXFunctionPointer.cpp b/lldb/source/DataFormatters/CXXFunctionPointer.cpp
index 3b7b0bc27cf8..d7df280e56ef 100644
--- a/lldb/source/DataFormatters/CXXFunctionPointer.cpp
+++ b/lldb/source/DataFormatters/CXXFunctionPointer.cpp
@@ -9,6 +9,7 @@
 #include "lldb/DataFormatters/CXXFunctionPointer.h"
 
 #include "lldb/Core/ValueObject.h"
+#include "lldb/Target/ABI.h"
 #include "lldb/Target/SectionLoadList.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Utility/Stream.h"
@@ -38,8 +39,34 @@ bool lldb_private::formatters::CXXFunctionPointerSummaryProvider(
       Address so_addr;
       Target *target = exe_ctx.GetTargetPtr();
       if (target && !target->GetSectionLoadList().IsEmpty()) {
-        if (target->GetSectionLoadList().ResolveLoadAddress(func_ptr_address,
-                                                            so_addr)) {
+        target->GetSectionLoadList().ResolveLoadAddress(func_ptr_address,
+                                                        so_addr);
+        if (so_addr.GetSection() == nullptr) {
+          // If we have an address that doesn't correspond to any symbol,
+          // it might have authentication bits.  Strip them & see if it
+          // now points to a symbol -- if so, do the SymbolContext lookup
+          // based on the stripped address.
+          // If we find a symbol with the ptrauth bits stripped, print the
+          // raw value into the stream, and replace the Address with the
+          // one that points to a symbol for a fuller description.
+          if (Process *process = exe_ctx.GetProcessPtr()) {
+            if (ABISP abi_sp = process->GetABI()) {
+              addr_t fixed_addr = abi_sp->FixCodeAddress(func_ptr_address);
+              if (fixed_addr != func_ptr_address) {
+                Address test_address;
+                test_address.SetLoadAddress(fixed_addr, target);
+                if (test_address.GetSection() != nullptr) {
+                  int addrsize = target->GetArchitecture().GetAddressByteSize();
+                  sstr.Printf("actual=0x%*.*" PRIx64 " ", addrsize * 2,
+                              addrsize * 2, fixed_addr);
+                  so_addr = test_address;
+                }
+              }
+            }
+          }
+        }
+
+        if (so_addr.IsValid()) {
           so_addr.Dump(&sstr, exe_ctx.GetBestExecutionContextScope(),
                        Address::DumpStyleResolvedDescription,
                        Address::DumpStyleSectionNameOffset);
diff --git a/lldb/source/DataFormatters/FormatManager.cpp b/lldb/source/DataFormatters/FormatManager.cpp
index cda1ae60d857..0ef5f0adc832 100644
--- a/lldb/source/DataFormatters/FormatManager.cpp
+++ b/lldb/source/DataFormatters/FormatManager.cpp
@@ -724,15 +724,14 @@ void FormatManager::LoadSystemFormatters() {
   lldb::TypeSummaryImplSP string_array_format(
       new StringSummaryFormat(string_array_flags, "${var%char[]}"));
 
-  RegularExpression any_size_char_arr(llvm::StringRef("char ?\\[[0-9]+\\]"));
+  RegularExpression any_size_char_arr(R"(^((un)?signed )?char ?\[[0-9]+\]$)");
 
   TypeCategoryImpl::SharedPointer sys_category_sp =
       GetCategory(m_system_category_name);
 
-  sys_category_sp->GetTypeSummariesContainer()->Add(ConstString("char *"),
-                                                    string_format);
-  sys_category_sp->GetTypeSummariesContainer()->Add(
-      ConstString("unsigned char *"), string_format);
+  sys_category_sp->GetRegexTypeSummariesContainer()->Add(
+      RegularExpression(R"(^((un)?signed )?char ?(\*|\[\])$)"), string_format);
+
   sys_category_sp->GetRegexTypeSummariesContainer()->Add(
       std::move(any_size_char_arr), string_array_format);
 
diff --git a/lldb/source/Expression/DWARFExpression.cpp b/lldb/source/Expression/DWARFExpression.cpp
index a10546c1deae..890d8b5d3107 100644
--- a/lldb/source/Expression/DWARFExpression.cpp
+++ b/lldb/source/Expression/DWARFExpression.cpp
@@ -460,22 +460,19 @@ bool DWARFExpression::Update_DW_OP_addr(lldb::addr_t file_addr) {
       // first, then modify it, and if all goes well, we then replace the data
       // for this expression
 
-      // So first we copy the data into a heap buffer
-      std::unique_ptr<DataBufferHeap> head_data_up(
-          new DataBufferHeap(m_data.GetDataStart(), m_data.GetByteSize()));
-
-      // Make en encoder so we can write the address into the buffer using the
-      // correct byte order (endianness)
-      DataEncoder encoder(head_data_up->GetBytes(), head_data_up->GetByteSize(),
+      // Make en encoder that contains a copy of the location expression data
+      // so we can write the address into the buffer using the correct byte
+      // order.
+      DataEncoder encoder(m_data.GetDataStart(), m_data.GetByteSize(),
                           m_data.GetByteOrder(), addr_byte_size);
 
       // Replace the address in the new buffer
-      if (encoder.PutUnsigned(offset, addr_byte_size, file_addr) == UINT32_MAX)
+      if (encoder.PutAddress(offset, file_addr) == UINT32_MAX)
         return false;
 
       // All went well, so now we can reset the data using a shared pointer to
       // the heap data so "m_data" will now correctly manage the heap data.
-      m_data.SetData(DataBufferSP(head_data_up.release()));
+      m_data.SetData(encoder.GetDataBuffer());
       return true;
     } else {
       const offset_t op_arg_size = GetOpcodeDataSize(m_data, offset, op);
@@ -521,15 +518,11 @@ bool DWARFExpression::LinkThreadLocalStorage(
   // We have to make a copy of the data as we don't know if this data is from a
   // read only memory mapped buffer, so we duplicate all of the data first,
   // then modify it, and if all goes well, we then replace the data for this
-  // expression
-
-  // So first we copy the data into a heap buffer
-  std::shared_ptr<DataBufferHeap> heap_data_sp(
-      new DataBufferHeap(m_data.GetDataStart(), m_data.GetByteSize()));
+  // expression.
 
-  // Make en encoder so we can write the address into the buffer using the
-  // correct byte order (endianness)
-  DataEncoder encoder(heap_data_sp->GetBytes(), heap_data_sp->GetByteSize(),
+  // Make en encoder that contains a copy of the location expression data so we
+  // can write the address into the buffer using the correct byte order.
+  DataEncoder encoder(m_data.GetDataStart(), m_data.GetByteSize(),
                       m_data.GetByteOrder(), addr_byte_size);
 
   lldb::offset_t offset = 0;
@@ -603,7 +596,7 @@ bool DWARFExpression::LinkThreadLocalStorage(
   // and read the
   // TLS data
   m_module_wp = new_module_sp;
-  m_data.SetData(heap_data_sp);
+  m_data.SetData(encoder.GetDataBuffer());
   return true;
 }
 
@@ -2817,4 +2810,3 @@ bool DWARFExpression::MatchesOperand(StackFrame &frame,
     return MatchRegOp(*reg)(operand);
   }
 }
-
diff --git a/lldb/source/Expression/IRExecutionUnit.cpp b/lldb/source/Expression/IRExecutionUnit.cpp
index f2d22f7ed9cc..3c4a87c27e20 100644
--- a/lldb/source/Expression/IRExecutionUnit.cpp
+++ b/lldb/source/Expression/IRExecutionUnit.cpp
@@ -483,8 +483,6 @@ void IRExecutionUnit::GetRunnableInfo(Status &error, lldb::addr_t &func_addr,
 
   func_addr = m_function_load_addr;
   func_end = m_function_end_load_addr;
-
-  return;
 }
 
 IRExecutionUnit::~IRExecutionUnit() {
diff --git a/lldb/source/Expression/IRMemoryMap.cpp b/lldb/source/Expression/IRMemoryMap.cpp
index 4ae2724d4dd8..9eee5cf5b9a2 100644
--- a/lldb/source/Expression/IRMemoryMap.cpp
+++ b/lldb/source/Expression/IRMemoryMap.cpp
@@ -609,7 +609,6 @@ void IRMemoryMap::WriteScalarToMemory(lldb::addr_t process_address,
     error.SetErrorToGenericError();
     error.SetErrorString("Couldn't write scalar: its size was zero");
   }
-  return;
 }
 
 void IRMemoryMap::WritePointerToMemory(lldb::addr_t process_address,
@@ -757,7 +756,6 @@ void IRMemoryMap::ReadScalarFromMemory(Scalar &scalar,
     error.SetErrorToGenericError();
     error.SetErrorString("Couldn't read scalar: its size was zero");
   }
-  return;
 }
 
 void IRMemoryMap::ReadPointerFromMemory(lldb::addr_t *address,
@@ -773,8 +771,6 @@ void IRMemoryMap::ReadPointerFromMemory(lldb::addr_t *address,
     return;
 
   *address = pointer_scalar.ULongLong();
-
-  return;
 }
 
 void IRMemoryMap::GetMemoryData(DataExtractor &extractor,
diff --git a/lldb/source/Expression/REPL.cpp b/lldb/source/Expression/REPL.cpp
index 9cd6129eedd7..be41c60ebb5f 100644
--- a/lldb/source/Expression/REPL.cpp
+++ b/lldb/source/Expression/REPL.cpp
@@ -39,7 +39,11 @@ lldb::REPLSP REPL::Create(Status &err, lldb::LanguageType language,
   lldb::REPLSP ret;
 
   while (REPLCreateInstance create_instance =
-             PluginManager::GetREPLCreateCallbackAtIndex(idx++)) {
+             PluginManager::GetREPLCreateCallbackAtIndex(idx)) {
+    LanguageSet supported_languages =
+        PluginManager::GetREPLSupportedLanguagesAtIndex(idx++);
+    if (!supported_languages[language])
+      continue;
     ret = (*create_instance)(err, language, debugger, target, repl_options);
     if (ret) {
       break;
diff --git a/lldb/source/Expression/UserExpression.cpp b/lldb/source/Expression/UserExpression.cpp
index b61781c0b82b..692594b03f16 100644
--- a/lldb/source/Expression/UserExpression.cpp
+++ b/lldb/source/Expression/UserExpression.cpp
@@ -254,9 +254,7 @@ UserExpression::Evaluate(ExecutionContext &exe_ctx,
   if (fixed_expression == nullptr)
     fixed_expression = &tmp_fixed_expression;
 
-  const char *fixed_text = user_expression_sp->GetFixedText();
-  if (fixed_text != nullptr)
-    fixed_expression->append(fixed_text);
+  *fixed_expression = user_expression_sp->GetFixedText().str();
 
   // If there is a fixed expression, try to parse it:
   if (!parse_success) {
@@ -265,8 +263,7 @@ UserExpression::Evaluate(ExecutionContext &exe_ctx,
     user_expression_sp.reset();
 
     execution_results = lldb::eExpressionParseError;
-    if (fixed_expression && !fixed_expression->empty() &&
-        options.GetAutoApplyFixIts()) {
+    if (!fixed_expression->empty() && options.GetAutoApplyFixIts()) {
       const uint64_t max_fix_retries = options.GetRetriesWithFixIts();
       for (uint64_t i = 0; i < max_fix_retries; ++i) {
         // Try parsing the fixed expression.
@@ -285,8 +282,8 @@ UserExpression::Evaluate(ExecutionContext &exe_ctx,
         } else {
           // The fixed expression also didn't parse. Let's check for any new
           // Fix-Its we could try.
-          if (fixed_expression_sp->GetFixedText()) {
-            *fixed_expression = fixed_expression_sp->GetFixedText();
+          if (!fixed_expression_sp->GetFixedText().empty()) {
+            *fixed_expression = fixed_expression_sp->GetFixedText().str();
           } else {
             // Fixed expression didn't compile without a fixit, don't retry and
             // don't tell the user about it.
diff --git a/lldb/source/Host/common/FileSystem.cpp b/lldb/source/Host/common/FileSystem.cpp
index 7687ad6c20a6..1e4a24abe301 100644
--- a/lldb/source/Host/common/FileSystem.cpp
+++ b/lldb/source/Host/common/FileSystem.cpp
@@ -513,3 +513,11 @@ void FileSystem::Collect(const llvm::Twine &file) {
 void FileSystem::SetHomeDirectory(std::string home_directory) {
   m_home_directory = std::move(home_directory);
 }
+
+Status FileSystem::RemoveFile(const FileSpec &file_spec) {
+  return RemoveFile(file_spec.GetPath());
+}
+
+Status FileSystem::RemoveFile(const llvm::Twine &path) {
+  return Status(llvm::sys::fs::remove(path));
+}
diff --git a/lldb/source/Host/common/ProcessLaunchInfo.cpp b/lldb/source/Host/common/ProcessLaunchInfo.cpp
index 8d281b80e8f0..07e4f15c9e9e 100644
--- a/lldb/source/Host/common/ProcessLaunchInfo.cpp
+++ b/lldb/source/Host/common/ProcessLaunchInfo.cpp
@@ -212,6 +212,14 @@ void ProcessLaunchInfo::SetDetachOnError(bool enable) {
 
 llvm::Error ProcessLaunchInfo::SetUpPtyRedirection() {
   Log *log = GetLogIfAllCategoriesSet(LIBLLDB_LOG_PROCESS);
+
+  bool stdin_free = GetFileActionForFD(STDIN_FILENO) == nullptr;
+  bool stdout_free = GetFileActionForFD(STDOUT_FILENO) == nullptr;
+  bool stderr_free = GetFileActionForFD(STDERR_FILENO) == nullptr;
+  bool any_free = stdin_free || stdout_free || stderr_free;
+  if (!any_free)
+    return llvm::Error::success();
+
   LLDB_LOG(log, "Generating a pty to use for stdin/out/err");
 
   int open_flags = O_RDWR | O_NOCTTY;
@@ -226,19 +234,13 @@ llvm::Error ProcessLaunchInfo::SetUpPtyRedirection() {
 
   const FileSpec secondary_file_spec(m_pty->GetSecondaryName());
 
-  // Only use the secondary tty if we don't have anything specified for
-  // input and don't have an action for stdin
-  if (GetFileActionForFD(STDIN_FILENO) == nullptr)
+  if (stdin_free)
     AppendOpenFileAction(STDIN_FILENO, secondary_file_spec, true, false);
 
-  // Only use the secondary tty if we don't have anything specified for
-  // output and don't have an action for stdout
-  if (GetFileActionForFD(STDOUT_FILENO) == nullptr)
+  if (stdout_free)
     AppendOpenFileAction(STDOUT_FILENO, secondary_file_spec, false, true);
 
-  // Only use the secondary tty if we don't have anything specified for
-  // error and don't have an action for stderr
-  if (GetFileActionForFD(STDERR_FILENO) == nullptr)
+  if (stderr_free)
     AppendOpenFileAction(STDERR_FILENO, secondary_file_spec, false, true);
   return llvm::Error::success();
 }
diff --git a/lldb/source/Host/posix/ProcessLauncherPosixFork.cpp b/lldb/source/Host/posix/ProcessLauncherPosixFork.cpp
index 63178e6c8a7a..2f08b9fa8857 100644
--- a/lldb/source/Host/posix/ProcessLauncherPosixFork.cpp
+++ b/lldb/source/Host/posix/ProcessLauncherPosixFork.cpp
@@ -86,7 +86,6 @@ static void DupDescriptor(int error_fd, const FileSpec &file_spec, int fd,
     ExitWithError(error_fd, "DupDescriptor-dup2");
 
   ::close(target_fd);
-  return;
 }
 
 [[noreturn]] static void ChildFunc(int error_fd,
diff --git a/lldb/source/Initialization/SystemInitializerCommon.cpp b/lldb/source/Initialization/SystemInitializerCommon.cpp
index 84c5a472332a..1c8406f68784 100644
--- a/lldb/source/Initialization/SystemInitializerCommon.cpp
+++ b/lldb/source/Initialization/SystemInitializerCommon.cpp
@@ -15,7 +15,7 @@
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/ReproducerProvider.h"
 #include "lldb/Utility/Timer.h"
-#include "lldb/lldb-private.h"
+#include "lldb/Version/Version.h"
 
 #if defined(__linux__) || defined(__FreeBSD__) || defined(__NetBSD__)
 #include "Plugins/Process/POSIX/ProcessPOSIXLog.h"
diff --git a/lldb/source/Interpreter/CommandInterpreter.cpp b/lldb/source/Interpreter/CommandInterpreter.cpp
index 301bf949feef..bd03f18b47c0 100644
--- a/lldb/source/Interpreter/CommandInterpreter.cpp
+++ b/lldb/source/Interpreter/CommandInterpreter.cpp
@@ -2216,7 +2216,6 @@ void CommandInterpreter::BuildAliasCommandArgs(CommandObject *alias_cmd_obj,
   }
 
   result.SetStatus(eReturnStatusSuccessFinishNoResult);
-  return;
 }
 
 int CommandInterpreter::GetOptionArgumentPosition(const char *in_string) {
@@ -2563,8 +2562,6 @@ void CommandInterpreter::HandleCommands(const StringList &commands,
 
   result.SetStatus(eReturnStatusSuccessFinishResult);
   m_debugger.SetAsyncExecution(old_async_execution);
-
-  return;
 }
 
 // Make flags that we can pass into the IOHandler so our delegates can do the
diff --git a/lldb/source/Interpreter/CommandReturnObject.cpp b/lldb/source/Interpreter/CommandReturnObject.cpp
index 506b0d6e7cde..1b1e6996764c 100644
--- a/lldb/source/Interpreter/CommandReturnObject.cpp
+++ b/lldb/source/Interpreter/CommandReturnObject.cpp
@@ -120,13 +120,13 @@ void CommandReturnObject::AppendRawError(llvm::StringRef in_string) {
 
 void CommandReturnObject::SetStatus(ReturnStatus status) { m_status = status; }
 
-ReturnStatus CommandReturnObject::GetStatus() { return m_status; }
+ReturnStatus CommandReturnObject::GetStatus() const { return m_status; }
 
-bool CommandReturnObject::Succeeded() {
+bool CommandReturnObject::Succeeded() const {
   return m_status <= eReturnStatusSuccessContinuingResult;
 }
 
-bool CommandReturnObject::HasResult() {
+bool CommandReturnObject::HasResult() const {
   return (m_status == eReturnStatusSuccessFinishResult ||
           m_status == eReturnStatusSuccessContinuingResult);
 }
@@ -145,7 +145,7 @@ void CommandReturnObject::Clear() {
   m_interactive = true;
 }
 
-bool CommandReturnObject::GetDidChangeProcessState() {
+bool CommandReturnObject::GetDidChangeProcessState() const {
   return m_did_change_process_state;
 }
 
diff --git a/lldb/source/Plugins/ABI/AArch64/ABIMacOSX_arm64.cpp b/lldb/source/Plugins/ABI/AArch64/ABIMacOSX_arm64.cpp
index ccfbeec3d589..804532acf9f7 100644
--- a/lldb/source/Plugins/ABI/AArch64/ABIMacOSX_arm64.cpp
+++ b/lldb/source/Plugins/ABI/AArch64/ABIMacOSX_arm64.cpp
@@ -817,6 +817,16 @@ ValueObjectSP ABIMacOSX_arm64::GetReturnValueObjectImpl(
 
 lldb::addr_t ABIMacOSX_arm64::FixAddress(addr_t pc, addr_t mask) {
   lldb::addr_t pac_sign_extension = 0x0080000000000000ULL;
+  // Darwin systems originally couldn't determine the proper value
+  // dynamically, so the most common value was hardcoded.  This has
+  // largely been cleaned up, but there are still a handful of
+  // environments that assume the default value is set to this value
+  // and there's no dynamic value to correct it.
+  // When no mask is specified, set it to 39 bits of addressing (0..38).
+  if (mask == 0) {
+    // ~((1ULL<<39)-1)
+    mask = 0xffffff8000000000;
+  }
   return (pc & pac_sign_extension) ? pc | mask : pc & (~mask);
 }
 
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ASTResultSynthesizer.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ASTResultSynthesizer.cpp
index 85e2fcfc838c..7844f27139cf 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ASTResultSynthesizer.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ASTResultSynthesizer.cpp
@@ -211,7 +211,7 @@ bool ASTResultSynthesizer::SynthesizeBodyResult(CompoundStmt *Body,
   Stmt **last_stmt_ptr = Body->body_end() - 1;
   Stmt *last_stmt = *last_stmt_ptr;
 
-  while (dyn_cast<NullStmt>(last_stmt)) {
+  while (isa<NullStmt>(last_stmt)) {
     if (last_stmt_ptr != Body->body_begin()) {
       last_stmt_ptr--;
       last_stmt = *last_stmt_ptr;
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTImporter.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTImporter.cpp
index 80469e292580..6ed3cc9384f0 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTImporter.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTImporter.cpp
@@ -293,7 +293,7 @@ public:
 
     NamedDecl *to_named_decl = dyn_cast<NamedDecl>(to);
     // Check if we already completed this type.
-    if (m_decls_already_completed.count(to_named_decl) != 0)
+    if (m_decls_already_completed.contains(to_named_decl))
       return;
     // Queue this type to be completed.
     m_decls_to_complete.insert(to_named_decl);
@@ -806,7 +806,7 @@ void ClangASTImporter::ForgetSource(clang::ASTContext *dst_ast,
   md->removeOriginsWithContext(src_ast);
 }
 
-ClangASTImporter::MapCompleter::~MapCompleter() { return; }
+ClangASTImporter::MapCompleter::~MapCompleter() {}
 
 llvm::Expected<Decl *>
 ClangASTImporter::ASTImporterDelegate::ImportImpl(Decl *From) {
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.cpp
index 410d8a95cb12..510352e8c173 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.cpp
@@ -513,8 +513,6 @@ void ClangASTSource::FindExternalLexicalDecls(
     // is consulted again when a clang::DeclContext::lookup is called.
     const_cast<DeclContext *>(decl_context)->setMustBuildLookupTable();
   }
-
-  return;
 }
 
 void ClangASTSource::FindExternalVisibleDecls(NameSearchContext &context) {
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.h b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.h
index 3afd1fd5f2d1..f3fec3f944a1 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.h
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.h
@@ -59,7 +59,7 @@ public:
   GetExternalCXXBaseSpecifiers(uint64_t Offset) override {
     return nullptr;
   }
-  void MaterializeVisibleDecls(const clang::DeclContext *DC) { return; }
+  void MaterializeVisibleDecls(const clang::DeclContext *DC) {}
 
   void InstallASTContext(TypeSystemClang &ast_context);
 
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.cpp
index 846c1597292b..4af5d41a5921 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.cpp
@@ -1954,8 +1954,6 @@ void ClangExpressionDeclMap::AddContextClassType(NameSearchContext &context,
     return;
 
   context.AddNamedDecl(typedef_decl);
-
-  return;
 }
 
 void ClangExpressionDeclMap::AddOneType(NameSearchContext &context,
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/IRDynamicChecks.cpp b/lldb/source/Plugins/ExpressionParser/Clang/IRDynamicChecks.cpp
index a6e36d81b950..8b132b54b7e6 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/IRDynamicChecks.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/IRDynamicChecks.cpp
@@ -353,7 +353,7 @@ protected:
   }
 
   bool InspectInstruction(llvm::Instruction &i) override {
-    if (dyn_cast<llvm::LoadInst>(&i) || dyn_cast<llvm::StoreInst>(&i))
+    if (isa<llvm::LoadInst>(&i) || isa<llvm::StoreInst>(&i))
       RegisterInstruction(i);
 
     return true;
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/IRForTarget.cpp b/lldb/source/Plugins/ExpressionParser/Clang/IRForTarget.cpp
index f80dc2b14467..e0e41925f7ef 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/IRForTarget.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/IRForTarget.cpp
@@ -1255,7 +1255,7 @@ bool IRForTarget::MaybeHandleVariable(Value *llvm_value_ptr) {
       m_decl_map->AddValueToStruct(named_decl, lldb_private::ConstString(name),
                                    llvm_value_ptr, *value_size,
                                    value_alignment);
-  } else if (dyn_cast<llvm::Function>(llvm_value_ptr)) {
+  } else if (isa<llvm::Function>(llvm_value_ptr)) {
     LLDB_LOG(log, "Function pointers aren't handled right now");
 
     return false;
diff --git a/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp b/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp
index f1925990e94a..df61cc3853eb 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp
@@ -35,6 +35,7 @@
 #include "BlockPointer.h"
 #include "CPlusPlusNameParser.h"
 #include "CxxStringTypes.h"
+#include "Generic.h"
 #include "LibCxx.h"
 #include "LibCxxAtomic.h"
 #include "LibCxxVariant.h"
@@ -655,7 +656,7 @@ static void LoadLibCxxFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
                   "libc++ std::tuple synthetic children",
                   ConstString("^std::__[[:alnum:]]+::tuple<.*>(( )?&)?$"),
                   stl_synth_flags, true);
-  AddCXXSynthetic(cpp_category_sp, LibcxxOptionalFrontEndCreator,
+  AddCXXSynthetic(cpp_category_sp, LibcxxOptionalSyntheticFrontEndCreator,
                   "libc++ std::optional synthetic children",
                   ConstString("^std::__[[:alnum:]]+::optional<.+>(( )?&)?$"),
                   stl_synth_flags, true);
@@ -772,7 +773,7 @@ static void LoadLibCxxFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
                 ConstString("^std::__[[:alnum:]]+::atomic<.+>$"),
                 stl_summary_flags, true);
   AddCXXSummary(cpp_category_sp,
-                lldb_private::formatters::LibcxxOptionalSummaryProvider,
+                lldb_private::formatters::GenericOptionalSummaryProvider,
                 "libc++ std::optional summary provider",
                 ConstString("^std::__[[:alnum:]]+::optional<.+>(( )?&)?$"),
                 stl_summary_flags, true);
@@ -904,6 +905,11 @@ static void LoadLibStdcppFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
           stl_synth_flags,
           "lldb.formatters.cpp.gnu_libstdcpp.StdMapLikeSynthProvider")));
   cpp_category_sp->GetRegexTypeSyntheticsContainer()->Add(
+      RegularExpression("^std::deque<.+>(( )?&)?$"),
+      SyntheticChildrenSP(new ScriptedSyntheticChildren(
+          stl_deref_flags,
+          "lldb.formatters.cpp.gnu_libstdcpp.StdDequeSynthProvider")));
+  cpp_category_sp->GetRegexTypeSyntheticsContainer()->Add(
       RegularExpression("^std::set<.+> >(( )?&)?$"),
       SyntheticChildrenSP(new ScriptedSyntheticChildren(
           stl_deref_flags,
@@ -914,11 +920,6 @@ static void LoadLibStdcppFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
           stl_deref_flags,
           "lldb.formatters.cpp.gnu_libstdcpp.StdMapLikeSynthProvider")));
   cpp_category_sp->GetRegexTypeSyntheticsContainer()->Add(
-      RegularExpression("^std::optional<.+>(( )?&)?$"),
-      SyntheticChildrenSP(new ScriptedSyntheticChildren(
-          stl_synth_flags,
-          "lldb.formatters.cpp.gnu_libstdcpp.StdOptionalSynthProvider")));
-  cpp_category_sp->GetRegexTypeSyntheticsContainer()->Add(
       RegularExpression("^std::multiset<.+> >(( )?&)?$"),
       SyntheticChildrenSP(new ScriptedSyntheticChildren(
           stl_deref_flags,
@@ -931,7 +932,7 @@ static void LoadLibStdcppFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
   cpp_category_sp->GetRegexTypeSyntheticsContainer()->Add(
       RegularExpression("^std::(__cxx11::)?list<.+>(( )?&)?$"),
       SyntheticChildrenSP(new ScriptedSyntheticChildren(
-          stl_synth_flags,
+          stl_deref_flags,
           "lldb.formatters.cpp.gnu_libstdcpp.StdListSynthProvider")));
   cpp_category_sp->GetRegexTypeSyntheticsContainer()->Add(
       RegularExpression("^std::(__cxx11::)?forward_list<.+>(( )?&)?$"),
@@ -942,11 +943,6 @@ static void LoadLibStdcppFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
   stl_summary_flags.SetDontShowChildren(false);
   stl_summary_flags.SetSkipPointers(false);
   cpp_category_sp->GetRegexTypeSummariesContainer()->Add(
-      RegularExpression("^std::optional<.+>(( )?&)?$"),
-      TypeSummaryImplSP(new ScriptSummaryFormat(
-          stl_summary_flags,
-          "lldb.formatters.cpp.gnu_libstdcpp.StdOptionalSummaryProvider")));
-  cpp_category_sp->GetRegexTypeSummariesContainer()->Add(
       RegularExpression("^std::bitset<.+>(( )?&)?$"),
       TypeSummaryImplSP(
           new StringSummaryFormat(stl_summary_flags, "size=${svar%#}")));
@@ -963,6 +959,10 @@ static void LoadLibStdcppFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
       TypeSummaryImplSP(
           new StringSummaryFormat(stl_summary_flags, "size=${svar%#}")));
   cpp_category_sp->GetRegexTypeSummariesContainer()->Add(
+      RegularExpression("^std::deque<.+>(( )?&)?$"),
+      TypeSummaryImplSP(
+          new StringSummaryFormat(stl_summary_flags, "size=${svar%#}")));
+  cpp_category_sp->GetRegexTypeSummariesContainer()->Add(
       RegularExpression("^std::multimap<.+> >(( )?&)?$"),
       TypeSummaryImplSP(
           new StringSummaryFormat(stl_summary_flags, "size=${svar%#}")));
@@ -1022,6 +1022,12 @@ static void LoadLibStdcppFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
       "std::bitset synthetic child", ConstString("^std::bitset<.+>(( )?&)?$"),
       stl_deref_flags, true);
 
+  AddCXXSynthetic(
+      cpp_category_sp,
+      lldb_private::formatters::LibStdcppOptionalSyntheticFrontEndCreator,
+      "std::optional synthetic child",
+      ConstString("^std::optional<.+>(( )?&)?$"), stl_deref_flags, true);
+
   AddCXXSummary(cpp_category_sp,
                 lldb_private::formatters::LibStdcppUniquePointerSummaryProvider,
                 "libstdc++ std::unique_ptr summary provider",
@@ -1037,6 +1043,10 @@ static void LoadLibStdcppFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
                 "libstdc++ std::weak_ptr summary provider",
                 ConstString("^std::weak_ptr<.+>(( )?&)?$"), stl_summary_flags,
                 true);
+  AddCXXSummary(
+      cpp_category_sp, lldb_private::formatters::GenericOptionalSummaryProvider,
+      "libstd++ std::optional summary provider",
+      ConstString("^std::optional<.+>(( )?&)?$"), stl_summary_flags, true);
 }
 
 static void LoadSystemFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
diff --git a/lldb/source/Plugins/Language/CPlusPlus/Generic.h b/lldb/source/Plugins/Language/CPlusPlus/Generic.h
new file mode 100644
index 000000000000..bfb28bebf90b
--- /dev/null
+++ b/lldb/source/Plugins/Language/CPlusPlus/Generic.h
@@ -0,0 +1,25 @@
+//===-- LibCxx.h -----------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_SOURCE_PLUGINS_LANGUAGE_CPLUSPLUS_GENERIC_H
+#define LLDB_SOURCE_PLUGINS_LANGUAGE_CPLUSPLUS_GENERIC_H
+
+#include "lldb/Core/ValueObject.h"
+#include "lldb/DataFormatters/TypeSummary.h"
+#include "lldb/Utility/Stream.h"
+
+namespace lldb_private {
+namespace formatters {
+
+bool GenericOptionalSummaryProvider(ValueObject &valobj, Stream &stream,
+                                    const TypeSummaryOptions &options);
+
+} // namespace formatters
+} // namespace lldb_private
+
+#endif // LLDB_SOURCE_PLUGINS_LANGUAGE_CPLUSPLUS_GENERIC_H
diff --git a/lldb/source/Plugins/Language/CPlusPlus/GenericOptional.cpp b/lldb/source/Plugins/Language/CPlusPlus/GenericOptional.cpp
new file mode 100644
index 000000000000..74b3f711de35
--- /dev/null
+++ b/lldb/source/Plugins/Language/CPlusPlus/GenericOptional.cpp
@@ -0,0 +1,139 @@
+//===-- GenericOptional.cpp ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#include "Generic.h"
+#include "LibCxx.h"
+#include "LibStdcpp.h"
+#include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
+#include "lldb/DataFormatters/FormattersHelpers.h"
+#include "lldb/Target/Target.h"
+
+using namespace lldb;
+using namespace lldb_private;
+
+bool lldb_private::formatters::GenericOptionalSummaryProvider(
+    ValueObject &valobj, Stream &stream, const TypeSummaryOptions &options) {
+  stream.Printf(" Has Value=%s ",
+                valobj.GetNumChildren() == 0 ? "false" : "true");
+
+  return true;
+}
+
+// Synthetic Children Provider
+namespace {
+
+class GenericOptionalFrontend : public SyntheticChildrenFrontEnd {
+public:
+  enum class StdLib {
+    LibCxx,
+    LibStdcpp,
+  };
+
+  GenericOptionalFrontend(ValueObject &valobj, StdLib stdlib);
+
+  size_t GetIndexOfChildWithName(ConstString name) override {
+    return formatters::ExtractIndexFromString(name.GetCString());
+  }
+
+  bool MightHaveChildren() override { return true; }
+  size_t CalculateNumChildren() override { return m_has_value ? 1U : 0U; }
+
+  ValueObjectSP GetChildAtIndex(size_t idx) override;
+  bool Update() override;
+
+private:
+  bool m_has_value = false;
+  StdLib m_stdlib;
+};
+
+} // namespace
+
+GenericOptionalFrontend::GenericOptionalFrontend(ValueObject &valobj,
+                                                 StdLib stdlib)
+    : SyntheticChildrenFrontEnd(valobj), m_stdlib(stdlib) {
+  if (auto target_sp = m_backend.GetTargetSP()) {
+    Update();
+  }
+}
+
+bool GenericOptionalFrontend::Update() {
+  ValueObjectSP engaged_sp;
+
+  if (m_stdlib == StdLib::LibCxx)
+    engaged_sp =
+        m_backend.GetChildMemberWithName(ConstString("__engaged_"), true);
+  else if (m_stdlib == StdLib::LibStdcpp)
+    engaged_sp =
+        m_backend.GetChildMemberWithName(ConstString("_M_payload"), true)
+            ->GetChildMemberWithName(ConstString("_M_engaged"), true);
+
+  if (!engaged_sp)
+    return false;
+
+  // _M_engaged/__engaged is a bool flag and is true if the optional contains a
+  // value. Converting it to unsigned gives us a size of 1 if it contains a
+  // value and 0 if not.
+  m_has_value = engaged_sp->GetValueAsUnsigned(0) != 0;
+
+  return false;
+}
+
+ValueObjectSP GenericOptionalFrontend::GetChildAtIndex(size_t _idx) {
+  if (!m_has_value)
+    return ValueObjectSP();
+
+  ValueObjectSP val_sp;
+
+  if (m_stdlib == StdLib::LibCxx)
+    // __val_ contains the underlying value of an optional if it has one.
+    // Currently because it is part of an anonymous union
+    // GetChildMemberWithName() does not peer through and find it unless we are
+    // at the parent itself. We can obtain the parent through __engaged_.
+    val_sp = m_backend.GetChildMemberWithName(ConstString("__engaged_"), true)
+                 ->GetParent()
+                 ->GetChildAtIndex(0, true)
+                 ->GetChildMemberWithName(ConstString("__val_"), true);
+  else if (m_stdlib == StdLib::LibStdcpp) {
+    val_sp = m_backend.GetChildMemberWithName(ConstString("_M_payload"), true)
+                 ->GetChildMemberWithName(ConstString("_M_payload"), true);
+
+    // In some implementations, _M_value contains the underlying value of an
+    // optional, and in other versions, it's in the payload member.
+    ValueObjectSP candidate =
+        val_sp->GetChildMemberWithName(ConstString("_M_value"), true);
+    if (candidate)
+      val_sp = candidate;
+  }
+
+  if (!val_sp)
+    return ValueObjectSP();
+
+  CompilerType holder_type = val_sp->GetCompilerType();
+
+  if (!holder_type)
+    return ValueObjectSP();
+
+  return val_sp->Clone(ConstString("Value"));
+}
+
+SyntheticChildrenFrontEnd *
+formatters::LibStdcppOptionalSyntheticFrontEndCreator(
+    CXXSyntheticChildren *, lldb::ValueObjectSP valobj_sp) {
+  if (valobj_sp)
+    return new GenericOptionalFrontend(
+        *valobj_sp, GenericOptionalFrontend::StdLib::LibStdcpp);
+  return nullptr;
+}
+
+SyntheticChildrenFrontEnd *formatters::LibcxxOptionalSyntheticFrontEndCreator(
+    CXXSyntheticChildren *, lldb::ValueObjectSP valobj_sp) {
+  if (valobj_sp)
+    return new GenericOptionalFrontend(*valobj_sp,
+                                       GenericOptionalFrontend::StdLib::LibCxx);
+  return nullptr;
+}
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
index 99e206543197..80dc71787ceb 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
@@ -170,8 +170,8 @@ SyntheticChildrenFrontEnd *LibcxxTupleFrontEndCreator(CXXSyntheticChildren *,
                                                       lldb::ValueObjectSP);
 
 SyntheticChildrenFrontEnd *
-LibcxxOptionalFrontEndCreator(CXXSyntheticChildren *,
-                              lldb::ValueObjectSP valobj_sp);
+LibcxxOptionalSyntheticFrontEndCreator(CXXSyntheticChildren *,
+                                       lldb::ValueObjectSP valobj_sp);
 
 SyntheticChildrenFrontEnd *
 LibcxxVariantFrontEndCreator(CXXSyntheticChildren *,
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxOptional.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxOptional.cpp
deleted file mode 100644
index c1b40ba65e7d..000000000000
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxOptional.cpp
+++ /dev/null
@@ -1,84 +0,0 @@
-//===-- LibCxxOptional.cpp ------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "LibCxx.h"
-#include "lldb/DataFormatters/FormattersHelpers.h"
-
-using namespace lldb;
-using namespace lldb_private;
-
-namespace {
-
-class OptionalFrontEnd : public SyntheticChildrenFrontEnd {
-public:
-  OptionalFrontEnd(ValueObject &valobj) : SyntheticChildrenFrontEnd(valobj) {
-    Update();
-  }
-
-  size_t GetIndexOfChildWithName(ConstString name) override {
-    return formatters::ExtractIndexFromString(name.GetCString());
-  }
-
-  bool MightHaveChildren() override { return true; }
-  bool Update() override;
-  size_t CalculateNumChildren() override { return m_has_value ? 1U : 0U; }
-  ValueObjectSP GetChildAtIndex(size_t idx) override;
-
-private:
-  /// True iff the option contains a value.
-  bool m_has_value = false;
-};
-} // namespace
-
-bool OptionalFrontEnd::Update() {
-  ValueObjectSP engaged_sp(
-      m_backend.GetChildMemberWithName(ConstString("__engaged_"), true));
-
-  if (!engaged_sp)
-    return false;
-
-  // __engaged_ is a bool flag and is true if the optional contains a value.
-  // Converting it to unsigned gives us a size of 1 if it contains a value
-  // and 0 if not.
-  m_has_value = engaged_sp->GetValueAsUnsigned(0) != 0;
-
-  return false;
-}
-
-ValueObjectSP OptionalFrontEnd::GetChildAtIndex(size_t idx) {
-  if (!m_has_value)
-    return ValueObjectSP();
-
-  // __val_ contains the underlying value of an optional if it has one.
-  // Currently because it is part of an anonymous union GetChildMemberWithName()
-  // does not peer through and find it unless we are at the parent itself.
-  // We can obtain the parent through __engaged_.
-  ValueObjectSP val_sp(
-      m_backend.GetChildMemberWithName(ConstString("__engaged_"), true)
-          ->GetParent()
-          ->GetChildAtIndex(0, true)
-          ->GetChildMemberWithName(ConstString("__val_"), true));
-
-  if (!val_sp)
-    return ValueObjectSP();
-
-  CompilerType holder_type = val_sp->GetCompilerType();
-
-  if (!holder_type)
-    return ValueObjectSP();
-
-  return val_sp->Clone(ConstString("Value"));
-}
-
-SyntheticChildrenFrontEnd *
-formatters::LibcxxOptionalFrontEndCreator(CXXSyntheticChildren *,
-                                          lldb::ValueObjectSP valobj_sp) {
-  if (valobj_sp)
-    return new OptionalFrontEnd(*valobj_sp);
-  return nullptr;
-}
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.h b/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.h
index b6f9c469fedd..1c1c8fdb9ea4 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.h
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.h
@@ -46,6 +46,10 @@ LibStdcppBitsetSyntheticFrontEndCreator(CXXSyntheticChildren *,
                                         lldb::ValueObjectSP);
 
 SyntheticChildrenFrontEnd *
+LibStdcppOptionalSyntheticFrontEndCreator(CXXSyntheticChildren *,
+                                          lldb::ValueObjectSP);
+
+SyntheticChildrenFrontEnd *
 LibStdcppVectorIteratorSyntheticFrontEndCreator(CXXSyntheticChildren *,
                                                 lldb::ValueObjectSP);
 
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCDeclVendor.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCDeclVendor.cpp
index 9bc40c16e5d0..af11109ae45d 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCDeclVendor.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCDeclVendor.cpp
@@ -80,8 +80,6 @@ public:
     LLDB_LOG(log, "  AOEAS::CT Before:\n{1}", ClangUtil::DumpDecl(tag_decl));
 
     LLDB_LOG(log, "  AOEAS::CT After:{1}", ClangUtil::DumpDecl(tag_decl));
-
-    return;
   }
 
   void CompleteType(clang::ObjCInterfaceDecl *interface_decl) override {
@@ -107,7 +105,6 @@ public:
       LLDB_LOGF(log, "  [CT] After:");
       LLDB_LOG(log, "    [CT] {0}", ClangUtil::DumpDecl(interface_decl));
     }
-    return;
   }
 
   bool layoutRecordType(
diff --git a/lldb/source/Plugins/Platform/POSIX/PlatformPOSIX.cpp b/lldb/source/Plugins/Platform/POSIX/PlatformPOSIX.cpp
index 719109c863e7..1d989b268b74 100644
--- a/lldb/source/Plugins/Platform/POSIX/PlatformPOSIX.cpp
+++ b/lldb/source/Plugins/Platform/POSIX/PlatformPOSIX.cpp
@@ -589,6 +589,8 @@ PlatformPOSIX::MakeLoadImageUtilityFunction(ExecutionContext &exe_ctx,
       result_ptr->image_ptr = dlopen(name, RTLD_LAZY);
       if (result_ptr->image_ptr)
         result_ptr->error_str = nullptr;
+      else
+        result_ptr->error_str = dlerror();
       return nullptr;
     }
     
diff --git a/lldb/source/Plugins/Platform/QemuUser/PlatformQemuUser.cpp b/lldb/source/Plugins/Platform/QemuUser/PlatformQemuUser.cpp
index 90c290b6fbc7..67c9484680a4 100644
--- a/lldb/source/Plugins/Platform/QemuUser/PlatformQemuUser.cpp
+++ b/lldb/source/Plugins/Platform/QemuUser/PlatformQemuUser.cpp
@@ -47,6 +47,27 @@ public:
     return m_collection_sp->GetPropertyAtIndexAsFileSpec(nullptr,
                                                          ePropertyEmulatorPath);
   }
+
+  Args GetEmulatorArgs() {
+    Args result;
+    m_collection_sp->GetPropertyAtIndexAsArgs(nullptr, ePropertyEmulatorArgs,
+                                              result);
+    return result;
+  }
+
+  Environment GetEmulatorEnvVars() {
+    Args args;
+    m_collection_sp->GetPropertyAtIndexAsArgs(nullptr, ePropertyEmulatorEnvVars,
+                                              args);
+    return Environment(args);
+  }
+
+  Environment GetTargetEnvVars() {
+    Args args;
+    m_collection_sp->GetPropertyAtIndexAsArgs(nullptr, ePropertyTargetEnvVars,
+                                              args);
+    return Environment(args);
+  }
 };
 
 static PluginProperties &GetGlobalProperties() {
@@ -98,6 +119,44 @@ static auto get_arg_range(const Args &args) {
                           args.GetArgumentArrayRef().end());
 }
 
+// Returns the emulator environment which result in the desired environment
+// being presented to the emulated process. We want to be careful about
+// preserving the host environment, as it may contain entries (LD_LIBRARY_PATH,
+// for example) needed for the operation of the emulator itself.
+static Environment ComputeLaunchEnvironment(Environment target,
+                                            Environment host) {
+  std::vector<std::string> set_env;
+  for (const auto &KV : target) {
+    // If the host value differs from the target (or is unset), then set it
+    // through QEMU_SET_ENV. Identical entries will be forwarded automatically.
+    auto host_it = host.find(KV.first());
+    if (host_it == host.end() || host_it->second != KV.second)
+      set_env.push_back(Environment::compose(KV));
+  }
+  llvm::sort(set_env);
+
+  std::vector<llvm::StringRef> unset_env;
+  for (const auto &KV : host) {
+    // If the target is missing some host entries, then unset them through
+    // QEMU_UNSET_ENV.
+    if (target.count(KV.first()) == 0)
+      unset_env.push_back(KV.first());
+  }
+  llvm::sort(unset_env);
+
+  // The actual QEMU_(UN)SET_ENV variables should not be forwarded to the
+  // target.
+  if (!set_env.empty()) {
+    host["QEMU_SET_ENV"] = llvm::join(set_env, ",");
+    unset_env.push_back("QEMU_SET_ENV");
+  }
+  if (!unset_env.empty()) {
+    unset_env.push_back("QEMU_UNSET_ENV");
+    host["QEMU_UNSET_ENV"] = llvm::join(unset_env, ",");
+  }
+  return host;
+}
+
 lldb::ProcessSP PlatformQemuUser::DebugProcess(ProcessLaunchInfo &launch_info,
                                                Debugger &debugger,
                                                Target &target, Status &error) {
@@ -112,8 +171,10 @@ lldb::ProcessSP PlatformQemuUser::DebugProcess(ProcessLaunchInfo &launch_info,
     llvm::sys::fs::createUniquePath(socket_model, socket_path, false);
   } while (FileSystem::Instance().Exists(socket_path));
 
-  Args args(
-      {qemu, "-g", socket_path, launch_info.GetExecutableFile().GetPath()});
+  Args args({qemu, "-g", socket_path});
+  args.AppendArguments(GetGlobalProperties().GetEmulatorArgs());
+  args.AppendArgument("--");
+  args.AppendArgument(launch_info.GetExecutableFile().GetPath());
   for (size_t i = 1; i < launch_info.GetArguments().size(); ++i)
     args.AppendArgument(launch_info.GetArguments()[i].ref());
 
@@ -121,11 +182,23 @@ lldb::ProcessSP PlatformQemuUser::DebugProcess(ProcessLaunchInfo &launch_info,
            get_arg_range(args));
 
   launch_info.SetArguments(args, true);
+
+  Environment emulator_env = Host::GetEnvironment();
+  for (const auto &KV : GetGlobalProperties().GetEmulatorEnvVars())
+    emulator_env[KV.first()] = KV.second;
+  launch_info.GetEnvironment() = ComputeLaunchEnvironment(
+      std::move(launch_info.GetEnvironment()), std::move(emulator_env));
+
   launch_info.SetLaunchInSeparateProcessGroup(true);
   launch_info.GetFlags().Clear(eLaunchFlagDebug);
   launch_info.SetMonitorProcessCallback(ProcessLaunchInfo::NoOpMonitorCallback,
                                         false);
 
+  // This is automatically done for host platform in
+  // Target::FinalizeFileActions, but we're not a host platform.
+  llvm::Error Err = launch_info.SetUpPtyRedirection();
+  LLDB_LOG_ERROR(log, std::move(Err), "SetUpPtyRedirection failed: {0}");
+
   error = Host::LaunchProcess(launch_info);
   if (error.Fail())
     return nullptr;
@@ -134,6 +207,7 @@ lldb::ProcessSP PlatformQemuUser::DebugProcess(ProcessLaunchInfo &launch_info,
       launch_info.GetListener(),
       process_gdb_remote::ProcessGDBRemote::GetPluginNameStatic(), nullptr,
       true);
+
   ListenerSP listener_sp =
       Listener::MakeListener("lldb.platform_qemu_user.debugprocess");
   launch_info.SetHijackListener(listener_sp);
@@ -143,6 +217,18 @@ lldb::ProcessSP PlatformQemuUser::DebugProcess(ProcessLaunchInfo &launch_info,
   if (error.Fail())
     return nullptr;
 
+  if (launch_info.GetPTY().GetPrimaryFileDescriptor() !=
+      PseudoTerminal::invalid_fd)
+    process_sp->SetSTDIOFileDescriptor(
+        launch_info.GetPTY().ReleasePrimaryFileDescriptor());
+
   process_sp->WaitForProcessToStop(llvm::None, nullptr, false, listener_sp);
   return process_sp;
 }
+
+Environment PlatformQemuUser::GetEnvironment() {
+  Environment env = Host::GetEnvironment();
+  for (const auto &KV : GetGlobalProperties().GetTargetEnvVars())
+    env[KV.first()] = KV.second;
+  return env;
+}
diff --git a/lldb/source/Plugins/Platform/QemuUser/PlatformQemuUser.h b/lldb/source/Plugins/Platform/QemuUser/PlatformQemuUser.h
index f4f5d224a8cd..71df1b7b7811 100644
--- a/lldb/source/Plugins/Platform/QemuUser/PlatformQemuUser.h
+++ b/lldb/source/Plugins/Platform/QemuUser/PlatformQemuUser.h
@@ -45,7 +45,7 @@ public:
 
   void CalculateTrapHandlerSymbolNames() override {}
 
-  Environment GetEnvironment() override { return Host::GetEnvironment(); }
+  Environment GetEnvironment() override;
 
 private:
   static lldb::PlatformSP CreateInstance(bool force, const ArchSpec *arch);
diff --git a/lldb/source/Plugins/Platform/QemuUser/PlatformQemuUserProperties.td b/lldb/source/Plugins/Platform/QemuUser/PlatformQemuUserProperties.td
index abfab7f59de4..4e8fbcfd6760 100644
--- a/lldb/source/Plugins/Platform/QemuUser/PlatformQemuUserProperties.td
+++ b/lldb/source/Plugins/Platform/QemuUser/PlatformQemuUserProperties.td
@@ -9,4 +9,16 @@ let Definition = "platformqemuuser" in {
     Global,
     DefaultStringValue<"">,
     Desc<"Path to the emulator binary.">;
+  def EmulatorArgs: Property<"emulator-args", "Args">,
+    Global,
+    DefaultStringValue<"">,
+    Desc<"Extra arguments to pass to the emulator.">;
+  def EmulatorEnvVars: Property<"emulator-env-vars", "Dictionary">,
+    Global,
+    ElementType<"String">,
+    Desc<"Extra variables to add to the emulator environment.">;
+  def TargetEnvVars: Property<"target-env-vars", "Dictionary">,
+    Global,
+    ElementType<"String">,
+    Desc<"Extra variables to add to emulated target environment.">;
 }
diff --git a/lldb/source/Plugins/Platform/gdb-server/PlatformRemoteGDBServer.cpp b/lldb/source/Plugins/Platform/gdb-server/PlatformRemoteGDBServer.cpp
index 987f7c7f57e7..3535a5ad739d 100644
--- a/lldb/source/Plugins/Platform/gdb-server/PlatformRemoteGDBServer.cpp
+++ b/lldb/source/Plugins/Platform/gdb-server/PlatformRemoteGDBServer.cpp
@@ -225,18 +225,12 @@ Status PlatformRemoteGDBServer::ConnectRemote(Args &args) {
   m_platform_hostname = parsed_url->hostname.str();
 
   m_gdb_client.SetConnection(std::make_unique<ConnectionFileDescriptor>());
-  if (repro::Reproducer::Instance().IsReplaying()) {
-    error = m_gdb_replay_server.Connect(m_gdb_client);
-    if (error.Success())
-      m_gdb_replay_server.StartAsyncThread();
-  } else {
-    if (repro::Generator *g = repro::Reproducer::Instance().GetGenerator()) {
-      repro::GDBRemoteProvider &provider =
-          g->GetOrCreate<repro::GDBRemoteProvider>();
-      m_gdb_client.SetPacketRecorder(provider.GetNewPacketRecorder());
-    }
-    m_gdb_client.Connect(url, &error);
+  if (repro::Generator *g = repro::Reproducer::Instance().GetGenerator()) {
+    repro::GDBRemoteProvider &provider =
+        g->GetOrCreate<repro::GDBRemoteProvider>();
+    m_gdb_client.SetPacketRecorder(provider.GetNewPacketRecorder());
   }
+  m_gdb_client.Connect(url, &error);
 
   if (error.Fail())
     return error;
diff --git a/lldb/source/Plugins/Process/FreeBSDKernel/ProcessFreeBSDKernel.cpp b/lldb/source/Plugins/Process/FreeBSDKernel/ProcessFreeBSDKernel.cpp
new file mode 100644
index 000000000000..339d33d25110
--- /dev/null
+++ b/lldb/source/Plugins/Process/FreeBSDKernel/ProcessFreeBSDKernel.cpp
@@ -0,0 +1,218 @@
+//===-- ProcessFreeBSDKernel.cpp ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "lldb/Core/Module.h"
+#include "lldb/Core/PluginManager.h"
+#include "lldb/Target/DynamicLoader.h"
+
+#include "Plugins/DynamicLoader/Static/DynamicLoaderStatic.h"
+#include "ProcessFreeBSDKernel.h"
+#include "ThreadFreeBSDKernel.h"
+
+#if LLDB_ENABLE_FBSDVMCORE
+#include <fvc.h>
+#endif
+#if defined(__FreeBSD__)
+#include <kvm.h>
+#endif
+
+using namespace lldb;
+using namespace lldb_private;
+
+LLDB_PLUGIN_DEFINE(ProcessFreeBSDKernel)
+
+namespace {
+
+#if LLDB_ENABLE_FBSDVMCORE
+class ProcessFreeBSDKernelFVC : public ProcessFreeBSDKernel {
+public:
+  ProcessFreeBSDKernelFVC(lldb::TargetSP target_sp, lldb::ListenerSP listener,
+                          fvc_t *fvc);
+
+  ~ProcessFreeBSDKernelFVC();
+
+  size_t DoReadMemory(lldb::addr_t addr, void *buf, size_t size,
+                      lldb_private::Status &error) override;
+
+private:
+  fvc_t *m_fvc;
+
+  const char *GetError();
+};
+#endif // LLDB_ENABLE_FBSDVMCORE
+
+#if defined(__FreeBSD__)
+class ProcessFreeBSDKernelKVM : public ProcessFreeBSDKernel {
+public:
+  ProcessFreeBSDKernelKVM(lldb::TargetSP target_sp, lldb::ListenerSP listener,
+                          kvm_t *fvc);
+
+  ~ProcessFreeBSDKernelKVM();
+
+  size_t DoReadMemory(lldb::addr_t addr, void *buf, size_t size,
+                      lldb_private::Status &error) override;
+
+private:
+  kvm_t *m_kvm;
+
+  const char *GetError();
+};
+#endif // defined(__FreeBSD__)
+
+} // namespace
+
+ProcessFreeBSDKernel::ProcessFreeBSDKernel(lldb::TargetSP target_sp,
+                                           ListenerSP listener_sp)
+    : PostMortemProcess(target_sp, listener_sp) {}
+
+lldb::ProcessSP ProcessFreeBSDKernel::CreateInstance(lldb::TargetSP target_sp,
+                                                     ListenerSP listener_sp,
+                                                     const FileSpec *crash_file,
+                                                     bool can_connect) {
+  ModuleSP executable = target_sp->GetExecutableModule();
+  if (crash_file && !can_connect && executable) {
+#if LLDB_ENABLE_FBSDVMCORE
+    fvc_t *fvc =
+        fvc_open(executable->GetFileSpec().GetPath().c_str(),
+                 crash_file->GetPath().c_str(), nullptr, nullptr, nullptr);
+    if (fvc)
+      return std::make_shared<ProcessFreeBSDKernelFVC>(target_sp, listener_sp,
+                                                       fvc);
+#endif
+
+#if defined(__FreeBSD__)
+    kvm_t *kvm =
+        kvm_open2(executable->GetFileSpec().GetPath().c_str(),
+                  crash_file->GetPath().c_str(), O_RDONLY, nullptr, nullptr);
+    if (kvm)
+      return std::make_shared<ProcessFreeBSDKernelKVM>(target_sp, listener_sp,
+                                                       kvm);
+#endif
+  }
+  return nullptr;
+}
+
+void ProcessFreeBSDKernel::Initialize() {
+  static llvm::once_flag g_once_flag;
+
+  llvm::call_once(g_once_flag, []() {
+    PluginManager::RegisterPlugin(GetPluginNameStatic(),
+                                  GetPluginDescriptionStatic(), CreateInstance);
+  });
+}
+
+void ProcessFreeBSDKernel::Terminate() {
+  PluginManager::UnregisterPlugin(ProcessFreeBSDKernel::CreateInstance);
+}
+
+Status ProcessFreeBSDKernel::DoDestroy() { return Status(); }
+
+bool ProcessFreeBSDKernel::CanDebug(lldb::TargetSP target_sp,
+                                    bool plugin_specified_by_name) {
+  return true;
+}
+
+void ProcessFreeBSDKernel::RefreshStateAfterStop() {}
+
+bool ProcessFreeBSDKernel::DoUpdateThreadList(ThreadList &old_thread_list,
+                                              ThreadList &new_thread_list) {
+  if (old_thread_list.GetSize(false) == 0) {
+    // Make up the thread the first time this is called so we can set our one
+    // and only core thread state up.
+
+    // We cannot construct a thread without a register context as that crashes
+    // LLDB but we can construct a process without threads to provide minimal
+    // memory reading support.
+    switch (GetTarget().GetArchitecture().GetMachine()) {
+    case llvm::Triple::aarch64:
+    case llvm::Triple::x86:
+    case llvm::Triple::x86_64:
+      break;
+    default:
+      return false;
+    }
+
+    const Symbol *pcb_sym =
+        GetTarget().GetExecutableModule()->FindFirstSymbolWithNameAndType(
+            ConstString("dumppcb"));
+    ThreadSP thread_sp(new ThreadFreeBSDKernel(
+        *this, 1, pcb_sym ? pcb_sym->GetFileAddress() : LLDB_INVALID_ADDRESS));
+    new_thread_list.AddThread(thread_sp);
+  } else {
+    const uint32_t num_threads = old_thread_list.GetSize(false);
+    for (uint32_t i = 0; i < num_threads; ++i)
+      new_thread_list.AddThread(old_thread_list.GetThreadAtIndex(i, false));
+  }
+  return new_thread_list.GetSize(false) > 0;
+}
+
+Status ProcessFreeBSDKernel::DoLoadCore() {
+  // The core is already loaded by CreateInstance().
+  return Status();
+}
+
+DynamicLoader *ProcessFreeBSDKernel::GetDynamicLoader() {
+  if (m_dyld_up.get() == nullptr)
+    m_dyld_up.reset(DynamicLoader::FindPlugin(
+        this, DynamicLoaderStatic::GetPluginNameStatic()));
+  return m_dyld_up.get();
+}
+
+#if LLDB_ENABLE_FBSDVMCORE
+
+ProcessFreeBSDKernelFVC::ProcessFreeBSDKernelFVC(lldb::TargetSP target_sp,
+                                                 ListenerSP listener_sp,
+                                                 fvc_t *fvc)
+    : ProcessFreeBSDKernel(target_sp, listener_sp), m_fvc(fvc) {}
+
+ProcessFreeBSDKernelFVC::~ProcessFreeBSDKernelFVC() {
+  if (m_fvc)
+    fvc_close(m_fvc);
+}
+
+size_t ProcessFreeBSDKernelFVC::DoReadMemory(lldb::addr_t addr, void *buf,
+                                             size_t size, Status &error) {
+  ssize_t rd = 0;
+  rd = fvc_read(m_fvc, addr, buf, size);
+  if (rd < 0 || static_cast<size_t>(rd) != size) {
+    error.SetErrorStringWithFormat("Reading memory failed: %s", GetError());
+    return rd > 0 ? rd : 0;
+  }
+  return rd;
+}
+
+const char *ProcessFreeBSDKernelFVC::GetError() { return fvc_geterr(m_fvc); }
+
+#endif // LLDB_ENABLE_FBSDVMCORE
+
+#if defined(__FreeBSD__)
+
+ProcessFreeBSDKernelKVM::ProcessFreeBSDKernelKVM(lldb::TargetSP target_sp,
+                                                 ListenerSP listener_sp,
+                                                 kvm_t *fvc)
+    : ProcessFreeBSDKernel(target_sp, listener_sp), m_kvm(fvc) {}
+
+ProcessFreeBSDKernelKVM::~ProcessFreeBSDKernelKVM() {
+  if (m_kvm)
+    kvm_close(m_kvm);
+}
+
+size_t ProcessFreeBSDKernelKVM::DoReadMemory(lldb::addr_t addr, void *buf,
+                                             size_t size, Status &error) {
+  ssize_t rd = 0;
+  rd = kvm_read2(m_kvm, addr, buf, size);
+  if (rd < 0 || static_cast<size_t>(rd) != size) {
+    error.SetErrorStringWithFormat("Reading memory failed: %s", GetError());
+    return rd > 0 ? rd : 0;
+  }
+  return rd;
+}
+
+const char *ProcessFreeBSDKernelKVM::GetError() { return kvm_geterr(m_kvm); }
+
+#endif // defined(__FreeBSD__)
diff --git a/lldb/source/Plugins/Process/FreeBSDKernel/ProcessFreeBSDKernel.h b/lldb/source/Plugins/Process/FreeBSDKernel/ProcessFreeBSDKernel.h
new file mode 100644
index 000000000000..558eec5403db
--- /dev/null
+++ b/lldb/source/Plugins/Process/FreeBSDKernel/ProcessFreeBSDKernel.h
@@ -0,0 +1,51 @@
+//===-- ProcessFreeBSDKernel.h ----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_SOURCE_PLUGINS_PROCESS_FREEBSDKERNEL_PROCESSFREEBSDKERNEL_H
+#define LLDB_SOURCE_PLUGINS_PROCESS_FREEBSDKERNEL_PROCESSFREEBSDKERNEL_H
+
+#include "lldb/Target/PostMortemProcess.h"
+
+class ProcessFreeBSDKernel : public lldb_private::PostMortemProcess {
+public:
+  ProcessFreeBSDKernel(lldb::TargetSP target_sp, lldb::ListenerSP listener);
+
+  static lldb::ProcessSP
+  CreateInstance(lldb::TargetSP target_sp, lldb::ListenerSP listener,
+                 const lldb_private::FileSpec *crash_file_path,
+                 bool can_connect);
+
+  static void Initialize();
+
+  static void Terminate();
+
+  static llvm::StringRef GetPluginNameStatic() { return "freebsd-kernel"; }
+
+  static llvm::StringRef GetPluginDescriptionStatic() {
+    return "FreeBSD kernel vmcore debugging plug-in.";
+  }
+
+  llvm::StringRef GetPluginName() override { return GetPluginNameStatic(); }
+
+  lldb_private::Status DoDestroy() override;
+
+  bool CanDebug(lldb::TargetSP target_sp,
+                bool plugin_specified_by_name) override;
+
+  void RefreshStateAfterStop() override;
+
+  lldb_private::Status DoLoadCore() override;
+
+  lldb_private::DynamicLoader *GetDynamicLoader() override;
+
+protected:
+  bool DoUpdateThreadList(lldb_private::ThreadList &old_thread_list,
+                          lldb_private::ThreadList &new_thread_list) override;
+};
+
+#endif // LLDB_SOURCE_PLUGINS_PROCESS_FREEBSDKERNEL_PROCESSFREEBSDKERNEL_H
diff --git a/lldb/source/Plugins/Process/FreeBSDKernel/RegisterContextFreeBSDKernel_arm64.cpp b/lldb/source/Plugins/Process/FreeBSDKernel/RegisterContextFreeBSDKernel_arm64.cpp
new file mode 100644
index 000000000000..11843ddc82d9
--- /dev/null
+++ b/lldb/source/Plugins/Process/FreeBSDKernel/RegisterContextFreeBSDKernel_arm64.cpp
@@ -0,0 +1,110 @@
+//===-- RegisterContextFreeBSDKernel_arm64.cpp ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RegisterContextFreeBSDKernel_arm64.h"
+#include "Plugins/Process/Utility/lldb-arm64-register-enums.h"
+
+#include "lldb/Target/Process.h"
+#include "lldb/Target/Thread.h"
+#include "lldb/Utility/RegisterValue.h"
+#include "llvm/Support/Endian.h"
+
+using namespace lldb;
+using namespace lldb_private;
+
+RegisterContextFreeBSDKernel_arm64::RegisterContextFreeBSDKernel_arm64(
+    Thread &thread, std::unique_ptr<RegisterInfoPOSIX_arm64> register_info_up,
+    lldb::addr_t pcb_addr)
+    : RegisterContextPOSIX_arm64(thread, std::move(register_info_up)),
+      m_pcb_addr(pcb_addr) {}
+
+bool RegisterContextFreeBSDKernel_arm64::ReadGPR() { return true; }
+
+bool RegisterContextFreeBSDKernel_arm64::ReadFPR() { return true; }
+
+bool RegisterContextFreeBSDKernel_arm64::WriteGPR() {
+  assert(0);
+  return false;
+}
+
+bool RegisterContextFreeBSDKernel_arm64::WriteFPR() {
+  assert(0);
+  return false;
+}
+
+bool RegisterContextFreeBSDKernel_arm64::ReadRegister(
+    const RegisterInfo *reg_info, RegisterValue &value) {
+  if (m_pcb_addr == LLDB_INVALID_ADDRESS)
+    return false;
+
+  struct {
+    llvm::support::ulittle64_t x[30];
+    llvm::support::ulittle64_t lr;
+    llvm::support::ulittle64_t _reserved;
+    llvm::support::ulittle64_t sp;
+  } pcb;
+
+  Status error;
+  size_t rd =
+      m_thread.GetProcess()->ReadMemory(m_pcb_addr, &pcb, sizeof(pcb), error);
+  if (rd != sizeof(pcb))
+    return false;
+
+  uint32_t reg = reg_info->kinds[lldb::eRegisterKindLLDB];
+  switch (reg) {
+  case gpr_x0_arm64:
+  case gpr_x1_arm64:
+  case gpr_x2_arm64:
+  case gpr_x3_arm64:
+  case gpr_x4_arm64:
+  case gpr_x5_arm64:
+  case gpr_x6_arm64:
+  case gpr_x7_arm64:
+  case gpr_x8_arm64:
+  case gpr_x9_arm64:
+  case gpr_x10_arm64:
+  case gpr_x11_arm64:
+  case gpr_x12_arm64:
+  case gpr_x13_arm64:
+  case gpr_x14_arm64:
+  case gpr_x15_arm64:
+  case gpr_x16_arm64:
+  case gpr_x17_arm64:
+  case gpr_x18_arm64:
+  case gpr_x19_arm64:
+  case gpr_x20_arm64:
+  case gpr_x21_arm64:
+  case gpr_x22_arm64:
+  case gpr_x23_arm64:
+  case gpr_x24_arm64:
+  case gpr_x25_arm64:
+  case gpr_x26_arm64:
+  case gpr_x27_arm64:
+  case gpr_x28_arm64:
+  case gpr_fp_arm64:
+    static_assert(gpr_fp_arm64 - gpr_x0_arm64 == 29,
+                  "nonconsecutive arm64 register numbers");
+    value = pcb.x[reg - gpr_x0_arm64];
+    break;
+  case gpr_sp_arm64:
+    value = pcb.sp;
+    break;
+  case gpr_pc_arm64:
+    // The pc of crashing thread is stored in lr.
+    value = pcb.lr;
+    break;
+  default:
+    return false;
+  }
+  return true;
+}
+
+bool RegisterContextFreeBSDKernel_arm64::WriteRegister(
+    const RegisterInfo *reg_info, const RegisterValue &value) {
+  return false;
+}
diff --git a/lldb/source/Plugins/Process/FreeBSDKernel/RegisterContextFreeBSDKernel_arm64.h b/lldb/source/Plugins/Process/FreeBSDKernel/RegisterContextFreeBSDKernel_arm64.h
new file mode 100644
index 000000000000..155dda6e748f
--- /dev/null
+++ b/lldb/source/Plugins/Process/FreeBSDKernel/RegisterContextFreeBSDKernel_arm64.h
@@ -0,0 +1,41 @@
+//===-- RegisterContextFreeBSDKernel_arm64.h --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_SOURCE_PLUGINS_PROCESS_FREEBSDKERNEL_REGISTERCONTEXTFREEBSDKERNEL_ARM64_H
+#define LLDB_SOURCE_PLUGINS_PROCESS_FREEBSDKERNEL_REGISTERCONTEXTFREEBSDKERNEL_ARM64_H
+
+#include "Plugins/Process/Utility/RegisterContextPOSIX_arm64.h"
+#include "Plugins/Process/elf-core/RegisterUtilities.h"
+
+class RegisterContextFreeBSDKernel_arm64 : public RegisterContextPOSIX_arm64 {
+public:
+  RegisterContextFreeBSDKernel_arm64(
+      lldb_private::Thread &thread,
+      std::unique_ptr<RegisterInfoPOSIX_arm64> register_info_up,
+      lldb::addr_t pcb_addr);
+
+  bool ReadRegister(const lldb_private::RegisterInfo *reg_info,
+                    lldb_private::RegisterValue &value) override;
+
+  bool WriteRegister(const lldb_private::RegisterInfo *reg_info,
+                     const lldb_private::RegisterValue &value) override;
+
+protected:
+  bool ReadGPR() override;
+
+  bool ReadFPR() override;
+
+  bool WriteGPR() override;
+
+  bool WriteFPR() override;
+
+private:
+  lldb::addr_t m_pcb_addr;
+};
+
+#endif // LLDB_SOURCE_PLUGINS_PROCESS_FREEBSDKERNEL_REGISTERCONTEXTFREEBSDKERNEL_ARM64_H
diff --git a/lldb/source/Plugins/Process/FreeBSDKernel/RegisterContextFreeBSDKernel_i386.cpp b/lldb/source/Plugins/Process/FreeBSDKernel/RegisterContextFreeBSDKernel_i386.cpp
new file mode 100644
index 000000000000..fde85c9c9f0d
--- /dev/null
+++ b/lldb/source/Plugins/Process/FreeBSDKernel/RegisterContextFreeBSDKernel_i386.cpp
@@ -0,0 +1,83 @@
+//===-- RegisterContextFreeBSDKernel_i386.cpp -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RegisterContextFreeBSDKernel_i386.h"
+
+#include "lldb/Target/Process.h"
+#include "lldb/Target/Thread.h"
+#include "lldb/Utility/RegisterValue.h"
+#include "llvm/Support/Endian.h"
+
+using namespace lldb;
+using namespace lldb_private;
+
+RegisterContextFreeBSDKernel_i386::RegisterContextFreeBSDKernel_i386(
+    Thread &thread, RegisterInfoInterface *register_info, lldb::addr_t pcb_addr)
+    : RegisterContextPOSIX_x86(thread, 0, register_info), m_pcb_addr(pcb_addr) {
+}
+
+bool RegisterContextFreeBSDKernel_i386::ReadGPR() { return true; }
+
+bool RegisterContextFreeBSDKernel_i386::ReadFPR() { return true; }
+
+bool RegisterContextFreeBSDKernel_i386::WriteGPR() {
+  assert(0);
+  return false;
+}
+
+bool RegisterContextFreeBSDKernel_i386::WriteFPR() {
+  assert(0);
+  return false;
+}
+
+bool RegisterContextFreeBSDKernel_i386::ReadRegister(
+    const RegisterInfo *reg_info, RegisterValue &value) {
+  if (m_pcb_addr == LLDB_INVALID_ADDRESS)
+    return false;
+
+  struct {
+    llvm::support::ulittle32_t edi;
+    llvm::support::ulittle32_t esi;
+    llvm::support::ulittle32_t ebp;
+    llvm::support::ulittle32_t esp;
+    llvm::support::ulittle32_t ebx;
+    llvm::support::ulittle32_t eip;
+  } pcb;
+
+  Status error;
+  size_t rd =
+      m_thread.GetProcess()->ReadMemory(m_pcb_addr, &pcb, sizeof(pcb), error);
+  if (rd != sizeof(pcb))
+    return false;
+
+  uint32_t reg = reg_info->kinds[lldb::eRegisterKindLLDB];
+  switch (reg) {
+#define REG(x)                                                                 \
+  case lldb_##x##_i386:                                                      \
+    value = pcb.x;                                                             \
+    break;
+
+    REG(edi);
+    REG(esi);
+    REG(ebp);
+    REG(esp);
+    REG(eip);
+
+#undef REG
+
+  default:
+    return false;
+  }
+
+  return true;
+}
+
+bool RegisterContextFreeBSDKernel_i386::WriteRegister(
+    const RegisterInfo *reg_info, const RegisterValue &value) {
+  return false;
+}
diff --git a/lldb/source/Plugins/Process/FreeBSDKernel/RegisterContextFreeBSDKernel_i386.h b/lldb/source/Plugins/Process/FreeBSDKernel/RegisterContextFreeBSDKernel_i386.h
new file mode 100644
index 000000000000..218e3374f8df
--- /dev/null
+++ b/lldb/source/Plugins/Process/FreeBSDKernel/RegisterContextFreeBSDKernel_i386.h
@@ -0,0 +1,41 @@
+//===-- RegisterContextFreeBSDKernel_i386.h ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_SOURCE_PLUGINS_PROCESS_FREEBSDKERNEL_REGISTERCONTEXTFREEBSDKERNEL_I386_H
+#define LLDB_SOURCE_PLUGINS_PROCESS_FREEBSDKERNEL_REGISTERCONTEXTFREEBSDKERNEL_I386_H
+
+#include "Plugins/Process/Utility/RegisterContextPOSIX_x86.h"
+#include "Plugins/Process/elf-core/RegisterUtilities.h"
+
+class RegisterContextFreeBSDKernel_i386 : public RegisterContextPOSIX_x86 {
+public:
+  RegisterContextFreeBSDKernel_i386(
+      lldb_private::Thread &thread,
+      lldb_private::RegisterInfoInterface *register_info,
+      lldb::addr_t pcb_addr);
+
+  bool ReadRegister(const lldb_private::RegisterInfo *reg_info,
+                    lldb_private::RegisterValue &value) override;
+
+  bool WriteRegister(const lldb_private::RegisterInfo *reg_info,
+                     const lldb_private::RegisterValue &value) override;
+
+protected:
+  bool ReadGPR() override;
+
+  bool ReadFPR() override;
+
+  bool WriteGPR() override;
+
+  bool WriteFPR() override;
+
+private:
+  lldb::addr_t m_pcb_addr;
+};
+
+#endif // LLDB_SOURCE_PLUGINS_PROCESS_FREEBSDKERNEL_REGISTERCONTEXTFREEBSDKERNEL_I386_H
diff --git a/lldb/source/Plugins/Process/FreeBSDKernel/RegisterContextFreeBSDKernel_x86_64.cpp b/lldb/source/Plugins/Process/FreeBSDKernel/RegisterContextFreeBSDKernel_x86_64.cpp
new file mode 100644
index 000000000000..ff57842e345c
--- /dev/null
+++ b/lldb/source/Plugins/Process/FreeBSDKernel/RegisterContextFreeBSDKernel_x86_64.cpp
@@ -0,0 +1,88 @@
+//===-- RegisterContextFreeBSDKernel_x86_64.cpp ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RegisterContextFreeBSDKernel_x86_64.h"
+
+#include "lldb/Target/Process.h"
+#include "lldb/Target/Thread.h"
+#include "lldb/Utility/RegisterValue.h"
+#include "llvm/Support/Endian.h"
+
+using namespace lldb;
+using namespace lldb_private;
+
+RegisterContextFreeBSDKernel_x86_64::RegisterContextFreeBSDKernel_x86_64(
+    Thread &thread, RegisterInfoInterface *register_info, lldb::addr_t pcb_addr)
+    : RegisterContextPOSIX_x86(thread, 0, register_info), m_pcb_addr(pcb_addr) {
+}
+
+bool RegisterContextFreeBSDKernel_x86_64::ReadGPR() { return true; }
+
+bool RegisterContextFreeBSDKernel_x86_64::ReadFPR() { return true; }
+
+bool RegisterContextFreeBSDKernel_x86_64::WriteGPR() {
+  assert(0);
+  return false;
+}
+
+bool RegisterContextFreeBSDKernel_x86_64::WriteFPR() {
+  assert(0);
+  return false;
+}
+
+bool RegisterContextFreeBSDKernel_x86_64::ReadRegister(
+    const RegisterInfo *reg_info, RegisterValue &value) {
+  if (m_pcb_addr == LLDB_INVALID_ADDRESS)
+    return false;
+
+  struct {
+    llvm::support::ulittle64_t r15;
+    llvm::support::ulittle64_t r14;
+    llvm::support::ulittle64_t r13;
+    llvm::support::ulittle64_t r12;
+    llvm::support::ulittle64_t rbp;
+    llvm::support::ulittle64_t rsp;
+    llvm::support::ulittle64_t rbx;
+    llvm::support::ulittle64_t rip;
+  } pcb;
+
+  Status error;
+  size_t rd =
+      m_thread.GetProcess()->ReadMemory(m_pcb_addr, &pcb, sizeof(pcb), error);
+  if (rd != sizeof(pcb))
+    return false;
+
+  uint32_t reg = reg_info->kinds[lldb::eRegisterKindLLDB];
+  switch (reg) {
+#define REG(x)                                                                 \
+  case lldb_##x##_x86_64:                                                      \
+    value = pcb.x;                                                             \
+    break;
+
+    REG(r15);
+    REG(r14);
+    REG(r13);
+    REG(r12);
+    REG(rbp);
+    REG(rsp);
+    REG(rbx);
+    REG(rip);
+
+#undef REG
+
+  default:
+    return false;
+  }
+
+  return true;
+}
+
+bool RegisterContextFreeBSDKernel_x86_64::WriteRegister(
+    const RegisterInfo *reg_info, const RegisterValue &value) {
+  return false;
+}
diff --git a/lldb/source/Plugins/Process/FreeBSDKernel/RegisterContextFreeBSDKernel_x86_64.h b/lldb/source/Plugins/Process/FreeBSDKernel/RegisterContextFreeBSDKernel_x86_64.h
new file mode 100644
index 000000000000..9a2ac638dfea
--- /dev/null
+++ b/lldb/source/Plugins/Process/FreeBSDKernel/RegisterContextFreeBSDKernel_x86_64.h
@@ -0,0 +1,41 @@
+//===-- RegisterContextFreeBSDKernel_x86_64.h -------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_SOURCE_PLUGINS_PROCESS_FREEBSDKERNEL_REGISTERCONTEXTFREEBSDKERNEL_X86_64_H
+#define LLDB_SOURCE_PLUGINS_PROCESS_FREEBSDKERNEL_REGISTERCONTEXTFREEBSDKERNEL_X86_64_H
+
+#include "Plugins/Process/Utility/RegisterContextPOSIX_x86.h"
+#include "Plugins/Process/elf-core/RegisterUtilities.h"
+
+class RegisterContextFreeBSDKernel_x86_64 : public RegisterContextPOSIX_x86 {
+public:
+  RegisterContextFreeBSDKernel_x86_64(
+      lldb_private::Thread &thread,
+      lldb_private::RegisterInfoInterface *register_info,
+      lldb::addr_t pcb_addr);
+
+  bool ReadRegister(const lldb_private::RegisterInfo *reg_info,
+                    lldb_private::RegisterValue &value) override;
+
+  bool WriteRegister(const lldb_private::RegisterInfo *reg_info,
+                     const lldb_private::RegisterValue &value) override;
+
+protected:
+  bool ReadGPR() override;
+
+  bool ReadFPR() override;
+
+  bool WriteGPR() override;
+
+  bool WriteFPR() override;
+
+private:
+  lldb::addr_t m_pcb_addr;
+};
+
+#endif // LLDB_SOURCE_PLUGINS_PROCESS_FREEBSDKERNEL_REGISTERCONTEXTFREEBSDKERNEL_X86_64_H
diff --git a/lldb/source/Plugins/Process/FreeBSDKernel/ThreadFreeBSDKernel.cpp b/lldb/source/Plugins/Process/FreeBSDKernel/ThreadFreeBSDKernel.cpp
new file mode 100644
index 000000000000..124c65d587ff
--- /dev/null
+++ b/lldb/source/Plugins/Process/FreeBSDKernel/ThreadFreeBSDKernel.cpp
@@ -0,0 +1,85 @@
+//===-- ThreadFreeBSDKernel.cpp -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "ThreadFreeBSDKernel.h"
+
+#include "lldb/Target/Unwind.h"
+#include "lldb/Utility/Log.h"
+
+#include "Plugins/Process/Utility/RegisterContextFreeBSD_i386.h"
+#include "Plugins/Process/Utility/RegisterContextFreeBSD_x86_64.h"
+#include "Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h"
+#include "ProcessFreeBSDKernel.h"
+#include "RegisterContextFreeBSDKernel_arm64.h"
+#include "RegisterContextFreeBSDKernel_i386.h"
+#include "RegisterContextFreeBSDKernel_x86_64.h"
+#include "ThreadFreeBSDKernel.h"
+
+using namespace lldb;
+using namespace lldb_private;
+
+ThreadFreeBSDKernel::ThreadFreeBSDKernel(Process &process, lldb::tid_t tid,
+                                         lldb::addr_t pcb_addr)
+    : Thread(process, tid), m_pcb_addr(pcb_addr) {}
+
+ThreadFreeBSDKernel::~ThreadFreeBSDKernel() {}
+
+void ThreadFreeBSDKernel::RefreshStateAfterStop() {}
+
+lldb::RegisterContextSP ThreadFreeBSDKernel::GetRegisterContext() {
+  if (!m_reg_context_sp)
+    m_reg_context_sp = CreateRegisterContextForFrame(nullptr);
+  return m_reg_context_sp;
+}
+
+lldb::RegisterContextSP
+ThreadFreeBSDKernel::CreateRegisterContextForFrame(StackFrame *frame) {
+  RegisterContextSP reg_ctx_sp;
+  uint32_t concrete_frame_idx = 0;
+
+  if (frame)
+    concrete_frame_idx = frame->GetConcreteFrameIndex();
+
+  if (concrete_frame_idx == 0) {
+    if (m_thread_reg_ctx_sp)
+      return m_thread_reg_ctx_sp;
+
+    ProcessFreeBSDKernel *process =
+        static_cast<ProcessFreeBSDKernel *>(GetProcess().get());
+    ArchSpec arch = process->GetTarget().GetArchitecture();
+
+    switch (arch.GetMachine()) {
+    case llvm::Triple::aarch64:
+      m_thread_reg_ctx_sp =
+          std::make_shared<RegisterContextFreeBSDKernel_arm64>(
+              *this, std::make_unique<RegisterInfoPOSIX_arm64>(arch, 0),
+              m_pcb_addr);
+      break;
+    case llvm::Triple::x86:
+      m_thread_reg_ctx_sp =
+          std::make_shared<RegisterContextFreeBSDKernel_i386>(
+              *this, new RegisterContextFreeBSD_i386(arch), m_pcb_addr);
+      break;
+    case llvm::Triple::x86_64:
+      m_thread_reg_ctx_sp =
+          std::make_shared<RegisterContextFreeBSDKernel_x86_64>(
+              *this, new RegisterContextFreeBSD_x86_64(arch), m_pcb_addr);
+      break;
+    default:
+      assert(false && "Unsupported architecture passed to ThreadFreeBSDKernel");
+      break;
+    }
+
+    reg_ctx_sp = m_thread_reg_ctx_sp;
+  } else {
+    reg_ctx_sp = GetUnwinder().CreateRegisterContextForFrame(frame);
+  }
+  return reg_ctx_sp;
+}
+
+bool ThreadFreeBSDKernel::CalculateStopInfo() { return false; }
diff --git a/lldb/source/Plugins/Process/FreeBSDKernel/ThreadFreeBSDKernel.h b/lldb/source/Plugins/Process/FreeBSDKernel/ThreadFreeBSDKernel.h
new file mode 100644
index 000000000000..2842eba64e56
--- /dev/null
+++ b/lldb/source/Plugins/Process/FreeBSDKernel/ThreadFreeBSDKernel.h
@@ -0,0 +1,36 @@
+//===-- ThreadFreeBSDKernel.h ------------------------------------- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_SOURCE_PLUGINS_PROCESS_FREEBSDKERNEL_THREADFREEBSDKERNEL_H
+#define LLDB_SOURCE_PLUGINS_PROCESS_FREEBSDKERNEL_THREADFREEBSDKERNEL_H
+
+#include "lldb/Target/Thread.h"
+
+class ThreadFreeBSDKernel : public lldb_private::Thread {
+public:
+  ThreadFreeBSDKernel(lldb_private::Process &process, lldb::tid_t tid,
+                      lldb::addr_t pcb_addr);
+
+  ~ThreadFreeBSDKernel() override;
+
+  void RefreshStateAfterStop() override;
+
+  lldb::RegisterContextSP GetRegisterContext() override;
+
+  lldb::RegisterContextSP
+  CreateRegisterContextForFrame(lldb_private::StackFrame *frame) override;
+
+protected:
+  bool CalculateStopInfo() override;
+
+private:
+  lldb::RegisterContextSP m_thread_reg_ctx_sp;
+  lldb::addr_t m_pcb_addr;
+};
+
+#endif // LLDB_SOURCE_PLUGINS_PROCESS_FREEBSDKERNEL_THREADFREEBSDKERNEL_H
diff --git a/lldb/source/Plugins/Process/elf-core/ProcessElfCore.cpp b/lldb/source/Plugins/Process/elf-core/ProcessElfCore.cpp
index b852a0164375..65dbc8ea95b3 100644
--- a/lldb/source/Plugins/Process/elf-core/ProcessElfCore.cpp
+++ b/lldb/source/Plugins/Process/elf-core/ProcessElfCore.cpp
@@ -64,6 +64,10 @@ lldb::ProcessSP ProcessElfCore::CreateInstance(lldb::TargetSP target_sp,
       DataExtractor data(data_sp, lldb::eByteOrderLittle, 4);
       lldb::offset_t data_offset = 0;
       if (elf_header.Parse(data, &data_offset)) {
+        // Check whether we're dealing with a raw FreeBSD "full memory dump"
+        // ELF vmcore that needs to be handled via FreeBSDKernel plugin instead.
+        if (elf_header.e_ident[7] == 0xFF && elf_header.e_version == 0)
+          return process_sp;
         if (elf_header.e_type == llvm::ELF::ET_CORE)
           process_sp = std::make_shared<ProcessElfCore>(target_sp, listener_sp,
                                                         *crash_file);
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp
index 07dfa5e04ee5..b5b105351de5 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp
@@ -1006,6 +1006,23 @@ GDBRemoteCommunicationClient::GetProcessArchitecture() {
   return m_process_arch;
 }
 
+bool GDBRemoteCommunicationClient::GetProcessStandaloneBinary(
+    UUID &uuid, addr_t &value, bool &value_is_offset) {
+  if (m_qProcessInfo_is_valid == eLazyBoolCalculate)
+    GetCurrentProcessInfo();
+
+  // Return true if we have a UUID or an address/offset of the
+  // main standalone / firmware binary being used.
+  if (!m_process_standalone_uuid.IsValid() &&
+      m_process_standalone_value == LLDB_INVALID_ADDRESS)
+    return false;
+
+  uuid = m_process_standalone_uuid;
+  value = m_process_standalone_value;
+  value_is_offset = m_process_standalone_value_is_offset;
+  return true;
+}
+
 bool GDBRemoteCommunicationClient::GetGDBServerVersion() {
   if (m_qGDBServerVersion_is_valid == eLazyBoolCalculate) {
     m_gdb_server_name.clear();
@@ -2147,6 +2164,25 @@ bool GDBRemoteCommunicationClient::GetCurrentProcessInfo(bool allow_lazy) {
         } else if (name.equals("elf_abi")) {
           elf_abi = std::string(value);
           ++num_keys_decoded;
+        } else if (name.equals("main-binary-uuid")) {
+          m_process_standalone_uuid.SetFromStringRef(value);
+          ++num_keys_decoded;
+        } else if (name.equals("main-binary-slide")) {
+          StringExtractor extractor(value);
+          m_process_standalone_value =
+              extractor.GetU64(LLDB_INVALID_ADDRESS, 16);
+          if (m_process_standalone_value != LLDB_INVALID_ADDRESS) {
+            m_process_standalone_value_is_offset = true;
+            ++num_keys_decoded;
+          }
+        } else if (name.equals("main-binary-address")) {
+          StringExtractor extractor(value);
+          m_process_standalone_value =
+              extractor.GetU64(LLDB_INVALID_ADDRESS, 16);
+          if (m_process_standalone_value != LLDB_INVALID_ADDRESS) {
+            m_process_standalone_value_is_offset = false;
+            ++num_keys_decoded;
+          }
         }
       }
       if (num_keys_decoded > 0)
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.h b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.h
index 6765372ce124..c69c33bb1c15 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.h
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.h
@@ -217,6 +217,9 @@ public:
 
   const ArchSpec &GetProcessArchitecture();
 
+  bool GetProcessStandaloneBinary(UUID &uuid, lldb::addr_t &value,
+                                  bool &value_is_offset);
+
   void GetRemoteQSupported();
 
   bool GetVContSupported(char flavor);
@@ -584,6 +587,9 @@ protected:
 
   ArchSpec m_host_arch;
   ArchSpec m_process_arch;
+  UUID m_process_standalone_uuid;
+  lldb::addr_t m_process_standalone_value = LLDB_INVALID_ADDRESS;
+  bool m_process_standalone_value_is_offset = false;
   llvm::VersionTuple m_os_version;
   llvm::VersionTuple m_maccatalyst_version;
   std::string m_os_build;
diff --git a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
index 3ade8c815feb..93fe36c0d9d6 100644
--- a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
@@ -526,18 +526,15 @@ Status ProcessGDBRemote::WillAttachToProcessWithName(const char *process_name,
 
 Status ProcessGDBRemote::DoConnectRemote(llvm::StringRef remote_url) {
   Log *log(ProcessGDBRemoteLog::GetLogIfAllCategoriesSet(GDBR_LOG_PROCESS));
-  Status error(WillLaunchOrAttach());
 
+  Status error(WillLaunchOrAttach());
   if (error.Fail())
     return error;
 
-  if (repro::Reproducer::Instance().IsReplaying())
-    error = ConnectToReplayServer();
-  else
-    error = ConnectToDebugserver(remote_url);
-
+  error = ConnectToDebugserver(remote_url);
   if (error.Fail())
     return error;
+
   StartAsyncThread();
 
   lldb::pid_t pid = m_gdb_comm.GetCurrentProcessID();
@@ -564,6 +561,94 @@ Status ProcessGDBRemote::DoConnectRemote(llvm::StringRef remote_url) {
         }
       }
 
+      // The remote stub may know about the "main binary" in
+      // the context of a firmware debug session, and can
+      // give us a UUID and an address/slide of where the
+      // binary is loaded in memory.
+      UUID standalone_uuid;
+      addr_t standalone_value;
+      bool standalone_value_is_offset;
+      if (m_gdb_comm.GetProcessStandaloneBinary(
+              standalone_uuid, standalone_value, standalone_value_is_offset)) {
+        ModuleSP module_sp;
+
+        if (standalone_uuid.IsValid()) {
+          ModuleSpec module_spec;
+          module_spec.GetUUID() = standalone_uuid;
+
+          // Look up UUID in global module cache before attempting
+          // a more expensive search.
+          Status error = ModuleList::GetSharedModule(module_spec, module_sp,
+                                                     nullptr, nullptr, nullptr);
+
+          if (!module_sp) {
+            // Force a an external lookup, if that tool is available.
+            if (!module_spec.GetSymbolFileSpec())
+              Symbols::DownloadObjectAndSymbolFile(module_spec, true);
+
+            if (FileSystem::Instance().Exists(module_spec.GetFileSpec())) {
+              module_sp = std::make_shared<Module>(module_spec);
+            }
+          }
+
+          // If we couldn't find the binary anywhere else, as a last resort,
+          // read it out of memory.
+          if (!module_sp.get() && standalone_value != LLDB_INVALID_ADDRESS &&
+              !standalone_value_is_offset) {
+            char namebuf[80];
+            snprintf(namebuf, sizeof(namebuf), "mem-image-0x%" PRIx64,
+                     standalone_value);
+            module_sp =
+                ReadModuleFromMemory(FileSpec(namebuf), standalone_value);
+          }
+
+          Log *log(ProcessGDBRemoteLog::GetLogIfAllCategoriesSet(
+              LIBLLDB_LOG_DYNAMIC_LOADER));
+          if (module_sp.get()) {
+            target.GetImages().AppendIfNeeded(module_sp, false);
+
+            bool changed = false;
+            if (module_sp->GetObjectFile()) {
+              if (standalone_value != LLDB_INVALID_ADDRESS) {
+                if (log)
+                  log->Printf("Loading binary UUID %s at %s 0x%" PRIx64,
+                              standalone_uuid.GetAsString().c_str(),
+                              standalone_value_is_offset ? "offset" : "address",
+                              standalone_value);
+                module_sp->SetLoadAddress(target, standalone_value,
+                                          standalone_value_is_offset, changed);
+              } else {
+                // No address/offset/slide, load the binary at file address,
+                // offset 0.
+                if (log)
+                  log->Printf("Loading binary UUID %s at file address",
+                              standalone_uuid.GetAsString().c_str());
+                const bool value_is_slide = true;
+                module_sp->SetLoadAddress(target, 0, value_is_slide, changed);
+              }
+            } else {
+              // In-memory image, load at its true address, offset 0.
+              if (log)
+                log->Printf("Loading binary UUID %s from memory",
+                            standalone_uuid.GetAsString().c_str());
+              const bool value_is_slide = true;
+              module_sp->SetLoadAddress(target, 0, value_is_slide, changed);
+            }
+
+            ModuleList added_module;
+            added_module.Append(module_sp, false);
+            target.ModulesDidLoad(added_module);
+          } else {
+            if (log)
+              log->Printf("Unable to find binary with UUID %s and load it at "
+                          "%s 0x%" PRIx64,
+                          standalone_uuid.GetAsString().c_str(),
+                          standalone_value_is_offset ? "offset" : "address",
+                          standalone_value);
+          }
+        }
+      }
+
       const StateType state = SetThreadStopInfo(response);
       if (state != eStateInvalid) {
         SetPrivateState(state);
@@ -3259,9 +3344,6 @@ ProcessGDBRemote::EstablishConnectionIfNeeded(const ProcessInfo &process_info) {
   if (platform_sp && !platform_sp->IsHost())
     return Status("Lost debug server connection");
 
-  if (repro::Reproducer::Instance().IsReplaying())
-    return ConnectToReplayServer();
-
   auto error = LaunchAndConnectToDebugserver(process_info);
   if (error.Fail()) {
     const char *error_string = error.AsCString();
@@ -3540,7 +3622,7 @@ thread_result_t ProcessGDBRemote::AsyncThread(void *arg) {
   // So it is safer to simply ignore any remaining packets by
   // explicitly checking for eStateExited before reentering the
   // fetch loop.
-  
+
   bool done = false;
   while (!done && process->GetPrivateState() != eStateExited) {
     LLDB_LOGF(log,
diff --git a/lldb/source/Plugins/Process/scripted/ScriptedProcess.cpp b/lldb/source/Plugins/Process/scripted/ScriptedProcess.cpp
index c1b7294a7f58..cb21a3e7e65f 100644
--- a/lldb/source/Plugins/Process/scripted/ScriptedProcess.cpp
+++ b/lldb/source/Plugins/Process/scripted/ScriptedProcess.cpp
@@ -311,6 +311,11 @@ bool ScriptedProcess::DoUpdateThreadList(ThreadList &old_thread_list,
     return GetInterface().ErrorWithMessage<bool>(LLVM_PRETTY_FUNCTION,
                                                  error.AsCString(), error);
 
+  RegisterContextSP reg_ctx_sp = thread_sp->GetRegisterContext();
+  if (!reg_ctx_sp)
+    return GetInterface().ErrorWithMessage<bool>(
+        LLVM_PRETTY_FUNCTION, "Invalid Register Context", error);
+
   new_thread_list.AddThread(thread_sp);
 
   return new_thread_list.GetSize(false) > 0;
diff --git a/lldb/source/Plugins/Process/scripted/ScriptedThread.cpp b/lldb/source/Plugins/Process/scripted/ScriptedThread.cpp
index 1adbd4e7799d..959b8c581885 100644
--- a/lldb/source/Plugins/Process/scripted/ScriptedThread.cpp
+++ b/lldb/source/Plugins/Process/scripted/ScriptedThread.cpp
@@ -198,13 +198,17 @@ std::shared_ptr<DynamicRegisterInfo> ScriptedThread::GetDynamicRegisterInfo() {
 
   if (!m_register_info_sp) {
     StructuredData::DictionarySP reg_info = GetInterface()->GetRegisterInfo();
+
+    Status error;
     if (!reg_info)
-      return nullptr;
+      return GetInterface()
+          ->ErrorWithMessage<std::shared_ptr<DynamicRegisterInfo>>(
+              LLVM_PRETTY_FUNCTION,
+              "Failed to get scripted thread registers info.", error,
+              LIBLLDB_LOG_THREAD);
 
     m_register_info_sp = std::make_shared<DynamicRegisterInfo>(
         *reg_info, m_scripted_process.GetTarget().GetArchitecture());
-    assert(m_register_info_sp->GetNumRegisters() > 0);
-    assert(m_register_info_sp->GetNumRegisterSets() > 0);
   }
 
   return m_register_info_sp;
diff --git a/lldb/source/Plugins/ScriptInterpreter/Lua/Lua.cpp b/lldb/source/Plugins/ScriptInterpreter/Lua/Lua.cpp
index e99b7b88379a..b82a2647e9a0 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Lua/Lua.cpp
+++ b/lldb/source/Plugins/ScriptInterpreter/Lua/Lua.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "Lua.h"
+#include "SWIGLuaBridge.h"
 #include "lldb/Host/FileSystem.h"
 #include "lldb/Utility/FileSpec.h"
 #include "llvm/Support/Error.h"
@@ -15,30 +16,6 @@
 using namespace lldb_private;
 using namespace lldb;
 
-#pragma clang diagnostic push
-#pragma clang diagnostic ignored "-Wreturn-type-c-linkage"
-
-// Disable warning C4190: 'LLDBSwigPythonBreakpointCallbackFunction' has
-// C-linkage specified, but returns UDT 'llvm::Expected<bool>' which is
-// incompatible with C
-#if _MSC_VER
-#pragma warning (push)
-#pragma warning (disable : 4190)
-#endif
-
-extern "C" llvm::Expected<bool> LLDBSwigLuaBreakpointCallbackFunction(
-    lua_State *L, lldb::StackFrameSP stop_frame_sp,
-    lldb::BreakpointLocationSP bp_loc_sp, StructuredDataImpl *extra_args_impl);
-
-extern "C" llvm::Expected<bool> LLDBSwigLuaWatchpointCallbackFunction(
-    lua_State *L, lldb::StackFrameSP stop_frame_sp, lldb::WatchpointSP wp_sp);
-
-#if _MSC_VER
-#pragma warning (pop)
-#endif
-
-#pragma clang diagnostic pop
-
 static int lldb_print(lua_State *L) {
   int n = lua_gettop(L);
   lua_getglobal(L, "io");
@@ -105,13 +82,7 @@ Lua::CallBreakpointCallback(void *baton, lldb::StackFrameSP stop_frame_sp,
 
   lua_pushlightuserdata(m_lua_state, baton);
   lua_gettable(m_lua_state, LUA_REGISTRYINDEX);
-  auto *extra_args_impl = [&]() -> StructuredDataImpl * {
-    if (extra_args_sp == nullptr)
-      return nullptr;
-    auto *extra_args_impl = new StructuredDataImpl();
-    extra_args_impl->SetObjectSP(extra_args_sp);
-    return extra_args_impl;
-  }();
+  StructuredDataImpl extra_args_impl(std::move(extra_args_sp));
   return LLDBSwigLuaBreakpointCallbackFunction(m_lua_state, stop_frame_sp,
                                                bp_loc_sp, extra_args_impl);
 }
diff --git a/lldb/source/Plugins/ScriptInterpreter/Lua/SWIGLuaBridge.h b/lldb/source/Plugins/ScriptInterpreter/Lua/SWIGLuaBridge.h
new file mode 100644
index 000000000000..5fca18f2dd6d
--- /dev/null
+++ b/lldb/source/Plugins/ScriptInterpreter/Lua/SWIGLuaBridge.h
@@ -0,0 +1,28 @@
+//===-- SWIGLuaBridge.h -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_PLUGINS_SCRIPTINTERPRETER_LUA_SWIGLUABRIDGE_H
+#define LLDB_PLUGINS_SCRIPTINTERPRETER_LUA_SWIGLUABRIDGE_H
+
+#include "lldb/lldb-forward.h"
+#include "lua.hpp"
+#include "llvm/Support/Error.h"
+
+namespace lldb_private {
+
+llvm::Expected<bool> LLDBSwigLuaBreakpointCallbackFunction(
+    lua_State *L, lldb::StackFrameSP stop_frame_sp,
+    lldb::BreakpointLocationSP bp_loc_sp,
+    const StructuredDataImpl &extra_args_impl);
+
+llvm::Expected<bool> LLDBSwigLuaWatchpointCallbackFunction(
+    lua_State *L, lldb::StackFrameSP stop_frame_sp, lldb::WatchpointSP wp_sp);
+
+} // namespace lldb_private
+
+#endif // LLDB_PLUGINS_SCRIPTINTERPRETER_LUA_SWIGLUABRIDGE_H
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/SWIGPythonBridge.h b/lldb/source/Plugins/ScriptInterpreter/Python/SWIGPythonBridge.h
index c7af13598843..2bb69dc47731 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/SWIGPythonBridge.h
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/SWIGPythonBridge.h
@@ -57,20 +57,20 @@ void *LLDBSWIGPython_CastPyObjectToSBMemoryRegionInfo(PyObject *data);
 void *LLDBSwigPythonCreateScriptedProcess(const char *python_class_name,
                                           const char *session_dictionary_name,
                                           const lldb::TargetSP &target_sp,
-                                          StructuredDataImpl *args_impl,
+                                          const StructuredDataImpl &args_impl,
                                           std::string &error_string);
 
 void *LLDBSwigPythonCreateScriptedThread(const char *python_class_name,
                                          const char *session_dictionary_name,
                                          const lldb::ProcessSP &process_sp,
-                                         StructuredDataImpl *args_impl,
+                                         const StructuredDataImpl &args_impl,
                                          std::string &error_string);
 
 llvm::Expected<bool> LLDBSwigPythonBreakpointCallbackFunction(
     const char *python_function_name, const char *session_dictionary_name,
     const lldb::StackFrameSP &sb_frame,
     const lldb::BreakpointLocationSP &sb_bp_loc,
-    lldb_private::StructuredDataImpl *args_impl);
+    const lldb_private::StructuredDataImpl &args_impl);
 
 bool LLDBSwigPythonWatchpointCallbackFunction(
     const char *python_function_name, const char *session_dictionary_name,
@@ -90,11 +90,11 @@ LLDBSwigPythonCreateSyntheticProvider(const char *python_class_name,
 
 void *LLDBSwigPythonCreateCommandObject(const char *python_class_name,
                                         const char *session_dictionary_name,
-                                        const lldb::DebuggerSP debugger_sp);
+                                        lldb::DebuggerSP debugger_sp);
 
 void *LLDBSwigPythonCreateScriptedThreadPlan(
     const char *python_class_name, const char *session_dictionary_name,
-    lldb_private::StructuredDataImpl *args_data, std::string &error_string,
+    const StructuredDataImpl &args_data, std::string &error_string,
     const lldb::ThreadPlanSP &thread_plan_sp);
 
 bool LLDBSWIGPythonCallThreadPlan(void *implementor, const char *method_name,
@@ -103,16 +103,17 @@ bool LLDBSWIGPythonCallThreadPlan(void *implementor, const char *method_name,
 
 void *LLDBSwigPythonCreateScriptedBreakpointResolver(
     const char *python_class_name, const char *session_dictionary_name,
-    lldb_private::StructuredDataImpl *args, const lldb::BreakpointSP &bkpt_sp);
+    const StructuredDataImpl &args, const lldb::BreakpointSP &bkpt_sp);
 
 unsigned int
 LLDBSwigPythonCallBreakpointResolver(void *implementor, const char *method_name,
                                      lldb_private::SymbolContext *sym_ctx);
 
-void *LLDBSwigPythonCreateScriptedStopHook(
-    lldb::TargetSP target_sp, const char *python_class_name,
-    const char *session_dictionary_name, lldb_private::StructuredDataImpl *args,
-    lldb_private::Status &error);
+void *LLDBSwigPythonCreateScriptedStopHook(lldb::TargetSP target_sp,
+                                           const char *python_class_name,
+                                           const char *session_dictionary_name,
+                                           const StructuredDataImpl &args,
+                                           lldb_private::Status &error);
 
 bool LLDBSwigPythonStopHookCallHandleStop(void *implementor,
                                           lldb::ExecutionContextRefSP exc_ctx,
@@ -136,18 +137,18 @@ PyObject *LLDBSwigPython_GetValueSynthProviderInstance(PyObject *implementor);
 
 bool LLDBSwigPythonCallCommand(const char *python_function_name,
                                const char *session_dictionary_name,
-                               lldb::DebuggerSP &debugger, const char *args,
+                               lldb::DebuggerSP debugger, const char *args,
                                lldb_private::CommandReturnObject &cmd_retobj,
                                lldb::ExecutionContextRefSP exe_ctx_ref_sp);
 
 bool LLDBSwigPythonCallCommandObject(
-    PyObject *implementor, lldb::DebuggerSP &debugger, const char *args,
+    PyObject *implementor, lldb::DebuggerSP debugger, const char *args,
     lldb_private::CommandReturnObject &cmd_retobj,
     lldb::ExecutionContextRefSP exe_ctx_ref_sp);
 
 bool LLDBSwigPythonCallModuleInit(const char *python_module_name,
                                   const char *session_dictionary_name,
-                                  lldb::DebuggerSP &debugger);
+                                  lldb::DebuggerSP debugger);
 
 void *LLDBSWIGPythonCreateOSPlugin(const char *python_class_name,
                                    const char *session_dictionary_name,
@@ -165,20 +166,20 @@ bool LLDBSWIGPythonRunScriptKeywordProcess(const char *python_function_name,
                                            const lldb::ProcessSP &process,
                                            std::string &output);
 
-bool LLDBSWIGPythonRunScriptKeywordThread(const char *python_function_name,
-                                          const char *session_dictionary_name,
-                                          lldb::ThreadSP &thread,
-                                          std::string &output);
+llvm::Optional<std::string>
+LLDBSWIGPythonRunScriptKeywordThread(const char *python_function_name,
+                                     const char *session_dictionary_name,
+                                     lldb::ThreadSP thread);
 
 bool LLDBSWIGPythonRunScriptKeywordTarget(const char *python_function_name,
                                           const char *session_dictionary_name,
                                           const lldb::TargetSP &target,
                                           std::string &output);
 
-bool LLDBSWIGPythonRunScriptKeywordFrame(const char *python_function_name,
-                                         const char *session_dictionary_name,
-                                         lldb::StackFrameSP &frame,
-                                         std::string &output);
+llvm::Optional<std::string>
+LLDBSWIGPythonRunScriptKeywordFrame(const char *python_function_name,
+                                    const char *session_dictionary_name,
+                                    lldb::StackFrameSP frame);
 
 bool LLDBSWIGPythonRunScriptKeywordValue(const char *python_function_name,
                                          const char *session_dictionary_name,
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
index 5f282d74e364..6afa4742698b 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
@@ -1718,7 +1718,7 @@ StructuredData::DictionarySP ScriptInterpreterPythonImpl::OSPlugin_CreateThread(
 }
 
 StructuredData::ObjectSP ScriptInterpreterPythonImpl::CreateScriptedThreadPlan(
-    const char *class_name, StructuredDataImpl *args_data,
+    const char *class_name, const StructuredDataImpl &args_data,
     std::string &error_str, lldb::ThreadPlanSP thread_plan_sp) {
   if (class_name == nullptr || class_name[0] == '\0')
     return StructuredData::ObjectSP();
@@ -1820,7 +1820,7 @@ lldb::StateType ScriptInterpreterPythonImpl::ScriptedThreadPlanGetRunState(
 
 StructuredData::GenericSP
 ScriptInterpreterPythonImpl::CreateScriptedBreakpointResolver(
-    const char *class_name, StructuredDataImpl *args_data,
+    const char *class_name, const StructuredDataImpl &args_data,
     lldb::BreakpointSP &bkpt_sp) {
 
   if (class_name == nullptr || class_name[0] == '\0')
@@ -1890,8 +1890,8 @@ ScriptInterpreterPythonImpl::ScriptedBreakpointResolverSearchDepth(
 }
 
 StructuredData::GenericSP ScriptInterpreterPythonImpl::CreateScriptedStopHook(
-    TargetSP target_sp, const char *class_name, StructuredDataImpl *args_data,
-    Status &error) {
+    TargetSP target_sp, const char *class_name,
+    const StructuredDataImpl &args_data, Status &error) {
 
   if (!target_sp) {
     error.SetErrorString("No target for scripted stop-hook.");
@@ -2197,7 +2197,7 @@ bool ScriptInterpreterPythonImpl::BreakpointCallbackFunction(
               LLDBSwigPythonBreakpointCallbackFunction(
                   python_function_name,
                   python_interpreter->m_dictionary_name.c_str(), stop_frame_sp,
-                  bp_loc_sp, bp_option_data->m_extra_args_up.get());
+                  bp_loc_sp, bp_option_data->m_extra_args);
 
           if (!maybe_ret_val) {
 
@@ -2521,7 +2521,6 @@ bool ScriptInterpreterPythonImpl::RunScriptFormatKeyword(
 bool ScriptInterpreterPythonImpl::RunScriptFormatKeyword(
     const char *impl_function, Thread *thread, std::string &output,
     Status &error) {
-  bool ret_val;
   if (!thread) {
     error.SetErrorString("no thread");
     return false;
@@ -2531,16 +2530,16 @@ bool ScriptInterpreterPythonImpl::RunScriptFormatKeyword(
     return false;
   }
 
-  {
-    ThreadSP thread_sp(thread->shared_from_this());
-    Locker py_lock(this,
-                   Locker::AcquireLock | Locker::InitSession | Locker::NoSTDIN);
-    ret_val = LLDBSWIGPythonRunScriptKeywordThread(
-        impl_function, m_dictionary_name.c_str(), thread_sp, output);
-    if (!ret_val)
-      error.SetErrorString("python script evaluation failed");
+  Locker py_lock(this,
+                 Locker::AcquireLock | Locker::InitSession | Locker::NoSTDIN);
+  if (llvm::Optional<std::string> result = LLDBSWIGPythonRunScriptKeywordThread(
+          impl_function, m_dictionary_name.c_str(),
+          thread->shared_from_this())) {
+    output = std::move(*result);
+    return true;
   }
-  return ret_val;
+  error.SetErrorString("python script evaluation failed");
+  return false;
 }
 
 bool ScriptInterpreterPythonImpl::RunScriptFormatKeyword(
@@ -2571,7 +2570,6 @@ bool ScriptInterpreterPythonImpl::RunScriptFormatKeyword(
 bool ScriptInterpreterPythonImpl::RunScriptFormatKeyword(
     const char *impl_function, StackFrame *frame, std::string &output,
     Status &error) {
-  bool ret_val;
   if (!frame) {
     error.SetErrorString("no frame");
     return false;
@@ -2581,16 +2579,16 @@ bool ScriptInterpreterPythonImpl::RunScriptFormatKeyword(
     return false;
   }
 
-  {
-    StackFrameSP frame_sp(frame->shared_from_this());
-    Locker py_lock(this,
-                   Locker::AcquireLock | Locker::InitSession | Locker::NoSTDIN);
-    ret_val = LLDBSWIGPythonRunScriptKeywordFrame(
-        impl_function, m_dictionary_name.c_str(), frame_sp, output);
-    if (!ret_val)
-      error.SetErrorString("python script evaluation failed");
+  Locker py_lock(this,
+                 Locker::AcquireLock | Locker::InitSession | Locker::NoSTDIN);
+  if (llvm::Optional<std::string> result = LLDBSWIGPythonRunScriptKeywordFrame(
+          impl_function, m_dictionary_name.c_str(),
+          frame->shared_from_this())) {
+    output = std::move(*result);
+    return true;
   }
-  return ret_val;
+  error.SetErrorString("python script evaluation failed");
+  return false;
 }
 
 bool ScriptInterpreterPythonImpl::RunScriptFormatKeyword(
@@ -2655,7 +2653,6 @@ bool ScriptInterpreterPythonImpl::LoadScriptingModule(
   }
 
   ScriptInterpreterIORedirect &io_redirect = **io_redirect_or_error;
-  lldb::DebuggerSP debugger_sp = m_debugger.shared_from_this();
 
   // Before executing Python code, lock the GIL.
   Locker py_lock(this,
@@ -2792,7 +2789,8 @@ bool ScriptInterpreterPythonImpl::LoadScriptingModule(
   // if we are here, everything worked
   // call __lldb_init_module(debugger,dict)
   if (!LLDBSwigPythonCallModuleInit(module_name.c_str(),
-                                    m_dictionary_name.c_str(), debugger_sp)) {
+                                    m_dictionary_name.c_str(),
+                                    m_debugger.shared_from_this())) {
     error.SetErrorString("calling __lldb_init_module failed");
     return false;
   }
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.h b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.h
index 8cfc24e71283..2e8301a85eb6 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.h
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.h
@@ -33,13 +33,12 @@ public:
     CommandDataPython() : BreakpointOptions::CommandData() {
       interpreter = lldb::eScriptLanguagePython;
     }
-    CommandDataPython(StructuredData::ObjectSP extra_args_sp) :
-        BreakpointOptions::CommandData(),
-        m_extra_args_up(new StructuredDataImpl()) {
-        interpreter = lldb::eScriptLanguagePython;
-        m_extra_args_up->SetObjectSP(extra_args_sp);
+    CommandDataPython(StructuredData::ObjectSP extra_args_sp)
+        : BreakpointOptions::CommandData(),
+          m_extra_args(std::move(extra_args_sp)) {
+      interpreter = lldb::eScriptLanguagePython;
     }
-    lldb::StructuredDataImplUP m_extra_args_up;
+    StructuredDataImpl m_extra_args;
   };
 
   ScriptInterpreterPython(Debugger &debugger)
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPythonImpl.h b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPythonImpl.h
index a3f83b696ed4..defc2acffcfa 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPythonImpl.h
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPythonImpl.h
@@ -79,7 +79,7 @@ public:
 
   StructuredData::ObjectSP
   CreateScriptedThreadPlan(const char *class_name,
-                           StructuredDataImpl *args_data,
+                           const StructuredDataImpl &args_data,
                            std::string &error_str,
                            lldb::ThreadPlanSP thread_plan) override;
 
@@ -99,7 +99,7 @@ public:
 
   StructuredData::GenericSP
   CreateScriptedBreakpointResolver(const char *class_name,
-                                   StructuredDataImpl *args_data,
+                                   const StructuredDataImpl &args_data,
                                    lldb::BreakpointSP &bkpt_sp) override;
   bool ScriptedBreakpointResolverSearchCallback(
       StructuredData::GenericSP implementor_sp,
@@ -110,7 +110,8 @@ public:
 
   StructuredData::GenericSP
   CreateScriptedStopHook(lldb::TargetSP target_sp, const char *class_name,
-                         StructuredDataImpl *args_data, Status &error) override;
+                         const StructuredDataImpl &args_data,
+                         Status &error) override;
 
   bool ScriptedStopHookHandleStop(StructuredData::GenericSP implementor_sp,
                                   ExecutionContext &exc_ctx,
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptedProcessPythonInterface.cpp b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptedProcessPythonInterface.cpp
index 29680dab5a14..e3c1931a565a 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptedProcessPythonInterface.cpp
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptedProcessPythonInterface.cpp
@@ -37,11 +37,7 @@ StructuredData::GenericSP ScriptedProcessPythonInterface::CreatePluginObject(
     return {};
 
   TargetSP target_sp = exe_ctx.GetTargetSP();
-  StructuredDataImpl *args_impl = nullptr;
-  if (args_sp) {
-    args_impl = new StructuredDataImpl();
-    args_impl->SetObjectSP(args_sp);
-  }
+  StructuredDataImpl args_impl(args_sp);
   std::string error_string;
 
   Locker py_lock(&m_interpreter, Locker::AcquireLock | Locker::NoSTDIN,
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptedThreadPythonInterface.cpp b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptedThreadPythonInterface.cpp
index d2c28bc426ee..6a881bfe625c 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptedThreadPythonInterface.cpp
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptedThreadPythonInterface.cpp
@@ -37,11 +37,7 @@ StructuredData::GenericSP ScriptedThreadPythonInterface::CreatePluginObject(
     return {};
 
   ProcessSP process_sp = exe_ctx.GetProcessSP();
-  StructuredDataImpl *args_impl = nullptr;
-  if (args_sp) {
-    args_impl = new StructuredDataImpl();
-    args_impl->SetObjectSP(args_sp);
-  }
+  StructuredDataImpl args_impl(args_sp);
   std::string error_string;
 
   Locker py_lock(&m_interpreter, Locker::AcquireLock | Locker::NoSTDIN,
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
index 4ac6e165dda3..b90f104c4d21 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
@@ -1530,7 +1530,6 @@ TypeSP DWARFASTParserClang::UpdateSymbolContextScopeForType(
     return type_sp;
 
   SymbolFileDWARF *dwarf = die.GetDWARF();
-  TypeList &type_list = dwarf->GetTypeList();
   DWARFDIE sc_parent_die = SymbolFileDWARF::GetParentSymbolContextDIE(die);
   dw_tag_t sc_parent_tag = sc_parent_die.Tag();
 
@@ -1550,10 +1549,6 @@ TypeSP DWARFASTParserClang::UpdateSymbolContextScopeForType(
   if (symbol_context_scope != nullptr)
     type_sp->SetSymbolContextScope(symbol_context_scope);
 
-  // We are ready to put this type into the uniqued list up at the module
-  // level.
-  type_list.Insert(type_sp);
-
   dwarf->GetDIEToType()[die.GetDIE()] = type_sp.get();
   return type_sp;
 }
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.h
index cece29dcf9ac..71d4c1e6c52f 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.h
@@ -153,7 +153,7 @@ public:
   const DWARFAbbreviationDeclarationSet *GetAbbreviations() const;
   dw_offset_t GetAbbrevOffset() const;
   uint8_t GetAddressByteSize() const { return m_header.GetAddressByteSize(); }
-  dw_addr_t GetAddrBase() const { return m_addr_base ? *m_addr_base : 0; }
+  dw_addr_t GetAddrBase() const { return m_addr_base.getValueOr(0); }
   dw_addr_t GetBaseAddress() const { return m_base_addr; }
   dw_offset_t GetLineTableOffset();
   dw_addr_t GetRangesBase() const { return m_ranges_base; }
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
index 8c20244a6c44..8c995ef2eb2a 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
@@ -1097,7 +1097,8 @@ bool SymbolFileDWARF::ParseImportedModules(
       if (const char *include_path = module_die.GetAttributeValueAsString(
               DW_AT_LLVM_include_path, nullptr)) {
         FileSpec include_spec(include_path, dwarf_cu->GetPathStyle());
-        MakeAbsoluteAndRemap(include_spec, *dwarf_cu, m_objfile_sp->GetModule());
+        MakeAbsoluteAndRemap(include_spec, *dwarf_cu,
+                             m_objfile_sp->GetModule());
         module.search_path = ConstString(include_spec.GetPath());
       }
       if (const char *sysroot = dwarf_cu->DIE().GetAttributeValueAsString(
@@ -1924,7 +1925,7 @@ void SymbolFileDWARF::ResolveFunctionAndBlock(lldb::addr_t file_vm_addr,
       block_die = function_die.LookupDeepestBlock(file_vm_addr);
   }
 
-  if (!sc.function || ! lookup_block)
+  if (!sc.function || !lookup_block)
     return;
 
   Block &block = sc.function->GetBlock(true);
@@ -2319,7 +2320,8 @@ void SymbolFileDWARF::FindFunctions(ConstString name,
   if (log) {
     GetObjectFile()->GetModule()->LogMessage(
         log,
-        "SymbolFileDWARF::FindFunctions (name=\"%s\", name_type_mask=0x%x, sc_list)",
+        "SymbolFileDWARF::FindFunctions (name=\"%s\", name_type_mask=0x%x, "
+        "sc_list)",
         name.GetCString(), name_type_mask);
   }
 
@@ -2352,8 +2354,7 @@ void SymbolFileDWARF::FindFunctions(ConstString name,
         log,
         "SymbolFileDWARF::FindFunctions (name=\"%s\", "
         "name_type_mask=0x%x, include_inlines=%d, sc_list) => %u",
-        name.GetCString(), name_type_mask, include_inlines,
-        num_matches);
+        name.GetCString(), name_type_mask, include_inlines, num_matches);
   }
 }
 
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
index 271ce7be1eea..e81ce28cb86e 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
@@ -253,8 +253,8 @@ public:
       ExternalTypeModuleMap;
 
   /// Return the list of Clang modules imported by this SymbolFile.
-  const ExternalTypeModuleMap& getExternalTypeModules() const {
-      return m_external_type_modules;
+  const ExternalTypeModuleMap &getExternalTypeModules() const {
+    return m_external_type_modules;
   }
 
   virtual DWARFDIE GetDIE(const DIERef &die_ref);
@@ -328,7 +328,6 @@ public:
     return m_parse_time;
   }
 
-
 protected:
   typedef llvm::DenseMap<const DWARFDebugInfoEntry *, lldb_private::Type *>
       DIEToTypePtr;
@@ -428,9 +427,10 @@ protected:
   virtual lldb::TypeSP
   FindDefinitionTypeForDWARFDeclContext(const DWARFDeclContext &die_decl_ctx);
 
-  virtual lldb::TypeSP FindCompleteObjCDefinitionTypeForDIE(
-      const DWARFDIE &die, lldb_private::ConstString type_name,
-      bool must_be_implementation);
+  virtual lldb::TypeSP
+  FindCompleteObjCDefinitionTypeForDIE(const DWARFDIE &die,
+                                       lldb_private::ConstString type_name,
+                                       bool must_be_implementation);
 
   lldb_private::Symbol *
   GetObjCClassSymbol(lldb_private::ConstString objc_class_name);
diff --git a/lldb/source/Plugins/SymbolFile/NativePDB/PdbAstBuilder.cpp b/lldb/source/Plugins/SymbolFile/NativePDB/PdbAstBuilder.cpp
index c29fc2230a67..9473befa6cc3 100644
--- a/lldb/source/Plugins/SymbolFile/NativePDB/PdbAstBuilder.cpp
+++ b/lldb/source/Plugins/SymbolFile/NativePDB/PdbAstBuilder.cpp
@@ -30,6 +30,70 @@ using namespace lldb_private::npdb;
 using namespace llvm::codeview;
 using namespace llvm::pdb;
 
+namespace {
+struct CreateMethodDecl : public TypeVisitorCallbacks {
+  CreateMethodDecl(PdbIndex &m_index, TypeSystemClang &m_clang,
+                   TypeIndex func_type_index,
+                   clang::FunctionDecl *&function_decl,
+                   lldb::opaque_compiler_type_t parent_ty,
+                   llvm::StringRef proc_name, CompilerType func_ct)
+      : m_index(m_index), m_clang(m_clang), func_type_index(func_type_index),
+        function_decl(function_decl), parent_ty(parent_ty),
+        proc_name(proc_name), func_ct(func_ct) {}
+  PdbIndex &m_index;
+  TypeSystemClang &m_clang;
+  TypeIndex func_type_index;
+  clang::FunctionDecl *&function_decl;
+  lldb::opaque_compiler_type_t parent_ty;
+  llvm::StringRef proc_name;
+  CompilerType func_ct;
+
+  llvm::Error visitKnownMember(CVMemberRecord &cvr,
+                               OverloadedMethodRecord &overloaded) override {
+    TypeIndex method_list_idx = overloaded.MethodList;
+
+    CVType method_list_type = m_index.tpi().getType(method_list_idx);
+    assert(method_list_type.kind() == LF_METHODLIST);
+
+    MethodOverloadListRecord method_list;
+    llvm::cantFail(TypeDeserializer::deserializeAs<MethodOverloadListRecord>(
+        method_list_type, method_list));
+
+    for (const OneMethodRecord &method : method_list.Methods) {
+      if (method.getType().getIndex() == func_type_index.getIndex())
+        AddMethod(overloaded.Name, method.getAccess(), method.getOptions(),
+                  method.Attrs);
+    }
+
+    return llvm::Error::success();
+  }
+
+  llvm::Error visitKnownMember(CVMemberRecord &cvr,
+                               OneMethodRecord &record) override {
+    AddMethod(record.getName(), record.getAccess(), record.getOptions(),
+              record.Attrs);
+    return llvm::Error::success();
+  }
+
+  void AddMethod(llvm::StringRef name, MemberAccess access,
+                 MethodOptions options, MemberAttributes attrs) {
+    if (name != proc_name || function_decl)
+      return;
+    lldb::AccessType access_type = TranslateMemberAccess(access);
+    bool is_virtual = attrs.isVirtual();
+    bool is_static = attrs.isStatic();
+    bool is_artificial = (options & MethodOptions::CompilerGenerated) ==
+                         MethodOptions::CompilerGenerated;
+    function_decl = m_clang.AddMethodToCXXRecordType(
+        parent_ty, proc_name,
+        /*mangled_name=*/nullptr, func_ct, /*access=*/access_type,
+        /*is_virtual=*/is_virtual, /*is_static=*/is_static,
+        /*is_inline=*/false, /*is_explicit=*/false,
+        /*is_attr_used=*/false, /*is_artificial=*/is_artificial);
+  }
+};
+} // namespace
+
 static llvm::Optional<PdbCompilandSymId> FindSymbolScope(PdbIndex &index,
                                                          PdbCompilandSymId id) {
   CVSymbol sym = index.ReadSymbolRecord(id);
@@ -681,7 +745,8 @@ bool PdbAstBuilder::CompleteTagDecl(clang::TagDecl &tag) {
   // Visit all members of this class, then perform any finalization necessary
   // to complete the class.
   CompilerType ct = ToCompilerType(tag_qt);
-  UdtRecordCompleter completer(best_ti, ct, tag, *this, m_index);
+  UdtRecordCompleter completer(best_ti, ct, tag, *this, m_index,
+                               m_cxx_record_map);
   auto error =
       llvm::codeview::visitMemberRecordStream(field_list_cvt.data(), completer);
   completer.complete();
@@ -1014,8 +1079,62 @@ PdbAstBuilder::GetOrCreateFunctionDecl(PdbCompilandSymId func_id) {
   proc_name.consume_front(context_name);
   proc_name.consume_front("::");
 
-  clang::FunctionDecl *function_decl = m_clang.CreateFunctionDeclaration(
-      parent, OptionalClangModuleID(), proc_name, func_ct, storage, false);
+  clang::FunctionDecl *function_decl = nullptr;
+  if (parent->isRecord()) {
+    clang::QualType parent_qt = llvm::dyn_cast<clang::TypeDecl>(parent)
+                                    ->getTypeForDecl()
+                                    ->getCanonicalTypeInternal();
+    lldb::opaque_compiler_type_t parent_opaque_ty =
+        ToCompilerType(parent_qt).GetOpaqueQualType();
+
+    auto iter = m_cxx_record_map.find(parent_opaque_ty);
+    if (iter != m_cxx_record_map.end()) {
+      if (iter->getSecond().contains({proc_name, func_ct})) {
+        return nullptr;
+      }
+    }
+
+    CVType cvt = m_index.tpi().getType(type_id.index);
+    MemberFunctionRecord func_record(static_cast<TypeRecordKind>(cvt.kind()));
+    llvm::cantFail(TypeDeserializer::deserializeAs<MemberFunctionRecord>(
+        cvt, func_record));
+    TypeIndex class_index = func_record.getClassType();
+    CVType parent_cvt = m_index.tpi().getType(class_index);
+    ClassRecord class_record = CVTagRecord::create(parent_cvt).asClass();
+    // If it's a forward reference, try to get the real TypeIndex.
+    if (class_record.isForwardRef()) {
+      llvm::Expected<TypeIndex> eti =
+          m_index.tpi().findFullDeclForForwardRef(class_index);
+      if (eti) {
+        class_record =
+            CVTagRecord::create(m_index.tpi().getType(*eti)).asClass();
+      }
+    }
+    if (!class_record.FieldList.isSimple()) {
+      CVType field_list = m_index.tpi().getType(class_record.FieldList);
+      CreateMethodDecl process(m_index, m_clang, type_id.index, function_decl,
+                               parent_opaque_ty, proc_name, func_ct);
+      if (llvm::Error err = visitMemberRecordStream(field_list.data(), process))
+        llvm::consumeError(std::move(err));
+    }
+
+    if (!function_decl) {
+      function_decl = m_clang.AddMethodToCXXRecordType(
+          parent_opaque_ty, proc_name,
+          /*mangled_name=*/nullptr, func_ct,
+          /*access=*/lldb::AccessType::eAccessPublic,
+          /*is_virtual=*/false, /*is_static=*/false,
+          /*is_inline=*/false, /*is_explicit=*/false,
+          /*is_attr_used=*/false, /*is_artificial=*/false);
+    }
+
+    m_cxx_record_map[parent_opaque_ty].insert({proc_name, func_ct});
+  } else {
+    function_decl = m_clang.CreateFunctionDeclaration(
+        parent, OptionalClangModuleID(), proc_name, func_ct, storage, false);
+    CreateFunctionParameters(func_id, *function_decl,
+                             func_type->getNumParams());
+  }
 
   lldbassert(m_uid_to_decl.count(toOpaqueUid(func_id)) == 0);
   m_uid_to_decl[toOpaqueUid(func_id)] = function_decl;
@@ -1024,8 +1143,6 @@ PdbAstBuilder::GetOrCreateFunctionDecl(PdbCompilandSymId func_id) {
   status.uid = toOpaqueUid(func_id);
   m_decl_to_status.insert({function_decl, status});
 
-  CreateFunctionParameters(func_id, *function_decl, func_type->getNumParams());
-
   return function_decl;
 }
 
@@ -1163,15 +1280,15 @@ clang::QualType PdbAstBuilder::CreateFunctionType(
 }
 
 static bool isTagDecl(clang::DeclContext &context) {
-  return !!llvm::dyn_cast<clang::TagDecl>(&context);
+  return llvm::isa<clang::TagDecl>(&context);
 }
 
 static bool isFunctionDecl(clang::DeclContext &context) {
-  return !!llvm::dyn_cast<clang::FunctionDecl>(&context);
+  return llvm::isa<clang::FunctionDecl>(&context);
 }
 
 static bool isBlockDecl(clang::DeclContext &context) {
-  return !!llvm::dyn_cast<clang::BlockDecl>(&context);
+  return llvm::isa<clang::BlockDecl>(&context);
 }
 
 void PdbAstBuilder::ParseAllNamespacesPlusChildrenOf(
diff --git a/lldb/source/Plugins/SymbolFile/NativePDB/PdbAstBuilder.h b/lldb/source/Plugins/SymbolFile/NativePDB/PdbAstBuilder.h
index 7bb2584d19a3..73accf5e5e68 100644
--- a/lldb/source/Plugins/SymbolFile/NativePDB/PdbAstBuilder.h
+++ b/lldb/source/Plugins/SymbolFile/NativePDB/PdbAstBuilder.h
@@ -137,6 +137,12 @@ private:
   llvm::DenseMap<clang::Decl *, DeclStatus> m_decl_to_status;
   llvm::DenseMap<lldb::user_id_t, clang::Decl *> m_uid_to_decl;
   llvm::DenseMap<lldb::user_id_t, clang::QualType> m_uid_to_type;
+
+  // From class/struct's opaque_compiler_type_t to a set containing the pairs of
+  // method's name and CompilerType.
+  llvm::DenseMap<lldb::opaque_compiler_type_t,
+                 llvm::SmallSet<std::pair<llvm::StringRef, CompilerType>, 8>>
+      m_cxx_record_map;
 };
 
 } // namespace npdb
diff --git a/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp b/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp
index bf101ac1acf1..e859b1d5a86c 100644
--- a/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp
+++ b/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp
@@ -900,7 +900,7 @@ lldb::LanguageType SymbolFileNativePDB::ParseLanguage(CompileUnit &comp_unit) {
   return TranslateLanguage(item->m_compile_opts->getLanguage());
 }
 
-void SymbolFileNativePDB::AddSymbols(Symtab &symtab) { return; }
+void SymbolFileNativePDB::AddSymbols(Symtab &symtab) {}
 
 size_t SymbolFileNativePDB::ParseFunctions(CompileUnit &comp_unit) {
   std::lock_guard<std::recursive_mutex> guard(GetModuleMutex());
diff --git a/lldb/source/Plugins/SymbolFile/NativePDB/UdtRecordCompleter.cpp b/lldb/source/Plugins/SymbolFile/NativePDB/UdtRecordCompleter.cpp
index c8fb46c75034..d0b27bc5bf79 100644
--- a/lldb/source/Plugins/SymbolFile/NativePDB/UdtRecordCompleter.cpp
+++ b/lldb/source/Plugins/SymbolFile/NativePDB/UdtRecordCompleter.cpp
@@ -28,13 +28,15 @@ using namespace lldb_private::npdb;
 
 using Error = llvm::Error;
 
-UdtRecordCompleter::UdtRecordCompleter(PdbTypeSymId id,
-                                       CompilerType &derived_ct,
-                                       clang::TagDecl &tag_decl,
-                                       PdbAstBuilder &ast_builder,
-                                       PdbIndex &index)
+UdtRecordCompleter::UdtRecordCompleter(
+    PdbTypeSymId id, CompilerType &derived_ct, clang::TagDecl &tag_decl,
+    PdbAstBuilder &ast_builder, PdbIndex &index,
+    llvm::DenseMap<lldb::opaque_compiler_type_t,
+                   llvm::SmallSet<std::pair<llvm::StringRef, CompilerType>, 8>>
+        &cxx_record_map)
     : m_id(id), m_derived_ct(derived_ct), m_tag_decl(tag_decl),
-      m_ast_builder(ast_builder), m_index(index) {
+      m_ast_builder(ast_builder), m_index(index),
+      m_cxx_record_map(cxx_record_map) {
   CVType cvt = m_index.tpi().getType(m_id.index);
   switch (cvt.kind()) {
   case LF_ENUM:
@@ -78,14 +80,24 @@ void UdtRecordCompleter::AddMethod(llvm::StringRef name, TypeIndex type_idx,
   clang::QualType method_qt =
       m_ast_builder.GetOrCreateType(PdbTypeSymId(type_idx));
   m_ast_builder.CompleteType(method_qt);
+  CompilerType method_ct = m_ast_builder.ToCompilerType(method_qt);
+  lldb::opaque_compiler_type_t derived_opaque_ty = m_derived_ct.GetOpaqueQualType();
+  auto iter = m_cxx_record_map.find(derived_opaque_ty);
+  if (iter != m_cxx_record_map.end()) {
+    if (iter->getSecond().contains({name, method_ct})) {
+      return;
+    }
+  }
 
   lldb::AccessType access_type = TranslateMemberAccess(access);
   bool is_artificial = (options & MethodOptions::CompilerGenerated) ==
                        MethodOptions::CompilerGenerated;
   m_ast_builder.clang().AddMethodToCXXRecordType(
-      m_derived_ct.GetOpaqueQualType(), name.data(), nullptr,
-      m_ast_builder.ToCompilerType(method_qt), access_type, attrs.isVirtual(),
-      attrs.isStatic(), false, false, false, is_artificial);
+      derived_opaque_ty, name.data(), nullptr, method_ct,
+      access_type, attrs.isVirtual(), attrs.isStatic(), false, false, false,
+      is_artificial);
+
+  m_cxx_record_map[derived_opaque_ty].insert({name, method_ct});
 }
 
 Error UdtRecordCompleter::visitKnownMember(CVMemberRecord &cvr,
diff --git a/lldb/source/Plugins/SymbolFile/NativePDB/UdtRecordCompleter.h b/lldb/source/Plugins/SymbolFile/NativePDB/UdtRecordCompleter.h
index ae7e47c82fe5..9c6b5ed28bc2 100644
--- a/lldb/source/Plugins/SymbolFile/NativePDB/UdtRecordCompleter.h
+++ b/lldb/source/Plugins/SymbolFile/NativePDB/UdtRecordCompleter.h
@@ -54,11 +54,17 @@ class UdtRecordCompleter : public llvm::codeview::TypeVisitorCallbacks {
   PdbIndex &m_index;
   std::vector<IndexedBase> m_bases;
   ClangASTImporter::LayoutInfo m_layout;
+  llvm::DenseMap<lldb::opaque_compiler_type_t,
+                 llvm::SmallSet<std::pair<llvm::StringRef, CompilerType>, 8>>
+      &m_cxx_record_map;
 
 public:
-  UdtRecordCompleter(PdbTypeSymId id, CompilerType &derived_ct,
-                     clang::TagDecl &tag_decl, PdbAstBuilder &ast_builder,
-                     PdbIndex &index);
+  UdtRecordCompleter(
+      PdbTypeSymId id, CompilerType &derived_ct, clang::TagDecl &tag_decl,
+      PdbAstBuilder &ast_builder, PdbIndex &index,
+      llvm::DenseMap<lldb::opaque_compiler_type_t,
+                     llvm::SmallSet<std::pair<llvm::StringRef, CompilerType>,
+                                    8>> &cxx_record_map);
 
 #define MEMBER_RECORD(EnumName, EnumVal, Name)                                 \
   llvm::Error visitKnownMember(llvm::codeview::CVMemberRecord &CVR,            \
diff --git a/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.cpp b/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.cpp
index db0ae241be7e..a40b6ec9a635 100644
--- a/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.cpp
+++ b/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.cpp
@@ -239,7 +239,6 @@ void SymbolFilePDB::GetCompileUnitIndex(
     }
   }
   index = UINT32_MAX;
-  return;
 }
 
 std::unique_ptr<llvm::pdb::PDBSymbolCompiland>
@@ -402,7 +401,7 @@ static size_t ParseFunctionBlocksForPDBSymbol(
         block = parent_block;
       else
         break;
-    } else if (llvm::dyn_cast<PDBSymbolBlock>(pdb_symbol)) {
+    } else if (llvm::isa<PDBSymbolBlock>(pdb_symbol)) {
       auto uid = pdb_symbol->getSymIndexId();
       if (parent_block->FindBlockByID(uid))
         break;
diff --git a/lldb/source/Plugins/TraceExporter/ctf/CommandObjectThreadTraceExportCTF.cpp b/lldb/source/Plugins/TraceExporter/ctf/CommandObjectThreadTraceExportCTF.cpp
index 919cdf46a5c0..a72e46a0b703 100644
--- a/lldb/source/Plugins/TraceExporter/ctf/CommandObjectThreadTraceExportCTF.cpp
+++ b/lldb/source/Plugins/TraceExporter/ctf/CommandObjectThreadTraceExportCTF.cpp
@@ -73,8 +73,7 @@ bool CommandObjectThreadTraceExportCTF::DoExecute(Args &command,
 
   if (thread == nullptr) {
     const uint32_t num_threads = process->GetThreadList().GetSize();
-    size_t tid = m_options.m_thread_index ? *m_options.m_thread_index
-                                          : LLDB_INVALID_THREAD_ID;
+    size_t tid = m_options.m_thread_index.getValueOr(LLDB_INVALID_THREAD_ID);
     result.AppendErrorWithFormatv(
         "Thread index {0} is out of range (valid values are 1 - {1}).\n", tid,
         num_threads);
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
index b1dbc382ff04..88c3aedb4c6b 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
@@ -2581,6 +2581,7 @@ RemoveWrappingTypes(QualType type, ArrayRef<clang::Type::TypeClass> mask = {}) {
     case clang::Type::Typedef:
     case clang::Type::TypeOf:
     case clang::Type::TypeOfExpr:
+    case clang::Type::Using:
       type = type->getLocallyUnqualifiedSingleStepDesugaredType();
       break;
     default:
@@ -4063,6 +4064,7 @@ TypeSystemClang::GetTypeClass(lldb::opaque_compiler_type_t type) {
   case clang::Type::Paren:
   case clang::Type::TypeOf:
   case clang::Type::TypeOfExpr:
+  case clang::Type::Using:
     llvm_unreachable("Handled in RemoveWrappingTypes!");
   case clang::Type::UnaryTransform:
     break;
@@ -4088,8 +4090,8 @@ TypeSystemClang::GetTypeClass(lldb::opaque_compiler_type_t type) {
     return lldb::eTypeClassVector;
   case clang::Type::Builtin:
   // Ext-Int is just an integer type.
-  case clang::Type::ExtInt:
-  case clang::Type::DependentExtInt:
+  case clang::Type::BitInt:
+  case clang::Type::DependentBitInt:
     return lldb::eTypeClassBuiltin;
   case clang::Type::ObjCObjectPointer:
     return lldb::eTypeClassObjCObjectPointer;
@@ -4722,6 +4724,7 @@ lldb::Encoding TypeSystemClang::GetEncoding(lldb::opaque_compiler_type_t type,
   case clang::Type::Typedef:
   case clang::Type::TypeOf:
   case clang::Type::TypeOfExpr:
+  case clang::Type::Using:
     llvm_unreachable("Handled in RemoveWrappingTypes!");
 
   case clang::Type::UnaryTransform:
@@ -4744,8 +4747,8 @@ lldb::Encoding TypeSystemClang::GetEncoding(lldb::opaque_compiler_type_t type,
     // TODO: Set this to more than one???
     break;
 
-  case clang::Type::ExtInt:
-  case clang::Type::DependentExtInt:
+  case clang::Type::BitInt:
+  case clang::Type::DependentBitInt:
     return qual_type->isUnsignedIntegerType() ? lldb::eEncodingUint
                                               : lldb::eEncodingSint;
 
@@ -5104,6 +5107,7 @@ lldb::Format TypeSystemClang::GetFormat(lldb::opaque_compiler_type_t type) {
   case clang::Type::Typedef:
   case clang::Type::TypeOf:
   case clang::Type::TypeOfExpr:
+  case clang::Type::Using:
     llvm_unreachable("Handled in RemoveWrappingTypes!");
   case clang::Type::UnaryTransform:
     break;
@@ -5124,8 +5128,8 @@ lldb::Format TypeSystemClang::GetFormat(lldb::opaque_compiler_type_t type) {
   case clang::Type::Vector:
     break;
 
-  case clang::Type::ExtInt:
-  case clang::Type::DependentExtInt:
+  case clang::Type::BitInt:
+  case clang::Type::DependentBitInt:
     return qual_type->isUnsignedIntegerType() ? lldb::eFormatUnsigned
                                               : lldb::eFormatDecimal;
 
@@ -5145,6 +5149,8 @@ lldb::Format TypeSystemClang::GetFormat(lldb::opaque_compiler_type_t type) {
     case clang::BuiltinType::UChar:
     case clang::BuiltinType::WChar_U:
       return lldb::eFormatChar;
+    case clang::BuiltinType::Char8:
+      return lldb::eFormatUnicode8;
     case clang::BuiltinType::Char16:
       return lldb::eFormatUnicode16;
     case clang::BuiltinType::Char32:
@@ -8953,6 +8959,7 @@ bool TypeSystemClang::DumpTypeValue(
         case eFormatCharPrintable:
         case eFormatCharArray:
         case eFormatBytes:
+        case eFormatUnicode8:
         case eFormatBytesWithASCII:
           item_count = byte_size;
           byte_size = 1;
diff --git a/lldb/source/Symbol/ObjectFile.cpp b/lldb/source/Symbol/ObjectFile.cpp
index bfab741b0d66..e7b19f5e0c51 100644
--- a/lldb/source/Symbol/ObjectFile.cpp
+++ b/lldb/source/Symbol/ObjectFile.cpp
@@ -23,6 +23,8 @@
 #include "lldb/Utility/Timer.h"
 #include "lldb/lldb-private.h"
 
+#include "llvm/Support/DJB.h"
+
 using namespace lldb;
 using namespace lldb_private;
 
@@ -737,13 +739,24 @@ Symtab *ObjectFile::GetSymtab() {
     // not be able to access the symbol table contents since all APIs in Symtab
     // are protected by a mutex in the Symtab object itself.
     llvm::call_once(*m_symtab_once_up, [&]() {
-       ElapsedTime elapsed(module_sp->GetSymtabParseTime());
-       Symtab *symtab = new Symtab(this);
-       std::lock_guard<std::recursive_mutex> symtab_guard(symtab->GetMutex());
-       m_symtab_up.reset(symtab);
-       ParseSymtab(*m_symtab_up);
-       m_symtab_up->Finalize();
+      Symtab *symtab = new Symtab(this);
+      std::lock_guard<std::recursive_mutex> symtab_guard(symtab->GetMutex());
+      m_symtab_up.reset(symtab);
+      if (!m_symtab_up->LoadFromCache()) {
+        ElapsedTime elapsed(module_sp->GetSymtabParseTime());
+        ParseSymtab(*m_symtab_up);
+        m_symtab_up->Finalize();
+      }
     });
   }
   return m_symtab_up.get();
 }
+
+uint32_t ObjectFile::GetCacheHash() {
+  if (m_cache_hash)
+    return *m_cache_hash;
+  StreamString strm;
+  strm.Format("{0}-{1}-{2}", m_file, GetType(), GetStrata());
+  m_cache_hash = llvm::djbHash(strm.GetString());
+  return *m_cache_hash;
+}
diff --git a/lldb/source/Symbol/Symbol.cpp b/lldb/source/Symbol/Symbol.cpp
index a8c81ee3082f..fa7226dfd046 100644
--- a/lldb/source/Symbol/Symbol.cpp
+++ b/lldb/source/Symbol/Symbol.cpp
@@ -17,6 +17,7 @@
 #include "lldb/Symbol/Symtab.h"
 #include "lldb/Target/Process.h"
 #include "lldb/Target/Target.h"
+#include "lldb/Utility/DataEncoder.h"
 #include "lldb/Utility/Stream.h"
 
 using namespace lldb;
@@ -595,3 +596,131 @@ void Symbol::SynthesizeNameIfNeeded() const {
     m_mangled.SetDemangledName(ConstString(os.str()));
   }
 }
+
+bool Symbol::Decode(const DataExtractor &data, lldb::offset_t *offset_ptr,
+                    const SectionList *section_list,
+                    const StringTableReader &strtab) {
+  if (!data.ValidOffsetForDataOfSize(*offset_ptr, 8))
+    return false;
+  m_uid = data.GetU32(offset_ptr);
+  m_type_data = data.GetU16(offset_ptr);
+  const uint16_t bitfields = data.GetU16(offset_ptr);
+  m_type_data_resolved = (1u << 15 & bitfields) != 0;
+  m_is_synthetic = (1u << 14 & bitfields) != 0;
+  m_is_debug = (1u << 13 & bitfields) != 0;
+  m_is_external = (1u << 12 & bitfields) != 0;
+  m_size_is_sibling = (1u << 11 & bitfields) != 0;
+  m_size_is_synthesized = (1u << 10 & bitfields) != 0;
+  m_size_is_valid = (1u << 9 & bitfields) != 0;
+  m_demangled_is_synthesized = (1u << 8 & bitfields) != 0;
+  m_contains_linker_annotations = (1u << 7 & bitfields) != 0;
+  m_is_weak = (1u << 6 & bitfields) != 0;
+  m_type = bitfields & 0x003f;
+  if (!m_mangled.Decode(data, offset_ptr, strtab))
+    return false;
+  if (!data.ValidOffsetForDataOfSize(*offset_ptr, 20))
+    return false;
+  const bool is_addr = data.GetU8(offset_ptr) != 0;
+  const uint64_t value = data.GetU64(offset_ptr);
+  if (is_addr) {
+    m_addr_range.GetBaseAddress().ResolveAddressUsingFileSections(
+        value, section_list);
+  } else {
+    m_addr_range.GetBaseAddress().Clear();
+    m_addr_range.GetBaseAddress().SetOffset(value);
+  }
+  m_addr_range.SetByteSize(data.GetU64(offset_ptr));
+  m_flags =  data.GetU32(offset_ptr);
+  return true;
+}
+
+/// The encoding format for the symbol is as follows:
+///
+/// uint32_t m_uid;
+/// uint16_t m_type_data;
+/// uint16_t bitfield_data;
+/// Mangled mangled;
+/// uint8_t is_addr;
+/// uint64_t file_addr_or_value;
+/// uint64_t size;
+/// uint32_t flags;
+///
+/// The only tricky thing in this encoding is encoding all of the bits in the
+/// bitfields. We use a trick to store all bitfields as a 16 bit value and we
+/// do the same thing when decoding the symbol. There are test that ensure this
+/// encoding works for each individual bit. Everything else is very easy to
+/// store.
+void Symbol::Encode(DataEncoder &file, ConstStringTable &strtab) const {
+  file.AppendU32(m_uid);
+  file.AppendU16(m_type_data);
+  uint16_t bitfields = m_type;
+  if (m_type_data_resolved)
+    bitfields |= 1u << 15;
+  if (m_is_synthetic)
+    bitfields |= 1u << 14;
+  if (m_is_debug)
+    bitfields |= 1u << 13;
+  if (m_is_external)
+    bitfields |= 1u << 12;
+  if (m_size_is_sibling)
+    bitfields |= 1u << 11;
+  if (m_size_is_synthesized)
+    bitfields |= 1u << 10;
+  if (m_size_is_valid)
+    bitfields |= 1u << 9;
+  if (m_demangled_is_synthesized)
+    bitfields |= 1u << 8;
+  if (m_contains_linker_annotations)
+    bitfields |= 1u << 7;
+  if (m_is_weak)
+    bitfields |= 1u << 6;
+  file.AppendU16(bitfields);
+  m_mangled.Encode(file, strtab);
+  // A symbol's value might be an address, or it might be a constant. If the
+  // symbol's base address doesn't have a section, then it is a constant value.
+  // If it does have a section, we will encode the file address and re-resolve
+  // the address when we decode it.
+  bool is_addr = m_addr_range.GetBaseAddress().GetSection().get() != NULL;
+  file.AppendU8(is_addr);
+  file.AppendU64(m_addr_range.GetBaseAddress().GetFileAddress());
+  file.AppendU64(m_addr_range.GetByteSize());
+  file.AppendU32(m_flags);
+}
+
+bool Symbol::operator==(const Symbol &rhs) const {
+  if (m_uid != rhs.m_uid)
+    return false;
+  if (m_type_data != rhs.m_type_data)
+    return false;
+  if (m_type_data_resolved != rhs.m_type_data_resolved)
+    return false;
+  if (m_is_synthetic != rhs.m_is_synthetic)
+    return false;
+  if (m_is_debug != rhs.m_is_debug)
+    return false;
+  if (m_is_external != rhs.m_is_external)
+    return false;
+  if (m_size_is_sibling != rhs.m_size_is_sibling)
+    return false;
+  if (m_size_is_synthesized != rhs.m_size_is_synthesized)
+    return false;
+  if (m_size_is_valid != rhs.m_size_is_valid)
+    return false;
+  if (m_demangled_is_synthesized != rhs.m_demangled_is_synthesized)
+    return false;
+  if (m_contains_linker_annotations != rhs.m_contains_linker_annotations)
+    return false;
+  if (m_is_weak != rhs.m_is_weak)
+    return false;
+  if (m_type != rhs.m_type)
+    return false;
+  if (m_mangled != rhs.m_mangled)
+    return false;
+  if (m_addr_range.GetBaseAddress() != rhs.m_addr_range.GetBaseAddress())
+    return false;
+  if (m_addr_range.GetByteSize() != rhs.m_addr_range.GetByteSize())
+    return false;
+  if (m_flags != rhs.m_flags)
+    return false;
+  return true;
+}
diff --git a/lldb/source/Symbol/SymbolFile.cpp b/lldb/source/Symbol/SymbolFile.cpp
index 53f8dd68c8b7..b85901af4d67 100644
--- a/lldb/source/Symbol/SymbolFile.cpp
+++ b/lldb/source/Symbol/SymbolFile.cpp
@@ -125,9 +125,7 @@ void SymbolFile::FindFunctions(const RegularExpression &regex,
 
 void SymbolFile::GetMangledNamesForFunction(
     const std::string &scope_qualified_name,
-    std::vector<ConstString> &mangled_names) {
-  return;
-}
+    std::vector<ConstString> &mangled_names) {}
 
 void SymbolFile::FindTypes(
     ConstString name, const CompilerDeclContext &parent_decl_ctx,
@@ -147,9 +145,11 @@ void SymbolFile::AssertModuleLock() {
   // We assert that we have to module lock by trying to acquire the lock from a
   // different thread. Note that we must abort if the result is true to
   // guarantee correctness.
-  assert(std::async(std::launch::async,
-                    [this] { return this->GetModuleMutex().try_lock(); })
-                 .get() == false &&
+  assert(std::async(
+             std::launch::async,
+             [this] {
+               return this->GetModuleMutex().try_lock();
+             }).get() == false &&
          "Module is not locked");
 #endif
 }
diff --git a/lldb/source/Symbol/Symtab.cpp b/lldb/source/Symbol/Symtab.cpp
index c67955523bfb..75450a156c28 100644
--- a/lldb/source/Symbol/Symtab.cpp
+++ b/lldb/source/Symbol/Symtab.cpp
@@ -9,6 +9,7 @@
 #include <map>
 #include <set>
 
+#include "lldb/Core/DataFileCache.h"
 #include "lldb/Core/Module.h"
 #include "lldb/Core/RichManglingContext.h"
 #include "lldb/Core/Section.h"
@@ -17,11 +18,15 @@
 #include "lldb/Symbol/SymbolContext.h"
 #include "lldb/Symbol/Symtab.h"
 #include "lldb/Target/Language.h"
+#include "lldb/Utility/DataEncoder.h"
+#include "lldb/Utility/Endian.h"
 #include "lldb/Utility/RegularExpression.h"
 #include "lldb/Utility/Stream.h"
 #include "lldb/Utility/Timer.h"
 
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/DJB.h"
 
 using namespace lldb;
 using namespace lldb_private;
@@ -109,7 +114,8 @@ void Symtab::Dump(Stream *s, Target *target, SortOrder sort_order,
         s->Indent();
         pos->Dump(s, target, std::distance(begin, pos), name_preference);
       }
-    } break;
+    }
+    break;
 
     case eSortOrderByName: {
       // Although we maintain a lookup by exact name map, the table isn't
@@ -663,7 +669,6 @@ uint32_t Symtab::AppendSymbolIndexesWithName(ConstString symbol_name,
                                              std::vector<uint32_t> &indexes) {
   std::lock_guard<std::recursive_mutex> guard(m_mutex);
 
-  LLDB_SCOPED_TIMER();
   if (symbol_name) {
     if (!m_name_indexes_computed)
       InitNameIndexes();
@@ -808,7 +813,6 @@ Symtab::FindAllSymbolsWithNameAndType(ConstString name,
                                       std::vector<uint32_t> &symbol_indexes) {
   std::lock_guard<std::recursive_mutex> guard(m_mutex);
 
-  LLDB_SCOPED_TIMER();
   // Initialize all of the lookup by name indexes before converting NAME to a
   // uniqued string NAME_STR below.
   if (!m_name_indexes_computed)
@@ -1006,6 +1010,7 @@ void Symtab::Finalize() {
     collection new_symbols(m_symbols.begin(), m_symbols.end());
     m_symbols.swap(new_symbols);
   }
+  SaveToCache();
 }
 
 Symbol *Symtab::FindSymbolAtFileAddress(addr_t file_addr) {
@@ -1152,3 +1157,191 @@ const Symbol *Symtab::GetParent(Symbol *child_symbol) const {
   }
   return nullptr;
 }
+
+std::string Symtab::GetCacheKey() {
+  std::string key;
+  llvm::raw_string_ostream strm(key);
+  // Symbol table can come from different object files for the same module. A
+  // module can have one object file as the main executable and might have
+  // another object file in a separate symbol file.
+  strm << m_objfile->GetModule()->GetCacheKey() << "-symtab-"
+      << llvm::format_hex(m_objfile->GetCacheHash(), 10);
+  return strm.str();
+}
+
+void Symtab::SaveToCache() {
+  DataFileCache *cache = Module::GetIndexCache();
+  if (!cache)
+    return; // Caching is not enabled.
+  InitNameIndexes(); // Init the name indexes so we can cache them as well.
+  const auto byte_order = endian::InlHostByteOrder();
+  DataEncoder file(byte_order, /*addr_size=*/8);
+  // Encode will return false if the symbol table's object file doesn't have
+  // anything to make a signature from.
+  if (Encode(file))
+    cache->SetCachedData(GetCacheKey(), file.GetData());
+}
+
+constexpr llvm::StringLiteral kIdentifierCStrMap("CMAP");
+
+static void EncodeCStrMap(DataEncoder &encoder, ConstStringTable &strtab,
+                          const UniqueCStringMap<uint32_t> &cstr_map) {
+  encoder.AppendData(kIdentifierCStrMap);
+  encoder.AppendU32(cstr_map.GetSize());
+  for (const auto &entry: cstr_map) {
+    // Make sure there are no empty strings.
+    assert((bool)entry.cstring);
+    encoder.AppendU32(strtab.Add(entry.cstring));
+    encoder.AppendU32(entry.value);
+  }
+}
+
+bool DecodeCStrMap(const DataExtractor &data, lldb::offset_t *offset_ptr,
+                   const StringTableReader &strtab,
+                   UniqueCStringMap<uint32_t> &cstr_map) {
+  llvm::StringRef identifier((const char *)data.GetData(offset_ptr, 4), 4);
+  if (identifier != kIdentifierCStrMap)
+    return false;
+  const uint32_t count = data.GetU32(offset_ptr);
+  for (uint32_t i=0; i<count; ++i)
+  {
+    llvm::StringRef str(strtab.Get(data.GetU32(offset_ptr)));
+    uint32_t value = data.GetU32(offset_ptr);
+    // No empty strings in the name indexes in Symtab
+    if (str.empty())
+      return false;
+    cstr_map.Append(ConstString(str), value);
+  }
+  return true;
+}
+
+constexpr llvm::StringLiteral kIdentifierSymbolTable("SYMB");
+constexpr uint32_t CURRENT_CACHE_VERSION = 1;
+
+/// The encoding format for the symbol table is as follows:
+///
+/// Signature signature;
+/// ConstStringTable strtab;
+/// Identifier four character code: 'SYMB'
+/// uint32_t version;
+/// uint32_t num_symbols;
+/// Symbol symbols[num_symbols];
+/// uint8_t num_cstr_maps;
+/// UniqueCStringMap<uint32_t> cstr_maps[num_cstr_maps]
+bool Symtab::Encode(DataEncoder &encoder) const {
+  // Name indexes must be computed before calling this function.
+  assert(m_name_indexes_computed);
+
+  // Encode the object file's signature
+  CacheSignature signature(m_objfile);
+  if (!signature.Encode(encoder))
+    return false;
+  ConstStringTable strtab;
+
+  // Encoder the symbol table into a separate encoder first. This allows us
+  // gather all of the strings we willl need in "strtab" as we will need to
+  // write the string table out before the symbol table.
+  DataEncoder symtab_encoder(encoder.GetByteOrder(),
+                              encoder.GetAddressByteSize());
+  symtab_encoder.AppendData(kIdentifierSymbolTable);
+  // Encode the symtab data version.
+  symtab_encoder.AppendU32(CURRENT_CACHE_VERSION);
+  // Encode the number of symbols.
+  symtab_encoder.AppendU32(m_symbols.size());
+  // Encode the symbol data for all symbols.
+  for (const auto &symbol: m_symbols)
+    symbol.Encode(symtab_encoder, strtab);
+
+  // Emit a byte for how many C string maps we emit. We will fix this up after
+  // we emit the C string maps since we skip emitting C string maps if they are
+  // empty.
+  size_t num_cmaps_offset = symtab_encoder.GetByteSize();
+  uint8_t num_cmaps = 0;
+  symtab_encoder.AppendU8(0);
+  for (const auto &pair: m_name_to_symbol_indices) {
+    if (pair.second.IsEmpty())
+      continue;
+    ++num_cmaps;
+    symtab_encoder.AppendU8(pair.first);
+    EncodeCStrMap(symtab_encoder, strtab, pair.second);
+  }
+  if (num_cmaps > 0)
+    symtab_encoder.PutU8(num_cmaps_offset, num_cmaps);
+
+  // Now that all strings have been gathered, we will emit the string table.
+  strtab.Encode(encoder);
+  // Followed the the symbol table data.
+  encoder.AppendData(symtab_encoder.GetData());
+  return true;
+}
+
+bool Symtab::Decode(const DataExtractor &data, lldb::offset_t *offset_ptr,
+                    bool &signature_mismatch) {
+  signature_mismatch = false;
+  CacheSignature signature;
+  StringTableReader strtab;
+  { // Scope for "elapsed" object below so it can measure the time parse.
+    ElapsedTime elapsed(m_objfile->GetModule()->GetSymtabParseTime());
+    if (!signature.Decode(data, offset_ptr))
+      return false;
+    if (CacheSignature(m_objfile) != signature) {
+      signature_mismatch = true;
+      return false;
+    }
+    // We now decode the string table for all strings in the data cache file.
+    if (!strtab.Decode(data, offset_ptr))
+      return false;
+
+    // And now we can decode the symbol table with string table we just decoded.
+    llvm::StringRef identifier((const char *)data.GetData(offset_ptr, 4), 4);
+    if (identifier != kIdentifierSymbolTable)
+      return false;
+    const uint32_t version = data.GetU32(offset_ptr);
+    if (version != CURRENT_CACHE_VERSION)
+      return false;
+    const uint32_t num_symbols = data.GetU32(offset_ptr);
+    if (num_symbols == 0)
+      return true;
+    m_symbols.resize(num_symbols);
+    SectionList *sections = m_objfile->GetModule()->GetSectionList();
+    for (uint32_t i=0; i<num_symbols; ++i) {
+      if (!m_symbols[i].Decode(data, offset_ptr, sections, strtab))
+        return false;
+    }
+  }
+
+  { // Scope for "elapsed" object below so it can measure the time to index.
+    ElapsedTime elapsed(m_objfile->GetModule()->GetSymtabIndexTime());
+    const uint8_t num_cstr_maps = data.GetU8(offset_ptr);
+    for (uint8_t i=0; i<num_cstr_maps; ++i) {
+      uint8_t type = data.GetU8(offset_ptr);
+      UniqueCStringMap<uint32_t> &cstr_map =
+          GetNameToSymbolIndexMap((lldb::FunctionNameType)type);
+      if (!DecodeCStrMap(data, offset_ptr, strtab, cstr_map))
+        return false;
+    }
+    m_name_indexes_computed = true;
+  }
+  return true;
+}
+
+bool Symtab::LoadFromCache() {
+  DataFileCache *cache = Module::GetIndexCache();
+  if (!cache)
+    return false;
+
+  std::unique_ptr<llvm::MemoryBuffer> mem_buffer_up =
+      cache->GetCachedData(GetCacheKey());
+  if (!mem_buffer_up)
+    return false;
+  DataExtractor data(mem_buffer_up->getBufferStart(),
+                     mem_buffer_up->getBufferSize(),
+                     m_objfile->GetByteOrder(),
+                     m_objfile->GetAddressByteSize());
+  bool signature_mismatch = false;
+  lldb::offset_t offset = 0;
+  const bool result = Decode(data, &offset, signature_mismatch);
+  if (signature_mismatch)
+    cache->RemoveCacheFile(GetCacheKey());
+  return result;
+}
diff --git a/lldb/source/Symbol/Type.cpp b/lldb/source/Symbol/Type.cpp
index 5b4169d256d2..d6c82ed1dd80 100644
--- a/lldb/source/Symbol/Type.cpp
+++ b/lldb/source/Symbol/Type.cpp
@@ -662,7 +662,7 @@ ConstString Type::GetQualifiedName() {
   return GetForwardCompilerType().GetTypeName();
 }
 
-bool Type::GetTypeScopeAndBasename(const llvm::StringRef& name,
+bool Type::GetTypeScopeAndBasename(llvm::StringRef name,
                                    llvm::StringRef &scope,
                                    llvm::StringRef &basename,
                                    TypeClass &type_class) {
diff --git a/lldb/source/Target/StackFrame.cpp b/lldb/source/Target/StackFrame.cpp
index cba51d266c5b..7b4295158425 100644
--- a/lldb/source/Target/StackFrame.cpp
+++ b/lldb/source/Target/StackFrame.cpp
@@ -1883,14 +1883,32 @@ bool StackFrame::GetStatus(Stream &strm, bool show_frame_info, bool show_source,
       if (m_sc.comp_unit && m_sc.line_entry.IsValid()) {
         have_debuginfo = true;
         if (source_lines_before > 0 || source_lines_after > 0) {
+          uint32_t start_line = m_sc.line_entry.line;
+          if (!start_line && m_sc.function) {
+            FileSpec source_file;
+            m_sc.function->GetStartLineSourceInfo(source_file, start_line);
+          }
+
           size_t num_lines =
               target->GetSourceManager().DisplaySourceLinesWithLineNumbers(
-                  m_sc.line_entry.file, m_sc.line_entry.line,
-                  m_sc.line_entry.column, source_lines_before,
-                  source_lines_after, "->", &strm);
+                  m_sc.line_entry.file, start_line, m_sc.line_entry.column,
+                  source_lines_before, source_lines_after, "->", &strm);
           if (num_lines != 0)
             have_source = true;
           // TODO: Give here a one time warning if source file is missing.
+          if (!m_sc.line_entry.line) {
+            ConstString fn_name = m_sc.GetFunctionName();
+
+            if (!fn_name.IsEmpty())
+              strm.Printf(
+                  "Note: this address is compiler-generated code in function "
+                  "%s that has no source code associated with it.",
+                  fn_name.AsCString());
+            else
+              strm.Printf("Note: this address is compiler-generated code that "
+                          "has no source code associated with it.");
+            strm.EOL();
+          }
         }
       }
       switch (disasm_display) {
diff --git a/lldb/source/Target/Target.cpp b/lldb/source/Target/Target.cpp
index 28575b50cf96..fa860399aca7 100644
--- a/lldb/source/Target/Target.cpp
+++ b/lldb/source/Target/Target.cpp
@@ -616,12 +616,8 @@ lldb::BreakpointSP Target::CreateScriptedBreakpoint(
         shared_from_this());
   }
 
-  StructuredDataImpl *extra_args_impl = new StructuredDataImpl();
-  if (extra_args_sp)
-    extra_args_impl->SetObjectSP(extra_args_sp);
-
   BreakpointResolverSP resolver_sp(new BreakpointResolverScripted(
-      nullptr, class_name, depth, extra_args_impl));
+      nullptr, class_name, depth, StructuredDataImpl(extra_args_sp)));
   return CreateBreakpoint(filter_sp, resolver_sp, internal, false, true);
 }
 
@@ -3321,8 +3317,7 @@ void Target::FinalizeFileActions(ProcessLaunchInfo &info) {
                  err_file_spec);
       }
 
-      if (default_to_use_pty &&
-          (!in_file_spec || !out_file_spec || !err_file_spec)) {
+      if (default_to_use_pty) {
         llvm::Error Err = info.SetUpPtyRedirection();
         LLDB_LOG_ERROR(log, std::move(Err), "SetUpPtyRedirection failed: {0}");
       }
@@ -3485,11 +3480,7 @@ Status Target::StopHookScripted::SetScriptCallback(
   }
 
   m_class_name = class_name;
-
-  m_extra_args = new StructuredDataImpl();
-
-  if (extra_args_sp)
-    m_extra_args->SetObjectSP(extra_args_sp);
+  m_extra_args.SetObjectSP(extra_args_sp);
 
   m_implementation_sp = script_interp->CreateScriptedStopHook(
       GetTarget(), m_class_name.c_str(), m_extra_args, error);
@@ -3527,9 +3518,9 @@ void Target::StopHookScripted::GetSubclassDescription(
   // Now print the extra args:
   // FIXME: We should use StructuredData.GetDescription on the m_extra_args
   // but that seems to rely on some printing plugin that doesn't exist.
-  if (!m_extra_args->IsValid())
+  if (!m_extra_args.IsValid())
     return;
-  StructuredData::ObjectSP object_sp = m_extra_args->GetObjectSP();
+  StructuredData::ObjectSP object_sp = m_extra_args.GetObjectSP();
   if (!object_sp || !object_sp->IsValid())
     return;
 
diff --git a/lldb/source/Target/Thread.cpp b/lldb/source/Target/Thread.cpp
index 1b32331d98f7..481a39a576e9 100644
--- a/lldb/source/Target/Thread.cpp
+++ b/lldb/source/Target/Thread.cpp
@@ -1370,15 +1370,9 @@ lldb::ThreadPlanSP Thread::QueueThreadPlanForStepScripted(
     bool abort_other_plans, const char *class_name, 
     StructuredData::ObjectSP extra_args_sp,  bool stop_other_threads,
     Status &status) {
-    
-  StructuredDataImpl *extra_args_impl = nullptr; 
-  if (extra_args_sp) {
-    extra_args_impl = new StructuredDataImpl();
-    extra_args_impl->SetObjectSP(extra_args_sp);
-  }
 
-  ThreadPlanSP thread_plan_sp(new ThreadPlanPython(*this, class_name, 
-                                                   extra_args_impl));
+  ThreadPlanSP thread_plan_sp(new ThreadPlanPython(
+      *this, class_name, StructuredDataImpl(extra_args_sp)));
   thread_plan_sp->SetStopOthers(stop_other_threads);
   status = QueueThreadPlan(thread_plan_sp, abort_other_plans);
   return thread_plan_sp;
diff --git a/lldb/source/Target/ThreadPlanPython.cpp b/lldb/source/Target/ThreadPlanPython.cpp
index cd63d28a3934..a8a36ae65c46 100644
--- a/lldb/source/Target/ThreadPlanPython.cpp
+++ b/lldb/source/Target/ThreadPlanPython.cpp
@@ -26,7 +26,7 @@ using namespace lldb_private;
 // ThreadPlanPython
 
 ThreadPlanPython::ThreadPlanPython(Thread &thread, const char *class_name,
-                                   StructuredDataImpl *args_data)
+                                   const StructuredDataImpl &args_data)
     : ThreadPlan(ThreadPlan::eKindPython, "Python based Thread Plan", thread,
                  eVoteNoOpinion, eVoteNoOpinion),
       m_class_name(class_name), m_args_data(args_data), m_did_push(false),
@@ -36,11 +36,6 @@ ThreadPlanPython::ThreadPlanPython(Thread &thread, const char *class_name,
   SetPrivate(false);
 }
 
-ThreadPlanPython::~ThreadPlanPython() {
-  // FIXME, do I need to decrement the ref count on this implementation object
-  // to make it go away?
-}
-
 bool ThreadPlanPython::ValidatePlan(Stream *error) {
   if (!m_did_push)
     return true;
diff --git a/lldb/source/Target/ThreadPlanStack.cpp b/lldb/source/Target/ThreadPlanStack.cpp
index f09583cc50cc..80634647f9e0 100644
--- a/lldb/source/Target/ThreadPlanStack.cpp
+++ b/lldb/source/Target/ThreadPlanStack.cpp
@@ -210,7 +210,6 @@ void ThreadPlanStack::DiscardAllPlans() {
   for (int i = stack_size - 1; i > 0; i--) {
     DiscardPlan();
   }
-  return;
 }
 
 void ThreadPlanStack::DiscardConsultingControllingPlans() {
diff --git a/lldb/source/Target/UnwindLLDB.cpp b/lldb/source/Target/UnwindLLDB.cpp
index 047147112f3b..77dd19b04ebd 100644
--- a/lldb/source/Target/UnwindLLDB.cpp
+++ b/lldb/source/Target/UnwindLLDB.cpp
@@ -312,7 +312,6 @@ void UnwindLLDB::UpdateUnwindPlanForFirstFrameIfInvalid(ABI *abi) {
   // Restore status after calling AddOneMoreFrame
   m_unwind_complete = old_m_unwind_complete;
   m_candidate_frame = old_m_candidate_frame;
-  return;
 }
 
 bool UnwindLLDB::AddOneMoreFrame(ABI *abi) {
diff --git a/lldb/source/Utility/DataEncoder.cpp b/lldb/source/Utility/DataEncoder.cpp
index e88cd23c1d84..b21453977496 100644
--- a/lldb/source/Utility/DataEncoder.cpp
+++ b/lldb/source/Utility/DataEncoder.cpp
@@ -8,7 +8,7 @@
 
 #include "lldb/Utility/DataEncoder.h"
 
-#include "lldb/Utility/DataBuffer.h"
+#include "lldb/Utility/DataBufferHeap.h"
 #include "lldb/Utility/Endian.h"
 
 #include "llvm/Support/Endian.h"
@@ -22,91 +22,34 @@ using namespace lldb;
 using namespace lldb_private;
 using namespace llvm::support::endian;
 
-// Default constructor.
 DataEncoder::DataEncoder()
-    : m_byte_order(endian::InlHostByteOrder()), m_addr_size(sizeof(void *)),
-      m_data_sp() {}
+    : m_data_sp(new DataBufferHeap()), m_byte_order(endian::InlHostByteOrder()),
+      m_addr_size(sizeof(void *)) {}
 
-// This constructor allows us to use data that is owned by someone else. The
-// data must stay around as long as this object is valid.
-DataEncoder::DataEncoder(void *data, uint32_t length, ByteOrder endian,
+DataEncoder::DataEncoder(const void *data, uint32_t length, ByteOrder endian,
                          uint8_t addr_size)
-    : m_start(static_cast<uint8_t *>(data)),
-      m_end(static_cast<uint8_t *>(data) + length), m_byte_order(endian),
-      m_addr_size(addr_size), m_data_sp() {}
-
-// Make a shared pointer reference to the shared data in "data_sp" and set the
-// endian swapping setting to "swap", and the address size to "addr_size". The
-// shared data reference will ensure the data lives as long as any DataEncoder
-// objects exist that have a reference to this data.
-DataEncoder::DataEncoder(const DataBufferSP &data_sp, ByteOrder endian,
-                         uint8_t addr_size)
-    : m_start(nullptr), m_end(nullptr), m_byte_order(endian),
-      m_addr_size(addr_size), m_data_sp() {
-  SetData(data_sp);
-}
-
-DataEncoder::~DataEncoder() = default;
+    : m_data_sp(new DataBufferHeap(data, length)), m_byte_order(endian),
+      m_addr_size(addr_size) {}
 
-// Clears the object contents back to a default invalid state, and release any
-// references to shared data that this object may contain.
-void DataEncoder::Clear() {
-  m_start = nullptr;
-  m_end = nullptr;
-  m_byte_order = endian::InlHostByteOrder();
-  m_addr_size = sizeof(void *);
-  m_data_sp.reset();
-}
-
-// Assign the data for this object to be a subrange of the shared data in
-// "data_sp" starting "data_offset" bytes into "data_sp" and ending
-// "data_length" bytes later. If "data_offset" is not a valid offset into
-// "data_sp", then this object will contain no bytes. If "data_offset" is
-// within "data_sp" yet "data_length" is too large, the length will be capped
-// at the number of bytes remaining in "data_sp". A ref counted pointer to the
-// data in "data_sp" will be made in this object IF the number of bytes this
-// object refers to in greater than zero (if at least one byte was available
-// starting at "data_offset") to ensure the data stays around as long as it is
-// needed. The address size and endian swap settings will remain unchanged from
-// their current settings.
-uint32_t DataEncoder::SetData(const DataBufferSP &data_sp, uint32_t data_offset,
-                              uint32_t data_length) {
-  m_start = m_end = nullptr;
-
-  if (data_length > 0) {
-    m_data_sp = data_sp;
-    if (data_sp) {
-      const size_t data_size = data_sp->GetByteSize();
-      if (data_offset < data_size) {
-        m_start = data_sp->GetBytes() + data_offset;
-        const size_t bytes_left = data_size - data_offset;
-        // Cap the length of we asked for too many
-        if (data_length <= bytes_left)
-          m_end = m_start + data_length; // We got all the bytes we wanted
-        else
-          m_end = m_start + bytes_left; // Not all the bytes requested were
-                                        // available in the shared data
-      }
-    }
-  }
-
-  uint32_t new_size = GetByteSize();
+DataEncoder::DataEncoder(ByteOrder endian, uint8_t addr_size)
+    : m_data_sp(new DataBufferHeap()), m_byte_order(endian),
+      m_addr_size(addr_size) {}
 
-  // Don't hold a shared pointer to the data buffer if we don't share any valid
-  // bytes in the shared buffer.
-  if (new_size == 0)
-    m_data_sp.reset();
+DataEncoder::~DataEncoder() = default;
 
-  return new_size;
+llvm::ArrayRef<uint8_t> DataEncoder::GetData() const {
+  return llvm::ArrayRef<uint8_t>(m_data_sp->GetBytes(), GetByteSize());
 }
 
+size_t DataEncoder::GetByteSize() const { return m_data_sp->GetByteSize(); }
+
 // Extract a single unsigned char from the binary data and update the offset
 // pointed to by "offset_ptr".
 //
 // RETURNS the byte that was extracted, or zero on failure.
 uint32_t DataEncoder::PutU8(uint32_t offset, uint8_t value) {
   if (ValidOffset(offset)) {
-    m_start[offset] = value;
+    m_data_sp->GetBytes()[offset] = value;
     return offset + 1;
   }
   return UINT32_MAX;
@@ -115,9 +58,9 @@ uint32_t DataEncoder::PutU8(uint32_t offset, uint8_t value) {
 uint32_t DataEncoder::PutU16(uint32_t offset, uint16_t value) {
   if (ValidOffsetForDataOfSize(offset, sizeof(value))) {
     if (m_byte_order != endian::InlHostByteOrder())
-      write16be(m_start + offset, value);
+      write16be(m_data_sp->GetBytes() + offset, value);
     else
-      write16le(m_start + offset, value);
+      write16le(m_data_sp->GetBytes() + offset, value);
 
     return offset + sizeof(value);
   }
@@ -127,9 +70,9 @@ uint32_t DataEncoder::PutU16(uint32_t offset, uint16_t value) {
 uint32_t DataEncoder::PutU32(uint32_t offset, uint32_t value) {
   if (ValidOffsetForDataOfSize(offset, sizeof(value))) {
     if (m_byte_order != endian::InlHostByteOrder())
-      write32be(m_start + offset, value);
+      write32be(m_data_sp->GetBytes() + offset, value);
     else
-      write32le(m_start + offset, value);
+      write32le(m_data_sp->GetBytes() + offset, value);
 
     return offset + sizeof(value);
   }
@@ -139,9 +82,9 @@ uint32_t DataEncoder::PutU32(uint32_t offset, uint32_t value) {
 uint32_t DataEncoder::PutU64(uint32_t offset, uint64_t value) {
   if (ValidOffsetForDataOfSize(offset, sizeof(value))) {
     if (m_byte_order != endian::InlHostByteOrder())
-      write64be(m_start + offset, value);
+      write64be(m_data_sp->GetBytes() + offset, value);
     else
-      write64le(m_start + offset, value);
+      write64le(m_data_sp->GetBytes() + offset, value);
 
     return offset + sizeof(value);
   }
@@ -171,7 +114,7 @@ uint32_t DataEncoder::PutData(uint32_t offset, const void *src,
     return offset;
 
   if (ValidOffsetForDataOfSize(offset, src_len)) {
-    memcpy(m_start + offset, src, src_len);
+    memcpy(m_data_sp->GetBytes() + offset, src, src_len);
     return offset + src_len;
   }
   return UINT32_MAX;
@@ -186,3 +129,63 @@ uint32_t DataEncoder::PutCString(uint32_t offset, const char *cstr) {
     return PutData(offset, cstr, strlen(cstr) + 1);
   return UINT32_MAX;
 }
+
+void DataEncoder::AppendU8(uint8_t value) {
+  m_data_sp->AppendData(&value, sizeof(value));
+}
+
+void DataEncoder::AppendU16(uint16_t value) {
+  uint32_t offset = m_data_sp->GetByteSize();
+  m_data_sp->SetByteSize(m_data_sp->GetByteSize() + sizeof(value));
+  PutU16(offset, value);
+}
+
+void DataEncoder::AppendU32(uint32_t value) {
+  uint32_t offset = m_data_sp->GetByteSize();
+  m_data_sp->SetByteSize(m_data_sp->GetByteSize() + sizeof(value));
+  PutU32(offset, value);
+}
+
+void DataEncoder::AppendU64(uint64_t value) {
+  uint32_t offset = m_data_sp->GetByteSize();
+  m_data_sp->SetByteSize(m_data_sp->GetByteSize() + sizeof(value));
+  PutU64(offset, value);
+}
+
+void DataEncoder::AppendAddress(lldb::addr_t addr) {
+  switch (m_addr_size) {
+  case 4:
+    AppendU32(addr);
+    break;
+  case 8:
+    AppendU64(addr);
+    break;
+  default:
+    llvm_unreachable("AppendAddress unhandled case!");
+  }
+}
+
+void DataEncoder::AppendData(llvm::StringRef data) {
+  const char *bytes = data.data();
+  const size_t length = data.size();
+  if (bytes && length > 0)
+    m_data_sp->AppendData(bytes, length);
+}
+
+void DataEncoder::AppendData(llvm::ArrayRef<uint8_t> data) {
+  const uint8_t *bytes = data.data();
+  const size_t length = data.size();
+  if (bytes && length > 0)
+    m_data_sp->AppendData(bytes, length);
+}
+
+void DataEncoder::AppendCString(llvm::StringRef data) {
+  const char *bytes = data.data();
+  const size_t length = data.size();
+  if (bytes) {
+    if (length > 0)
+      m_data_sp->AppendData(bytes, length);
+    if (length == 0 || bytes[length - 1] != '\0')
+      AppendU8(0);
+  }
+}
diff --git a/lldb/source/Utility/FileSpec.cpp b/lldb/source/Utility/FileSpec.cpp
index 601edb86c1b0..24f8c2b1c23f 100644
--- a/lldb/source/Utility/FileSpec.cpp
+++ b/lldb/source/Utility/FileSpec.cpp
@@ -310,7 +310,7 @@ llvm::Optional<FileSpec::Style> FileSpec::GuessPathStyle(llvm::StringRef absolut
     return Style::posix;
   if (absolute_path.startswith(R"(\\)"))
     return Style::windows;
-  if (absolute_path.size() > 3 && llvm::isAlpha(absolute_path[0]) &&
+  if (absolute_path.size() >= 3 && llvm::isAlpha(absolute_path[0]) &&
       absolute_path.substr(1, 2) == R"(:\)")
     return Style::windows;
   return llvm::None;
diff --git a/lldb/source/Utility/Reproducer.cpp b/lldb/source/Utility/Reproducer.cpp
index b63863c535fa..a306d6c1ef25 100644
--- a/lldb/source/Utility/Reproducer.cpp
+++ b/lldb/source/Utility/Reproducer.cpp
@@ -44,10 +44,6 @@ llvm::Error Reproducer::Initialize(ReproducerMode mode,
     }
     return Instance().SetCapture(root);
   } break;
-  case ReproducerMode::Replay:
-    return Instance().SetReplay(root, /*passive*/ false);
-  case ReproducerMode::PassiveReplay:
-    return Instance().SetReplay(root, /*passive*/ true);
   case ReproducerMode::Off:
     break;
   };
@@ -116,26 +112,6 @@ llvm::Error Reproducer::SetCapture(llvm::Optional<FileSpec> root) {
   return Error::success();
 }
 
-llvm::Error Reproducer::SetReplay(llvm::Optional<FileSpec> root, bool passive) {
-  std::lock_guard<std::mutex> guard(m_mutex);
-
-  if (root && m_generator)
-    return make_error<StringError>(
-        "cannot replay a reproducer when generating one",
-        inconvertibleErrorCode());
-
-  if (!root) {
-    m_loader.reset();
-    return Error::success();
-  }
-
-  m_loader.emplace(*root, passive);
-  if (auto e = m_loader->LoadIndex())
-    return e;
-
-  return Error::success();
-}
-
 FileSpec Reproducer::GetReproducerPath() const {
   if (auto g = GetGenerator())
     return g->GetRoot();
@@ -222,8 +198,7 @@ void Generator::AddProvidersToIndex() {
 }
 
 Loader::Loader(FileSpec root, bool passive)
-    : m_root(MakeAbsolute(std::move(root))), m_loaded(false),
-      m_passive_replay(passive) {}
+    : m_root(MakeAbsolute(std::move(root))), m_loaded(false) {}
 
 llvm::Error Loader::LoadIndex() {
   if (m_loaded)
diff --git a/lldb/source/lldb.cpp b/lldb/source/Version/Version.cpp
index 371902f6c1b5..b391fe1eacd8 100644
--- a/lldb/source/lldb.cpp
+++ b/lldb/source/Version/Version.cpp
@@ -1,4 +1,4 @@
-//===-- lldb.cpp ----------------------------------------------------------===//
+//===-- Version.cpp -------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,23 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "lldb/Version/Version.h"
 #include "VCSVersion.inc"
-#include "lldb/lldb-private.h"
+#include "lldb/Version/Version.inc"
 #include "clang/Basic/Version.h"
 
-using namespace lldb;
-using namespace lldb_private;
-
-// LLDB_VERSION_STRING is set through a define so unlike the other defines
-// expanded with CMake, it lacks the double quotes.
-#define QUOTE(str) #str
-#define EXPAND_AND_QUOTE(str) QUOTE(str)
-
 static const char *GetLLDBVersion() {
-#ifdef LLDB_VERSION_STRING
-  return EXPAND_AND_QUOTE(LLDB_VERSION_STRING);
+#ifdef LLDB_FULL_VERSION_STRING
+  return LLDB_FULL_VERSION_STRING;
 #else
-  return "lldb version " CLANG_VERSION_STRING;
+  return "lldb version " LLDB_VERSION_STRING;
 #endif
 }
 
@@ -30,7 +23,7 @@ static const char *GetLLDBRevision() {
 #ifdef LLDB_REVISION
   return LLDB_REVISION;
 #else
-  return NULL;
+  return nullptr;
 #endif
 }
 
@@ -38,12 +31,13 @@ static const char *GetLLDBRepository() {
 #ifdef LLDB_REPOSITORY
   return LLDB_REPOSITORY;
 #else
-  return NULL;
+  return nullptr;
 #endif
 }
 
 const char *lldb_private::GetVersion() {
   static std::string g_version_str;
+
   if (g_version_str.empty()) {
     const char *lldb_version = GetLLDBVersion();
     const char *lldb_repo = GetLLDBRepository();
@@ -67,11 +61,13 @@ const char *lldb_private::GetVersion() {
       g_version_str += "\n  clang revision ";
       g_version_str += clang_rev;
     }
+
     std::string llvm_rev(clang::getLLVMRevision());
     if (llvm_rev.length() > 0) {
       g_version_str += "\n  llvm revision ";
       g_version_str += llvm_rev;
     }
   }
+
   return g_version_str.c_str();
 }
diff --git a/lldb/tools/driver/Driver.cpp b/lldb/tools/driver/Driver.cpp
index 977cc306bb4e..2ed9958e51da 100644
--- a/lldb/tools/driver/Driver.cpp
+++ b/lldb/tools/driver/Driver.cpp
@@ -736,43 +736,6 @@ EXAMPLES:
 
 static llvm::Optional<int> InitializeReproducer(llvm::StringRef argv0,
                                                 opt::InputArgList &input_args) {
-  if (auto *finalize_path = input_args.getLastArg(OPT_reproducer_finalize)) {
-    if (const char *error = SBReproducer::Finalize(finalize_path->getValue())) {
-      WithColor::error() << "reproducer finalization failed: " << error << '\n';
-      return 1;
-    }
-
-    llvm::outs() << "********************\n";
-    llvm::outs() << "Crash reproducer for ";
-    llvm::outs() << lldb::SBDebugger::GetVersionString() << '\n';
-    llvm::outs() << '\n';
-    llvm::outs() << "Reproducer written to '" << SBReproducer::GetPath()
-                 << "'\n";
-    llvm::outs() << '\n';
-    llvm::outs() << "Before attaching the reproducer to a bug report:\n";
-    llvm::outs() << " - Look at the directory to ensure you're willing to "
-                    "share its content.\n";
-    llvm::outs()
-        << " - Make sure the reproducer works by replaying the reproducer.\n";
-    llvm::outs() << '\n';
-    llvm::outs() << "Replay the reproducer with the following command:\n";
-    llvm::outs() << argv0 << " -replay " << finalize_path->getValue() << "\n";
-    llvm::outs() << "********************\n";
-    return 0;
-  }
-
-  if (auto *replay_path = input_args.getLastArg(OPT_replay)) {
-    SBReplayOptions replay_options;
-    replay_options.SetCheckVersion(!input_args.hasArg(OPT_no_version_check));
-    replay_options.SetVerify(!input_args.hasArg(OPT_no_verification));
-    if (const char *error =
-            SBReproducer::Replay(replay_path->getValue(), replay_options)) {
-      WithColor::error() << "reproducer replay failed: " << error << '\n';
-      return 1;
-    }
-    return 0;
-  }
-
   bool capture = input_args.hasArg(OPT_capture);
   bool generate_on_exit = input_args.hasArg(OPT_generate_on_exit);
   auto *capture_path = input_args.getLastArg(OPT_capture_path);
@@ -799,18 +762,6 @@ static llvm::Optional<int> InitializeReproducer(llvm::StringRef argv0,
     }
     if (generate_on_exit)
       SBReproducer::SetAutoGenerate(true);
-
-    // Register the reproducer signal handler.
-    if (!input_args.hasArg(OPT_no_generate_on_signal)) {
-      if (const char *reproducer_path = SBReproducer::GetPath()) {
-        static std::string *finalize_cmd = new std::string(argv0);
-        finalize_cmd->append(" --reproducer-finalize '");
-        finalize_cmd->append(reproducer_path);
-        finalize_cmd->append("'");
-        llvm::sys::AddSignalHandler(reproducer_handler,
-                                    const_cast<char *>(finalize_cmd->c_str()));
-      }
-    }
   }
 
   return llvm::None;
diff --git a/lldb/tools/driver/Options.td b/lldb/tools/driver/Options.td
index d59ac314d594..9e6ee390eead 100644
--- a/lldb/tools/driver/Options.td
+++ b/lldb/tools/driver/Options.td
@@ -233,17 +233,6 @@ def capture: F<"capture">,
 def capture_path: Separate<["--", "-"], "capture-path">,
   MetaVarName<"<filename>">,
   HelpText<"Tells the debugger to use the given filename for the reproducer.">;
-def replay: Separate<["--", "-"], "replay">,
-  MetaVarName<"<filename>">,
-  HelpText<"Tells the debugger to replay a reproducer from <filename>.">;
-def reproducer_finalize: Separate<["--", "-"], "reproducer-finalize">,
-  MetaVarName<"<filename>">;
-def no_version_check: F<"reproducer-no-version-check">,
-  HelpText<"Disable the reproducer version check.">;
-def no_verification: F<"reproducer-no-verify">,
-  HelpText<"Disable the reproducer verification.">;
-def no_generate_on_signal: F<"reproducer-no-generate-on-signal">,
-  HelpText<"Don't generate reproducer when a signal is received.">;
 def generate_on_exit: F<"reproducer-generate-on-exit">,
   HelpText<"Generate reproducer on exit.">;
 
diff --git a/lldb/tools/lldb-server/lldb-server.cpp b/lldb/tools/lldb-server/lldb-server.cpp
index 1e001ac7185b..1808ffc0c979 100644
--- a/lldb/tools/lldb-server/lldb-server.cpp
+++ b/lldb/tools/lldb-server/lldb-server.cpp
@@ -8,7 +8,7 @@
 
 #include "SystemInitializerLLGS.h"
 #include "lldb/Initialization/SystemLifetimeManager.h"
-#include "lldb/lldb-private.h"
+#include "lldb/Version/Version.h"
 
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringRef.h"
diff --git a/llvm/include/llvm-c/Core.h b/llvm/include/llvm-c/Core.h
index f2183ff52bfb..ae2bcb8444b4 100644
--- a/llvm/include/llvm-c/Core.h
+++ b/llvm/include/llvm-c/Core.h
@@ -15,6 +15,7 @@
 #ifndef LLVM_C_CORE_H
 #define LLVM_C_CORE_H
 
+#include "llvm-c/Deprecated.h"
 #include "llvm-c/ErrorHandling.h"
 #include "llvm-c/ExternC.h"
 #include "llvm-c/Types.h"
@@ -2151,13 +2152,18 @@ LLVMValueRef LLVMConstFCmp(LLVMRealPredicate Predicate,
 LLVMValueRef LLVMConstShl(LLVMValueRef LHSConstant, LLVMValueRef RHSConstant);
 LLVMValueRef LLVMConstLShr(LLVMValueRef LHSConstant, LLVMValueRef RHSConstant);
 LLVMValueRef LLVMConstAShr(LLVMValueRef LHSConstant, LLVMValueRef RHSConstant);
-LLVMValueRef LLVMConstGEP(LLVMValueRef ConstantVal,
-                          LLVMValueRef *ConstantIndices, unsigned NumIndices);
+LLVM_ATTRIBUTE_C_DEPRECATED(
+    LLVMValueRef LLVMConstGEP(LLVMValueRef ConstantVal,
+                              LLVMValueRef *ConstantIndices,
+                              unsigned NumIndices),
+    "Use LLVMConstGEP2 instead to support opaque pointers");
 LLVMValueRef LLVMConstGEP2(LLVMTypeRef Ty, LLVMValueRef ConstantVal,
                            LLVMValueRef *ConstantIndices, unsigned NumIndices);
-LLVMValueRef LLVMConstInBoundsGEP(LLVMValueRef ConstantVal,
-                                  LLVMValueRef *ConstantIndices,
-                                  unsigned NumIndices);
+LLVM_ATTRIBUTE_C_DEPRECATED(
+    LLVMValueRef LLVMConstInBoundsGEP(LLVMValueRef ConstantVal,
+                                      LLVMValueRef *ConstantIndices,
+                                      unsigned NumIndices),
+    "Use LLVMConstInBoundsGEP2 instead to support opaque pointers");
 LLVMValueRef LLVMConstInBoundsGEP2(LLVMTypeRef Ty, LLVMValueRef ConstantVal,
                                    LLVMValueRef *ConstantIndices,
                                    unsigned NumIndices);
@@ -2378,9 +2384,10 @@ void LLVMSetExternallyInitialized(LLVMValueRef GlobalVar, LLVMBool IsExtInit);
  * @{
  */
 
-/** Deprecated: Use LLVMAddAlias2 instead. */
-LLVMValueRef LLVMAddAlias(LLVMModuleRef M, LLVMTypeRef Ty, LLVMValueRef Aliasee,
-                          const char *Name);
+LLVM_ATTRIBUTE_C_DEPRECATED(
+    LLVMValueRef LLVMAddAlias(LLVMModuleRef M, LLVMTypeRef Ty,
+                              LLVMValueRef Aliasee, const char *Name),
+    "Use LLVMAddAlias2 instead to support opaque pointers");
 
 /**
  * Add a GlobalAlias with the given value type, address space and aliasee.
@@ -3497,7 +3504,7 @@ LLVMTypeRef LLVMGetAllocatedType(LLVMValueRef Alloca);
  */
 
 /**
- * Check whether the given GEP instruction is inbounds.
+ * Check whether the given GEP operator is inbounds.
  */
 LLVMBool LLVMIsInBounds(LLVMValueRef GEP);
 
@@ -3507,6 +3514,11 @@ LLVMBool LLVMIsInBounds(LLVMValueRef GEP);
 void LLVMSetIsInBounds(LLVMValueRef GEP, LLVMBool InBounds);
 
 /**
+ * Get the source element type of the given GEP operator.
+ */
+LLVMTypeRef LLVMGetGEPSourceElementType(LLVMValueRef GEP);
+
+/**
  * @}
  */
 
@@ -3556,7 +3568,7 @@ LLVMBasicBlockRef LLVMGetIncomingBlock(LLVMValueRef PhiNode, unsigned Index);
 
 /**
  * Obtain the number of indices.
- * NB: This also works on GEP.
+ * NB: This also works on GEP operators.
  */
 unsigned LLVMGetNumIndices(LLVMValueRef Inst);
 
@@ -3676,12 +3688,12 @@ LLVMValueRef LLVMBuildSwitch(LLVMBuilderRef, LLVMValueRef V,
                              LLVMBasicBlockRef Else, unsigned NumCases);
 LLVMValueRef LLVMBuildIndirectBr(LLVMBuilderRef B, LLVMValueRef Addr,
                                  unsigned NumDests);
-// LLVMBuildInvoke is deprecated in favor of LLVMBuildInvoke2, in preparation
-// for opaque pointer types.
-LLVMValueRef LLVMBuildInvoke(LLVMBuilderRef, LLVMValueRef Fn,
-                             LLVMValueRef *Args, unsigned NumArgs,
-                             LLVMBasicBlockRef Then, LLVMBasicBlockRef Catch,
-                             const char *Name);
+LLVM_ATTRIBUTE_C_DEPRECATED(
+    LLVMValueRef LLVMBuildInvoke(LLVMBuilderRef, LLVMValueRef Fn,
+                                 LLVMValueRef *Args, unsigned NumArgs,
+                                 LLVMBasicBlockRef Then,
+                                 LLVMBasicBlockRef Catch, const char *Name),
+    "Use LLVMBuildInvoke2 instead to support opaque pointers");
 LLVMValueRef LLVMBuildInvoke2(LLVMBuilderRef, LLVMTypeRef Ty, LLVMValueRef Fn,
                               LLVMValueRef *Args, unsigned NumArgs,
                               LLVMBasicBlockRef Then, LLVMBasicBlockRef Catch,
@@ -3875,23 +3887,27 @@ LLVMValueRef LLVMBuildAlloca(LLVMBuilderRef, LLVMTypeRef Ty, const char *Name);
 LLVMValueRef LLVMBuildArrayAlloca(LLVMBuilderRef, LLVMTypeRef Ty,
                                   LLVMValueRef Val, const char *Name);
 LLVMValueRef LLVMBuildFree(LLVMBuilderRef, LLVMValueRef PointerVal);
-// LLVMBuildLoad is deprecated in favor of LLVMBuildLoad2, in preparation for
-// opaque pointer types.
-LLVMValueRef LLVMBuildLoad(LLVMBuilderRef, LLVMValueRef PointerVal,
-                           const char *Name);
+LLVM_ATTRIBUTE_C_DEPRECATED(
+    LLVMValueRef LLVMBuildLoad(LLVMBuilderRef, LLVMValueRef PointerVal,
+                               const char *Name),
+    "Use LLVMBuildLoad2 instead to support opaque pointers");
 LLVMValueRef LLVMBuildLoad2(LLVMBuilderRef, LLVMTypeRef Ty,
                             LLVMValueRef PointerVal, const char *Name);
 LLVMValueRef LLVMBuildStore(LLVMBuilderRef, LLVMValueRef Val, LLVMValueRef Ptr);
-// LLVMBuildGEP, LLVMBuildInBoundsGEP, and LLVMBuildStructGEP are deprecated in
-// favor of LLVMBuild*GEP2, in preparation for opaque pointer types.
-LLVMValueRef LLVMBuildGEP(LLVMBuilderRef B, LLVMValueRef Pointer,
-                          LLVMValueRef *Indices, unsigned NumIndices,
-                          const char *Name);
-LLVMValueRef LLVMBuildInBoundsGEP(LLVMBuilderRef B, LLVMValueRef Pointer,
-                                  LLVMValueRef *Indices, unsigned NumIndices,
-                                  const char *Name);
-LLVMValueRef LLVMBuildStructGEP(LLVMBuilderRef B, LLVMValueRef Pointer,
-                                unsigned Idx, const char *Name);
+LLVM_ATTRIBUTE_C_DEPRECATED(
+    LLVMValueRef LLVMBuildGEP(LLVMBuilderRef B, LLVMValueRef Pointer,
+                              LLVMValueRef *Indices, unsigned NumIndices,
+                              const char *Name),
+    "Use LLVMBuildGEP2 instead to support opaque pointers");
+LLVM_ATTRIBUTE_C_DEPRECATED(
+    LLVMValueRef LLVMBuildInBoundsGEP(LLVMBuilderRef B, LLVMValueRef Pointer,
+                                      LLVMValueRef *Indices,
+                                      unsigned NumIndices, const char *Name),
+    "Use LLVMBuildInBoundsGEP2 instead to support opaque pointers");
+LLVM_ATTRIBUTE_C_DEPRECATED(
+    LLVMValueRef LLVMBuildStructGEP(LLVMBuilderRef B, LLVMValueRef Pointer,
+                                    unsigned Idx, const char *Name),
+    "Use LLVMBuildStructGEP2 instead to support opaque pointers");
 LLVMValueRef LLVMBuildGEP2(LLVMBuilderRef B, LLVMTypeRef Ty,
                            LLVMValueRef Pointer, LLVMValueRef *Indices,
                            unsigned NumIndices, const char *Name);
@@ -3971,11 +3987,11 @@ LLVMValueRef LLVMBuildFCmp(LLVMBuilderRef, LLVMRealPredicate Op,
 
 /* Miscellaneous instructions */
 LLVMValueRef LLVMBuildPhi(LLVMBuilderRef, LLVMTypeRef Ty, const char *Name);
-// LLVMBuildCall is deprecated in favor of LLVMBuildCall2, in preparation for
-// opaque pointer types.
-LLVMValueRef LLVMBuildCall(LLVMBuilderRef, LLVMValueRef Fn,
-                           LLVMValueRef *Args, unsigned NumArgs,
-                           const char *Name);
+LLVM_ATTRIBUTE_C_DEPRECATED(
+    LLVMValueRef LLVMBuildCall(LLVMBuilderRef, LLVMValueRef Fn,
+                               LLVMValueRef *Args, unsigned NumArgs,
+                               const char *Name),
+    "Use LLVMBuildCall2 instead to support opaque pointers");
 LLVMValueRef LLVMBuildCall2(LLVMBuilderRef, LLVMTypeRef, LLVMValueRef Fn,
                             LLVMValueRef *Args, unsigned NumArgs,
                             const char *Name);
diff --git a/llvm/include/llvm-c/Deprecated.h b/llvm/include/llvm-c/Deprecated.h
new file mode 100644
index 000000000000..1ed5d11550d2
--- /dev/null
+++ b/llvm/include/llvm-c/Deprecated.h
@@ -0,0 +1,38 @@
+/*===-- llvm-c/Deprecated.h - Deprecation macro -------------------*- C -*-===*\
+|*                                                                            *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
+|*                                                                            *|
+|*===----------------------------------------------------------------------===*|
+|*                                                                            *|
+|* This header declares LLVM_ATTRIBUTE_C_DEPRECATED() macro, which can be     *|
+|* used to deprecate functions in the C interface.                            *|
+|*                                                                            *|
+\*===----------------------------------------------------------------------===*/
+
+#ifndef LLVM_C_DEPRECATED_H
+#define LLVM_C_DEPRECATED_H
+
+#ifndef __has_feature
+# define __has_feature(x) 0
+#endif
+
+// This is a variant of LLVM_ATTRIBUTE_DEPRECATED() that is compatible with
+// C compilers.
+#if __has_feature(attribute_deprecated_with_message)
+# define LLVM_ATTRIBUTE_C_DEPRECATED(decl, message) \
+  decl __attribute__((deprecated(message)))
+#elif defined(__GNUC__)
+# define LLVM_ATTRIBUTE_C_DEPRECATED(decl, message) \
+  decl __attribute__((deprecated))
+#elif defined(_MSC_VER)
+# define LLVM_ATTRIBUTE_C_DEPRECATED(decl, message) \
+  __declspec(deprecated(message)) decl
+#else
+# define LLVM_ATTRIBUTE_C_DEPRECATED(decl, message) \
+  decl
+#endif
+
+#endif /* LLVM_C_DEPRECATED_H */
diff --git a/llvm/include/llvm/ADT/GenericCycleImpl.h b/llvm/include/llvm/ADT/GenericCycleImpl.h
new file mode 100644
index 000000000000..5f29236eac47
--- /dev/null
+++ b/llvm/include/llvm/ADT/GenericCycleImpl.h
@@ -0,0 +1,411 @@
+//===- GenericCycleImpl.h -------------------------------------*- C++ -*---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This template implementation resides in a separate file so that it
+// does not get injected into every .cpp file that includes the
+// generic header.
+//
+// DO NOT INCLUDE THIS FILE WHEN MERELY USING CYCLEINFO.
+//
+// This file should only be included by files that implement a
+// specialization of the relevant templates. Currently these are:
+// - CycleAnalysis.cpp
+// - MachineCycleAnalysis.cpp
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ADT_GENERICCYCLEIMPL_H
+#define LLVM_ADT_GENERICCYCLEIMPL_H
+
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/DepthFirstIterator.h"
+#include "llvm/ADT/GenericCycleInfo.h"
+
+#define DEBUG_TYPE "generic-cycle-impl"
+
+namespace llvm {
+
+template <typename ContextT>
+bool GenericCycle<ContextT>::contains(const GenericCycle *C) const {
+  if (!C)
+    return false;
+
+  if (Depth > C->Depth)
+    return false;
+  while (Depth < C->Depth)
+    C = C->ParentCycle;
+  return this == C;
+}
+
+template <typename ContextT>
+void GenericCycle<ContextT>::getExitBlocks(
+    SmallVectorImpl<BlockT *> &TmpStorage) const {
+  TmpStorage.clear();
+
+  size_t NumExitBlocks = 0;
+  for (BlockT *Block : blocks()) {
+    llvm::append_range(TmpStorage, successors(Block));
+
+    for (size_t Idx = NumExitBlocks, End = TmpStorage.size(); Idx < End;
+         ++Idx) {
+      BlockT *Succ = TmpStorage[Idx];
+      if (!contains(Succ)) {
+        auto ExitEndIt = TmpStorage.begin() + NumExitBlocks;
+        if (std::find(TmpStorage.begin(), ExitEndIt, Succ) == ExitEndIt)
+          TmpStorage[NumExitBlocks++] = Succ;
+      }
+    }
+
+    TmpStorage.resize(NumExitBlocks);
+  }
+}
+
+/// \brief Helper class for computing cycle information.
+template <typename ContextT> class GenericCycleInfoCompute {
+  using BlockT = typename ContextT::BlockT;
+  using CycleInfoT = GenericCycleInfo<ContextT>;
+  using CycleT = typename CycleInfoT::CycleT;
+
+  CycleInfoT &Info;
+
+  struct DFSInfo {
+    unsigned Start = 0; // DFS start; positive if block is found
+    unsigned End = 0;   // DFS end
+
+    DFSInfo() {}
+    explicit DFSInfo(unsigned Start) : Start(Start) {}
+
+    /// Whether this node is an ancestor (or equal to) the node \p Other
+    /// in the DFS tree.
+    bool isAncestorOf(const DFSInfo &Other) const {
+      return Start <= Other.Start && Other.End <= End;
+    }
+  };
+
+  DenseMap<BlockT *, DFSInfo> BlockDFSInfo;
+  SmallVector<BlockT *, 8> BlockPreorder;
+
+  GenericCycleInfoCompute(const GenericCycleInfoCompute &) = delete;
+  GenericCycleInfoCompute &operator=(const GenericCycleInfoCompute &) = delete;
+
+public:
+  GenericCycleInfoCompute(CycleInfoT &Info) : Info(Info) {}
+
+  void run(BlockT *EntryBlock);
+
+  static void updateDepth(CycleT *SubTree);
+
+private:
+  void dfs(BlockT *EntryBlock);
+};
+
+template <typename ContextT>
+auto GenericCycleInfo<ContextT>::getTopLevelParentCycle(
+    const BlockT *Block) const -> CycleT * {
+  auto MapIt = BlockMap.find(Block);
+  if (MapIt == BlockMap.end())
+    return nullptr;
+
+  auto *C = MapIt->second;
+  while (C->ParentCycle)
+    C = C->ParentCycle;
+  return C;
+}
+
+template <typename ContextT>
+void GenericCycleInfo<ContextT>::moveToNewParent(CycleT *NewParent,
+                                                 CycleT *Child) {
+  auto &CurrentContainer =
+      Child->ParentCycle ? Child->ParentCycle->Children : TopLevelCycles;
+  auto Pos = llvm::find_if(CurrentContainer, [=](const auto &Ptr) -> bool {
+    return Child == Ptr.get();
+  });
+  assert(Pos != CurrentContainer.end());
+  NewParent->Children.push_back(std::move(*Pos));
+  *Pos = std::move(CurrentContainer.back());
+  CurrentContainer.pop_back();
+  Child->ParentCycle = NewParent;
+}
+
+/// \brief Main function of the cycle info computations.
+template <typename ContextT>
+void GenericCycleInfoCompute<ContextT>::run(BlockT *EntryBlock) {
+  LLVM_DEBUG(errs() << "Entry block: " << Info.Context.print(EntryBlock)
+                    << "\n");
+  dfs(EntryBlock);
+
+  SmallVector<BlockT *, 8> Worklist;
+
+  for (BlockT *HeaderCandidate : llvm::reverse(BlockPreorder)) {
+    const DFSInfo CandidateInfo = BlockDFSInfo.lookup(HeaderCandidate);
+
+    for (BlockT *Pred : predecessors(HeaderCandidate)) {
+      const DFSInfo PredDFSInfo = BlockDFSInfo.lookup(Pred);
+      if (CandidateInfo.isAncestorOf(PredDFSInfo))
+        Worklist.push_back(Pred);
+    }
+    if (Worklist.empty()) {
+      continue;
+    }
+
+    // Found a cycle with the candidate as its header.
+    LLVM_DEBUG(errs() << "Found cycle for header: "
+                      << Info.Context.print(HeaderCandidate) << "\n");
+    std::unique_ptr<CycleT> NewCycle = std::make_unique<CycleT>();
+    NewCycle->appendEntry(HeaderCandidate);
+    NewCycle->appendBlock(HeaderCandidate);
+    Info.BlockMap.try_emplace(HeaderCandidate, NewCycle.get());
+
+    // Helper function to process (non-back-edge) predecessors of a discovered
+    // block and either add them to the worklist or recognize that the given
+    // block is an additional cycle entry.
+    auto ProcessPredecessors = [&](BlockT *Block) {
+      LLVM_DEBUG(errs() << "  block " << Info.Context.print(Block) << ": ");
+
+      bool IsEntry = false;
+      for (BlockT *Pred : predecessors(Block)) {
+        const DFSInfo PredDFSInfo = BlockDFSInfo.lookup(Pred);
+        if (CandidateInfo.isAncestorOf(PredDFSInfo)) {
+          Worklist.push_back(Pred);
+        } else {
+          IsEntry = true;
+        }
+      }
+      if (IsEntry) {
+        assert(!NewCycle->isEntry(Block));
+        LLVM_DEBUG(errs() << "append as entry\n");
+        NewCycle->appendEntry(Block);
+      } else {
+        LLVM_DEBUG(errs() << "append as child\n");
+      }
+    };
+
+    do {
+      BlockT *Block = Worklist.pop_back_val();
+      if (Block == HeaderCandidate)
+        continue;
+
+      // If the block has already been discovered by some cycle
+      // (possibly by ourself), then the outermost cycle containing it
+      // should become our child.
+      if (auto *BlockParent = Info.getTopLevelParentCycle(Block)) {
+        LLVM_DEBUG(errs() << "  block " << Info.Context.print(Block) << ": ");
+
+        if (BlockParent != NewCycle.get()) {
+          LLVM_DEBUG(errs()
+                     << "discovered child cycle "
+                     << Info.Context.print(BlockParent->getHeader()) << "\n");
+          // Make BlockParent the child of NewCycle.
+          Info.moveToNewParent(NewCycle.get(), BlockParent);
+          NewCycle->Blocks.insert(NewCycle->Blocks.end(),
+                                  BlockParent->block_begin(),
+                                  BlockParent->block_end());
+
+          for (auto *ChildEntry : BlockParent->entries())
+            ProcessPredecessors(ChildEntry);
+        } else {
+          LLVM_DEBUG(errs()
+                     << "known child cycle "
+                     << Info.Context.print(BlockParent->getHeader()) << "\n");
+        }
+      } else {
+        Info.BlockMap.try_emplace(Block, NewCycle.get());
+        assert(!is_contained(NewCycle->Blocks, Block));
+        NewCycle->Blocks.push_back(Block);
+        ProcessPredecessors(Block);
+      }
+    } while (!Worklist.empty());
+
+    Info.TopLevelCycles.push_back(std::move(NewCycle));
+  }
+
+  // Fix top-level cycle links and compute cycle depths.
+  for (auto *TLC : Info.toplevel_cycles()) {
+    LLVM_DEBUG(errs() << "top-level cycle: "
+                      << Info.Context.print(TLC->getHeader()) << "\n");
+
+    TLC->ParentCycle = nullptr;
+    updateDepth(TLC);
+  }
+}
+
+/// \brief Recompute depth values of \p SubTree and all descendants.
+template <typename ContextT>
+void GenericCycleInfoCompute<ContextT>::updateDepth(CycleT *SubTree) {
+  for (CycleT *Cycle : depth_first(SubTree))
+    Cycle->Depth = Cycle->ParentCycle ? Cycle->ParentCycle->Depth + 1 : 1;
+}
+
+/// \brief Compute a DFS of basic blocks starting at the function entry.
+///
+/// Fills BlockDFSInfo with start/end counters and BlockPreorder.
+template <typename ContextT>
+void GenericCycleInfoCompute<ContextT>::dfs(BlockT *EntryBlock) {
+  SmallVector<unsigned, 8> DFSTreeStack;
+  SmallVector<BlockT *, 8> TraverseStack;
+  unsigned Counter = 0;
+  TraverseStack.emplace_back(EntryBlock);
+
+  do {
+    BlockT *Block = TraverseStack.back();
+    LLVM_DEBUG(errs() << "DFS visiting block: " << Info.Context.print(Block)
+                      << "\n");
+    if (!BlockDFSInfo.count(Block)) {
+      // We're visiting the block for the first time. Open its DFSInfo, add
+      // successors to the traversal stack, and remember the traversal stack
+      // depth at which the block was opened, so that we can correctly record
+      // its end time.
+      LLVM_DEBUG(errs() << "  first encountered at depth "
+                        << TraverseStack.size() << "\n");
+
+      DFSTreeStack.emplace_back(TraverseStack.size());
+      llvm::append_range(TraverseStack, successors(Block));
+
+      LLVM_ATTRIBUTE_UNUSED
+      bool Added = BlockDFSInfo.try_emplace(Block, ++Counter).second;
+      assert(Added);
+      BlockPreorder.push_back(Block);
+      LLVM_DEBUG(errs() << "  preorder number: " << Counter << "\n");
+    } else {
+      assert(!DFSTreeStack.empty());
+      if (DFSTreeStack.back() == TraverseStack.size()) {
+        LLVM_DEBUG(errs() << "  ended at " << Counter << "\n");
+        BlockDFSInfo.find(Block)->second.End = Counter;
+        DFSTreeStack.pop_back();
+      } else {
+        LLVM_DEBUG(errs() << "  already done\n");
+      }
+      TraverseStack.pop_back();
+    }
+  } while (!TraverseStack.empty());
+  assert(DFSTreeStack.empty());
+
+  LLVM_DEBUG(
+    errs() << "Preorder:\n";
+    for (int i = 0, e = BlockPreorder.size(); i != e; ++i) {
+      errs() << "  " << Info.Context.print(BlockPreorder[i]) << ": " << i << "\n";
+    }
+  );
+}
+
+/// \brief Reset the object to its initial state.
+template <typename ContextT> void GenericCycleInfo<ContextT>::clear() {
+  TopLevelCycles.clear();
+  BlockMap.clear();
+}
+
+/// \brief Compute the cycle info for a function.
+template <typename ContextT>
+void GenericCycleInfo<ContextT>::compute(FunctionT &F) {
+  GenericCycleInfoCompute<ContextT> Compute(*this);
+  Context.setFunction(F);
+
+  LLVM_DEBUG(errs() << "Computing cycles for function: " << F.getName()
+                    << "\n");
+  Compute.run(ContextT::getEntryBlock(F));
+
+  assert(validateTree());
+}
+
+/// \brief Find the innermost cycle containing a given block.
+///
+/// \returns the innermost cycle containing \p Block or nullptr if
+///          it is not contained in any cycle.
+template <typename ContextT>
+auto GenericCycleInfo<ContextT>::getCycle(const BlockT *Block) const
+    -> CycleT * {
+  auto MapIt = BlockMap.find(Block);
+  if (MapIt != BlockMap.end())
+    return MapIt->second;
+  return nullptr;
+}
+
+/// \brief Validate the internal consistency of the cycle tree.
+///
+/// Note that this does \em not check that cycles are really cycles in the CFG,
+/// or that the right set of cycles in the CFG were found.
+template <typename ContextT>
+bool GenericCycleInfo<ContextT>::validateTree() const {
+  DenseSet<BlockT *> Blocks;
+  DenseSet<BlockT *> Entries;
+
+  auto reportError = [](const char *File, int Line, const char *Cond) {
+    errs() << File << ':' << Line
+           << ": GenericCycleInfo::validateTree: " << Cond << '\n';
+  };
+#define check(cond)                                                            \
+  do {                                                                         \
+    if (!(cond)) {                                                             \
+      reportError(__FILE__, __LINE__, #cond);                                  \
+      return false;                                                            \
+    }                                                                          \
+  } while (false)
+
+  for (const auto *TLC : toplevel_cycles()) {
+    for (const CycleT *Cycle : depth_first(TLC)) {
+      if (Cycle->ParentCycle)
+        check(is_contained(Cycle->ParentCycle->children(), Cycle));
+
+      for (BlockT *Block : Cycle->Blocks) {
+        auto MapIt = BlockMap.find(Block);
+        check(MapIt != BlockMap.end());
+        check(Cycle->contains(MapIt->second));
+        check(Blocks.insert(Block).second); // duplicates in block list?
+      }
+      Blocks.clear();
+
+      check(!Cycle->Entries.empty());
+      for (BlockT *Entry : Cycle->Entries) {
+        check(Entries.insert(Entry).second); // duplicate entry?
+        check(is_contained(Cycle->Blocks, Entry));
+      }
+      Entries.clear();
+
+      unsigned ChildDepth = 0;
+      for (const CycleT *Child : Cycle->children()) {
+        check(Child->Depth > Cycle->Depth);
+        if (!ChildDepth) {
+          ChildDepth = Child->Depth;
+        } else {
+          check(ChildDepth == Child->Depth);
+        }
+      }
+    }
+  }
+
+  for (const auto &Entry : BlockMap) {
+    BlockT *Block = Entry.first;
+    for (const CycleT *Cycle = Entry.second; Cycle;
+         Cycle = Cycle->ParentCycle) {
+      check(is_contained(Cycle->Blocks, Block));
+    }
+  }
+
+#undef check
+
+  return true;
+}
+
+/// \brief Print the cycle info.
+template <typename ContextT>
+void GenericCycleInfo<ContextT>::print(raw_ostream &Out) const {
+  for (const auto *TLC : toplevel_cycles()) {
+    for (const CycleT *Cycle : depth_first(TLC)) {
+      for (unsigned I = 0; I < Cycle->Depth; ++I)
+        Out << "    ";
+
+      Out << Cycle->print(Context) << '\n';
+    }
+  }
+}
+
+} // namespace llvm
+
+#undef DEBUG_TYPE
+
+#endif // LLVM_ADT_GENERICCYCLEIMPL_H
diff --git a/llvm/include/llvm/ADT/GenericCycleInfo.h b/llvm/include/llvm/ADT/GenericCycleInfo.h
new file mode 100644
index 000000000000..aad704301e43
--- /dev/null
+++ b/llvm/include/llvm/ADT/GenericCycleInfo.h
@@ -0,0 +1,334 @@
+//===- GenericCycleInfo.h - Info for Cycles in any IR ------*- C++ -*------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// \brief Find all cycles in a control-flow graph, including irreducible loops.
+///
+/// See docs/CycleTerminology.rst for a formal definition of cycles.
+///
+/// Briefly:
+/// - A cycle is a generalization of a loop which can represent
+///   irreducible control flow.
+/// - Cycles identified in a program are implementation defined,
+///   depending on the DFS traversal chosen.
+/// - Cycles are well-nested, and form a forest with a parent-child
+///   relationship.
+/// - In any choice of DFS, every natural loop L is represented by a
+///   unique cycle C which is a superset of L.
+/// - In the absence of irreducible control flow, the cycles are
+///   exactly the natural loops in the program.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ADT_GENERICCYCLEINFO_H
+#define LLVM_ADT_GENERICCYCLEINFO_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/GenericSSAContext.h"
+#include "llvm/ADT/GraphTraits.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/iterator.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/Printable.h"
+#include "llvm/Support/raw_ostream.h"
+#include <vector>
+
+namespace llvm {
+
+template <typename ContexT> class GenericCycleInfo;
+template <typename ContexT> class GenericCycleInfoCompute;
+
+/// A possibly irreducible generalization of a \ref Loop.
+template <typename ContextT> class GenericCycle {
+public:
+  using BlockT = typename ContextT::BlockT;
+  using FunctionT = typename ContextT::FunctionT;
+  template <typename> friend class GenericCycleInfo;
+  template <typename> friend class GenericCycleInfoCompute;
+
+private:
+  /// The parent cycle. Is null for the root "cycle". Top-level cycles point
+  /// at the root.
+  GenericCycle *ParentCycle = nullptr;
+
+  /// The entry block(s) of the cycle. The header is the only entry if
+  /// this is a loop. Is empty for the root "cycle", to avoid
+  /// unnecessary memory use.
+  SmallVector<BlockT *, 1> Entries;
+
+  /// Child cycles, if any.
+  std::vector<std::unique_ptr<GenericCycle>> Children;
+
+  /// Basic blocks that are contained in the cycle, including entry blocks,
+  /// and including blocks that are part of a child cycle.
+  std::vector<BlockT *> Blocks;
+
+  /// Depth of the cycle in the tree. The root "cycle" is at depth 0.
+  ///
+  /// \note Depths are not necessarily contiguous. However, child loops always
+  ///       have strictly greater depth than their parents, and sibling loops
+  ///       always have the same depth.
+  unsigned Depth = 0;
+
+  void clear() {
+    Entries.clear();
+    Children.clear();
+    Blocks.clear();
+    Depth = 0;
+    ParentCycle = nullptr;
+  }
+
+  void appendEntry(BlockT *Block) { Entries.push_back(Block); }
+  void appendBlock(BlockT *Block) { Blocks.push_back(Block); }
+
+  GenericCycle(const GenericCycle &) = delete;
+  GenericCycle &operator=(const GenericCycle &) = delete;
+  GenericCycle(GenericCycle &&Rhs) = delete;
+  GenericCycle &operator=(GenericCycle &&Rhs) = delete;
+
+public:
+  GenericCycle() = default;
+
+  /// \brief Whether the cycle is a natural loop.
+  bool isReducible() const { return Entries.size() == 1; }
+
+  BlockT *getHeader() const { return Entries[0]; }
+
+  /// \brief Return whether \p Block is an entry block of the cycle.
+  bool isEntry(BlockT *Block) const { return is_contained(Entries, Block); }
+
+  /// \brief Return whether \p Block is contained in the cycle.
+  bool contains(const BlockT *Block) const {
+    return is_contained(Blocks, Block);
+  }
+
+  /// \brief Returns true iff this cycle contains \p C.
+  ///
+  /// Note: Non-strict containment check, i.e. returns true if C is the
+  /// same cycle.
+  bool contains(const GenericCycle *C) const;
+
+  const GenericCycle *getParentCycle() const { return ParentCycle; }
+  GenericCycle *getParentCycle() { return ParentCycle; }
+  unsigned getDepth() const { return Depth; }
+
+  /// Return all of the successor blocks of this cycle.
+  ///
+  /// These are the blocks _outside of the current cycle_ which are
+  /// branched to.
+  void getExitBlocks(SmallVectorImpl<BlockT *> &TmpStorage) const;
+
+  /// Iteration over child cycles.
+  //@{
+  using const_child_iterator_base =
+      typename std::vector<std::unique_ptr<GenericCycle>>::const_iterator;
+  struct const_child_iterator
+      : iterator_adaptor_base<const_child_iterator, const_child_iterator_base> {
+    using Base =
+        iterator_adaptor_base<const_child_iterator, const_child_iterator_base>;
+
+    const_child_iterator() = default;
+    explicit const_child_iterator(const_child_iterator_base I) : Base(I) {}
+
+    const const_child_iterator_base &wrapped() { return Base::wrapped(); }
+    GenericCycle *operator*() const { return Base::I->get(); }
+  };
+
+  const_child_iterator child_begin() const {
+    return const_child_iterator{Children.begin()};
+  }
+  const_child_iterator child_end() const {
+    return const_child_iterator{Children.end()};
+  }
+  size_t getNumChildren() const { return Children.size(); }
+  iterator_range<const_child_iterator> children() const {
+    return llvm::make_range(const_child_iterator{Children.begin()},
+                            const_child_iterator{Children.end()});
+  }
+  //@}
+
+  /// Iteration over blocks in the cycle (including entry blocks).
+  //@{
+  using const_block_iterator = typename std::vector<BlockT *>::const_iterator;
+
+  const_block_iterator block_begin() const {
+    return const_block_iterator{Blocks.begin()};
+  }
+  const_block_iterator block_end() const {
+    return const_block_iterator{Blocks.end()};
+  }
+  size_t getNumBlocks() const { return Blocks.size(); }
+  iterator_range<const_block_iterator> blocks() const {
+    return llvm::make_range(block_begin(), block_end());
+  }
+  //@}
+
+  /// Iteration over entry blocks.
+  //@{
+  using const_entry_iterator =
+      typename SmallVectorImpl<BlockT *>::const_iterator;
+
+  size_t getNumEntries() const { return Entries.size(); }
+  iterator_range<const_entry_iterator> entries() const {
+    return llvm::make_range(Entries.begin(), Entries.end());
+  }
+
+  Printable printEntries(const ContextT &Ctx) const {
+    return Printable([this, &Ctx](raw_ostream &Out) {
+      bool First = true;
+      for (auto *Entry : Entries) {
+        if (!First)
+          Out << ' ';
+        First = false;
+        Out << Ctx.print(Entry);
+      }
+    });
+  }
+
+  Printable print(const ContextT &Ctx) const {
+    return Printable([this, &Ctx](raw_ostream &Out) {
+      Out << "depth=" << Depth << ": entries(" << printEntries(Ctx) << ')';
+
+      for (auto *Block : Blocks) {
+        if (isEntry(Block))
+          continue;
+
+        Out << ' ' << Ctx.print(Block);
+      }
+    });
+  }
+};
+
+/// \brief Cycle information for a function.
+template <typename ContextT> class GenericCycleInfo {
+public:
+  using BlockT = typename ContextT::BlockT;
+  using CycleT = GenericCycle<ContextT>;
+  using FunctionT = typename ContextT::FunctionT;
+  template <typename> friend class GenericCycle;
+  template <typename> friend class GenericCycleInfoCompute;
+
+private:
+  ContextT Context;
+
+  /// Map basic blocks to their inner-most containing loop.
+  DenseMap<BlockT *, CycleT *> BlockMap;
+
+  /// Outermost cycles discovered by any DFS.
+  ///
+  /// Note: The implementation treats the nullptr as the parent of
+  /// every top-level cycle. See \ref contains for an example.
+  std::vector<std::unique_ptr<CycleT>> TopLevelCycles;
+
+public:
+  GenericCycleInfo() = default;
+  GenericCycleInfo(GenericCycleInfo &&) = default;
+  GenericCycleInfo &operator=(GenericCycleInfo &&) = default;
+
+  void clear();
+  void compute(FunctionT &F);
+
+  FunctionT *getFunction() const { return Context.getFunction(); }
+  const ContextT &getSSAContext() const { return Context; }
+
+  CycleT *getCycle(const BlockT *Block) const;
+  CycleT *getTopLevelParentCycle(const BlockT *Block) const;
+
+  /// Move \p Child to \p NewParent by manipulating Children vectors.
+  ///
+  /// Note: This is an incomplete operation that does not update the
+  /// list of blocks in the new parent or the depth of the subtree.
+  void moveToNewParent(CycleT *NewParent, CycleT *Child);
+
+  /// Methods for debug and self-test.
+  //@{
+  bool validateTree() const;
+  void print(raw_ostream &Out) const;
+  void dump() const { print(dbgs()); }
+  //@}
+
+  /// Iteration over top-level cycles.
+  //@{
+  using const_toplevel_iterator_base =
+      typename std::vector<std::unique_ptr<CycleT>>::const_iterator;
+  struct const_toplevel_iterator
+      : iterator_adaptor_base<const_toplevel_iterator,
+                              const_toplevel_iterator_base> {
+    using Base = iterator_adaptor_base<const_toplevel_iterator,
+                                       const_toplevel_iterator_base>;
+
+    const_toplevel_iterator() = default;
+    explicit const_toplevel_iterator(const_toplevel_iterator_base I)
+        : Base(I) {}
+
+    const const_toplevel_iterator_base &wrapped() { return Base::wrapped(); }
+    CycleT *operator*() const { return Base::I->get(); }
+  };
+
+  const_toplevel_iterator toplevel_begin() const {
+    return const_toplevel_iterator{TopLevelCycles.begin()};
+  }
+  const_toplevel_iterator toplevel_end() const {
+    return const_toplevel_iterator{TopLevelCycles.end()};
+  }
+
+  iterator_range<const_toplevel_iterator> toplevel_cycles() const {
+    return llvm::make_range(const_toplevel_iterator{TopLevelCycles.begin()},
+                            const_toplevel_iterator{TopLevelCycles.end()});
+  }
+  //@}
+};
+
+/// \brief GraphTraits for iterating over a sub-tree of the CycleT tree.
+template <typename CycleRefT, typename ChildIteratorT> struct CycleGraphTraits {
+  using NodeRef = CycleRefT;
+
+  using nodes_iterator = ChildIteratorT;
+  using ChildIteratorType = nodes_iterator;
+
+  static NodeRef getEntryNode(NodeRef Graph) { return Graph; }
+
+  static ChildIteratorType child_begin(NodeRef Ref) {
+    return Ref->child_begin();
+  }
+  static ChildIteratorType child_end(NodeRef Ref) { return Ref->child_end(); }
+
+  // Not implemented:
+  // static nodes_iterator nodes_begin(GraphType *G)
+  // static nodes_iterator nodes_end  (GraphType *G)
+  //    nodes_iterator/begin/end - Allow iteration over all nodes in the graph
+
+  // typedef EdgeRef           - Type of Edge token in the graph, which should
+  //                             be cheap to copy.
+  // typedef ChildEdgeIteratorType - Type used to iterate over children edges in
+  //                             graph, dereference to a EdgeRef.
+
+  // static ChildEdgeIteratorType child_edge_begin(NodeRef)
+  // static ChildEdgeIteratorType child_edge_end(NodeRef)
+  //     Return iterators that point to the beginning and ending of the
+  //     edge list for the given callgraph node.
+  //
+  // static NodeRef edge_dest(EdgeRef)
+  //     Return the destination node of an edge.
+  // static unsigned       size       (GraphType *G)
+  //    Return total number of nodes in the graph
+};
+
+template <typename BlockT>
+struct GraphTraits<const GenericCycle<BlockT> *>
+    : CycleGraphTraits<const GenericCycle<BlockT> *,
+                       typename GenericCycle<BlockT>::const_child_iterator> {};
+template <typename BlockT>
+struct GraphTraits<GenericCycle<BlockT> *>
+    : CycleGraphTraits<GenericCycle<BlockT> *,
+                       typename GenericCycle<BlockT>::const_child_iterator> {};
+
+} // namespace llvm
+
+#endif // LLVM_ADT_GENERICCYCLEINFO_H
diff --git a/llvm/include/llvm/ADT/GenericSSAContext.h b/llvm/include/llvm/ADT/GenericSSAContext.h
new file mode 100644
index 000000000000..409222547d5c
--- /dev/null
+++ b/llvm/include/llvm/ADT/GenericSSAContext.h
@@ -0,0 +1,74 @@
+//===- GenericSSAContext.h --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+///
+/// This file defines the little GenericSSAContext<X> template class
+/// that can be used to implement IR analyses as templates.
+/// Specializing these templates allows the analyses to be used over
+/// both LLVM IR and Machine IR.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ADT_GENERICSSACONTEXT_H
+#define LLVM_ADT_GENERICSSACONTEXT_H
+
+#include "llvm/Support/Printable.h"
+
+namespace llvm {
+
+template <typename _FunctionT> class GenericSSAContext {
+public:
+  // Specializations should provide the following types that are similar to how
+  // LLVM IR is structured:
+
+  // The smallest unit of the IR is a ValueT. The SSA context uses a ValueRefT,
+  // which is a pointer to a ValueT, since Machine IR does not have the
+  // equivalent of a ValueT.
+  //
+  // using ValueRefT = ...
+
+  // An InstT is a subclass of ValueT that itself defines one or more ValueT
+  // objects.
+  //
+  // using InstT = ... must be a subclass of Value
+
+  // A BlockT is a sequence of InstT, and forms a node of the CFG. It
+  // has global methods predecessors() and successors() that return
+  // the list of incoming CFG edges and outgoing CFG edges
+  // respectively.
+  //
+  // using BlockT = ...
+
+  // A FunctionT represents a CFG along with arguments and return values. It is
+  // the smallest complete unit of code in a Module.
+  //
+  // The compiler produces an error here if this class is implicitly
+  // specialized due to an instantiation. An explicit specialization
+  // of this template needs to be added before the instantiation point
+  // indicated by the compiler.
+  using FunctionT = typename _FunctionT::invalidTemplateInstanceError;
+
+  // Every FunctionT has a unique BlockT marked as its entry.
+  //
+  // static BlockT* getEntryBlock(FunctionT &F);
+
+  // Initialize the SSA context with information about the FunctionT being
+  // processed.
+  //
+  // void setFunction(FunctionT &function);
+  // FunctionT* getFunction() const;
+
+  // Methods to print various objects.
+  //
+  // Printable print(BlockT *block) const;
+  // Printable print(InstructionT *inst) const;
+  // Printable print(ValueRefT value) const;
+};
+} // namespace llvm
+
+#endif // LLVM_ADT_GENERICSSACONTEXT_H
diff --git a/llvm/include/llvm/ADT/PointerUnion.h b/llvm/include/llvm/ADT/PointerUnion.h
index 0874f67db3fe..5ce2dbee4b3a 100644
--- a/llvm/include/llvm/ADT/PointerUnion.h
+++ b/llvm/include/llvm/ADT/PointerUnion.h
@@ -16,6 +16,7 @@
 
 #include "llvm/ADT/DenseMapInfo.h"
 #include "llvm/ADT/PointerIntPair.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/PointerLikeTypeTraits.h"
 #include <algorithm>
 #include <cassert>
@@ -35,21 +36,6 @@ namespace pointer_union_detail {
     return std::min<int>({PointerLikeTypeTraits<Ts>::NumLowBitsAvailable...});
   }
 
-  /// Find the index of a type in a list of types. TypeIndex<T, Us...>::Index
-  /// is the index of T in Us, or sizeof...(Us) if T does not appear in the
-  /// list.
-  template <typename T, typename ...Us> struct TypeIndex;
-  template <typename T, typename ...Us> struct TypeIndex<T, T, Us...> {
-    static constexpr int Index = 0;
-  };
-  template <typename T, typename U, typename... Us>
-  struct TypeIndex<T, U, Us...> {
-    static constexpr int Index = 1 + TypeIndex<T, Us...>::Index;
-  };
-  template <typename T> struct TypeIndex<T> {
-    static constexpr int Index = 0;
-  };
-
   /// Find the first type in a list of types.
   template <typename T, typename...> struct GetFirstType {
     using type = T;
@@ -116,6 +102,7 @@ namespace pointer_union_detail {
 ///    P = (float*)0;
 ///    Y = P.get<float*>();   // ok.
 ///    X = P.get<int*>();     // runtime assertion failure.
+///    PointerUnion<int*, int*> Q; // compile time failure.
 template <typename... PTs>
 class PointerUnion
     : public pointer_union_detail::PointerUnionMembers<
@@ -124,12 +111,14 @@ class PointerUnion
               void *, pointer_union_detail::bitsRequired(sizeof...(PTs)), int,
               pointer_union_detail::PointerUnionUIntTraits<PTs...>>,
           0, PTs...> {
+  static_assert(TypesAreDistinct<PTs...>::value,
+                "PointerUnion alternative types cannot be repeated");
   // The first type is special because we want to directly cast a pointer to a
   // default-initialized union to a pointer to the first type. But we don't
   // want PointerUnion to be a 'template <typename First, typename ...Rest>'
   // because it's much more convenient to have a name for the whole pack. So
   // split off the first type here.
-  using First = typename pointer_union_detail::GetFirstType<PTs...>::type;
+  using First = TypeAtIndex<0, PTs...>;
   using Base = typename PointerUnion::PointerUnionMembers;
 
 public:
@@ -146,10 +135,7 @@ public:
 
   /// Test if the Union currently holds the type matching T.
   template <typename T> bool is() const {
-    constexpr int Index = pointer_union_detail::TypeIndex<T, PTs...>::Index;
-    static_assert(Index < sizeof...(PTs),
-                  "PointerUnion::is<T> given type not in the union");
-    return this->Val.getInt() == Index;
+    return this->Val.getInt() == FirstIndexOfType<T, PTs...>::value;
   }
 
   /// Returns the value of the specified pointer type.
diff --git a/llvm/include/llvm/ADT/STLExtras.h b/llvm/include/llvm/ADT/STLExtras.h
index f9b658ca960a..2d38e153c79e 100644
--- a/llvm/include/llvm/ADT/STLExtras.h
+++ b/llvm/include/llvm/ADT/STLExtras.h
@@ -144,6 +144,61 @@ template <typename ReturnType, typename... Args>
 struct function_traits<ReturnType (&)(Args...), false>
     : public function_traits<ReturnType (*)(Args...)> {};
 
+/// traits class for checking whether type T is one of any of the given
+/// types in the variadic list.
+template <typename T, typename... Ts>
+using is_one_of = disjunction<std::is_same<T, Ts>...>;
+
+/// traits class for checking whether type T is a base class for all
+///  the given types in the variadic list.
+template <typename T, typename... Ts>
+using are_base_of = conjunction<std::is_base_of<T, Ts>...>;
+
+namespace detail {
+template <typename T, typename... Us> struct TypesAreDistinct;
+template <typename T, typename... Us>
+struct TypesAreDistinct
+    : std::integral_constant<bool, !is_one_of<T, Us...>::value &&
+                                       TypesAreDistinct<Us...>::value> {};
+template <typename T> struct TypesAreDistinct<T> : std::true_type {};
+} // namespace detail
+
+/// Determine if all types in Ts are distinct.
+///
+/// Useful to statically assert when Ts is intended to describe a non-multi set
+/// of types.
+///
+/// Expensive (currently quadratic in sizeof(Ts...)), and so should only be
+/// asserted once per instantiation of a type which requires it.
+template <typename... Ts> struct TypesAreDistinct;
+template <> struct TypesAreDistinct<> : std::true_type {};
+template <typename... Ts>
+struct TypesAreDistinct
+    : std::integral_constant<bool, detail::TypesAreDistinct<Ts...>::value> {};
+
+/// Find the first index where a type appears in a list of types.
+///
+/// FirstIndexOfType<T, Us...>::value is the first index of T in Us.
+///
+/// Typically only meaningful when it is otherwise statically known that the
+/// type pack has no duplicate types. This should be guaranteed explicitly with
+/// static_assert(TypesAreDistinct<Us...>::value).
+///
+/// It is a compile-time error to instantiate when T is not present in Us, i.e.
+/// if is_one_of<T, Us...>::value is false.
+template <typename T, typename... Us> struct FirstIndexOfType;
+template <typename T, typename U, typename... Us>
+struct FirstIndexOfType<T, U, Us...>
+    : std::integral_constant<size_t, 1 + FirstIndexOfType<T, Us...>::value> {};
+template <typename T, typename... Us>
+struct FirstIndexOfType<T, T, Us...> : std::integral_constant<size_t, 0> {};
+
+/// Find the type at a given index in a list of types.
+///
+/// TypeAtIndex<I, Ts...> is the type at index I in Ts.
+template <size_t I, typename... Ts>
+using TypeAtIndex = std::tuple_element_t<I, std::tuple<Ts...>>;
+
 //===----------------------------------------------------------------------===//
 //     Extra additions to <functional>
 //===----------------------------------------------------------------------===//
diff --git a/llvm/include/llvm/ADT/SmallVector.h b/llvm/include/llvm/ADT/SmallVector.h
index 0d13524f25ce..804567ebe3f1 100644
--- a/llvm/include/llvm/ADT/SmallVector.h
+++ b/llvm/include/llvm/ADT/SmallVector.h
@@ -589,17 +589,21 @@ public:
 
 private:
   template <bool ForOverwrite> void resizeImpl(size_type N) {
+    if (N == this->size())
+      return;
+
     if (N < this->size()) {
-      this->pop_back_n(this->size() - N);
-    } else if (N > this->size()) {
-      this->reserve(N);
-      for (auto I = this->end(), E = this->begin() + N; I != E; ++I)
-        if (ForOverwrite)
-          new (&*I) T;
-        else
-          new (&*I) T();
-      this->set_size(N);
+      this->truncate(N);
+      return;
     }
+
+    this->reserve(N);
+    for (auto I = this->end(), E = this->begin() + N; I != E; ++I)
+      if (ForOverwrite)
+        new (&*I) T;
+      else
+        new (&*I) T();
+    this->set_size(N);
   }
 
 public:
@@ -608,12 +612,19 @@ public:
   /// Like resize, but \ref T is POD, the new values won't be initialized.
   void resize_for_overwrite(size_type N) { resizeImpl<true>(N); }
 
+  /// Like resize, but requires that \p N is less than \a size().
+  void truncate(size_type N) {
+    assert(this->size() >= N && "Cannot increase size with truncate");
+    this->destroy_range(this->begin() + N, this->end());
+    this->set_size(N);
+  }
+
   void resize(size_type N, ValueParamT NV) {
     if (N == this->size())
       return;
 
     if (N < this->size()) {
-      this->pop_back_n(this->size() - N);
+      this->truncate(N);
       return;
     }
 
@@ -628,8 +639,7 @@ public:
 
   void pop_back_n(size_type NumItems) {
     assert(this->size() >= NumItems);
-    this->destroy_range(this->end() - NumItems, this->end());
-    this->set_size(this->size() - NumItems);
+    truncate(this->size() - NumItems);
   }
 
   LLVM_NODISCARD T pop_back_val() {
diff --git a/llvm/include/llvm/ADT/StringRef.h b/llvm/include/llvm/ADT/StringRef.h
index 9f4b89218042..3950910f0635 100644
--- a/llvm/include/llvm/ADT/StringRef.h
+++ b/llvm/include/llvm/ADT/StringRef.h
@@ -149,11 +149,11 @@ namespace llvm {
 
     /// empty - Check if the string is empty.
     LLVM_NODISCARD
-    bool empty() const { return Length == 0; }
+    constexpr bool empty() const { return Length == 0; }
 
     /// size - Get the string size.
     LLVM_NODISCARD
-    size_t size() const { return Length; }
+    constexpr size_t size() const { return Length; }
 
     /// front - Get the first character in the string.
     LLVM_NODISCARD
diff --git a/llvm/include/llvm/ADT/Triple.h b/llvm/include/llvm/ADT/Triple.h
index 2fd3047acbfd..5dbd4f16bfd5 100644
--- a/llvm/include/llvm/ADT/Triple.h
+++ b/llvm/include/llvm/ADT/Triple.h
@@ -10,6 +10,7 @@
 #define LLVM_ADT_TRIPLE_H
 
 #include "llvm/ADT/Twine.h"
+#include "llvm/Support/VersionTuple.h"
 
 // Some system headers or GCC predefined macros conflict with identifiers in
 // this file.  Undefine them here.
@@ -19,8 +20,6 @@
 
 namespace llvm {
 
-class VersionTuple;
-
 /// Triple - Helper class for working with autoconf configuration names. For
 /// historical reasons, we also call these 'triples' (they used to contain
 /// exactly three fields).
@@ -332,10 +331,7 @@ public:
   /// triple, if present.
   ///
   /// For example, "fooos1.2.3" would return (1, 2, 3).
-  ///
-  /// If an entry is not defined, it will be returned as 0.
-  void getEnvironmentVersion(unsigned &Major, unsigned &Minor,
-                             unsigned &Micro) const;
+  VersionTuple getEnvironmentVersion() const;
 
   /// Get the object format for this triple.
   ObjectFormatType getObjectFormat() const { return ObjectFormat; }
@@ -344,34 +340,25 @@ public:
   /// present.
   ///
   /// For example, "fooos1.2.3" would return (1, 2, 3).
-  ///
-  /// If an entry is not defined, it will be returned as 0.
-  void getOSVersion(unsigned &Major, unsigned &Minor, unsigned &Micro) const;
+  VersionTuple getOSVersion() const;
 
   /// Return just the major version number, this is specialized because it is a
   /// common query.
-  unsigned getOSMajorVersion() const {
-    unsigned Maj, Min, Micro;
-    getOSVersion(Maj, Min, Micro);
-    return Maj;
-  }
+  unsigned getOSMajorVersion() const { return getOSVersion().getMajor(); }
 
   /// Parse the version number as with getOSVersion and then translate generic
   /// "darwin" versions to the corresponding OS X versions.  This may also be
   /// called with IOS triples but the OS X version number is just set to a
   /// constant 10.4.0 in that case.  Returns true if successful.
-  bool getMacOSXVersion(unsigned &Major, unsigned &Minor,
-                        unsigned &Micro) const;
+  bool getMacOSXVersion(VersionTuple &Version) const;
 
   /// Parse the version number as with getOSVersion.  This should only be called
   /// with IOS or generic triples.
-  void getiOSVersion(unsigned &Major, unsigned &Minor,
-                     unsigned &Micro) const;
+  VersionTuple getiOSVersion() const;
 
   /// Parse the version number as with getOSVersion.  This should only be called
   /// with WatchOS or generic triples.
-  void getWatchOSVersion(unsigned &Major, unsigned &Minor,
-                         unsigned &Micro) const;
+  VersionTuple getWatchOSVersion() const;
 
   /// @}
   /// @name Direct Component Access
@@ -428,23 +415,17 @@ public:
   /// the target triple.
   bool isOSVersionLT(unsigned Major, unsigned Minor = 0,
                      unsigned Micro = 0) const {
-    unsigned LHS[3];
-    getOSVersion(LHS[0], LHS[1], LHS[2]);
-
-    if (LHS[0] != Major)
-      return LHS[0] < Major;
-    if (LHS[1] != Minor)
-      return LHS[1] < Minor;
-    if (LHS[2] != Micro)
-      return LHS[2] < Micro;
-
-    return false;
+    if (Minor == 0) {
+      return getOSVersion() < VersionTuple(Major);
+    }
+    if (Micro == 0) {
+      return getOSVersion() < VersionTuple(Major, Minor);
+    }
+    return getOSVersion() < VersionTuple(Major, Minor, Micro);
   }
 
   bool isOSVersionLT(const Triple &Other) const {
-    unsigned RHS[3];
-    Other.getOSVersion(RHS[0], RHS[1], RHS[2]);
-    return isOSVersionLT(RHS[0], RHS[1], RHS[2]);
+    return getOSVersion() < Other.getOSVersion();
   }
 
   /// Comparison function for checking OS X version compatibility, which handles
@@ -678,14 +659,13 @@ public:
   bool isAndroidVersionLT(unsigned Major) const {
     assert(isAndroid() && "Not an Android triple!");
 
-    unsigned Env[3];
-    getEnvironmentVersion(Env[0], Env[1], Env[2]);
+    VersionTuple Version = getEnvironmentVersion();
 
     // 64-bit targets did not exist before API level 21 (Lollipop).
-    if (isArch64Bit() && Env[0] < 21)
-      Env[0] = 21;
+    if (isArch64Bit() && Version.getMajor() < 21)
+      return VersionTuple(21) < VersionTuple(Major);
 
-    return Env[0] < Major;
+    return Version < VersionTuple(Major);
   }
 
   /// Tests whether the environment is musl-libc
diff --git a/llvm/include/llvm/Analysis/CycleAnalysis.h b/llvm/include/llvm/Analysis/CycleAnalysis.h
new file mode 100644
index 000000000000..e16b908d6a10
--- /dev/null
+++ b/llvm/include/llvm/Analysis/CycleAnalysis.h
@@ -0,0 +1,77 @@
+//===- CycleAnalysis.h - Cycle Info for LLVM IR -----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+///
+/// This file declares an analysis pass that computes CycleInfo for
+/// LLVM IR, specialized from GenericCycleInfo.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ANALYSIS_CYCLEANALYSIS_H
+#define LLVM_ANALYSIS_CYCLEANALYSIS_H
+
+#include "llvm/ADT/GenericCycleInfo.h"
+#include "llvm/IR/PassManager.h"
+#include "llvm/IR/SSAContext.h"
+
+namespace llvm {
+extern template class GenericCycleInfo<SSAContext>;
+extern template class GenericCycle<SSAContext>;
+
+using CycleInfo = GenericCycleInfo<SSAContext>;
+using Cycle = CycleInfo::CycleT;
+
+/// Analysis pass which computes a \ref CycleInfo.
+class CycleAnalysis : public AnalysisInfoMixin<CycleAnalysis> {
+  friend AnalysisInfoMixin<CycleAnalysis>;
+  static AnalysisKey Key;
+
+public:
+  /// Provide the result typedef for this analysis pass.
+  using Result = CycleInfo;
+
+  /// Run the analysis pass over a function and produce a dominator tree.
+  CycleInfo run(Function &F, FunctionAnalysisManager &);
+
+  // TODO: verify analysis?
+};
+
+/// Printer pass for the \c DominatorTree.
+class CycleInfoPrinterPass : public PassInfoMixin<CycleInfoPrinterPass> {
+  raw_ostream &OS;
+
+public:
+  explicit CycleInfoPrinterPass(raw_ostream &OS);
+
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+};
+
+/// Legacy analysis pass which computes a \ref CycleInfo.
+class CycleInfoWrapperPass : public FunctionPass {
+  Function *F = nullptr;
+  CycleInfo CI;
+
+public:
+  static char ID;
+
+  CycleInfoWrapperPass();
+
+  CycleInfo &getCycleInfo() { return CI; }
+  const CycleInfo &getCycleInfo() const { return CI; }
+
+  bool runOnFunction(Function &F) override;
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
+  void releaseMemory() override;
+  void print(raw_ostream &OS, const Module *M = nullptr) const override;
+
+  // TODO: verify analysis?
+};
+
+} // end namespace llvm
+
+#endif // LLVM_ANALYSIS_CYCLEANALYSIS_H
diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h
index ea4c0312e073..9858a46d16a2 100644
--- a/llvm/include/llvm/Analysis/IVDescriptors.h
+++ b/llvm/include/llvm/Analysis/IVDescriptors.h
@@ -157,7 +157,7 @@ public:
   static InstDesc isConditionalRdxPattern(RecurKind Kind, Instruction *I);
 
   /// Returns identity corresponding to the RecurrenceKind.
-  Value *getRecurrenceIdentity(RecurKind K, Type *Tp, FastMathFlags FMF);
+  Value *getRecurrenceIdentity(RecurKind K, Type *Tp, FastMathFlags FMF) const;
 
   /// Returns the opcode corresponding to the RecurrenceKind.
   static unsigned getOpcode(RecurKind Kind);
diff --git a/llvm/include/llvm/Analysis/InlineCost.h b/llvm/include/llvm/Analysis/InlineCost.h
index b22841343b1a..776749b9a07f 100644
--- a/llvm/include/llvm/Analysis/InlineCost.h
+++ b/llvm/include/llvm/Analysis/InlineCost.h
@@ -212,7 +212,7 @@ struct InlineParams {
   Optional<bool> ComputeFullInlineCost;
 
   /// Indicate whether we should allow inline deferral.
-  Optional<bool> EnableDeferral = true;
+  Optional<bool> EnableDeferral;
 
   /// Indicate whether we allow inlining for recursive call.
   Optional<bool> AllowRecursiveCall = false;
diff --git a/llvm/include/llvm/Analysis/MLModelRunner.h b/llvm/include/llvm/Analysis/MLModelRunner.h
index 7cfa6efedf10..90b3cc7e76e6 100644
--- a/llvm/include/llvm/Analysis/MLModelRunner.h
+++ b/llvm/include/llvm/Analysis/MLModelRunner.h
@@ -10,7 +10,6 @@
 #ifndef LLVM_ANALYSIS_MLMODELRUNNER_H
 #define LLVM_ANALYSIS_MLMODELRUNNER_H
 
-#include "llvm/Analysis/InlineModelFeatureMaps.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/PassManager.h"
 
@@ -18,6 +17,9 @@ namespace llvm {
 
 /// MLModelRunner interface: abstraction of a mechanism for evaluating a
 /// tensorflow "saved model".
+/// NOTE: feature indices are expected to be consistent all accross
+/// MLModelRunners (pertaining to the same model), and also Loggers (see
+/// TFUtils.h)
 class MLModelRunner {
 public:
   // Disallows copy and assign.
@@ -25,12 +27,27 @@ public:
   MLModelRunner &operator=(const MLModelRunner &) = delete;
   virtual ~MLModelRunner() = default;
 
-  virtual bool run() = 0;
-  virtual void setFeature(FeatureIndex Index, int64_t Value) = 0;
-  virtual int64_t getFeature(int Index) const = 0;
+  template <typename T> T evaluate() {
+    return *reinterpret_cast<T *>(evaluateUntyped());
+  }
+
+  template <typename T, typename I> T *getTensor(I FeatureID) {
+    return reinterpret_cast<T *>(
+        getTensorUntyped(static_cast<size_t>(FeatureID)));
+  }
+
+  template <typename T, typename I> const T *getTensor(I FeatureID) const {
+    return reinterpret_cast<const T *>(
+        getTensorUntyped(static_cast<size_t>(FeatureID)));
+  }
 
 protected:
   MLModelRunner(LLVMContext &Ctx) : Ctx(Ctx) {}
+  virtual void *evaluateUntyped() = 0;
+  virtual void *getTensorUntyped(size_t Index) = 0;
+  const void *getTensorUntyped(size_t Index) const {
+    return (const_cast<MLModelRunner *>(this))->getTensorUntyped(Index);
+  }
 
   LLVMContext &Ctx;
 };
diff --git a/llvm/include/llvm/Analysis/MemoryBuiltins.h b/llvm/include/llvm/Analysis/MemoryBuiltins.h
index 39ade20df53f..94495a518042 100644
--- a/llvm/include/llvm/Analysis/MemoryBuiltins.h
+++ b/llvm/include/llvm/Analysis/MemoryBuiltins.h
@@ -241,7 +241,7 @@ class ObjectSizeOffsetVisitor
   APInt Zero;
   SmallPtrSet<Instruction *, 8> SeenInsts;
 
-  APInt align(APInt Size, uint64_t Align);
+  APInt align(APInt Size, MaybeAlign Align);
 
   SizeOffsetType unknown() {
     return std::make_pair(APInt(), APInt());
diff --git a/llvm/include/llvm/Analysis/MemoryLocation.h b/llvm/include/llvm/Analysis/MemoryLocation.h
index 3b188d763ef2..833fce1b1726 100644
--- a/llvm/include/llvm/Analysis/MemoryLocation.h
+++ b/llvm/include/llvm/Analysis/MemoryLocation.h
@@ -253,6 +253,8 @@ public:
   static MemoryLocation getForDest(const MemIntrinsic *MI);
   static MemoryLocation getForDest(const AtomicMemIntrinsic *MI);
   static MemoryLocation getForDest(const AnyMemIntrinsic *MI);
+  static Optional<MemoryLocation> getForDest(const CallBase *CI,
+                                             const TargetLibraryInfo &TLI);
 
   /// Return a location representing a particular argument of a call.
   static MemoryLocation getForArgument(const CallBase *Call, unsigned ArgIdx,
diff --git a/llvm/include/llvm/Analysis/ModelUnderTrainingRunner.h b/llvm/include/llvm/Analysis/ModelUnderTrainingRunner.h
new file mode 100644
index 000000000000..ca99d5d01eef
--- /dev/null
+++ b/llvm/include/llvm/Analysis/ModelUnderTrainingRunner.h
@@ -0,0 +1,59 @@
+//===- ModelUnderTrainingRunner.h -- 'development' mode runner --*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+
+#ifndef LLVM_ANALYSIS_MODELUNDERTRAININGRUNNER_H
+#define LLVM_ANALYSIS_MODELUNDERTRAININGRUNNER_H
+
+#include "llvm/Config/llvm-config.h"
+
+#ifdef LLVM_HAVE_TF_API
+#include "llvm/Analysis/MLModelRunner.h"
+#include "llvm/Analysis/Utils/TFUtils.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+
+/// ModelUnderTrainingRunner - training mode implementation. It uses TF C APIs
+/// to dynamically load and evaluate a TF SavedModel
+/// (https://www.tensorflow.org/guide/saved_model). Runtime performance is
+/// sacrificed for ease of use while training.
+class ModelUnderTrainingRunner final : public MLModelRunner {
+public:
+  ModelUnderTrainingRunner(LLVMContext &Ctx, const std::string &ModelPath,
+                           const std::vector<TensorSpec> &InputSpecs,
+                           const std::vector<LoggedFeatureSpec> &OutputSpecs);
+
+  // Disallows copy and assign.
+  ModelUnderTrainingRunner(const ModelUnderTrainingRunner &) = delete;
+  ModelUnderTrainingRunner &
+  operator=(const ModelUnderTrainingRunner &) = delete;
+
+  bool isValid() const { return !!Evaluator; }
+
+  const std::vector<LoggedFeatureSpec> &outputLoggedFeatureSpecs() const {
+    return OutputSpecs;
+  }
+
+  const Optional<TFModelEvaluator::EvaluationResult> &
+  lastEvaluationResult() const {
+    return LastEvaluationResult;
+  }
+
+private:
+  std::unique_ptr<TFModelEvaluator> Evaluator;
+  const std::vector<LoggedFeatureSpec> OutputSpecs;
+  Optional<TFModelEvaluator::EvaluationResult> LastEvaluationResult;
+  void *evaluateUntyped() override;
+  void *getTensorUntyped(size_t Index) override;
+};
+
+} // namespace llvm
+#endif // define(LLVM_HAVE_TF_API)
+#endif // LLVM_ANALYSIS_MODELUNDERTRAININGRUNNER_H
diff --git a/llvm/include/llvm/Analysis/NoInferenceModelRunner.h b/llvm/include/llvm/Analysis/NoInferenceModelRunner.h
new file mode 100644
index 000000000000..60d6777c765b
--- /dev/null
+++ b/llvm/include/llvm/Analysis/NoInferenceModelRunner.h
@@ -0,0 +1,39 @@
+//===- NoInferenceModelRunner.h ---- noop ML model runner  ------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+
+#ifndef LLVM_ANALYSIS_NOINFERENCEMODELRUNNER_H
+#define LLVM_ANALYSIS_NOINFERENCEMODELRUNNER_H
+
+#include "llvm/Config/llvm-config.h"
+
+/// While not strictly necessary to conditionally compile this, it really
+/// has no usecase outside the 'development' mode.
+#ifdef LLVM_HAVE_TF_API
+#include "llvm/Analysis/MLModelRunner.h"
+#include "llvm/Analysis/Utils/TFUtils.h"
+namespace llvm {
+/// A pseudo model runner. We use it to store feature values when collecting
+/// logs for the default policy, in 'development' mode, but never ask it to
+/// 'run'.
+class NoInferenceModelRunner : public MLModelRunner {
+public:
+  NoInferenceModelRunner(LLVMContext &Ctx,
+                         const std::vector<TensorSpec> &Inputs);
+
+private:
+  void *evaluateUntyped() override {
+    llvm_unreachable("We shouldn't call run on this model runner.");
+  }
+  void *getTensorUntyped(size_t Index) override;
+
+  std::vector<std::unique_ptr<char[]>> ValuesBuffer;
+};
+} // namespace llvm
+#endif // defined(LLVM_HAVE_TF_API)
+#endif // defined(LLVM_ANALYSIS_NOINFERENCEMODELRUNNER_H)
diff --git a/llvm/include/llvm/Analysis/ReleaseModeModelRunner.h b/llvm/include/llvm/Analysis/ReleaseModeModelRunner.h
new file mode 100644
index 000000000000..b684f87ea5cb
--- /dev/null
+++ b/llvm/include/llvm/Analysis/ReleaseModeModelRunner.h
@@ -0,0 +1,67 @@
+//===- ReleaseModeModelRunner.h - Fast, precompiled model runner  ---------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a model runner wrapping an AOT compiled ML model.
+// Only inference is supported.
+//
+//===----------------------------------------------------------------------===//
+#include "llvm/Analysis/MLModelRunner.h"
+
+#include <memory>
+#include <vector>
+
+using namespace llvm;
+namespace llvm {
+
+/// ReleaseModeModelRunner - production mode implementation of the
+/// MLModelRunner. It uses an AOT-compiled SavedModel for efficient execution.
+template <class TGen>
+class ReleaseModeModelRunner final : public MLModelRunner {
+public:
+  /// FeatureNames' type should be an indexed collection of std::string, like
+  /// std::array or std::vector, that has a size() method.
+  template <class FType>
+  ReleaseModeModelRunner(LLVMContext &Ctx, const FType &FeatureNames,
+                         StringRef DecisionName, StringRef FeedPrefix = "feed_",
+                         StringRef FetchPrefix = "fetch_")
+      : MLModelRunner(Ctx), CompiledModel(std::make_unique<TGen>()) {
+    assert(CompiledModel && "The CompiledModel should be valid");
+
+    const size_t FeatureCount = FeatureNames.size();
+    FeatureIndices.resize(FeatureCount);
+
+    for (size_t I = 0; I < FeatureCount; ++I) {
+      const int Index =
+          CompiledModel->LookupArgIndex(FeedPrefix.str() + FeatureNames[I]);
+      assert(Index >= 0 && "Cannot find Feature in inlining model");
+      FeatureIndices[I] = Index;
+    }
+
+    ResultIndex = CompiledModel->LookupResultIndex(FetchPrefix.str() +
+                                                   DecisionName.str());
+    assert(ResultIndex >= 0 && "Cannot find DecisionName in inlining model");
+  }
+
+  virtual ~ReleaseModeModelRunner() = default;
+
+private:
+  void *evaluateUntyped() override {
+    CompiledModel->Run();
+    return CompiledModel->result_data(ResultIndex);
+  }
+
+  void *getTensorUntyped(size_t Index) override {
+    return reinterpret_cast<char *>(
+        CompiledModel->arg_data(FeatureIndices[Index]));
+  }
+
+  std::vector<int32_t> FeatureIndices;
+  int32_t ResultIndex = -1;
+  std::unique_ptr<TGen> CompiledModel;
+};
+} // namespace llvm
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 170d6b8f35ff..d9f5c9689d5c 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1137,6 +1137,13 @@ public:
                   TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput,
                   const Instruction *I = nullptr) const;
 
+  /// \return The cost of VP Load and Store instructions.
+  InstructionCost
+  getVPMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
+                    unsigned AddressSpace,
+                    TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput,
+                    const Instruction *I = nullptr) const;
+
   /// \return The cost of masked Load and Store instructions.
   InstructionCost getMaskedMemoryOpCost(
       unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
@@ -1291,13 +1298,12 @@ public:
   bool areInlineCompatible(const Function *Caller,
                            const Function *Callee) const;
 
-  /// \returns True if the caller and callee agree on how \p Args will be passed
+  /// \returns True if the caller and callee agree on how \p Types will be
+  /// passed to or returned from the callee.
   /// to the callee.
-  /// \param[out] Args The list of compatible arguments.  The implementation may
-  /// filter out any incompatible args from this list.
-  bool areFunctionArgsABICompatible(const Function *Caller,
-                                    const Function *Callee,
-                                    SmallPtrSetImpl<Argument *> &Args) const;
+  /// \param Types List of types to check.
+  bool areTypesABICompatible(const Function *Caller, const Function *Callee,
+                             const ArrayRef<Type *> &Types) const;
 
   /// The type of load/store indexing.
   enum MemIndexedMode {
@@ -1388,12 +1394,17 @@ public:
   /// \returns True if the target supports scalable vectors.
   bool supportsScalableVectors() const;
 
+  /// \return true when scalable vectorization is preferred.
+  bool enableScalableVectorization() const;
+
   /// \name Vector Predication Information
   /// @{
   /// Whether the target supports the %evl parameter of VP intrinsic efficiently
-  /// in hardware. (see LLVM Language Reference - "Vector Predication
-  /// Intrinsics") Use of %evl is discouraged when that is not the case.
-  bool hasActiveVectorLength() const;
+  /// in hardware, for the given opcode and type/alignment. (see LLVM Language
+  /// Reference - "Vector Predication Intrinsics").
+  /// Use of %evl is discouraged when that is not the case.
+  bool hasActiveVectorLength(unsigned Opcode, Type *DataType,
+                             Align Alignment) const;
 
   struct VPLegalization {
     enum VPTransform {
@@ -1667,6 +1678,11 @@ public:
                                           unsigned AddressSpace,
                                           TTI::TargetCostKind CostKind,
                                           const Instruction *I) = 0;
+  virtual InstructionCost getVPMemoryOpCost(unsigned Opcode, Type *Src,
+                                            Align Alignment,
+                                            unsigned AddressSpace,
+                                            TTI::TargetCostKind CostKind,
+                                            const Instruction *I) = 0;
   virtual InstructionCost
   getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
                         unsigned AddressSpace,
@@ -1718,9 +1734,9 @@ public:
       unsigned SrcAlign, unsigned DestAlign) const = 0;
   virtual bool areInlineCompatible(const Function *Caller,
                                    const Function *Callee) const = 0;
-  virtual bool
-  areFunctionArgsABICompatible(const Function *Caller, const Function *Callee,
-                               SmallPtrSetImpl<Argument *> &Args) const = 0;
+  virtual bool areTypesABICompatible(const Function *Caller,
+                                     const Function *Callee,
+                                     const ArrayRef<Type *> &Types) const = 0;
   virtual bool isIndexedLoadLegal(MemIndexedMode Mode, Type *Ty) const = 0;
   virtual bool isIndexedStoreLegal(MemIndexedMode Mode, Type *Ty) const = 0;
   virtual unsigned getLoadStoreVecRegBitWidth(unsigned AddrSpace) const = 0;
@@ -1747,8 +1763,10 @@ public:
                                                ReductionFlags) const = 0;
   virtual bool shouldExpandReduction(const IntrinsicInst *II) const = 0;
   virtual unsigned getGISelRematGlobalCost() const = 0;
+  virtual bool enableScalableVectorization() const = 0;
   virtual bool supportsScalableVectors() const = 0;
-  virtual bool hasActiveVectorLength() const = 0;
+  virtual bool hasActiveVectorLength(unsigned Opcode, Type *DataType,
+                                     Align Alignment) const = 0;
   virtual InstructionCost getInstructionLatency(const Instruction *I) = 0;
   virtual VPLegalization
   getVPLegalizationStrategy(const VPIntrinsic &PI) const = 0;
@@ -2185,6 +2203,13 @@ public:
     return Impl.getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
                                 CostKind, I);
   }
+  InstructionCost getVPMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
+                                    unsigned AddressSpace,
+                                    TTI::TargetCostKind CostKind,
+                                    const Instruction *I) override {
+    return Impl.getVPMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
+                                  CostKind, I);
+  }
   InstructionCost getMaskedMemoryOpCost(unsigned Opcode, Type *Src,
                                         Align Alignment, unsigned AddressSpace,
                                         TTI::TargetCostKind CostKind) override {
@@ -2273,10 +2298,9 @@ public:
                            const Function *Callee) const override {
     return Impl.areInlineCompatible(Caller, Callee);
   }
-  bool areFunctionArgsABICompatible(
-      const Function *Caller, const Function *Callee,
-      SmallPtrSetImpl<Argument *> &Args) const override {
-    return Impl.areFunctionArgsABICompatible(Caller, Callee, Args);
+  bool areTypesABICompatible(const Function *Caller, const Function *Callee,
+                             const ArrayRef<Type *> &Types) const override {
+    return Impl.areTypesABICompatible(Caller, Callee, Types);
   }
   bool isIndexedLoadLegal(MemIndexedMode Mode, Type *Ty) const override {
     return Impl.isIndexedLoadLegal(Mode, Ty, getDataLayout());
@@ -2340,8 +2364,13 @@ public:
     return Impl.supportsScalableVectors();
   }
 
-  bool hasActiveVectorLength() const override {
-    return Impl.hasActiveVectorLength();
+  bool enableScalableVectorization() const override {
+    return Impl.enableScalableVectorization();
+  }
+
+  bool hasActiveVectorLength(unsigned Opcode, Type *DataType,
+                             Align Alignment) const override {
+    return Impl.hasActiveVectorLength(Opcode, DataType, Alignment);
   }
 
   InstructionCost getInstructionLatency(const Instruction *I) override {
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 05ef2495475f..26a696f09b3d 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -564,6 +564,13 @@ public:
     return 1;
   }
 
+  InstructionCost getVPMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
+                                    unsigned AddressSpace,
+                                    TTI::TargetCostKind CostKind,
+                                    const Instruction *I) const {
+    return 1;
+  }
+
   InstructionCost getMaskedMemoryOpCost(unsigned Opcode, Type *Src,
                                         Align Alignment, unsigned AddressSpace,
                                         TTI::TargetCostKind CostKind) const {
@@ -618,7 +625,6 @@ public:
     case Intrinsic::coro_frame:
     case Intrinsic::coro_size:
     case Intrinsic::coro_suspend:
-    case Intrinsic::coro_param:
     case Intrinsic::coro_subfn_addr:
       // These intrinsics don't actually represent code after lowering.
       return 0;
@@ -702,9 +708,8 @@ public:
             Callee->getFnAttribute("target-features"));
   }
 
-  bool areFunctionArgsABICompatible(const Function *Caller,
-                                    const Function *Callee,
-                                    SmallPtrSetImpl<Argument *> &Args) const {
+  bool areTypesABICompatible(const Function *Caller, const Function *Callee,
+                             const ArrayRef<Type *> &Types) const {
     return (Caller->getFnAttribute("target-cpu") ==
             Callee->getFnAttribute("target-cpu")) &&
            (Caller->getFnAttribute("target-features") ==
@@ -772,7 +777,12 @@ public:
 
   bool supportsScalableVectors() const { return false; }
 
-  bool hasActiveVectorLength() const { return false; }
+  bool enableScalableVectorization() const { return false; }
+
+  bool hasActiveVectorLength(unsigned Opcode, Type *DataType,
+                             Align Alignment) const {
+    return false;
+  }
 
   TargetTransformInfo::VPLegalization
   getVPLegalizationStrategy(const VPIntrinsic &PI) const {
diff --git a/llvm/include/llvm/Analysis/Utils/TFUtils.h b/llvm/include/llvm/Analysis/Utils/TFUtils.h
index 1f6be0e60eb9..012fca53a200 100644
--- a/llvm/include/llvm/Analysis/Utils/TFUtils.h
+++ b/llvm/include/llvm/Analysis/Utils/TFUtils.h
@@ -147,6 +147,9 @@ public:
   /// Construct a Logger. If IncludeReward is false, then logReward or
   /// logFinalReward shouldn't be called, and the reward feature won't be
   /// printed out.
+  /// NOTE: the FeatureSpecs are expected to be in the same order (i.e. have
+  /// corresponding indices) with any MLModelRunner implementations
+  /// corresponding to the model being trained/logged.
   Logger(const std::vector<LoggedFeatureSpec> &FeatureSpecs,
          const TensorSpec &RewardSpec, bool IncludeReward);
 
@@ -246,8 +249,10 @@ public:
   /// otherwise.
   bool isValid() const { return !!Impl; }
 
-private:
+  /// Untyped access to input.
   void *getUntypedInput(size_t Index);
+
+private:
   std::unique_ptr<TFModelEvaluatorImpl> Impl;
 };
 
diff --git a/llvm/include/llvm/AsmParser/LLParser.h b/llvm/include/llvm/AsmParser/LLParser.h
index d621c232378c..62af3afbc142 100644
--- a/llvm/include/llvm/AsmParser/LLParser.h
+++ b/llvm/include/llvm/AsmParser/LLParser.h
@@ -62,12 +62,14 @@ namespace llvm {
     APFloat APFloatVal{0.0};
     Constant *ConstantVal;
     std::unique_ptr<Constant *[]> ConstantStructElts;
+    bool NoCFI = false;
 
     ValID() = default;
     ValID(const ValID &RHS)
         : Kind(RHS.Kind), Loc(RHS.Loc), UIntVal(RHS.UIntVal), FTy(RHS.FTy),
           StrVal(RHS.StrVal), StrVal2(RHS.StrVal2), APSIntVal(RHS.APSIntVal),
-          APFloatVal(RHS.APFloatVal), ConstantVal(RHS.ConstantVal) {
+          APFloatVal(RHS.APFloatVal), ConstantVal(RHS.ConstantVal),
+          NoCFI(RHS.NoCFI) {
       assert(!RHS.ConstantStructElts);
     }
 
diff --git a/llvm/include/llvm/AsmParser/LLToken.h b/llvm/include/llvm/AsmParser/LLToken.h
index f8ca054863ac..78ebb35e0ea4 100644
--- a/llvm/include/llvm/AsmParser/LLToken.h
+++ b/llvm/include/llvm/AsmParser/LLToken.h
@@ -370,6 +370,7 @@ enum Kind {
   kw_insertvalue,
   kw_blockaddress,
   kw_dso_local_equivalent,
+  kw_no_cfi,
 
   kw_freeze,
 
@@ -407,6 +408,7 @@ enum Kind {
   kw_noUnwind,
   kw_mayThrow,
   kw_hasUnknownCall,
+  kw_mustBeUnreachable,
   kw_calls,
   kw_callee,
   kw_params,
diff --git a/llvm/include/llvm/BinaryFormat/ELF.h b/llvm/include/llvm/BinaryFormat/ELF.h
index c199e933116a..065661cbd188 100644
--- a/llvm/include/llvm/BinaryFormat/ELF.h
+++ b/llvm/include/llvm/BinaryFormat/ELF.h
@@ -608,6 +608,8 @@ enum {
   EF_HEXAGON_MACH_V67 = 0x00000067,  // Hexagon V67
   EF_HEXAGON_MACH_V67T = 0x00008067, // Hexagon V67T
   EF_HEXAGON_MACH_V68 = 0x00000068,  // Hexagon V68
+  EF_HEXAGON_MACH_V69 = 0x00000069,  // Hexagon V69
+  EF_HEXAGON_MACH = 0x000003ff,      // Hexagon V..
 
   // Highest ISA version flags
   EF_HEXAGON_ISA_MACH = 0x00000000, // Same as specified in bits[11:0]
@@ -623,6 +625,8 @@ enum {
   EF_HEXAGON_ISA_V66 = 0x00000066,  // Hexagon V66 ISA
   EF_HEXAGON_ISA_V67 = 0x00000067,  // Hexagon V67 ISA
   EF_HEXAGON_ISA_V68 = 0x00000068,  // Hexagon V68 ISA
+  EF_HEXAGON_ISA_V69 = 0x00000069,  // Hexagon V69 ISA
+  EF_HEXAGON_ISA = 0x000003ff,      // Hexagon V.. ISA
 };
 
 // Hexagon-specific section indexes for common small data
diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
index 04eb2739cbd5..7301618d337a 100644
--- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h
+++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
@@ -381,9 +381,10 @@ enum ConstantsCodes {
   CST_CODE_CE_UNOP = 25,                   // CE_UNOP:      [opcode, opval]
   CST_CODE_POISON = 26,                    // POISON
   CST_CODE_DSO_LOCAL_EQUIVALENT = 27,      // DSO_LOCAL_EQUIVALENT [gvty, gv]
-  CST_CODE_INLINEASM = 28, // INLINEASM:     [sideeffect|alignstack|
-                           //                 asmdialect|unwind,
-                           //                 asmstr,conststr]
+  CST_CODE_INLINEASM = 28,    // INLINEASM:     [sideeffect|alignstack|
+                              //                 asmdialect|unwind,
+                              //                 asmstr,conststr]
+  CST_CODE_NO_CFI_VALUE = 29, // NO_CFI [ fty, f ]
 };
 
 /// CastOpcodes - These are values used in the bitcode files to encode which
diff --git a/llvm/include/llvm/CodeGen/AsmPrinter.h b/llvm/include/llvm/CodeGen/AsmPrinter.h
index d7d3692877de..281ecb8de251 100644
--- a/llvm/include/llvm/CodeGen/AsmPrinter.h
+++ b/llvm/include/llvm/CodeGen/AsmPrinter.h
@@ -799,6 +799,11 @@ private:
 
   /// This method decides whether the specified basic block requires a label.
   bool shouldEmitLabelForBasicBlock(const MachineBasicBlock &MBB) const;
+
+protected:
+  virtual bool shouldEmitWeakSwiftAsyncExtendedFramePointerFlags() const {
+    return false;
+  }
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h
index 8a603de2f91d..886b3af834d7 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h
@@ -1248,7 +1248,7 @@ private:
     for (auto *DeadMI : DeadInsts) {
       LLVM_DEBUG(dbgs() << *DeadMI << "Is dead, eagerly deleting\n");
       WrapperObserver.erasingInstr(*DeadMI);
-      DeadMI->eraseFromParentAndMarkDBGValuesForRemoval();
+      DeadMI->eraseFromParent();
     }
     DeadInsts.clear();
   }
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
index 74615c73741a..044f2e22cfdd 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
@@ -192,6 +192,10 @@ private:
                     SmallVectorImpl<Register> &VRegs,
                     SmallVectorImpl<Register> &LeftoverVRegs);
 
+  /// Version which handles irregular sub-vector splits.
+  void extractVectorParts(Register Reg, unsigned NumElst,
+                          SmallVectorImpl<Register> &VRegs);
+
   /// Helper function to build a wide generic register \p DstReg of type \p
   /// RegTy from smaller parts. This will produce a G_MERGE_VALUES,
   /// G_BUILD_VECTOR, G_CONCAT_VECTORS, or sequence of G_INSERT as appropriate
@@ -205,6 +209,11 @@ private:
                    LLT PartTy, ArrayRef<Register> PartRegs,
                    LLT LeftoverTy = LLT(), ArrayRef<Register> LeftoverRegs = {});
 
+  /// Merge \p PartRegs with different types into \p DstReg.
+  void mergeMixedSubvectors(Register DstReg, ArrayRef<Register> PartRegs);
+
+  void appendVectorElts(SmallVectorImpl<Register> &Elts, Register Reg);
+
   /// Unmerge \p SrcReg into smaller sized values, and append them to \p
   /// Parts. The elements of \p Parts will be the greatest common divisor type
   /// of \p DstTy, \p NarrowTy and the type of \p SrcReg. This will compute and
@@ -285,26 +294,18 @@ public:
   /// vector bounds.
   Register getVectorElementPointer(Register VecPtr, LLT VecTy, Register Index);
 
-  LegalizeResult fewerElementsVectorImplicitDef(MachineInstr &MI,
-                                                unsigned TypeIdx, LLT NarrowTy);
-
-  /// Legalize a instruction with a vector type where each operand may have a
-  /// different element type. All type indexes must have the same number of
-  /// elements.
-  LegalizeResult fewerElementsVectorMultiEltType(MachineInstr &MI,
-                                                 unsigned TypeIdx, LLT NarrowTy);
+  /// Handles most opcodes. Split \p MI into same instruction on sub-vectors or
+  /// scalars with \p NumElts elements (1 for scalar). Supports uneven splits:
+  /// there can be leftover sub-vector with fewer then \p NumElts or a leftover
+  /// scalar. To avoid this use moreElements first and set MI number of elements
+  /// to multiple of \p NumElts. Non-vector operands that should be used on all
+  /// sub-instructions without split are listed in \p NonVecOpIndices.
+  LegalizeResult fewerElementsVectorMultiEltType(
+      GenericMachineInstr &MI, unsigned NumElts,
+      std::initializer_list<unsigned> NonVecOpIndices = {});
 
-  LegalizeResult fewerElementsVectorCasts(MachineInstr &MI, unsigned TypeIdx,
-                                          LLT NarrowTy);
-
-  LegalizeResult
-  fewerElementsVectorCmp(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy);
-
-  LegalizeResult
-  fewerElementsVectorSelect(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy);
-
-  LegalizeResult fewerElementsVectorPhi(MachineInstr &MI,
-                                        unsigned TypeIdx, LLT NarrowTy);
+  LegalizeResult fewerElementsVectorPhi(GenericMachineInstr &MI,
+                                        unsigned NumElts);
 
   LegalizeResult moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx,
                                        LLT MoreTy);
@@ -320,22 +321,9 @@ public:
                                                            unsigned TypeIdx,
                                                            LLT NarrowTy);
 
-  LegalizeResult fewerElementsVectorMulo(MachineInstr &MI, unsigned TypeIdx,
-                                         LLT NarrowTy);
-
   LegalizeResult reduceLoadStoreWidth(GLoadStore &MI, unsigned TypeIdx,
                                       LLT NarrowTy);
 
-  /// Legalize an instruction by reducing the operation width, either by
-  /// narrowing the type of the operation or by reducing the number of elements
-  /// of a vector.
-  /// The used strategy (narrow vs. fewerElements) is decided by \p NarrowTy.
-  /// Narrow is used if the scalar type of \p NarrowTy and \p DstTy differ,
-  /// fewerElements is used when the scalar type is the same but the number of
-  /// elements between \p NarrowTy and \p DstTy differ.
-  LegalizeResult reduceOperationWidth(MachineInstr &MI, unsigned TypeIdx,
-                                      LLT NarrowTy);
-
   LegalizeResult fewerElementsVectorSextInReg(MachineInstr &MI, unsigned TypeIdx,
                                               LLT NarrowTy);
 
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
index 68c14240ebc7..0b37539030b1 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
@@ -58,7 +58,10 @@ enum LegalizeAction : std::uint8_t {
 
   /// The (vector) operation should be implemented by splitting it into
   /// sub-vectors where the operation is legal. For example a <8 x s64> add
-  /// might be implemented as 4 separate <2 x s64> adds.
+  /// might be implemented as 4 separate <2 x s64> adds. There can be a leftover
+  /// if there are not enough elements for last sub-vector e.g. <7 x s64> add
+  /// will be implemented as 3 separate <2 x s64> adds and one s64 add. Leftover
+  /// types can be avoided by doing MoreElements first.
   FewerElements,
 
   /// The (vector) operation should be implemented by widening the input
@@ -1050,6 +1053,26 @@ public:
               TypeIdx, LLT::fixed_vector(MinElements, VecTy.getElementType()));
         });
   }
+
+  /// Set number of elements to nearest larger multiple of NumElts.
+  LegalizeRuleSet &alignNumElementsTo(unsigned TypeIdx, const LLT EltTy,
+                                      unsigned NumElts) {
+    typeIdx(TypeIdx);
+    return actionIf(
+        LegalizeAction::MoreElements,
+        [=](const LegalityQuery &Query) {
+          LLT VecTy = Query.Types[TypeIdx];
+          return VecTy.isVector() && VecTy.getElementType() == EltTy &&
+                 (VecTy.getNumElements() % NumElts != 0);
+        },
+        [=](const LegalityQuery &Query) {
+          LLT VecTy = Query.Types[TypeIdx];
+          unsigned NewSize = alignTo(VecTy.getNumElements(), NumElts);
+          return std::make_pair(
+              TypeIdx, LLT::fixed_vector(NewSize, VecTy.getElementType()));
+        });
+  }
+
   /// Limit the number of elements in EltTy vectors to at most MaxElements.
   LegalizeRuleSet &clampMaxNumElements(unsigned TypeIdx, const LLT EltTy,
                                        unsigned MaxElements) {
@@ -1085,6 +1108,19 @@ public:
         .clampMaxNumElements(TypeIdx, EltTy, MaxTy.getNumElements());
   }
 
+  /// Express \p EltTy vectors strictly using vectors with \p NumElts elements
+  /// (or scalars when \p NumElts equals 1).
+  /// First pad with undef elements to nearest larger multiple of \p NumElts.
+  /// Then perform split with all sub-instructions having the same type.
+  /// Using clampMaxNumElements (non-strict) can result in leftover instruction
+  /// with different type (fewer elements then \p NumElts or scalar).
+  /// No effect if the type is not a vector.
+  LegalizeRuleSet &clampMaxNumElementsStrict(unsigned TypeIdx, const LLT EltTy,
+                                             unsigned NumElts) {
+    return alignNumElementsTo(TypeIdx, EltTy, NumElts)
+        .clampMaxNumElements(TypeIdx, EltTy, NumElts);
+  }
+
   /// Fallback on the previous implementation. This should only be used while
   /// porting a rule.
   LegalizeRuleSet &fallback() {
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h b/llvm/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h
index a41166bb4c6b..28bb8de11762 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h
@@ -361,9 +361,9 @@ m_GAdd(const LHS &L, const RHS &R) {
 }
 
 template <typename LHS, typename RHS>
-inline BinaryOp_match<LHS, RHS, TargetOpcode::G_PTR_ADD, true>
+inline BinaryOp_match<LHS, RHS, TargetOpcode::G_PTR_ADD, false>
 m_GPtrAdd(const LHS &L, const RHS &R) {
-  return BinaryOp_match<LHS, RHS, TargetOpcode::G_PTR_ADD, true>(L, R);
+  return BinaryOp_match<LHS, RHS, TargetOpcode::G_PTR_ADD, false>(L, R);
 }
 
 template <typename LHS, typename RHS>
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
index 069f71b54328..fde0cb3cf1af 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
@@ -497,6 +497,34 @@ public:
   MachineInstrBuilder buildMaskLowPtrBits(const DstOp &Res, const SrcOp &Op0,
                                           uint32_t NumBits);
 
+  /// Build and insert
+  /// a, b, ..., x = G_UNMERGE_VALUES \p Op0
+  /// \p Res = G_BUILD_VECTOR a, b, ..., x, undef, ..., undef
+  ///
+  /// Pad \p Op0 with undef elements to match number of elements in \p Res.
+  ///
+  /// \pre setBasicBlock or setMI must have been called.
+  /// \pre \p Res and \p Op0 must be generic virtual registers with vector type,
+  ///      same vector element type and Op0 must have fewer elements then Res.
+  ///
+  /// \return a MachineInstrBuilder for the newly created build vector instr.
+  MachineInstrBuilder buildPadVectorWithUndefElements(const DstOp &Res,
+                                                      const SrcOp &Op0);
+
+  /// Build and insert
+  /// a, b, ..., x, y, z = G_UNMERGE_VALUES \p Op0
+  /// \p Res = G_BUILD_VECTOR a, b, ..., x
+  ///
+  /// Delete trailing elements in \p Op0 to match number of elements in \p Res.
+  ///
+  /// \pre setBasicBlock or setMI must have been called.
+  /// \pre \p Res and \p Op0 must be generic virtual registers with vector type,
+  ///      same vector element type and Op0 must have more elements then Res.
+  ///
+  /// \return a MachineInstrBuilder for the newly created build vector instr.
+  MachineInstrBuilder buildDeleteTrailingVectorElements(const DstOp &Res,
+                                                        const SrcOp &Op0);
+
   /// Build and insert \p Res, \p CarryOut = G_UADDO \p Op0, \p Op1
   ///
   /// G_UADDO sets \p Res to \p Op0 + \p Op1 (truncated to the bit width) and
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
index 4126e2ac7b8f..8fed79585fe9 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
@@ -323,6 +323,11 @@ Register getFunctionLiveInPhysReg(MachineFunction &MF, const TargetInstrInfo &TI
 LLVM_READNONE
 LLT getLCMType(LLT OrigTy, LLT TargetTy);
 
+LLVM_READNONE
+/// Return smallest type that covers both \p OrigTy and \p TargetTy and is
+/// multiple of TargetTy.
+LLT getCoverTy(LLT OrigTy, LLT TargetTy);
+
 /// Return a type where the total size is the greatest common divisor of \p
 /// OrigTy and \p TargetTy. This will try to either change the number of vector
 /// elements, or bitwidth of scalars. The intent is the result type can be used
diff --git a/llvm/include/llvm/CodeGen/MIRYamlMapping.h b/llvm/include/llvm/CodeGen/MIRYamlMapping.h
index b6d7c2487126..05a375bc251b 100644
--- a/llvm/include/llvm/CodeGen/MIRYamlMapping.h
+++ b/llvm/include/llvm/CodeGen/MIRYamlMapping.h
@@ -695,6 +695,7 @@ struct MachineFunction {
   bool TracksRegLiveness = false;
   bool HasWinCFI = false;
   bool FailsVerification = false;
+  bool TracksDebugUserValues = false;
   std::vector<VirtualRegisterDefinition> VirtualRegisters;
   std::vector<MachineFunctionLiveIn> LiveIns;
   Optional<std::vector<FlowStringValue>> CalleeSavedRegisters;
@@ -724,6 +725,8 @@ template <> struct MappingTraits<MachineFunction> {
     YamlIO.mapOptional("tracksRegLiveness", MF.TracksRegLiveness, false);
     YamlIO.mapOptional("hasWinCFI", MF.HasWinCFI, false);
     YamlIO.mapOptional("failsVerification", MF.FailsVerification, false);
+    YamlIO.mapOptional("tracksDebugUserValues", MF.TracksDebugUserValues,
+                       false);
     YamlIO.mapOptional("registers", MF.VirtualRegisters,
                        std::vector<VirtualRegisterDefinition>());
     YamlIO.mapOptional("liveins", MF.LiveIns,
diff --git a/llvm/include/llvm/CodeGen/MachineCycleAnalysis.h b/llvm/include/llvm/CodeGen/MachineCycleAnalysis.h
new file mode 100644
index 000000000000..d3816bbc0780
--- /dev/null
+++ b/llvm/include/llvm/CodeGen/MachineCycleAnalysis.h
@@ -0,0 +1,31 @@
+//===- MachineCycleAnalysis.h - Cycle Info for Machine IR -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the MachineCycleInfo class, which is a thin wrapper over
+// the Machine IR instance of GenericCycleInfo.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_MACHINECYCLEANALYSIS_H
+#define LLVM_CODEGEN_MACHINECYCLEANALYSIS_H
+
+#include "llvm/ADT/GenericCycleInfo.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineSSAContext.h"
+
+namespace llvm {
+
+extern template class GenericCycleInfo<MachineSSAContext>;
+extern template class GenericCycle<MachineSSAContext>;
+
+using MachineCycleInfo = GenericCycleInfo<MachineSSAContext>;
+using MachineCycle = MachineCycleInfo::CycleT;
+
+} // end namespace llvm
+
+#endif // LLVM_CODEGEN_MACHINECYCLEANALYSIS_H
diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h
index ec23dde0c6c0..c4767a51b094 100644
--- a/llvm/include/llvm/CodeGen/MachineFunction.h
+++ b/llvm/include/llvm/CodeGen/MachineFunction.h
@@ -152,6 +152,12 @@ public:
   // FailsVerification: Means that the function is not expected to pass machine
   //  verification. This can be set by passes that introduce known problems that
   //  have not been fixed yet.
+  // TracksDebugUserValues: Without this property enabled, debug instructions
+  // such as DBG_VALUE are allowed to reference virtual registers even if those
+  // registers do not have a definition. With the property enabled virtual
+  // registers must only be used if they have a definition. This property
+  // allows earlier passes in the pipeline to skip updates of `DBG_VALUE`
+  // instructions to save compile time.
   enum class Property : unsigned {
     IsSSA,
     NoPHIs,
@@ -163,7 +169,8 @@ public:
     Selected,
     TiedOpsRewritten,
     FailsVerification,
-    LastProperty = FailsVerification,
+    TracksDebugUserValues,
+    LastProperty = TracksDebugUserValues,
   };
 
   bool hasProperty(Property P) const {
@@ -883,7 +890,7 @@ public:
 
   /// CreateMachineInstr - Allocate a new MachineInstr. Use this instead
   /// of `new MachineInstr'.
-  MachineInstr *CreateMachineInstr(const MCInstrDesc &MCID, const DebugLoc &DL,
+  MachineInstr *CreateMachineInstr(const MCInstrDesc &MCID, DebugLoc DL,
                                    bool NoImplicit = false);
 
   /// Create a new MachineInstr which is a copy of \p Orig, identical in all
@@ -900,18 +907,20 @@ public:
   ///
   /// Note: Does not perform target specific adjustments; consider using
   /// TargetInstrInfo::duplicate() intead.
-  MachineInstr &CloneMachineInstrBundle(MachineBasicBlock &MBB,
-      MachineBasicBlock::iterator InsertBefore, const MachineInstr &Orig);
+  MachineInstr &
+  cloneMachineInstrBundle(MachineBasicBlock &MBB,
+                          MachineBasicBlock::iterator InsertBefore,
+                          const MachineInstr &Orig);
 
   /// DeleteMachineInstr - Delete the given MachineInstr.
-  void DeleteMachineInstr(MachineInstr *MI);
+  void deleteMachineInstr(MachineInstr *MI);
 
   /// CreateMachineBasicBlock - Allocate a new MachineBasicBlock. Use this
   /// instead of `new MachineBasicBlock'.
   MachineBasicBlock *CreateMachineBasicBlock(const BasicBlock *bb = nullptr);
 
   /// DeleteMachineBasicBlock - Delete the given MachineBasicBlock.
-  void DeleteMachineBasicBlock(MachineBasicBlock *MBB);
+  void deleteMachineBasicBlock(MachineBasicBlock *MBB);
 
   /// getMachineMemOperand - Allocate a new MachineMemOperand.
   /// MachineMemOperands are owned by the MachineFunction and need not be
diff --git a/llvm/include/llvm/CodeGen/MachineInstr.h b/llvm/include/llvm/CodeGen/MachineInstr.h
index 0ac934e208b6..2893e138a95c 100644
--- a/llvm/include/llvm/CodeGen/MachineInstr.h
+++ b/llvm/include/llvm/CodeGen/MachineInstr.h
@@ -249,7 +249,7 @@ private:
                  PointerSumTypeMember<EIIK_OutOfLine, ExtraInfo *>>
       Info;
 
-  DebugLoc debugLoc;                    // Source line information.
+  DebugLoc DbgLoc; // Source line information.
 
   /// Unique instruction number. Used by DBG_INSTR_REFs to refer to the values
   /// defined by this instruction.
@@ -267,7 +267,7 @@ private:
   /// This constructor create a MachineInstr and add the implicit operands.
   /// It reserves space for number of operands specified by
   /// MCInstrDesc.  An explicit DebugLoc is supplied.
-  MachineInstr(MachineFunction &, const MCInstrDesc &tid, DebugLoc dl,
+  MachineInstr(MachineFunction &, const MCInstrDesc &TID, DebugLoc DL,
                bool NoImp = false);
 
   // MachineInstrs are pool-allocated and owned by MachineFunction.
@@ -415,7 +415,7 @@ public:
   void unbundleFromSucc();
 
   /// Returns the debug location id of this MachineInstr.
-  const DebugLoc &getDebugLoc() const { return debugLoc; }
+  const DebugLoc &getDebugLoc() const { return DbgLoc; }
 
   /// Return the operand containing the offset to be used if this DBG_VALUE
   /// instruction is indirect; will be an invalid register if this value is
@@ -1173,12 +1173,6 @@ public:
   /// eraseFromBundle() to erase individual bundled instructions.
   void eraseFromParent();
 
-  /// Unlink 'this' from the containing basic block and delete it.
-  ///
-  /// For all definitions mark their uses in DBG_VALUE nodes
-  /// as undefined. Otherwise like eraseFromParent().
-  void eraseFromParentAndMarkDBGValuesForRemoval();
-
   /// Unlink 'this' form its basic block and delete it.
   ///
   /// If the instruction is part of a bundle, the other instructions in the
@@ -1739,13 +1733,13 @@ public:
 
   /// Replace the instruction descriptor (thus opcode) of
   /// the current instruction with a new one.
-  void setDesc(const MCInstrDesc &tid) { MCID = &tid; }
+  void setDesc(const MCInstrDesc &TID) { MCID = &TID; }
 
   /// Replace current source information with new such.
   /// Avoid using this, the constructor argument is preferable.
-  void setDebugLoc(DebugLoc dl) {
-    debugLoc = std::move(dl);
-    assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor");
+  void setDebugLoc(DebugLoc DL) {
+    DbgLoc = std::move(DL);
+    assert(DbgLoc.hasTrivialDestructor() && "Expected trivial destructor");
   }
 
   /// Erase an operand from an instruction, leaving it with one
diff --git a/llvm/include/llvm/CodeGen/MachinePassRegistry.def b/llvm/include/llvm/CodeGen/MachinePassRegistry.def
index d79303c771d6..e6763899a083 100644
--- a/llvm/include/llvm/CodeGen/MachinePassRegistry.def
+++ b/llvm/include/llvm/CodeGen/MachinePassRegistry.def
@@ -197,4 +197,6 @@ DUMMY_MACHINE_FUNCTION_PASS("regbankselect", RegBankSelectPass, ())
 DUMMY_MACHINE_FUNCTION_PASS("instruction-select", InstructionSelectPass, ())
 DUMMY_MACHINE_FUNCTION_PASS("reset-machine-function", ResetMachineFunctionPass, ())
 DUMMY_MACHINE_FUNCTION_PASS("machineverifier", MachineVerifierPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("machine-cycles", MachineCycleInfoWrapperPass, ())
+DUMMY_MACHINE_FUNCTION_PASS("print-machine-cycles", MachineCycleInfoPrinterPass, ())
 #undef DUMMY_MACHINE_FUNCTION_PASS
diff --git a/llvm/include/llvm/CodeGen/MachineSSAContext.h b/llvm/include/llvm/CodeGen/MachineSSAContext.h
new file mode 100644
index 000000000000..6dbf321bdeaa
--- /dev/null
+++ b/llvm/include/llvm/CodeGen/MachineSSAContext.h
@@ -0,0 +1,58 @@
+//===- MachineSSAContext.h --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+///
+/// This file declares a specialization of the GenericSSAContext<X>
+/// template class for Machine IR.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_MACHINESSACONTEXT_H
+#define LLVM_CODEGEN_MACHINESSACONTEXT_H
+
+#include "llvm/ADT/GenericSSAContext.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/Support/Printable.h"
+
+#include <memory>
+
+namespace llvm {
+class MachineInstr;
+class MachineBasicBlock;
+class MachineFunction;
+class Register;
+template <typename, bool> class DominatorTreeBase;
+
+inline auto successors(MachineBasicBlock *BB) { return BB->successors(); }
+inline auto predecessors(MachineBasicBlock *BB) { return BB->predecessors(); }
+
+template <> class GenericSSAContext<MachineFunction> {
+  const MachineRegisterInfo *RegInfo = nullptr;
+  MachineFunction *MF;
+
+public:
+  using BlockT = MachineBasicBlock;
+  using FunctionT = MachineFunction;
+  using InstructionT = MachineInstr;
+  using ValueRefT = Register;
+  using DominatorTreeT = DominatorTreeBase<BlockT, false>;
+
+  static MachineBasicBlock *getEntryBlock(MachineFunction &F);
+
+  void setFunction(MachineFunction &Fn);
+  MachineFunction *getFunction() const { return MF; }
+
+  Printable print(MachineBasicBlock *Block) const;
+  Printable print(MachineInstr *Inst) const;
+  Printable print(Register Value) const;
+};
+
+using MachineSSAContext = GenericSSAContext<MachineFunction>;
+} // namespace llvm
+
+#endif // LLVM_CODEGEN_MACHINESSACONTEXT_H
diff --git a/llvm/include/llvm/CodeGen/MachineSSAUpdater.h b/llvm/include/llvm/CodeGen/MachineSSAUpdater.h
index 0af356e376ab..3f0b55e0abb8 100644
--- a/llvm/include/llvm/CodeGen/MachineSSAUpdater.h
+++ b/llvm/include/llvm/CodeGen/MachineSSAUpdater.h
@@ -77,7 +77,9 @@ public:
   Register GetValueAtEndOfBlock(MachineBasicBlock *BB);
 
   /// GetValueInMiddleOfBlock - Construct SSA form, materializing a value that
-  /// is live in the middle of the specified block.
+  /// is live in the middle of the specified block. If ExistingValueOnly is
+  /// true then this will only return an existing value or $noreg; otherwise new
+  /// instructions may be inserted to materialize a value.
   ///
   /// GetValueInMiddleOfBlock is the same as GetValueAtEndOfBlock except in one
   /// important case: if there is a definition of the rewritten value after the
@@ -94,7 +96,8 @@ public:
   /// their respective blocks.  However, the use of X happens in the *middle* of
   /// a block.  Because of this, we need to insert a new PHI node in SomeBB to
   /// merge the appropriate values, and this value isn't live out of the block.
-  Register GetValueInMiddleOfBlock(MachineBasicBlock *BB);
+  Register GetValueInMiddleOfBlock(MachineBasicBlock *BB,
+                                   bool ExistingValueOnly = false);
 
   /// RewriteUse - Rewrite a use of the symbolic value.  This handles PHI nodes,
   /// which use their value in the corresponding predecessor.  Note that this
@@ -104,7 +107,10 @@ public:
   void RewriteUse(MachineOperand &U);
 
 private:
-  Register GetValueAtEndOfBlockInternal(MachineBasicBlock *BB);
+  // If ExistingValueOnly is true, will not create any new instructions. Used
+  // for debug values, which cannot modify Codegen.
+  Register GetValueAtEndOfBlockInternal(MachineBasicBlock *BB,
+                                        bool ExistingValueOnly = false);
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/CodeGen/MachineScheduler.h b/llvm/include/llvm/CodeGen/MachineScheduler.h
index 5bd5c8aa757a..e368fd7d056a 100644
--- a/llvm/include/llvm/CodeGen/MachineScheduler.h
+++ b/llvm/include/llvm/CodeGen/MachineScheduler.h
@@ -101,6 +101,11 @@ namespace llvm {
 extern cl::opt<bool> ForceTopDown;
 extern cl::opt<bool> ForceBottomUp;
 extern cl::opt<bool> VerifyScheduling;
+#ifndef NDEBUG
+extern cl::opt<bool> ViewMISchedDAGs;
+#else
+extern const bool ViewMISchedDAGs;
+#endif
 
 class AAResults;
 class LiveIntervals;
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index 5a3f4e9a23ff..d21844555f5b 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -1912,10 +1912,10 @@ public:
   ///
   /// NOTE: The function will return true for a demanded splat of UNDEF values.
   bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts,
-                    unsigned Depth = 0);
+                    unsigned Depth = 0) const;
 
   /// Test whether \p V has a splatted value.
-  bool isSplatValue(SDValue V, bool AllowUndefs = false);
+  bool isSplatValue(SDValue V, bool AllowUndefs = false) const;
 
   /// If V is a splatted value, return the source vector and its splat index.
   SDValue getSplatSourceVector(SDValue V, int &SplatIndex);
diff --git a/llvm/include/llvm/CodeGen/StackProtector.h b/llvm/include/llvm/CodeGen/StackProtector.h
index f6513e8d4ea0..57456b3f6c16 100644
--- a/llvm/include/llvm/CodeGen/StackProtector.h
+++ b/llvm/include/llvm/CodeGen/StackProtector.h
@@ -95,7 +95,7 @@ private:
                                 bool InStruct = false) const;
 
   /// Check whether a stack allocation has its address taken.
-  bool HasAddressTaken(const Instruction *AI, uint64_t AllocSize);
+  bool HasAddressTaken(const Instruction *AI, TypeSize AllocSize);
 
   /// RequiresStackProtector - Check whether or not this function needs a
   /// stack protector based upon the stack protector level.
diff --git a/llvm/include/llvm/CodeGen/TargetInstrInfo.h b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
index d43dd9fac85d..58b8e59b68d7 100644
--- a/llvm/include/llvm/CodeGen/TargetInstrInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
@@ -1190,8 +1190,6 @@ public:
                                      MachineInstr &NewMI1,
                                      MachineInstr &NewMI2) const {}
 
-  virtual void setSpecialOperandAttr(MachineInstr &MI, uint16_t Flags) const {}
-
   /// Return true when a target supports MachineCombiner.
   virtual bool useMachineCombiner() const { return false; }
 
@@ -1929,9 +1927,7 @@ public:
   /// Optional target hook that returns true if \p MBB is safe to outline from,
   /// and returns any target-specific information in \p Flags.
   virtual bool isMBBSafeToOutlineFrom(MachineBasicBlock &MBB,
-                                      unsigned &Flags) const {
-    return true;
-  }
+                                      unsigned &Flags) const;
 
   /// Insert a custom frame for outlined functions.
   virtual void buildOutlinedFrame(MachineBasicBlock &MBB, MachineFunction &MF,
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index d862701c37d7..b2d82e0cc6e8 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -852,6 +852,20 @@ public:
     return getBooleanContents(Type.isVector(), Type.isFloatingPoint());
   }
 
+  /// Promote the given target boolean to a target boolean of the given type.
+  /// A target boolean is an integer value, not necessarily of type i1, the bits
+  /// of which conform to getBooleanContents.
+  ///
+  /// ValVT is the type of values that produced the boolean.
+  SDValue promoteTargetBoolean(SelectionDAG &DAG, SDValue Bool,
+                               EVT ValVT) const {
+    SDLoc dl(Bool);
+    EVT BoolVT =
+        getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ValVT);
+    ISD::NodeType ExtendCode = getExtendForContent(getBooleanContents(ValVT));
+    return DAG.getNode(ExtendCode, dl, BoolVT, Bool);
+  }
+
   /// Return target scheduling preference.
   Sched::Preference getSchedulingPreference() const {
     return SchedPreferenceInfo;
@@ -3606,6 +3620,13 @@ public:
                                             const SelectionDAG &DAG,
                                             bool SNaN = false,
                                             unsigned Depth = 0) const;
+
+  /// Return true if vector \p Op has the same value across all \p DemandedElts,
+  /// indicating any elements which may be undef in the output \p UndefElts.
+  virtual bool isSplatValueForTargetNode(SDValue Op, const APInt &DemandedElts,
+                                         APInt &UndefElts,
+                                         unsigned Depth = 0) const;
+
   struct DAGCombinerInfo {
     void *DC;  // The DAG Combiner object.
     CombineLevel Level;
@@ -4460,18 +4481,15 @@ public:
 
   /// Expand funnel shift.
   /// \param N Node to expand
-  /// \param Result output after conversion
-  /// \returns True, if the expansion was successful, false otherwise
-  bool expandFunnelShift(SDNode *N, SDValue &Result, SelectionDAG &DAG) const;
+  /// \returns The expansion if successful, SDValue() otherwise
+  SDValue expandFunnelShift(SDNode *N, SelectionDAG &DAG) const;
 
   /// Expand rotations.
   /// \param N Node to expand
   /// \param AllowVectorOps expand vector rotate, this should only be performed
   ///        if the legalization is happening outside of LegalizeVectorOps
-  /// \param Result output after conversion
-  /// \returns True, if the expansion was successful, false otherwise
-  bool expandROT(SDNode *N, bool AllowVectorOps, SDValue &Result,
-                 SelectionDAG &DAG) const;
+  /// \returns The expansion if successful, SDValue() otherwise
+  SDValue expandROT(SDNode *N, bool AllowVectorOps, SelectionDAG &DAG) const;
 
   /// Expand shift-by-parts.
   /// \param N Node to expand
diff --git a/llvm/include/llvm/CodeGen/VLIWMachineScheduler.h b/llvm/include/llvm/CodeGen/VLIWMachineScheduler.h
new file mode 100644
index 000000000000..a39f04f6db6c
--- /dev/null
+++ b/llvm/include/llvm/CodeGen/VLIWMachineScheduler.h
@@ -0,0 +1,268 @@
+//===- VLIWMachineScheduler.h - VLIW-Focused Scheduling Pass ----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//                                                                            //
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_VLIWMACHINESCHEDULER_H
+#define LLVM_CODEGEN_VLIWMACHINESCHEDULER_H
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/Twine.h"
+#include "llvm/CodeGen/MachineScheduler.h"
+#include "llvm/CodeGen/TargetSchedule.h"
+#include <limits>
+#include <memory>
+#include <utility>
+
+namespace llvm {
+
+class DFAPacketizer;
+class RegisterClassInfo;
+class ScheduleHazardRecognizer;
+class SUnit;
+class TargetInstrInfo;
+class TargetSubtargetInfo;
+
+class VLIWResourceModel {
+protected:
+  const TargetInstrInfo *TII;
+
+  /// ResourcesModel - Represents VLIW state.
+  /// Not limited to VLIW targets per se, but assumes definition of resource
+  /// model by a target.
+  DFAPacketizer *ResourcesModel;
+
+  const TargetSchedModel *SchedModel;
+
+  /// Local packet/bundle model. Purely
+  /// internal to the MI scheduler at the time.
+  SmallVector<SUnit *> Packet;
+
+  /// Total packets created.
+  unsigned TotalPackets = 0;
+
+public:
+  VLIWResourceModel(const TargetSubtargetInfo &STI, const TargetSchedModel *SM);
+
+  virtual ~VLIWResourceModel();
+
+  virtual void reset();
+
+  virtual bool hasDependence(const SUnit *SUd, const SUnit *SUu);
+  virtual bool isResourceAvailable(SUnit *SU, bool IsTop);
+  virtual bool reserveResources(SUnit *SU, bool IsTop);
+  unsigned getTotalPackets() const { return TotalPackets; }
+  size_t getPacketInstCount() const { return Packet.size(); }
+  bool isInPacket(SUnit *SU) const { return is_contained(Packet, SU); }
+
+protected:
+  virtual DFAPacketizer *createPacketizer(const TargetSubtargetInfo &STI) const;
+};
+
+/// Extend the standard ScheduleDAGMILive to provide more context and override
+/// the top-level schedule() driver.
+class VLIWMachineScheduler : public ScheduleDAGMILive {
+public:
+  VLIWMachineScheduler(MachineSchedContext *C,
+                       std::unique_ptr<MachineSchedStrategy> S)
+      : ScheduleDAGMILive(C, std::move(S)) {}
+
+  /// Schedule - This is called back from ScheduleDAGInstrs::Run() when it's
+  /// time to do some work.
+  void schedule() override;
+
+  RegisterClassInfo *getRegClassInfo() { return RegClassInfo; }
+  int getBBSize() { return BB->size(); }
+};
+
+//===----------------------------------------------------------------------===//
+// ConvergingVLIWScheduler - Implementation of a VLIW-aware
+// MachineSchedStrategy.
+//===----------------------------------------------------------------------===//
+
+class ConvergingVLIWScheduler : public MachineSchedStrategy {
+protected:
+  /// Store the state used by ConvergingVLIWScheduler heuristics, required
+  ///  for the lifetime of one invocation of pickNode().
+  struct SchedCandidate {
+    // The best SUnit candidate.
+    SUnit *SU = nullptr;
+
+    // Register pressure values for the best candidate.
+    RegPressureDelta RPDelta;
+
+    // Best scheduling cost.
+    int SCost = 0;
+
+    SchedCandidate() = default;
+  };
+  /// Represent the type of SchedCandidate found within a single queue.
+  enum CandResult {
+    NoCand,
+    NodeOrder,
+    SingleExcess,
+    SingleCritical,
+    SingleMax,
+    MultiPressure,
+    BestCost,
+    Weak
+  };
+
+  // Constants used to denote relative importance of
+  // heuristic components for cost computation.
+  static constexpr unsigned PriorityOne = 200;
+  static constexpr unsigned PriorityTwo = 50;
+  static constexpr unsigned PriorityThree = 75;
+  static constexpr unsigned ScaleTwo = 10;
+
+  /// Each Scheduling boundary is associated with ready queues. It tracks the
+  /// current cycle in whichever direction at has moved, and maintains the state
+  /// of "hazards" and other interlocks at the current cycle.
+  struct VLIWSchedBoundary {
+    VLIWMachineScheduler *DAG = nullptr;
+    const TargetSchedModel *SchedModel = nullptr;
+
+    ReadyQueue Available;
+    ReadyQueue Pending;
+    bool CheckPending = false;
+
+    ScheduleHazardRecognizer *HazardRec = nullptr;
+    VLIWResourceModel *ResourceModel = nullptr;
+
+    unsigned CurrCycle = 0;
+    unsigned IssueCount = 0;
+    unsigned CriticalPathLength = 0;
+
+    /// MinReadyCycle - Cycle of the soonest available instruction.
+    unsigned MinReadyCycle = std::numeric_limits<unsigned>::max();
+
+    // Remember the greatest min operand latency.
+    unsigned MaxMinLatency = 0;
+
+    /// Pending queues extend the ready queues with the same ID and the
+    /// PendingFlag set.
+    VLIWSchedBoundary(unsigned ID, const Twine &Name)
+        : Available(ID, Name + ".A"),
+          Pending(ID << ConvergingVLIWScheduler::LogMaxQID, Name + ".P") {}
+
+    ~VLIWSchedBoundary();
+
+    void init(VLIWMachineScheduler *dag, const TargetSchedModel *smodel) {
+      DAG = dag;
+      SchedModel = smodel;
+      CurrCycle = 0;
+      IssueCount = 0;
+      // Initialize the critical path length limit, which used by the scheduling
+      // cost model to determine the value for scheduling an instruction. We use
+      // a slightly different heuristic for small and large functions. For small
+      // functions, it's important to use the height/depth of the instruction.
+      // For large functions, prioritizing by height or depth increases spills.
+      CriticalPathLength = DAG->getBBSize() / SchedModel->getIssueWidth();
+      if (DAG->getBBSize() < 50)
+        // We divide by two as a cheap and simple heuristic to reduce the
+        // critcal path length, which increases the priority of using the graph
+        // height/depth in the scheduler's cost computation.
+        CriticalPathLength >>= 1;
+      else {
+        // For large basic blocks, we prefer a larger critical path length to
+        // decrease the priority of using the graph height/depth.
+        unsigned MaxPath = 0;
+        for (auto &SU : DAG->SUnits)
+          MaxPath = std::max(MaxPath, isTop() ? SU.getHeight() : SU.getDepth());
+        CriticalPathLength = std::max(CriticalPathLength, MaxPath) + 1;
+      }
+    }
+
+    bool isTop() const {
+      return Available.getID() == ConvergingVLIWScheduler::TopQID;
+    }
+
+    bool checkHazard(SUnit *SU);
+
+    void releaseNode(SUnit *SU, unsigned ReadyCycle);
+
+    void bumpCycle();
+
+    void bumpNode(SUnit *SU);
+
+    void releasePending();
+
+    void removeReady(SUnit *SU);
+
+    SUnit *pickOnlyChoice();
+
+    bool isLatencyBound(SUnit *SU) {
+      if (CurrCycle >= CriticalPathLength)
+        return true;
+      unsigned PathLength = isTop() ? SU->getHeight() : SU->getDepth();
+      return CriticalPathLength - CurrCycle <= PathLength;
+    }
+  };
+
+  VLIWMachineScheduler *DAG = nullptr;
+  const TargetSchedModel *SchedModel = nullptr;
+
+  // State of the top and bottom scheduled instruction boundaries.
+  VLIWSchedBoundary Top;
+  VLIWSchedBoundary Bot;
+
+  /// List of pressure sets that have a high pressure level in the region.
+  SmallVector<bool> HighPressureSets;
+
+public:
+  /// SUnit::NodeQueueId: 0 (none), 1 (top), 2 (bot), 3 (both)
+  enum { TopQID = 1, BotQID = 2, LogMaxQID = 2 };
+
+  ConvergingVLIWScheduler() : Top(TopQID, "TopQ"), Bot(BotQID, "BotQ") {}
+  virtual ~ConvergingVLIWScheduler() = default;
+
+  void initialize(ScheduleDAGMI *dag) override;
+
+  SUnit *pickNode(bool &IsTopNode) override;
+
+  void schedNode(SUnit *SU, bool IsTopNode) override;
+
+  void releaseTopNode(SUnit *SU) override;
+
+  void releaseBottomNode(SUnit *SU) override;
+
+  unsigned reportPackets() {
+    return Top.ResourceModel->getTotalPackets() +
+           Bot.ResourceModel->getTotalPackets();
+  }
+
+protected:
+  virtual VLIWResourceModel *
+  createVLIWResourceModel(const TargetSubtargetInfo &STI,
+                          const TargetSchedModel *SchedModel) const;
+
+  SUnit *pickNodeBidrectional(bool &IsTopNode);
+
+  int pressureChange(const SUnit *SU, bool isBotUp);
+
+  virtual int SchedulingCost(ReadyQueue &Q, SUnit *SU,
+                             SchedCandidate &Candidate, RegPressureDelta &Delta,
+                             bool verbose);
+
+  CandResult pickNodeFromQueue(VLIWSchedBoundary &Zone,
+                               const RegPressureTracker &RPTracker,
+                               SchedCandidate &Candidate);
+#ifndef NDEBUG
+  void traceCandidate(const char *Label, const ReadyQueue &Q, SUnit *SU,
+                      int Cost, PressureChange P = PressureChange());
+
+  void readyQueueVerboseDump(const RegPressureTracker &RPTracker,
+                             SchedCandidate &Candidate, ReadyQueue &Q);
+#endif
+};
+
+ScheduleDAGMILive *createVLIWSched(MachineSchedContext *C);
+
+} // end namespace llvm
+
+#endif // LLVM_CODEGEN_VLIWMACHINESCHEDULER_H
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h
index ae1afeb668be..24714ac3d101 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h
@@ -189,6 +189,11 @@ public:
                                DWOUnits.begin() + DWOUnits.getNumInfoUnits());
   }
 
+  const DWARFUnitVector &getDWOUnitsVector() {
+    parseDWOUnits();
+    return DWOUnits;
+  }
+
   /// Get units from .debug_types.dwo in the DWO context.
   unit_iterator_range dwo_types_section_units() {
     parseDWOUnits();
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h
index dbc11c51a789..1794f6649827 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h
@@ -15,6 +15,7 @@
 #include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h"
 #include "llvm/DebugInfo/DWARF/DWARFLocationExpression.h"
 #include "llvm/DebugInfo/DWARF/DWARFRelocMap.h"
+#include "llvm/Support/Errc.h"
 #include <cstdint>
 
 namespace llvm {
@@ -142,6 +143,22 @@ private:
   uint16_t Version;
 };
 
+class ResolverError : public ErrorInfo<ResolverError> {
+public:
+  static char ID;
+
+  ResolverError(uint32_t Index, dwarf::LoclistEntries Kind) : Index(Index), Kind(Kind) {}
+
+  void log(raw_ostream &OS) const override;
+  std::error_code convertToErrorCode() const override {
+    return llvm::errc::invalid_argument;
+  }
+
+private:
+  uint32_t Index;
+  dwarf::LoclistEntries Kind;
+};
+
 } // end namespace llvm
 
 #endif // LLVM_DEBUGINFO_DWARF_DWARFDEBUGLOC_H
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFFormValue.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFFormValue.h
index 3c051c3ea018..130cdb8800a9 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFFormValue.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFFormValue.h
@@ -112,7 +112,7 @@ public:
   Optional<UnitOffset> getAsRelativeReference() const;
   Optional<uint64_t> getAsUnsignedConstant() const;
   Optional<int64_t> getAsSignedConstant() const;
-  Optional<const char *> getAsCString() const;
+  Expected<const char *> getAsCString() const;
   Optional<uint64_t> getAsAddress() const;
   Optional<object::SectionedAddress> getAsSectionedAddress() const;
   Optional<uint64_t> getAsSectionOffset() const;
@@ -173,9 +173,14 @@ namespace dwarf {
 /// \returns an optional value that contains a value if the form value
 /// was valid and was a string.
 inline Optional<const char *> toString(const Optional<DWARFFormValue> &V) {
-  if (V)
-    return V->getAsCString();
-  return None;
+  if (!V)
+    return None;
+  Expected<const char*> E = V->getAsCString();
+  if (!E) {
+    consumeError(E.takeError());
+    return None;
+  }
+  return *E;
 }
 
 /// Take an optional DWARFFormValue and try to extract a string value from it.
@@ -185,10 +190,16 @@ inline Optional<const char *> toString(const Optional<DWARFFormValue> &V) {
 /// was valid and was a string.
 inline StringRef toStringRef(const Optional<DWARFFormValue> &V,
                              StringRef Default = {}) {
-  if (V)
-    if (auto S = V->getAsCString())
-      return *S;
-  return Default;
+  if (!V)
+    return Default;
+  auto S = V->getAsCString();
+  if (!S) {
+    consumeError(S.takeError());
+    return Default;
+  }
+  if (!*S)
+    return Default;
+  return *S;
 }
 
 /// Take an optional DWARFFormValue and extract a string value from it.
@@ -199,7 +210,9 @@ inline StringRef toStringRef(const Optional<DWARFFormValue> &V,
 /// form value's encoding wasn't a string.
 inline const char *toString(const Optional<DWARFFormValue> &V,
                             const char *Default) {
-  return toString(V).getValueOr(Default);
+  if (auto E = toString(V))
+    return *E;
+  return Default;
 }
 
 /// Take an optional DWARFFormValue and try to extract an unsigned constant.
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h
index 1292bfbc0591..b96a4c19758f 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h
@@ -331,7 +331,7 @@ public:
 
   Optional<object::SectionedAddress>
   getAddrOffsetSectionItem(uint32_t Index) const;
-  Optional<uint64_t> getStringOffsetSectionItem(uint32_t Index) const;
+  Expected<uint64_t> getStringOffsetSectionItem(uint32_t Index) const;
 
   DWARFDataExtractor getDebugInfoExtractor() const;
 
diff --git a/llvm/include/llvm/DebugInfo/MSF/MSFCommon.h b/llvm/include/llvm/DebugInfo/MSF/MSFCommon.h
index a922839a999d..0520b94ea3dd 100644
--- a/llvm/include/llvm/DebugInfo/MSF/MSFCommon.h
+++ b/llvm/include/llvm/DebugInfo/MSF/MSFCommon.h
@@ -101,6 +101,26 @@ inline bool isValidBlockSize(uint32_t Size) {
   return false;
 }
 
+/// Given the specified block size, returns the maximum possible file size.
+/// Block Size  |  Max File Size
+/// <= 4096     |      4GB
+///    8192     |      8GB
+///   16384     |      16GB
+///   32768     |      32GB
+/// \p Size - the block size of the MSF
+inline uint64_t getMaxFileSizeFromBlockSize(uint32_t Size) {
+  switch (Size) {
+  case 8192:
+    return (uint64_t)UINT32_MAX * 2ULL;
+  case 16384:
+    return (uint64_t)UINT32_MAX * 3ULL;
+  case 32768:
+    return (uint64_t)UINT32_MAX * 4ULL;
+  default:
+    return (uint64_t)UINT32_MAX;
+  }
+}
+
 // Super Block, Fpm0, Fpm1, and Block Map
 inline uint32_t getMinimumBlockCount() { return 4; }
 
diff --git a/llvm/include/llvm/DebugInfo/MSF/MSFError.h b/llvm/include/llvm/DebugInfo/MSF/MSFError.h
index 0ef30f10bc68..b84f9d7c4fee 100644
--- a/llvm/include/llvm/DebugInfo/MSF/MSFError.h
+++ b/llvm/include/llvm/DebugInfo/MSF/MSFError.h
@@ -16,7 +16,10 @@ namespace msf {
 enum class msf_error_code {
   unspecified = 1,
   insufficient_buffer,
-  size_overflow,
+  size_overflow_4096,
+  size_overflow_8192,
+  size_overflow_16384,
+  size_overflow_32768,
   not_writable,
   no_stream,
   invalid_format,
diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/PDBFile.h b/llvm/include/llvm/DebugInfo/PDB/Native/PDBFile.h
index 2124e6a46ed5..c5ee73280c46 100644
--- a/llvm/include/llvm/DebugInfo/PDB/Native/PDBFile.h
+++ b/llvm/include/llvm/DebugInfo/PDB/Native/PDBFile.h
@@ -65,7 +65,7 @@ public:
   uint32_t getStreamByteSize(uint32_t StreamIndex) const override;
   ArrayRef<support::ulittle32_t>
   getStreamBlockList(uint32_t StreamIndex) const override;
-  uint32_t getFileSize() const;
+  uint64_t getFileSize() const;
 
   Expected<ArrayRef<uint8_t>> getBlockData(uint32_t BlockIndex,
                                            uint32_t NumBytes) const override;
diff --git a/llvm/include/llvm/Debuginfod/Debuginfod.h b/llvm/include/llvm/Debuginfod/Debuginfod.h
new file mode 100644
index 000000000000..fcb8ed3a9222
--- /dev/null
+++ b/llvm/include/llvm/Debuginfod/Debuginfod.h
@@ -0,0 +1,71 @@
+//===-- llvm/Debuginfod/Debuginfod.h - Debuginfod client --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the declarations of getCachedOrDownloadArtifact and
+/// several convenience functions for specific artifact types:
+/// getCachedOrDownloadSource, getCachedOrDownloadExecutable, and
+/// getCachedOrDownloadDebuginfo. This file also declares
+/// getDefaultDebuginfodUrls and getDefaultDebuginfodCacheDirectory.
+///
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFOD_DEBUGINFOD_H
+#define LLVM_DEBUGINFOD_DEBUGINFOD_H
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/MemoryBuffer.h"
+
+namespace llvm {
+
+typedef ArrayRef<uint8_t> BuildIDRef;
+
+typedef SmallVector<uint8_t, 10> BuildID;
+
+/// Finds default array of Debuginfod server URLs by checking DEBUGINFOD_URLS
+/// environment variable.
+Expected<SmallVector<StringRef>> getDefaultDebuginfodUrls();
+
+/// Finds a default local file caching directory for the debuginfod client,
+/// first checking DEBUGINFOD_CACHE_PATH.
+Expected<std::string> getDefaultDebuginfodCacheDirectory();
+
+/// Finds a default timeout for debuginfod HTTP requests. Checks
+/// DEBUGINFOD_TIMEOUT environment variable, default is 90 seconds (90000 ms).
+std::chrono::milliseconds getDefaultDebuginfodTimeout();
+
+/// Fetches a specified source file by searching the default local cache
+/// directory and server URLs.
+Expected<std::string> getCachedOrDownloadSource(BuildIDRef ID,
+                                                StringRef SourceFilePath);
+
+/// Fetches an executable by searching the default local cache directory and
+/// server URLs.
+Expected<std::string> getCachedOrDownloadExecutable(BuildIDRef ID);
+
+/// Fetches a debug binary by searching the default local cache directory and
+/// server URLs.
+Expected<std::string> getCachedOrDownloadDebuginfo(BuildIDRef ID);
+
+/// Fetches any debuginfod artifact using the default local cache directory and
+/// server URLs.
+Expected<std::string> getCachedOrDownloadArtifact(StringRef UniqueKey,
+                                                  StringRef UrlPath);
+
+/// Fetches any debuginfod artifact using the specified local cache directory,
+/// server URLs, and request timeout (in milliseconds). If the artifact is
+/// found, uses the UniqueKey for the local cache file.
+Expected<std::string> getCachedOrDownloadArtifact(
+    StringRef UniqueKey, StringRef UrlPath, StringRef CacheDirectoryPath,
+    ArrayRef<StringRef> DebuginfodUrls, std::chrono::milliseconds Timeout);
+
+} // end namespace llvm
+
+#endif
diff --git a/llvm/include/llvm/Support/HTTPClient.h b/llvm/include/llvm/Debuginfod/HTTPClient.h
index 3172610c2d8b..51de66629544 100644
--- a/llvm/include/llvm/Support/HTTPClient.h
+++ b/llvm/include/llvm/Debuginfod/HTTPClient.h
@@ -77,10 +77,16 @@ public:
 
 /// A reusable client that can perform HTTPRequests through a network socket.
 class HTTPClient {
+#ifdef LLVM_ENABLE_CURL
+  void *Curl = nullptr;
+#endif
+
 public:
   HTTPClient();
   ~HTTPClient();
 
+  static bool IsInitialized;
+
   /// Returns true only if LLVM has been compiled with a working HTTPClient.
   static bool isAvailable();
 
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Core.h b/llvm/include/llvm/ExecutionEngine/Orc/Core.h
index 2180be3341e1..b5f5636800df 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/Core.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/Core.h
@@ -670,14 +670,22 @@ class MaterializationUnit {
 public:
   static char ID;
 
-  MaterializationUnit(SymbolFlagsMap InitalSymbolFlags,
-                      SymbolStringPtr InitSymbol)
-      : SymbolFlags(std::move(InitalSymbolFlags)),
-        InitSymbol(std::move(InitSymbol)) {
-    assert((!this->InitSymbol || this->SymbolFlags.count(this->InitSymbol)) &&
-           "If set, InitSymbol should appear in InitialSymbolFlags map");
-  }
+  struct Interface {
+    Interface() = default;
+    Interface(SymbolFlagsMap InitalSymbolFlags, SymbolStringPtr InitSymbol)
+        : SymbolFlags(std::move(InitalSymbolFlags)),
+          InitSymbol(std::move(InitSymbol)) {
+      assert((!this->InitSymbol || this->SymbolFlags.count(this->InitSymbol)) &&
+             "If set, InitSymbol should appear in InitialSymbolFlags map");
+    }
+
+    SymbolFlagsMap SymbolFlags;
+    SymbolStringPtr InitSymbol;
+  };
 
+  MaterializationUnit(Interface I)
+      : SymbolFlags(std::move(I.SymbolFlags)),
+        InitSymbol(std::move(I.InitSymbol)) {}
   virtual ~MaterializationUnit() {}
 
   /// Return the name of this materialization unit. Useful for debugging
@@ -730,7 +738,7 @@ public:
 private:
   void materialize(std::unique_ptr<MaterializationResponsibility> R) override;
   void discard(const JITDylib &JD, const SymbolStringPtr &Name) override;
-  static SymbolFlagsMap extractFlags(const SymbolMap &Symbols);
+  static MaterializationUnit::Interface extractFlags(const SymbolMap &Symbols);
 
   SymbolMap Symbols;
 };
@@ -772,7 +780,8 @@ public:
 private:
   void materialize(std::unique_ptr<MaterializationResponsibility> R) override;
   void discard(const JITDylib &JD, const SymbolStringPtr &Name) override;
-  static SymbolFlagsMap extractFlags(const SymbolAliasMap &Aliases);
+  static MaterializationUnit::Interface
+  extractFlags(const SymbolAliasMap &Aliases);
 
   JITDylib *SourceJD = nullptr;
   JITDylibLookupFlags SourceJDLookupFlags;
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h
index 1946aed9733e..8e572ea1d0c1 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h
@@ -259,12 +259,18 @@ private:
 /// the containing object being added to the JITDylib.
 class StaticLibraryDefinitionGenerator : public DefinitionGenerator {
 public:
+  // Interface builder function for objects loaded from this archive.
+  using GetObjectFileInterface =
+      unique_function<Expected<MaterializationUnit::Interface>(
+          ExecutionSession &ES, MemoryBufferRef ObjBuffer)>;
+
   /// Try to create a StaticLibraryDefinitionGenerator from the given path.
   ///
   /// This call will succeed if the file at the given path is a static library
   /// is a valid archive, otherwise it will return an error.
   static Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>>
-  Load(ObjectLayer &L, const char *FileName);
+  Load(ObjectLayer &L, const char *FileName,
+       GetObjectFileInterface GetObjFileInterface = GetObjectFileInterface());
 
   /// Try to create a StaticLibraryDefinitionGenerator from the given path.
   ///
@@ -272,13 +278,15 @@ public:
   /// or a MachO universal binary containing a static library that is compatible
   /// with the given triple. Otherwise it will return an error.
   static Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>>
-  Load(ObjectLayer &L, const char *FileName, const Triple &TT);
+  Load(ObjectLayer &L, const char *FileName, const Triple &TT,
+       GetObjectFileInterface GetObjFileInterface = GetObjectFileInterface());
 
   /// Try to create a StaticLibrarySearchGenerator from the given memory buffer.
   /// This call will succeed if the buffer contains a valid archive, otherwise
   /// it will return an error.
   static Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>>
-  Create(ObjectLayer &L, std::unique_ptr<MemoryBuffer> ArchiveBuffer);
+  Create(ObjectLayer &L, std::unique_ptr<MemoryBuffer> ArchiveBuffer,
+         GetObjectFileInterface GetObjFileInterface = GetObjectFileInterface());
 
   Error tryToGenerate(LookupState &LS, LookupKind K, JITDylib &JD,
                       JITDylibLookupFlags JDLookupFlags,
@@ -287,9 +295,11 @@ public:
 private:
   StaticLibraryDefinitionGenerator(ObjectLayer &L,
                                    std::unique_ptr<MemoryBuffer> ArchiveBuffer,
+                                   GetObjectFileInterface GetObjFileInterface,
                                    Error &Err);
 
   ObjectLayer &L;
+  GetObjectFileInterface GetObjFileInterface;
   std::unique_ptr<MemoryBuffer> ArchiveBuffer;
   std::unique_ptr<object::Archive> Archive;
 };
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Layer.h b/llvm/include/llvm/ExecutionEngine/Orc/Layer.h
index dccbb4be9b52..cfeedc2a0bda 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/Layer.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/Layer.h
@@ -43,8 +43,7 @@ public:
   /// entries for each definition in M.
   /// This constructor is useful for delegating work from one
   /// IRMaterializationUnit to another.
-  IRMaterializationUnit(ThreadSafeModule TSM, SymbolFlagsMap SymbolFlags,
-                        SymbolStringPtr InitSymbol,
+  IRMaterializationUnit(ThreadSafeModule TSM, Interface I,
                         SymbolNameToDefinitionMap SymbolToDefinition);
 
   /// Return the ModuleIdentifier as the name for this MaterializationUnit.
@@ -141,14 +140,28 @@ public:
   /// Returns the execution session for this layer.
   ExecutionSession &getExecutionSession() { return ES; }
 
-  /// Adds a MaterializationUnit representing the given IR to the given
-  /// JITDylib.
-  virtual Error add(ResourceTrackerSP RT, std::unique_ptr<MemoryBuffer> O);
-
-  Error add(JITDylib &JD, std::unique_ptr<MemoryBuffer> O) {
-    return add(JD.getDefaultResourceTracker(), std::move(O));
+  /// Adds a MaterializationUnit for the object file in the given memory buffer
+  /// to the JITDylib for the given ResourceTracker.
+  virtual Error add(ResourceTrackerSP RT, std::unique_ptr<MemoryBuffer> O,
+                    MaterializationUnit::Interface I);
+
+  /// Adds a MaterializationUnit for the object file in the given memory buffer
+  /// to the JITDylib for the given ResourceTracker. The interface for the
+  /// object will be built using the default object interface builder.
+  Error add(ResourceTrackerSP RT, std::unique_ptr<MemoryBuffer> O);
+
+  /// Adds a MaterializationUnit for the object file in the given memory buffer
+  /// to the given JITDylib.
+  Error add(JITDylib &JD, std::unique_ptr<MemoryBuffer> O,
+            MaterializationUnit::Interface I) {
+    return add(JD.getDefaultResourceTracker(), std::move(O), std::move(I));
   }
 
+  /// Adds a MaterializationUnit for the object file in the given memory buffer
+  /// to the given JITDylib. The interface for the object will be built using
+  /// the default object interface builder.
+  Error add(JITDylib &JD, std::unique_ptr<MemoryBuffer> O);
+
   /// Emit should materialize the given IR.
   virtual void emit(std::unique_ptr<MaterializationResponsibility> R,
                     std::unique_ptr<MemoryBuffer> O) = 0;
@@ -161,13 +174,13 @@ private:
 /// instance) by calling 'emit' on the given ObjectLayer.
 class BasicObjectLayerMaterializationUnit : public MaterializationUnit {
 public:
+  /// Create using the default object interface builder function.
   static Expected<std::unique_ptr<BasicObjectLayerMaterializationUnit>>
   Create(ObjectLayer &L, std::unique_ptr<MemoryBuffer> O);
 
   BasicObjectLayerMaterializationUnit(ObjectLayer &L,
                                       std::unique_ptr<MemoryBuffer> O,
-                                      SymbolFlagsMap SymbolFlags,
-                                      SymbolStringPtr InitSymbol);
+                                      Interface I);
 
   /// Return the buffer's identifier as the name for this MaterializationUnit.
   StringRef getName() const override;
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/LazyReexports.h b/llvm/include/llvm/ExecutionEngine/Orc/LazyReexports.h
index e6a9d8945285..f81cdcef6655 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/LazyReexports.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/LazyReexports.h
@@ -151,7 +151,8 @@ public:
 private:
   void materialize(std::unique_ptr<MaterializationResponsibility> R) override;
   void discard(const JITDylib &JD, const SymbolStringPtr &Name) override;
-  static SymbolFlagsMap extractFlags(const SymbolAliasMap &Aliases);
+  static MaterializationUnit::Interface
+  extractFlags(const SymbolAliasMap &Aliases);
 
   LazyCallThroughManager &LCTManager;
   IndirectStubsManager &ISManager;
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Mangling.h b/llvm/include/llvm/ExecutionEngine/Orc/Mangling.h
index e0f770a601fb..77429f4b11ee 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/Mangling.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/Mangling.h
@@ -55,11 +55,6 @@ public:
                   SymbolNameToDefinitionMap *SymbolToDefinition = nullptr);
 };
 
-/// Returns a SymbolFlagsMap for the object file represented by the given
-/// buffer, or an error if the buffer does not contain a valid object file.
-Expected<std::pair<SymbolFlagsMap, SymbolStringPtr>>
-getObjectSymbolInfo(ExecutionSession &ES, MemoryBufferRef ObjBuffer);
-
 } // End namespace orc
 } // End namespace llvm
 
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/ObjectFileInterface.h b/llvm/include/llvm/ExecutionEngine/Orc/ObjectFileInterface.h
new file mode 100644
index 000000000000..1bf09069163e
--- /dev/null
+++ b/llvm/include/llvm/ExecutionEngine/Orc/ObjectFileInterface.h
@@ -0,0 +1,38 @@
+//===-- ObjectFileInterface.h - MU interface utils for objects --*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Utilities for building MaterializationUnit::Interface objects from
+// object files.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_EXECUTIONENGINE_ORC_OBJECTFILEINTERFACE_H
+#define LLVM_EXECUTIONENGINE_ORC_OBJECTFILEINTERFACE_H
+
+#include "llvm/ExecutionEngine/Orc/Core.h"
+#include "llvm/Support/MemoryBuffer.h"
+
+namespace llvm {
+namespace orc {
+
+/// Adds an initializer symbol to the given MU interface.
+/// The init symbol's name is guaranteed to be unique within I, and will be of
+/// the form $.<ObjFileName>.__inits.<N>, where N is some integer.
+void addInitSymbol(MaterializationUnit::Interface &I, ExecutionSession &ES,
+                   StringRef ObjFileName);
+
+/// Returns a MaterializationUnit::Interface for the object file contained in
+/// the given buffer, or an error if the buffer does not contain a valid object
+/// file.
+Expected<MaterializationUnit::Interface>
+getObjectFileInterface(ExecutionSession &ES, MemoryBufferRef ObjBuffer);
+
+} // End namespace orc
+} // End namespace llvm
+
+#endif // LLVM_EXECUTIONENGINE_ORC_OBJECTFILEINTERFACE_H
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index 5ee379b7fcad..18d577dff497 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -180,6 +180,7 @@ def OMPC_Read : Clause<"read"> { let clangClass = "OMPReadClause"; }
 def OMPC_Write : Clause<"write"> { let clangClass = "OMPWriteClause"; }
 def OMPC_Update : Clause<"update"> { let clangClass = "OMPUpdateClause"; }
 def OMPC_Capture : Clause<"capture"> { let clangClass = "OMPCaptureClause"; }
+def OMPC_Compare : Clause<"compare"> { let clangClass = "OMPCompareClause"; }
 def OMPC_SeqCst : Clause<"seq_cst"> { let clangClass = "OMPSeqCstClause"; }
 def OMPC_AcqRel : Clause<"acq_rel"> { let clangClass = "OMPAcqRelClause"; }
 def OMPC_Acquire : Clause<"acquire"> { let clangClass = "OMPAcquireClause"; }
@@ -282,7 +283,7 @@ def OMPC_Allocate : Clause<"allocate"> {
 def OMPC_NonTemporal : Clause<"nontemporal"> {
   let clangClass = "OMPNontemporalClause";
   let flangClass = "Name";
-  let isValueList = true; 
+  let isValueList = true;
 }
 
 def OMP_ORDER_concurrent : ClauseVal<"concurrent",1,1> {}
@@ -536,6 +537,7 @@ def OMP_Atomic : Directive<"atomic"> {
     VersionedClause<OMPC_Write>,
     VersionedClause<OMPC_Update>,
     VersionedClause<OMPC_Capture>,
+    VersionedClause<OMPC_Compare, 51>
   ];
   let allowedOnceClauses = [
     VersionedClause<OMPC_SeqCst>,
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
index 563e0eed1762..9976d1961ed1 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -539,24 +539,27 @@ public:
       function_ref<InsertPointTy(InsertPointTy, Value *, Value *, Value *&)>;
 
   /// Functions used to generate atomic reductions. Such functions take two
-  /// Values representing pointers to LHS and RHS of the reduction. They are
-  /// expected to atomically update the LHS to the reduced value.
+  /// Values representing pointers to LHS and RHS of the reduction, as well as
+  /// the element type of these pointers. They are expected to atomically
+  /// update the LHS to the reduced value.
   using AtomicReductionGenTy =
-      function_ref<InsertPointTy(InsertPointTy, Value *, Value *)>;
+      function_ref<InsertPointTy(InsertPointTy, Type *, Value *, Value *)>;
 
   /// Information about an OpenMP reduction.
   struct ReductionInfo {
-    ReductionInfo(Value *Variable, Value *PrivateVariable,
+    ReductionInfo(Type *ElementType, Value *Variable, Value *PrivateVariable,
                   ReductionGenTy ReductionGen,
                   AtomicReductionGenTy AtomicReductionGen)
-        : Variable(Variable), PrivateVariable(PrivateVariable),
-          ReductionGen(ReductionGen), AtomicReductionGen(AtomicReductionGen) {}
-
-    /// Returns the type of the element being reduced.
-    Type *getElementType() const {
-      return Variable->getType()->getPointerElementType();
+        : ElementType(ElementType), Variable(Variable),
+          PrivateVariable(PrivateVariable), ReductionGen(ReductionGen),
+          AtomicReductionGen(AtomicReductionGen) {
+      assert(cast<PointerType>(Variable->getType())
+          ->isOpaqueOrPointeeTypeMatches(ElementType) && "Invalid elem type");
     }
 
+    /// Reduction element type, must match pointee type of variable.
+    Type *ElementType;
+
     /// Reduction variable of pointer type.
     Value *Variable;
 
@@ -1166,9 +1169,9 @@ private:
   /// \param UpdateOp 	Code generator for complex expressions that cannot be
   ///                   expressed through atomicrmw instruction.
   /// \param VolatileX	     true if \a X volatile?
-  /// \param IsXLHSInRHSPart true if \a X is Left H.S. in Right H.S. part of
-  ///                        the update expression, false otherwise.
-  ///                        (e.g. true for X = X BinOp Expr)
+  /// \param IsXBinopExpr true if \a X is Left H.S. in Right H.S. part of the
+  ///                     update expression, false otherwise.
+  ///                     (e.g. true for X = X BinOp Expr)
   ///
   /// \returns A pair of the old value of X before the update, and the value
   ///          used for the update.
@@ -1177,7 +1180,7 @@ private:
                                                AtomicRMWInst::BinOp RMWOp,
                                                AtomicUpdateCallbackTy &UpdateOp,
                                                bool VolatileX,
-                                               bool IsXLHSInRHSPart);
+                                               bool IsXBinopExpr);
 
   /// Emit the binary op. described by \p RMWOp, using \p Src1 and \p Src2 .
   ///
@@ -1235,9 +1238,9 @@ public:
   ///                 atomic will be generated.
   /// \param UpdateOp 	Code generator for complex expressions that cannot be
   ///                   expressed through atomicrmw instruction.
-  /// \param IsXLHSInRHSPart true if \a X is Left H.S. in Right H.S. part of
-  ///                        the update expression, false otherwise.
-  ///	                       (e.g. true for X = X BinOp Expr)
+  /// \param IsXBinopExpr true if \a X is Left H.S. in Right H.S. part of the
+  ///                     update expression, false otherwise.
+  ///	                    (e.g. true for X = X BinOp Expr)
   ///
   /// \return Insertion point after generated atomic update IR.
   InsertPointTy createAtomicUpdate(const LocationDescription &Loc,
@@ -1245,7 +1248,7 @@ public:
                                    Value *Expr, AtomicOrdering AO,
                                    AtomicRMWInst::BinOp RMWOp,
                                    AtomicUpdateCallbackTy &UpdateOp,
-                                   bool IsXLHSInRHSPart);
+                                   bool IsXBinopExpr);
 
   /// Emit atomic update for constructs: --- Only Scalar data types
   /// V = X; X = X BinOp Expr ,
@@ -1269,9 +1272,9 @@ public:
   ///                   expressed through atomicrmw instruction.
   /// \param UpdateExpr true if X is an in place update of the form
   ///                   X = X BinOp Expr or X = Expr BinOp X
-  /// \param IsXLHSInRHSPart true if X is Left H.S. in Right H.S. part of the
-  ///                        update expression, false otherwise.
-  ///                        (e.g. true for X = X BinOp Expr)
+  /// \param IsXBinopExpr true if X is Left H.S. in Right H.S. part of the
+  ///                     update expression, false otherwise.
+  ///                     (e.g. true for X = X BinOp Expr)
   /// \param IsPostfixUpdate true if original value of 'x' must be stored in
   ///                        'v', not an updated one.
   ///
@@ -1281,7 +1284,7 @@ public:
                       AtomicOpValue &X, AtomicOpValue &V, Value *Expr,
                       AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
                       AtomicUpdateCallbackTy &UpdateOp, bool UpdateExpr,
-                      bool IsPostfixUpdate, bool IsXLHSInRHSPart);
+                      bool IsPostfixUpdate, bool IsXBinopExpr);
 
   /// Create the control flow structure of a canonical OpenMP loop.
   ///
@@ -1408,13 +1411,10 @@ class CanonicalLoopInfo {
   friend class OpenMPIRBuilder;
 
 private:
-  BasicBlock *Preheader = nullptr;
   BasicBlock *Header = nullptr;
   BasicBlock *Cond = nullptr;
-  BasicBlock *Body = nullptr;
   BasicBlock *Latch = nullptr;
   BasicBlock *Exit = nullptr;
-  BasicBlock *After = nullptr;
 
   /// Add the control blocks of this loop to \p BBs.
   ///
@@ -1436,10 +1436,7 @@ public:
   /// Code that must be execute before any loop iteration can be emitted here,
   /// such as computing the loop trip count and begin lifetime markers. Code in
   /// the preheader is not considered part of the canonical loop.
-  BasicBlock *getPreheader() const {
-    assert(isValid() && "Requires a valid canonical loop");
-    return Preheader;
-  }
+  BasicBlock *getPreheader() const;
 
   /// The header is the entry for each iteration. In the canonical control flow,
   /// it only contains the PHINode for the induction variable.
@@ -1460,7 +1457,7 @@ public:
   /// eventually branch to the \p Latch block.
   BasicBlock *getBody() const {
     assert(isValid() && "Requires a valid canonical loop");
-    return Body;
+    return cast<BranchInst>(Cond->getTerminator())->getSuccessor(0);
   }
 
   /// Reaching the latch indicates the end of the loop body code. In the
@@ -1484,7 +1481,7 @@ public:
   /// statements/cancellations).
   BasicBlock *getAfter() const {
     assert(isValid() && "Requires a valid canonical loop");
-    return After;
+    return Exit->getSingleSuccessor();
   }
 
   /// Returns the llvm::Value containing the number of loop iterations. It must
@@ -1515,18 +1512,21 @@ public:
   /// Return the insertion point for user code before the loop.
   OpenMPIRBuilder::InsertPointTy getPreheaderIP() const {
     assert(isValid() && "Requires a valid canonical loop");
+    BasicBlock *Preheader = getPreheader();
     return {Preheader, std::prev(Preheader->end())};
   };
 
   /// Return the insertion point for user code in the body.
   OpenMPIRBuilder::InsertPointTy getBodyIP() const {
     assert(isValid() && "Requires a valid canonical loop");
+    BasicBlock *Body = getBody();
     return {Body, Body->begin()};
   };
 
   /// Return the insertion point for user code after the loop.
   OpenMPIRBuilder::InsertPointTy getAfterIP() const {
     assert(isValid() && "Requires a valid canonical loop");
+    BasicBlock *After = getAfter();
     return {After, After->begin()};
   };
 
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
index 8e4f7568fb9c..08bf5981cdc3 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
@@ -207,6 +207,7 @@ __OMP_RTL(__kmpc_omp_reg_task_with_affinity, false, Int32, IdentPtr, Int32,
 
 __OMP_RTL(__kmpc_get_hardware_num_blocks, false, Int32, )
 __OMP_RTL(__kmpc_get_hardware_num_threads_in_block, false, Int32, )
+__OMP_RTL(__kmpc_get_warp_size, false, Int32, )
 
 __OMP_RTL(omp_get_thread_num, false, Int32, )
 __OMP_RTL(omp_get_num_threads, false, Int32, )
@@ -455,8 +456,6 @@ __OMP_RTL(__kmpc_barrier_simple_generic, false, Void, IdentPtr, Int32)
 __OMP_RTL(__kmpc_warp_active_thread_mask, false, Int64,)
 __OMP_RTL(__kmpc_syncwarp, false, Void, Int64)
 
-__OMP_RTL(__kmpc_get_warp_size, false, Int32, )
-
 __OMP_RTL(__kmpc_is_generic_main_thread_id, false, Int8, Int32)
 
 __OMP_RTL(__last, false, Void, )
@@ -629,6 +628,7 @@ __OMP_RTL_ATTRS(__kmpc_omp_reg_task_with_affinity, DefaultAttrs, AttributeSet(),
 
 __OMP_RTL_ATTRS(__kmpc_get_hardware_num_blocks, GetterAttrs, AttributeSet(), ParamAttrs())
 __OMP_RTL_ATTRS(__kmpc_get_hardware_num_threads_in_block, GetterAttrs, AttributeSet(), ParamAttrs())
+__OMP_RTL_ATTRS(__kmpc_get_warp_size, GetterAttrs, AttributeSet(), ParamAttrs())
 
 __OMP_RTL_ATTRS(omp_get_thread_num, GetterAttrs, AttributeSet(), ParamAttrs())
 __OMP_RTL_ATTRS(omp_get_num_threads, GetterAttrs, AttributeSet(), ParamAttrs())
diff --git a/llvm/include/llvm/IR/Attributes.h b/llvm/include/llvm/IR/Attributes.h
index 282be640d8be..f64f15bd38ba 100644
--- a/llvm/include/llvm/IR/Attributes.h
+++ b/llvm/include/llvm/IR/Attributes.h
@@ -216,9 +216,12 @@ public:
   /// if not known).
   std::pair<unsigned, Optional<unsigned>> getAllocSizeArgs() const;
 
-  /// Returns the argument numbers for the vscale_range attribute (or pair(0, 0)
-  /// if not known).
-  std::pair<unsigned, unsigned> getVScaleRangeArgs() const;
+  /// Returns the minimum value for the vscale_range attribute.
+  unsigned getVScaleRangeMin() const;
+
+  /// Returns the maximum value for the vscale_range attribute or None when
+  /// unknown.
+  Optional<unsigned> getVScaleRangeMax() const;
 
   /// The Attribute is converted to a string of equivalent mnemonic. This
   /// is, presumably, for writing out the mnemonics for the assembly writer.
@@ -348,7 +351,8 @@ public:
   Type *getInAllocaType() const;
   Type *getElementType() const;
   std::pair<unsigned, Optional<unsigned>> getAllocSizeArgs() const;
-  std::pair<unsigned, unsigned> getVScaleRangeArgs() const;
+  unsigned getVScaleRangeMin() const;
+  Optional<unsigned> getVScaleRangeMax() const;
   std::string getAsString(bool InAttrGrp = false) const;
 
   /// Return true if this attribute set belongs to the LLVMContext.
@@ -452,6 +456,8 @@ public:
   static AttributeList get(LLVMContext &C, unsigned Index,
                            ArrayRef<StringRef> Kind);
   static AttributeList get(LLVMContext &C, unsigned Index,
+                           AttributeSet Attrs);
+  static AttributeList get(LLVMContext &C, unsigned Index,
                            const AttrBuilder &B);
 
   // TODO: remove non-AtIndex versions of these methods.
@@ -938,6 +944,8 @@ class AttrBuilder {
 
 public:
   AttrBuilder() = default;
+  AttrBuilder(const AttrBuilder &) = delete;
+  AttrBuilder(AttrBuilder &&) = default;
 
   AttrBuilder(const Attribute &A) {
     addAttribute(A);
@@ -1053,9 +1061,11 @@ public:
   /// doesn't exist, pair(0, 0) is returned.
   std::pair<unsigned, Optional<unsigned>> getAllocSizeArgs() const;
 
-  /// Retrieve the vscale_range args, if the vscale_range attribute exists.  If
-  /// it doesn't exist, pair(0, 0) is returned.
-  std::pair<unsigned, unsigned> getVScaleRangeArgs() const;
+  /// Retrieve the minimum value of 'vscale_range'.
+  unsigned getVScaleRangeMin() const;
+
+  /// Retrieve the maximum value of 'vscale_range' or None when unknown.
+  Optional<unsigned> getVScaleRangeMax() const;
 
   /// Add integer attribute with raw value (packed/encoded if necessary).
   AttrBuilder &addRawIntAttr(Attribute::AttrKind Kind, uint64_t Value);
@@ -1097,7 +1107,8 @@ public:
                                 const Optional<unsigned> &NumElemsArg);
 
   /// This turns two ints into the form used internally in Attribute.
-  AttrBuilder &addVScaleRangeAttr(unsigned MinValue, unsigned MaxValue);
+  AttrBuilder &addVScaleRangeAttr(unsigned MinValue,
+                                  Optional<unsigned> MaxValue);
 
   /// Add a type attribute with the given type.
   AttrBuilder &addTypeAttr(Attribute::AttrKind Kind, Type *Ty);
diff --git a/llvm/include/llvm/IR/Attributes.td b/llvm/include/llvm/IR/Attributes.td
index de25b51a6292..40c554c269ca 100644
--- a/llvm/include/llvm/IR/Attributes.td
+++ b/llvm/include/llvm/IR/Attributes.td
@@ -345,3 +345,6 @@ def : MergeRule<"adjustCallerStackProbeSize">;
 def : MergeRule<"adjustMinLegalVectorWidth">;
 def : MergeRule<"adjustNullPointerValidAttr">;
 def : MergeRule<"setAND<MustProgressAttr>">;
+
+// Target dependent attributes
+include "llvm/IR/AttributesAMDGPU.td"
diff --git a/llvm/include/llvm/IR/AttributesAMDGPU.td b/llvm/include/llvm/IR/AttributesAMDGPU.td
new file mode 100644
index 000000000000..e2a0f045b656
--- /dev/null
+++ b/llvm/include/llvm/IR/AttributesAMDGPU.td
@@ -0,0 +1,14 @@
+//===- AttributesAMDGPU.td - Defines AMDGPU attributes -----*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines AMDGPU specific attributes.
+//
+//===----------------------------------------------------------------------===//
+
+def AMDGPUUnsafeFPAtomics : StrBoolAttr<"amdgpu-unsafe-fp-atomics">;
+def : MergeRule<"setAND<AMDGPUUnsafeFPAtomicsAttr>">;
diff --git a/llvm/include/llvm/IR/Constants.h b/llvm/include/llvm/IR/Constants.h
index 71414d95d9a3..65d453861628 100644
--- a/llvm/include/llvm/IR/Constants.h
+++ b/llvm/include/llvm/IR/Constants.h
@@ -926,6 +926,41 @@ struct OperandTraits<DSOLocalEquivalent>
 
 DEFINE_TRANSPARENT_OPERAND_ACCESSORS(DSOLocalEquivalent, Value)
 
+/// Wrapper for a value that won't be replaced with a CFI jump table
+/// pointer in LowerTypeTestsModule.
+class NoCFIValue final : public Constant {
+  friend class Constant;
+
+  NoCFIValue(GlobalValue *GV);
+
+  void *operator new(size_t S) { return User::operator new(S, 1); }
+
+  void destroyConstantImpl();
+  Value *handleOperandChangeImpl(Value *From, Value *To);
+
+public:
+  /// Return a NoCFIValue for the specified function.
+  static NoCFIValue *get(GlobalValue *GV);
+
+  /// Transparently provide more efficient getOperand methods.
+  DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
+
+  GlobalValue *getGlobalValue() const {
+    return cast<GlobalValue>(Op<0>().get());
+  }
+
+  /// Methods for support type inquiry through isa, cast, and dyn_cast:
+  static bool classof(const Value *V) {
+    return V->getValueID() == NoCFIValueVal;
+  }
+};
+
+template <>
+struct OperandTraits<NoCFIValue> : public FixedNumOperandTraits<NoCFIValue, 1> {
+};
+
+DEFINE_TRANSPARENT_OPERAND_ACCESSORS(NoCFIValue, Value)
+
 //===----------------------------------------------------------------------===//
 /// A constant value that is initialized with an expression using
 /// other constant values.
diff --git a/llvm/include/llvm/IR/DataLayout.h b/llvm/include/llvm/IR/DataLayout.h
index 46acd403bef1..36438fc4f4e0 100644
--- a/llvm/include/llvm/IR/DataLayout.h
+++ b/llvm/include/llvm/IR/DataLayout.h
@@ -92,14 +92,14 @@ struct LayoutAlignElem {
 struct PointerAlignElem {
   Align ABIAlign;
   Align PrefAlign;
-  uint32_t TypeByteWidth;
+  uint32_t TypeBitWidth;
   uint32_t AddressSpace;
-  uint32_t IndexWidth;
+  uint32_t IndexBitWidth;
 
   /// Initializer
-  static PointerAlignElem get(uint32_t AddressSpace, Align ABIAlign,
-                              Align PrefAlign, uint32_t TypeByteWidth,
-                              uint32_t IndexWidth);
+  static PointerAlignElem getInBits(uint32_t AddressSpace, Align ABIAlign,
+                                    Align PrefAlign, uint32_t TypeBitWidth,
+                                    uint32_t IndexBitWidth);
 
   bool operator==(const PointerAlignElem &rhs) const;
 };
@@ -180,8 +180,9 @@ private:
 
   /// Attempts to set the alignment of a pointer in the given address space.
   /// Returns an error description on failure.
-  Error setPointerAlignment(uint32_t AddrSpace, Align ABIAlign, Align PrefAlign,
-                            uint32_t TypeByteWidth, uint32_t IndexWidth);
+  Error setPointerAlignmentInBits(uint32_t AddrSpace, Align ABIAlign,
+                                  Align PrefAlign, uint32_t TypeBitWidth,
+                                  uint32_t IndexBitWidth);
 
   /// Internal helper to get alignment for integer of given bitwidth.
   Align getIntegerAlignment(uint32_t BitWidth, bool abi_or_pref) const;
@@ -372,7 +373,8 @@ public:
   /// the backends/clients are updated.
   Align getPointerPrefAlignment(unsigned AS = 0) const;
 
-  /// Layout pointer size
+  /// Layout pointer size in bytes, rounded up to a whole
+  /// number of bytes.
   /// FIXME: The defaults need to be removed once all of
   /// the backends/clients are updated.
   unsigned getPointerSize(unsigned AS = 0) const;
@@ -380,7 +382,8 @@ public:
   /// Returns the maximum index size over all address spaces.
   unsigned getMaxIndexSize() const;
 
-  // Index size used for address calculation.
+  // Index size in bytes used for address calculation,
+  /// rounded up to a whole number of bytes.
   unsigned getIndexSize(unsigned AS) const;
 
   /// Return the address spaces containing non-integral pointers.  Pointers in
@@ -407,7 +410,7 @@ public:
   /// FIXME: The defaults need to be removed once all of
   /// the backends/clients are updated.
   unsigned getPointerSizeInBits(unsigned AS = 0) const {
-    return getPointerSize(AS) * 8;
+    return getPointerAlignElem(AS).TypeBitWidth;
   }
 
   /// Returns the maximum index size over all address spaces.
@@ -417,7 +420,7 @@ public:
 
   /// Size in bits of index used for address calculation in getelementptr.
   unsigned getIndexSizeInBits(unsigned AS) const {
-    return getIndexSize(AS) * 8;
+    return getPointerAlignElem(AS).IndexBitWidth;
   }
 
   /// Layout pointer size, in bits, based on the type.  If this function is
@@ -470,7 +473,7 @@ public:
   /// For example, returns 5 for i36 and 10 for x86_fp80.
   TypeSize getTypeStoreSize(Type *Ty) const {
     TypeSize BaseSize = getTypeSizeInBits(Ty);
-    return { (BaseSize.getKnownMinSize() + 7) / 8, BaseSize.isScalable() };
+    return {divideCeil(BaseSize.getKnownMinSize(), 8), BaseSize.isScalable()};
   }
 
   /// Returns the maximum number of bits that may be overwritten by
@@ -588,6 +591,12 @@ public:
   /// the result element type and Offset to be the residual offset.
   SmallVector<APInt> getGEPIndicesForOffset(Type *&ElemTy, APInt &Offset) const;
 
+  /// Get single GEP index to access Offset inside ElemTy. Returns None if
+  /// index cannot be computed, e.g. because the type is not an aggregate.
+  /// ElemTy is updated to be the result element type and Offset to be the
+  /// residual offset.
+  Optional<APInt> getGEPIndexForOffset(Type *&ElemTy, APInt &Offset) const;
+
   /// Returns a StructLayout object, indicating the alignment of the
   /// struct, its size, and the offsets of its fields.
   ///
diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h
index 046e9b5e809e..84ebb461ebef 100644
--- a/llvm/include/llvm/IR/Instructions.h
+++ b/llvm/include/llvm/IR/Instructions.h
@@ -105,6 +105,11 @@ public:
     return cast<PointerType>(Instruction::getType());
   }
 
+  /// Return the address space for the allocation.
+  unsigned getAddressSpace() const {
+    return getType()->getAddressSpace();
+  }
+
   /// Get allocation size in bits. Returns None if size can't be determined,
   /// e.g. in case of a VLA.
   Optional<TypeSize> getAllocationSizeInBits(const DataLayout &DL) const;
@@ -1451,6 +1456,10 @@ public:
   ///
   static auto predicates() { return FCmpPredicates(); }
 
+  /// Return result of `LHS Pred RHS` comparison.
+  static bool compare(const APFloat &LHS, const APFloat &RHS,
+                      FCmpInst::Predicate Pred);
+
   /// Methods for support type inquiry through isa, cast, and dyn_cast:
   static bool classof(const Instruction *I) {
     return I->getOpcode() == Instruction::FCmp;
diff --git a/llvm/include/llvm/IR/IntrinsicInst.h b/llvm/include/llvm/IR/IntrinsicInst.h
index d186029db8cf..647a912b72f6 100644
--- a/llvm/include/llvm/IR/IntrinsicInst.h
+++ b/llvm/include/llvm/IR/IntrinsicInst.h
@@ -390,8 +390,10 @@ public:
 class VPIntrinsic : public IntrinsicInst {
 public:
   /// \brief Declares a llvm.vp.* intrinsic in \p M that matches the parameters
-  /// \p Params.
+  /// \p Params. Additionally, the load and gather intrinsics require
+  /// \p ReturnType to be specified.
   static Function *getDeclarationForParams(Module *M, Intrinsic::ID,
+                                           Type *ReturnType,
                                            ArrayRef<Value *> Params);
 
   static Optional<unsigned> getMaskParamPos(Intrinsic::ID IntrinsicID);
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index 637e6d8f6cf5..da580de3dbd3 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -319,6 +319,7 @@ def llvm_v4bf16_ty     : LLVMType<v4bf16>;   //  4 x bfloat (__bf16)
 def llvm_v8bf16_ty     : LLVMType<v8bf16>;   //  8 x bfloat (__bf16)
 def llvm_v1f32_ty      : LLVMType<v1f32>;    //  1 x float
 def llvm_v2f32_ty      : LLVMType<v2f32>;    //  2 x float
+def llvm_v3f32_ty      : LLVMType<v3f32>;    //  3 x float
 def llvm_v4f32_ty      : LLVMType<v4f32>;    //  4 x float
 def llvm_v8f32_ty      : LLVMType<v8f32>;    //  8 x float
 def llvm_v16f32_ty     : LLVMType<v16f32>;   // 16 x float
@@ -331,6 +332,9 @@ def llvm_v16f64_ty     : LLVMType<v16f64>;   // 16 x double
 
 def llvm_vararg_ty     : LLVMType<isVoid>;   // this means vararg here
 
+def llvm_externref_ty  : LLVMType<externref>;
+def llvm_funcref_ty    : LLVMType<funcref>;
+
 //===----------------------------------------------------------------------===//
 // Intrinsic Definitions.
 //===----------------------------------------------------------------------===//
@@ -1013,14 +1017,15 @@ def int_codeview_annotation : DefaultAttrsIntrinsic<[], [llvm_metadata_ty],
 
 //===------------------------ Trampoline Intrinsics -----------------------===//
 //
-def int_init_trampoline : Intrinsic<[],
-                                    [llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty],
-                                    [IntrArgMemOnly, NoCapture<ArgIndex<0>>]>,
-                                    GCCBuiltin<"__builtin_init_trampoline">;
+def int_init_trampoline : DefaultAttrsIntrinsic<
+    [], [llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty],
+    [IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>,
+     ReadNone<ArgIndex<1>>, ReadNone<ArgIndex<2>>]>,
+    GCCBuiltin<"__builtin_init_trampoline">;
 
-def int_adjust_trampoline : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty],
-                                      [IntrReadMem, IntrArgMemOnly]>,
-                                      GCCBuiltin<"__builtin_adjust_trampoline">;
+def int_adjust_trampoline : DefaultAttrsIntrinsic<
+    [llvm_ptr_ty], [llvm_ptr_ty], [IntrReadMem, IntrArgMemOnly]>,
+    GCCBuiltin<"__builtin_adjust_trampoline">;
 
 //===------------------------ Overflow Intrinsics -------------------------===//
 //
@@ -1278,10 +1283,6 @@ def int_coro_alloca_alloc : Intrinsic<[llvm_token_ty],
 def int_coro_alloca_get : Intrinsic<[llvm_ptr_ty], [llvm_token_ty], []>;
 def int_coro_alloca_free : Intrinsic<[], [llvm_token_ty], []>;
 
-def int_coro_param : Intrinsic<[llvm_i1_ty], [llvm_ptr_ty, llvm_ptr_ty],
-                               [IntrNoMem, ReadNone<ArgIndex<0>>,
-                                ReadNone<ArgIndex<1>>]>;
-
 // Coroutine Manipulation Intrinsics.
 
 def int_coro_resume : Intrinsic<[], [llvm_ptr_ty], [Throws]>;
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index 0a44670de76e..2f2564702b87 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -1284,7 +1284,7 @@ def int_amdgcn_s_dcache_inv :
 
 def int_amdgcn_s_memtime :
   GCCBuiltin<"__builtin_amdgcn_s_memtime">,
-  Intrinsic<[llvm_i64_ty], [], [IntrWillReturn]>;
+  Intrinsic<[llvm_i64_ty], [], [IntrNoMem, IntrHasSideEffects, IntrWillReturn]>;
 
 def int_amdgcn_s_sleep :
   GCCBuiltin<"__builtin_amdgcn_s_sleep">,
@@ -1726,7 +1726,7 @@ def int_amdgcn_s_dcache_wb_vol :
 
 def int_amdgcn_s_memrealtime :
   GCCBuiltin<"__builtin_amdgcn_s_memrealtime">,
-  Intrinsic<[llvm_i64_ty], [], [IntrWillReturn]>;
+  Intrinsic<[llvm_i64_ty], [], [IntrNoMem, IntrHasSideEffects, IntrWillReturn]>;
 
 // llvm.amdgcn.ds.permute <index> <src>
 def int_amdgcn_ds_permute :
@@ -1789,9 +1789,11 @@ def int_amdgcn_global_atomic_csub : AMDGPUGlobalAtomicRtn<llvm_i32_ty>;
 
 // uint4 llvm.amdgcn.image.bvh.intersect.ray <node_ptr>, <ray_extent>, <ray_origin>,
 //                                           <ray_dir>, <ray_inv_dir>, <texture_descr>
+// <node_ptr> is i32 or i64.
+// <ray_dir> and <ray_inv_dir> are both v3f16 or both v3f32.
 def int_amdgcn_image_bvh_intersect_ray :
   Intrinsic<[llvm_v4i32_ty],
-            [llvm_anyint_ty, llvm_float_ty, llvm_v4f32_ty, llvm_anyvector_ty,
+            [llvm_anyint_ty, llvm_float_ty, llvm_v3f32_ty, llvm_anyvector_ty,
              LLVMMatchType<1>, llvm_v4i32_ty],
             [IntrReadMem, IntrWillReturn]>;
 
diff --git a/llvm/include/llvm/IR/IntrinsicsARM.td b/llvm/include/llvm/IR/IntrinsicsARM.td
index 52702fe7e731..cf375b9280db 100644
--- a/llvm/include/llvm/IR/IntrinsicsARM.td
+++ b/llvm/include/llvm/IR/IntrinsicsARM.td
@@ -809,8 +809,7 @@ def int_arm_cls64: Intrinsic<[llvm_i32_ty], [llvm_i64_ty], [IntrNoMem]>;
 def int_arm_mve_vctp8  : Intrinsic<[llvm_v16i1_ty], [llvm_i32_ty], [IntrNoMem]>;
 def int_arm_mve_vctp16 : Intrinsic<[llvm_v8i1_ty], [llvm_i32_ty], [IntrNoMem]>;
 def int_arm_mve_vctp32 : Intrinsic<[llvm_v4i1_ty], [llvm_i32_ty], [IntrNoMem]>;
-// vctp64 takes v4i1, to work around v2i1 not being a legal MVE type
-def int_arm_mve_vctp64 : Intrinsic<[llvm_v4i1_ty], [llvm_i32_ty], [IntrNoMem]>;
+def int_arm_mve_vctp64 : Intrinsic<[llvm_v2i1_ty], [llvm_i32_ty], [IntrNoMem]>;
 
 // v8.3-A Floating-point complex add
 def int_arm_neon_vcadd_rot90  : Neon_2Arg_Intrinsic;
diff --git a/llvm/include/llvm/IR/IntrinsicsHexagonDep.td b/llvm/include/llvm/IR/IntrinsicsHexagonDep.td
index 6799273bf805..177114636a50 100644
--- a/llvm/include/llvm/IR/IntrinsicsHexagonDep.td
+++ b/llvm/include/llvm/IR/IntrinsicsHexagonDep.td
@@ -23,13 +23,6 @@ class Hexagon_i64_i64_Intrinsic<string GCCIntSuffix,
        intr_properties>;
 
 // tag : A2_add
-class Hexagon_custom_i32_i32i32_Intrinsic<
-      list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
-       [llvm_i32_ty], [llvm_i32_ty,llvm_i32_ty],
-       intr_properties>;
-
-// tag : A2_addh_h16_hh
 class Hexagon_i32_i32i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
   : Hexagon_Intrinsic<GCCIntSuffix,
@@ -37,13 +30,6 @@ class Hexagon_i32_i32i32_Intrinsic<string GCCIntSuffix,
        intr_properties>;
 
 // tag : A2_addp
-class Hexagon_custom_i64_i64i64_Intrinsic<
-      list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
-       [llvm_i64_ty], [llvm_i64_ty,llvm_i64_ty],
-       intr_properties>;
-
-// tag : A2_addpsat
 class Hexagon_i64_i64i64_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
   : Hexagon_Intrinsic<GCCIntSuffix,
@@ -64,13 +50,6 @@ class Hexagon_i64_i32i32_Intrinsic<string GCCIntSuffix,
        [llvm_i64_ty], [llvm_i32_ty,llvm_i32_ty],
        intr_properties>;
 
-// tag : A2_neg
-class Hexagon_custom_i32_i32_Intrinsic<
-      list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
-       [llvm_i32_ty], [llvm_i32_ty],
-       intr_properties>;
-
 // tag : A2_roundsat
 class Hexagon_i32_i64_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
@@ -288,20 +267,6 @@ class Hexagon_i64_i64i32i32_Intrinsic<string GCCIntSuffix,
        [llvm_i64_ty], [llvm_i64_ty,llvm_i32_ty,llvm_i32_ty],
        intr_properties>;
 
-// tag : M2_dpmpyss_s0
-class Hexagon_custom_i64_i32i32_Intrinsic<
-      list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
-       [llvm_i64_ty], [llvm_i32_ty,llvm_i32_ty],
-       intr_properties>;
-
-// tag : S2_asl_i_p
-class Hexagon_custom_i64_i64i32_Intrinsic<
-      list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
-       [llvm_i64_ty], [llvm_i64_ty,llvm_i32_ty],
-       intr_properties>;
-
 // tag : S2_insert
 class Hexagon_i32_i32i32i32i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
@@ -351,7 +316,7 @@ class Hexagon_v32i32_v64i32_Intrinsic<string GCCIntSuffix,
        [llvm_v32i32_ty], [llvm_v64i32_ty],
        intr_properties>;
 
-// tag : V6_lvsplatb
+// tag : V6_lvsplatw
 class Hexagon_v16i32_i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
   : Hexagon_Intrinsic<GCCIntSuffix,
@@ -366,44 +331,44 @@ class Hexagon_v32i32_i32_Intrinsic<string GCCIntSuffix,
        intr_properties>;
 
 // tag : V6_pred_and
-class Hexagon_custom_v64i1_v64i1v64i1_Intrinsic<
+class Hexagon_v64i1_v64i1v64i1_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
+  : Hexagon_Intrinsic<GCCIntSuffix,
        [llvm_v64i1_ty], [llvm_v64i1_ty,llvm_v64i1_ty],
        intr_properties>;
 
 // tag : V6_pred_and
-class Hexagon_custom_v128i1_v128i1v128i1_Intrinsic_128B<
+class Hexagon_v128i1_v128i1v128i1_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
+  : Hexagon_Intrinsic<GCCIntSuffix,
        [llvm_v128i1_ty], [llvm_v128i1_ty,llvm_v128i1_ty],
        intr_properties>;
 
 // tag : V6_pred_not
-class Hexagon_custom_v64i1_v64i1_Intrinsic<
+class Hexagon_v64i1_v64i1_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
+  : Hexagon_Intrinsic<GCCIntSuffix,
        [llvm_v64i1_ty], [llvm_v64i1_ty],
        intr_properties>;
 
 // tag : V6_pred_not
-class Hexagon_custom_v128i1_v128i1_Intrinsic_128B<
+class Hexagon_v128i1_v128i1_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
+  : Hexagon_Intrinsic<GCCIntSuffix,
        [llvm_v128i1_ty], [llvm_v128i1_ty],
        intr_properties>;
 
 // tag : V6_pred_scalar2
-class Hexagon_custom_v64i1_i32_Intrinsic<
+class Hexagon_v64i1_i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
+  : Hexagon_Intrinsic<GCCIntSuffix,
        [llvm_v64i1_ty], [llvm_i32_ty],
        intr_properties>;
 
 // tag : V6_pred_scalar2
-class Hexagon_custom_v128i1_i32_Intrinsic_128B<
+class Hexagon_v128i1_i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
+  : Hexagon_Intrinsic<GCCIntSuffix,
        [llvm_v128i1_ty], [llvm_i32_ty],
        intr_properties>;
 
@@ -436,27 +401,27 @@ class Hexagon_v64i32_v64i32v64i32v64i32i32_Intrinsic<string GCCIntSuffix,
        intr_properties>;
 
 // tag : V6_vS32b_nqpred_ai
-class Hexagon_custom__v64i1ptrv16i32_Intrinsic<
+class Hexagon__v64i1ptrv16i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
+  : Hexagon_Intrinsic<GCCIntSuffix,
        [], [llvm_v64i1_ty,llvm_ptr_ty,llvm_v16i32_ty],
        intr_properties>;
 
 // tag : V6_vS32b_nqpred_ai
-class Hexagon_custom__v128i1ptrv32i32_Intrinsic_128B<
+class Hexagon__v128i1ptrv32i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
+  : Hexagon_Intrinsic<GCCIntSuffix,
        [], [llvm_v128i1_ty,llvm_ptr_ty,llvm_v32i32_ty],
        intr_properties>;
 
-// tag : V6_vabsb
+// tag : V6_vabs_hf
 class Hexagon_v16i32_v16i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
   : Hexagon_Intrinsic<GCCIntSuffix,
        [llvm_v16i32_ty], [llvm_v16i32_ty],
        intr_properties>;
 
-// tag : V6_vabsb
+// tag : V6_vabs_hf
 class Hexagon_v32i32_v32i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
   : Hexagon_Intrinsic<GCCIntSuffix,
@@ -477,6 +442,20 @@ class Hexagon_v32i32_v32i32v32i32_Intrinsic<string GCCIntSuffix,
        [llvm_v32i32_ty], [llvm_v32i32_ty,llvm_v32i32_ty],
        intr_properties>;
 
+// tag : V6_vadd_sf_hf
+class Hexagon_v32i32_v16i32v16i32_Intrinsic<string GCCIntSuffix,
+      list<IntrinsicProperty> intr_properties = [IntrNoMem]>
+  : Hexagon_Intrinsic<GCCIntSuffix,
+       [llvm_v32i32_ty], [llvm_v16i32_ty,llvm_v16i32_ty],
+       intr_properties>;
+
+// tag : V6_vadd_sf_hf
+class Hexagon_v64i32_v32i32v32i32_Intrinsic<string GCCIntSuffix,
+      list<IntrinsicProperty> intr_properties = [IntrNoMem]>
+  : Hexagon_Intrinsic<GCCIntSuffix,
+       [llvm_v64i32_ty], [llvm_v32i32_ty,llvm_v32i32_ty],
+       intr_properties>;
+
 // tag : V6_vaddb_dv
 class Hexagon_v64i32_v64i32v64i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
@@ -485,16 +464,16 @@ class Hexagon_v64i32_v64i32v64i32_Intrinsic<string GCCIntSuffix,
        intr_properties>;
 
 // tag : V6_vaddbnq
-class Hexagon_custom_v16i32_v64i1v16i32v16i32_Intrinsic<
+class Hexagon_v16i32_v64i1v16i32v16i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
+  : Hexagon_Intrinsic<GCCIntSuffix,
        [llvm_v16i32_ty], [llvm_v64i1_ty,llvm_v16i32_ty,llvm_v16i32_ty],
        intr_properties>;
 
 // tag : V6_vaddbnq
-class Hexagon_custom_v32i32_v128i1v32i32v32i32_Intrinsic_128B<
+class Hexagon_v32i32_v128i1v32i32v32i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
+  : Hexagon_Intrinsic<GCCIntSuffix,
        [llvm_v32i32_ty], [llvm_v128i1_ty,llvm_v32i32_ty,llvm_v32i32_ty],
        intr_properties>;
 
@@ -513,31 +492,17 @@ class Hexagon_custom_v32i32v128i1_v32i32v32i32v128i1_Intrinsic_128B<
        intr_properties>;
 
 // tag : V6_vaddcarrysat
-class Hexagon_custom_v16i32_v16i32v16i32v64i1_Intrinsic<
+class Hexagon_v16i32_v16i32v16i32v64i1_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
+  : Hexagon_Intrinsic<GCCIntSuffix,
        [llvm_v16i32_ty], [llvm_v16i32_ty,llvm_v16i32_ty,llvm_v64i1_ty],
        intr_properties>;
 
 // tag : V6_vaddcarrysat
-class Hexagon_custom_v32i32_v32i32v32i32v128i1_Intrinsic_128B<
-      list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
-       [llvm_v32i32_ty], [llvm_v32i32_ty,llvm_v32i32_ty,llvm_v128i1_ty],
-       intr_properties>;
-
-// tag : V6_vaddhw
-class Hexagon_v32i32_v16i32v16i32_Intrinsic<string GCCIntSuffix,
-      list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_Intrinsic<GCCIntSuffix,
-       [llvm_v32i32_ty], [llvm_v16i32_ty,llvm_v16i32_ty],
-       intr_properties>;
-
-// tag : V6_vaddhw
-class Hexagon_v64i32_v32i32v32i32_Intrinsic<string GCCIntSuffix,
+class Hexagon_v32i32_v32i32v32i32v128i1_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
   : Hexagon_Intrinsic<GCCIntSuffix,
-       [llvm_v64i32_ty], [llvm_v32i32_ty,llvm_v32i32_ty],
+       [llvm_v32i32_ty], [llvm_v32i32_ty,llvm_v32i32_ty,llvm_v128i1_ty],
        intr_properties>;
 
 // tag : V6_vaddhw_acc
@@ -562,72 +527,72 @@ class Hexagon_v16i32_v16i32v16i32i32_Intrinsic<string GCCIntSuffix,
        intr_properties>;
 
 // tag : V6_vandnqrt
-class Hexagon_custom_v16i32_v64i1i32_Intrinsic<
+class Hexagon_v16i32_v64i1i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
+  : Hexagon_Intrinsic<GCCIntSuffix,
        [llvm_v16i32_ty], [llvm_v64i1_ty,llvm_i32_ty],
        intr_properties>;
 
 // tag : V6_vandnqrt
-class Hexagon_custom_v32i32_v128i1i32_Intrinsic_128B<
+class Hexagon_v32i32_v128i1i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
+  : Hexagon_Intrinsic<GCCIntSuffix,
        [llvm_v32i32_ty], [llvm_v128i1_ty,llvm_i32_ty],
        intr_properties>;
 
 // tag : V6_vandnqrt_acc
-class Hexagon_custom_v16i32_v16i32v64i1i32_Intrinsic<
+class Hexagon_v16i32_v16i32v64i1i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
+  : Hexagon_Intrinsic<GCCIntSuffix,
        [llvm_v16i32_ty], [llvm_v16i32_ty,llvm_v64i1_ty,llvm_i32_ty],
        intr_properties>;
 
 // tag : V6_vandnqrt_acc
-class Hexagon_custom_v32i32_v32i32v128i1i32_Intrinsic_128B<
+class Hexagon_v32i32_v32i32v128i1i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
+  : Hexagon_Intrinsic<GCCIntSuffix,
        [llvm_v32i32_ty], [llvm_v32i32_ty,llvm_v128i1_ty,llvm_i32_ty],
        intr_properties>;
 
 // tag : V6_vandvnqv
-class Hexagon_custom_v16i32_v64i1v16i32_Intrinsic<
+class Hexagon_v16i32_v64i1v16i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
+  : Hexagon_Intrinsic<GCCIntSuffix,
        [llvm_v16i32_ty], [llvm_v64i1_ty,llvm_v16i32_ty],
        intr_properties>;
 
 // tag : V6_vandvnqv
-class Hexagon_custom_v32i32_v128i1v32i32_Intrinsic_128B<
+class Hexagon_v32i32_v128i1v32i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
+  : Hexagon_Intrinsic<GCCIntSuffix,
        [llvm_v32i32_ty], [llvm_v128i1_ty,llvm_v32i32_ty],
        intr_properties>;
 
 // tag : V6_vandvrt
-class Hexagon_custom_v64i1_v16i32i32_Intrinsic<
+class Hexagon_v64i1_v16i32i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
+  : Hexagon_Intrinsic<GCCIntSuffix,
        [llvm_v64i1_ty], [llvm_v16i32_ty,llvm_i32_ty],
        intr_properties>;
 
 // tag : V6_vandvrt
-class Hexagon_custom_v128i1_v32i32i32_Intrinsic_128B<
+class Hexagon_v128i1_v32i32i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
+  : Hexagon_Intrinsic<GCCIntSuffix,
        [llvm_v128i1_ty], [llvm_v32i32_ty,llvm_i32_ty],
        intr_properties>;
 
 // tag : V6_vandvrt_acc
-class Hexagon_custom_v64i1_v64i1v16i32i32_Intrinsic<
+class Hexagon_v64i1_v64i1v16i32i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
+  : Hexagon_Intrinsic<GCCIntSuffix,
        [llvm_v64i1_ty], [llvm_v64i1_ty,llvm_v16i32_ty,llvm_i32_ty],
        intr_properties>;
 
 // tag : V6_vandvrt_acc
-class Hexagon_custom_v128i1_v128i1v32i32i32_Intrinsic_128B<
+class Hexagon_v128i1_v128i1v32i32i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
+  : Hexagon_Intrinsic<GCCIntSuffix,
        [llvm_v128i1_ty], [llvm_v128i1_ty,llvm_v32i32_ty,llvm_i32_ty],
        intr_properties>;
 
@@ -645,6 +610,20 @@ class Hexagon_v32i32_v32i32i32_Intrinsic<string GCCIntSuffix,
        [llvm_v32i32_ty], [llvm_v32i32_ty,llvm_i32_ty],
        intr_properties>;
 
+// tag : V6_vasrvuhubrndsat
+class Hexagon_v16i32_v32i32v16i32_Intrinsic<string GCCIntSuffix,
+      list<IntrinsicProperty> intr_properties = [IntrNoMem]>
+  : Hexagon_Intrinsic<GCCIntSuffix,
+       [llvm_v16i32_ty], [llvm_v32i32_ty,llvm_v16i32_ty],
+       intr_properties>;
+
+// tag : V6_vasrvuhubrndsat
+class Hexagon_v32i32_v64i32v32i32_Intrinsic<string GCCIntSuffix,
+      list<IntrinsicProperty> intr_properties = [IntrNoMem]>
+  : Hexagon_Intrinsic<GCCIntSuffix,
+       [llvm_v32i32_ty], [llvm_v64i32_ty,llvm_v32i32_ty],
+       intr_properties>;
+
 // tag : V6_vassignp
 class Hexagon_v64i32_v64i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
@@ -652,6 +631,20 @@ class Hexagon_v64i32_v64i32_Intrinsic<string GCCIntSuffix,
        [llvm_v64i32_ty], [llvm_v64i32_ty],
        intr_properties>;
 
+// tag : V6_vcvt_hf_b
+class Hexagon_v32i32_v16i32_Intrinsic<string GCCIntSuffix,
+      list<IntrinsicProperty> intr_properties = [IntrNoMem]>
+  : Hexagon_Intrinsic<GCCIntSuffix,
+       [llvm_v32i32_ty], [llvm_v16i32_ty],
+       intr_properties>;
+
+// tag : V6_vcvt_hf_b
+class Hexagon_v64i32_v32i32_Intrinsic<string GCCIntSuffix,
+      list<IntrinsicProperty> intr_properties = [IntrNoMem]>
+  : Hexagon_Intrinsic<GCCIntSuffix,
+       [llvm_v64i32_ty], [llvm_v32i32_ty],
+       intr_properties>;
+
 // tag : V6_vd0
 class Hexagon_v16i32__Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
@@ -687,6 +680,20 @@ class Hexagon_v64i32_v32i32v32i32i32_Intrinsic<string GCCIntSuffix,
        [llvm_v64i32_ty], [llvm_v32i32_ty,llvm_v32i32_ty,llvm_i32_ty],
        intr_properties>;
 
+// tag : V6_vdmpy_sf_hf_acc
+class Hexagon_v16i32_v16i32v16i32v16i32_Intrinsic<string GCCIntSuffix,
+      list<IntrinsicProperty> intr_properties = [IntrNoMem]>
+  : Hexagon_Intrinsic<GCCIntSuffix,
+       [llvm_v16i32_ty], [llvm_v16i32_ty,llvm_v16i32_ty,llvm_v16i32_ty],
+       intr_properties>;
+
+// tag : V6_vdmpy_sf_hf_acc
+class Hexagon_v32i32_v32i32v32i32v32i32_Intrinsic<string GCCIntSuffix,
+      list<IntrinsicProperty> intr_properties = [IntrNoMem]>
+  : Hexagon_Intrinsic<GCCIntSuffix,
+       [llvm_v32i32_ty], [llvm_v32i32_ty,llvm_v32i32_ty,llvm_v32i32_ty],
+       intr_properties>;
+
 // tag : V6_vdmpybus_dv
 class Hexagon_v64i32_v64i32i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
@@ -722,45 +729,31 @@ class Hexagon_v32i32_v32i32v64i32i32_Intrinsic<string GCCIntSuffix,
        [llvm_v32i32_ty], [llvm_v32i32_ty,llvm_v64i32_ty,llvm_i32_ty],
        intr_properties>;
 
-// tag : V6_vdmpyhvsat_acc
-class Hexagon_v16i32_v16i32v16i32v16i32_Intrinsic<string GCCIntSuffix,
-      list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_Intrinsic<GCCIntSuffix,
-       [llvm_v16i32_ty], [llvm_v16i32_ty,llvm_v16i32_ty,llvm_v16i32_ty],
-       intr_properties>;
-
-// tag : V6_vdmpyhvsat_acc
-class Hexagon_v32i32_v32i32v32i32v32i32_Intrinsic<string GCCIntSuffix,
-      list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_Intrinsic<GCCIntSuffix,
-       [llvm_v32i32_ty], [llvm_v32i32_ty,llvm_v32i32_ty,llvm_v32i32_ty],
-       intr_properties>;
-
 // tag : V6_veqb
-class Hexagon_custom_v64i1_v16i32v16i32_Intrinsic<
+class Hexagon_v64i1_v16i32v16i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
+  : Hexagon_Intrinsic<GCCIntSuffix,
        [llvm_v64i1_ty], [llvm_v16i32_ty,llvm_v16i32_ty],
        intr_properties>;
 
 // tag : V6_veqb
-class Hexagon_custom_v128i1_v32i32v32i32_Intrinsic_128B<
+class Hexagon_v128i1_v32i32v32i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
+  : Hexagon_Intrinsic<GCCIntSuffix,
        [llvm_v128i1_ty], [llvm_v32i32_ty,llvm_v32i32_ty],
        intr_properties>;
 
 // tag : V6_veqb_and
-class Hexagon_custom_v64i1_v64i1v16i32v16i32_Intrinsic<
+class Hexagon_v64i1_v64i1v16i32v16i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
+  : Hexagon_Intrinsic<GCCIntSuffix,
        [llvm_v64i1_ty], [llvm_v64i1_ty,llvm_v16i32_ty,llvm_v16i32_ty],
        intr_properties>;
 
 // tag : V6_veqb_and
-class Hexagon_custom_v128i1_v128i1v32i32v32i32_Intrinsic_128B<
+class Hexagon_v128i1_v128i1v32i32v32i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
+  : Hexagon_Intrinsic<GCCIntSuffix,
        [llvm_v128i1_ty], [llvm_v128i1_ty,llvm_v32i32_ty,llvm_v32i32_ty],
        intr_properties>;
 
@@ -779,16 +772,16 @@ class Hexagon__ptri32i32v32i32_Intrinsic<string GCCIntSuffix,
        intr_properties>;
 
 // tag : V6_vgathermhq
-class Hexagon_custom__ptrv64i1i32i32v16i32_Intrinsic<
+class Hexagon__ptrv64i1i32i32v16i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
+  : Hexagon_Intrinsic<GCCIntSuffix,
        [], [llvm_ptr_ty,llvm_v64i1_ty,llvm_i32_ty,llvm_i32_ty,llvm_v16i32_ty],
        intr_properties>;
 
 // tag : V6_vgathermhq
-class Hexagon_custom__ptrv128i1i32i32v32i32_Intrinsic_128B<
+class Hexagon__ptrv128i1i32i32v32i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
+  : Hexagon_Intrinsic<GCCIntSuffix,
        [], [llvm_ptr_ty,llvm_v128i1_ty,llvm_i32_ty,llvm_i32_ty,llvm_v32i32_ty],
        intr_properties>;
 
@@ -800,16 +793,16 @@ class Hexagon__ptri32i32v64i32_Intrinsic<string GCCIntSuffix,
        intr_properties>;
 
 // tag : V6_vgathermhwq
-class Hexagon_custom__ptrv64i1i32i32v32i32_Intrinsic<
+class Hexagon__ptrv64i1i32i32v32i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
+  : Hexagon_Intrinsic<GCCIntSuffix,
        [], [llvm_ptr_ty,llvm_v64i1_ty,llvm_i32_ty,llvm_i32_ty,llvm_v32i32_ty],
        intr_properties>;
 
 // tag : V6_vgathermhwq
-class Hexagon_custom__ptrv128i1i32i32v64i32_Intrinsic_128B<
+class Hexagon__ptrv128i1i32i32v64i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
+  : Hexagon_Intrinsic<GCCIntSuffix,
        [], [llvm_ptr_ty,llvm_v128i1_ty,llvm_i32_ty,llvm_i32_ty,llvm_v64i32_ty],
        intr_properties>;
 
@@ -891,16 +884,16 @@ class Hexagon_v64i32_v64i32v32i32i32_Intrinsic<string GCCIntSuffix,
        intr_properties>;
 
 // tag : V6_vprefixqb
-class Hexagon_custom_v16i32_v64i1_Intrinsic<
+class Hexagon_v16i32_v64i1_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
+  : Hexagon_Intrinsic<GCCIntSuffix,
        [llvm_v16i32_ty], [llvm_v64i1_ty],
        intr_properties>;
 
 // tag : V6_vprefixqb
-class Hexagon_custom_v32i32_v128i1_Intrinsic_128B<
+class Hexagon_v32i32_v128i1_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
+  : Hexagon_Intrinsic<GCCIntSuffix,
        [llvm_v32i32_ty], [llvm_v128i1_ty],
        intr_properties>;
 
@@ -932,20 +925,6 @@ class Hexagon_v64i32_v64i32v64i32i32i32_Intrinsic<string GCCIntSuffix,
        [llvm_v64i32_ty], [llvm_v64i32_ty,llvm_v64i32_ty,llvm_i32_ty,llvm_i32_ty],
        intr_properties>;
 
-// tag : V6_vsb
-class Hexagon_v32i32_v16i32_Intrinsic<string GCCIntSuffix,
-      list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_Intrinsic<GCCIntSuffix,
-       [llvm_v32i32_ty], [llvm_v16i32_ty],
-       intr_properties>;
-
-// tag : V6_vsb
-class Hexagon_v64i32_v32i32_Intrinsic<string GCCIntSuffix,
-      list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_Intrinsic<GCCIntSuffix,
-       [llvm_v64i32_ty], [llvm_v32i32_ty],
-       intr_properties>;
-
 // tag : V6_vscattermh
 class Hexagon__i32i32v16i32v16i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
@@ -961,16 +940,16 @@ class Hexagon__i32i32v32i32v32i32_Intrinsic<string GCCIntSuffix,
        intr_properties>;
 
 // tag : V6_vscattermhq
-class Hexagon_custom__v64i1i32i32v16i32v16i32_Intrinsic<
+class Hexagon__v64i1i32i32v16i32v16i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
+  : Hexagon_Intrinsic<GCCIntSuffix,
        [], [llvm_v64i1_ty,llvm_i32_ty,llvm_i32_ty,llvm_v16i32_ty,llvm_v16i32_ty],
        intr_properties>;
 
 // tag : V6_vscattermhq
-class Hexagon_custom__v128i1i32i32v32i32v32i32_Intrinsic_128B<
+class Hexagon__v128i1i32i32v32i32v32i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
+  : Hexagon_Intrinsic<GCCIntSuffix,
        [], [llvm_v128i1_ty,llvm_i32_ty,llvm_i32_ty,llvm_v32i32_ty,llvm_v32i32_ty],
        intr_properties>;
 
@@ -989,30 +968,30 @@ class Hexagon__i32i32v64i32v32i32_Intrinsic<string GCCIntSuffix,
        intr_properties>;
 
 // tag : V6_vscattermhwq
-class Hexagon_custom__v64i1i32i32v32i32v16i32_Intrinsic<
+class Hexagon__v64i1i32i32v32i32v16i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
+  : Hexagon_Intrinsic<GCCIntSuffix,
        [], [llvm_v64i1_ty,llvm_i32_ty,llvm_i32_ty,llvm_v32i32_ty,llvm_v16i32_ty],
        intr_properties>;
 
 // tag : V6_vscattermhwq
-class Hexagon_custom__v128i1i32i32v64i32v32i32_Intrinsic_128B<
+class Hexagon__v128i1i32i32v64i32v32i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
+  : Hexagon_Intrinsic<GCCIntSuffix,
        [], [llvm_v128i1_ty,llvm_i32_ty,llvm_i32_ty,llvm_v64i32_ty,llvm_v32i32_ty],
        intr_properties>;
 
 // tag : V6_vswap
-class Hexagon_custom_v32i32_v64i1v16i32v16i32_Intrinsic<
+class Hexagon_v32i32_v64i1v16i32v16i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
+  : Hexagon_Intrinsic<GCCIntSuffix,
        [llvm_v32i32_ty], [llvm_v64i1_ty,llvm_v16i32_ty,llvm_v16i32_ty],
        intr_properties>;
 
 // tag : V6_vswap
-class Hexagon_custom_v64i32_v128i1v32i32v32i32_Intrinsic_128B<
+class Hexagon_v64i32_v128i1v32i32v32i32_Intrinsic<string GCCIntSuffix,
       list<IntrinsicProperty> intr_properties = [IntrNoMem]>
-  : Hexagon_NonGCC_Intrinsic<
+  : Hexagon_Intrinsic<GCCIntSuffix,
        [llvm_v64i32_ty], [llvm_v128i1_ty,llvm_v32i32_ty,llvm_v32i32_ty],
        intr_properties>;
 
@@ -1077,7 +1056,7 @@ def int_hexagon_A2_abssat :
 Hexagon_i32_i32_Intrinsic<"HEXAGON_A2_abssat">;
 
 def int_hexagon_A2_add :
-Hexagon_custom_i32_i32i32_Intrinsic;
+Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A2_add">;
 
 def int_hexagon_A2_addh_h16_hh :
 Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A2_addh_h16_hh">;
@@ -1116,10 +1095,10 @@ def int_hexagon_A2_addh_l16_sat_ll :
 Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A2_addh_l16_sat_ll">;
 
 def int_hexagon_A2_addi :
-Hexagon_custom_i32_i32i32_Intrinsic<[IntrNoMem, ImmArg<ArgIndex<1>>]>;
+Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A2_addi", [IntrNoMem, ImmArg<ArgIndex<1>>]>;
 
 def int_hexagon_A2_addp :
-Hexagon_custom_i64_i64i64_Intrinsic;
+Hexagon_i64_i64i64_Intrinsic<"HEXAGON_A2_addp">;
 
 def int_hexagon_A2_addpsat :
 Hexagon_i64_i64i64_Intrinsic<"HEXAGON_A2_addpsat">;
@@ -1131,10 +1110,10 @@ def int_hexagon_A2_addsp :
 Hexagon_i64_i32i64_Intrinsic<"HEXAGON_A2_addsp">;
 
 def int_hexagon_A2_and :
-Hexagon_custom_i32_i32i32_Intrinsic;
+Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A2_and">;
 
 def int_hexagon_A2_andir :
-Hexagon_custom_i32_i32i32_Intrinsic<[IntrNoMem, ImmArg<ArgIndex<1>>]>;
+Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A2_andir", [IntrNoMem, ImmArg<ArgIndex<1>>]>;
 
 def int_hexagon_A2_andp :
 Hexagon_i64_i64i64_Intrinsic<"HEXAGON_A2_andp">;
@@ -1188,7 +1167,7 @@ def int_hexagon_A2_minup :
 Hexagon_i64_i64i64_Intrinsic<"HEXAGON_A2_minup">;
 
 def int_hexagon_A2_neg :
-Hexagon_custom_i32_i32_Intrinsic;
+Hexagon_i32_i32_Intrinsic<"HEXAGON_A2_neg">;
 
 def int_hexagon_A2_negp :
 Hexagon_i64_i64_Intrinsic<"HEXAGON_A2_negp">;
@@ -1197,16 +1176,16 @@ def int_hexagon_A2_negsat :
 Hexagon_i32_i32_Intrinsic<"HEXAGON_A2_negsat">;
 
 def int_hexagon_A2_not :
-Hexagon_custom_i32_i32_Intrinsic;
+Hexagon_i32_i32_Intrinsic<"HEXAGON_A2_not">;
 
 def int_hexagon_A2_notp :
 Hexagon_i64_i64_Intrinsic<"HEXAGON_A2_notp">;
 
 def int_hexagon_A2_or :
-Hexagon_custom_i32_i32i32_Intrinsic;
+Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A2_or">;
 
 def int_hexagon_A2_orir :
-Hexagon_custom_i32_i32i32_Intrinsic<[IntrNoMem, ImmArg<ArgIndex<1>>]>;
+Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A2_orir", [IntrNoMem, ImmArg<ArgIndex<1>>]>;
 
 def int_hexagon_A2_orp :
 Hexagon_i64_i64i64_Intrinsic<"HEXAGON_A2_orp">;
@@ -1230,7 +1209,7 @@ def int_hexagon_A2_satuh :
 Hexagon_i32_i32_Intrinsic<"HEXAGON_A2_satuh">;
 
 def int_hexagon_A2_sub :
-Hexagon_custom_i32_i32i32_Intrinsic;
+Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A2_sub">;
 
 def int_hexagon_A2_subh_h16_hh :
 Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A2_subh_h16_hh">;
@@ -1269,10 +1248,10 @@ def int_hexagon_A2_subh_l16_sat_ll :
 Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A2_subh_l16_sat_ll">;
 
 def int_hexagon_A2_subp :
-Hexagon_custom_i64_i64i64_Intrinsic;
+Hexagon_i64_i64i64_Intrinsic<"HEXAGON_A2_subp">;
 
 def int_hexagon_A2_subri :
-Hexagon_custom_i32_i32i32_Intrinsic<[IntrNoMem, ImmArg<ArgIndex<0>>]>;
+Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A2_subri", [IntrNoMem, ImmArg<ArgIndex<0>>]>;
 
 def int_hexagon_A2_subsat :
 Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A2_subsat">;
@@ -1308,10 +1287,10 @@ def int_hexagon_A2_swiz :
 Hexagon_i32_i32_Intrinsic<"HEXAGON_A2_swiz">;
 
 def int_hexagon_A2_sxtb :
-Hexagon_custom_i32_i32_Intrinsic;
+Hexagon_i32_i32_Intrinsic<"HEXAGON_A2_sxtb">;
 
 def int_hexagon_A2_sxth :
-Hexagon_custom_i32_i32_Intrinsic;
+Hexagon_i32_i32_Intrinsic<"HEXAGON_A2_sxth">;
 
 def int_hexagon_A2_sxtw :
 Hexagon_i64_i32_Intrinsic<"HEXAGON_A2_sxtw">;
@@ -1524,16 +1503,16 @@ def int_hexagon_A2_vsubws :
 Hexagon_i64_i64i64_Intrinsic<"HEXAGON_A2_vsubws">;
 
 def int_hexagon_A2_xor :
-Hexagon_custom_i32_i32i32_Intrinsic;
+Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A2_xor">;
 
 def int_hexagon_A2_xorp :
 Hexagon_i64_i64i64_Intrinsic<"HEXAGON_A2_xorp">;
 
 def int_hexagon_A2_zxtb :
-Hexagon_custom_i32_i32_Intrinsic;
+Hexagon_i32_i32_Intrinsic<"HEXAGON_A2_zxtb">;
 
 def int_hexagon_A2_zxth :
-Hexagon_custom_i32_i32_Intrinsic;
+Hexagon_i32_i32_Intrinsic<"HEXAGON_A2_zxth">;
 
 def int_hexagon_A4_andn :
 Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A4_andn">;
@@ -2088,7 +2067,7 @@ def int_hexagon_M2_dpmpyss_rnd_s0 :
 Hexagon_i32_i32i32_Intrinsic<"HEXAGON_M2_dpmpyss_rnd_s0">;
 
 def int_hexagon_M2_dpmpyss_s0 :
-Hexagon_custom_i64_i32i32_Intrinsic;
+Hexagon_i64_i32i32_Intrinsic<"HEXAGON_M2_dpmpyss_s0">;
 
 def int_hexagon_M2_dpmpyuu_acc_s0 :
 Hexagon_i64_i64i32i32_Intrinsic<"HEXAGON_M2_dpmpyuu_acc_s0">;
@@ -2097,7 +2076,7 @@ def int_hexagon_M2_dpmpyuu_nac_s0 :
 Hexagon_i64_i64i32i32_Intrinsic<"HEXAGON_M2_dpmpyuu_nac_s0">;
 
 def int_hexagon_M2_dpmpyuu_s0 :
-Hexagon_custom_i64_i32i32_Intrinsic;
+Hexagon_i64_i32i32_Intrinsic<"HEXAGON_M2_dpmpyuu_s0">;
 
 def int_hexagon_M2_hmmpyh_rs1 :
 Hexagon_i32_i32i32_Intrinsic<"HEXAGON_M2_hmmpyh_rs1">;
@@ -2514,10 +2493,10 @@ def int_hexagon_M2_mpyd_rnd_ll_s1 :
 Hexagon_i64_i32i32_Intrinsic<"HEXAGON_M2_mpyd_rnd_ll_s1">;
 
 def int_hexagon_M2_mpyi :
-Hexagon_custom_i32_i32i32_Intrinsic;
+Hexagon_i32_i32i32_Intrinsic<"HEXAGON_M2_mpyi">;
 
 def int_hexagon_M2_mpysmi :
-Hexagon_custom_i32_i32i32_Intrinsic<[IntrNoMem, ImmArg<ArgIndex<1>>]>;
+Hexagon_i32_i32i32_Intrinsic<"HEXAGON_M2_mpysmi", [IntrNoMem, ImmArg<ArgIndex<1>>]>;
 
 def int_hexagon_M2_mpysu_up :
 Hexagon_i32_i32i32_Intrinsic<"HEXAGON_M2_mpysu_up">;
@@ -2670,7 +2649,7 @@ def int_hexagon_M2_mpyud_nac_ll_s1 :
 Hexagon_i64_i64i32i32_Intrinsic<"HEXAGON_M2_mpyud_nac_ll_s1">;
 
 def int_hexagon_M2_mpyui :
-Hexagon_custom_i32_i32i32_Intrinsic;
+Hexagon_i32_i32i32_Intrinsic<"HEXAGON_M2_mpyui">;
 
 def int_hexagon_M2_nacci :
 Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_M2_nacci">;
@@ -2958,7 +2937,7 @@ def int_hexagon_S2_addasl_rrri :
 Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_addasl_rrri", [IntrNoMem, ImmArg<ArgIndex<2>>]>;
 
 def int_hexagon_S2_asl_i_p :
-Hexagon_custom_i64_i64i32_Intrinsic<[IntrNoMem, ImmArg<ArgIndex<1>>]>;
+Hexagon_i64_i64i32_Intrinsic<"HEXAGON_S2_asl_i_p", [IntrNoMem, ImmArg<ArgIndex<1>>]>;
 
 def int_hexagon_S2_asl_i_p_acc :
 Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_asl_i_p_acc", [IntrNoMem, ImmArg<ArgIndex<2>>]>;
@@ -2976,7 +2955,7 @@ def int_hexagon_S2_asl_i_p_xacc :
 Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_asl_i_p_xacc", [IntrNoMem, ImmArg<ArgIndex<2>>]>;
 
 def int_hexagon_S2_asl_i_r :
-Hexagon_custom_i32_i32i32_Intrinsic<[IntrNoMem, ImmArg<ArgIndex<1>>]>;
+Hexagon_i32_i32i32_Intrinsic<"HEXAGON_S2_asl_i_r", [IntrNoMem, ImmArg<ArgIndex<1>>]>;
 
 def int_hexagon_S2_asl_i_r_acc :
 Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_asl_i_r_acc", [IntrNoMem, ImmArg<ArgIndex<2>>]>;
@@ -3045,7 +3024,7 @@ def int_hexagon_S2_asl_r_vw :
 Hexagon_i64_i64i32_Intrinsic<"HEXAGON_S2_asl_r_vw">;
 
 def int_hexagon_S2_asr_i_p :
-Hexagon_custom_i64_i64i32_Intrinsic<[IntrNoMem, ImmArg<ArgIndex<1>>]>;
+Hexagon_i64_i64i32_Intrinsic<"HEXAGON_S2_asr_i_p", [IntrNoMem, ImmArg<ArgIndex<1>>]>;
 
 def int_hexagon_S2_asr_i_p_acc :
 Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_asr_i_p_acc", [IntrNoMem, ImmArg<ArgIndex<2>>]>;
@@ -3066,7 +3045,7 @@ def int_hexagon_S2_asr_i_p_rnd_goodsyntax :
 Hexagon_i64_i64i32_Intrinsic<"HEXAGON_S2_asr_i_p_rnd_goodsyntax", [IntrNoMem, ImmArg<ArgIndex<1>>]>;
 
 def int_hexagon_S2_asr_i_r :
-Hexagon_custom_i32_i32i32_Intrinsic<[IntrNoMem, ImmArg<ArgIndex<1>>]>;
+Hexagon_i32_i32i32_Intrinsic<"HEXAGON_S2_asr_i_r", [IntrNoMem, ImmArg<ArgIndex<1>>]>;
 
 def int_hexagon_S2_asr_i_r_acc :
 Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_asr_i_r_acc", [IntrNoMem, ImmArg<ArgIndex<2>>]>;
@@ -3258,7 +3237,7 @@ def int_hexagon_S2_lsl_r_vw :
 Hexagon_i64_i64i32_Intrinsic<"HEXAGON_S2_lsl_r_vw">;
 
 def int_hexagon_S2_lsr_i_p :
-Hexagon_custom_i64_i64i32_Intrinsic<[IntrNoMem, ImmArg<ArgIndex<1>>]>;
+Hexagon_i64_i64i32_Intrinsic<"HEXAGON_S2_lsr_i_p", [IntrNoMem, ImmArg<ArgIndex<1>>]>;
 
 def int_hexagon_S2_lsr_i_p_acc :
 Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_lsr_i_p_acc", [IntrNoMem, ImmArg<ArgIndex<2>>]>;
@@ -3276,7 +3255,7 @@ def int_hexagon_S2_lsr_i_p_xacc :
 Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_lsr_i_p_xacc", [IntrNoMem, ImmArg<ArgIndex<2>>]>;
 
 def int_hexagon_S2_lsr_i_r :
-Hexagon_custom_i32_i32i32_Intrinsic<[IntrNoMem, ImmArg<ArgIndex<1>>]>;
+Hexagon_i32_i32i32_Intrinsic<"HEXAGON_S2_lsr_i_r", [IntrNoMem, ImmArg<ArgIndex<1>>]>;
 
 def int_hexagon_S2_lsr_i_r_acc :
 Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_lsr_i_r_acc", [IntrNoMem, ImmArg<ArgIndex<2>>]>;
@@ -3809,70 +3788,70 @@ def int_hexagon_V6_lvsplatw_128B :
 Hexagon_v32i32_i32_Intrinsic<"HEXAGON_V6_lvsplatw_128B">;
 
 def int_hexagon_V6_pred_and :
-Hexagon_custom_v64i1_v64i1v64i1_Intrinsic;
+Hexagon_v64i1_v64i1v64i1_Intrinsic<"HEXAGON_V6_pred_and">;
 
 def int_hexagon_V6_pred_and_128B :
-Hexagon_custom_v128i1_v128i1v128i1_Intrinsic_128B;
+Hexagon_v128i1_v128i1v128i1_Intrinsic<"HEXAGON_V6_pred_and_128B">;
 
 def int_hexagon_V6_pred_and_n :
-Hexagon_custom_v64i1_v64i1v64i1_Intrinsic;
+Hexagon_v64i1_v64i1v64i1_Intrinsic<"HEXAGON_V6_pred_and_n">;
 
 def int_hexagon_V6_pred_and_n_128B :
-Hexagon_custom_v128i1_v128i1v128i1_Intrinsic_128B;
+Hexagon_v128i1_v128i1v128i1_Intrinsic<"HEXAGON_V6_pred_and_n_128B">;
 
 def int_hexagon_V6_pred_not :
-Hexagon_custom_v64i1_v64i1_Intrinsic;
+Hexagon_v64i1_v64i1_Intrinsic<"HEXAGON_V6_pred_not">;
 
 def int_hexagon_V6_pred_not_128B :
-Hexagon_custom_v128i1_v128i1_Intrinsic_128B;
+Hexagon_v128i1_v128i1_Intrinsic<"HEXAGON_V6_pred_not_128B">;
 
 def int_hexagon_V6_pred_or :
-Hexagon_custom_v64i1_v64i1v64i1_Intrinsic;
+Hexagon_v64i1_v64i1v64i1_Intrinsic<"HEXAGON_V6_pred_or">;
 
 def int_hexagon_V6_pred_or_128B :
-Hexagon_custom_v128i1_v128i1v128i1_Intrinsic_128B;
+Hexagon_v128i1_v128i1v128i1_Intrinsic<"HEXAGON_V6_pred_or_128B">;
 
 def int_hexagon_V6_pred_or_n :
-Hexagon_custom_v64i1_v64i1v64i1_Intrinsic;
+Hexagon_v64i1_v64i1v64i1_Intrinsic<"HEXAGON_V6_pred_or_n">;
 
 def int_hexagon_V6_pred_or_n_128B :
-Hexagon_custom_v128i1_v128i1v128i1_Intrinsic_128B;
+Hexagon_v128i1_v128i1v128i1_Intrinsic<"HEXAGON_V6_pred_or_n_128B">;
 
 def int_hexagon_V6_pred_scalar2 :
-Hexagon_custom_v64i1_i32_Intrinsic;
+Hexagon_v64i1_i32_Intrinsic<"HEXAGON_V6_pred_scalar2">;
 
 def int_hexagon_V6_pred_scalar2_128B :
-Hexagon_custom_v128i1_i32_Intrinsic_128B;
+Hexagon_v128i1_i32_Intrinsic<"HEXAGON_V6_pred_scalar2_128B">;
 
 def int_hexagon_V6_pred_xor :
-Hexagon_custom_v64i1_v64i1v64i1_Intrinsic;
+Hexagon_v64i1_v64i1v64i1_Intrinsic<"HEXAGON_V6_pred_xor">;
 
 def int_hexagon_V6_pred_xor_128B :
-Hexagon_custom_v128i1_v128i1v128i1_Intrinsic_128B;
+Hexagon_v128i1_v128i1v128i1_Intrinsic<"HEXAGON_V6_pred_xor_128B">;
 
 def int_hexagon_V6_vS32b_nqpred_ai :
-Hexagon_custom__v64i1ptrv16i32_Intrinsic<[IntrWriteMem]>;
+Hexagon__v64i1ptrv16i32_Intrinsic<"HEXAGON_V6_vS32b_nqpred_ai", [IntrWriteMem]>;
 
 def int_hexagon_V6_vS32b_nqpred_ai_128B :
-Hexagon_custom__v128i1ptrv32i32_Intrinsic_128B<[IntrWriteMem]>;
+Hexagon__v128i1ptrv32i32_Intrinsic<"HEXAGON_V6_vS32b_nqpred_ai_128B", [IntrWriteMem]>;
 
 def int_hexagon_V6_vS32b_nt_nqpred_ai :
-Hexagon_custom__v64i1ptrv16i32_Intrinsic<[IntrWriteMem]>;
+Hexagon__v64i1ptrv16i32_Intrinsic<"HEXAGON_V6_vS32b_nt_nqpred_ai", [IntrWriteMem]>;
 
 def int_hexagon_V6_vS32b_nt_nqpred_ai_128B :
-Hexagon_custom__v128i1ptrv32i32_Intrinsic_128B<[IntrWriteMem]>;
+Hexagon__v128i1ptrv32i32_Intrinsic<"HEXAGON_V6_vS32b_nt_nqpred_ai_128B", [IntrWriteMem]>;
 
 def int_hexagon_V6_vS32b_nt_qpred_ai :
-Hexagon_custom__v64i1ptrv16i32_Intrinsic<[IntrWriteMem]>;
+Hexagon__v64i1ptrv16i32_Intrinsic<"HEXAGON_V6_vS32b_nt_qpred_ai", [IntrWriteMem]>;
 
 def int_hexagon_V6_vS32b_nt_qpred_ai_128B :
-Hexagon_custom__v128i1ptrv32i32_Intrinsic_128B<[IntrWriteMem]>;
+Hexagon__v128i1ptrv32i32_Intrinsic<"HEXAGON_V6_vS32b_nt_qpred_ai_128B", [IntrWriteMem]>;
 
 def int_hexagon_V6_vS32b_qpred_ai :
-Hexagon_custom__v64i1ptrv16i32_Intrinsic<[IntrWriteMem]>;
+Hexagon__v64i1ptrv16i32_Intrinsic<"HEXAGON_V6_vS32b_qpred_ai", [IntrWriteMem]>;
 
 def int_hexagon_V6_vS32b_qpred_ai_128B :
-Hexagon_custom__v128i1ptrv32i32_Intrinsic_128B<[IntrWriteMem]>;
+Hexagon__v128i1ptrv32i32_Intrinsic<"HEXAGON_V6_vS32b_qpred_ai_128B", [IntrWriteMem]>;
 
 def int_hexagon_V6_vabsdiffh :
 Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vabsdiffh">;
@@ -3935,16 +3914,16 @@ def int_hexagon_V6_vaddb_dv_128B :
 Hexagon_v64i32_v64i32v64i32_Intrinsic<"HEXAGON_V6_vaddb_dv_128B">;
 
 def int_hexagon_V6_vaddbnq :
-Hexagon_custom_v16i32_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v16i32_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_vaddbnq">;
 
 def int_hexagon_V6_vaddbnq_128B :
-Hexagon_custom_v32i32_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v32i32_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vaddbnq_128B">;
 
 def int_hexagon_V6_vaddbq :
-Hexagon_custom_v16i32_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v16i32_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_vaddbq">;
 
 def int_hexagon_V6_vaddbq_128B :
-Hexagon_custom_v32i32_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v32i32_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vaddbq_128B">;
 
 def int_hexagon_V6_vaddh :
 Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vaddh">;
@@ -3959,16 +3938,16 @@ def int_hexagon_V6_vaddh_dv_128B :
 Hexagon_v64i32_v64i32v64i32_Intrinsic<"HEXAGON_V6_vaddh_dv_128B">;
 
 def int_hexagon_V6_vaddhnq :
-Hexagon_custom_v16i32_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v16i32_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_vaddhnq">;
 
 def int_hexagon_V6_vaddhnq_128B :
-Hexagon_custom_v32i32_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v32i32_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vaddhnq_128B">;
 
 def int_hexagon_V6_vaddhq :
-Hexagon_custom_v16i32_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v16i32_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_vaddhq">;
 
 def int_hexagon_V6_vaddhq_128B :
-Hexagon_custom_v32i32_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v32i32_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vaddhq_128B">;
 
 def int_hexagon_V6_vaddhsat :
 Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vaddhsat">;
@@ -4037,16 +4016,16 @@ def int_hexagon_V6_vaddw_dv_128B :
 Hexagon_v64i32_v64i32v64i32_Intrinsic<"HEXAGON_V6_vaddw_dv_128B">;
 
 def int_hexagon_V6_vaddwnq :
-Hexagon_custom_v16i32_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v16i32_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_vaddwnq">;
 
 def int_hexagon_V6_vaddwnq_128B :
-Hexagon_custom_v32i32_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v32i32_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vaddwnq_128B">;
 
 def int_hexagon_V6_vaddwq :
-Hexagon_custom_v16i32_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v16i32_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_vaddwq">;
 
 def int_hexagon_V6_vaddwq_128B :
-Hexagon_custom_v32i32_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v32i32_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vaddwq_128B">;
 
 def int_hexagon_V6_vaddwsat :
 Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vaddwsat">;
@@ -4079,28 +4058,28 @@ def int_hexagon_V6_vand_128B :
 Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vand_128B">;
 
 def int_hexagon_V6_vandqrt :
-Hexagon_custom_v16i32_v64i1i32_Intrinsic;
+Hexagon_v16i32_v64i1i32_Intrinsic<"HEXAGON_V6_vandqrt">;
 
 def int_hexagon_V6_vandqrt_128B :
-Hexagon_custom_v32i32_v128i1i32_Intrinsic_128B;
+Hexagon_v32i32_v128i1i32_Intrinsic<"HEXAGON_V6_vandqrt_128B">;
 
 def int_hexagon_V6_vandqrt_acc :
-Hexagon_custom_v16i32_v16i32v64i1i32_Intrinsic;
+Hexagon_v16i32_v16i32v64i1i32_Intrinsic<"HEXAGON_V6_vandqrt_acc">;
 
 def int_hexagon_V6_vandqrt_acc_128B :
-Hexagon_custom_v32i32_v32i32v128i1i32_Intrinsic_128B;
+Hexagon_v32i32_v32i32v128i1i32_Intrinsic<"HEXAGON_V6_vandqrt_acc_128B">;
 
 def int_hexagon_V6_vandvrt :
-Hexagon_custom_v64i1_v16i32i32_Intrinsic;
+Hexagon_v64i1_v16i32i32_Intrinsic<"HEXAGON_V6_vandvrt">;
 
 def int_hexagon_V6_vandvrt_128B :
-Hexagon_custom_v128i1_v32i32i32_Intrinsic_128B;
+Hexagon_v128i1_v32i32i32_Intrinsic<"HEXAGON_V6_vandvrt_128B">;
 
 def int_hexagon_V6_vandvrt_acc :
-Hexagon_custom_v64i1_v64i1v16i32i32_Intrinsic;
+Hexagon_v64i1_v64i1v16i32i32_Intrinsic<"HEXAGON_V6_vandvrt_acc">;
 
 def int_hexagon_V6_vandvrt_acc_128B :
-Hexagon_custom_v128i1_v128i1v32i32i32_Intrinsic_128B;
+Hexagon_v128i1_v128i1v32i32i32_Intrinsic<"HEXAGON_V6_vandvrt_acc_128B">;
 
 def int_hexagon_V6_vaslh :
 Hexagon_v16i32_v16i32i32_Intrinsic<"HEXAGON_V6_vaslh">;
@@ -4439,220 +4418,220 @@ def int_hexagon_V6_vdsaduh_acc_128B :
 Hexagon_v64i32_v64i32v64i32i32_Intrinsic<"HEXAGON_V6_vdsaduh_acc_128B">;
 
 def int_hexagon_V6_veqb :
-Hexagon_custom_v64i1_v16i32v16i32_Intrinsic;
+Hexagon_v64i1_v16i32v16i32_Intrinsic<"HEXAGON_V6_veqb">;
 
 def int_hexagon_V6_veqb_128B :
-Hexagon_custom_v128i1_v32i32v32i32_Intrinsic_128B;
+Hexagon_v128i1_v32i32v32i32_Intrinsic<"HEXAGON_V6_veqb_128B">;
 
 def int_hexagon_V6_veqb_and :
-Hexagon_custom_v64i1_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v64i1_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_veqb_and">;
 
 def int_hexagon_V6_veqb_and_128B :
-Hexagon_custom_v128i1_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v128i1_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_veqb_and_128B">;
 
 def int_hexagon_V6_veqb_or :
-Hexagon_custom_v64i1_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v64i1_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_veqb_or">;
 
 def int_hexagon_V6_veqb_or_128B :
-Hexagon_custom_v128i1_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v128i1_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_veqb_or_128B">;
 
 def int_hexagon_V6_veqb_xor :
-Hexagon_custom_v64i1_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v64i1_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_veqb_xor">;
 
 def int_hexagon_V6_veqb_xor_128B :
-Hexagon_custom_v128i1_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v128i1_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_veqb_xor_128B">;
 
 def int_hexagon_V6_veqh :
-Hexagon_custom_v64i1_v16i32v16i32_Intrinsic;
+Hexagon_v64i1_v16i32v16i32_Intrinsic<"HEXAGON_V6_veqh">;
 
 def int_hexagon_V6_veqh_128B :
-Hexagon_custom_v128i1_v32i32v32i32_Intrinsic_128B;
+Hexagon_v128i1_v32i32v32i32_Intrinsic<"HEXAGON_V6_veqh_128B">;
 
 def int_hexagon_V6_veqh_and :
-Hexagon_custom_v64i1_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v64i1_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_veqh_and">;
 
 def int_hexagon_V6_veqh_and_128B :
-Hexagon_custom_v128i1_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v128i1_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_veqh_and_128B">;
 
 def int_hexagon_V6_veqh_or :
-Hexagon_custom_v64i1_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v64i1_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_veqh_or">;
 
 def int_hexagon_V6_veqh_or_128B :
-Hexagon_custom_v128i1_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v128i1_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_veqh_or_128B">;
 
 def int_hexagon_V6_veqh_xor :
-Hexagon_custom_v64i1_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v64i1_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_veqh_xor">;
 
 def int_hexagon_V6_veqh_xor_128B :
-Hexagon_custom_v128i1_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v128i1_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_veqh_xor_128B">;
 
 def int_hexagon_V6_veqw :
-Hexagon_custom_v64i1_v16i32v16i32_Intrinsic;
+Hexagon_v64i1_v16i32v16i32_Intrinsic<"HEXAGON_V6_veqw">;
 
 def int_hexagon_V6_veqw_128B :
-Hexagon_custom_v128i1_v32i32v32i32_Intrinsic_128B;
+Hexagon_v128i1_v32i32v32i32_Intrinsic<"HEXAGON_V6_veqw_128B">;
 
 def int_hexagon_V6_veqw_and :
-Hexagon_custom_v64i1_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v64i1_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_veqw_and">;
 
 def int_hexagon_V6_veqw_and_128B :
-Hexagon_custom_v128i1_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v128i1_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_veqw_and_128B">;
 
 def int_hexagon_V6_veqw_or :
-Hexagon_custom_v64i1_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v64i1_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_veqw_or">;
 
 def int_hexagon_V6_veqw_or_128B :
-Hexagon_custom_v128i1_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v128i1_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_veqw_or_128B">;
 
 def int_hexagon_V6_veqw_xor :
-Hexagon_custom_v64i1_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v64i1_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_veqw_xor">;
 
 def int_hexagon_V6_veqw_xor_128B :
-Hexagon_custom_v128i1_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v128i1_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_veqw_xor_128B">;
 
 def int_hexagon_V6_vgtb :
-Hexagon_custom_v64i1_v16i32v16i32_Intrinsic;
+Hexagon_v64i1_v16i32v16i32_Intrinsic<"HEXAGON_V6_vgtb">;
 
 def int_hexagon_V6_vgtb_128B :
-Hexagon_custom_v128i1_v32i32v32i32_Intrinsic_128B;
+Hexagon_v128i1_v32i32v32i32_Intrinsic<"HEXAGON_V6_vgtb_128B">;
 
 def int_hexagon_V6_vgtb_and :
-Hexagon_custom_v64i1_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v64i1_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_vgtb_and">;
 
 def int_hexagon_V6_vgtb_and_128B :
-Hexagon_custom_v128i1_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v128i1_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vgtb_and_128B">;
 
 def int_hexagon_V6_vgtb_or :
-Hexagon_custom_v64i1_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v64i1_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_vgtb_or">;
 
 def int_hexagon_V6_vgtb_or_128B :
-Hexagon_custom_v128i1_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v128i1_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vgtb_or_128B">;
 
 def int_hexagon_V6_vgtb_xor :
-Hexagon_custom_v64i1_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v64i1_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_vgtb_xor">;
 
 def int_hexagon_V6_vgtb_xor_128B :
-Hexagon_custom_v128i1_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v128i1_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vgtb_xor_128B">;
 
 def int_hexagon_V6_vgth :
-Hexagon_custom_v64i1_v16i32v16i32_Intrinsic;
+Hexagon_v64i1_v16i32v16i32_Intrinsic<"HEXAGON_V6_vgth">;
 
 def int_hexagon_V6_vgth_128B :
-Hexagon_custom_v128i1_v32i32v32i32_Intrinsic_128B;
+Hexagon_v128i1_v32i32v32i32_Intrinsic<"HEXAGON_V6_vgth_128B">;
 
 def int_hexagon_V6_vgth_and :
-Hexagon_custom_v64i1_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v64i1_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_vgth_and">;
 
 def int_hexagon_V6_vgth_and_128B :
-Hexagon_custom_v128i1_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v128i1_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vgth_and_128B">;
 
 def int_hexagon_V6_vgth_or :
-Hexagon_custom_v64i1_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v64i1_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_vgth_or">;
 
 def int_hexagon_V6_vgth_or_128B :
-Hexagon_custom_v128i1_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v128i1_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vgth_or_128B">;
 
 def int_hexagon_V6_vgth_xor :
-Hexagon_custom_v64i1_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v64i1_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_vgth_xor">;
 
 def int_hexagon_V6_vgth_xor_128B :
-Hexagon_custom_v128i1_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v128i1_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vgth_xor_128B">;
 
 def int_hexagon_V6_vgtub :
-Hexagon_custom_v64i1_v16i32v16i32_Intrinsic;
+Hexagon_v64i1_v16i32v16i32_Intrinsic<"HEXAGON_V6_vgtub">;
 
 def int_hexagon_V6_vgtub_128B :
-Hexagon_custom_v128i1_v32i32v32i32_Intrinsic_128B;
+Hexagon_v128i1_v32i32v32i32_Intrinsic<"HEXAGON_V6_vgtub_128B">;
 
 def int_hexagon_V6_vgtub_and :
-Hexagon_custom_v64i1_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v64i1_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_vgtub_and">;
 
 def int_hexagon_V6_vgtub_and_128B :
-Hexagon_custom_v128i1_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v128i1_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vgtub_and_128B">;
 
 def int_hexagon_V6_vgtub_or :
-Hexagon_custom_v64i1_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v64i1_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_vgtub_or">;
 
 def int_hexagon_V6_vgtub_or_128B :
-Hexagon_custom_v128i1_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v128i1_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vgtub_or_128B">;
 
 def int_hexagon_V6_vgtub_xor :
-Hexagon_custom_v64i1_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v64i1_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_vgtub_xor">;
 
 def int_hexagon_V6_vgtub_xor_128B :
-Hexagon_custom_v128i1_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v128i1_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vgtub_xor_128B">;
 
 def int_hexagon_V6_vgtuh :
-Hexagon_custom_v64i1_v16i32v16i32_Intrinsic;
+Hexagon_v64i1_v16i32v16i32_Intrinsic<"HEXAGON_V6_vgtuh">;
 
 def int_hexagon_V6_vgtuh_128B :
-Hexagon_custom_v128i1_v32i32v32i32_Intrinsic_128B;
+Hexagon_v128i1_v32i32v32i32_Intrinsic<"HEXAGON_V6_vgtuh_128B">;
 
 def int_hexagon_V6_vgtuh_and :
-Hexagon_custom_v64i1_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v64i1_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_vgtuh_and">;
 
 def int_hexagon_V6_vgtuh_and_128B :
-Hexagon_custom_v128i1_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v128i1_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vgtuh_and_128B">;
 
 def int_hexagon_V6_vgtuh_or :
-Hexagon_custom_v64i1_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v64i1_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_vgtuh_or">;
 
 def int_hexagon_V6_vgtuh_or_128B :
-Hexagon_custom_v128i1_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v128i1_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vgtuh_or_128B">;
 
 def int_hexagon_V6_vgtuh_xor :
-Hexagon_custom_v64i1_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v64i1_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_vgtuh_xor">;
 
 def int_hexagon_V6_vgtuh_xor_128B :
-Hexagon_custom_v128i1_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v128i1_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vgtuh_xor_128B">;
 
 def int_hexagon_V6_vgtuw :
-Hexagon_custom_v64i1_v16i32v16i32_Intrinsic;
+Hexagon_v64i1_v16i32v16i32_Intrinsic<"HEXAGON_V6_vgtuw">;
 
 def int_hexagon_V6_vgtuw_128B :
-Hexagon_custom_v128i1_v32i32v32i32_Intrinsic_128B;
+Hexagon_v128i1_v32i32v32i32_Intrinsic<"HEXAGON_V6_vgtuw_128B">;
 
 def int_hexagon_V6_vgtuw_and :
-Hexagon_custom_v64i1_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v64i1_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_vgtuw_and">;
 
 def int_hexagon_V6_vgtuw_and_128B :
-Hexagon_custom_v128i1_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v128i1_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vgtuw_and_128B">;
 
 def int_hexagon_V6_vgtuw_or :
-Hexagon_custom_v64i1_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v64i1_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_vgtuw_or">;
 
 def int_hexagon_V6_vgtuw_or_128B :
-Hexagon_custom_v128i1_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v128i1_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vgtuw_or_128B">;
 
 def int_hexagon_V6_vgtuw_xor :
-Hexagon_custom_v64i1_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v64i1_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_vgtuw_xor">;
 
 def int_hexagon_V6_vgtuw_xor_128B :
-Hexagon_custom_v128i1_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v128i1_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vgtuw_xor_128B">;
 
 def int_hexagon_V6_vgtw :
-Hexagon_custom_v64i1_v16i32v16i32_Intrinsic;
+Hexagon_v64i1_v16i32v16i32_Intrinsic<"HEXAGON_V6_vgtw">;
 
 def int_hexagon_V6_vgtw_128B :
-Hexagon_custom_v128i1_v32i32v32i32_Intrinsic_128B;
+Hexagon_v128i1_v32i32v32i32_Intrinsic<"HEXAGON_V6_vgtw_128B">;
 
 def int_hexagon_V6_vgtw_and :
-Hexagon_custom_v64i1_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v64i1_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_vgtw_and">;
 
 def int_hexagon_V6_vgtw_and_128B :
-Hexagon_custom_v128i1_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v128i1_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vgtw_and_128B">;
 
 def int_hexagon_V6_vgtw_or :
-Hexagon_custom_v64i1_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v64i1_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_vgtw_or">;
 
 def int_hexagon_V6_vgtw_or_128B :
-Hexagon_custom_v128i1_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v128i1_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vgtw_or_128B">;
 
 def int_hexagon_V6_vgtw_xor :
-Hexagon_custom_v64i1_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v64i1_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_vgtw_xor">;
 
 def int_hexagon_V6_vgtw_xor_128B :
-Hexagon_custom_v128i1_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v128i1_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vgtw_xor_128B">;
 
 def int_hexagon_V6_vinsertwr :
 Hexagon_v16i32_v16i32i32_Intrinsic<"HEXAGON_V6_vinsertwr">;
@@ -5051,10 +5030,10 @@ def int_hexagon_V6_vmpyuhv_acc_128B :
 Hexagon_v64i32_v64i32v32i32v32i32_Intrinsic<"HEXAGON_V6_vmpyuhv_acc_128B">;
 
 def int_hexagon_V6_vmux :
-Hexagon_custom_v16i32_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v16i32_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_vmux">;
 
 def int_hexagon_V6_vmux_128B :
-Hexagon_custom_v32i32_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v32i32_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vmux_128B">;
 
 def int_hexagon_V6_vnavgh :
 Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vnavgh">;
@@ -5375,16 +5354,16 @@ def int_hexagon_V6_vsubb_dv_128B :
 Hexagon_v64i32_v64i32v64i32_Intrinsic<"HEXAGON_V6_vsubb_dv_128B">;
 
 def int_hexagon_V6_vsubbnq :
-Hexagon_custom_v16i32_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v16i32_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_vsubbnq">;
 
 def int_hexagon_V6_vsubbnq_128B :
-Hexagon_custom_v32i32_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v32i32_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vsubbnq_128B">;
 
 def int_hexagon_V6_vsubbq :
-Hexagon_custom_v16i32_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v16i32_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_vsubbq">;
 
 def int_hexagon_V6_vsubbq_128B :
-Hexagon_custom_v32i32_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v32i32_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vsubbq_128B">;
 
 def int_hexagon_V6_vsubh :
 Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vsubh">;
@@ -5399,16 +5378,16 @@ def int_hexagon_V6_vsubh_dv_128B :
 Hexagon_v64i32_v64i32v64i32_Intrinsic<"HEXAGON_V6_vsubh_dv_128B">;
 
 def int_hexagon_V6_vsubhnq :
-Hexagon_custom_v16i32_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v16i32_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_vsubhnq">;
 
 def int_hexagon_V6_vsubhnq_128B :
-Hexagon_custom_v32i32_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v32i32_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vsubhnq_128B">;
 
 def int_hexagon_V6_vsubhq :
-Hexagon_custom_v16i32_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v16i32_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_vsubhq">;
 
 def int_hexagon_V6_vsubhq_128B :
-Hexagon_custom_v32i32_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v32i32_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vsubhq_128B">;
 
 def int_hexagon_V6_vsubhsat :
 Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vsubhsat">;
@@ -5477,16 +5456,16 @@ def int_hexagon_V6_vsubw_dv_128B :
 Hexagon_v64i32_v64i32v64i32_Intrinsic<"HEXAGON_V6_vsubw_dv_128B">;
 
 def int_hexagon_V6_vsubwnq :
-Hexagon_custom_v16i32_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v16i32_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_vsubwnq">;
 
 def int_hexagon_V6_vsubwnq_128B :
-Hexagon_custom_v32i32_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v32i32_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vsubwnq_128B">;
 
 def int_hexagon_V6_vsubwq :
-Hexagon_custom_v16i32_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v16i32_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_vsubwq">;
 
 def int_hexagon_V6_vsubwq_128B :
-Hexagon_custom_v32i32_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v32i32_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vsubwq_128B">;
 
 def int_hexagon_V6_vsubwsat :
 Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vsubwsat">;
@@ -5501,10 +5480,10 @@ def int_hexagon_V6_vsubwsat_dv_128B :
 Hexagon_v64i32_v64i32v64i32_Intrinsic<"HEXAGON_V6_vsubwsat_dv_128B">;
 
 def int_hexagon_V6_vswap :
-Hexagon_custom_v32i32_v64i1v16i32v16i32_Intrinsic;
+Hexagon_v32i32_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_vswap">;
 
 def int_hexagon_V6_vswap_128B :
-Hexagon_custom_v64i32_v128i1v32i32v32i32_Intrinsic_128B;
+Hexagon_v64i32_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vswap_128B">;
 
 def int_hexagon_V6_vtmpyb :
 Hexagon_v32i32_v32i32i32_Intrinsic<"HEXAGON_V6_vtmpyb">;
@@ -5611,22 +5590,22 @@ def int_hexagon_V6_lvsplath_128B :
 Hexagon_v32i32_i32_Intrinsic<"HEXAGON_V6_lvsplath_128B">;
 
 def int_hexagon_V6_pred_scalar2v2 :
-Hexagon_custom_v64i1_i32_Intrinsic;
+Hexagon_v64i1_i32_Intrinsic<"HEXAGON_V6_pred_scalar2v2">;
 
 def int_hexagon_V6_pred_scalar2v2_128B :
-Hexagon_custom_v128i1_i32_Intrinsic_128B;
+Hexagon_v128i1_i32_Intrinsic<"HEXAGON_V6_pred_scalar2v2_128B">;
 
 def int_hexagon_V6_shuffeqh :
-Hexagon_custom_v64i1_v64i1v64i1_Intrinsic;
+Hexagon_v64i1_v64i1v64i1_Intrinsic<"HEXAGON_V6_shuffeqh">;
 
 def int_hexagon_V6_shuffeqh_128B :
-Hexagon_custom_v128i1_v128i1v128i1_Intrinsic_128B;
+Hexagon_v128i1_v128i1v128i1_Intrinsic<"HEXAGON_V6_shuffeqh_128B">;
 
 def int_hexagon_V6_shuffeqw :
-Hexagon_custom_v64i1_v64i1v64i1_Intrinsic;
+Hexagon_v64i1_v64i1v64i1_Intrinsic<"HEXAGON_V6_shuffeqw">;
 
 def int_hexagon_V6_shuffeqw_128B :
-Hexagon_custom_v128i1_v128i1v128i1_Intrinsic_128B;
+Hexagon_v128i1_v128i1v128i1_Intrinsic<"HEXAGON_V6_shuffeqw_128B">;
 
 def int_hexagon_V6_vaddbsat :
 Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vaddbsat">;
@@ -5695,28 +5674,28 @@ def int_hexagon_V6_vadduwsat_dv_128B :
 Hexagon_v64i32_v64i32v64i32_Intrinsic<"HEXAGON_V6_vadduwsat_dv_128B">;
 
 def int_hexagon_V6_vandnqrt :
-Hexagon_custom_v16i32_v64i1i32_Intrinsic;
+Hexagon_v16i32_v64i1i32_Intrinsic<"HEXAGON_V6_vandnqrt">;
 
 def int_hexagon_V6_vandnqrt_128B :
-Hexagon_custom_v32i32_v128i1i32_Intrinsic_128B;
+Hexagon_v32i32_v128i1i32_Intrinsic<"HEXAGON_V6_vandnqrt_128B">;
 
 def int_hexagon_V6_vandnqrt_acc :
-Hexagon_custom_v16i32_v16i32v64i1i32_Intrinsic;
+Hexagon_v16i32_v16i32v64i1i32_Intrinsic<"HEXAGON_V6_vandnqrt_acc">;
 
 def int_hexagon_V6_vandnqrt_acc_128B :
-Hexagon_custom_v32i32_v32i32v128i1i32_Intrinsic_128B;
+Hexagon_v32i32_v32i32v128i1i32_Intrinsic<"HEXAGON_V6_vandnqrt_acc_128B">;
 
 def int_hexagon_V6_vandvnqv :
-Hexagon_custom_v16i32_v64i1v16i32_Intrinsic;
+Hexagon_v16i32_v64i1v16i32_Intrinsic<"HEXAGON_V6_vandvnqv">;
 
 def int_hexagon_V6_vandvnqv_128B :
-Hexagon_custom_v32i32_v128i1v32i32_Intrinsic_128B;
+Hexagon_v32i32_v128i1v32i32_Intrinsic<"HEXAGON_V6_vandvnqv_128B">;
 
 def int_hexagon_V6_vandvqv :
-Hexagon_custom_v16i32_v64i1v16i32_Intrinsic;
+Hexagon_v16i32_v64i1v16i32_Intrinsic<"HEXAGON_V6_vandvqv">;
 
 def int_hexagon_V6_vandvqv_128B :
-Hexagon_custom_v32i32_v128i1v32i32_Intrinsic_128B;
+Hexagon_v32i32_v128i1v32i32_Intrinsic<"HEXAGON_V6_vandvqv_128B">;
 
 def int_hexagon_V6_vasrhbsat :
 Hexagon_v16i32_v16i32v16i32i32_Intrinsic<"HEXAGON_V6_vasrhbsat">;
@@ -5961,10 +5940,10 @@ def int_hexagon_V6_vgathermh_128B :
 Hexagon__ptri32i32v32i32_Intrinsic<"HEXAGON_V6_vgathermh_128B", [IntrArgMemOnly]>;
 
 def int_hexagon_V6_vgathermhq :
-Hexagon_custom__ptrv64i1i32i32v16i32_Intrinsic<[IntrArgMemOnly]>;
+Hexagon__ptrv64i1i32i32v16i32_Intrinsic<"HEXAGON_V6_vgathermhq", [IntrArgMemOnly]>;
 
 def int_hexagon_V6_vgathermhq_128B :
-Hexagon_custom__ptrv128i1i32i32v32i32_Intrinsic_128B<[IntrArgMemOnly]>;
+Hexagon__ptrv128i1i32i32v32i32_Intrinsic<"HEXAGON_V6_vgathermhq_128B", [IntrArgMemOnly]>;
 
 def int_hexagon_V6_vgathermhw :
 Hexagon__ptri32i32v32i32_Intrinsic<"HEXAGON_V6_vgathermhw", [IntrArgMemOnly]>;
@@ -5973,10 +5952,10 @@ def int_hexagon_V6_vgathermhw_128B :
 Hexagon__ptri32i32v64i32_Intrinsic<"HEXAGON_V6_vgathermhw_128B", [IntrArgMemOnly]>;
 
 def int_hexagon_V6_vgathermhwq :
-Hexagon_custom__ptrv64i1i32i32v32i32_Intrinsic<[IntrArgMemOnly]>;
+Hexagon__ptrv64i1i32i32v32i32_Intrinsic<"HEXAGON_V6_vgathermhwq", [IntrArgMemOnly]>;
 
 def int_hexagon_V6_vgathermhwq_128B :
-Hexagon_custom__ptrv128i1i32i32v64i32_Intrinsic_128B<[IntrArgMemOnly]>;
+Hexagon__ptrv128i1i32i32v64i32_Intrinsic<"HEXAGON_V6_vgathermhwq_128B", [IntrArgMemOnly]>;
 
 def int_hexagon_V6_vgathermw :
 Hexagon__ptri32i32v16i32_Intrinsic<"HEXAGON_V6_vgathermw", [IntrArgMemOnly]>;
@@ -5985,10 +5964,10 @@ def int_hexagon_V6_vgathermw_128B :
 Hexagon__ptri32i32v32i32_Intrinsic<"HEXAGON_V6_vgathermw_128B", [IntrArgMemOnly]>;
 
 def int_hexagon_V6_vgathermwq :
-Hexagon_custom__ptrv64i1i32i32v16i32_Intrinsic<[IntrArgMemOnly]>;
+Hexagon__ptrv64i1i32i32v16i32_Intrinsic<"HEXAGON_V6_vgathermwq", [IntrArgMemOnly]>;
 
 def int_hexagon_V6_vgathermwq_128B :
-Hexagon_custom__ptrv128i1i32i32v32i32_Intrinsic_128B<[IntrArgMemOnly]>;
+Hexagon__ptrv128i1i32i32v32i32_Intrinsic<"HEXAGON_V6_vgathermwq_128B", [IntrArgMemOnly]>;
 
 def int_hexagon_V6_vlut4 :
 Hexagon_v16i32_v16i32i64_Intrinsic<"HEXAGON_V6_vlut4">;
@@ -6051,22 +6030,22 @@ def int_hexagon_V6_vnavgb_128B :
 Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vnavgb_128B">;
 
 def int_hexagon_V6_vprefixqb :
-Hexagon_custom_v16i32_v64i1_Intrinsic;
+Hexagon_v16i32_v64i1_Intrinsic<"HEXAGON_V6_vprefixqb">;
 
 def int_hexagon_V6_vprefixqb_128B :
-Hexagon_custom_v32i32_v128i1_Intrinsic_128B;
+Hexagon_v32i32_v128i1_Intrinsic<"HEXAGON_V6_vprefixqb_128B">;
 
 def int_hexagon_V6_vprefixqh :
-Hexagon_custom_v16i32_v64i1_Intrinsic;
+Hexagon_v16i32_v64i1_Intrinsic<"HEXAGON_V6_vprefixqh">;
 
 def int_hexagon_V6_vprefixqh_128B :
-Hexagon_custom_v32i32_v128i1_Intrinsic_128B;
+Hexagon_v32i32_v128i1_Intrinsic<"HEXAGON_V6_vprefixqh_128B">;
 
 def int_hexagon_V6_vprefixqw :
-Hexagon_custom_v16i32_v64i1_Intrinsic;
+Hexagon_v16i32_v64i1_Intrinsic<"HEXAGON_V6_vprefixqw">;
 
 def int_hexagon_V6_vprefixqw_128B :
-Hexagon_custom_v32i32_v128i1_Intrinsic_128B;
+Hexagon_v32i32_v128i1_Intrinsic<"HEXAGON_V6_vprefixqw_128B">;
 
 def int_hexagon_V6_vscattermh :
 Hexagon__i32i32v16i32v16i32_Intrinsic<"HEXAGON_V6_vscattermh", [IntrWriteMem]>;
@@ -6081,10 +6060,10 @@ def int_hexagon_V6_vscattermh_add_128B :
 Hexagon__i32i32v32i32v32i32_Intrinsic<"HEXAGON_V6_vscattermh_add_128B", [IntrWriteMem]>;
 
 def int_hexagon_V6_vscattermhq :
-Hexagon_custom__v64i1i32i32v16i32v16i32_Intrinsic<[IntrWriteMem]>;
+Hexagon__v64i1i32i32v16i32v16i32_Intrinsic<"HEXAGON_V6_vscattermhq", [IntrWriteMem]>;
 
 def int_hexagon_V6_vscattermhq_128B :
-Hexagon_custom__v128i1i32i32v32i32v32i32_Intrinsic_128B<[IntrWriteMem]>;
+Hexagon__v128i1i32i32v32i32v32i32_Intrinsic<"HEXAGON_V6_vscattermhq_128B", [IntrWriteMem]>;
 
 def int_hexagon_V6_vscattermhw :
 Hexagon__i32i32v32i32v16i32_Intrinsic<"HEXAGON_V6_vscattermhw", [IntrWriteMem]>;
@@ -6099,10 +6078,10 @@ def int_hexagon_V6_vscattermhw_add_128B :
 Hexagon__i32i32v64i32v32i32_Intrinsic<"HEXAGON_V6_vscattermhw_add_128B", [IntrWriteMem]>;
 
 def int_hexagon_V6_vscattermhwq :
-Hexagon_custom__v64i1i32i32v32i32v16i32_Intrinsic<[IntrWriteMem]>;
+Hexagon__v64i1i32i32v32i32v16i32_Intrinsic<"HEXAGON_V6_vscattermhwq", [IntrWriteMem]>;
 
 def int_hexagon_V6_vscattermhwq_128B :
-Hexagon_custom__v128i1i32i32v64i32v32i32_Intrinsic_128B<[IntrWriteMem]>;
+Hexagon__v128i1i32i32v64i32v32i32_Intrinsic<"HEXAGON_V6_vscattermhwq_128B", [IntrWriteMem]>;
 
 def int_hexagon_V6_vscattermw :
 Hexagon__i32i32v16i32v16i32_Intrinsic<"HEXAGON_V6_vscattermw", [IntrWriteMem]>;
@@ -6117,18 +6096,18 @@ def int_hexagon_V6_vscattermw_add_128B :
 Hexagon__i32i32v32i32v32i32_Intrinsic<"HEXAGON_V6_vscattermw_add_128B", [IntrWriteMem]>;
 
 def int_hexagon_V6_vscattermwq :
-Hexagon_custom__v64i1i32i32v16i32v16i32_Intrinsic<[IntrWriteMem]>;
+Hexagon__v64i1i32i32v16i32v16i32_Intrinsic<"HEXAGON_V6_vscattermwq", [IntrWriteMem]>;
 
 def int_hexagon_V6_vscattermwq_128B :
-Hexagon_custom__v128i1i32i32v32i32v32i32_Intrinsic_128B<[IntrWriteMem]>;
+Hexagon__v128i1i32i32v32i32v32i32_Intrinsic<"HEXAGON_V6_vscattermwq_128B", [IntrWriteMem]>;
 
 // V66 HVX Instructions.
 
 def int_hexagon_V6_vaddcarrysat :
-Hexagon_custom_v16i32_v16i32v16i32v64i1_Intrinsic;
+Hexagon_v16i32_v16i32v16i32v64i1_Intrinsic<"HEXAGON_V6_vaddcarrysat">;
 
 def int_hexagon_V6_vaddcarrysat_128B :
-Hexagon_custom_v32i32_v32i32v32i32v128i1_Intrinsic_128B;
+Hexagon_v32i32_v32i32v32i32v128i1_Intrinsic<"HEXAGON_V6_vaddcarrysat_128B">;
 
 def int_hexagon_V6_vasr_into :
 Hexagon_v32i32_v32i32v16i32v16i32_Intrinsic<"HEXAGON_V6_vasr_into">;
@@ -6174,3 +6153,437 @@ Hexagon_v32i32_v32i32v32i32v32i32i32_Intrinsic<"HEXAGON_V6_v6mpyvubs10_vxx", [In
 def int_hexagon_V6_v6mpyvubs10_vxx_128B :
 Hexagon_v64i32_v64i32v64i32v64i32i32_Intrinsic<"HEXAGON_V6_v6mpyvubs10_vxx_128B", [IntrNoMem, ImmArg<ArgIndex<3>>]>;
 
+def int_hexagon_V6_vabs_hf :
+Hexagon_v16i32_v16i32_Intrinsic<"HEXAGON_V6_vabs_hf">;
+
+def int_hexagon_V6_vabs_hf_128B :
+Hexagon_v32i32_v32i32_Intrinsic<"HEXAGON_V6_vabs_hf_128B">;
+
+def int_hexagon_V6_vabs_sf :
+Hexagon_v16i32_v16i32_Intrinsic<"HEXAGON_V6_vabs_sf">;
+
+def int_hexagon_V6_vabs_sf_128B :
+Hexagon_v32i32_v32i32_Intrinsic<"HEXAGON_V6_vabs_sf_128B">;
+
+def int_hexagon_V6_vadd_hf :
+Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vadd_hf">;
+
+def int_hexagon_V6_vadd_hf_128B :
+Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vadd_hf_128B">;
+
+def int_hexagon_V6_vadd_hf_hf :
+Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vadd_hf_hf">;
+
+def int_hexagon_V6_vadd_hf_hf_128B :
+Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vadd_hf_hf_128B">;
+
+def int_hexagon_V6_vadd_qf16 :
+Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vadd_qf16">;
+
+def int_hexagon_V6_vadd_qf16_128B :
+Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vadd_qf16_128B">;
+
+def int_hexagon_V6_vadd_qf16_mix :
+Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vadd_qf16_mix">;
+
+def int_hexagon_V6_vadd_qf16_mix_128B :
+Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vadd_qf16_mix_128B">;
+
+def int_hexagon_V6_vadd_qf32 :
+Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vadd_qf32">;
+
+def int_hexagon_V6_vadd_qf32_128B :
+Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vadd_qf32_128B">;
+
+def int_hexagon_V6_vadd_qf32_mix :
+Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vadd_qf32_mix">;
+
+def int_hexagon_V6_vadd_qf32_mix_128B :
+Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vadd_qf32_mix_128B">;
+
+def int_hexagon_V6_vadd_sf :
+Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vadd_sf">;
+
+def int_hexagon_V6_vadd_sf_128B :
+Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vadd_sf_128B">;
+
+def int_hexagon_V6_vadd_sf_hf :
+Hexagon_v32i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vadd_sf_hf">;
+
+def int_hexagon_V6_vadd_sf_hf_128B :
+Hexagon_v64i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vadd_sf_hf_128B">;
+
+def int_hexagon_V6_vadd_sf_sf :
+Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vadd_sf_sf">;
+
+def int_hexagon_V6_vadd_sf_sf_128B :
+Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vadd_sf_sf_128B">;
+
+def int_hexagon_V6_vassign_fp :
+Hexagon_v16i32_v16i32_Intrinsic<"HEXAGON_V6_vassign_fp">;
+
+def int_hexagon_V6_vassign_fp_128B :
+Hexagon_v32i32_v32i32_Intrinsic<"HEXAGON_V6_vassign_fp_128B">;
+
+def int_hexagon_V6_vconv_hf_qf16 :
+Hexagon_v16i32_v16i32_Intrinsic<"HEXAGON_V6_vconv_hf_qf16">;
+
+def int_hexagon_V6_vconv_hf_qf16_128B :
+Hexagon_v32i32_v32i32_Intrinsic<"HEXAGON_V6_vconv_hf_qf16_128B">;
+
+def int_hexagon_V6_vconv_hf_qf32 :
+Hexagon_v16i32_v32i32_Intrinsic<"HEXAGON_V6_vconv_hf_qf32">;
+
+def int_hexagon_V6_vconv_hf_qf32_128B :
+Hexagon_v32i32_v64i32_Intrinsic<"HEXAGON_V6_vconv_hf_qf32_128B">;
+
+def int_hexagon_V6_vconv_sf_qf32 :
+Hexagon_v16i32_v16i32_Intrinsic<"HEXAGON_V6_vconv_sf_qf32">;
+
+def int_hexagon_V6_vconv_sf_qf32_128B :
+Hexagon_v32i32_v32i32_Intrinsic<"HEXAGON_V6_vconv_sf_qf32_128B">;
+
+def int_hexagon_V6_vcvt_b_hf :
+Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vcvt_b_hf">;
+
+def int_hexagon_V6_vcvt_b_hf_128B :
+Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vcvt_b_hf_128B">;
+
+def int_hexagon_V6_vcvt_h_hf :
+Hexagon_v16i32_v16i32_Intrinsic<"HEXAGON_V6_vcvt_h_hf">;
+
+def int_hexagon_V6_vcvt_h_hf_128B :
+Hexagon_v32i32_v32i32_Intrinsic<"HEXAGON_V6_vcvt_h_hf_128B">;
+
+def int_hexagon_V6_vcvt_hf_b :
+Hexagon_v32i32_v16i32_Intrinsic<"HEXAGON_V6_vcvt_hf_b">;
+
+def int_hexagon_V6_vcvt_hf_b_128B :
+Hexagon_v64i32_v32i32_Intrinsic<"HEXAGON_V6_vcvt_hf_b_128B">;
+
+def int_hexagon_V6_vcvt_hf_h :
+Hexagon_v16i32_v16i32_Intrinsic<"HEXAGON_V6_vcvt_hf_h">;
+
+def int_hexagon_V6_vcvt_hf_h_128B :
+Hexagon_v32i32_v32i32_Intrinsic<"HEXAGON_V6_vcvt_hf_h_128B">;
+
+def int_hexagon_V6_vcvt_hf_sf :
+Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vcvt_hf_sf">;
+
+def int_hexagon_V6_vcvt_hf_sf_128B :
+Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vcvt_hf_sf_128B">;
+
+def int_hexagon_V6_vcvt_hf_ub :
+Hexagon_v32i32_v16i32_Intrinsic<"HEXAGON_V6_vcvt_hf_ub">;
+
+def int_hexagon_V6_vcvt_hf_ub_128B :
+Hexagon_v64i32_v32i32_Intrinsic<"HEXAGON_V6_vcvt_hf_ub_128B">;
+
+def int_hexagon_V6_vcvt_hf_uh :
+Hexagon_v16i32_v16i32_Intrinsic<"HEXAGON_V6_vcvt_hf_uh">;
+
+def int_hexagon_V6_vcvt_hf_uh_128B :
+Hexagon_v32i32_v32i32_Intrinsic<"HEXAGON_V6_vcvt_hf_uh_128B">;
+
+def int_hexagon_V6_vcvt_sf_hf :
+Hexagon_v32i32_v16i32_Intrinsic<"HEXAGON_V6_vcvt_sf_hf">;
+
+def int_hexagon_V6_vcvt_sf_hf_128B :
+Hexagon_v64i32_v32i32_Intrinsic<"HEXAGON_V6_vcvt_sf_hf_128B">;
+
+def int_hexagon_V6_vcvt_ub_hf :
+Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vcvt_ub_hf">;
+
+def int_hexagon_V6_vcvt_ub_hf_128B :
+Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vcvt_ub_hf_128B">;
+
+def int_hexagon_V6_vcvt_uh_hf :
+Hexagon_v16i32_v16i32_Intrinsic<"HEXAGON_V6_vcvt_uh_hf">;
+
+def int_hexagon_V6_vcvt_uh_hf_128B :
+Hexagon_v32i32_v32i32_Intrinsic<"HEXAGON_V6_vcvt_uh_hf_128B">;
+
+def int_hexagon_V6_vdmpy_sf_hf :
+Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vdmpy_sf_hf">;
+
+def int_hexagon_V6_vdmpy_sf_hf_128B :
+Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vdmpy_sf_hf_128B">;
+
+def int_hexagon_V6_vdmpy_sf_hf_acc :
+Hexagon_v16i32_v16i32v16i32v16i32_Intrinsic<"HEXAGON_V6_vdmpy_sf_hf_acc">;
+
+def int_hexagon_V6_vdmpy_sf_hf_acc_128B :
+Hexagon_v32i32_v32i32v32i32v32i32_Intrinsic<"HEXAGON_V6_vdmpy_sf_hf_acc_128B">;
+
+def int_hexagon_V6_vfmax_hf :
+Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vfmax_hf">;
+
+def int_hexagon_V6_vfmax_hf_128B :
+Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vfmax_hf_128B">;
+
+def int_hexagon_V6_vfmax_sf :
+Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vfmax_sf">;
+
+def int_hexagon_V6_vfmax_sf_128B :
+Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vfmax_sf_128B">;
+
+def int_hexagon_V6_vfmin_hf :
+Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vfmin_hf">;
+
+def int_hexagon_V6_vfmin_hf_128B :
+Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vfmin_hf_128B">;
+
+def int_hexagon_V6_vfmin_sf :
+Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vfmin_sf">;
+
+def int_hexagon_V6_vfmin_sf_128B :
+Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vfmin_sf_128B">;
+
+def int_hexagon_V6_vfneg_hf :
+Hexagon_v16i32_v16i32_Intrinsic<"HEXAGON_V6_vfneg_hf">;
+
+def int_hexagon_V6_vfneg_hf_128B :
+Hexagon_v32i32_v32i32_Intrinsic<"HEXAGON_V6_vfneg_hf_128B">;
+
+def int_hexagon_V6_vfneg_sf :
+Hexagon_v16i32_v16i32_Intrinsic<"HEXAGON_V6_vfneg_sf">;
+
+def int_hexagon_V6_vfneg_sf_128B :
+Hexagon_v32i32_v32i32_Intrinsic<"HEXAGON_V6_vfneg_sf_128B">;
+
+def int_hexagon_V6_vgthf :
+Hexagon_v64i1_v16i32v16i32_Intrinsic<"HEXAGON_V6_vgthf">;
+
+def int_hexagon_V6_vgthf_128B :
+Hexagon_v128i1_v32i32v32i32_Intrinsic<"HEXAGON_V6_vgthf_128B">;
+
+def int_hexagon_V6_vgthf_and :
+Hexagon_v64i1_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_vgthf_and">;
+
+def int_hexagon_V6_vgthf_and_128B :
+Hexagon_v128i1_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vgthf_and_128B">;
+
+def int_hexagon_V6_vgthf_or :
+Hexagon_v64i1_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_vgthf_or">;
+
+def int_hexagon_V6_vgthf_or_128B :
+Hexagon_v128i1_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vgthf_or_128B">;
+
+def int_hexagon_V6_vgthf_xor :
+Hexagon_v64i1_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_vgthf_xor">;
+
+def int_hexagon_V6_vgthf_xor_128B :
+Hexagon_v128i1_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vgthf_xor_128B">;
+
+def int_hexagon_V6_vgtsf :
+Hexagon_v64i1_v16i32v16i32_Intrinsic<"HEXAGON_V6_vgtsf">;
+
+def int_hexagon_V6_vgtsf_128B :
+Hexagon_v128i1_v32i32v32i32_Intrinsic<"HEXAGON_V6_vgtsf_128B">;
+
+def int_hexagon_V6_vgtsf_and :
+Hexagon_v64i1_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_vgtsf_and">;
+
+def int_hexagon_V6_vgtsf_and_128B :
+Hexagon_v128i1_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vgtsf_and_128B">;
+
+def int_hexagon_V6_vgtsf_or :
+Hexagon_v64i1_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_vgtsf_or">;
+
+def int_hexagon_V6_vgtsf_or_128B :
+Hexagon_v128i1_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vgtsf_or_128B">;
+
+def int_hexagon_V6_vgtsf_xor :
+Hexagon_v64i1_v64i1v16i32v16i32_Intrinsic<"HEXAGON_V6_vgtsf_xor">;
+
+def int_hexagon_V6_vgtsf_xor_128B :
+Hexagon_v128i1_v128i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vgtsf_xor_128B">;
+
+def int_hexagon_V6_vmax_hf :
+Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vmax_hf">;
+
+def int_hexagon_V6_vmax_hf_128B :
+Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vmax_hf_128B">;
+
+def int_hexagon_V6_vmax_sf :
+Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vmax_sf">;
+
+def int_hexagon_V6_vmax_sf_128B :
+Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vmax_sf_128B">;
+
+def int_hexagon_V6_vmin_hf :
+Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vmin_hf">;
+
+def int_hexagon_V6_vmin_hf_128B :
+Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vmin_hf_128B">;
+
+def int_hexagon_V6_vmin_sf :
+Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vmin_sf">;
+
+def int_hexagon_V6_vmin_sf_128B :
+Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vmin_sf_128B">;
+
+def int_hexagon_V6_vmpy_hf_hf :
+Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vmpy_hf_hf">;
+
+def int_hexagon_V6_vmpy_hf_hf_128B :
+Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vmpy_hf_hf_128B">;
+
+def int_hexagon_V6_vmpy_hf_hf_acc :
+Hexagon_v16i32_v16i32v16i32v16i32_Intrinsic<"HEXAGON_V6_vmpy_hf_hf_acc">;
+
+def int_hexagon_V6_vmpy_hf_hf_acc_128B :
+Hexagon_v32i32_v32i32v32i32v32i32_Intrinsic<"HEXAGON_V6_vmpy_hf_hf_acc_128B">;
+
+def int_hexagon_V6_vmpy_qf16 :
+Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vmpy_qf16">;
+
+def int_hexagon_V6_vmpy_qf16_128B :
+Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vmpy_qf16_128B">;
+
+def int_hexagon_V6_vmpy_qf16_hf :
+Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vmpy_qf16_hf">;
+
+def int_hexagon_V6_vmpy_qf16_hf_128B :
+Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vmpy_qf16_hf_128B">;
+
+def int_hexagon_V6_vmpy_qf16_mix_hf :
+Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vmpy_qf16_mix_hf">;
+
+def int_hexagon_V6_vmpy_qf16_mix_hf_128B :
+Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vmpy_qf16_mix_hf_128B">;
+
+def int_hexagon_V6_vmpy_qf32 :
+Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vmpy_qf32">;
+
+def int_hexagon_V6_vmpy_qf32_128B :
+Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vmpy_qf32_128B">;
+
+def int_hexagon_V6_vmpy_qf32_hf :
+Hexagon_v32i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vmpy_qf32_hf">;
+
+def int_hexagon_V6_vmpy_qf32_hf_128B :
+Hexagon_v64i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vmpy_qf32_hf_128B">;
+
+def int_hexagon_V6_vmpy_qf32_mix_hf :
+Hexagon_v32i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vmpy_qf32_mix_hf">;
+
+def int_hexagon_V6_vmpy_qf32_mix_hf_128B :
+Hexagon_v64i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vmpy_qf32_mix_hf_128B">;
+
+def int_hexagon_V6_vmpy_qf32_qf16 :
+Hexagon_v32i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vmpy_qf32_qf16">;
+
+def int_hexagon_V6_vmpy_qf32_qf16_128B :
+Hexagon_v64i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vmpy_qf32_qf16_128B">;
+
+def int_hexagon_V6_vmpy_qf32_sf :
+Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vmpy_qf32_sf">;
+
+def int_hexagon_V6_vmpy_qf32_sf_128B :
+Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vmpy_qf32_sf_128B">;
+
+def int_hexagon_V6_vmpy_sf_hf :
+Hexagon_v32i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vmpy_sf_hf">;
+
+def int_hexagon_V6_vmpy_sf_hf_128B :
+Hexagon_v64i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vmpy_sf_hf_128B">;
+
+def int_hexagon_V6_vmpy_sf_hf_acc :
+Hexagon_v32i32_v32i32v16i32v16i32_Intrinsic<"HEXAGON_V6_vmpy_sf_hf_acc">;
+
+def int_hexagon_V6_vmpy_sf_hf_acc_128B :
+Hexagon_v64i32_v64i32v32i32v32i32_Intrinsic<"HEXAGON_V6_vmpy_sf_hf_acc_128B">;
+
+def int_hexagon_V6_vmpy_sf_sf :
+Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vmpy_sf_sf">;
+
+def int_hexagon_V6_vmpy_sf_sf_128B :
+Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vmpy_sf_sf_128B">;
+
+def int_hexagon_V6_vsub_hf :
+Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vsub_hf">;
+
+def int_hexagon_V6_vsub_hf_128B :
+Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vsub_hf_128B">;
+
+def int_hexagon_V6_vsub_hf_hf :
+Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vsub_hf_hf">;
+
+def int_hexagon_V6_vsub_hf_hf_128B :
+Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vsub_hf_hf_128B">;
+
+def int_hexagon_V6_vsub_qf16 :
+Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vsub_qf16">;
+
+def int_hexagon_V6_vsub_qf16_128B :
+Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vsub_qf16_128B">;
+
+def int_hexagon_V6_vsub_qf16_mix :
+Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vsub_qf16_mix">;
+
+def int_hexagon_V6_vsub_qf16_mix_128B :
+Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vsub_qf16_mix_128B">;
+
+def int_hexagon_V6_vsub_qf32 :
+Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vsub_qf32">;
+
+def int_hexagon_V6_vsub_qf32_128B :
+Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vsub_qf32_128B">;
+
+def int_hexagon_V6_vsub_qf32_mix :
+Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vsub_qf32_mix">;
+
+def int_hexagon_V6_vsub_qf32_mix_128B :
+Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vsub_qf32_mix_128B">;
+
+def int_hexagon_V6_vsub_sf :
+Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vsub_sf">;
+
+def int_hexagon_V6_vsub_sf_128B :
+Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vsub_sf_128B">;
+
+def int_hexagon_V6_vsub_sf_hf :
+Hexagon_v32i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vsub_sf_hf">;
+
+def int_hexagon_V6_vsub_sf_hf_128B :
+Hexagon_v64i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vsub_sf_hf_128B">;
+
+def int_hexagon_V6_vsub_sf_sf :
+Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vsub_sf_sf">;
+
+def int_hexagon_V6_vsub_sf_sf_128B :
+Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vsub_sf_sf_128B">;
+
+// V69 HVX Instructions.
+
+def int_hexagon_V6_vasrvuhubrndsat :
+Hexagon_v16i32_v32i32v16i32_Intrinsic<"HEXAGON_V6_vasrvuhubrndsat">;
+
+def int_hexagon_V6_vasrvuhubrndsat_128B :
+Hexagon_v32i32_v64i32v32i32_Intrinsic<"HEXAGON_V6_vasrvuhubrndsat_128B">;
+
+def int_hexagon_V6_vasrvuhubsat :
+Hexagon_v16i32_v32i32v16i32_Intrinsic<"HEXAGON_V6_vasrvuhubsat">;
+
+def int_hexagon_V6_vasrvuhubsat_128B :
+Hexagon_v32i32_v64i32v32i32_Intrinsic<"HEXAGON_V6_vasrvuhubsat_128B">;
+
+def int_hexagon_V6_vasrvwuhrndsat :
+Hexagon_v16i32_v32i32v16i32_Intrinsic<"HEXAGON_V6_vasrvwuhrndsat">;
+
+def int_hexagon_V6_vasrvwuhrndsat_128B :
+Hexagon_v32i32_v64i32v32i32_Intrinsic<"HEXAGON_V6_vasrvwuhrndsat_128B">;
+
+def int_hexagon_V6_vasrvwuhsat :
+Hexagon_v16i32_v32i32v16i32_Intrinsic<"HEXAGON_V6_vasrvwuhsat">;
+
+def int_hexagon_V6_vasrvwuhsat_128B :
+Hexagon_v32i32_v64i32v32i32_Intrinsic<"HEXAGON_V6_vasrvwuhsat_128B">;
+
+def int_hexagon_V6_vmpyuhvs :
+Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vmpyuhvs">;
+
+def int_hexagon_V6_vmpyuhvs_128B :
+Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vmpyuhvs_128B">;
+
diff --git a/llvm/include/llvm/IR/IntrinsicsRISCV.td b/llvm/include/llvm/IR/IntrinsicsRISCV.td
index 3ceb347e97bf..747049b1035b 100644
--- a/llvm/include/llvm/IR/IntrinsicsRISCV.td
+++ b/llvm/include/llvm/IR/IntrinsicsRISCV.td
@@ -642,20 +642,6 @@ let TargetPrefix = "riscv" in {
                      LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty,
                      LLVMMatchType<2>],
                     [ImmArg<ArgIndex<4>>, IntrNoMem]>, RISCVVIntrinsic;
-  // For atomic operations without mask
-  // Input: (base, index, value, vl)
-  class RISCVAMONoMask
-        : Intrinsic<[llvm_anyvector_ty],
-                    [LLVMPointerType<LLVMMatchType<0>>, llvm_anyvector_ty, LLVMMatchType<0>,
-                     llvm_anyint_ty],
-                    [NoCapture<ArgIndex<0>>]>, RISCVVIntrinsic;
-  // For atomic operations with mask
-  // Input: (base, index, value, mask, vl)
-  class RISCVAMOMask
-        : Intrinsic<[llvm_anyvector_ty],
-                    [LLVMPointerType<LLVMMatchType<0>>, llvm_anyvector_ty, LLVMMatchType<0>,
-                     LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty],
-                    [NoCapture<ArgIndex<0>>]>, RISCVVIntrinsic;
 
   // For unit stride segment load
   // Input: (pointer, vl)
@@ -930,10 +916,6 @@ let TargetPrefix = "riscv" in {
     def "int_riscv_" #NAME :RISCVConversionNoMask;
     def "int_riscv_" # NAME # "_mask" : RISCVConversionMask;
   }
-  multiclass RISCVAMO {
-    def "int_riscv_" # NAME : RISCVAMONoMask;
-    def "int_riscv_" # NAME # "_mask" : RISCVAMOMask;
-  }
   multiclass RISCVUSSegLoad<int nf> {
     def "int_riscv_" # NAME : RISCVUSSegLoad<nf>;
     def "int_riscv_" # NAME # "_mask" : RISCVUSSegLoadMask<nf>;
@@ -976,16 +958,6 @@ let TargetPrefix = "riscv" in {
   def int_riscv_vlm : RISCVUSLoad;
   def int_riscv_vsm : RISCVUSStore;
 
-  defm vamoswap : RISCVAMO;
-  defm vamoadd : RISCVAMO;
-  defm vamoxor : RISCVAMO;
-  defm vamoand : RISCVAMO;
-  defm vamoor : RISCVAMO;
-  defm vamomin : RISCVAMO;
-  defm vamomax : RISCVAMO;
-  defm vamominu : RISCVAMO;
-  defm vamomaxu : RISCVAMO;
-
   defm vadd : RISCVBinaryAAX;
   defm vsub : RISCVBinaryAAX;
   defm vrsub : RISCVBinaryAAX;
diff --git a/llvm/include/llvm/IR/IntrinsicsWebAssembly.td b/llvm/include/llvm/IR/IntrinsicsWebAssembly.td
index 6a8e6c797f85..aecc3d91fae7 100644
--- a/llvm/include/llvm/IR/IntrinsicsWebAssembly.td
+++ b/llvm/include/llvm/IR/IntrinsicsWebAssembly.td
@@ -11,6 +11,9 @@
 ///
 //===----------------------------------------------------------------------===//
 
+// Type definition for a table in an intrinsic
+def llvm_table_ty : LLVMQualPointerType<llvm_i8_ty, 1>;
+
 let TargetPrefix = "wasm" in {  // All intrinsics start with "llvm.wasm.".
 
 // Query the current memory size, and increase the current memory size.
@@ -24,6 +27,35 @@ def int_wasm_memory_grow : Intrinsic<[llvm_anyint_ty],
                                      []>;
 
 //===----------------------------------------------------------------------===//
+// ref.null intrinsics
+//===----------------------------------------------------------------------===//
+def int_wasm_ref_null_extern : Intrinsic<[llvm_externref_ty], [], [IntrNoMem]>;
+def int_wasm_ref_null_func : Intrinsic<[llvm_funcref_ty], [], [IntrNoMem]>;
+
+//===----------------------------------------------------------------------===//
+// Table intrinsics
+//===----------------------------------------------------------------------===//
+// Query the current table size, and increase the current table size.
+def int_wasm_table_size : Intrinsic<[llvm_i32_ty],
+                                    [llvm_table_ty],
+                                    [IntrReadMem]>;
+def int_wasm_table_copy : Intrinsic<[],
+                                    [llvm_table_ty, llvm_table_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
+                                    []>;
+def int_wasm_table_grow_externref : Intrinsic<[llvm_i32_ty],
+                                              [llvm_table_ty, llvm_externref_ty, llvm_i32_ty],
+                                              []>;
+def int_wasm_table_grow_funcref : Intrinsic<[llvm_i32_ty],
+                                            [llvm_table_ty, llvm_funcref_ty, llvm_i32_ty],
+                                            []>;
+def int_wasm_table_fill_externref : Intrinsic<[],
+                                              [llvm_table_ty, llvm_i32_ty, llvm_externref_ty, llvm_i32_ty],
+                                              []>;
+def int_wasm_table_fill_funcref : Intrinsic<[],
+                                            [llvm_table_ty, llvm_i32_ty, llvm_funcref_ty, llvm_i32_ty],
+                                            []>;
+
+//===----------------------------------------------------------------------===//
 // Trapping float-to-int conversions
 //===----------------------------------------------------------------------===//
 
diff --git a/llvm/include/llvm/IR/Module.h b/llvm/include/llvm/IR/Module.h
index bd3a196c7181..4ddbd6ff14b3 100644
--- a/llvm/include/llvm/IR/Module.h
+++ b/llvm/include/llvm/IR/Module.h
@@ -934,6 +934,17 @@ public:
   /// Set the partial sample profile ratio in the profile summary module flag,
   /// if applicable.
   void setPartialSampleProfileRatio(const ModuleSummaryIndex &Index);
+
+  /// Get the target variant triple which is a string describing a variant of
+  /// the target host platform. For example, Mac Catalyst can be a variant
+  /// target triple for a macOS target.
+  /// @returns a string containing the target variant triple.
+  StringRef getDarwinTargetVariantTriple() const;
+
+  /// Get the target variant version build SDK version metadata.
+  ///
+  /// An empty version is returned if no such metadata is attached.
+  VersionTuple getDarwinTargetVariantSDKVersion() const;
 };
 
 /// Given "llvm.used" or "llvm.compiler.used" as a global name, collect the
diff --git a/llvm/include/llvm/IR/ModuleSummaryIndex.h b/llvm/include/llvm/IR/ModuleSummaryIndex.h
index e00b78d45c63..ec1d5ef79eed 100644
--- a/llvm/include/llvm/IR/ModuleSummaryIndex.h
+++ b/llvm/include/llvm/IR/ModuleSummaryIndex.h
@@ -581,6 +581,13 @@ public:
     // If there are calls to unknown targets (e.g. indirect)
     unsigned HasUnknownCall : 1;
 
+    // Indicate if a function must be an unreachable function.
+    //
+    // This bit is sufficient but not necessary;
+    // if this bit is on, the function must be regarded as unreachable;
+    // if this bit is off, the function might be reachable or unreachable.
+    unsigned MustBeUnreachable : 1;
+
     FFlags &operator&=(const FFlags &RHS) {
       this->ReadNone &= RHS.ReadNone;
       this->ReadOnly &= RHS.ReadOnly;
@@ -591,13 +598,15 @@ public:
       this->NoUnwind &= RHS.NoUnwind;
       this->MayThrow &= RHS.MayThrow;
       this->HasUnknownCall &= RHS.HasUnknownCall;
+      this->MustBeUnreachable &= RHS.MustBeUnreachable;
       return *this;
     }
 
     bool anyFlagSet() {
       return this->ReadNone | this->ReadOnly | this->NoRecurse |
              this->ReturnDoesNotAlias | this->NoInline | this->AlwaysInline |
-             this->NoUnwind | this->MayThrow | this->HasUnknownCall;
+             this->NoUnwind | this->MayThrow | this->HasUnknownCall |
+             this->MustBeUnreachable;
     }
 
     operator std::string() {
@@ -613,6 +622,7 @@ public:
       OS << ", noUnwind: " << this->NoUnwind;
       OS << ", mayThrow: " << this->MayThrow;
       OS << ", hasUnknownCall: " << this->HasUnknownCall;
+      OS << ", mustBeUnreachable: " << this->MustBeUnreachable;
       OS << ")";
       return OS.str();
     }
diff --git a/llvm/include/llvm/IR/SSAContext.h b/llvm/include/llvm/IR/SSAContext.h
new file mode 100644
index 000000000000..9d9290a2c1d7
--- /dev/null
+++ b/llvm/include/llvm/IR/SSAContext.h
@@ -0,0 +1,56 @@
+//===- SSAContext.h ---------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+///
+/// This file declares a specialization of the GenericSSAContext<X>
+/// class template for LLVM IR.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_IR_SSACONTEXT_H
+#define LLVM_IR_SSACONTEXT_H
+
+#include "llvm/ADT/GenericSSAContext.h"
+#include "llvm/IR/ModuleSlotTracker.h"
+#include "llvm/Support/Printable.h"
+
+#include <memory>
+
+namespace llvm {
+class BasicBlock;
+class Function;
+class Instruction;
+class Value;
+template <typename> class SmallVectorImpl;
+template <typename, bool> class DominatorTreeBase;
+
+template <> class GenericSSAContext<Function> {
+  Function *F;
+
+public:
+  using BlockT = BasicBlock;
+  using FunctionT = Function;
+  using InstructionT = Instruction;
+  using ValueRefT = Value *;
+  using DominatorTreeT = DominatorTreeBase<BlockT, false>;
+
+  static BasicBlock *getEntryBlock(Function &F);
+
+  void setFunction(Function &Fn);
+  Function *getFunction() const { return F; }
+
+  Printable print(BasicBlock *Block) const;
+  Printable print(Instruction *Inst) const;
+  Printable print(Value *Value) const;
+};
+
+using SSAContext = GenericSSAContext<Function>;
+
+} // namespace llvm
+
+#endif // LLVM_IR_SSACONTEXT_H
diff --git a/llvm/include/llvm/IR/VPIntrinsics.def b/llvm/include/llvm/IR/VPIntrinsics.def
index a3c6b4e70bf5..121c8bbc6c27 100644
--- a/llvm/include/llvm/IR/VPIntrinsics.def
+++ b/llvm/include/llvm/IR/VPIntrinsics.def
@@ -214,7 +214,7 @@ HELPER_REGISTER_BINARY_FP_VP(frem, VP_FREM, FRem)
 ///// } Floating-Point Arithmetic
 
 ///// Memory Operations {
-// llvm.vp.store(ptr,val,mask,vlen)
+// llvm.vp.store(val,ptr,mask,vlen)
 BEGIN_REGISTER_VP_INTRINSIC(vp_store, 2, 3)
 // chain = VP_STORE chain,val,base,offset,mask,evl
 BEGIN_REGISTER_VP_SDNODE(VP_STORE, 0, vp_store, 4, 5)
@@ -345,10 +345,9 @@ HELPER_REGISTER_REDUCTION_SEQ_VP(vp_reduce_fmul, VP_REDUCE_FMUL,
 ///// Shuffles {
 
 // llvm.vp.select(mask,on_true,on_false,vlen)
-BEGIN_REGISTER_VP_INTRINSIC(vp_select, 0, 3)
-// BEGIN_REGISTER_VP_SDNODE(VP_SELECT, -1, vp_select, 0, 4)
-// END_REGISTER_CASES(vp_select, VP_SELECT)
-END_REGISTER_VP_INTRINSIC(vp_select)
+BEGIN_REGISTER_VP(vp_select, 0, 3, VP_SELECT, -1)
+VP_PROPERTY_FUNCTIONAL_OPC(Select)
+END_REGISTER_VP(vp_select, VP_SELECT)
 
 BEGIN_REGISTER_VP(experimental_vp_splice, 3, 5, EXPERIMENTAL_VP_SPLICE, -1)
 END_REGISTER_VP(experimental_vp_splice, EXPERIMENTAL_VP_SPLICE)
diff --git a/llvm/include/llvm/IR/Value.def b/llvm/include/llvm/IR/Value.def
index 0a0125d319c3..97d15260f36b 100644
--- a/llvm/include/llvm/IR/Value.def
+++ b/llvm/include/llvm/IR/Value.def
@@ -80,6 +80,7 @@ HANDLE_GLOBAL_VALUE(GlobalVariable)
 HANDLE_CONSTANT(BlockAddress)
 HANDLE_CONSTANT(ConstantExpr)
 HANDLE_CONSTANT_EXCLUDE_LLVM_C_API(DSOLocalEquivalent)
+HANDLE_CONSTANT_EXCLUDE_LLVM_C_API(NoCFIValue)
 
 // ConstantAggregate.
 HANDLE_CONSTANT(ConstantArray)
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index 845d7dcdebd2..0c5ebc9a2f28 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -122,6 +122,7 @@ void initializeControlHeightReductionLegacyPassPass(PassRegistry&);
 void initializeCorrelatedValuePropagationPass(PassRegistry&);
 void initializeCostModelAnalysisPass(PassRegistry&);
 void initializeCrossDSOCFIPass(PassRegistry&);
+void initializeCycleInfoWrapperPassPass(PassRegistry &);
 void initializeDAEPass(PassRegistry&);
 void initializeDAHPass(PassRegistry&);
 void initializeDCELegacyPassPass(PassRegistry&);
@@ -164,7 +165,7 @@ void initializeFinalizeISelPass(PassRegistry&);
 void initializeFinalizeMachineBundlesPass(PassRegistry&);
 void initializeFixIrreduciblePass(PassRegistry &);
 void initializeFixupStatepointCallerSavedPass(PassRegistry&);
-void initializeFlattenCFGPassPass(PassRegistry&);
+void initializeFlattenCFGLegacyPassPass(PassRegistry &);
 void initializeFloat2IntLegacyPassPass(PassRegistry&);
 void initializeForceFunctionAttrsLegacyPassPass(PassRegistry&);
 void initializeForwardControlFlowIntegrityPass(PassRegistry&);
@@ -291,6 +292,8 @@ void initializeMachineBranchProbabilityInfoPass(PassRegistry&);
 void initializeMachineCSEPass(PassRegistry&);
 void initializeMachineCombinerPass(PassRegistry&);
 void initializeMachineCopyPropagationPass(PassRegistry&);
+void initializeMachineCycleInfoPrinterPassPass(PassRegistry &);
+void initializeMachineCycleInfoWrapperPassPass(PassRegistry &);
 void initializeMachineDominanceFrontierPass(PassRegistry&);
 void initializeMachineDominatorTreePass(PassRegistry&);
 void initializeMachineFunctionPrinterPassPass(PassRegistry&);
@@ -375,6 +378,7 @@ void initializeRAGreedyPass(PassRegistry&);
 void initializeReachingDefAnalysisPass(PassRegistry&);
 void initializeReassociateLegacyPassPass(PassRegistry&);
 void initializeRedundantDbgInstEliminationPass(PassRegistry&);
+void initializeRegAllocEvictionAdvisorAnalysisPass(PassRegistry &);
 void initializeRegAllocFastPass(PassRegistry&);
 void initializeRegBankSelectPass(PassRegistry&);
 void initializeRegToMemLegacyPass(PassRegistry&);
diff --git a/llvm/include/llvm/MC/MCAssembler.h b/llvm/include/llvm/MC/MCAssembler.h
index 1f670e3973ce..9d5cb620c9de 100644
--- a/llvm/include/llvm/MC/MCAssembler.h
+++ b/llvm/include/llvm/MC/MCAssembler.h
@@ -153,6 +153,7 @@ private:
   MCLOHContainer LOHContainer;
 
   VersionInfoType VersionInfo;
+  VersionInfoType DarwinTargetVariantVersionInfo;
 
   /// Evaluate a fixup to a relocatable expression and the value which should be
   /// placed into the fixup.
@@ -285,6 +286,21 @@ public:
     VersionInfo.SDKVersion = SDKVersion;
   }
 
+  const VersionInfoType &getDarwinTargetVariantVersionInfo() const {
+    return DarwinTargetVariantVersionInfo;
+  }
+  void setDarwinTargetVariantBuildVersion(MachO::PlatformType Platform,
+                                          unsigned Major, unsigned Minor,
+                                          unsigned Update,
+                                          VersionTuple SDKVersion) {
+    DarwinTargetVariantVersionInfo.EmitBuildVersion = true;
+    DarwinTargetVariantVersionInfo.TypeOrPlatform.Platform = Platform;
+    DarwinTargetVariantVersionInfo.Major = Major;
+    DarwinTargetVariantVersionInfo.Minor = Minor;
+    DarwinTargetVariantVersionInfo.Update = Update;
+    DarwinTargetVariantVersionInfo.SDKVersion = SDKVersion;
+  }
+
   /// Reuse an assembler instance
   ///
   void reset();
diff --git a/llvm/include/llvm/MC/MCObjectFileInfo.h b/llvm/include/llvm/MC/MCObjectFileInfo.h
index ba7450ac64f1..5e0cccaba77f 100644
--- a/llvm/include/llvm/MC/MCObjectFileInfo.h
+++ b/llvm/include/llvm/MC/MCObjectFileInfo.h
@@ -427,6 +427,8 @@ private:
   bool PositionIndependent = false;
   MCContext *Ctx = nullptr;
   VersionTuple SDKVersion;
+  Optional<Triple> DarwinTargetVariantTriple;
+  VersionTuple DarwinTargetVariantSDKVersion;
 
   void initMachOMCObjectFileInfo(const Triple &T);
   void initELFMCObjectFileInfo(const Triple &T, bool Large);
@@ -442,6 +444,23 @@ public:
   }
 
   const VersionTuple &getSDKVersion() const { return SDKVersion; }
+
+  void setDarwinTargetVariantTriple(const Triple &T) {
+    DarwinTargetVariantTriple = T;
+  }
+
+  const Triple *getDarwinTargetVariantTriple() const {
+    return DarwinTargetVariantTriple ? DarwinTargetVariantTriple.getPointer()
+                                     : nullptr;
+  }
+
+  void setDarwinTargetVariantSDKVersion(const VersionTuple &TheSDKVersion) {
+    DarwinTargetVariantSDKVersion = TheSDKVersion;
+  }
+
+  const VersionTuple &getDarwinTargetVariantSDKVersion() const {
+    return DarwinTargetVariantSDKVersion;
+  }
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/MC/MCObjectStreamer.h b/llvm/include/llvm/MC/MCObjectStreamer.h
index 9d6416e4a18d..183fd79fb9fc 100644
--- a/llvm/include/llvm/MC/MCObjectStreamer.h
+++ b/llvm/include/llvm/MC/MCObjectStreamer.h
@@ -50,6 +50,16 @@ class MCObjectStreamer : public MCStreamer {
   };
   SmallVector<PendingMCFixup, 2> PendingFixups;
 
+  struct PendingAssignment {
+    MCSymbol *Symbol;
+    const MCExpr *Value;
+  };
+
+  /// A list of conditional assignments we may need to emit if the target
+  /// symbol is later emitted.
+  DenseMap<const MCSymbol *, SmallVector<PendingAssignment, 1>>
+      pendingAssignments;
+
   virtual void emitInstToData(const MCInst &Inst, const MCSubtargetInfo&) = 0;
   void emitCFIStartProcImpl(MCDwarfFrameInfo &Frame) override;
   void emitCFIEndProcImpl(MCDwarfFrameInfo &Frame) override;
@@ -118,6 +128,8 @@ public:
   virtual void emitLabelAtPos(MCSymbol *Symbol, SMLoc Loc, MCFragment *F,
                               uint64_t Offset);
   void emitAssignment(MCSymbol *Symbol, const MCExpr *Value) override;
+  void emitConditionalAssignment(MCSymbol *Symbol,
+                                 const MCExpr *Value) override;
   void emitValueImpl(const MCExpr *Value, unsigned Size,
                      SMLoc Loc = SMLoc()) override;
   void emitULEB128Value(const MCExpr *Value) override;
@@ -208,6 +220,10 @@ public:
                                        const MCSymbol *Lo) override;
 
   bool mayHaveInstructions(MCSection &Sec) const override;
+
+  /// Emits pending conditional assignments that depend on \p Symbol
+  /// being emitted.
+  void emitPendingAssignments(MCSymbol *Symbol);
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h b/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h
index abb95628c2a9..faf0a4474c8a 100644
--- a/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h
+++ b/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h
@@ -10,6 +10,7 @@
 #define LLVM_MC_MCPARSER_MCPARSEDASMOPERAND_H
 
 #include "llvm/ADT/StringRef.h"
+#include "llvm/MC/MCInstrDesc.h"
 #include "llvm/Support/SMLoc.h"
 #include <string>
 
@@ -76,6 +77,10 @@ public:
   /// assembly.
   virtual bool isOffsetOfLocal() const { return false; }
 
+  /// isMemPlaceholder - Do we need to ignore the constraint, rather than emit
+  /// code? Only valid when parsing MS-style inline assembly.
+  virtual bool isMemPlaceholder(const MCInstrDesc &Desc) const { return false; }
+
   /// getOffsetOfLoc - Get the location of the offset operator.
   virtual SMLoc getOffsetOfLoc() const { return SMLoc(); }
 
diff --git a/llvm/include/llvm/MC/MCStreamer.h b/llvm/include/llvm/MC/MCStreamer.h
index e00f50f617fa..7bfbdb880098 100644
--- a/llvm/include/llvm/MC/MCStreamer.h
+++ b/llvm/include/llvm/MC/MCStreamer.h
@@ -496,8 +496,16 @@ public:
                                 unsigned Minor, unsigned Update,
                                 VersionTuple SDKVersion) {}
 
+  virtual void emitDarwinTargetVariantBuildVersion(unsigned Platform,
+                                                   unsigned Major,
+                                                   unsigned Minor,
+                                                   unsigned Update,
+                                                   VersionTuple SDKVersion) {}
+
   void emitVersionForTarget(const Triple &Target,
-                            const VersionTuple &SDKVersion);
+                            const VersionTuple &SDKVersion,
+                            const Triple *DarwinTargetVariantTriple,
+                            const VersionTuple &DarwinTargetVariantSDKVersion);
 
   /// Note in the output that the specified \p Func is a Thumb mode
   /// function (ARM target only).
@@ -516,6 +524,10 @@ public:
   /// \param Value - The value for the symbol.
   virtual void emitAssignment(MCSymbol *Symbol, const MCExpr *Value);
 
+  /// Emit an assignment of \p Value to \p Symbol, but only if \p Value is also
+  /// emitted.
+  virtual void emitConditionalAssignment(MCSymbol *Symbol, const MCExpr *Value);
+
   /// Emit an weak reference from \p Alias to \p Symbol.
   ///
   /// This corresponds to an assembler statement such as:
diff --git a/llvm/include/llvm/MC/MCTargetOptions.h b/llvm/include/llvm/MC/MCTargetOptions.h
index cd97cb0e2992..3510eeca8953 100644
--- a/llvm/include/llvm/MC/MCTargetOptions.h
+++ b/llvm/include/llvm/MC/MCTargetOptions.h
@@ -62,9 +62,10 @@ public:
   std::string ABIName;
   std::string AssemblyLanguage;
   std::string SplitDwarfFile;
+  std::string COFFOutputFilename;
 
   const char *Argv0 = nullptr;
-  ArrayRef<const char *> CommandLineArgs;
+  ArrayRef<std::string> CommandLineArgs;
 
   /// Additional paths to search for `.include` directives when using the
   /// integrated assembler.
diff --git a/llvm/include/llvm/Object/MachO.h b/llvm/include/llvm/Object/MachO.h
index ca5d63e4074f..ede742c47f97 100644
--- a/llvm/include/llvm/Object/MachO.h
+++ b/llvm/include/llvm/Object/MachO.h
@@ -652,6 +652,13 @@ public:
     return std::string(std::string(Version.str()));
   }
 
+  /// If the input path is a .dSYM bundle (as created by the dsymutil tool),
+  /// return the paths to the object files found in the bundle, otherwise return
+  /// an empty vector. If the path appears to be a .dSYM bundle but no objects
+  /// were found or there was a filesystem error, then return an error.
+  static Expected<std::vector<std::string>>
+  findDsymObjectMembers(StringRef Path);
+
 private:
   MachOObjectFile(MemoryBufferRef Object, bool IsLittleEndian, bool Is64Bits,
                   Error &Err, uint32_t UniversalCputype = 0,
diff --git a/llvm/include/llvm/Option/ArgList.h b/llvm/include/llvm/Option/ArgList.h
index f6abf2a62aa5..74897de52a93 100644
--- a/llvm/include/llvm/Option/ArgList.h
+++ b/llvm/include/llvm/Option/ArgList.h
@@ -245,6 +245,12 @@ public:
     return getLastArg(Ids...) != nullptr;
   }
 
+  /// Return true if the arg list contains multiple arguments matching \p Id.
+  bool hasMultipleArgs(OptSpecifier Id) const {
+    auto Args = filtered(Id);
+    return (Args.begin() != Args.end()) && (++Args.begin()) != Args.end();
+  }
+
   /// Return the last argument matching \p Id, or null.
   template<typename ...OptSpecifiers>
   Arg *getLastArg(OptSpecifiers ...Ids) const {
diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h
index 7c7883e98183..66b0b149fa25 100644
--- a/llvm/include/llvm/Passes/PassBuilder.h
+++ b/llvm/include/llvm/Passes/PassBuilder.h
@@ -199,7 +199,7 @@ public:
 
   /// Construct the module pipeline that performs inlining with
   /// module inliner pass.
-  ModuleInlinerPass buildModuleInlinerPipeline(OptimizationLevel Level,
+  ModulePassManager buildModuleInlinerPipeline(OptimizationLevel Level,
                                                ThinOrFullLTOPhase Phase);
 
   /// Construct the core LLVM module optimization pipeline.
diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h
index 4395c2abb33e..6c5efb2f6d5d 100644
--- a/llvm/include/llvm/ProfileData/InstrProf.h
+++ b/llvm/include/llvm/ProfileData/InstrProf.h
@@ -290,6 +290,10 @@ enum class instrprof_error {
   too_large,
   truncated,
   malformed,
+  missing_debug_info_for_correlation,
+  unexpected_debug_info_for_correlation,
+  unable_to_correlate_profile,
+  unsupported_debug_format,
   unknown_function,
   invalid_prof,
   hash_mismatch,
@@ -1149,7 +1153,8 @@ void getMemOPSizeRangeFromOption(StringRef Str, int64_t &RangeStart,
 // Create a COMDAT variable INSTR_PROF_RAW_VERSION_VAR to make the runtime
 // aware this is an ir_level profile so it can set the version flag.
 GlobalVariable *createIRLevelProfileFlagVar(Module &M, bool IsCS,
-                                            bool InstrEntryBBEnabled);
+                                            bool InstrEntryBBEnabled,
+                                            bool DebugInfoCorrelate);
 
 // Create the variable for the profile file name.
 void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput);
diff --git a/llvm/include/llvm/ProfileData/InstrProfCorrelator.h b/llvm/include/llvm/ProfileData/InstrProfCorrelator.h
new file mode 100644
index 000000000000..eae7b4e0322c
--- /dev/null
+++ b/llvm/include/llvm/ProfileData/InstrProfCorrelator.h
@@ -0,0 +1,170 @@
+//===- InstrProfCorrelator.h ------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// This file defines InstrProfCorrelator used to generate PGO profiles from
+// raw profile data and debug info.
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_PROFILEDATA_INSTRPROFCORRELATOR_H
+#define LLVM_PROFILEDATA_INSTRPROFCORRELATOR_H
+
+#include "llvm/DebugInfo/DWARF/DWARFContext.h"
+#include "llvm/Object/Binary.h"
+#include "llvm/Object/ObjectFile.h"
+#include "llvm/ProfileData/InstrProf.h"
+#include "llvm/Support/Casting.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include <vector>
+
+namespace llvm {
+
+/// InstrProfCorrelator - A base class used to create raw instrumentation data
+/// to their functions.
+class InstrProfCorrelator {
+public:
+  static llvm::Expected<std::unique_ptr<InstrProfCorrelator>>
+  get(StringRef DebugInfoFilename);
+
+  /// Construct a ProfileData vector used to correlate raw instrumentation data
+  /// to their functions.
+  virtual Error correlateProfileData() = 0;
+
+  static const char *FunctionNameAttributeName;
+  static const char *CFGHashAttributeName;
+  static const char *NumCountersAttributeName;
+
+  enum InstrProfCorrelatorKind { CK_32Bit, CK_64Bit };
+  InstrProfCorrelatorKind getKind() const { return Kind; }
+  virtual ~InstrProfCorrelator() {}
+
+protected:
+  struct Context {
+    static llvm::Expected<std::unique_ptr<Context>>
+    get(std::unique_ptr<MemoryBuffer> Buffer, const object::ObjectFile &Obj);
+    std::unique_ptr<MemoryBuffer> Buffer;
+    /// The address range of the __llvm_prf_cnts section.
+    uint64_t CountersSectionStart;
+    uint64_t CountersSectionEnd;
+    /// True if target and host have different endian orders.
+    bool ShouldSwapBytes;
+  };
+  const std::unique_ptr<InstrProfCorrelator::Context> Ctx;
+
+  InstrProfCorrelator(InstrProfCorrelatorKind K, std::unique_ptr<Context> Ctx)
+      : Ctx(std::move(Ctx)), Kind(K) {}
+
+private:
+  static llvm::Expected<std::unique_ptr<InstrProfCorrelator>>
+  get(std::unique_ptr<MemoryBuffer> Buffer);
+
+  const InstrProfCorrelatorKind Kind;
+};
+
+/// InstrProfCorrelatorImpl - A child of InstrProfCorrelator with a template
+/// pointer type so that the ProfileData vector can be materialized.
+template <class IntPtrT>
+class InstrProfCorrelatorImpl : public InstrProfCorrelator {
+public:
+  InstrProfCorrelatorImpl(std::unique_ptr<InstrProfCorrelator::Context> Ctx);
+  static bool classof(const InstrProfCorrelator *C);
+
+  /// Return a pointer to the underlying ProfileData vector that this class
+  /// constructs.
+  const RawInstrProf::ProfileData<IntPtrT> *getDataPointer() const {
+    return Data.empty() ? nullptr : Data.data();
+  }
+
+  /// Return the number of ProfileData elements.
+  size_t getDataSize() const { return Data.size(); }
+
+  /// Return a pointer to the compressed names string that this class
+  /// constructs.
+  const char *getCompressedNamesPointer() const {
+    return CompressedNames.c_str();
+  }
+
+  /// Return the number of bytes in the compressed names string.
+  size_t getCompressedNamesSize() const { return CompressedNames.size(); }
+
+  static llvm::Expected<std::unique_ptr<InstrProfCorrelatorImpl<IntPtrT>>>
+  get(std::unique_ptr<InstrProfCorrelator::Context> Ctx,
+      const object::ObjectFile &Obj);
+
+protected:
+  std::vector<RawInstrProf::ProfileData<IntPtrT>> Data;
+  std::string CompressedNames;
+
+  Error correlateProfileData() override;
+  virtual void correlateProfileDataImpl() = 0;
+
+  void addProbe(StringRef FunctionName, uint64_t CFGHash, IntPtrT CounterOffset,
+                IntPtrT FunctionPtr, uint32_t NumCounters);
+
+private:
+  InstrProfCorrelatorImpl(InstrProfCorrelatorKind Kind,
+                          std::unique_ptr<InstrProfCorrelator::Context> Ctx)
+      : InstrProfCorrelator(Kind, std::move(Ctx)){};
+  std::vector<std::string> Names;
+
+  // Byte-swap the value if necessary.
+  template <class T> T maybeSwap(T Value) const {
+    return Ctx->ShouldSwapBytes ? sys::getSwappedBytes(Value) : Value;
+  }
+};
+
+/// DwarfInstrProfCorrelator - A child of InstrProfCorrelatorImpl that takes
+/// DWARF debug info as input to correlate profiles.
+template <class IntPtrT>
+class DwarfInstrProfCorrelator : public InstrProfCorrelatorImpl<IntPtrT> {
+public:
+  DwarfInstrProfCorrelator(std::unique_ptr<DWARFContext> DICtx,
+                           std::unique_ptr<InstrProfCorrelator::Context> Ctx)
+      : InstrProfCorrelatorImpl<IntPtrT>(std::move(Ctx)),
+        DICtx(std::move(DICtx)) {}
+
+private:
+  std::unique_ptr<DWARFContext> DICtx;
+
+  /// Return the address of the object that the provided DIE symbolizes.
+  llvm::Optional<uint64_t> getLocation(const DWARFDie &Die) const;
+
+  /// Returns true if the provided DIE symbolizes an instrumentation probe
+  /// symbol.
+  static bool isDIEOfProbe(const DWARFDie &Die);
+
+  /// Iterate over DWARF DIEs to find those that symbolize instrumentation
+  /// probes and construct the ProfileData vector and CompressedNames string.
+  ///
+  /// Here is some example DWARF for an instrumentation probe we are looking
+  /// for:
+  /// \code
+  ///   DW_TAG_subprogram
+  ///   DW_AT_low_pc	(0x0000000000000000)
+  ///   DW_AT_high_pc	(0x0000000000000014)
+  ///   DW_AT_name	("foo")
+  ///     DW_TAG_variable
+  ///       DW_AT_name	("__profc_foo")
+  ///       DW_AT_location	(DW_OP_addr 0x0)
+  ///       DW_TAG_LLVM_annotation
+  ///         DW_AT_name	("Function Name")
+  ///         DW_AT_const_value	("foo")
+  ///       DW_TAG_LLVM_annotation
+  ///         DW_AT_name	("CFG Hash")
+  ///         DW_AT_const_value	(12345678)
+  ///       DW_TAG_LLVM_annotation
+  ///         DW_AT_name	("Num Counters")
+  ///         DW_AT_const_value	(2)
+  ///       NULL
+  ///     NULL
+  /// \endcode
+  void correlateProfileDataImpl() override;
+};
+
+} // end namespace llvm
+
+#endif // LLVM_PROFILEDATA_INSTRPROFCORRELATOR_H
diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc
index 008b8dde5820..44719126b596 100644
--- a/llvm/include/llvm/ProfileData/InstrProfData.inc
+++ b/llvm/include/llvm/ProfileData/InstrProfData.inc
@@ -653,15 +653,17 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
 
 /* Profile version is always of type uint64_t. Reserve the upper 8 bits in the
  * version for other variants of profile. We set the lowest bit of the upper 8
- * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton
+ * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentation
  * generated profile, and 0 if this is a Clang FE generated profile.
  * 1 in bit 57 indicates there are context-sensitive records in the profile.
+ * The 59th bit indicates whether to use debug info to correlate profiles.
  */
 #define VARIANT_MASKS_ALL 0xff00000000000000ULL
 #define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL)
 #define VARIANT_MASK_IR_PROF (0x1ULL << 56)
 #define VARIANT_MASK_CSIR_PROF (0x1ULL << 57)
 #define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58)
+#define VARIANT_MASK_DBG_CORRELATE (0x1ULL << 59)
 #define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version
 #define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime
 #define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias
diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h
index b62d4ff044a3..c615e8533178 100644
--- a/llvm/include/llvm/ProfileData/InstrProfReader.h
+++ b/llvm/include/llvm/ProfileData/InstrProfReader.h
@@ -18,6 +18,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/IR/ProfileSummary.h"
 #include "llvm/ProfileData/InstrProf.h"
+#include "llvm/ProfileData/InstrProfCorrelator.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/LineIterator.h"
@@ -96,6 +97,9 @@ public:
 
   virtual bool instrEntryBBEnabled() const = 0;
 
+  /// Return true if we must provide debug info to create PGO profiles.
+  virtual bool useDebugInfoCorrelate() const { return false; }
+
   /// Return the PGO symtab. There are three different readers:
   /// Raw, Text, and Indexed profile readers. The first two types
   /// of readers are used only by llvm-profdata tool, while the indexed
@@ -150,10 +154,12 @@ public:
 
   /// Factory method to create an appropriately typed reader for the given
   /// instrprof file.
-  static Expected<std::unique_ptr<InstrProfReader>> create(const Twine &Path);
+  static Expected<std::unique_ptr<InstrProfReader>>
+  create(const Twine &Path, const InstrProfCorrelator *Correlator = nullptr);
 
   static Expected<std::unique_ptr<InstrProfReader>>
-  create(std::unique_ptr<MemoryBuffer> Buffer);
+  create(std::unique_ptr<MemoryBuffer> Buffer,
+         const InstrProfCorrelator *Correlator = nullptr);
 };
 
 /// Reader for the simple text based instrprof format.
@@ -215,6 +221,9 @@ class RawInstrProfReader : public InstrProfReader {
 private:
   /// The profile data file contents.
   std::unique_ptr<MemoryBuffer> DataBuffer;
+  /// If available, this hold the ProfileData array used to correlate raw
+  /// instrumentation data to their functions.
+  const InstrProfCorrelatorImpl<IntPtrT> *Correlator;
   bool ShouldSwapBytes;
   // The value of the version field of the raw profile data header. The lower 56
   // bits specifies the format version and the most significant 8 bits specify
@@ -226,7 +235,7 @@ private:
   const RawInstrProf::ProfileData<IntPtrT> *DataEnd;
   const uint64_t *CountersStart;
   const char *NamesStart;
-  uint64_t NamesSize;
+  const char *NamesEnd;
   // After value profile is all read, this pointer points to
   // the header of next profile data (if exists)
   const uint8_t *ValueDataStart;
@@ -237,8 +246,11 @@ private:
   const uint8_t *BinaryIdsStart;
 
 public:
-  RawInstrProfReader(std::unique_ptr<MemoryBuffer> DataBuffer)
-      : DataBuffer(std::move(DataBuffer)) {}
+  RawInstrProfReader(std::unique_ptr<MemoryBuffer> DataBuffer,
+                     const InstrProfCorrelator *Correlator)
+      : DataBuffer(std::move(DataBuffer)),
+        Correlator(dyn_cast_or_null<const InstrProfCorrelatorImpl<IntPtrT>>(
+            Correlator)) {}
   RawInstrProfReader(const RawInstrProfReader &) = delete;
   RawInstrProfReader &operator=(const RawInstrProfReader &) = delete;
 
@@ -259,6 +271,10 @@ public:
     return (Version & VARIANT_MASK_INSTR_ENTRY) != 0;
   }
 
+  bool useDebugInfoCorrelate() const override {
+    return (Version & VARIANT_MASK_DBG_CORRELATE) != 0;
+  }
+
   InstrProfSymtab &getSymtab() override {
     assert(Symtab.get());
     return *Symtab.get();
diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h
index 7ac9eccf8ac2..dc6522f4ec4c 100644
--- a/llvm/include/llvm/ProfileData/SampleProf.h
+++ b/llvm/include/llvm/ProfileData/SampleProf.h
@@ -206,7 +206,8 @@ enum class SecProfSummaryFlags : uint32_t {
 enum class SecFuncMetadataFlags : uint32_t {
   SecFlagInvalid = 0,
   SecFlagIsProbeBased = (1 << 0),
-  SecFlagHasAttribute = (1 << 1)
+  SecFlagHasAttribute = (1 << 1),
+  SecFlagIsCSNested = (1 << 2),
 };
 
 enum class SecFuncOffsetFlags : uint32_t {
@@ -591,11 +592,11 @@ public:
                         : hash_value(getName());
   }
 
-  /// Set the name of the function.
+  /// Set the name of the function and clear the current context.
   void setName(StringRef FunctionName) {
-    assert(FullContext.empty() &&
-           "setName should only be called for non-CS profile");
     Name = FunctionName;
+    FullContext = SampleContextFrames();
+    State = UnknownContext;
   }
 
   void setContext(SampleContextFrames Context,
@@ -745,6 +746,16 @@ public:
     }
   }
 
+  // Set current context and all callee contexts to be synthetic.
+  void SetContextSynthetic() {
+    Context.setState(SyntheticContext);
+    for (auto &I : CallsiteSamples) {
+      for (auto &CS : I.second) {
+        CS.second.SetContextSynthetic();
+      }
+    }
+  }
+
   /// Return the number of samples collected at the given location.
   /// Each location is specified by \p LineOffset and \p Discriminator.
   /// If the location is not found in profile, return error.
@@ -816,7 +827,7 @@ public:
   /// Return the sample count of the first instruction of the function.
   /// The function can be either a standalone symbol or an inlined function.
   uint64_t getEntrySamples() const {
-    if (FunctionSamples::ProfileIsCS && getHeadSamples()) {
+    if (FunctionSamples::ProfileIsCSFlat && getHeadSamples()) {
       // For CS profile, if we already have more accurate head samples
       // counted by branch sample from caller, use them as entry samples.
       return getHeadSamples();
@@ -1008,7 +1019,13 @@ public:
   /// instruction. This is wrapper of two scenarios, the probe-based profile and
   /// regular profile, to hide implementation details from the sample loader and
   /// the context tracker.
-  static LineLocation getCallSiteIdentifier(const DILocation *DIL);
+  static LineLocation getCallSiteIdentifier(const DILocation *DIL,
+                                            bool ProfileIsFS = false);
+
+  /// Returns a unique hash code for a combination of a callsite location and
+  /// the callee function name.
+  static uint64_t getCallSiteHash(StringRef CalleeName,
+                                  const LineLocation &Callsite);
 
   /// Get the FunctionSamples of the inline instance where DIL originates
   /// from.
@@ -1027,7 +1044,9 @@ public:
 
   static bool ProfileIsProbeBased;
 
-  static bool ProfileIsCS;
+  static bool ProfileIsCSFlat;
+
+  static bool ProfileIsCSNested;
 
   SampleContext &getContext() const { return Context; }
 
@@ -1161,6 +1180,40 @@ private:
   SampleProfileMap &ProfileMap;
 };
 
+// CSProfileConverter converts a full context-sensitive flat sample profile into
+// a nested context-sensitive sample profile.
+class CSProfileConverter {
+public:
+  CSProfileConverter(SampleProfileMap &Profiles);
+  void convertProfiles();
+  struct FrameNode {
+    FrameNode(StringRef FName = StringRef(),
+              FunctionSamples *FSamples = nullptr,
+              LineLocation CallLoc = {0, 0})
+        : FuncName(FName), FuncSamples(FSamples), CallSiteLoc(CallLoc){};
+
+    // Map line+discriminator location to child frame
+    std::map<uint64_t, FrameNode> AllChildFrames;
+    // Function name for current frame
+    StringRef FuncName;
+    // Function Samples for current frame
+    FunctionSamples *FuncSamples;
+    // Callsite location in parent context
+    LineLocation CallSiteLoc;
+
+    FrameNode *getOrCreateChildFrame(const LineLocation &CallSite,
+                                     StringRef CalleeName);
+  };
+
+private:
+  // Nest all children profiles into the profile of Node.
+  void convertProfiles(FrameNode &Node);
+  FrameNode *getOrCreateContextPath(const SampleContext &Context);
+
+  SampleProfileMap &ProfileMap;
+  FrameNode RootFrame;
+};
+
 /// ProfileSymbolList records the list of function symbols shown up
 /// in the binary used to generate the profile. It is useful to
 /// to discriminate a function being so cold as not to shown up
diff --git a/llvm/include/llvm/ProfileData/SampleProfReader.h b/llvm/include/llvm/ProfileData/SampleProfReader.h
index e6d31f1b9098..a2caca246d93 100644
--- a/llvm/include/llvm/ProfileData/SampleProfReader.h
+++ b/llvm/include/llvm/ProfileData/SampleProfReader.h
@@ -473,8 +473,11 @@ public:
   /// Whether input profile is based on pseudo probes.
   bool profileIsProbeBased() const { return ProfileIsProbeBased; }
 
-  /// Whether input profile is fully context-sensitive
-  bool profileIsCS() const { return ProfileIsCS; }
+  /// Whether input profile is fully context-sensitive and flat.
+  bool profileIsCSFlat() const { return ProfileIsCSFlat; }
+
+  /// Whether input profile is fully context-sensitive and nested.
+  bool profileIsCSNested() const { return ProfileIsCSNested; }
 
   virtual std::unique_ptr<ProfileSymbolList> getProfileSymbolList() {
     return nullptr;
@@ -533,8 +536,11 @@ protected:
   /// \brief Whether samples are collected based on pseudo probes.
   bool ProfileIsProbeBased = false;
 
-  /// Whether function profiles are context-sensitive.
-  bool ProfileIsCS = false;
+  /// Whether function profiles are context-sensitive flat profiles.
+  bool ProfileIsCSFlat = false;
+
+  /// Whether function profiles are context-sensitive nested profiles.
+  bool ProfileIsCSNested = false;
 
   /// Number of context-sensitive profiles.
   uint32_t CSProfileCount = 0;
@@ -698,6 +704,8 @@ protected:
   std::error_code readSecHdrTable();
 
   std::error_code readFuncMetadata(bool ProfileHasAttribute);
+  std::error_code readFuncMetadata(bool ProfileHasAttribute,
+                                   FunctionSamples *FProfile);
   std::error_code readFuncOffsetTable();
   std::error_code readFuncProfiles();
   std::error_code readMD5NameTable();
diff --git a/llvm/include/llvm/ProfileData/SampleProfWriter.h b/llvm/include/llvm/ProfileData/SampleProfWriter.h
index 773beac24ebc..42decd255203 100644
--- a/llvm/include/llvm/ProfileData/SampleProfWriter.h
+++ b/llvm/include/llvm/ProfileData/SampleProfWriter.h
@@ -269,6 +269,7 @@ protected:
   std::error_code writeCSNameTableSection();
 
   std::error_code writeFuncMetadata(const SampleProfileMap &Profiles);
+  std::error_code writeFuncMetadata(const FunctionSamples &Profile);
 
   // Functions to write various kinds of sections.
   std::error_code writeNameTableSection(const SampleProfileMap &ProfileMap);
diff --git a/llvm/include/llvm/Support/ARMEHABI.h b/llvm/include/llvm/Support/ARMEHABI.h
index 3fbb56d65eb8..1a7778fe4a1c 100644
--- a/llvm/include/llvm/Support/ARMEHABI.h
+++ b/llvm/include/llvm/Support/ARMEHABI.h
@@ -71,6 +71,10 @@ namespace EHABI {
     // Purpose: finish
     UNWIND_OPCODE_FINISH = 0xb0,
 
+    // Format: 10110100
+    // Purpose: Pop Return Address Authetication Code
+    UNWIND_OPCODE_POP_RA_AUTH_CODE = 0xb4,
+
     // Format: 10110001 0000xxxx
     // Purpose: pop r[3:0]
     // Constraint: x != 0
diff --git a/llvm/include/llvm/Support/Caching.h b/llvm/include/llvm/Support/Caching.h
index 1e5fea17f708..5c30a822ef38 100644
--- a/llvm/include/llvm/Support/Caching.h
+++ b/llvm/include/llvm/Support/Caching.h
@@ -27,8 +27,11 @@ class MemoryBuffer;
 /// that can be done by deriving from this class and overriding the destructor.
 class CachedFileStream {
 public:
-  CachedFileStream(std::unique_ptr<raw_pwrite_stream> OS) : OS(std::move(OS)) {}
+  CachedFileStream(std::unique_ptr<raw_pwrite_stream> OS,
+                   std::string OSPath = "")
+      : OS(std::move(OS)), ObjectPathName(OSPath) {}
   std::unique_ptr<raw_pwrite_stream> OS;
+  std::string ObjectPathName;
   virtual ~CachedFileStream() = default;
 };
 
@@ -63,9 +66,10 @@ using AddBufferFn =
 /// the cache directory if it does not already exist. The cache name appears in
 /// error messages for errors during caching. The temporary file prefix is used
 /// in the temporary file naming scheme used when writing files atomically.
-Expected<FileCache> localCache(Twine CacheNameRef, Twine TempFilePrefixRef,
-                               Twine CacheDirectoryPathRef,
-                               AddBufferFn AddBuffer);
+Expected<FileCache> localCache(
+    Twine CacheNameRef, Twine TempFilePrefixRef, Twine CacheDirectoryPathRef,
+    AddBufferFn AddBuffer = [](size_t Task, std::unique_ptr<MemoryBuffer> MB) {
+    });
 } // namespace llvm
 
 #endif
diff --git a/llvm/include/llvm/Support/Chrono.h b/llvm/include/llvm/Support/Chrono.h
index f478549a7e4e..629a37a90aae 100644
--- a/llvm/include/llvm/Support/Chrono.h
+++ b/llvm/include/llvm/Support/Chrono.h
@@ -22,13 +22,13 @@ class raw_ostream;
 namespace sys {
 
 /// A time point on the system clock. This is provided for two reasons:
-/// - to insulate us agains subtle differences in behavoir to differences in
-///   system clock precision (which is implementation-defined and differs between
-///   platforms).
+/// - to insulate us against subtle differences in behavior to differences in
+///   system clock precision (which is implementation-defined and differs
+///   between platforms).
 /// - to shorten the type name
-/// The default precision is nanoseconds. If need a specific precision specify
-/// it explicitly. If unsure, use the default. If you need a time point on a
-/// clock other than the system_clock, use std::chrono directly.
+/// The default precision is nanoseconds. If you need a specific precision
+/// specify it explicitly. If unsure, use the default. If you need a time point
+/// on a clock other than the system_clock, use std::chrono directly.
 template <typename D = std::chrono::nanoseconds>
 using TimePoint = std::chrono::time_point<std::chrono::system_clock, D>;
 
diff --git a/llvm/include/llvm/Support/Compiler.h b/llvm/include/llvm/Support/Compiler.h
index c5318137ed3d..b31ba6bc7fc2 100644
--- a/llvm/include/llvm/Support/Compiler.h
+++ b/llvm/include/llvm/Support/Compiler.h
@@ -126,7 +126,11 @@
 #if __has_attribute(visibility) && !defined(__MINGW32__) &&                    \
     !defined(__CYGWIN__) && !defined(_WIN32)
 #define LLVM_LIBRARY_VISIBILITY __attribute__ ((visibility("hidden")))
-#define LLVM_EXTERNAL_VISIBILITY __attribute__ ((visibility("default")))
+#if defined(LLVM_BUILD_LLVM_DYLIB) || defined(LLVM_BUILD_SHARED_LIBS)
+#define LLVM_EXTERNAL_VISIBILITY __attribute__((visibility("default")))
+#else
+#define LLVM_EXTERNAL_VISIBILITY
+#endif
 #else
 #define LLVM_LIBRARY_VISIBILITY
 #define LLVM_EXTERNAL_VISIBILITY
diff --git a/llvm/include/llvm/Support/GraphWriter.h b/llvm/include/llvm/Support/GraphWriter.h
index 11a31bf40160..1c0f5f702c6d 100644
--- a/llvm/include/llvm/Support/GraphWriter.h
+++ b/llvm/include/llvm/Support/GraphWriter.h
@@ -265,10 +265,9 @@ public:
           << DOT::EscapeString(DTraits.getEdgeDestLabel(Node, i));
       }
 
-      if (RenderUsingHTML)
-        O << "<td colspan=\"1\">... truncated</td>";
-      else if (i != e)
-        O << "|<d64>truncated...}";
+      if (i != e)
+        O << "|<d64>truncated...";
+      O << "}";
     }
 
     if (RenderUsingHTML)
diff --git a/llvm/include/llvm/Support/RISCVISAInfo.h b/llvm/include/llvm/Support/RISCVISAInfo.h
index 7110de601123..1ba4d449b709 100644
--- a/llvm/include/llvm/Support/RISCVISAInfo.h
+++ b/llvm/include/llvm/Support/RISCVISAInfo.h
@@ -81,6 +81,9 @@ private:
   void addExtension(StringRef ExtName, unsigned MajorVersion,
                     unsigned MinorVersion);
 
+  Error checkDependency();
+
+  void updateImplication();
   void updateFLen();
 };
 
diff --git a/llvm/include/llvm/Support/ScopedPrinter.h b/llvm/include/llvm/Support/ScopedPrinter.h
index 0dfe1245f7d6..865337e3cc7f 100644
--- a/llvm/include/llvm/Support/ScopedPrinter.h
+++ b/llvm/include/llvm/Support/ScopedPrinter.h
@@ -16,6 +16,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/DataTypes.h"
 #include "llvm/Support/Endian.h"
+#include "llvm/Support/JSON.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
 
@@ -57,19 +58,65 @@ struct HexNumber {
   uint64_t Value;
 };
 
+struct FlagEntry {
+  FlagEntry(StringRef Name, char Value)
+      : Name(Name), Value(static_cast<unsigned char>(Value)) {}
+  FlagEntry(StringRef Name, signed char Value)
+      : Name(Name), Value(static_cast<unsigned char>(Value)) {}
+  FlagEntry(StringRef Name, signed short Value)
+      : Name(Name), Value(static_cast<unsigned short>(Value)) {}
+  FlagEntry(StringRef Name, signed int Value)
+      : Name(Name), Value(static_cast<unsigned int>(Value)) {}
+  FlagEntry(StringRef Name, signed long Value)
+      : Name(Name), Value(static_cast<unsigned long>(Value)) {}
+  FlagEntry(StringRef Name, signed long long Value)
+      : Name(Name), Value(static_cast<unsigned long long>(Value)) {}
+  FlagEntry(StringRef Name, unsigned char Value) : Name(Name), Value(Value) {}
+  FlagEntry(StringRef Name, unsigned short Value) : Name(Name), Value(Value) {}
+  FlagEntry(StringRef Name, unsigned int Value) : Name(Name), Value(Value) {}
+  FlagEntry(StringRef Name, unsigned long Value) : Name(Name), Value(Value) {}
+  FlagEntry(StringRef Name, unsigned long long Value)
+      : Name(Name), Value(Value) {}
+  StringRef Name;
+  uint64_t Value;
+};
+
 raw_ostream &operator<<(raw_ostream &OS, const HexNumber &Value);
 std::string to_hexString(uint64_t Value, bool UpperCase = true);
 
 template <class T> std::string to_string(const T &Value) {
   std::string number;
-  llvm::raw_string_ostream stream(number);
+  raw_string_ostream stream(number);
   stream << Value;
   return stream.str();
 }
 
+template <typename T, typename TEnum>
+std::string enumToString(T Value, ArrayRef<EnumEntry<TEnum>> EnumValues) {
+  for (const EnumEntry<TEnum> &EnumItem : EnumValues)
+    if (EnumItem.Value == Value)
+      return std::string(EnumItem.AltName);
+  return to_hexString(Value, false);
+}
+
 class ScopedPrinter {
 public:
-  ScopedPrinter(raw_ostream &OS) : OS(OS), IndentLevel(0) {}
+  enum class ScopedPrinterKind {
+    Base,
+    JSON,
+  };
+
+  ScopedPrinter(raw_ostream &OS,
+                ScopedPrinterKind Kind = ScopedPrinterKind::Base)
+      : OS(OS), IndentLevel(0), Kind(Kind) {}
+
+  ScopedPrinterKind getKind() const { return Kind; }
+
+  static bool classof(const ScopedPrinter *SP) {
+    return SP->getKind() == ScopedPrinterKind::Base;
+  }
+
+  virtual ~ScopedPrinter() {}
 
   void flush() { OS.flush(); }
 
@@ -106,20 +153,17 @@ public:
       }
     }
 
-    if (Found) {
-      startLine() << Label << ": " << Name << " (" << hex(Value) << ")\n";
-    } else {
-      startLine() << Label << ": " << hex(Value) << "\n";
-    }
+    if (Found)
+      printHex(Label, Name, Value);
+    else
+      printHex(Label, Value);
   }
 
   template <typename T, typename TFlag>
   void printFlags(StringRef Label, T Value, ArrayRef<EnumEntry<TFlag>> Flags,
                   TFlag EnumMask1 = {}, TFlag EnumMask2 = {},
                   TFlag EnumMask3 = {}) {
-    typedef EnumEntry<TFlag> FlagEntry;
-    typedef SmallVector<FlagEntry, 10> FlagVector;
-    FlagVector SetFlags;
+    SmallVector<FlagEntry, 10> SetFlags;
 
     for (const auto &Flag : Flags) {
       if (Flag.Value == 0)
@@ -135,69 +179,69 @@ public:
       bool IsEnum = (Flag.Value & EnumMask) != 0;
       if ((!IsEnum && (Value & Flag.Value) == Flag.Value) ||
           (IsEnum && (Value & EnumMask) == Flag.Value)) {
-        SetFlags.push_back(Flag);
+        SetFlags.emplace_back(Flag.Name, Flag.Value);
       }
     }
 
-    llvm::sort(SetFlags, &flagName<TFlag>);
-
-    startLine() << Label << " [ (" << hex(Value) << ")\n";
-    for (const auto &Flag : SetFlags) {
-      startLine() << "  " << Flag.Name << " (" << hex(Flag.Value) << ")\n";
-    }
-    startLine() << "]\n";
+    llvm::sort(SetFlags, &flagName);
+    printFlagsImpl(Label, hex(Value), SetFlags);
   }
 
   template <typename T> void printFlags(StringRef Label, T Value) {
-    startLine() << Label << " [ (" << hex(Value) << ")\n";
+    SmallVector<HexNumber, 10> SetFlags;
     uint64_t Flag = 1;
     uint64_t Curr = Value;
     while (Curr > 0) {
       if (Curr & 1)
-        startLine() << "  " << hex(Flag) << "\n";
+        SetFlags.emplace_back(Flag);
       Curr >>= 1;
       Flag <<= 1;
     }
-    startLine() << "]\n";
+    printFlagsImpl(Label, hex(Value), SetFlags);
   }
 
-  void printNumber(StringRef Label, uint64_t Value) {
+  virtual void printNumber(StringRef Label, uint64_t Value) {
     startLine() << Label << ": " << Value << "\n";
   }
 
-  void printNumber(StringRef Label, uint32_t Value) {
+  virtual void printNumber(StringRef Label, uint32_t Value) {
     startLine() << Label << ": " << Value << "\n";
   }
 
-  void printNumber(StringRef Label, uint16_t Value) {
+  virtual void printNumber(StringRef Label, uint16_t Value) {
     startLine() << Label << ": " << Value << "\n";
   }
 
-  void printNumber(StringRef Label, uint8_t Value) {
+  virtual void printNumber(StringRef Label, uint8_t Value) {
     startLine() << Label << ": " << unsigned(Value) << "\n";
   }
 
-  void printNumber(StringRef Label, int64_t Value) {
+  virtual void printNumber(StringRef Label, int64_t Value) {
     startLine() << Label << ": " << Value << "\n";
   }
 
-  void printNumber(StringRef Label, int32_t Value) {
+  virtual void printNumber(StringRef Label, int32_t Value) {
     startLine() << Label << ": " << Value << "\n";
   }
 
-  void printNumber(StringRef Label, int16_t Value) {
+  virtual void printNumber(StringRef Label, int16_t Value) {
     startLine() << Label << ": " << Value << "\n";
   }
 
-  void printNumber(StringRef Label, int8_t Value) {
+  virtual void printNumber(StringRef Label, int8_t Value) {
     startLine() << Label << ": " << int(Value) << "\n";
   }
 
-  void printNumber(StringRef Label, const APSInt &Value) {
+  virtual void printNumber(StringRef Label, const APSInt &Value) {
     startLine() << Label << ": " << Value << "\n";
   }
 
-  void printBoolean(StringRef Label, bool Value) {
+  template <typename T>
+  void printNumber(StringRef Label, StringRef Str, T Value) {
+    printNumberImpl(Label, Str, to_string(Value));
+  }
+
+  virtual void printBoolean(StringRef Label, bool Value) {
     startLine() << Label << ": " << (Value ? "Yes" : "No") << '\n';
   }
 
@@ -207,12 +251,62 @@ public:
     getOStream() << "\n";
   }
 
-  template <typename T> void printList(StringRef Label, const T &List) {
-    startLine() << Label << ": [";
-    ListSeparator LS;
+  template <typename T>
+  void printList(StringRef Label, const ArrayRef<T> List) {
+    SmallVector<std::string, 10> StringList;
     for (const auto &Item : List)
-      OS << LS << Item;
-    OS << "]\n";
+      StringList.emplace_back(to_string(Item));
+    printList(Label, StringList);
+  }
+
+  virtual void printList(StringRef Label, const ArrayRef<bool> List) {
+    printListImpl(Label, List);
+  }
+
+  virtual void printList(StringRef Label, const ArrayRef<std::string> List) {
+    printListImpl(Label, List);
+  }
+
+  virtual void printList(StringRef Label, const ArrayRef<uint64_t> List) {
+    printListImpl(Label, List);
+  }
+
+  virtual void printList(StringRef Label, const ArrayRef<uint32_t> List) {
+    printListImpl(Label, List);
+  }
+
+  virtual void printList(StringRef Label, const ArrayRef<uint16_t> List) {
+    printListImpl(Label, List);
+  }
+
+  virtual void printList(StringRef Label, const ArrayRef<uint8_t> List) {
+    SmallVector<unsigned> NumberList;
+    for (const uint8_t &Item : List)
+      NumberList.emplace_back(Item);
+    printListImpl(Label, NumberList);
+  }
+
+  virtual void printList(StringRef Label, const ArrayRef<int64_t> List) {
+    printListImpl(Label, List);
+  }
+
+  virtual void printList(StringRef Label, const ArrayRef<int32_t> List) {
+    printListImpl(Label, List);
+  }
+
+  virtual void printList(StringRef Label, const ArrayRef<int16_t> List) {
+    printListImpl(Label, List);
+  }
+
+  virtual void printList(StringRef Label, const ArrayRef<int8_t> List) {
+    SmallVector<int> NumberList;
+    for (const int8_t &Item : List)
+      NumberList.emplace_back(Item);
+    printListImpl(Label, NumberList);
+  }
+
+  virtual void printList(StringRef Label, const ArrayRef<APSInt> List) {
+    printListImpl(Label, List);
   }
 
   template <typename T, typename U>
@@ -227,45 +321,31 @@ public:
   }
 
   template <typename T> void printHexList(StringRef Label, const T &List) {
-    startLine() << Label << ": [";
-    ListSeparator LS;
+    SmallVector<HexNumber> HexList;
     for (const auto &Item : List)
-      OS << LS << hex(Item);
-    OS << "]\n";
+      HexList.emplace_back(Item);
+    printHexListImpl(Label, HexList);
   }
 
   template <typename T> void printHex(StringRef Label, T Value) {
-    startLine() << Label << ": " << hex(Value) << "\n";
+    printHexImpl(Label, hex(Value));
   }
 
   template <typename T> void printHex(StringRef Label, StringRef Str, T Value) {
-    startLine() << Label << ": " << Str << " (" << hex(Value) << ")\n";
+    printHexImpl(Label, Str, hex(Value));
   }
 
   template <typename T>
   void printSymbolOffset(StringRef Label, StringRef Symbol, T Value) {
-    startLine() << Label << ": " << Symbol << '+' << hex(Value) << '\n';
+    printSymbolOffsetImpl(Label, Symbol, hex(Value));
   }
 
-  void printString(StringRef Value) { startLine() << Value << "\n"; }
+  virtual void printString(StringRef Value) { startLine() << Value << "\n"; }
 
-  void printString(StringRef Label, StringRef Value) {
+  virtual void printString(StringRef Label, StringRef Value) {
     startLine() << Label << ": " << Value << "\n";
   }
 
-  void printString(StringRef Label, const std::string &Value) {
-    printString(Label, StringRef(Value));
-  }
-
-  void printString(StringRef Label, const char* Value) {
-    printString(Label, StringRef(Value));
-  }
-
-  template <typename T>
-  void printNumber(StringRef Label, StringRef Str, T Value) {
-    startLine() << Label << ": " << Str << " (" << Value << ")\n";
-  }
-
   void printBinary(StringRef Label, StringRef Str, ArrayRef<uint8_t> Value) {
     printBinaryImpl(Label, Str, Value, false);
   }
@@ -308,15 +388,27 @@ public:
   }
 
   template <typename T> void printObject(StringRef Label, const T &Value) {
-    startLine() << Label << ": " << Value << "\n";
+    printString(Label, to_string(Value));
   }
 
-  raw_ostream &startLine() {
+  virtual void objectBegin() { scopedBegin('{'); }
+
+  virtual void objectBegin(StringRef Label) { scopedBegin(Label, '{'); }
+
+  virtual void objectEnd() { scopedEnd('}'); }
+
+  virtual void arrayBegin() { scopedBegin('['); }
+
+  virtual void arrayBegin(StringRef Label) { scopedBegin(Label, '['); }
+
+  virtual void arrayEnd() { scopedEnd(']'); }
+
+  virtual raw_ostream &startLine() {
     printIndent();
     return OS;
   }
 
-  raw_ostream &getOStream() { return OS; }
+  virtual raw_ostream &getOStream() { return OS; }
 
 private:
   template <typename T> void printVersionInternal(T Value) {
@@ -329,17 +421,87 @@ private:
     printVersionInternal(Value2, Args...);
   }
 
-  template <typename T>
-  static bool flagName(const EnumEntry<T> &lhs, const EnumEntry<T> &rhs) {
-    return lhs.Name < rhs.Name;
+  static bool flagName(const FlagEntry &LHS, const FlagEntry &RHS) {
+    return LHS.Name < RHS.Name;
   }
 
-  void printBinaryImpl(StringRef Label, StringRef Str, ArrayRef<uint8_t> Value,
-                       bool Block, uint32_t StartOffset = 0);
+  virtual void printBinaryImpl(StringRef Label, StringRef Str,
+                               ArrayRef<uint8_t> Value, bool Block,
+                               uint32_t StartOffset = 0);
+
+  virtual void printFlagsImpl(StringRef Label, HexNumber Value,
+                              ArrayRef<FlagEntry> Flags) {
+    startLine() << Label << " [ (" << Value << ")\n";
+    for (const auto &Flag : Flags)
+      startLine() << "  " << Flag.Name << " (" << hex(Flag.Value) << ")\n";
+    startLine() << "]\n";
+  }
+
+  virtual void printFlagsImpl(StringRef Label, HexNumber Value,
+                              ArrayRef<HexNumber> Flags) {
+    startLine() << Label << " [ (" << Value << ")\n";
+    for (const auto &Flag : Flags)
+      startLine() << "  " << Flag << '\n';
+    startLine() << "]\n";
+  }
+
+  template <typename T> void printListImpl(StringRef Label, const T List) {
+    startLine() << Label << ": [";
+    ListSeparator LS;
+    for (const auto &Item : List)
+      OS << LS << Item;
+    OS << "]\n";
+  }
+
+  virtual void printHexListImpl(StringRef Label,
+                                const ArrayRef<HexNumber> List) {
+    startLine() << Label << ": [";
+    ListSeparator LS;
+    for (const auto &Item : List)
+      OS << LS << hex(Item);
+    OS << "]\n";
+  }
+
+  virtual void printHexImpl(StringRef Label, HexNumber Value) {
+    startLine() << Label << ": " << Value << "\n";
+  }
+
+  virtual void printHexImpl(StringRef Label, StringRef Str, HexNumber Value) {
+    startLine() << Label << ": " << Str << " (" << Value << ")\n";
+  }
+
+  virtual void printSymbolOffsetImpl(StringRef Label, StringRef Symbol,
+                                     HexNumber Value) {
+    startLine() << Label << ": " << Symbol << '+' << Value << '\n';
+  }
+
+  virtual void printNumberImpl(StringRef Label, StringRef Str,
+                               StringRef Value) {
+    startLine() << Label << ": " << Str << " (" << Value << ")\n";
+  }
+
+  void scopedBegin(char Symbol) {
+    startLine() << Symbol << '\n';
+    indent();
+  }
+
+  void scopedBegin(StringRef Label, char Symbol) {
+    startLine() << Label;
+    if (!Label.empty())
+      OS << ' ';
+    OS << Symbol << '\n';
+    indent();
+  }
+
+  void scopedEnd(char Symbol) {
+    unindent();
+    startLine() << Symbol << '\n';
+  }
 
   raw_ostream &OS;
   int IndentLevel;
   StringRef Prefix;
+  ScopedPrinterKind Kind;
 };
 
 template <>
@@ -349,31 +511,330 @@ ScopedPrinter::printHex<support::ulittle16_t>(StringRef Label,
   startLine() << Label << ": " << hex(Value) << "\n";
 }
 
-template<char Open, char Close>
-struct DelimitedScope {
-  explicit DelimitedScope(ScopedPrinter &W) : W(W) {
-    W.startLine() << Open << '\n';
-    W.indent();
+struct DelimitedScope;
+
+class JSONScopedPrinter : public ScopedPrinter {
+private:
+  enum class Scope {
+    Array,
+    Object,
+  };
+
+  enum class ScopeKind {
+    NoAttribute,
+    Attribute,
+    NestedAttribute,
+  };
+
+  struct ScopeContext {
+    Scope Context;
+    ScopeKind Kind;
+    ScopeContext(Scope Context, ScopeKind Kind = ScopeKind::NoAttribute)
+        : Context(Context), Kind(Kind) {}
+  };
+
+  SmallVector<ScopeContext, 8> ScopeHistory;
+  json::OStream JOS;
+  std::unique_ptr<DelimitedScope> OuterScope;
+
+public:
+  JSONScopedPrinter(raw_ostream &OS, bool PrettyPrint = false,
+                    std::unique_ptr<DelimitedScope> &&OuterScope =
+                        std::unique_ptr<DelimitedScope>{});
+
+  static bool classof(const ScopedPrinter *SP) {
+    return SP->getKind() == ScopedPrinter::ScopedPrinterKind::JSON;
+  }
+
+  void printNumber(StringRef Label, uint64_t Value) override {
+    JOS.attribute(Label, Value);
+  }
+
+  void printNumber(StringRef Label, uint32_t Value) override {
+    JOS.attribute(Label, Value);
+  }
+
+  void printNumber(StringRef Label, uint16_t Value) override {
+    JOS.attribute(Label, Value);
+  }
+
+  void printNumber(StringRef Label, uint8_t Value) override {
+    JOS.attribute(Label, Value);
+  }
+
+  void printNumber(StringRef Label, int64_t Value) override {
+    JOS.attribute(Label, Value);
+  }
+
+  void printNumber(StringRef Label, int32_t Value) override {
+    JOS.attribute(Label, Value);
+  }
+
+  void printNumber(StringRef Label, int16_t Value) override {
+    JOS.attribute(Label, Value);
+  }
+
+  void printNumber(StringRef Label, int8_t Value) override {
+    JOS.attribute(Label, Value);
+  }
+
+  void printNumber(StringRef Label, const APSInt &Value) override {
+    JOS.attributeBegin(Label);
+    printAPSInt(Value);
+    JOS.attributeEnd();
+  }
+
+  void printBoolean(StringRef Label, bool Value) override {
+    JOS.attribute(Label, Value);
+  }
+
+  void printList(StringRef Label, const ArrayRef<bool> List) override {
+    printListImpl(Label, List);
+  }
+
+  void printList(StringRef Label, const ArrayRef<std::string> List) override {
+    printListImpl(Label, List);
+  }
+
+  void printList(StringRef Label, const ArrayRef<uint64_t> List) override {
+    printListImpl(Label, List);
+  }
+
+  void printList(StringRef Label, const ArrayRef<uint32_t> List) override {
+    printListImpl(Label, List);
+  }
+
+  void printList(StringRef Label, const ArrayRef<uint16_t> List) override {
+    printListImpl(Label, List);
+  }
+
+  void printList(StringRef Label, const ArrayRef<uint8_t> List) override {
+    printListImpl(Label, List);
+  }
+
+  void printList(StringRef Label, const ArrayRef<int64_t> List) override {
+    printListImpl(Label, List);
+  }
+
+  void printList(StringRef Label, const ArrayRef<int32_t> List) override {
+    printListImpl(Label, List);
   }
 
-  DelimitedScope(ScopedPrinter &W, StringRef N) : W(W) {
-    W.startLine() << N;
-    if (!N.empty())
-      W.getOStream() << ' ';
-    W.getOStream() << Open << '\n';
-    W.indent();
+  void printList(StringRef Label, const ArrayRef<int16_t> List) override {
+    printListImpl(Label, List);
   }
 
-  ~DelimitedScope() {
-    W.unindent();
-    W.startLine() << Close << '\n';
+  void printList(StringRef Label, const ArrayRef<int8_t> List) override {
+    printListImpl(Label, List);
   }
 
-  ScopedPrinter &W;
+  void printList(StringRef Label, const ArrayRef<APSInt> List) override {
+    JOS.attributeArray(Label, [&]() {
+      for (const APSInt &Item : List) {
+        printAPSInt(Item);
+      }
+    });
+  }
+
+  void printString(StringRef Value) override { JOS.value(Value); }
+
+  void printString(StringRef Label, StringRef Value) override {
+    JOS.attribute(Label, Value);
+  }
+
+  void objectBegin() override {
+    scopedBegin({Scope::Object, ScopeKind::NoAttribute});
+  }
+
+  void objectBegin(StringRef Label) override {
+    scopedBegin(Label, Scope::Object);
+  }
+
+  void objectEnd() override { scopedEnd(); }
+
+  void arrayBegin() override {
+    scopedBegin({Scope::Array, ScopeKind::NoAttribute});
+  }
+
+  void arrayBegin(StringRef Label) override {
+    scopedBegin(Label, Scope::Array);
+  }
+
+  void arrayEnd() override { scopedEnd(); }
+
+private:
+  // Output HexNumbers as decimals so that they're easier to parse.
+  uint64_t hexNumberToInt(HexNumber Hex) { return Hex.Value; }
+
+  void printAPSInt(const APSInt &Value) {
+    JOS.rawValueBegin() << Value;
+    JOS.rawValueEnd();
+  }
+
+  void printFlagsImpl(StringRef Label, HexNumber Value,
+                      ArrayRef<FlagEntry> Flags) override {
+    JOS.attributeObject(Label, [&]() {
+      JOS.attribute("RawFlags", hexNumberToInt(Value));
+      JOS.attributeArray("Flags", [&]() {
+        for (const FlagEntry &Flag : Flags) {
+          JOS.objectBegin();
+          JOS.attribute("Name", Flag.Name);
+          JOS.attribute("Value", Flag.Value);
+          JOS.objectEnd();
+        }
+      });
+    });
+  }
+
+  void printFlagsImpl(StringRef Label, HexNumber Value,
+                      ArrayRef<HexNumber> Flags) override {
+    JOS.attributeObject(Label, [&]() {
+      JOS.attribute("RawFlags", hexNumberToInt(Value));
+      JOS.attributeArray("Flags", [&]() {
+        for (const HexNumber &Flag : Flags) {
+          JOS.value(Flag.Value);
+        }
+      });
+    });
+  }
+
+  template <typename T> void printListImpl(StringRef Label, const T &List) {
+    JOS.attributeArray(Label, [&]() {
+      for (const auto &Item : List)
+        JOS.value(Item);
+    });
+  }
+
+  void printHexListImpl(StringRef Label,
+                        const ArrayRef<HexNumber> List) override {
+    JOS.attributeArray(Label, [&]() {
+      for (const HexNumber &Item : List) {
+        JOS.value(hexNumberToInt(Item));
+      }
+    });
+  }
+
+  void printHexImpl(StringRef Label, HexNumber Value) override {
+    JOS.attribute(Label, hexNumberToInt(Value));
+  }
+
+  void printHexImpl(StringRef Label, StringRef Str, HexNumber Value) override {
+    JOS.attributeObject(Label, [&]() {
+      JOS.attribute("Value", Str);
+      JOS.attribute("RawValue", hexNumberToInt(Value));
+    });
+  }
+
+  void printSymbolOffsetImpl(StringRef Label, StringRef Symbol,
+                             HexNumber Value) override {
+    JOS.attributeObject(Label, [&]() {
+      JOS.attribute("SymName", Symbol);
+      JOS.attribute("Offset", hexNumberToInt(Value));
+    });
+  }
+
+  void printNumberImpl(StringRef Label, StringRef Str,
+                       StringRef Value) override {
+    JOS.attributeObject(Label, [&]() {
+      JOS.attribute("Value", Str);
+      JOS.attributeBegin("RawValue");
+      JOS.rawValueBegin() << Value;
+      JOS.rawValueEnd();
+      JOS.attributeEnd();
+    });
+  }
+
+  void printBinaryImpl(StringRef Label, StringRef Str, ArrayRef<uint8_t> Value,
+                       bool Block, uint32_t StartOffset = 0) override {
+    JOS.attributeObject(Label, [&]() {
+      if (!Str.empty())
+        JOS.attribute("Value", Str);
+      JOS.attribute("Offset", StartOffset);
+      JOS.attributeArray("Bytes", [&]() {
+        for (uint8_t Val : Value)
+          JOS.value(Val);
+      });
+    });
+  }
+
+  void scopedBegin(ScopeContext ScopeCtx) {
+    if (ScopeCtx.Context == Scope::Object)
+      JOS.objectBegin();
+    else if (ScopeCtx.Context == Scope::Array)
+      JOS.arrayBegin();
+    ScopeHistory.push_back(ScopeCtx);
+  }
+
+  void scopedBegin(StringRef Label, Scope Ctx) {
+    ScopeKind Kind = ScopeKind::Attribute;
+    if (ScopeHistory.empty() || ScopeHistory.back().Context != Scope::Object) {
+      JOS.objectBegin();
+      Kind = ScopeKind::NestedAttribute;
+    }
+    JOS.attributeBegin(Label);
+    scopedBegin({Ctx, Kind});
+  }
+
+  void scopedEnd() {
+    ScopeContext ScopeCtx = ScopeHistory.back();
+    if (ScopeCtx.Context == Scope::Object)
+      JOS.objectEnd();
+    else if (ScopeCtx.Context == Scope::Array)
+      JOS.arrayEnd();
+    if (ScopeCtx.Kind == ScopeKind::Attribute ||
+        ScopeCtx.Kind == ScopeKind::NestedAttribute)
+      JOS.attributeEnd();
+    if (ScopeCtx.Kind == ScopeKind::NestedAttribute)
+      JOS.objectEnd();
+    ScopeHistory.pop_back();
+  }
+};
+
+struct DelimitedScope {
+  DelimitedScope(ScopedPrinter &W) : W(&W) {}
+  DelimitedScope() : W(nullptr) {}
+  virtual ~DelimitedScope(){};
+  virtual void setPrinter(ScopedPrinter &W) = 0;
+  ScopedPrinter *W;
+};
+
+struct DictScope : DelimitedScope {
+  explicit DictScope() : DelimitedScope() {}
+  explicit DictScope(ScopedPrinter &W) : DelimitedScope(W) { W.objectBegin(); }
+
+  DictScope(ScopedPrinter &W, StringRef N) : DelimitedScope(W) {
+    W.objectBegin(N);
+  }
+
+  void setPrinter(ScopedPrinter &W) override {
+    this->W = &W;
+    W.objectBegin();
+  }
+
+  ~DictScope() {
+    if (W)
+      W->objectEnd();
+  }
 };
 
-using DictScope = DelimitedScope<'{', '}'>;
-using ListScope = DelimitedScope<'[', ']'>;
+struct ListScope : DelimitedScope {
+  explicit ListScope() : DelimitedScope() {}
+  explicit ListScope(ScopedPrinter &W) : DelimitedScope(W) { W.arrayBegin(); }
+
+  ListScope(ScopedPrinter &W, StringRef N) : DelimitedScope(W) {
+    W.arrayBegin(N);
+  }
+
+  void setPrinter(ScopedPrinter &W) override {
+    this->W = &W;
+    W.arrayBegin();
+  }
+
+  ~ListScope() {
+    if (W)
+      W->arrayEnd();
+  }
+};
 
 } // namespace llvm
 
diff --git a/llvm/include/llvm/Support/SmallVectorMemoryBuffer.h b/llvm/include/llvm/Support/SmallVectorMemoryBuffer.h
index 9aa4e9aec266..f7f2d4e54e70 100644
--- a/llvm/include/llvm/Support/SmallVectorMemoryBuffer.h
+++ b/llvm/include/llvm/Support/SmallVectorMemoryBuffer.h
@@ -28,23 +28,21 @@ namespace llvm {
 /// MemoryBuffer).
 class SmallVectorMemoryBuffer : public MemoryBuffer {
 public:
-  /// Construct an SmallVectorMemoryBuffer from the given SmallVector
-  /// r-value.
-  ///
-  /// FIXME: It'd be nice for this to be a non-templated constructor taking a
-  /// SmallVectorImpl here instead of a templated one taking a SmallVector<N>,
-  /// but SmallVector's move-construction/assignment currently only take
-  /// SmallVectors. If/when that is fixed we can simplify this constructor and
-  /// the following one.
-  SmallVectorMemoryBuffer(SmallVectorImpl<char> &&SV)
-      : SV(std::move(SV)), BufferName("<in-memory object>") {
-    init(this->SV.begin(), this->SV.end(), false);
-  }
-
-  /// Construct a named SmallVectorMemoryBuffer from the given
-  /// SmallVector r-value and StringRef.
-  SmallVectorMemoryBuffer(SmallVectorImpl<char> &&SV, StringRef Name)
+  /// Construct a SmallVectorMemoryBuffer from the given SmallVector r-value.
+  SmallVectorMemoryBuffer(SmallVectorImpl<char> &&SV,
+                          bool RequiresNullTerminator = true)
+      : SmallVectorMemoryBuffer(std::move(SV), "<in-memory object>",
+                                RequiresNullTerminator) {}
+
+  /// Construct a named SmallVectorMemoryBuffer from the given SmallVector
+  /// r-value and StringRef.
+  SmallVectorMemoryBuffer(SmallVectorImpl<char> &&SV, StringRef Name,
+                          bool RequiresNullTerminator = true)
       : SV(std::move(SV)), BufferName(std::string(Name)) {
+    if (RequiresNullTerminator) {
+      this->SV.push_back('\0');
+      this->SV.pop_back();
+    }
     init(this->SV.begin(), this->SV.end(), false);
   }
 
diff --git a/llvm/include/llvm/Support/TargetParser.h b/llvm/include/llvm/Support/TargetParser.h
index b11467dcce28..01e25a0ea857 100644
--- a/llvm/include/llvm/Support/TargetParser.h
+++ b/llvm/include/llvm/Support/TargetParser.h
@@ -17,8 +17,9 @@
 // FIXME: vector is used because that's what clang uses for subtarget feature
 // lists, but SmallVector would probably be better
 #include "llvm/ADT/Triple.h"
-#include "llvm/Support/ARMTargetParser.h"
 #include "llvm/Support/AArch64TargetParser.h"
+#include "llvm/Support/ARMTargetParser.h"
+#include "llvm/Support/RISCVISAInfo.h"
 #include <vector>
 
 namespace llvm {
@@ -174,6 +175,7 @@ void fillValidCPUArchList(SmallVectorImpl<StringRef> &Values, bool IsRV64);
 void fillValidTuneCPUArchList(SmallVectorImpl<StringRef> &Values, bool IsRV64);
 bool getCPUFeaturesExceptStdExt(CPUKind Kind, std::vector<StringRef> &Features);
 StringRef resolveTuneCPUAlias(StringRef TuneCPU, bool IsRV64);
+StringRef computeDefaultABIFromArch(const llvm::RISCVISAInfo &ISAInfo);
 
 } // namespace RISCV
 
diff --git a/llvm/include/llvm/Support/ThreadPool.h b/llvm/include/llvm/Support/ThreadPool.h
index 8d30e8e92755..aecff122d3cb 100644
--- a/llvm/include/llvm/Support/ThreadPool.h
+++ b/llvm/include/llvm/Support/ThreadPool.h
@@ -65,7 +65,10 @@ public:
   /// It is an error to try to add new tasks while blocking on this call.
   void wait();
 
-  unsigned getThreadCount() const { return ThreadCount; }
+  // TODO: misleading legacy name warning!
+  // Returns the maximum number of worker threads in the pool, not the current
+  // number of threads!
+  unsigned getThreadCount() const { return MaxThreadCount; }
 
   /// Returns true if the current thread is a worker thread of this thread pool.
   bool isWorkerThread() const;
@@ -108,6 +111,7 @@ private:
     /// corresponding future.
     auto R = createTaskAndFuture(Task);
 
+    int requestedThreads;
     {
       // Lock the queue and push the new task
       std::unique_lock<std::mutex> LockGuard(QueueLock);
@@ -115,8 +119,10 @@ private:
       // Don't allow enqueueing after disabling the pool
       assert(EnableFlag && "Queuing a thread during ThreadPool destruction");
       Tasks.push(std::move(R.first));
+      requestedThreads = ActiveThreads + Tasks.size();
     }
     QueueCondition.notify_one();
+    grow(requestedThreads);
     return R.second.share();
 
 #else // LLVM_ENABLE_THREADS Disabled
@@ -130,8 +136,16 @@ private:
 #endif
   }
 
+#if LLVM_ENABLE_THREADS
+  // Grow to ensure that we have at least `requested` Threads, but do not go
+  // over MaxThreadCount.
+  void grow(int requested);
+#endif
+
   /// Threads in flight
   std::vector<llvm::thread> Threads;
+  /// Lock protecting access to the Threads vector.
+  mutable std::mutex ThreadsLock;
 
   /// Tasks waiting for execution in the pool.
   std::queue<std::function<void()>> Tasks;
@@ -151,7 +165,10 @@ private:
   bool EnableFlag = true;
 #endif
 
-  unsigned ThreadCount;
+  const ThreadPoolStrategy Strategy;
+
+  /// Maximum number of threads to potentially grow this pool to.
+  const unsigned MaxThreadCount;
 };
 }
 
diff --git a/llvm/include/llvm/Support/ToolOutputFile.h b/llvm/include/llvm/Support/ToolOutputFile.h
index ec1d6ae52268..6b7222550b9f 100644
--- a/llvm/include/llvm/Support/ToolOutputFile.h
+++ b/llvm/include/llvm/Support/ToolOutputFile.h
@@ -29,9 +29,10 @@ class ToolOutputFile {
   /// raw_fd_ostream is destructed. It installs cleanups in its constructor and
   /// uninstalls them in its destructor.
   class CleanupInstaller {
+  public:
     /// The name of the file.
     std::string Filename;
-  public:
+
     /// The flag which indicates whether we should not delete the file.
     bool Keep;
 
@@ -64,6 +65,8 @@ public:
   /// Indicate that the tool's job wrt this output file has been successful and
   /// the file should not be deleted.
   void keep() { Installer.Keep = true; }
+
+  const std::string &outputFilename() { return Installer.Filename; }
 };
 
 } // end llvm namespace
diff --git a/llvm/include/llvm/Support/VirtualFileSystem.h b/llvm/include/llvm/Support/VirtualFileSystem.h
index 10d2389ee079..78bcdbf3e932 100644
--- a/llvm/include/llvm/Support/VirtualFileSystem.h
+++ b/llvm/include/llvm/Support/VirtualFileSystem.h
@@ -64,6 +64,8 @@ public:
          uint64_t Size, llvm::sys::fs::file_type Type,
          llvm::sys::fs::perms Perms);
 
+  /// Get a copy of a Status with a different size.
+  static Status copyWithNewSize(const Status &In, uint64_t NewSize);
   /// Get a copy of a Status with a different name.
   static Status copyWithNewName(const Status &In, const Twine &NewName);
   static Status copyWithNewName(const llvm::sys::fs::file_status &In,
diff --git a/llvm/include/llvm/Target/TargetOptions.h b/llvm/include/llvm/Target/TargetOptions.h
index 912f6d1c153a..c639f326abc9 100644
--- a/llvm/include/llvm/Target/TargetOptions.h
+++ b/llvm/include/llvm/Target/TargetOptions.h
@@ -418,6 +418,11 @@ namespace llvm {
 
     /// Machine level options.
     MCTargetOptions MCOptions;
+
+    /// Stores the filename/path of the final .o/.obj file, to be written in the
+    /// debug information. This is used for emitting the CodeView S_OBJNAME
+    /// record.
+    std::string ObjectFilenameForDebug;
   };
 
 } // End llvm namespace
diff --git a/llvm/include/llvm/TextAPI/InterfaceFile.h b/llvm/include/llvm/TextAPI/InterfaceFile.h
index 03a541454e1a..6ef4db2ae158 100644
--- a/llvm/include/llvm/TextAPI/InterfaceFile.h
+++ b/llvm/include/llvm/TextAPI/InterfaceFile.h
@@ -381,6 +381,8 @@ public:
     return {Symbols.begin(), Symbols.end()};
   }
 
+  size_t symbolsCount() const { return Symbols.size(); }
+
   const_filtered_symbol_range exports() const {
     std::function<bool(const Symbol *)> fn = [](const Symbol *Symbol) {
       return !Symbol->isUndefined();
diff --git a/llvm/include/llvm/Transforms/IPO/ProfiledCallGraph.h b/llvm/include/llvm/Transforms/IPO/ProfiledCallGraph.h
index 429fcbd81b45..893654650caa 100644
--- a/llvm/include/llvm/Transforms/IPO/ProfiledCallGraph.h
+++ b/llvm/include/llvm/Transforms/IPO/ProfiledCallGraph.h
@@ -68,7 +68,8 @@ public:
 
   // Constructor for non-CS profile.
   ProfiledCallGraph(SampleProfileMap &ProfileMap) {
-    assert(!FunctionSamples::ProfileIsCS && "CS profile is not handled here");
+    assert(!FunctionSamples::ProfileIsCSFlat &&
+           "CS flat profile is not handled here");
     for (const auto &Samples : ProfileMap) {
       addProfiledCalls(Samples.second);
     }
diff --git a/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h b/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h
index 5d80da407d7e..cf87d028600f 100644
--- a/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h
+++ b/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h
@@ -66,8 +66,6 @@ public:
   void dumpTree();
 
 private:
-  static uint64_t nodeHash(StringRef ChildName, const LineLocation &Callsite);
-
   // Map line+discriminator location to child context
   std::map<uint64_t, ContextTrieNode> AllChildContext;
 
diff --git a/llvm/include/llvm/Transforms/Scalar/FlattenCFG.h b/llvm/include/llvm/Transforms/Scalar/FlattenCFG.h
new file mode 100644
index 000000000000..ff49a4ab7ceb
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Scalar/FlattenCFG.h
@@ -0,0 +1,25 @@
+//===- FlattenCFG.h -------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// The FlattenCFG pass flattens a function's CFG using the FlattenCFG utility
+// function, iteratively flattening until no further changes are made.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_SCALAR_FLATTENCFG_H
+#define LLVM_TRANSFORMS_SCALAR_FLATTENCFG_H
+
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+struct FlattenCFGPass : PassInfoMixin<FlattenCFGPass> {
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+};
+} // namespace llvm
+
+#endif // LLVM_TRANSFORMS_SCALAR_FLATTENCFG_H
diff --git a/llvm/include/llvm/Transforms/Utils/CodeLayout.h b/llvm/include/llvm/Transforms/Utils/CodeLayout.h
new file mode 100644
index 000000000000..987a5651a8b6
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Utils/CodeLayout.h
@@ -0,0 +1,58 @@
+//===- CodeLayout.h - Code layout/placement algorithms  ---------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// Declares methods and data structures for code layout algorithms.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_UTILS_CODELAYOUT_H
+#define LLVM_TRANSFORMS_UTILS_CODELAYOUT_H
+
+#include "llvm/ADT/DenseMap.h"
+
+#include <vector>
+
+namespace llvm {
+
+class MachineBasicBlock;
+
+/// Find a layout of nodes (basic blocks) of a given CFG optimizing jump
+/// locality and thus processor I-cache utilization. This is achieved via
+/// increasing the number of fall-through jumps and co-locating frequently
+/// executed nodes together.
+/// The nodes are assumed to be indexed by integers from [0, |V|) so that the
+/// current order is the identity permutation.
+/// \p NodeSizes: The sizes of the nodes (in bytes).
+/// \p NodeCounts: The execution counts of the nodes in the profile.
+/// \p EdgeCounts: The execution counts of every edge (jump) in the profile. The
+///    map also defines the edges in CFG and should include 0-count edges.
+/// \returns The best block order found.
+std::vector<uint64_t> applyExtTspLayout(
+    const std::vector<uint64_t> &NodeSizes,
+    const std::vector<uint64_t> &NodeCounts,
+    const DenseMap<std::pair<uint64_t, uint64_t>, uint64_t> &EdgeCounts);
+
+/// Estimate the "quality" of a given node order in CFG. The higher the score,
+/// the better the order is. The score is designed to reflect the locality of
+/// the given order, which is anti-correlated with the number of I-cache misses
+/// in a typical execution of the function.
+double calcExtTspScore(
+    const std::vector<uint64_t> &Order, const std::vector<uint64_t> &NodeSizes,
+    const std::vector<uint64_t> &NodeCounts,
+    const DenseMap<std::pair<uint64_t, uint64_t>, uint64_t> &EdgeCounts);
+
+/// Estimate the "quality" of the current node order in CFG.
+double calcExtTspScore(
+    const std::vector<uint64_t> &NodeSizes,
+    const std::vector<uint64_t> &NodeCounts,
+    const DenseMap<std::pair<uint64_t, uint64_t>, uint64_t> &EdgeCounts);
+
+} // end namespace llvm
+
+#endif // LLVM_TRANSFORMS_UTILS_CODELAYOUT_H
diff --git a/llvm/include/llvm/Transforms/Utils/FunctionComparator.h b/llvm/include/llvm/Transforms/Utils/FunctionComparator.h
index e808a50b320f..964fdce45744 100644
--- a/llvm/include/llvm/Transforms/Utils/FunctionComparator.h
+++ b/llvm/include/llvm/Transforms/Utils/FunctionComparator.h
@@ -320,6 +320,7 @@ protected:
   int cmpTypes(Type *TyL, Type *TyR) const;
 
   int cmpNumbers(uint64_t L, uint64_t R) const;
+  int cmpAligns(Align L, Align R) const;
   int cmpAPInts(const APInt &L, const APInt &R) const;
   int cmpAPFloats(const APFloat &L, const APFloat &R) const;
   int cmpMem(StringRef L, StringRef R) const;
diff --git a/llvm/include/llvm/Transforms/Utils/Local.h b/llvm/include/llvm/Transforms/Utils/Local.h
index 3c529abce85a..a914c6e0925f 100644
--- a/llvm/include/llvm/Transforms/Utils/Local.h
+++ b/llvm/include/llvm/Transforms/Utils/Local.h
@@ -89,6 +89,14 @@ bool isInstructionTriviallyDead(Instruction *I,
 bool wouldInstructionBeTriviallyDead(Instruction *I,
                                      const TargetLibraryInfo *TLI = nullptr);
 
+/// Return true if the result produced by the instruction has no side effects on
+/// any paths other than where it is used. This is less conservative than 
+/// wouldInstructionBeTriviallyDead which is based on the assumption
+/// that the use count will be 0. An example usage of this API is for
+/// identifying instructions that can be sunk down to use(s).
+bool wouldInstructionBeTriviallyDeadOnUnusedPaths(
+    Instruction *I, const TargetLibraryInfo *TLI = nullptr);
+
 /// If the specified value is a trivially dead instruction, delete it.
 /// If that makes any of its operands trivially dead, delete them too,
 /// recursively. Return true if any instructions were deleted.
diff --git a/llvm/include/llvm/Transforms/Utils/LoopUtils.h b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
index 30c3f71e0947..e0a9115f61b0 100644
--- a/llvm/include/llvm/Transforms/Utils/LoopUtils.h
+++ b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
@@ -367,14 +367,12 @@ Value *createMinMaxOp(IRBuilderBase &Builder, RecurKind RK, Value *Left,
 
 /// Generates an ordered vector reduction using extracts to reduce the value.
 Value *getOrderedReduction(IRBuilderBase &Builder, Value *Acc, Value *Src,
-                           unsigned Op, RecurKind MinMaxKind = RecurKind::None,
-                           ArrayRef<Value *> RedOps = None);
+                           unsigned Op, RecurKind MinMaxKind = RecurKind::None);
 
 /// Generates a vector reduction using shufflevectors to reduce the value.
 /// Fast-math-flags are propagated using the IRBuilder's setting.
 Value *getShuffleReduction(IRBuilderBase &Builder, Value *Src, unsigned Op,
-                           RecurKind MinMaxKind = RecurKind::None,
-                           ArrayRef<Value *> RedOps = None);
+                           RecurKind MinMaxKind = RecurKind::None);
 
 /// Create a target reduction of the given vector. The reduction operation
 /// is described by the \p Opcode parameter. min/max reductions require
@@ -384,8 +382,7 @@ Value *getShuffleReduction(IRBuilderBase &Builder, Value *Src, unsigned Op,
 /// Fast-math-flags are propagated using the IRBuilder's setting.
 Value *createSimpleTargetReduction(IRBuilderBase &B,
                                    const TargetTransformInfo *TTI, Value *Src,
-                                   RecurKind RdxKind,
-                                   ArrayRef<Value *> RedOps = None);
+                                   RecurKind RdxKind);
 
 /// Create a target reduction of the given vector \p Src for a reduction of the
 /// kind RecurKind::SelectICmp or RecurKind::SelectFCmp. The reduction operation
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index ed9e0beb0339..32d295a2dd16 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -29,6 +29,7 @@
 #include "llvm/ADT/MapVector.h"
 #include "llvm/Analysis/LoopAccessAnalysis.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Support/TypeSize.h"
 #include "llvm/Transforms/Utils/LoopUtils.h"
 
@@ -104,14 +105,12 @@ public:
     /// Vectorize loops using scalable vectors or fixed-width vectors, but favor
     /// scalable vectors when the cost-model is inconclusive. This is the
     /// default when the scalable.enable hint is enabled through a pragma.
-    SK_PreferScalable = 1,
-    /// Vectorize loops using scalable vectors or fixed-width  vectors, but
-    /// favor fixed-width vectors when the cost is inconclusive.
-    SK_PreferFixedWidth = 2,
+    SK_PreferScalable = 1
   };
 
   LoopVectorizeHints(const Loop *L, bool InterleaveOnlyWhenForced,
-                     OptimizationRemarkEmitter &ORE);
+                     OptimizationRemarkEmitter &ORE,
+                     const TargetTransformInfo *TTI = nullptr);
 
   /// Mark the loop L as already vectorized by setting the width to 1.
   void setAlreadyVectorized();
@@ -123,9 +122,10 @@ public:
   void emitRemarkWithHints() const;
 
   ElementCount getWidth() const {
-    return ElementCount::get(Width.Value,
-                             isScalableVectorizationExplicitlyEnabled());
+    return ElementCount::get(Width.Value, (ScalableForceKind)Scalable.Value ==
+                                              SK_PreferScalable);
   }
+
   unsigned getInterleave() const {
     if (Interleave.Value)
       return Interleave.Value;
@@ -144,22 +144,9 @@ public:
     return (ForceKind)Force.Value;
   }
 
-  /// \return true if the cost-model for scalable vectorization should
-  /// favor vectorization with scalable vectors over fixed-width vectors when
-  /// the cost-model is inconclusive.
-  bool isScalableVectorizationPreferred() const {
-    return Scalable.Value == SK_PreferScalable;
-  }
-
-  /// \return true if scalable vectorization has been explicitly enabled.
-  bool isScalableVectorizationExplicitlyEnabled() const {
-    return Scalable.Value == SK_PreferFixedWidth ||
-           Scalable.Value == SK_PreferScalable;
-  }
-
   /// \return true if scalable vectorization has been explicitly disabled.
   bool isScalableVectorizationDisabled() const {
-    return Scalable.Value == SK_FixedWidthOnly;
+    return (ScalableForceKind)Scalable.Value == SK_FixedWidthOnly;
   }
 
   /// If hints are provided that force vectorization, use the AlwaysPrint
@@ -293,10 +280,10 @@ public:
   PHINode *getPrimaryInduction() { return PrimaryInduction; }
 
   /// Returns the reduction variables found in the loop.
-  ReductionList &getReductionVars() { return Reductions; }
+  const ReductionList &getReductionVars() const { return Reductions; }
 
   /// Returns the induction variables found in the loop.
-  InductionList &getInductionVars() { return Inductions; }
+  const InductionList &getInductionVars() const { return Inductions; }
 
   /// Return the first-order recurrences found in the loop.
   RecurrenceSet &getFirstOrderRecurrences() { return FirstOrderRecurrences; }
@@ -308,23 +295,27 @@ public:
   Type *getWidestInductionType() { return WidestIndTy; }
 
   /// Returns True if V is a Phi node of an induction variable in this loop.
-  bool isInductionPhi(const Value *V);
+  bool isInductionPhi(const Value *V) const;
+
+  /// Returns a pointer to the induction descriptor, if \p Phi is an integer or
+  /// floating point induction.
+  const InductionDescriptor *getIntOrFpInductionDescriptor(PHINode *Phi) const;
 
   /// Returns True if V is a cast that is part of an induction def-use chain,
   /// and had been proven to be redundant under a runtime guard (in other
   /// words, the cast has the same SCEV expression as the induction phi).
-  bool isCastedInductionVariable(const Value *V);
+  bool isCastedInductionVariable(const Value *V) const;
 
   /// Returns True if V can be considered as an induction variable in this
   /// loop. V can be the induction phi, or some redundant cast in the def-use
   /// chain of the inducion phi.
-  bool isInductionVariable(const Value *V);
+  bool isInductionVariable(const Value *V) const;
 
   /// Returns True if PN is a reduction variable in this loop.
-  bool isReductionVariable(PHINode *PN) { return Reductions.count(PN); }
+  bool isReductionVariable(PHINode *PN) const { return Reductions.count(PN); }
 
   /// Returns True if Phi is a first-order recurrence in this loop.
-  bool isFirstOrderRecurrence(const PHINode *Phi);
+  bool isFirstOrderRecurrence(const PHINode *Phi) const;
 
   /// Return true if the block BB needs to be predicated in order for the loop
   /// to be vectorized.
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h
index d105496ad47f..463b5a058052 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h
@@ -80,6 +80,38 @@ class TargetTransformInfo;
 extern cl::opt<bool> EnableLoopInterleaving;
 extern cl::opt<bool> EnableLoopVectorization;
 
+/// A marker to determine if extra passes after loop vectorization should be
+/// run.
+struct ShouldRunExtraVectorPasses
+    : public AnalysisInfoMixin<ShouldRunExtraVectorPasses> {
+  static AnalysisKey Key;
+  struct Result {
+    bool invalidate(Function &F, const PreservedAnalyses &PA,
+                    FunctionAnalysisManager::Invalidator &) {
+      // Check whether the analysis has been explicitly invalidated. Otherwise,
+      // it remains preserved.
+      auto PAC = PA.getChecker<ShouldRunExtraVectorPasses>();
+      return !PAC.preservedWhenStateless();
+    }
+  };
+
+  Result run(Function &F, FunctionAnalysisManager &FAM) { return Result(); }
+};
+
+/// A pass manager to run a set of extra function simplification passes after
+/// vectorization, if requested. LoopVectorize caches the
+/// ShouldRunExtraVectorPasses analysis to request extra simplifications, if
+/// they could be beneficial.
+struct ExtraVectorPassManager : public FunctionPassManager {
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM) {
+    auto PA = PreservedAnalyses::all();
+    if (AM.getCachedResult<ShouldRunExtraVectorPasses>(F))
+      PA.intersect(FunctionPassManager::run(F, AM));
+    PA.abandon<ShouldRunExtraVectorPasses>();
+    return PA;
+  }
+};
+
 struct LoopVectorizeOptions {
   /// If false, consider all loops for interleaving.
   /// If true, only loops that explicitly request interleaving are considered.
diff --git a/llvm/include/llvm/module.modulemap b/llvm/include/llvm/module.modulemap
index 6cbbb9a4028e..b0f7f2120606 100644
--- a/llvm/include/llvm/module.modulemap
+++ b/llvm/include/llvm/module.modulemap
@@ -354,6 +354,7 @@ module LLVM_ProfileData {
   module * { export * }
 
   textual header "ProfileData/InstrProfData.inc"
+  textual header "ProfileData/MemProfData.inc"
 }
 
 // FIXME: Mislayered?
diff --git a/llvm/lib/Analysis/AliasAnalysis.cpp b/llvm/lib/Analysis/AliasAnalysis.cpp
index d030f74481cf..49199060786c 100644
--- a/llvm/lib/Analysis/AliasAnalysis.cpp
+++ b/llvm/lib/Analysis/AliasAnalysis.cpp
@@ -249,11 +249,11 @@ ModRefInfo AAResults::getModRefInfo(const CallBase *Call,
     bool IsMustAlias = true;
     ModRefInfo AllArgsMask = ModRefInfo::NoModRef;
     if (doesAccessArgPointees(MRB)) {
-      for (auto AI = Call->arg_begin(), AE = Call->arg_end(); AI != AE; ++AI) {
-        const Value *Arg = *AI;
+      for (const auto &I : llvm::enumerate(Call->args())) {
+        const Value *Arg = I.value();
         if (!Arg->getType()->isPointerTy())
           continue;
-        unsigned ArgIdx = std::distance(Call->arg_begin(), AI);
+        unsigned ArgIdx = I.index();
         MemoryLocation ArgLoc =
             MemoryLocation::getForArgument(Call, ArgIdx, TLI);
         AliasResult ArgAlias = alias(ArgLoc, Loc, AAQI);
@@ -696,14 +696,16 @@ ModRefInfo AAResults::getModRefInfo(const Instruction *I,
   case Instruction::AtomicRMW:
     return getModRefInfo((const AtomicRMWInst *)I, Loc, AAQIP);
   case Instruction::Call:
-    return getModRefInfo((const CallInst *)I, Loc, AAQIP);
+  case Instruction::CallBr:
   case Instruction::Invoke:
-    return getModRefInfo((const InvokeInst *)I, Loc, AAQIP);
+    return getModRefInfo((const CallBase *)I, Loc, AAQIP);
   case Instruction::CatchPad:
     return getModRefInfo((const CatchPadInst *)I, Loc, AAQIP);
   case Instruction::CatchRet:
     return getModRefInfo((const CatchReturnInst *)I, Loc, AAQIP);
   default:
+    assert(!I->mayReadOrWriteMemory() &&
+           "Unhandled memory access instruction!");
     return ModRefInfo::NoModRef;
   }
 }
diff --git a/llvm/lib/Analysis/Analysis.cpp b/llvm/lib/Analysis/Analysis.cpp
index db5167061509..177f38af13d8 100644
--- a/llvm/lib/Analysis/Analysis.cpp
+++ b/llvm/lib/Analysis/Analysis.cpp
@@ -35,6 +35,7 @@ void llvm::initializeAnalysis(PassRegistry &Registry) {
   initializeCFGOnlyPrinterLegacyPassPass(Registry);
   initializeCFLAndersAAWrapperPassPass(Registry);
   initializeCFLSteensAAWrapperPassPass(Registry);
+  initializeCycleInfoWrapperPassPass(Registry);
   initializeDependenceAnalysisWrapperPassPass(Registry);
   initializeDelinearizationPass(Registry);
   initializeDemandedBitsWrapperPassPass(Registry);
diff --git a/llvm/lib/Analysis/BasicAliasAnalysis.cpp b/llvm/lib/Analysis/BasicAliasAnalysis.cpp
index 88b0f37b1d48..5f1bf2001d47 100644
--- a/llvm/lib/Analysis/BasicAliasAnalysis.cpp
+++ b/llvm/lib/Analysis/BasicAliasAnalysis.cpp
@@ -1699,6 +1699,7 @@ AliasResult BasicAAResult::aliasCheckRecursive(
       return Result;
   } else if (const GEPOperator *GV2 = dyn_cast<GEPOperator>(V2)) {
     AliasResult Result = aliasGEP(GV2, V2Size, V1, V1Size, O2, O1, AAQI);
+    Result.swap();
     if (Result != AliasResult::MayAlias)
       return Result;
   }
@@ -1709,6 +1710,7 @@ AliasResult BasicAAResult::aliasCheckRecursive(
       return Result;
   } else if (const PHINode *PN = dyn_cast<PHINode>(V2)) {
     AliasResult Result = aliasPHI(PN, V2Size, V1, V1Size, AAQI);
+    Result.swap();
     if (Result != AliasResult::MayAlias)
       return Result;
   }
@@ -1719,6 +1721,7 @@ AliasResult BasicAAResult::aliasCheckRecursive(
       return Result;
   } else if (const SelectInst *S2 = dyn_cast<SelectInst>(V2)) {
     AliasResult Result = aliasSelect(S2, V2Size, V1, V1Size, AAQI);
+    Result.swap();
     if (Result != AliasResult::MayAlias)
       return Result;
   }
diff --git a/llvm/lib/Analysis/CaptureTracking.cpp b/llvm/lib/Analysis/CaptureTracking.cpp
index 8955658cb9e7..9b45f455be08 100644
--- a/llvm/lib/Analysis/CaptureTracking.cpp
+++ b/llvm/lib/Analysis/CaptureTracking.cpp
@@ -346,13 +346,16 @@ void llvm::PointerMayBeCaptured(const Value *V, CaptureTracker *Tracker,
           if (Tracker->captured(U))
             return;
 
-      // Not captured if only passed via 'nocapture' arguments.  Note that
-      // calling a function pointer does not in itself cause the pointer to
+      // Calling a function pointer does not in itself cause the pointer to
       // be captured.  This is a subtle point considering that (for example)
       // the callee might return its own address.  It is analogous to saying
       // that loading a value from a pointer does not cause the pointer to be
       // captured, even though the loaded value might be the pointer itself
       // (think of self-referential objects).
+      if (Call->isCallee(U))
+        break;
+
+      // Not captured if only passed via 'nocapture' arguments.
       if (Call->isDataOperand(U) &&
           !Call->doesNotCapture(Call->getDataOperandNo(U))) {
         // The parameter is not marked 'nocapture' - captured.
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index 3ed3b8902343..922b38e92785 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -352,6 +352,9 @@ Constant *llvm::ConstantFoldLoadThroughBitcast(Constant *C, Type *DestTy,
                                          const DataLayout &DL) {
   do {
     Type *SrcTy = C->getType();
+    if (SrcTy == DestTy)
+      return C;
+
     TypeSize DestSize = DL.getTypeSizeInBits(DestTy);
     TypeSize SrcSize = DL.getTypeSizeInBits(SrcTy);
     if (!TypeSize::isKnownGE(SrcSize, DestSize))
@@ -705,7 +708,8 @@ Constant *llvm::ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty,
   // is all undef or zero, we know what it loads.
   if (auto *GV = dyn_cast<GlobalVariable>(getUnderlyingObject(C))) {
     if (GV->isConstant() && GV->hasDefinitiveInitializer()) {
-      if (GV->getInitializer()->isNullValue())
+      if (GV->getInitializer()->isNullValue() && !Ty->isX86_MMXTy() &&
+          !Ty->isX86_AMXTy())
         return Constant::getNullValue(Ty);
       if (isa<UndefValue>(GV->getInitializer()))
         return UndefValue::get(Ty);
@@ -881,7 +885,7 @@ Constant *SymbolicallyEvaluateGEP(const GEPOperator *GEP,
     InnermostGEP = GEP;
     InBounds &= GEP->isInBounds();
 
-    SmallVector<Value *, 4> NestedOps(GEP->op_begin() + 1, GEP->op_end());
+    SmallVector<Value *, 4> NestedOps(llvm::drop_begin(GEP->operands()));
 
     // Do not try the incorporate the sub-GEP if some index is not a number.
     bool AllConstantInt = true;
@@ -1774,15 +1778,8 @@ static bool mayFoldConstrained(ConstrainedFPIntrinsic *CI,
 
   // If the operation does not change exception status flags, it is safe
   // to fold.
-  if (St == APFloat::opStatus::opOK) {
-    // When FP exceptions are not ignored, intrinsic call will not be
-    // eliminated, because it is considered as having side effect. But we
-    // know that its evaluation does not raise exceptions, so side effect
-    // is absent. To allow removing the call, mark it as not accessing memory.
-    if (EB && *EB != fp::ExceptionBehavior::ebIgnore)
-      CI->addFnAttr(Attribute::ReadNone);
+  if (St == APFloat::opStatus::opOK)
     return true;
-  }
 
   // If evaluation raised FP exception, the result can depend on rounding
   // mode. If the latter is unknown, folding is not possible.
@@ -2960,10 +2957,6 @@ static Constant *ConstantFoldFixedVectorCall(
     if (auto *Op = dyn_cast<ConstantInt>(Operands[0])) {
       unsigned Lanes = FVTy->getNumElements();
       uint64_t Limit = Op->getZExtValue();
-      // vctp64 are currently modelled as returning a v4i1, not a v2i1. Make
-      // sure we get the limit right in that case and set all relevant lanes.
-      if (IntrinsicID == Intrinsic::arm_mve_vctp64)
-        Limit *= 2;
 
       SmallVector<Constant *, 16> NCs;
       for (unsigned i = 0; i < Lanes; i++) {
diff --git a/llvm/lib/Analysis/CycleAnalysis.cpp b/llvm/lib/Analysis/CycleAnalysis.cpp
new file mode 100644
index 000000000000..09c7ee67e05c
--- /dev/null
+++ b/llvm/lib/Analysis/CycleAnalysis.cpp
@@ -0,0 +1,77 @@
+//===- CycleAnalysis.cpp - Compute CycleInfo for LLVM IR ------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Analysis/CycleAnalysis.h"
+#include "llvm/ADT/GenericCycleImpl.h"
+#include "llvm/IR/CFG.h"
+#include "llvm/InitializePasses.h"
+
+using namespace llvm;
+
+template class llvm::GenericCycleInfo<SSAContext>;
+template class llvm::GenericCycle<SSAContext>;
+
+CycleInfo CycleAnalysis::run(Function &F, FunctionAnalysisManager &) {
+  CycleInfo CI;
+  CI.compute(F);
+  return CI;
+}
+
+AnalysisKey CycleAnalysis::Key;
+
+CycleInfoPrinterPass::CycleInfoPrinterPass(raw_ostream &OS) : OS(OS) {}
+
+PreservedAnalyses CycleInfoPrinterPass::run(Function &F,
+                                            FunctionAnalysisManager &AM) {
+  OS << "CycleInfo for function: " << F.getName() << "\n";
+  AM.getResult<CycleAnalysis>(F).print(OS);
+
+  return PreservedAnalyses::all();
+}
+
+//===----------------------------------------------------------------------===//
+//  CycleInfoWrapperPass Implementation
+//===----------------------------------------------------------------------===//
+//
+// The implementation details of the wrapper pass that holds a CycleInfo
+// suitable for use with the legacy pass manager.
+//
+//===----------------------------------------------------------------------===//
+
+char CycleInfoWrapperPass::ID = 0;
+
+CycleInfoWrapperPass::CycleInfoWrapperPass() : FunctionPass(ID) {
+  initializeCycleInfoWrapperPassPass(*PassRegistry::getPassRegistry());
+}
+
+INITIALIZE_PASS_BEGIN(CycleInfoWrapperPass, "cycles", "Cycle Info Analysis",
+                      true, true)
+INITIALIZE_PASS_END(CycleInfoWrapperPass, "cycles", "Cycle Info Analysis", true,
+                    true)
+
+void CycleInfoWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const {
+  AU.setPreservesAll();
+}
+
+bool CycleInfoWrapperPass::runOnFunction(Function &Func) {
+  CI.clear();
+
+  F = &Func;
+  CI.compute(Func);
+  return false;
+}
+
+void CycleInfoWrapperPass::print(raw_ostream &OS, const Module *) const {
+  OS << "CycleInfo for function: " << F->getName() << "\n";
+  CI.print(OS);
+}
+
+void CycleInfoWrapperPass::releaseMemory() {
+  CI.clear();
+  F = nullptr;
+}
diff --git a/llvm/lib/Analysis/DevelopmentModeInlineAdvisor.cpp b/llvm/lib/Analysis/DevelopmentModeInlineAdvisor.cpp
index d87fa849d839..31b2dafa29b4 100644
--- a/llvm/lib/Analysis/DevelopmentModeInlineAdvisor.cpp
+++ b/llvm/lib/Analysis/DevelopmentModeInlineAdvisor.cpp
@@ -16,6 +16,8 @@
 #include "llvm/Analysis/CallGraph.h"
 #include "llvm/Analysis/InlineSizeEstimatorAnalysis.h"
 #include "llvm/Analysis/MLInlineAdvisor.h"
+#include "llvm/Analysis/ModelUnderTrainingRunner.h"
+#include "llvm/Analysis/NoInferenceModelRunner.h"
 #include "llvm/Analysis/Utils/TFUtils.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/Support/CommandLine.h"
@@ -94,7 +96,6 @@ struct InlineEvent {
 /// Because this is a protobuf, we cannot just stream the events as they come.
 /// Internally, TrainingLogger stores data in column-major format, because that
 /// lines up with how TF SequenceExample represents it.
-class ModelUnderTrainingRunner;
 class TrainingLogger final {
 public:
   TrainingLogger(StringRef LogFileName, const ModelUnderTrainingRunner *MUTR);
@@ -261,65 +262,21 @@ private:
   const int64_t Mandatory;
 };
 
-/// A pseudo model runner. We use it to store feature values when collecting
-/// logs for the default policy, but never ask it to 'run'.
-class NoInferenceModelRunner : public MLModelRunner {
-public:
-  NoInferenceModelRunner(LLVMContext &Ctx)
-      : MLModelRunner(Ctx), Features(NumberOfFeatures) {}
-  void setFeature(FeatureIndex Index, int64_t Value) override {
-    Features[static_cast<int>(Index)] = Value;
-  }
-
-  int64_t getFeature(int Index) const override { return Features[Index]; }
-  bool run() override {
-    llvm_unreachable("We shouldn't call run on this model runner.");
-  }
-
-private:
-  InlineFeatures Features;
-};
-
-/// ModelUnderTrainingRunner - training mode implementation. It uses TF C APIs
-/// to dynamically load and evaluate a TF SavedModel
-/// (https://www.tensorflow.org/guide/saved_model). Runtime performance is
-/// sacrificed for ease of use while training.
-class ModelUnderTrainingRunner final : public MLModelRunner {
-public:
-  ModelUnderTrainingRunner(LLVMContext &Ctx, const std::string &ModelPath);
-
-  bool run() override;
+static const std::vector<TensorSpec> TrainingOnlyFeatures{
+    TensorSpec::createSpec<int64_t>(TFFeedPrefix + "inlining_default", {1}),
+    TensorSpec::createSpec<float>(TFFeedPrefix + "discount", {1}),
+    TensorSpec::createSpec<float>(TFFeedPrefix + "reward", {1}),
+    TensorSpec::createSpec<int32_t>(TFFeedPrefix + "step_type", {1})};
 
-  // Disallows copy and assign.
-  ModelUnderTrainingRunner(const ModelUnderTrainingRunner &) = delete;
-  ModelUnderTrainingRunner &
-  operator=(const ModelUnderTrainingRunner &) = delete;
-
-  void setFeature(FeatureIndex Index, int64_t Value) override;
-  int64_t getFeature(int Index) const override;
-  bool isValid() const { return !!Evaluator; }
-
-  const std::vector<LoggedFeatureSpec> &outputLoggedFeatureSpecs() const {
-    return OutputSpecs;
-  }
-
-  const Optional<TFModelEvaluator::EvaluationResult> &
-  lastEvaluationResult() const {
-    return LastEvaluationResult;
-  }
+static const std::vector<TensorSpec> getInputFeatures() {
+  std::vector<TensorSpec> InputSpecs;
+  for (size_t I = 0; I < NumberOfFeatures; ++I)
+    InputSpecs.push_back(
+        TensorSpec::createSpec<int64_t>(TFFeedPrefix + FeatureNameMap[I], {1}));
+  append_range(InputSpecs, TrainingOnlyFeatures);
+  return InputSpecs;
+}
 
-private:
-  std::unique_ptr<TFModelEvaluator> Evaluator;
-  std::vector<LoggedFeatureSpec> OutputSpecs;
-  Optional<TFModelEvaluator::EvaluationResult> LastEvaluationResult;
-
-  // The training framework needs some additional features.
-  const std::vector<TensorSpec> TrainingOnlyFeatures{
-      TensorSpec::createSpec<int64_t>(TFFeedPrefix + "inlining_default", {1}),
-      TensorSpec::createSpec<float>(TFFeedPrefix + "discount", {1}),
-      TensorSpec::createSpec<float>(TFFeedPrefix + "reward", {1}),
-      TensorSpec::createSpec<int32_t>(TFFeedPrefix + "step_type", {1})};
-};
 } // namespace
 
 TrainingLogger::TrainingLogger(StringRef LogFileName,
@@ -353,7 +310,7 @@ void TrainingLogger::logInlineEvent(const InlineEvent &Event,
                                     const MLModelRunner &ModelRunner) {
   size_t CurrentFeature = 0;
   for (; CurrentFeature < NumberOfFeatures; ++CurrentFeature) {
-    int64_t F = ModelRunner.getFeature(CurrentFeature);
+    int64_t F = *ModelRunner.getTensor<int64_t>(CurrentFeature);
     L->logInt64Value(CurrentFeature, &F);
   }
 
@@ -433,7 +390,9 @@ DevelopmentModeMLInlineAdvisor::getAdviceFromModel(
     return MLInlineAdvisor::getAdviceFromModel(CB, ORE);
 
   bool DefaultAdvice = GetDefaultAdvice(CB);
-  auto Recommendation = IsDoingInference ? ModelRunner->run() : DefaultAdvice;
+  auto Recommendation =
+      IsDoingInference ? static_cast<bool>(ModelRunner->evaluate<int64_t>())
+                       : DefaultAdvice;
   return std::make_unique<LoggingMLInlineAdvice>(
       /*Advisor=*/this,
       /*CB=*/CB, /*ORE=*/ORE, /*Recommendation=*/Recommendation,
@@ -458,49 +417,6 @@ size_t DevelopmentModeMLInlineAdvisor::getTotalSizeEstimate() {
   return Ret;
 }
 
-ModelUnderTrainingRunner::ModelUnderTrainingRunner(LLVMContext &Ctx,
-                                                   const std::string &ModelPath)
-    : MLModelRunner(Ctx) {
-  std::vector<TensorSpec> InputSpecs;
-  for (size_t I = 0; I < NumberOfFeatures; ++I)
-    InputSpecs.push_back(
-        TensorSpec::createSpec<int64_t>(TFFeedPrefix + FeatureNameMap[I], {1}));
-  append_range(InputSpecs, TrainingOnlyFeatures);
-  if (auto MaybeOutSpecs =
-          loadOutputSpecs(Ctx, DecisionName, ModelPath, TFOutputSpecOverride))
-    OutputSpecs = std::move(*MaybeOutSpecs);
-  else
-    return;
-
-  Evaluator = std::make_unique<TFModelEvaluator>(
-      ModelPath, InputSpecs, [&](size_t I) { return OutputSpecs[I].Spec; },
-      OutputSpecs.size());
-  if (!Evaluator || !Evaluator->isValid()) {
-    Ctx.emitError("Failed to create inliner saved model evaluator");
-    Evaluator.reset();
-    return;
-  }
-}
-
-bool ModelUnderTrainingRunner::run() {
-  LastEvaluationResult = Evaluator->evaluate();
-  if (!LastEvaluationResult.hasValue()) {
-    Ctx.emitError("Error evaluating model.");
-    return false;
-  }
-  int64_t Decision = *LastEvaluationResult->getTensorValue<int64_t>(0);
-  return static_cast<bool>(Decision);
-}
-
-int64_t ModelUnderTrainingRunner::getFeature(int Index) const {
-  return *Evaluator->getInput<int64_t>(Index);
-}
-
-void ModelUnderTrainingRunner::setFeature(FeatureIndex Index, int64_t Value) {
-  size_t NumericIndex = static_cast<size_t>(Index);
-  *(Evaluator->getInput<int64_t>(NumericIndex)) = Value;
-}
-
 std::unique_ptr<InlineAdvisor> llvm::getDevelopmentModeAdvisor(
     Module &M, ModuleAnalysisManager &MAM,
     std::function<bool(CallBase &)> GetDefaultAdvice) {
@@ -509,10 +425,13 @@ std::unique_ptr<InlineAdvisor> llvm::getDevelopmentModeAdvisor(
   ModelUnderTrainingRunner *MUTRPtr = nullptr;
   bool IsDoingInference = false;
   if (TFModelUnderTrainingPath.empty())
-    Runner.reset(new NoInferenceModelRunner(Ctx));
+    Runner.reset(new NoInferenceModelRunner(Ctx, getInputFeatures()));
   else {
-    auto MUTR = std::make_unique<ModelUnderTrainingRunner>(
-        Ctx, TFModelUnderTrainingPath);
+    std::unique_ptr<ModelUnderTrainingRunner> MUTR;
+    if (auto MaybeOutputSpecs = loadOutputSpecs(
+            Ctx, DecisionName, TFModelUnderTrainingPath, TFOutputSpecOverride))
+      MUTR = std::make_unique<ModelUnderTrainingRunner>(
+          Ctx, TFModelUnderTrainingPath, getInputFeatures(), *MaybeOutputSpecs);
     if (!MUTR || !MUTR->isValid()) {
       Ctx.emitError("Could not load the policy model from the provided path");
       return nullptr;
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index cfe910df4e91..f5fa6748d053 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -933,7 +933,7 @@ bool RecurrenceDescriptor::isFirstOrderRecurrence(
 /// This function returns the identity element (or neutral element) for
 /// the operation K.
 Value *RecurrenceDescriptor::getRecurrenceIdentity(RecurKind K, Type *Tp,
-                                                   FastMathFlags FMF) {
+                                                   FastMathFlags FMF) const {
   switch (K) {
   case RecurKind::Xor:
   case RecurKind::Add:
diff --git a/llvm/lib/Analysis/InlineAdvisor.cpp b/llvm/lib/Analysis/InlineAdvisor.cpp
index 73d1eff1b968..140c88eb8b0d 100644
--- a/llvm/lib/Analysis/InlineAdvisor.cpp
+++ b/llvm/lib/Analysis/InlineAdvisor.cpp
@@ -40,6 +40,10 @@ static cl::opt<bool>
                                    " callsites processed by inliner but decided"
                                    " to be not inlined"));
 
+static cl::opt<bool> EnableInlineDeferral("inline-deferral", cl::init(false),
+                                          cl::Hidden,
+                                          cl::desc("Enable deferred inlining"));
+
 // An integer used to limit the cost of inline deferral.  The default negative
 // number tells shouldBeDeferred to only take the secondary cost into account.
 static cl::opt<int>
@@ -136,8 +140,9 @@ llvm::Optional<llvm::InlineCost> static getDefaultInlineAdvice(
     return getInlineCost(CB, Params, CalleeTTI, GetAssumptionCache, GetTLI,
                          GetBFI, PSI, RemarksEnabled ? &ORE : nullptr);
   };
-  return llvm::shouldInline(CB, GetInlineCost, ORE,
-                            Params.EnableDeferral.getValueOr(false));
+  return llvm::shouldInline(
+      CB, GetInlineCost, ORE,
+      Params.EnableDeferral.getValueOr(EnableInlineDeferral));
 }
 
 std::unique_ptr<InlineAdvice>
@@ -409,8 +414,6 @@ llvm::shouldInline(CallBase &CB,
              << "' in other contexts";
     });
     setInlineRemark(CB, "deferred");
-    // IC does not bool() to false, so get an InlineCost that will.
-    // This will not be inspected to make an error message.
     return None;
   }
 
diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index 22d2ce11cc90..4831b22b1d46 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -2173,6 +2173,15 @@ static Value *SimplifyAndInst(Value *Op0, Value *Op1, const SimplifyQuery &Q,
     }
   }
 
+  // ((X | Y) ^ X ) & ((X | Y) ^ Y) --> 0
+  // ((X | Y) ^ Y ) & ((X | Y) ^ X) --> 0
+  BinaryOperator *Or;
+  if (match(Op0, m_c_Xor(m_Value(X),
+                         m_CombineAnd(m_BinOp(Or),
+                                      m_c_Or(m_Deferred(X), m_Value(Y))))) &&
+      match(Op1, m_c_Xor(m_Specific(Or), m_Specific(Y))))
+    return Constant::getNullValue(Op0->getType());
+
   return nullptr;
 }
 
@@ -2198,6 +2207,18 @@ static Value *simplifyOrLogic(Value *X, Value *Y) {
 
   Value *A, *B;
 
+  // (A ^ B) | (A | B) --> A | B
+  // (A ^ B) | (B | A) --> B | A
+  if (match(X, m_Xor(m_Value(A), m_Value(B))) &&
+      match(Y, m_c_Or(m_Specific(A), m_Specific(B))))
+    return Y;
+
+  // ~(A ^ B) | (A | B) --> -1
+  // ~(A ^ B) | (B | A) --> -1
+  if (match(X, m_Not(m_Xor(m_Value(A), m_Value(B)))) &&
+      match(Y, m_c_Or(m_Specific(A), m_Specific(B))))
+    return ConstantInt::getAllOnesValue(Ty);
+
   // (A & ~B) | (A ^ B) --> A ^ B
   // (~B & A) | (A ^ B) --> A ^ B
   // (A & ~B) | (B ^ A) --> B ^ A
@@ -2214,18 +2235,33 @@ static Value *simplifyOrLogic(Value *X, Value *Y) {
       match(Y, m_c_And(m_Specific(A), m_Specific(B))))
     return X;
 
-  // (A ^ B) | (A | B) --> A | B
-  // (A ^ B) | (B | A) --> B | A
-  if (match(X, m_Xor(m_Value(A), m_Value(B))) &&
-      match(Y, m_c_Or(m_Specific(A), m_Specific(B))))
-    return Y;
-
-  // ~(A ^ B) | (A | B) --> -1
-  // ~(A ^ B) | (B | A) --> -1
-  if (match(X, m_Not(m_Xor(m_Value(A), m_Value(B)))) &&
-      match(Y, m_c_Or(m_Specific(A), m_Specific(B))))
+  // (~A | B) | (A ^ B) --> -1
+  // (~A | B) | (B ^ A) --> -1
+  // (B | ~A) | (A ^ B) --> -1
+  // (B | ~A) | (B ^ A) --> -1
+  if (match(X, m_c_Or(m_Not(m_Value(A)), m_Value(B))) &&
+      match(Y, m_c_Xor(m_Specific(A), m_Specific(B))))
     return ConstantInt::getAllOnesValue(Ty);
 
+  // (~A & B) | ~(A | B) --> ~A
+  // (~A & B) | ~(B | A) --> ~A
+  // (B & ~A) | ~(A | B) --> ~A
+  // (B & ~A) | ~(B | A) --> ~A
+  Value *NotA;
+  if (match(X,
+            m_c_And(m_CombineAnd(m_Value(NotA), m_NotForbidUndef(m_Value(A))),
+                    m_Value(B))) &&
+      match(Y, m_Not(m_c_Or(m_Specific(A), m_Specific(B)))))
+    return NotA;
+
+  // ~(A ^ B) | (A & B) --> ~(A & B)
+  // ~(A ^ B) | (B & A) --> ~(A & B)
+  Value *NotAB;
+  if (match(X, m_CombineAnd(m_NotForbidUndef(m_Xor(m_Value(A), m_Value(B))),
+                            m_Value(NotAB))) &&
+      match(Y, m_c_And(m_Specific(A), m_Specific(B))))
+    return NotAB;
+
   return nullptr;
 }
 
@@ -2259,27 +2295,6 @@ static Value *SimplifyOrInst(Value *Op0, Value *Op1, const SimplifyQuery &Q,
   if (Value *V = simplifyLogicOfAddSub(Op0, Op1, Instruction::Or))
     return V;
 
-  Value *A, *B, *NotA;
-
-  // (~A & B) | ~(A | B) --> ~A
-  // (~A & B) | ~(B | A) --> ~A
-  // (B & ~A) | ~(A | B) --> ~A
-  // (B & ~A) | ~(B | A) --> ~A
-  if (match(Op0, m_c_And(m_CombineAnd(m_Value(NotA), m_Not(m_Value(A))),
-                         m_Value(B))) &&
-      match(Op1, m_Not(m_c_Or(m_Specific(A), m_Specific(B)))))
-    return NotA;
-
-  // Commute the 'or' operands.
-  // ~(A | B) | (~A & B) --> ~A
-  // ~(B | A) | (~A & B) --> ~A
-  // ~(A | B) | (B & ~A) --> ~A
-  // ~(B | A) | (B & ~A) --> ~A
-  if (match(Op1, m_c_And(m_CombineAnd(m_Value(NotA), m_Not(m_Value(A))),
-                         m_Value(B))) &&
-      match(Op0, m_Not(m_c_Or(m_Specific(A), m_Specific(B)))))
-    return NotA;
-
   // Rotated -1 is still -1:
   // (-1 << X) | (-1 >> (C - X)) --> -1
   // (-1 >> X) | (-1 << (C - X)) --> -1
@@ -2335,6 +2350,7 @@ static Value *SimplifyOrInst(Value *Op0, Value *Op1, const SimplifyQuery &Q,
   }
 
   // (A & C1)|(B & C2)
+  Value *A, *B;
   const APInt *C1, *C2;
   if (match(Op0, m_And(m_Value(A), m_APInt(C1))) &&
       match(Op1, m_And(m_Value(B), m_APInt(C2)))) {
@@ -2696,9 +2712,17 @@ static Value *simplifyICmpOfBools(CmpInst::Predicate Pred, Value *LHS,
   if (!OpTy->isIntOrIntVectorTy(1))
     return nullptr;
 
-  // A boolean compared to true/false can be simplified in 14 out of the 20
-  // (10 predicates * 2 constants) possible combinations. Cases not handled here
-  // require a 'not' of the LHS, so those must be transformed in InstCombine.
+  // A boolean compared to true/false can be reduced in 14 out of the 20
+  // (10 predicates * 2 constants) possible combinations. The other
+  // 6 cases require a 'not' of the LHS.
+
+  auto ExtractNotLHS = [](Value *V) -> Value * {
+    Value *X;
+    if (match(V, m_Not(m_Value(X))))
+      return X;
+    return nullptr;
+  };
+
   if (match(RHS, m_Zero())) {
     switch (Pred) {
     case CmpInst::ICMP_NE:  // X !=  0 -> X
@@ -2706,6 +2730,13 @@ static Value *simplifyICmpOfBools(CmpInst::Predicate Pred, Value *LHS,
     case CmpInst::ICMP_SLT: // X <s  0 -> X
       return LHS;
 
+    case CmpInst::ICMP_EQ:  // not(X) ==  0 -> X != 0 -> X
+    case CmpInst::ICMP_ULE: // not(X) <=u 0 -> X >u 0 -> X
+    case CmpInst::ICMP_SGE: // not(X) >=s 0 -> X <s 0 -> X
+      if (Value *X = ExtractNotLHS(LHS))
+        return X;
+      break;
+
     case CmpInst::ICMP_ULT: // X <u  0 -> false
     case CmpInst::ICMP_SGT: // X >s  0 -> false
       return getFalse(ITy);
@@ -2723,6 +2754,13 @@ static Value *simplifyICmpOfBools(CmpInst::Predicate Pred, Value *LHS,
     case CmpInst::ICMP_SLE: // X <=s -1 -> X
       return LHS;
 
+    case CmpInst::ICMP_NE:  // not(X) !=  1 -> X ==   1 -> X
+    case CmpInst::ICMP_ULT: // not(X) <=u 1 -> X >=u  1 -> X
+    case CmpInst::ICMP_SGT: // not(X) >s  1 -> X <=s -1 -> X
+      if (Value *X = ExtractNotLHS(LHS))
+        return X;
+      break;
+
     case CmpInst::ICMP_UGT: // X >u   1 -> false
     case CmpInst::ICMP_SLT: // X <s  -1 -> false
       return getFalse(ITy);
@@ -5887,9 +5925,9 @@ static Value *simplifyIntrinsic(CallBase *Call, const SimplifyQuery &Q) {
       auto Attr = Call->getFunction()->getFnAttribute(Attribute::VScaleRange);
       if (!Attr.isValid())
         return nullptr;
-      unsigned VScaleMin, VScaleMax;
-      std::tie(VScaleMin, VScaleMax) = Attr.getVScaleRangeArgs();
-      if (VScaleMin == VScaleMax && VScaleMax != 0)
+      unsigned VScaleMin = Attr.getVScaleRangeMin();
+      Optional<unsigned> VScaleMax = Attr.getVScaleRangeMax();
+      if (VScaleMax && VScaleMin == VScaleMax)
         return ConstantInt::get(F->getReturnType(), VScaleMin);
       return nullptr;
     }
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 19a24ac6a484..6444518dc70c 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -1568,11 +1568,12 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
 
   auto &DL = InnermostLoop->getHeader()->getModule()->getDataLayout();
   uint64_t TypeByteSize = DL.getTypeAllocSize(ATy);
+  bool HasSameSize =
+      DL.getTypeStoreSizeInBits(ATy) == DL.getTypeStoreSizeInBits(BTy);
   uint64_t Stride = std::abs(StrideAPtr);
   const SCEVConstant *C = dyn_cast<SCEVConstant>(Dist);
   if (!C) {
-    if (!isa<SCEVCouldNotCompute>(Dist) &&
-        TypeByteSize == DL.getTypeAllocSize(BTy) &&
+    if (!isa<SCEVCouldNotCompute>(Dist) && HasSameSize &&
         isSafeDependenceDistance(DL, *(PSE.getSE()),
                                  *(PSE.getBackedgeTakenCount()), *Dist, Stride,
                                  TypeByteSize))
@@ -1587,7 +1588,7 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
   int64_t Distance = Val.getSExtValue();
 
   // Attempt to prove strided accesses independent.
-  if (std::abs(Distance) > 0 && Stride > 1 && ATy == BTy &&
+  if (std::abs(Distance) > 0 && Stride > 1 && HasSameSize &&
       areStridedAccessesIndependent(std::abs(Distance), Stride, TypeByteSize)) {
     LLVM_DEBUG(dbgs() << "LAA: Strided accesses are independent\n");
     return Dependence::NoDep;
@@ -1598,7 +1599,7 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
     bool IsTrueDataDependence = (AIsWrite && !BIsWrite);
     if (IsTrueDataDependence && EnableForwardingConflictDetection &&
         (couldPreventStoreLoadForward(Val.abs().getZExtValue(), TypeByteSize) ||
-         ATy != BTy)) {
+         !HasSameSize)) {
       LLVM_DEBUG(dbgs() << "LAA: Forward but may prevent st->ld forwarding\n");
       return Dependence::ForwardButPreventsForwarding;
     }
@@ -1608,21 +1609,19 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
   }
 
   // Write to the same location with the same size.
-  // Could be improved to assert type sizes are the same (i32 == float, etc).
   if (Val == 0) {
-    if (ATy == BTy)
+    if (HasSameSize)
       return Dependence::Forward;
     LLVM_DEBUG(
-        dbgs() << "LAA: Zero dependence difference but different types\n");
+        dbgs() << "LAA: Zero dependence difference but different type sizes\n");
     return Dependence::Unknown;
   }
 
   assert(Val.isStrictlyPositive() && "Expect a positive value");
 
-  if (ATy != BTy) {
-    LLVM_DEBUG(
-        dbgs()
-        << "LAA: ReadWrite-Write positive dependency with different types\n");
+  if (!HasSameSize) {
+    LLVM_DEBUG(dbgs() << "LAA: ReadWrite-Write positive dependency with "
+                         "different type sizes\n");
     return Dependence::Unknown;
   }
 
diff --git a/llvm/lib/Analysis/MLInlineAdvisor.cpp b/llvm/lib/Analysis/MLInlineAdvisor.cpp
index 6fc4c42bdd71..f5a65cd2b689 100644
--- a/llvm/lib/Analysis/MLInlineAdvisor.cpp
+++ b/llvm/lib/Analysis/MLInlineAdvisor.cpp
@@ -35,6 +35,21 @@
 
 using namespace llvm;
 
+#ifdef LLVM_HAVE_TF_AOT
+#include "llvm/Analysis/ReleaseModeModelRunner.h"
+// codegen-ed file
+#include "InlinerSizeModel.h" // NOLINT
+#include "llvm/Analysis/InlineModelFeatureMaps.h"
+
+std::unique_ptr<InlineAdvisor>
+llvm::getReleaseModeAdvisor(Module &M, ModuleAnalysisManager &MAM) {
+  auto AOTRunner =
+      std::make_unique<ReleaseModeModelRunner<llvm::InlinerSizeModel>>(
+          M.getContext(), FeatureNameMap, DecisionName);
+  return std::make_unique<MLInlineAdvisor>(M, MAM, std::move(AOTRunner));
+}
+#endif
+
 #define DEBUG_TYPE "inline-ml"
 
 static cl::opt<float> SizeIncreaseThreshold(
@@ -245,29 +260,32 @@ std::unique_ptr<InlineAdvice> MLInlineAdvisor::getAdviceImpl(CallBase &CB) {
   auto &CallerBefore = FAM.getResult<FunctionPropertiesAnalysis>(Caller);
   auto &CalleeBefore = FAM.getResult<FunctionPropertiesAnalysis>(Callee);
 
-  ModelRunner->setFeature(FeatureIndex::CalleeBasicBlockCount,
-                          CalleeBefore.BasicBlockCount);
-  ModelRunner->setFeature(FeatureIndex::CallSiteHeight,
-                          FunctionLevels[&Caller]);
-  ModelRunner->setFeature(FeatureIndex::NodeCount, NodeCount);
-  ModelRunner->setFeature(FeatureIndex::NrCtantParams, NrCtantParams);
-  ModelRunner->setFeature(FeatureIndex::EdgeCount, EdgeCount);
-  ModelRunner->setFeature(FeatureIndex::CallerUsers, CallerBefore.Uses);
-  ModelRunner->setFeature(FeatureIndex::CallerConditionallyExecutedBlocks,
-                          CallerBefore.BlocksReachedFromConditionalInstruction);
-  ModelRunner->setFeature(FeatureIndex::CallerBasicBlockCount,
-                          CallerBefore.BasicBlockCount);
-  ModelRunner->setFeature(FeatureIndex::CalleeConditionallyExecutedBlocks,
-                          CalleeBefore.BlocksReachedFromConditionalInstruction);
-  ModelRunner->setFeature(FeatureIndex::CalleeUsers, CalleeBefore.Uses);
-  ModelRunner->setFeature(FeatureIndex::CostEstimate, CostEstimate);
+  *ModelRunner->getTensor<int64_t>(FeatureIndex::CalleeBasicBlockCount) =
+      CalleeBefore.BasicBlockCount;
+  *ModelRunner->getTensor<int64_t>(FeatureIndex::CallSiteHeight) =
+      FunctionLevels[&Caller];
+  *ModelRunner->getTensor<int64_t>(FeatureIndex::NodeCount) = NodeCount;
+  *ModelRunner->getTensor<int64_t>(FeatureIndex::NrCtantParams) = NrCtantParams;
+  *ModelRunner->getTensor<int64_t>(FeatureIndex::EdgeCount) = EdgeCount;
+  *ModelRunner->getTensor<int64_t>(FeatureIndex::CallerUsers) =
+      CallerBefore.Uses;
+  *ModelRunner->getTensor<int64_t>(
+      FeatureIndex::CallerConditionallyExecutedBlocks) =
+      CallerBefore.BlocksReachedFromConditionalInstruction;
+  *ModelRunner->getTensor<int64_t>(FeatureIndex::CallerBasicBlockCount) =
+      CallerBefore.BasicBlockCount;
+  *ModelRunner->getTensor<int64_t>(
+      FeatureIndex::CalleeConditionallyExecutedBlocks) =
+      CalleeBefore.BlocksReachedFromConditionalInstruction;
+  *ModelRunner->getTensor<int64_t>(FeatureIndex::CalleeUsers) =
+      CalleeBefore.Uses;
+  *ModelRunner->getTensor<int64_t>(FeatureIndex::CostEstimate) = CostEstimate;
 
   // Add the cost features
   for (size_t I = 0;
        I < static_cast<size_t>(InlineCostFeatureIndex::NumberOfFeatures); ++I) {
-    ModelRunner->setFeature(
-        inlineCostFeatureToMlFeature(static_cast<InlineCostFeatureIndex>(I)),
-        CostFeatures->at(I));
+    *ModelRunner->getTensor<int64_t>(inlineCostFeatureToMlFeature(
+        static_cast<InlineCostFeatureIndex>(I))) = CostFeatures->at(I);
   }
 
   return getAdviceFromModel(CB, ORE);
@@ -276,7 +294,8 @@ std::unique_ptr<InlineAdvice> MLInlineAdvisor::getAdviceImpl(CallBase &CB) {
 std::unique_ptr<MLInlineAdvice>
 MLInlineAdvisor::getAdviceFromModel(CallBase &CB,
                                     OptimizationRemarkEmitter &ORE) {
-  return std::make_unique<MLInlineAdvice>(this, CB, ORE, ModelRunner->run());
+  return std::make_unique<MLInlineAdvice>(
+      this, CB, ORE, static_cast<bool>(ModelRunner->evaluate<int64_t>()));
 }
 
 std::unique_ptr<InlineAdvice> MLInlineAdvisor::getMandatoryAdvice(CallBase &CB,
@@ -302,7 +321,8 @@ void MLInlineAdvice::reportContextForRemark(
   using namespace ore;
   OR << NV("Callee", Callee->getName());
   for (size_t I = 0; I < NumberOfFeatures; ++I)
-    OR << NV(FeatureNameMap[I], getAdvisor()->getModelRunner().getFeature(I));
+    OR << NV(FeatureNameMap[I],
+             *getAdvisor()->getModelRunner().getTensor<int64_t>(I));
   OR << NV("ShouldInline", isInliningRecommended());
 }
 
diff --git a/llvm/lib/Analysis/MemDerefPrinter.cpp b/llvm/lib/Analysis/MemDerefPrinter.cpp
index 1b16e1a9bcb2..30937a2e4931 100644
--- a/llvm/lib/Analysis/MemDerefPrinter.cpp
+++ b/llvm/lib/Analysis/MemDerefPrinter.cpp
@@ -59,8 +59,8 @@ bool MemDerefPrinter::runOnFunction(Function &F) {
       Value *PO = LI->getPointerOperand();
       if (isDereferenceablePointer(PO, LI->getType(), DL))
         Deref.push_back(PO);
-      if (isDereferenceableAndAlignedPointer(
-              PO, LI->getType(), MaybeAlign(LI->getAlignment()), DL))
+      if (isDereferenceableAndAlignedPointer(PO, LI->getType(),
+                                             MaybeAlign(LI->getAlign()), DL))
         DerefAndAligned.insert(PO);
     }
   }
@@ -94,8 +94,8 @@ PreservedAnalyses MemDerefPrinterPass::run(Function &F,
       Value *PO = LI->getPointerOperand();
       if (isDereferenceablePointer(PO, LI->getType(), DL))
         Deref.push_back(PO);
-      if (isDereferenceableAndAlignedPointer(
-              PO, LI->getType(), MaybeAlign(LI->getAlignment()), DL))
+      if (isDereferenceableAndAlignedPointer(PO, LI->getType(),
+                                             MaybeAlign(LI->getAlign()), DL))
         DerefAndAligned.insert(PO);
     }
   }
diff --git a/llvm/lib/Analysis/MemoryBuiltins.cpp b/llvm/lib/Analysis/MemoryBuiltins.cpp
index 4f2b5b34304d..ffdd7a2cfd4b 100644
--- a/llvm/lib/Analysis/MemoryBuiltins.cpp
+++ b/llvm/lib/Analysis/MemoryBuiltins.cpp
@@ -592,9 +592,9 @@ STATISTIC(ObjectVisitorArgument,
 STATISTIC(ObjectVisitorLoad,
           "Number of load instructions with unsolved size and offset");
 
-APInt ObjectSizeOffsetVisitor::align(APInt Size, uint64_t Alignment) {
+APInt ObjectSizeOffsetVisitor::align(APInt Size, MaybeAlign Alignment) {
   if (Options.RoundToAlign && Alignment)
-    return APInt(IntTyBits, alignTo(Size.getZExtValue(), Align(Alignment)));
+    return APInt(IntTyBits, alignTo(Size.getZExtValue(), Alignment));
   return Size;
 }
 
@@ -669,7 +669,7 @@ SizeOffsetType ObjectSizeOffsetVisitor::visitAllocaInst(AllocaInst &I) {
 
   APInt Size(IntTyBits, DL.getTypeAllocSize(I.getAllocatedType()));
   if (!I.isArrayAllocation())
-    return std::make_pair(align(Size, I.getAlignment()), Zero);
+    return std::make_pair(align(Size, I.getAlign()), Zero);
 
   Value *ArraySize = I.getArraySize();
   if (const ConstantInt *C = dyn_cast<ConstantInt>(ArraySize)) {
@@ -679,8 +679,8 @@ SizeOffsetType ObjectSizeOffsetVisitor::visitAllocaInst(AllocaInst &I) {
 
     bool Overflow;
     Size = Size.umul_ov(NumElems, Overflow);
-    return Overflow ? unknown() : std::make_pair(align(Size, I.getAlignment()),
-                                                 Zero);
+    return Overflow ? unknown()
+                    : std::make_pair(align(Size, I.getAlign()), Zero);
   }
   return unknown();
 }
@@ -694,7 +694,7 @@ SizeOffsetType ObjectSizeOffsetVisitor::visitArgument(Argument &A) {
   }
 
   APInt Size(IntTyBits, DL.getTypeAllocSize(MemoryTy));
-  return std::make_pair(align(Size, A.getParamAlignment()), Zero);
+  return std::make_pair(align(Size, A.getParamAlign()), Zero);
 }
 
 SizeOffsetType ObjectSizeOffsetVisitor::visitCallBase(CallBase &CB) {
@@ -800,7 +800,7 @@ SizeOffsetType ObjectSizeOffsetVisitor::visitGlobalVariable(GlobalVariable &GV){
     return unknown();
 
   APInt Size(IntTyBits, DL.getTypeAllocSize(GV.getValueType()));
-  return std::make_pair(align(Size, GV.getAlignment()), Zero);
+  return std::make_pair(align(Size, GV.getAlign()), Zero);
 }
 
 SizeOffsetType ObjectSizeOffsetVisitor::visitIntToPtrInst(IntToPtrInst&) {
diff --git a/llvm/lib/Analysis/MemoryLocation.cpp b/llvm/lib/Analysis/MemoryLocation.cpp
index 854ba83bd34a..a877b19df866 100644
--- a/llvm/lib/Analysis/MemoryLocation.cpp
+++ b/llvm/lib/Analysis/MemoryLocation.cpp
@@ -101,13 +101,8 @@ MemoryLocation MemoryLocation::getForSource(const AtomicMemTransferInst *MTI) {
 }
 
 MemoryLocation MemoryLocation::getForSource(const AnyMemTransferInst *MTI) {
-  auto Size = LocationSize::afterPointer();
-  if (ConstantInt *C = dyn_cast<ConstantInt>(MTI->getLength()))
-    Size = LocationSize::precise(C->getValue().getZExtValue());
-
-  // memcpy/memmove can have AA tags. For memcpy, they apply
-  // to both the source and the destination.
-  return MemoryLocation(MTI->getRawSource(), Size, MTI->getAAMetadata());
+  assert(MTI->getRawSource() == MTI->getArgOperand(1));
+  return getForArgument(MTI, 1, nullptr);
 }
 
 MemoryLocation MemoryLocation::getForDest(const MemIntrinsic *MI) {
@@ -119,13 +114,47 @@ MemoryLocation MemoryLocation::getForDest(const AtomicMemIntrinsic *MI) {
 }
 
 MemoryLocation MemoryLocation::getForDest(const AnyMemIntrinsic *MI) {
-  auto Size = LocationSize::afterPointer();
-  if (ConstantInt *C = dyn_cast<ConstantInt>(MI->getLength()))
-    Size = LocationSize::precise(C->getValue().getZExtValue());
+  assert(MI->getRawDest() == MI->getArgOperand(0));
+  return getForArgument(MI, 0, nullptr);
+}
+
+Optional<MemoryLocation>
+MemoryLocation::getForDest(const CallBase *CB, const TargetLibraryInfo &TLI) {
+  if (!CB->onlyAccessesArgMemory())
+    return None;
+
+  if (CB->hasOperandBundles())
+    // TODO: remove implementation restriction
+    return None;
+
+  Value *UsedV = nullptr;
+  Optional<unsigned> UsedIdx;
+  for (unsigned i = 0; i < CB->arg_size(); i++) {
+    if (!CB->getArgOperand(i)->getType()->isPointerTy())
+      continue;
+     if (CB->onlyReadsMemory(i))
+       continue;
+    if (!UsedV) {
+      // First potentially writing parameter
+      UsedV = CB->getArgOperand(i);
+      UsedIdx = i;
+      continue;
+    }
+    UsedIdx = None;
+    if (UsedV != CB->getArgOperand(i))
+      // Can't describe writing to two distinct locations.
+      // TODO: This results in an inprecision when two values derived from the
+      // same object are passed as arguments to the same function.
+      return None;
+  }
+  if (!UsedV)
+    // We don't currently have a way to represent a "does not write" result
+    // and thus have to be conservative and return unknown.
+    return None;
 
-  // memcpy/memmove can have AA tags. For memcpy, they apply
-  // to both the source and the destination.
-  return MemoryLocation(MI->getRawDest(), Size, MI->getAAMetadata());
+  if (UsedIdx)
+    return getForArgument(CB, *UsedIdx, &TLI);
+  return MemoryLocation::getBeforeOrAfter(UsedV, CB->getAAMetadata());
 }
 
 MemoryLocation MemoryLocation::getForArgument(const CallBase *Call,
@@ -145,6 +174,9 @@ MemoryLocation MemoryLocation::getForArgument(const CallBase *Call,
     case Intrinsic::memcpy:
     case Intrinsic::memcpy_inline:
     case Intrinsic::memmove:
+    case Intrinsic::memcpy_element_unordered_atomic:
+    case Intrinsic::memmove_element_unordered_atomic:
+    case Intrinsic::memset_element_unordered_atomic:
       assert((ArgIdx == 0 || ArgIdx == 1) &&
              "Invalid argument index for memory intrinsic");
       if (ConstantInt *LenCI = dyn_cast<ConstantInt>(II->getArgOperand(2)))
@@ -204,6 +236,10 @@ MemoryLocation MemoryLocation::getForArgument(const CallBase *Call,
                                 II->getArgOperand(1)->getType())),
                             AATags);
     }
+
+    assert(
+        !isa<AnyMemTransferInst>(II) &&
+        "all memory transfer intrinsics should be handled by the switch above");
   }
 
   // We can bound the aliasing properties of memset_pattern16 just as we can
@@ -213,6 +249,12 @@ MemoryLocation MemoryLocation::getForArgument(const CallBase *Call,
   LibFunc F;
   if (TLI && TLI->getLibFunc(*Call, F) && TLI->has(F)) {
     switch (F) {
+    case LibFunc_strcpy:
+    case LibFunc_strcat:
+    case LibFunc_strncat:
+      assert((ArgIdx == 0 || ArgIdx == 1) && "Invalid argument index for str function");
+      return MemoryLocation::getAfter(Arg, AATags);
+
     case LibFunc_memset_chk: {
       assert(ArgIdx == 0 && "Invalid argument index for memset_chk");
       LocationSize Size = LocationSize::afterPointer();
@@ -236,10 +278,18 @@ MemoryLocation MemoryLocation::getForArgument(const CallBase *Call,
       return MemoryLocation(Arg, Size, AATags);
     }
     case LibFunc_memset_pattern16:
+    case LibFunc_memset_pattern4:
+    case LibFunc_memset_pattern8:
       assert((ArgIdx == 0 || ArgIdx == 1) &&
              "Invalid argument index for memset_pattern16");
-      if (ArgIdx == 1)
-        return MemoryLocation(Arg, LocationSize::precise(16), AATags);
+      if (ArgIdx == 1) {
+        unsigned Size = 16;
+        if (F == LibFunc_memset_pattern4)
+          Size = 4;
+        else if (F == LibFunc_memset_pattern8)
+          Size = 8;
+        return MemoryLocation(Arg, LocationSize::precise(Size), AATags);
+      }
       if (const ConstantInt *LenCI =
               dyn_cast<ConstantInt>(Call->getArgOperand(2)))
         return MemoryLocation(Arg, LocationSize::precise(LenCI->getZExtValue()),
@@ -274,7 +324,6 @@ MemoryLocation MemoryLocation::getForArgument(const CallBase *Call,
       break;
     };
   }
-  // FIXME: Handle memset_pattern4 and memset_pattern8 also.
 
   return MemoryLocation::getBeforeOrAfter(Call->getArgOperand(ArgIdx), AATags);
 }
diff --git a/llvm/lib/Analysis/ModelUnderTrainingRunner.cpp b/llvm/lib/Analysis/ModelUnderTrainingRunner.cpp
new file mode 100644
index 000000000000..941458f648bc
--- /dev/null
+++ b/llvm/lib/Analysis/ModelUnderTrainingRunner.cpp
@@ -0,0 +1,49 @@
+//===- ModelUnderTrainingRunner.cpp - 'development' mode runner -----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implementation of a MLModelRunner for 'development' mode, i.e. evaluation
+// happens off a model that's provided from the command line and is interpreted.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Config/config.h"
+#if defined(LLVM_HAVE_TF_API)
+
+#include "llvm/Analysis/ModelUnderTrainingRunner.h"
+
+using namespace llvm;
+
+ModelUnderTrainingRunner::ModelUnderTrainingRunner(
+    LLVMContext &Ctx, const std::string &ModelPath,
+    const std::vector<TensorSpec> &InputSpecs,
+    const std::vector<LoggedFeatureSpec> &OutputSpecs)
+    : MLModelRunner(Ctx), OutputSpecs(OutputSpecs) {
+  Evaluator = std::make_unique<TFModelEvaluator>(
+      ModelPath, InputSpecs, [&](size_t I) { return OutputSpecs[I].Spec; },
+      OutputSpecs.size());
+  if (!Evaluator || !Evaluator->isValid()) {
+    Ctx.emitError("Failed to create inliner saved model evaluator");
+    Evaluator.reset();
+    return;
+  }
+}
+
+void *ModelUnderTrainingRunner::evaluateUntyped() {
+  LastEvaluationResult = Evaluator->evaluate();
+  if (!LastEvaluationResult.hasValue()) {
+    Ctx.emitError("Error evaluating model.");
+    return nullptr;
+  }
+  return LastEvaluationResult->getUntypedTensorValue(0);
+}
+
+void *ModelUnderTrainingRunner::getTensorUntyped(size_t Index) {
+  return Evaluator->getUntypedInput(Index);
+}
+
+#endif // defined(LLVM_HAVE_TF_API)
diff --git a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
index d80814852e19..2880ca62a7f8 100644
--- a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
+++ b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
@@ -234,6 +234,18 @@ static bool isNonVolatileStore(const Instruction *I) {
   return false;
 }
 
+// Returns true if the function definition must be unreachable.
+//
+// Note if this helper function returns true, `F` is guaranteed
+// to be unreachable; if it returns false, `F` might still
+// be unreachable but not covered by this helper function.
+static bool mustBeUnreachableFunction(const Function &F) {
+  // A function must be unreachable if its entry block ends with an
+  // 'unreachable'.
+  assert(!F.isDeclaration());
+  return isa<UnreachableInst>(F.getEntryBlock().getTerminator());
+}
+
 static void computeFunctionSummary(
     ModuleSummaryIndex &Index, const Module &M, const Function &F,
     BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, DominatorTree &DT,
@@ -488,7 +500,8 @@ static void computeFunctionSummary(
       // Don't try to import functions with noinline attribute.
       F.getAttributes().hasFnAttr(Attribute::NoInline),
       F.hasFnAttribute(Attribute::AlwaysInline),
-      F.hasFnAttribute(Attribute::NoUnwind), MayThrow, HasUnknownCall};
+      F.hasFnAttribute(Attribute::NoUnwind), MayThrow, HasUnknownCall,
+      mustBeUnreachableFunction(F)};
   std::vector<FunctionSummary::ParamAccess> ParamAccesses;
   if (auto *SSI = GetSSICallback(F))
     ParamAccesses = SSI->getParamAccesses(Index);
@@ -737,7 +750,8 @@ ModuleSummaryIndex llvm::buildModuleSummaryIndex(
                         F->hasFnAttribute(Attribute::AlwaysInline),
                         F->hasFnAttribute(Attribute::NoUnwind),
                         /* MayThrow */ true,
-                        /* HasUnknownCall */ true},
+                        /* HasUnknownCall */ true,
+                        /* MustBeUnreachable */ false},
                     /*EntryCount=*/0, ArrayRef<ValueInfo>{},
                     ArrayRef<FunctionSummary::EdgeTy>{},
                     ArrayRef<GlobalValue::GUID>{},
diff --git a/llvm/lib/Analysis/NoInferenceModelRunner.cpp b/llvm/lib/Analysis/NoInferenceModelRunner.cpp
new file mode 100644
index 000000000000..02ece6aa3900
--- /dev/null
+++ b/llvm/lib/Analysis/NoInferenceModelRunner.cpp
@@ -0,0 +1,33 @@
+//===- NoInferenceModelRunner.cpp - noop ML model runner   ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// A pseudo model runner. We use it to store feature values when collecting
+// logs for the default policy, in 'development' mode, but never ask it to
+// 'run'.
+//===----------------------------------------------------------------------===//
+#include "llvm/Config/config.h"
+#if defined(LLVM_HAVE_TF_API)
+
+#include "llvm/Analysis/NoInferenceModelRunner.h"
+#include "llvm/Analysis/Utils/TFUtils.h"
+
+using namespace llvm;
+
+NoInferenceModelRunner::NoInferenceModelRunner(
+    LLVMContext &Ctx, const std::vector<TensorSpec> &Inputs)
+    : MLModelRunner(Ctx) {
+  ValuesBuffer.reserve(Inputs.size());
+  for (const auto &TS : Inputs)
+    ValuesBuffer.push_back(std::make_unique<char[]>(TS.getElementCount() *
+                                                    TS.getElementByteSize()));
+}
+
+void *NoInferenceModelRunner::getTensorUntyped(size_t Index) {
+  return ValuesBuffer[Index].get();
+}
+#endif // defined(LLVM_HAVE_TF_API)
diff --git a/llvm/lib/Analysis/ReleaseModeModelRunner.cpp b/llvm/lib/Analysis/ReleaseModeModelRunner.cpp
deleted file mode 100644
index d2bf95388066..000000000000
--- a/llvm/lib/Analysis/ReleaseModeModelRunner.cpp
+++ /dev/null
@@ -1,90 +0,0 @@
-//===- ReleaseModeModelRunner.cpp - Fast, precompiled model runner  -------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements a model runner wrapping an AOT compiled ML model.
-// Only inference is supported.
-//
-//===----------------------------------------------------------------------===//
-#include "llvm/Config/config.h"
-#if defined(LLVM_HAVE_TF_AOT)
-
-#include "llvm/Analysis/InlineModelFeatureMaps.h"
-#include "llvm/Analysis/MLInlineAdvisor.h"
-
-// codegen-ed file
-#include "InlinerSizeModel.h" // NOLINT
-
-#include <memory>
-#include <vector>
-
-using namespace llvm;
-namespace {
-
-const char FeedPrefix[] = "feed_";
-const char FetchPrefix[] = "fetch_";
-
-/// MLModelRunner - production mode implementation. It uses a AOT-compiled
-/// SavedModel for efficient execution.
-class ReleaseModeModelRunner final : public MLModelRunner {
-public:
-  ReleaseModeModelRunner(LLVMContext &Ctx);
-  virtual ~ReleaseModeModelRunner() = default;
-
-  bool run() override;
-
-  void setFeature(FeatureIndex Index, int64_t Value) override;
-  int64_t getFeature(int Index) const override;
-
-private:
-  std::vector<int32_t> FeatureIndices;
-  int32_t ResultIndex = -1;
-  std::unique_ptr<llvm::InlinerSizeModel> CompiledModel;
-};
-} // namespace
-
-ReleaseModeModelRunner::ReleaseModeModelRunner(LLVMContext &Ctx)
-    : MLModelRunner(Ctx),
-      CompiledModel(std::make_unique<llvm::InlinerSizeModel>()) {
-  assert(CompiledModel && "The CompiledModel should be valid");
-
-  FeatureIndices.resize(NumberOfFeatures);
-
-  for (size_t I = 0; I < NumberOfFeatures; ++I) {
-    const int Index =
-        CompiledModel->LookupArgIndex(FeedPrefix + FeatureNameMap[I]);
-    assert(Index >= 0 && "Cannot find Feature in inlining model");
-    FeatureIndices[I] = Index;
-  }
-
-  ResultIndex =
-      CompiledModel->LookupResultIndex(std::string(FetchPrefix) + DecisionName);
-  assert(ResultIndex >= 0 && "Cannot find DecisionName in inlining model");
-}
-
-int64_t ReleaseModeModelRunner::getFeature(int Index) const {
-  return *static_cast<int64_t *>(
-      CompiledModel->arg_data(FeatureIndices[Index]));
-}
-
-void ReleaseModeModelRunner::setFeature(FeatureIndex Index, int64_t Value) {
-  *static_cast<int64_t *>(CompiledModel->arg_data(
-      FeatureIndices[static_cast<size_t>(Index)])) = Value;
-}
-
-bool ReleaseModeModelRunner::run() {
-  CompiledModel->Run();
-  return static_cast<bool>(
-      *static_cast<int64_t *>(CompiledModel->result_data(ResultIndex)));
-}
-
-std::unique_ptr<InlineAdvisor>
-llvm::getReleaseModeAdvisor(Module &M, ModuleAnalysisManager &MAM) {
-  auto AOTRunner = std::make_unique<ReleaseModeModelRunner>(M.getContext());
-  return std::make_unique<MLInlineAdvisor>(M, MAM, std::move(AOTRunner));
-}
-#endif // defined(LLVM_HAVE_TF_AOT)
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index 7dc7f9904c70..0c3f32295ae1 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -8829,11 +8829,10 @@ const SCEV *ScalarEvolution::getSCEVAtScope(const SCEV *V, const Loop *L) {
   for (auto &LS : reverse(ValuesAtScopes[V]))
     if (LS.first == L) {
       LS.second = C;
+      if (!isa<SCEVConstant>(C))
+        ValuesAtScopesUsers[C].push_back({L, V});
       break;
     }
-
-  if (!isa<SCEVConstant>(C))
-    ValuesAtScopesUsers[C].push_back({L, V});
   return C;
 }
 
@@ -13058,11 +13057,13 @@ void ScalarEvolution::verify() const {
     Worklist.append(L->begin(), L->end());
   }
   for (auto &KV : ValueExprMap) {
+#ifndef NDEBUG
     // Check for SCEV expressions referencing invalid/deleted loops.
     if (auto *AR = dyn_cast<SCEVAddRecExpr>(KV.second)) {
       assert(ValidLoops.contains(AR->getLoop()) &&
              "AddRec references invalid loop");
     }
+#endif
 
     // Check that the value is also part of the reverse map.
     auto It = ExprValueMap.find(KV.second);
@@ -13122,7 +13123,7 @@ void ScalarEvolution::verify() const {
             is_contained(It->second, std::make_pair(L, Value)))
           continue;
         dbgs() << "Value: " << *Value << ", Loop: " << *L << ", ValueAtScope: "
-               << ValueAtScope << " missing in ValuesAtScopesUsers\n";
+               << *ValueAtScope << " missing in ValuesAtScopesUsers\n";
         std::abort();
       }
     }
@@ -13139,7 +13140,7 @@ void ScalarEvolution::verify() const {
           is_contained(It->second, std::make_pair(L, ValueAtScope)))
         continue;
       dbgs() << "Value: " << *Value << ", Loop: " << *L << ", ValueAtScope: "
-             << ValueAtScope << " missing in ValuesAtScopes\n";
+             << *ValueAtScope << " missing in ValuesAtScopes\n";
       std::abort();
     }
   }
@@ -13958,11 +13959,12 @@ const SCEV *ScalarEvolution::applyLoopGuards(const SCEV *Expr, const Loop *L) {
         ExprsToRewrite.push_back(LHS);
     }
   };
-  // Starting at the loop predecessor, climb up the predecessor chain, as long
-  // as there are predecessors that can be found that have unique successors
-  // leading to the original header.
+  // First, collect conditions from dominating branches. Starting at the loop
+  // predecessor, climb up the predecessor chain, as long as there are
+  // predecessors that can be found that have unique successors leading to the
+  // original header.
   // TODO: share this logic with isLoopEntryGuardedByCond.
-  DenseMap<const SCEV *, const SCEV *> RewriteMap;
+  SmallVector<std::pair<Value *, bool>> Terms;
   for (std::pair<const BasicBlock *, const BasicBlock *> Pair(
            L->getLoopPredecessor(), L->getHeader());
        Pair.first; Pair = getPredecessorWithUniqueSuccessorForBB(Pair.first)) {
@@ -13972,10 +13974,20 @@ const SCEV *ScalarEvolution::applyLoopGuards(const SCEV *Expr, const Loop *L) {
     if (!LoopEntryPredicate || LoopEntryPredicate->isUnconditional())
       continue;
 
-    bool EnterIfTrue = LoopEntryPredicate->getSuccessor(0) == Pair.second;
+    Terms.emplace_back(LoopEntryPredicate->getCondition(),
+                       LoopEntryPredicate->getSuccessor(0) == Pair.second);
+  }
+
+  // Now apply the information from the collected conditions to RewriteMap.
+  // Conditions are processed in reverse order, so the earliest conditions is
+  // processed first. This ensures the SCEVs with the shortest dependency chains
+  // are constructed first.
+  DenseMap<const SCEV *, const SCEV *> RewriteMap;
+  for (auto &E : reverse(Terms)) {
+    bool EnterIfTrue = E.second;
     SmallVector<Value *, 8> Worklist;
     SmallPtrSet<Value *, 8> Visited;
-    Worklist.push_back(LoopEntryPredicate->getCondition());
+    Worklist.push_back(E.first);
     while (!Worklist.empty()) {
       Value *Cond = Worklist.pop_back_val();
       if (!Visited.insert(Cond).second)
diff --git a/llvm/lib/Analysis/TargetLibraryInfo.cpp b/llvm/lib/Analysis/TargetLibraryInfo.cpp
index 72fbd5ad3f68..02923c2c7eb1 100644
--- a/llvm/lib/Analysis/TargetLibraryInfo.cpp
+++ b/llvm/lib/Analysis/TargetLibraryInfo.cpp
@@ -238,9 +238,8 @@ static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T,
     // e.g., x86_64-pc-windows-msvc18.
     bool hasPartialC99 = true;
     if (T.isKnownWindowsMSVCEnvironment()) {
-      unsigned Major, Minor, Micro;
-      T.getEnvironmentVersion(Major, Minor, Micro);
-      hasPartialC99 = (Major == 0 || Major >= 19);
+      VersionTuple Version = T.getEnvironmentVersion();
+      hasPartialC99 = (Version.getMajor() == 0 || Version.getMajor() >= 19);
     }
 
     // Latest targets support C89 math functions, in part.
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 5067f493f02d..6aa9a77391dc 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -982,10 +982,10 @@ bool TargetTransformInfo::areInlineCompatible(const Function *Caller,
   return TTIImpl->areInlineCompatible(Caller, Callee);
 }
 
-bool TargetTransformInfo::areFunctionArgsABICompatible(
+bool TargetTransformInfo::areTypesABICompatible(
     const Function *Caller, const Function *Callee,
-    SmallPtrSetImpl<Argument *> &Args) const {
-  return TTIImpl->areFunctionArgsABICompatible(Caller, Callee, Args);
+    const ArrayRef<Type *> &Types) const {
+  return TTIImpl->areTypesABICompatible(Caller, Callee, Types);
 }
 
 bool TargetTransformInfo::isIndexedLoadLegal(MemIndexedMode Mode,
@@ -1072,8 +1072,13 @@ bool TargetTransformInfo::supportsScalableVectors() const {
   return TTIImpl->supportsScalableVectors();
 }
 
-bool TargetTransformInfo::hasActiveVectorLength() const {
-  return TTIImpl->hasActiveVectorLength();
+bool TargetTransformInfo::enableScalableVectorization() const {
+  return TTIImpl->enableScalableVectorization();
+}
+
+bool TargetTransformInfo::hasActiveVectorLength(unsigned Opcode, Type *DataType,
+                                                Align Alignment) const {
+  return TTIImpl->hasActiveVectorLength(Opcode, DataType, Alignment);
 }
 
 InstructionCost
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 1c41c77a8cfb..fc378f97de0b 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -1154,7 +1154,7 @@ static void computeKnownBitsFromOperator(const Operator *I,
       // If the negate has an NSW flag we can assume the sign bit of the result
       // will be 0 because that makes abs(INT_MIN) undefined.
       if (match(RHS, m_Neg(m_Specific(LHS))) &&
-          Q.IIQ.hasNoSignedWrap(cast<Instruction>(RHS)))
+          Q.IIQ.hasNoSignedWrap(cast<OverflowingBinaryOperator>(RHS)))
         Known.Zero.setSignBit();
     }
 
@@ -1709,23 +1709,25 @@ static void computeKnownBitsFromOperator(const Operator *I,
             !II->getFunction()->hasFnAttribute(Attribute::VScaleRange))
           break;
 
-        auto VScaleRange = II->getFunction()
-                               ->getFnAttribute(Attribute::VScaleRange)
-                               .getVScaleRangeArgs();
+        auto Attr = II->getFunction()->getFnAttribute(Attribute::VScaleRange);
+        Optional<unsigned> VScaleMax = Attr.getVScaleRangeMax();
 
-        if (VScaleRange.second == 0)
+        if (!VScaleMax)
           break;
 
+        unsigned VScaleMin = Attr.getVScaleRangeMin();
+
         // If vscale min = max then we know the exact value at compile time
         // and hence we know the exact bits.
-        if (VScaleRange.first == VScaleRange.second) {
-          Known.One = VScaleRange.first;
-          Known.Zero = VScaleRange.first;
+        if (VScaleMin == VScaleMax) {
+          Known.One = VScaleMin;
+          Known.Zero = VScaleMin;
           Known.Zero.flipAllBits();
           break;
         }
 
-        unsigned FirstZeroHighBit = 32 - countLeadingZeros(VScaleRange.second);
+        unsigned FirstZeroHighBit =
+            32 - countLeadingZeros(VScaleMax.getValue());
         if (FirstZeroHighBit < BitWidth)
           Known.Zero.setBitsFrom(FirstZeroHighBit);
 
@@ -4676,8 +4678,8 @@ bool llvm::isSafeToSpeculativelyExecute(const Value *V,
       return false;
     const DataLayout &DL = LI->getModule()->getDataLayout();
     return isDereferenceableAndAlignedPointer(
-        LI->getPointerOperand(), LI->getType(), MaybeAlign(LI->getAlignment()),
-        DL, CtxI, DT, TLI);
+        LI->getPointerOperand(), LI->getType(), MaybeAlign(LI->getAlign()), DL,
+        CtxI, DT, TLI);
   }
   case Instruction::Call: {
     auto *CI = cast<const CallInst>(Inst);
@@ -4975,14 +4977,6 @@ static bool canCreateUndefOrPoison(const Operator *Op, bool PoisonOnly,
   if (ConsiderFlags && Op->hasPoisonGeneratingFlags())
     return true;
 
-  // TODO: this should really be under the ConsiderFlags block, but currently
-  // these are not dropped by dropPoisonGeneratingFlags
-  if (const auto *FP = dyn_cast<FPMathOperator>(Op)) {
-    auto FMF = FP->getFastMathFlags();
-    if (FMF.noNaNs() || FMF.noInfs())
-      return true;
-  }
-
   unsigned Opcode = Op->getOpcode();
 
   // Check whether opcode is a poison/undef-generating operation
diff --git a/llvm/lib/AsmParser/LLLexer.cpp b/llvm/lib/AsmParser/LLLexer.cpp
index 41fb0b9008be..e3bf41c9721b 100644
--- a/llvm/lib/AsmParser/LLLexer.cpp
+++ b/llvm/lib/AsmParser/LLLexer.cpp
@@ -733,6 +733,7 @@ lltok::Kind LLLexer::LexIdentifier() {
   KEYWORD(x);
   KEYWORD(blockaddress);
   KEYWORD(dso_local_equivalent);
+  KEYWORD(no_cfi);
 
   // Metadata types.
   KEYWORD(distinct);
@@ -773,6 +774,7 @@ lltok::Kind LLLexer::LexIdentifier() {
   KEYWORD(noUnwind);
   KEYWORD(mayThrow);
   KEYWORD(hasUnknownCall);
+  KEYWORD(mustBeUnreachable);
   KEYWORD(calls);
   KEYWORD(callee);
   KEYWORD(params);
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index 5feabd876e3a..35c615522fe2 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -152,28 +152,28 @@ bool LLParser::validateEndOfModule(bool UpgradeDebugInfo) {
         FnAttrs.removeAttribute(Attribute::Alignment);
       }
 
-      AS = AS.addFnAttributes(Context, AttributeSet::get(Context, FnAttrs));
+      AS = AS.addFnAttributes(Context, FnAttrs);
       Fn->setAttributes(AS);
     } else if (CallInst *CI = dyn_cast<CallInst>(V)) {
       AttributeList AS = CI->getAttributes();
       AttrBuilder FnAttrs(AS.getFnAttrs());
       AS = AS.removeFnAttributes(Context);
       FnAttrs.merge(B);
-      AS = AS.addFnAttributes(Context, AttributeSet::get(Context, FnAttrs));
+      AS = AS.addFnAttributes(Context, FnAttrs);
       CI->setAttributes(AS);
     } else if (InvokeInst *II = dyn_cast<InvokeInst>(V)) {
       AttributeList AS = II->getAttributes();
       AttrBuilder FnAttrs(AS.getFnAttrs());
       AS = AS.removeFnAttributes(Context);
       FnAttrs.merge(B);
-      AS = AS.addFnAttributes(Context, AttributeSet::get(Context, FnAttrs));
+      AS = AS.addFnAttributes(Context, FnAttrs);
       II->setAttributes(AS);
     } else if (CallBrInst *CBI = dyn_cast<CallBrInst>(V)) {
       AttributeList AS = CBI->getAttributes();
       AttrBuilder FnAttrs(AS.getFnAttrs());
       AS = AS.removeFnAttributes(Context);
       FnAttrs.merge(B);
-      AS = AS.addFnAttributes(Context, AttributeSet::get(Context, FnAttrs));
+      AS = AS.addFnAttributes(Context, FnAttrs);
       CBI->setAttributes(AS);
     } else if (auto *GV = dyn_cast<GlobalVariable>(V)) {
       AttrBuilder Attrs(GV->getAttributes());
@@ -1306,7 +1306,8 @@ bool LLParser::parseEnumAttribute(Attribute::AttrKind Attr, AttrBuilder &B,
     unsigned MinValue, MaxValue;
     if (parseVScaleRangeArguments(MinValue, MaxValue))
       return true;
-    B.addVScaleRangeAttr(MinValue, MaxValue);
+    B.addVScaleRangeAttr(MinValue,
+                         MaxValue > 0 ? MaxValue : Optional<unsigned>());
     return false;
   }
   case Attribute::Dereferenceable: {
@@ -3287,6 +3288,20 @@ bool LLParser::parseValID(ValID &ID, PerFunctionState *PFS, Type *ExpectedTy) {
     return false;
   }
 
+  case lltok::kw_no_cfi: {
+    // ValID ::= 'no_cfi' @foo
+    Lex.Lex();
+
+    if (parseValID(ID, PFS))
+      return true;
+
+    if (ID.Kind != ValID::t_GlobalID && ID.Kind != ValID::t_GlobalName)
+      return error(ID.Loc, "expected global value name in no_cfi");
+
+    ID.NoCFI = true;
+    return false;
+  }
+
   case lltok::kw_trunc:
   case lltok::kw_zext:
   case lltok::kw_sext:
@@ -5267,9 +5282,13 @@ bool LLParser::convertValIDToValue(Type *Ty, ValID &ID, Value *&V,
   }
   case ValID::t_GlobalName:
     V = getGlobalVal(ID.StrVal, Ty, ID.Loc);
+    if (V && ID.NoCFI)
+      V = NoCFIValue::get(cast<GlobalValue>(V));
     return V == nullptr;
   case ValID::t_GlobalID:
     V = getGlobalVal(ID.UIntVal, Ty, ID.Loc);
+    if (V && ID.NoCFI)
+      V = NoCFIValue::get(cast<GlobalValue>(V));
     return V == nullptr;
   case ValID::t_APSInt:
     if (!Ty->isIntegerTy())
@@ -8533,6 +8552,7 @@ bool LLParser::parseFlag(unsigned &Val) {
 ///        [',' 'noUnwind' ':' Flag]? ')'
 ///        [',' 'mayThrow' ':' Flag]? ')'
 ///        [',' 'hasUnknownCall' ':' Flag]? ')'
+///        [',' 'mustBeUnreachable' ':' Flag]? ')'
 
 bool LLParser::parseOptionalFFlags(FunctionSummary::FFlags &FFlags) {
   assert(Lex.getKind() == lltok::kw_funcFlags);
@@ -8599,6 +8619,12 @@ bool LLParser::parseOptionalFFlags(FunctionSummary::FFlags &FFlags) {
         return true;
       FFlags.HasUnknownCall = Val;
       break;
+    case lltok::kw_mustBeUnreachable:
+      Lex.Lex();
+      if (parseToken(lltok::colon, "expected ':'") || parseFlag(Val))
+        return true;
+      FFlags.MustBeUnreachable = Val;
+      break;
     default:
       return error(Lex.getLoc(), "expected function flag type");
     }
diff --git a/llvm/lib/BinaryFormat/AMDGPUMetadataVerifier.cpp b/llvm/lib/BinaryFormat/AMDGPUMetadataVerifier.cpp
index cd1d872cc219..284e469a1d2f 100644
--- a/llvm/lib/BinaryFormat/AMDGPUMetadataVerifier.cpp
+++ b/llvm/lib/BinaryFormat/AMDGPUMetadataVerifier.cpp
@@ -57,11 +57,7 @@ bool MetadataVerifier::verifyArray(
   auto &Array = Node.getArray();
   if (Size && Array.size() != *Size)
     return false;
-  for (auto &Item : Array)
-    if (!verifyNode(Item))
-      return false;
-
-  return true;
+  return llvm::all_of(Array, verifyNode);
 }
 
 bool MetadataVerifier::verifyEntry(
diff --git a/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp b/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp
index d7bcb0d7f575..a36b256c29b6 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp
@@ -107,9 +107,9 @@ static Optional<const char *> GetCodeName(unsigned CodeID, unsigned BlockID,
   // Check to see if we have a blockinfo record for this record, with a name.
   if (const BitstreamBlockInfo::BlockInfo *Info =
           BlockInfo.getBlockInfo(BlockID)) {
-    for (unsigned i = 0, e = Info->RecordNames.size(); i != e; ++i)
-      if (Info->RecordNames[i].first == CodeID)
-        return Info->RecordNames[i].second.c_str();
+    for (const std::pair<unsigned, std::string> &RN : Info->RecordNames)
+      if (RN.first == CodeID)
+        return RN.second.c_str();
   }
 
   if (CurStreamType != LLVMIRBitstream)
@@ -219,6 +219,7 @@ static Optional<const char *> GetCodeName(unsigned CodeID, unsigned BlockID,
       STRINGIFY_CODE(CST_CODE, CE_SHUFVEC_EX)
       STRINGIFY_CODE(CST_CODE, CE_UNOP)
       STRINGIFY_CODE(CST_CODE, DSO_LOCAL_EQUIVALENT)
+      STRINGIFY_CODE(CST_CODE, NO_CFI_VALUE)
     case bitc::CST_CODE_BLOCKADDRESS:
       return "CST_CODE_BLOCKADDRESS";
       STRINGIFY_CODE(CST_CODE, DATA)
@@ -646,16 +647,14 @@ void BitcodeAnalyzer::printStats(BCDumpOptions O,
 
   // Emit per-block stats.
   O.OS << "Per-block Summary:\n";
-  for (std::map<unsigned, PerBlockIDStats>::iterator I = BlockIDStats.begin(),
-                                                     E = BlockIDStats.end();
-       I != E; ++I) {
-    O.OS << "  Block ID #" << I->first;
+  for (const auto &Stat : BlockIDStats) {
+    O.OS << "  Block ID #" << Stat.first;
     if (Optional<const char *> BlockName =
-            GetBlockName(I->first, BlockInfo, CurStreamType))
+            GetBlockName(Stat.first, BlockInfo, CurStreamType))
       O.OS << " (" << *BlockName << ")";
     O.OS << ":\n";
 
-    const PerBlockIDStats &Stats = I->second;
+    const PerBlockIDStats &Stats = Stat.second;
     O.OS << "      Num Instances: " << Stats.NumInstances << "\n";
     O.OS << "         Total Size: ";
     printSize(O.OS, Stats.NumBits);
@@ -694,8 +693,8 @@ void BitcodeAnalyzer::printStats(BCDumpOptions O,
 
       O.OS << "\tRecord Histogram:\n";
       O.OS << "\t\t  Count    # Bits     b/Rec   % Abv  Record Kind\n";
-      for (unsigned i = 0, e = FreqPairs.size(); i != e; ++i) {
-        const PerRecordStats &RecStats = Stats.CodeFreq[FreqPairs[i].second];
+      for (const auto &FreqPair : FreqPairs) {
+        const PerRecordStats &RecStats = Stats.CodeFreq[FreqPair.second];
 
         O.OS << format("\t\t%7d %9lu", RecStats.NumInstances,
                        (unsigned long)RecStats.TotalBits);
@@ -714,10 +713,10 @@ void BitcodeAnalyzer::printStats(BCDumpOptions O,
 
         O.OS << "  ";
         if (Optional<const char *> CodeName = GetCodeName(
-                FreqPairs[i].second, I->first, BlockInfo, CurStreamType))
+                FreqPair.second, Stat.first, BlockInfo, CurStreamType))
           O.OS << *CodeName << "\n";
         else
-          O.OS << "UnknownCode" << FreqPairs[i].second << "\n";
+          O.OS << "UnknownCode" << FreqPair.second << "\n";
       }
       O.OS << "\n";
     }
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index 993cb1de8c02..f5a878f8788a 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -488,6 +488,7 @@ class BitcodeReader : public BitcodeReaderBase, public GVMaterializer {
   BitcodeReaderValueList ValueList;
   Optional<MetadataLoader> MDLoader;
   std::vector<Comdat *> ComdatList;
+  DenseSet<GlobalObject *> ImplicitComdatObjects;
   SmallVector<Instruction *, 64> InstructionList;
 
   std::vector<std::pair<GlobalVariable *, unsigned>> GlobalInits;
@@ -932,6 +933,7 @@ static FunctionSummary::FFlags getDecodedFFlags(uint64_t RawFlags) {
   Flags.NoUnwind = (RawFlags >> 6) & 0x1;
   Flags.MayThrow = (RawFlags >> 7) & 0x1;
   Flags.HasUnknownCall = (RawFlags >> 8) & 0x1;
+  Flags.MustBeUnreachable = (RawFlags >> 9) & 0x1;
   return Flags;
 }
 
@@ -2037,14 +2039,8 @@ Expected<Value *> BitcodeReader::recordValue(SmallVectorImpl<uint64_t> &Record,
     return error("Invalid value name");
   V->setName(NameStr);
   auto *GO = dyn_cast<GlobalObject>(V);
-  if (GO) {
-    if (GO->getComdat() == reinterpret_cast<Comdat *>(1)) {
-      if (TT.supportsCOMDAT())
-        GO->setComdat(TheModule->getOrInsertComdat(V->getName()));
-      else
-        GO->setComdat(nullptr);
-    }
-  }
+  if (GO && ImplicitComdatObjects.contains(GO) && TT.supportsCOMDAT())
+    GO->setComdat(TheModule->getOrInsertComdat(V->getName()));
   return V;
 }
 
@@ -2942,6 +2938,19 @@ Error BitcodeReader::parseConstants() {
       V = DSOLocalEquivalent::get(GV);
       break;
     }
+    case bitc::CST_CODE_NO_CFI_VALUE: {
+      if (Record.size() < 2)
+        return error("Invalid record");
+      Type *GVTy = getTypeByID(Record[0]);
+      if (!GVTy)
+        return error("Invalid record");
+      GlobalValue *GV = dyn_cast_or_null<GlobalValue>(
+          ValueList.getConstantFwdRef(Record[1], GVTy));
+      if (!GV)
+        return error("Invalid record");
+      V = NoCFIValue::get(GV);
+      break;
+    }
     }
 
     ValueList.assignValue(V, NextCstNo);
@@ -3292,7 +3301,7 @@ Error BitcodeReader::parseGlobalVarRecord(ArrayRef<uint64_t> Record) {
       NewGV->setComdat(ComdatList[ComdatID - 1]);
     }
   } else if (hasImplicitComdat(RawLinkage)) {
-    NewGV->setComdat(reinterpret_cast<Comdat *>(1));
+    ImplicitComdatObjects.insert(NewGV);
   }
 
   if (Record.size() > 12) {
@@ -3426,7 +3435,7 @@ Error BitcodeReader::parseFunctionRecord(ArrayRef<uint64_t> Record) {
       Func->setComdat(ComdatList[ComdatID - 1]);
     }
   } else if (hasImplicitComdat(RawLinkage)) {
-    Func->setComdat(reinterpret_cast<Comdat *>(1));
+    ImplicitComdatObjects.insert(Func);
   }
 
   if (Record.size() > 13)
@@ -6733,10 +6742,10 @@ llvm::getBitcodeFileContents(MemoryBufferRef Buffer) {
         // not have its own string table. A bitcode file may have multiple
         // string tables if it was created by binary concatenation, for example
         // with "llvm-cat -b".
-        for (auto I = F.Mods.rbegin(), E = F.Mods.rend(); I != E; ++I) {
-          if (!I->Strtab.empty())
+        for (BitcodeModule &I : llvm::reverse(F.Mods)) {
+          if (!I.Strtab.empty())
             break;
-          I->Strtab = *Strtab;
+          I.Strtab = *Strtab;
         }
         // Similarly, the string table is used by every preceding symbol table;
         // normally there will be just one unless the bitcode file was created
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index e2354c40844a..dc06bc10cf95 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -833,8 +833,7 @@ void ModuleBitcodeWriter::writeAttributeTable() {
   Stream.EnterSubblock(bitc::PARAMATTR_BLOCK_ID, 3);
 
   SmallVector<uint64_t, 64> Record;
-  for (unsigned i = 0, e = Attrs.size(); i != e; ++i) {
-    AttributeList AL = Attrs[i];
+  for (const AttributeList &AL : Attrs) {
     for (unsigned i : AL.indexes()) {
       AttributeSet AS = AL.getAttributes(i);
       if (AS.hasAttributes())
@@ -1067,6 +1066,7 @@ static uint64_t getEncodedFFlags(FunctionSummary::FFlags Flags) {
   RawFlags |= (Flags.NoUnwind << 6);
   RawFlags |= (Flags.MayThrow << 7);
   RawFlags |= (Flags.HasUnknownCall << 8);
+  RawFlags |= (Flags.MustBeUnreachable << 9);
   return RawFlags;
 }
 
@@ -2657,6 +2657,10 @@ void ModuleBitcodeWriter::writeConstants(unsigned FirstVal, unsigned LastVal,
       Code = bitc::CST_CODE_DSO_LOCAL_EQUIVALENT;
       Record.push_back(VE.getTypeID(Equiv->getGlobalValue()->getType()));
       Record.push_back(VE.getValueID(Equiv->getGlobalValue()));
+    } else if (const auto *NC = dyn_cast<NoCFIValue>(C)) {
+      Code = bitc::CST_CODE_NO_CFI_VALUE;
+      Record.push_back(VE.getTypeID(NC->getGlobalValue()->getType()));
+      Record.push_back(VE.getValueID(NC->getGlobalValue()));
     } else {
 #ifndef NDEBUG
       C->dump();
diff --git a/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp b/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp
index 07e0708e68c3..df4f1a1873d7 100644
--- a/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp
+++ b/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp
@@ -310,8 +310,7 @@ static UseListOrderStack predictUseListOrder(const Module &M) {
   // We want to visit the functions backward now so we can list function-local
   // constants in the last Function they're used in.  Module-level constants
   // have already been visited above.
-  for (auto I = M.rbegin(), E = M.rend(); I != E; ++I) {
-    const Function &F = *I;
+  for (const Function &F : llvm::reverse(M)) {
     if (F.isDeclaration())
       continue;
     for (const BasicBlock &BB : F)
@@ -541,9 +540,8 @@ void ValueEnumerator::print(raw_ostream &OS, const ValueMapType &Map,
                             const char *Name) const {
   OS << "Map Name: " << Name << "\n";
   OS << "Size: " << Map.size() << "\n";
-  for (ValueMapType::const_iterator I = Map.begin(),
-         E = Map.end(); I != E; ++I) {
-    const Value *V = I->first;
+  for (const auto &I : Map) {
+    const Value *V = I.first;
     if (V->hasName())
       OS << "Value: " << V->getName();
     else
@@ -569,10 +567,10 @@ void ValueEnumerator::print(raw_ostream &OS, const MetadataMapType &Map,
                             const char *Name) const {
   OS << "Map Name: " << Name << "\n";
   OS << "Size: " << Map.size() << "\n";
-  for (auto I = Map.begin(), E = Map.end(); I != E; ++I) {
-    const Metadata *MD = I->first;
-    OS << "Metadata: slot = " << I->second.ID << "\n";
-    OS << "Metadata: function = " << I->second.F << "\n";
+  for (const auto &I : Map) {
+    const Metadata *MD = I.first;
+    OS << "Metadata: slot = " << I.second.ID << "\n";
+    OS << "Metadata: function = " << I.second.F << "\n";
     MD->print(OS);
     OS << "\n";
   }
diff --git a/llvm/lib/CodeGen/AggressiveAntiDepBreaker.cpp b/llvm/lib/CodeGen/AggressiveAntiDepBreaker.cpp
index 5984063627b0..5c64622c7245 100644
--- a/llvm/lib/CodeGen/AggressiveAntiDepBreaker.cpp
+++ b/llvm/lib/CodeGen/AggressiveAntiDepBreaker.cpp
@@ -561,8 +561,7 @@ bool AggressiveAntiDepBreaker::FindSuitableFreeRegisters(
                     << ":\n");
   std::map<unsigned, BitVector> RenameRegisterMap;
   unsigned SuperReg = 0;
-  for (unsigned i = 0, e = Regs.size(); i != e; ++i) {
-    unsigned Reg = Regs[i];
+  for (unsigned Reg : Regs) {
     if ((SuperReg == 0) || TRI->isSuperRegister(SuperReg, Reg))
       SuperReg = Reg;
 
@@ -584,8 +583,7 @@ bool AggressiveAntiDepBreaker::FindSuitableFreeRegisters(
   }
 
   // All group registers should be a subreg of SuperReg.
-  for (unsigned i = 0, e = Regs.size(); i != e; ++i) {
-    unsigned Reg = Regs[i];
+  for (unsigned Reg : Regs) {
     if (Reg == SuperReg) continue;
     bool IsSub = TRI->isSubRegister(SuperReg, Reg);
     // FIXME: remove this once PR18663 has been properly fixed. For now,
@@ -646,8 +644,7 @@ bool AggressiveAntiDepBreaker::FindSuitableFreeRegisters(
     // For each referenced group register (which must be a SuperReg or
     // a subregister of SuperReg), find the corresponding subregister
     // of NewSuperReg and make sure it is free to be renamed.
-    for (unsigned i = 0, e = Regs.size(); i != e; ++i) {
-      unsigned Reg = Regs[i];
+    for (unsigned Reg : Regs) {
       unsigned NewReg = 0;
       if (Reg == SuperReg) {
         NewReg = NewSuperReg;
diff --git a/llvm/lib/CodeGen/Analysis.cpp b/llvm/lib/CodeGen/Analysis.cpp
index 7d8a73e12d3a..7e68e5e22879 100644
--- a/llvm/lib/CodeGen/Analysis.cpp
+++ b/llvm/lib/CodeGen/Analysis.cpp
@@ -712,8 +712,8 @@ bool llvm::returnTypeIsEligibleForTailCall(const Function *F,
     // The manipulations performed when we're looking through an insertvalue or
     // an extractvalue would happen at the front of the RetPath list, so since
     // we have to copy it anyway it's more efficient to create a reversed copy.
-    SmallVector<unsigned, 4> TmpRetPath(RetPath.rbegin(), RetPath.rend());
-    SmallVector<unsigned, 4> TmpCallPath(CallPath.rbegin(), CallPath.rend());
+    SmallVector<unsigned, 4> TmpRetPath(llvm::reverse(RetPath));
+    SmallVector<unsigned, 4> TmpCallPath(llvm::reverse(CallPath));
 
     // Finally, we can check whether the value produced by the tail call at this
     // index is compatible with the value we return.
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index 828cb760b82e..533f20535655 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -180,7 +180,7 @@ Align AsmPrinter::getGVAlignment(const GlobalObject *GV, const DataLayout &DL,
     Alignment = InAlign;
 
   // If the GV has a specified alignment, take it into account.
-  const MaybeAlign GVAlign(GV->getAlignment());
+  const MaybeAlign GVAlign(GV->getAlign());
   if (!GVAlign)
     return Alignment;
 
@@ -288,7 +288,11 @@ bool AsmPrinter::doInitialization(Module &M) {
   // use the directive, where it would need the same conditionalization
   // anyway.
   const Triple &Target = TM.getTargetTriple();
-  OutStreamer->emitVersionForTarget(Target, M.getSDKVersion());
+  Triple TVT(M.getDarwinTargetVariantTriple());
+  OutStreamer->emitVersionForTarget(
+      Target, M.getSDKVersion(),
+      M.getDarwinTargetVariantTriple().empty() ? nullptr : &TVT,
+      M.getDarwinTargetVariantSDKVersion());
 
   // Allow the target to emit any magic that it wants at the start of the file.
   emitStartOfAsmFile(M);
@@ -1856,6 +1860,17 @@ bool AsmPrinter::doFinalization(Module &M) {
         continue;
       OutStreamer->emitSymbolAttribute(getSymbol(&GO), MCSA_WeakReference);
     }
+    if (shouldEmitWeakSwiftAsyncExtendedFramePointerFlags()) {
+      auto SymbolName = "swift_async_extendedFramePointerFlags";
+      auto Global = M.getGlobalVariable(SymbolName);
+      if (!Global) {
+        auto Int8PtrTy = Type::getInt8PtrTy(M.getContext());
+        Global = new GlobalVariable(M, Int8PtrTy, false,
+                                    GlobalValue::ExternalWeakLinkage, nullptr,
+                                    SymbolName);
+        OutStreamer->emitSymbolAttribute(getSymbol(Global), MCSA_WeakReference);
+      }
+    }
   }
 
   // Print aliases in topological order, that is, for each alias a = b,
@@ -2502,6 +2517,9 @@ const MCExpr *AsmPrinter::lowerConstant(const Constant *CV) {
   if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(CV))
     return getObjFileLowering().lowerDSOLocalEquivalent(Equiv, TM);
 
+  if (const NoCFIValue *NC = dyn_cast<NoCFIValue>(CV))
+    return MCSymbolRefExpr::create(getSymbol(NC->getGlobalValue()), Ctx);
+
   const ConstantExpr *CE = dyn_cast<ConstantExpr>(CV);
   if (!CE) {
     llvm_unreachable("Unknown constant value to lower!");
diff --git a/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp
index 85ff84484ced..d621108408f0 100644
--- a/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp
@@ -611,8 +611,8 @@ static SourceLanguage MapDWLangToCVLang(unsigned DWLang) {
 void CodeViewDebug::beginModule(Module *M) {
   // If module doesn't have named metadata anchors or COFF debug section
   // is not available, skip any debug info related stuff.
-  if (!M->getNamedMetadata("llvm.dbg.cu") ||
-      !Asm->getObjFileLowering().getCOFFDebugSymbolsSection()) {
+  NamedMDNode *CUs = M->getNamedMetadata("llvm.dbg.cu");
+  if (!CUs || !Asm->getObjFileLowering().getCOFFDebugSymbolsSection()) {
     Asm = nullptr;
     return;
   }
@@ -622,7 +622,6 @@ void CodeViewDebug::beginModule(Module *M) {
   TheCPU = mapArchToCVCPUType(Triple(M->getTargetTriple()).getArch());
 
   // Get the current source language.
-  NamedMDNode *CUs = MMI->getModule()->getNamedMetadata("llvm.dbg.cu");
   const MDNode *Node = *CUs->operands().begin();
   const auto *CU = cast<DICompileUnit>(Node);
 
@@ -650,6 +649,7 @@ void CodeViewDebug::endModule() {
   switchToDebugSectionForSymbol(nullptr);
 
   MCSymbol *CompilerInfo = beginCVSubsection(DebugSubsectionKind::Symbols);
+  emitObjName();
   emitCompilerInformation();
   endCVSubsection(CompilerInfo);
 
@@ -785,6 +785,29 @@ void CodeViewDebug::emitTypeGlobalHashes() {
   }
 }
 
+void CodeViewDebug::emitObjName() {
+  MCSymbol *CompilerEnd = beginSymbolRecord(SymbolKind::S_OBJNAME);
+
+  StringRef PathRef(Asm->TM.Options.ObjectFilenameForDebug);
+  llvm::SmallString<256> PathStore(PathRef);
+
+  if (PathRef.empty() || PathRef == "-") {
+    // Don't emit the filename if we're writing to stdout or to /dev/null.
+    PathRef = {};
+  } else {
+    llvm::sys::path::remove_dots(PathStore, /*remove_dot_dot=*/true);
+    PathRef = PathStore;
+  }
+
+  OS.AddComment("Signature");
+  OS.emitIntValue(0, 4);
+
+  OS.AddComment("Object name");
+  emitNullTerminatedSymbolName(OS, PathRef);
+
+  endSymbolRecord(CompilerEnd);
+}
+
 namespace {
 struct Version {
   int Part[4];
diff --git a/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.h b/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.h
index 6f88e15ee8fe..d1fc3cdccb20 100644
--- a/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.h
+++ b/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.h
@@ -302,6 +302,8 @@ class LLVM_LIBRARY_VISIBILITY CodeViewDebug : public DebugHandlerBase {
 
   void emitTypeGlobalHashes();
 
+  void emitObjName();
+
   void emitCompilerInformation();
 
   void emitBuildInfo();
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
index 0d2736178f0f..9b73f0ab2f05 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
@@ -779,7 +779,7 @@ DIE *DwarfCompileUnit::constructVariableDIEImpl(const DbgVariable &DV,
     const TargetRegisterInfo &TRI = *Asm->MF->getSubtarget().getRegisterInfo();
 
     auto AddEntry = [&](const DbgValueLocEntry &Entry,
-                                            DIExpressionCursor &Cursor) {
+                        DIExpressionCursor &Cursor) {
       if (Entry.isLocation()) {
         if (!DwarfExpr.addMachineRegExpression(TRI, Cursor,
                                                Entry.getLoc().getReg()))
@@ -788,11 +788,19 @@ DIE *DwarfCompileUnit::constructVariableDIEImpl(const DbgVariable &DV,
         // If there is an expression, emit raw unsigned bytes.
         DwarfExpr.addUnsignedConstant(Entry.getInt());
       } else if (Entry.isConstantFP()) {
+        // DwarfExpression does not support arguments wider than 64 bits
+        // (see PR52584).
+        // TODO: Consider chunking expressions containing overly wide
+        // arguments into separate pointer-sized fragment expressions.
         APInt RawBytes = Entry.getConstantFP()->getValueAPF().bitcastToAPInt();
-        DwarfExpr.addUnsignedConstant(RawBytes);
+        if (RawBytes.getBitWidth() > 64)
+          return false;
+        DwarfExpr.addUnsignedConstant(RawBytes.getZExtValue());
       } else if (Entry.isConstantInt()) {
         APInt RawBytes = Entry.getConstantInt()->getValue();
-        DwarfExpr.addUnsignedConstant(RawBytes);
+        if (RawBytes.getBitWidth() > 64)
+          return false;
+        DwarfExpr.addUnsignedConstant(RawBytes.getZExtValue());
       } else if (Entry.isTargetIndexLocation()) {
         TargetIndexLocation Loc = Entry.getTargetIndexLocation();
         // TODO TargetIndexLocation is a target-independent. Currently only the
@@ -805,11 +813,12 @@ DIE *DwarfCompileUnit::constructVariableDIEImpl(const DbgVariable &DV,
       return true;
     };
 
-    DwarfExpr.addExpression(
-        std::move(Cursor),
-        [&](unsigned Idx, DIExpressionCursor &Cursor) -> bool {
-          return AddEntry(DVal->getLocEntries()[Idx], Cursor);
-        });
+    if (!DwarfExpr.addExpression(
+            std::move(Cursor),
+            [&](unsigned Idx, DIExpressionCursor &Cursor) -> bool {
+              return AddEntry(DVal->getLocEntries()[Idx], Cursor);
+            }))
+      return VariableDie;
 
     // Now attach the location information to the DIE.
     addBlock(*VariableDie, dwarf::DW_AT_location, DwarfExpr.finalize());
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
index 047676d4c11e..48134f1fd774 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
@@ -1224,17 +1224,15 @@ void DwarfDebug::beginModule(Module *M) {
         CU.getOrCreateGlobalVariableDIE(GV, sortGlobalExprs(GVMap[GV]));
     }
 
-    for (auto *Ty : CUNode->getEnumTypes()) {
-      // The enum types array by design contains pointers to
-      // MDNodes rather than DIRefs. Unique them here.
+    for (auto *Ty : CUNode->getEnumTypes())
       CU.getOrCreateTypeDIE(cast<DIType>(Ty));
-    }
+
     for (auto *Ty : CUNode->getRetainedTypes()) {
       // The retained types array by design contains pointers to
       // MDNodes rather than DIRefs. Unique them here.
       if (DIType *RT = dyn_cast<DIType>(Ty))
-          // There is no point in force-emitting a forward declaration.
-          CU.getOrCreateTypeDIE(RT);
+        // There is no point in force-emitting a forward declaration.
+        CU.getOrCreateTypeDIE(RT);
     }
     // Emit imported_modules last so that the relevant context is already
     // available.
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp
index 6409c39e7849..37407c98e75f 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp
@@ -463,15 +463,14 @@ static bool isMemoryLocation(DIExpressionCursor ExprCursor) {
   return true;
 }
 
-void DwarfExpression::addExpression(DIExpressionCursor &&ExprCursor,
-                                    unsigned FragmentOffsetInBits) {
+void DwarfExpression::addExpression(DIExpressionCursor &&ExprCursor) {
   addExpression(std::move(ExprCursor),
                 [](unsigned Idx, DIExpressionCursor &Cursor) -> bool {
                   llvm_unreachable("unhandled opcode found in expression");
                 });
 }
 
-void DwarfExpression::addExpression(
+bool DwarfExpression::addExpression(
     DIExpressionCursor &&ExprCursor,
     llvm::function_ref<bool(unsigned, DIExpressionCursor &)> InsertArg) {
   // Entry values can currently only cover the initial register location,
@@ -496,7 +495,7 @@ void DwarfExpression::addExpression(
     case dwarf::DW_OP_LLVM_arg:
       if (!InsertArg(Op->getArg(0), ExprCursor)) {
         LocationKind = Unknown;
-        return;
+        return false;
       }
       break;
     case dwarf::DW_OP_LLVM_fragment: {
@@ -527,7 +526,7 @@ void DwarfExpression::addExpression(
       setSubRegisterPiece(0, 0);
       // Reset the location description kind.
       LocationKind = Unknown;
-      return;
+      return true;
     }
     case dwarf::DW_OP_plus_uconst:
       assert(!isRegisterLocation());
@@ -630,6 +629,8 @@ void DwarfExpression::addExpression(
   if (isImplicitLocation() && !isParameterValue())
     // Turn this into an implicit location description.
     addStackValue();
+
+  return true;
 }
 
 /// add masking operations to stencil out a subregister.
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.h b/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.h
index 513e9072309e..e605fe2f7d39 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.h
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.h
@@ -340,16 +340,17 @@ public:
   /// create one if necessary.
   unsigned getOrCreateBaseType(unsigned BitSize, dwarf::TypeKind Encoding);
 
+  /// Emit all remaining operations in the DIExpressionCursor. The
+  /// cursor must not contain any DW_OP_LLVM_arg operations.
+  void addExpression(DIExpressionCursor &&Expr);
+
   /// Emit all remaining operations in the DIExpressionCursor.
-  ///
-  /// \param FragmentOffsetInBits     If this is one fragment out of multiple
-  ///                                 locations, this is the offset of the
-  ///                                 fragment inside the entire variable.
-  void addExpression(DIExpressionCursor &&Expr,
-                     unsigned FragmentOffsetInBits = 0);
-  void
-  addExpression(DIExpressionCursor &&Expr,
-                llvm::function_ref<bool(unsigned, DIExpressionCursor &)> InsertArg);
+  /// DW_OP_LLVM_arg operations are resolved by calling (\p InsertArg).
+  //
+  /// \return false if any call to (\p InsertArg) returns false.
+  bool addExpression(
+      DIExpressionCursor &&Expr,
+      llvm::function_ref<bool(unsigned, DIExpressionCursor &)> InsertArg);
 
   /// If applicable, emit an empty DW_OP_piece / DW_OP_bit_piece to advance to
   /// the fragment described by \c Expr.
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
index 976e35905144..6b6d63f14f87 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
@@ -536,6 +536,18 @@ void DwarfUnit::addThrownTypes(DIE &Die, DINodeArray ThrownTypes) {
   }
 }
 
+void DwarfUnit::addAccess(DIE &Die, DINode::DIFlags Flags) {
+  if ((Flags & DINode::FlagAccessibility) == DINode::FlagProtected)
+    addUInt(Die, dwarf::DW_AT_accessibility, dwarf::DW_FORM_data1,
+            dwarf::DW_ACCESS_protected);
+  else if ((Flags & DINode::FlagAccessibility) == DINode::FlagPrivate)
+    addUInt(Die, dwarf::DW_AT_accessibility, dwarf::DW_FORM_data1,
+            dwarf::DW_ACCESS_private);
+  else if ((Flags & DINode::FlagAccessibility) == DINode::FlagPublic)
+    addUInt(Die, dwarf::DW_AT_accessibility, dwarf::DW_FORM_data1,
+            dwarf::DW_ACCESS_public);
+}
+
 DIE *DwarfUnit::getOrCreateContextDIE(const DIScope *Context) {
   if (!Context || isa<DIFile>(Context))
     return &getUnitDie();
@@ -842,13 +854,17 @@ void DwarfUnit::addAnnotation(DIE &Buffer, DINodeArray Annotations) {
   for (const Metadata *Annotation : Annotations->operands()) {
     const MDNode *MD = cast<MDNode>(Annotation);
     const MDString *Name = cast<MDString>(MD->getOperand(0));
-
-    // Currently, only MDString is supported with btf_decl_tag attribute.
-    const MDString *Value = cast<MDString>(MD->getOperand(1));
+    const auto &Value = MD->getOperand(1);
 
     DIE &AnnotationDie = createAndAddDIE(dwarf::DW_TAG_LLVM_annotation, Buffer);
     addString(AnnotationDie, dwarf::DW_AT_name, Name->getString());
-    addString(AnnotationDie, dwarf::DW_AT_const_value, Value->getString());
+    if (const auto *Data = dyn_cast<MDString>(Value))
+      addString(AnnotationDie, dwarf::DW_AT_const_value, Data->getString());
+    else if (const auto *Data = dyn_cast<ConstantAsMetadata>(Value))
+      addConstantValue(AnnotationDie, Data->getValue()->getUniqueInteger(),
+                       /*Unsigned=*/true);
+    else
+      assert(false && "Unsupported annotation value type");
   }
 }
 
@@ -1007,6 +1023,9 @@ void DwarfUnit::constructTypeDIE(DIE &Buffer, const DICompositeType *CTy) {
     if (CTy->isForwardDecl())
       addFlag(Buffer, dwarf::DW_AT_declaration);
 
+    // Add accessibility info if available.
+    addAccess(Buffer, CTy->getFlags());
+
     // Add source line info if available.
     if (!CTy->isForwardDecl())
       addSourceLine(Buffer, CTy);
@@ -1308,15 +1327,7 @@ void DwarfUnit::applySubprogramAttributes(const DISubprogram *SP, DIE &SPDie,
   if (SP->isNoReturn())
     addFlag(SPDie, dwarf::DW_AT_noreturn);
 
-  if (SP->isProtected())
-    addUInt(SPDie, dwarf::DW_AT_accessibility, dwarf::DW_FORM_data1,
-            dwarf::DW_ACCESS_protected);
-  else if (SP->isPrivate())
-    addUInt(SPDie, dwarf::DW_AT_accessibility, dwarf::DW_FORM_data1,
-            dwarf::DW_ACCESS_private);
-  else if (SP->isPublic())
-    addUInt(SPDie, dwarf::DW_AT_accessibility, dwarf::DW_FORM_data1,
-            dwarf::DW_ACCESS_public);
+  addAccess(SPDie, SP->getFlags());
 
   if (SP->isExplicit())
     addFlag(SPDie, dwarf::DW_AT_explicit);
@@ -1666,16 +1677,8 @@ DIE &DwarfUnit::constructMemberDIE(DIE &Buffer, const DIDerivedType *DT) {
     }
   }
 
-  if (DT->isProtected())
-    addUInt(MemberDie, dwarf::DW_AT_accessibility, dwarf::DW_FORM_data1,
-            dwarf::DW_ACCESS_protected);
-  else if (DT->isPrivate())
-    addUInt(MemberDie, dwarf::DW_AT_accessibility, dwarf::DW_FORM_data1,
-            dwarf::DW_ACCESS_private);
-  // Otherwise C++ member and base classes are considered public.
-  else if (DT->isPublic())
-    addUInt(MemberDie, dwarf::DW_AT_accessibility, dwarf::DW_FORM_data1,
-            dwarf::DW_ACCESS_public);
+  addAccess(MemberDie, DT->getFlags());
+
   if (DT->isVirtual())
     addUInt(MemberDie, dwarf::DW_AT_virtuality, dwarf::DW_FORM_data1,
             dwarf::DW_VIRTUALITY_virtual);
@@ -1717,15 +1720,7 @@ DIE *DwarfUnit::getOrCreateStaticMemberDIE(const DIDerivedType *DT) {
 
   // FIXME: We could omit private if the parent is a class_type, and
   // public if the parent is something else.
-  if (DT->isProtected())
-    addUInt(StaticMemberDIE, dwarf::DW_AT_accessibility, dwarf::DW_FORM_data1,
-            dwarf::DW_ACCESS_protected);
-  else if (DT->isPrivate())
-    addUInt(StaticMemberDIE, dwarf::DW_AT_accessibility, dwarf::DW_FORM_data1,
-            dwarf::DW_ACCESS_private);
-  else if (DT->isPublic())
-    addUInt(StaticMemberDIE, dwarf::DW_AT_accessibility, dwarf::DW_FORM_data1,
-            dwarf::DW_ACCESS_public);
+  addAccess(StaticMemberDIE, DT->getFlags());
 
   if (const ConstantInt *CI = dyn_cast_or_null<ConstantInt>(DT->getConstant()))
     addConstantValue(StaticMemberDIE, CI, Ty);
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.h b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.h
index 8140279adaef..54b0079dd7ce 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.h
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.h
@@ -226,6 +226,9 @@ public:
   /// Add thrown types.
   void addThrownTypes(DIE &Die, DINodeArray ThrownTypes);
 
+  /// Add the accessibility attribute.
+  void addAccess(DIE &Die, DINode::DIFlags Flags);
+
   /// Add a new type attribute to the specified entity.
   ///
   /// This takes and attribute parameter because DW_AT_friend attributes are
diff --git a/llvm/lib/CodeGen/AsmPrinter/OcamlGCPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/OcamlGCPrinter.cpp
index a9fb31d42679..3ade262d9af2 100644
--- a/llvm/lib/CodeGen/AsmPrinter/OcamlGCPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/OcamlGCPrinter.cpp
@@ -112,16 +112,12 @@ void OcamlGCMetadataPrinter::finishAssembly(Module &M, GCModuleInfo &Info,
   EmitCamlGlobal(M, AP, "frametable");
 
   int NumDescriptors = 0;
-  for (GCModuleInfo::FuncInfoVec::iterator I = Info.funcinfo_begin(),
-                                           IE = Info.funcinfo_end();
-       I != IE; ++I) {
-    GCFunctionInfo &FI = **I;
-    if (FI.getStrategy().getName() != getStrategy().getName())
+  for (std::unique_ptr<GCFunctionInfo> &FI :
+       llvm::make_range(Info.funcinfo_begin(), Info.funcinfo_end())) {
+    if (FI->getStrategy().getName() != getStrategy().getName())
       // this function is managed by some other GC
       continue;
-    for (GCFunctionInfo::iterator J = FI.begin(), JE = FI.end(); J != JE; ++J) {
-      NumDescriptors++;
-    }
+    NumDescriptors += FI->size();
   }
 
   if (NumDescriptors >= 1 << 16) {
@@ -131,35 +127,34 @@ void OcamlGCMetadataPrinter::finishAssembly(Module &M, GCModuleInfo &Info,
   AP.emitInt16(NumDescriptors);
   AP.emitAlignment(IntPtrSize == 4 ? Align(4) : Align(8));
 
-  for (GCModuleInfo::FuncInfoVec::iterator I = Info.funcinfo_begin(),
-                                           IE = Info.funcinfo_end();
-       I != IE; ++I) {
-    GCFunctionInfo &FI = **I;
-    if (FI.getStrategy().getName() != getStrategy().getName())
+  for (std::unique_ptr<GCFunctionInfo> &FI :
+       llvm::make_range(Info.funcinfo_begin(), Info.funcinfo_end())) {
+    if (FI->getStrategy().getName() != getStrategy().getName())
       // this function is managed by some other GC
       continue;
 
-    uint64_t FrameSize = FI.getFrameSize();
+    uint64_t FrameSize = FI->getFrameSize();
     if (FrameSize >= 1 << 16) {
       // Very rude!
-      report_fatal_error("Function '" + FI.getFunction().getName() +
+      report_fatal_error("Function '" + FI->getFunction().getName() +
                          "' is too large for the ocaml GC! "
                          "Frame size " +
                          Twine(FrameSize) +
                          ">= 65536.\n"
                          "(" +
-                         Twine(reinterpret_cast<uintptr_t>(&FI)) + ")");
+                         Twine(reinterpret_cast<uintptr_t>(FI.get())) + ")");
     }
 
     AP.OutStreamer->AddComment("live roots for " +
-                               Twine(FI.getFunction().getName()));
+                               Twine(FI->getFunction().getName()));
     AP.OutStreamer->AddBlankLine();
 
-    for (GCFunctionInfo::iterator J = FI.begin(), JE = FI.end(); J != JE; ++J) {
-      size_t LiveCount = FI.live_size(J);
+    for (GCFunctionInfo::iterator J = FI->begin(), JE = FI->end(); J != JE;
+         ++J) {
+      size_t LiveCount = FI->live_size(J);
       if (LiveCount >= 1 << 16) {
         // Very rude!
-        report_fatal_error("Function '" + FI.getFunction().getName() +
+        report_fatal_error("Function '" + FI->getFunction().getName() +
                            "' is too large for the ocaml GC! "
                            "Live root count " +
                            Twine(LiveCount) + " >= 65536.");
@@ -169,8 +164,8 @@ void OcamlGCMetadataPrinter::finishAssembly(Module &M, GCModuleInfo &Info,
       AP.emitInt16(FrameSize);
       AP.emitInt16(LiveCount);
 
-      for (GCFunctionInfo::live_iterator K = FI.live_begin(J),
-                                         KE = FI.live_end(J);
+      for (GCFunctionInfo::live_iterator K = FI->live_begin(J),
+                                         KE = FI->live_end(J);
            K != KE; ++K) {
         if (K->StackOffset >= 1 << 16) {
           // Very rude!
diff --git a/llvm/lib/CodeGen/AsmPrinter/PseudoProbePrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/PseudoProbePrinter.cpp
index 9e6f1a537de3..bab187f46535 100644
--- a/llvm/lib/CodeGen/AsmPrinter/PseudoProbePrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/PseudoProbePrinter.cpp
@@ -47,7 +47,6 @@ void PseudoProbeHandler::emitPseudoProbe(uint64_t Guid, uint64_t Index,
     InlinedAt = InlinedAt->getInlinedAt();
   }
 
-  SmallVector<InlineSite, 8> InlineStack(ReversedInlineStack.rbegin(),
-                                         ReversedInlineStack.rend());
+  SmallVector<InlineSite, 8> InlineStack(llvm::reverse(ReversedInlineStack));
   Asm->OutStreamer->emitPseudoProbe(Guid, Index, Type, Attr, InlineStack);
 }
diff --git a/llvm/lib/CodeGen/BranchFolding.cpp b/llvm/lib/CodeGen/BranchFolding.cpp
index 64dadc82b48b..0ff67f7ca00a 100644
--- a/llvm/lib/CodeGen/BranchFolding.cpp
+++ b/llvm/lib/CodeGen/BranchFolding.cpp
@@ -1125,8 +1125,8 @@ bool BranchFolder::TailMergeBlocks(MachineFunction &MF) {
     // If this is a large problem, avoid visiting the same basic blocks multiple
     // times.
     if (MergePotentials.size() == TailMergeThreshold)
-      for (unsigned i = 0, e = MergePotentials.size(); i != e; ++i)
-        TriedMerging.insert(MergePotentials[i].getBlock());
+      for (MergePotentialsElt &Elt : MergePotentials)
+        TriedMerging.insert(Elt.getBlock());
 
     if (MergePotentials.size() >= 2)
       MadeChange |= TryTailMergeBlocks(IBB, PredBB, MinCommonTailLength);
diff --git a/llvm/lib/CodeGen/CalcSpillWeights.cpp b/llvm/lib/CodeGen/CalcSpillWeights.cpp
index 863a0e1e0b56..5f9982cd155d 100644
--- a/llvm/lib/CodeGen/CalcSpillWeights.cpp
+++ b/llvm/lib/CodeGen/CalcSpillWeights.cpp
@@ -15,13 +15,13 @@
 #include "llvm/CodeGen/MachineLoopInfo.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/StackMaps.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/CodeGen/VirtRegMap.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/CodeGen/StackMaps.h"
 #include <cassert>
 #include <tuple>
 
@@ -35,7 +35,7 @@ void VirtRegAuxInfo::calculateSpillWeightsAndHints() {
 
   MachineRegisterInfo &MRI = MF.getRegInfo();
   for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
-    unsigned Reg = Register::index2VirtReg(I);
+    Register Reg = Register::index2VirtReg(I);
     if (MRI.reg_nodbg_empty(Reg))
       continue;
     calculateSpillWeightAndHint(LIS.getInterval(Reg));
@@ -64,14 +64,14 @@ static Register copyHint(const MachineInstr *MI, unsigned Reg,
   if (Register::isVirtualRegister(HReg))
     return Sub == HSub ? HReg : Register();
 
-  const TargetRegisterClass *rc = MRI.getRegClass(Reg);
+  const TargetRegisterClass *RC = MRI.getRegClass(Reg);
   MCRegister CopiedPReg = HSub ? TRI.getSubReg(HReg, HSub) : HReg.asMCReg();
-  if (rc->contains(CopiedPReg))
+  if (RC->contains(CopiedPReg))
     return CopiedPReg;
 
   // Check if reg:sub matches so that a super register could be hinted.
   if (Sub)
-    return TRI.getMatchingSuperReg(CopiedPReg, Sub, rc);
+    return TRI.getMatchingSuperReg(CopiedPReg, Sub, RC);
 
   return 0;
 }
@@ -80,8 +80,8 @@ static Register copyHint(const MachineInstr *MI, unsigned Reg,
 static bool isRematerializable(const LiveInterval &LI, const LiveIntervals &LIS,
                                const VirtRegMap &VRM,
                                const TargetInstrInfo &TII) {
-  unsigned Reg = LI.reg();
-  unsigned Original = VRM.getOriginal(Reg);
+  Register Reg = LI.reg();
+  Register Original = VRM.getOriginal(Reg);
   for (LiveInterval::const_vni_iterator I = LI.vni_begin(), E = LI.vni_end();
        I != E; ++I) {
     const VNInfo *VNI = *I;
@@ -183,8 +183,8 @@ float VirtRegAuxInfo::weightCalcHelper(LiveInterval &LI, SlotIndex *Start,
   bool ShouldUpdateLI = !IsLocalSplitArtifact;
 
   if (IsLocalSplitArtifact) {
-    MachineBasicBlock *localMBB = LIS.getMBBFromIndex(*End);
-    assert(localMBB == LIS.getMBBFromIndex(*Start) &&
+    MachineBasicBlock *LocalMBB = LIS.getMBBFromIndex(*End);
+    assert(LocalMBB == LIS.getMBBFromIndex(*Start) &&
            "start and end are expected to be in the same basic block");
 
     // Local split artifact will have 2 additional copy instructions and they
@@ -192,8 +192,8 @@ float VirtRegAuxInfo::weightCalcHelper(LiveInterval &LI, SlotIndex *Start,
     // localLI = COPY other
     // ...
     // other   = COPY localLI
-    TotalWeight += LiveIntervals::getSpillWeight(true, false, &MBFI, localMBB);
-    TotalWeight += LiveIntervals::getSpillWeight(false, true, &MBFI, localMBB);
+    TotalWeight += LiveIntervals::getSpillWeight(true, false, &MBFI, LocalMBB);
+    TotalWeight += LiveIntervals::getSpillWeight(false, true, &MBFI, LocalMBB);
 
     NumInstr += 2;
   }
diff --git a/llvm/lib/CodeGen/CodeGen.cpp b/llvm/lib/CodeGen/CodeGen.cpp
index bbdd8aab502e..7c236a9785d8 100644
--- a/llvm/lib/CodeGen/CodeGen.cpp
+++ b/llvm/lib/CodeGen/CodeGen.cpp
@@ -68,6 +68,8 @@ void llvm::initializeCodeGen(PassRegistry &Registry) {
   initializeMachineCSEPass(Registry);
   initializeMachineCombinerPass(Registry);
   initializeMachineCopyPropagationPass(Registry);
+  initializeMachineCycleInfoPrinterPassPass(Registry);
+  initializeMachineCycleInfoWrapperPassPass(Registry);
   initializeMachineDominatorTreePass(Registry);
   initializeMachineFunctionPrinterPassPass(Registry);
   initializeMachineLICMPass(Registry);
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index ac4180c4c3ab..747f4e4fdecc 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -4831,9 +4831,7 @@ static bool IsOperandAMemoryOperand(CallInst *CI, InlineAsm *IA, Value *OpVal,
   TargetLowering::AsmOperandInfoVector TargetConstraints =
       TLI.ParseConstraints(F->getParent()->getDataLayout(), &TRI, *CI);
 
-  for (unsigned i = 0, e = TargetConstraints.size(); i != e; ++i) {
-    TargetLowering::AsmOperandInfo &OpInfo = TargetConstraints[i];
-
+  for (TargetLowering::AsmOperandInfo &OpInfo : TargetConstraints) {
     // Compute the constraint code and ConstraintType to use.
     TLI.ComputeConstraintToUse(OpInfo, SDValue());
 
@@ -5617,9 +5615,7 @@ bool CodeGenPrepare::optimizeInlineAsmInst(CallInst *CS) {
   TargetLowering::AsmOperandInfoVector TargetConstraints =
       TLI->ParseConstraints(*DL, TRI, *CS);
   unsigned ArgNo = 0;
-  for (unsigned i = 0, e = TargetConstraints.size(); i != e; ++i) {
-    TargetLowering::AsmOperandInfo &OpInfo = TargetConstraints[i];
-
+  for (TargetLowering::AsmOperandInfo &OpInfo : TargetConstraints) {
     // Compute the constraint code and ConstraintType to use.
     TLI->ComputeConstraintToUse(OpInfo, SDValue());
 
@@ -6856,8 +6852,7 @@ bool CodeGenPrepare::optimizeSelectInst(SelectInst *SI) {
   // Use reverse iterator because later select may use the value of the
   // earlier select, and we need to propagate value through earlier select
   // to get the PHI operand.
-  for (auto It = ASI.rbegin(); It != ASI.rend(); ++It) {
-    SelectInst *SI = *It;
+  for (SelectInst *SI : llvm::reverse(ASI)) {
     // The select itself is replaced with a PHI Node.
     PHINode *PN = PHINode::Create(SI->getType(), 2, "", &EndBlock->front());
     PN->takeName(SI);
diff --git a/llvm/lib/CodeGen/CriticalAntiDepBreaker.cpp b/llvm/lib/CodeGen/CriticalAntiDepBreaker.cpp
index 4e98d49206b5..901409ea9f8f 100644
--- a/llvm/lib/CodeGen/CriticalAntiDepBreaker.cpp
+++ b/llvm/lib/CodeGen/CriticalAntiDepBreaker.cpp
@@ -405,8 +405,7 @@ findSuitableFreeRegister(RegRefIter RegRefBegin,
                          const TargetRegisterClass *RC,
                          SmallVectorImpl<unsigned> &Forbid) {
   ArrayRef<MCPhysReg> Order = RegClassInfo.getOrder(RC);
-  for (unsigned i = 0; i != Order.size(); ++i) {
-    unsigned NewReg = Order[i];
+  for (unsigned NewReg : Order) {
     // Don't replace a register with itself.
     if (NewReg == AntiDepReg) continue;
     // Don't replace a register with one that was recently used to repair
diff --git a/llvm/lib/CodeGen/DeadMachineInstructionElim.cpp b/llvm/lib/CodeGen/DeadMachineInstructionElim.cpp
index 0bb186a02416..5579152f1ce0 100644
--- a/llvm/lib/CodeGen/DeadMachineInstructionElim.cpp
+++ b/llvm/lib/CodeGen/DeadMachineInstructionElim.cpp
@@ -142,9 +142,9 @@ bool DeadMachineInstructionElim::eliminateDeadMI(MachineFunction &MF) {
       if (isDead(&MI)) {
         LLVM_DEBUG(dbgs() << "DeadMachineInstructionElim: DELETING: " << MI);
         // It is possible that some DBG_VALUE instructions refer to this
-        // instruction.  They get marked as undef and will be deleted
-        // in the live debug variable analysis.
-        MI.eraseFromParentAndMarkDBGValuesForRemoval();
+        // instruction. They will be deleted in the live debug variable
+        // analysis.
+        MI.eraseFromParent();
         AnyChanges = true;
         ++NumDeletes;
         continue;
diff --git a/llvm/lib/CodeGen/EarlyIfConversion.cpp b/llvm/lib/CodeGen/EarlyIfConversion.cpp
index 90883212a275..0b5469b02637 100644
--- a/llvm/lib/CodeGen/EarlyIfConversion.cpp
+++ b/llvm/lib/CodeGen/EarlyIfConversion.cpp
@@ -210,9 +210,9 @@ bool SSAIfConv::canSpeculateInstrs(MachineBasicBlock *MBB) {
 
   // Check all instructions, except the terminators. It is assumed that
   // terminators never have side effects or define any used register values.
-  for (MachineBasicBlock::iterator I = MBB->begin(),
-       E = MBB->getFirstTerminator(); I != E; ++I) {
-    if (I->isDebugInstr())
+  for (MachineInstr &MI :
+       llvm::make_range(MBB->begin(), MBB->getFirstTerminator())) {
+    if (MI.isDebugInstr())
       continue;
 
     if (++InstrCount > BlockInstrLimit && !Stress) {
@@ -222,28 +222,28 @@ bool SSAIfConv::canSpeculateInstrs(MachineBasicBlock *MBB) {
     }
 
     // There shouldn't normally be any phis in a single-predecessor block.
-    if (I->isPHI()) {
-      LLVM_DEBUG(dbgs() << "Can't hoist: " << *I);
+    if (MI.isPHI()) {
+      LLVM_DEBUG(dbgs() << "Can't hoist: " << MI);
       return false;
     }
 
     // Don't speculate loads. Note that it may be possible and desirable to
     // speculate GOT or constant pool loads that are guaranteed not to trap,
     // but we don't support that for now.
-    if (I->mayLoad()) {
-      LLVM_DEBUG(dbgs() << "Won't speculate load: " << *I);
+    if (MI.mayLoad()) {
+      LLVM_DEBUG(dbgs() << "Won't speculate load: " << MI);
       return false;
     }
 
     // We never speculate stores, so an AA pointer isn't necessary.
     bool DontMoveAcrossStore = true;
-    if (!I->isSafeToMove(nullptr, DontMoveAcrossStore)) {
-      LLVM_DEBUG(dbgs() << "Can't speculate: " << *I);
+    if (!MI.isSafeToMove(nullptr, DontMoveAcrossStore)) {
+      LLVM_DEBUG(dbgs() << "Can't speculate: " << MI);
       return false;
     }
 
     // Check for any dependencies on Head instructions.
-    if (!InstrDependenciesAllowIfConv(&(*I)))
+    if (!InstrDependenciesAllowIfConv(&MI))
       return false;
   }
   return true;
diff --git a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
index 17094a8e44f8..d061664e8c5d 100644
--- a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
@@ -256,7 +256,7 @@ mergeVectorRegsToResultRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs,
   LLT PartLLT = MRI.getType(SrcRegs[0]);
 
   // Deal with v3s16 split into v2s16
-  LLT LCMTy = getLCMType(LLTy, PartLLT);
+  LLT LCMTy = getCoverTy(LLTy, PartLLT);
   if (LCMTy == LLTy) {
     // Common case where no padding is needed.
     assert(DstRegs.size() == 1);
@@ -267,21 +267,9 @@ mergeVectorRegsToResultRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs,
   // widening the original value.
   Register UnmergeSrcReg;
   if (LCMTy != PartLLT) {
-    // e.g. A <3 x s16> value was split to <2 x s16>
-    // %register_value0:_(<2 x s16>)
-    // %register_value1:_(<2 x s16>)
-    // %undef:_(<2 x s16>) = G_IMPLICIT_DEF
-    // %concat:_<6 x s16>) = G_CONCAT_VECTORS %reg_value0, %reg_value1, %undef
-    // %dst_reg:_(<3 x s16>), %dead:_(<3 x s16>) = G_UNMERGE_VALUES %concat
-    const int NumWide = LCMTy.getSizeInBits() / PartLLT.getSizeInBits();
-    Register Undef = B.buildUndef(PartLLT).getReg(0);
-
-    // Build vector of undefs.
-    SmallVector<Register, 8> WidenedSrcs(NumWide, Undef);
-
-    // Replace the first sources with the real registers.
-    std::copy(SrcRegs.begin(), SrcRegs.end(), WidenedSrcs.begin());
-    UnmergeSrcReg = B.buildConcatVectors(LCMTy, WidenedSrcs).getReg(0);
+    assert(DstRegs.size() == 1);
+    return B.buildDeleteTrailingVectorElements(DstRegs[0],
+                                               B.buildMerge(LCMTy, SrcRegs));
   } else {
     // We don't need to widen anything if we're extracting a scalar which was
     // promoted to a vector e.g. s8 -> v4s8 -> s8
@@ -298,6 +286,8 @@ mergeVectorRegsToResultRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs,
   for (int I = DstRegs.size(); I != NumDst; ++I)
     PadDstRegs[I] = MRI.createGenericVirtualRegister(LLTy);
 
+  if (PadDstRegs.size() == 1)
+    return B.buildDeleteTrailingVectorElements(DstRegs[0], UnmergeSrcReg);
   return B.buildUnmerge(PadDstRegs, UnmergeSrcReg);
 }
 
@@ -485,7 +475,7 @@ static void buildCopyToRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs,
 
   MachineRegisterInfo &MRI = *B.getMRI();
   LLT DstTy = MRI.getType(DstRegs[0]);
-  LLT LCMTy = getLCMType(SrcTy, PartTy);
+  LLT LCMTy = getCoverTy(SrcTy, PartTy);
 
   const unsigned DstSize = DstTy.getSizeInBits();
   const unsigned SrcSize = SrcTy.getSizeInBits();
@@ -493,7 +483,7 @@ static void buildCopyToRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs,
 
   Register UnmergeSrc = SrcReg;
 
-  if (CoveringSize != SrcSize) {
+  if (!LCMTy.isVector() && CoveringSize != SrcSize) {
     // For scalars, it's common to be able to use a simple extension.
     if (SrcTy.isScalar() && DstTy.isScalar()) {
       CoveringSize = alignTo(SrcSize, DstSize);
@@ -510,14 +500,10 @@ static void buildCopyToRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs,
     }
   }
 
-  // Unmerge to the original registers and pad with dead defs.
-  SmallVector<Register, 8> UnmergeResults(DstRegs.begin(), DstRegs.end());
-  for (unsigned Size = DstSize * DstRegs.size(); Size != CoveringSize;
-       Size += DstSize) {
-    UnmergeResults.push_back(MRI.createGenericVirtualRegister(DstTy));
-  }
+  if (LCMTy.isVector() && CoveringSize != SrcSize)
+    UnmergeSrc = B.buildPadVectorWithUndefElements(LCMTy, SrcReg).getReg(0);
 
-  B.buildUnmerge(UnmergeResults, UnmergeSrc);
+  B.buildUnmerge(DstRegs, UnmergeSrc);
 }
 
 bool CallLowering::determineAndHandleAssignments(
diff --git a/llvm/lib/CodeGen/GlobalISel/Combiner.cpp b/llvm/lib/CodeGen/GlobalISel/Combiner.cpp
index 381c6df5c97a..dd1ef74e8ad0 100644
--- a/llvm/lib/CodeGen/GlobalISel/Combiner.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Combiner.cpp
@@ -135,7 +135,7 @@ bool Combiner::combineMachineInstrs(MachineFunction &MF,
         // Erase dead insts before even adding to the list.
         if (isTriviallyDead(CurMI, *MRI)) {
           LLVM_DEBUG(dbgs() << CurMI << "Is dead; erasing.\n");
-          CurMI.eraseFromParentAndMarkDBGValuesForRemoval();
+          CurMI.eraseFromParent();
           continue;
         }
         WorkList.deferred_insert(&CurMI);
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 755b3b844570..f7a634dad61a 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -1551,8 +1551,8 @@ void CombinerHelper::applyShiftOfShiftedLogic(MachineInstr &MI,
   Builder.buildInstr(MatchInfo.Logic->getOpcode(), {Dest}, {Shift1, Shift2});
 
   // These were one use so it's safe to remove them.
-  MatchInfo.Shift2->eraseFromParentAndMarkDBGValuesForRemoval();
-  MatchInfo.Logic->eraseFromParentAndMarkDBGValuesForRemoval();
+  MatchInfo.Shift2->eraseFromParent();
+  MatchInfo.Logic->eraseFromParent();
 
   MI.eraseFromParent();
 }
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 87cc60d51bc2..6d415c9c7f90 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -338,9 +338,10 @@ bool IRTranslator::translateCompare(const User &U,
     MIRBuilder.buildCopy(
         Res, getOrCreateVReg(*Constant::getAllOnesValue(U.getType())));
   else {
-    assert(CI && "Instruction should be CmpInst");
-    MIRBuilder.buildFCmp(Pred, Res, Op0, Op1,
-                         MachineInstr::copyFlagsFromInstruction(*CI));
+    uint16_t Flags = 0;
+    if (CI)
+      Flags = MachineInstr::copyFlagsFromInstruction(*CI);
+    MIRBuilder.buildFCmp(Pred, Res, Op0, Op1, Flags);
   }
 
   return true;
@@ -3502,7 +3503,7 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
   // Get rid of the now empty basic block.
   EntryBB->removeSuccessor(&NewEntryBB);
   MF->remove(EntryBB);
-  MF->DeleteMachineBasicBlock(EntryBB);
+  MF->deleteMachineBasicBlock(EntryBB);
 
   assert(&MF->front() == &NewEntryBB &&
          "New entry wasn't next in the list of basic block!");
diff --git a/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp b/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
index 9b2692486384..b10c9272a508 100644
--- a/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
@@ -163,7 +163,7 @@ bool InstructionSelect::runOnMachineFunction(MachineFunction &MF) {
       // If so, erase it.
       if (isTriviallyDead(MI, MRI)) {
         LLVM_DEBUG(dbgs() << "Is dead; erasing.\n");
-        MI.eraseFromParentAndMarkDBGValuesForRemoval();
+        MI.eraseFromParent();
         continue;
       }
 
@@ -255,8 +255,12 @@ bool InstructionSelect::runOnMachineFunction(MachineFunction &MF) {
     MachineInstr *MI = nullptr;
     if (!MRI.def_empty(VReg))
       MI = &*MRI.def_instr_begin(VReg);
-    else if (!MRI.use_empty(VReg))
+    else if (!MRI.use_empty(VReg)) {
       MI = &*MRI.use_instr_begin(VReg);
+      // Debug value instruction is permitted to use undefined vregs.
+      if (MI->isDebugValue())
+        continue;
+    }
     if (!MI)
       continue;
 
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index e09cd26eb0c1..e8a8efd5dad4 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -176,16 +176,18 @@ bool LegalizerHelper::extractParts(Register Reg, LLT RegTy,
     return true;
   }
 
+  // Perform irregular split. Leftover is last element of RegPieces.
   if (MainTy.isVector()) {
-    unsigned EltSize = MainTy.getScalarSizeInBits();
-    if (LeftoverSize % EltSize != 0)
-      return false;
-    LeftoverTy = LLT::scalarOrVector(
-        ElementCount::getFixed(LeftoverSize / EltSize), EltSize);
-  } else {
-    LeftoverTy = LLT::scalar(LeftoverSize);
+    SmallVector<Register, 8> RegPieces;
+    extractVectorParts(Reg, MainTy.getNumElements(), RegPieces);
+    for (unsigned i = 0; i < RegPieces.size() - 1; ++i)
+      VRegs.push_back(RegPieces[i]);
+    LeftoverRegs.push_back(RegPieces[RegPieces.size() - 1]);
+    LeftoverTy = MRI.getType(LeftoverRegs[0]);
+    return true;
   }
 
+  LeftoverTy = LLT::scalar(LeftoverSize);
   // For irregular sizes, extract the individual parts.
   for (unsigned I = 0; I != NumParts; ++I) {
     Register NewReg = MRI.createGenericVirtualRegister(MainTy);
@@ -203,6 +205,44 @@ bool LegalizerHelper::extractParts(Register Reg, LLT RegTy,
   return true;
 }
 
+void LegalizerHelper::extractVectorParts(Register Reg, unsigned NumElts,
+                                         SmallVectorImpl<Register> &VRegs) {
+  LLT RegTy = MRI.getType(Reg);
+  assert(RegTy.isVector() && "Expected a vector type");
+
+  LLT EltTy = RegTy.getElementType();
+  LLT NarrowTy = (NumElts == 1) ? EltTy : LLT::fixed_vector(NumElts, EltTy);
+  unsigned RegNumElts = RegTy.getNumElements();
+  unsigned LeftoverNumElts = RegNumElts % NumElts;
+  unsigned NumNarrowTyPieces = RegNumElts / NumElts;
+
+  // Perfect split without leftover
+  if (LeftoverNumElts == 0)
+    return extractParts(Reg, NarrowTy, NumNarrowTyPieces, VRegs);
+
+  // Irregular split. Provide direct access to all elements for artifact
+  // combiner using unmerge to elements. Then build vectors with NumElts
+  // elements. Remaining element(s) will be (used to build vector) Leftover.
+  SmallVector<Register, 8> Elts;
+  extractParts(Reg, EltTy, RegNumElts, Elts);
+
+  unsigned Offset = 0;
+  // Requested sub-vectors of NarrowTy.
+  for (unsigned i = 0; i < NumNarrowTyPieces; ++i, Offset += NumElts) {
+    ArrayRef<Register> Pieces(&Elts[Offset], NumElts);
+    VRegs.push_back(MIRBuilder.buildMerge(NarrowTy, Pieces).getReg(0));
+  }
+
+  // Leftover element(s).
+  if (LeftoverNumElts == 1) {
+    VRegs.push_back(Elts[Offset]);
+  } else {
+    LLT LeftoverTy = LLT::fixed_vector(LeftoverNumElts, EltTy);
+    ArrayRef<Register> Pieces(&Elts[Offset], LeftoverNumElts);
+    VRegs.push_back(MIRBuilder.buildMerge(LeftoverTy, Pieces).getReg(0));
+  }
+}
+
 void LegalizerHelper::insertParts(Register DstReg,
                                   LLT ResultTy, LLT PartTy,
                                   ArrayRef<Register> PartRegs,
@@ -223,6 +263,15 @@ void LegalizerHelper::insertParts(Register DstReg,
     return;
   }
 
+  // Merge sub-vectors with different number of elements and insert into DstReg.
+  if (ResultTy.isVector()) {
+    assert(LeftoverRegs.size() == 1 && "Expected one leftover register");
+    SmallVector<Register, 8> AllRegs;
+    for (auto Reg : concat<const Register>(PartRegs, LeftoverRegs))
+      AllRegs.push_back(Reg);
+    return mergeMixedSubvectors(DstReg, AllRegs);
+  }
+
   SmallVector<Register> GCDRegs;
   LLT GCDTy = getGCDType(getGCDType(ResultTy, LeftoverTy), PartTy);
   for (auto PartReg : concat<const Register>(PartRegs, LeftoverRegs))
@@ -231,6 +280,30 @@ void LegalizerHelper::insertParts(Register DstReg,
   buildWidenedRemergeToDst(DstReg, ResultLCMTy, GCDRegs);
 }
 
+void LegalizerHelper::appendVectorElts(SmallVectorImpl<Register> &Elts,
+                                       Register Reg) {
+  LLT Ty = MRI.getType(Reg);
+  SmallVector<Register, 8> RegElts;
+  extractParts(Reg, Ty.getScalarType(), Ty.getNumElements(), RegElts);
+  Elts.append(RegElts);
+}
+
+/// Merge \p PartRegs with different types into \p DstReg.
+void LegalizerHelper::mergeMixedSubvectors(Register DstReg,
+                                           ArrayRef<Register> PartRegs) {
+  SmallVector<Register, 8> AllElts;
+  for (unsigned i = 0; i < PartRegs.size() - 1; ++i)
+    appendVectorElts(AllElts, PartRegs[i]);
+
+  Register Leftover = PartRegs[PartRegs.size() - 1];
+  if (MRI.getType(Leftover).isScalar())
+    AllElts.push_back(Leftover);
+  else
+    appendVectorElts(AllElts, Leftover);
+
+  MIRBuilder.buildMerge(DstReg, AllElts);
+}
+
 /// Append the result registers of G_UNMERGE_VALUES \p MI to \p Regs.
 static void getUnmergeResults(SmallVectorImpl<Register> &Regs,
                               const MachineInstr &MI) {
@@ -916,8 +989,26 @@ LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
     return Legalized;
   }
 
-  case TargetOpcode::G_FREEZE:
-    return reduceOperationWidth(MI, TypeIdx, NarrowTy);
+  case TargetOpcode::G_FREEZE: {
+    if (TypeIdx != 0)
+      return UnableToLegalize;
+
+    LLT Ty = MRI.getType(MI.getOperand(0).getReg());
+    // Should widen scalar first
+    if (Ty.getSizeInBits() % NarrowTy.getSizeInBits() != 0)
+      return UnableToLegalize;
+
+    auto Unmerge = MIRBuilder.buildUnmerge(NarrowTy, MI.getOperand(1).getReg());
+    SmallVector<Register, 8> Parts;
+    for (unsigned i = 0; i < Unmerge->getNumDefs(); ++i) {
+      Parts.push_back(
+          MIRBuilder.buildFreeze(NarrowTy, Unmerge.getReg(i)).getReg(0));
+    }
+
+    MIRBuilder.buildMerge(MI.getOperand(0).getReg(), Parts);
+    MI.eraseFromParent();
+    return Legalized;
+  }
   case TargetOpcode::G_ADD:
   case TargetOpcode::G_SUB:
   case TargetOpcode::G_SADDO:
@@ -1372,37 +1463,17 @@ void LegalizerHelper::moreElementsVectorDst(MachineInstr &MI, LLT WideTy,
                                             unsigned OpIdx) {
   MachineOperand &MO = MI.getOperand(OpIdx);
   MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
-  MO.setReg(widenWithUnmerge(WideTy, MO.getReg()));
+  Register Dst = MO.getReg();
+  Register DstExt = MRI.createGenericVirtualRegister(WideTy);
+  MO.setReg(DstExt);
+  MIRBuilder.buildDeleteTrailingVectorElements(Dst, DstExt);
 }
 
 void LegalizerHelper::moreElementsVectorSrc(MachineInstr &MI, LLT MoreTy,
                                             unsigned OpIdx) {
   MachineOperand &MO = MI.getOperand(OpIdx);
-
-  LLT OldTy = MRI.getType(MO.getReg());
-  unsigned OldElts = OldTy.getNumElements();
-  unsigned NewElts = MoreTy.getNumElements();
-
-  unsigned NumParts = NewElts / OldElts;
-
-  // Use concat_vectors if the result is a multiple of the number of elements.
-  if (NumParts * OldElts == NewElts) {
-    SmallVector<Register, 8> Parts;
-    Parts.push_back(MO.getReg());
-
-    Register ImpDef = MIRBuilder.buildUndef(OldTy).getReg(0);
-    for (unsigned I = 1; I != NumParts; ++I)
-      Parts.push_back(ImpDef);
-
-    auto Concat = MIRBuilder.buildConcatVectors(MoreTy, Parts);
-    MO.setReg(Concat.getReg(0));
-    return;
-  }
-
-  Register MoreReg = MRI.createGenericVirtualRegister(MoreTy);
-  Register ImpDef = MIRBuilder.buildUndef(MoreTy).getReg(0);
-  MIRBuilder.buildInsert(MoreReg, ImpDef, MO.getReg(), 0);
-  MO.setReg(MoreReg);
+  SmallVector<Register, 8> Regs;
+  MO.setReg(MIRBuilder.buildPadVectorWithUndefElements(MoreTy, MO).getReg(0));
 }
 
 void LegalizerHelper::bitcastSrc(MachineInstr &MI, LLT CastTy, unsigned OpIdx) {
@@ -3558,20 +3629,83 @@ Register LegalizerHelper::getVectorElementPointer(Register VecPtr, LLT VecTy,
   return MIRBuilder.buildPtrAdd(PtrTy, VecPtr, Mul).getReg(0);
 }
 
-LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorImplicitDef(
-    MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy) {
-  Register DstReg = MI.getOperand(0).getReg();
-  LLT DstTy = MRI.getType(DstReg);
-  LLT LCMTy = getLCMType(DstTy, NarrowTy);
+#ifndef NDEBUG
+/// Check that all vector operands have same number of elements. Other operands
+/// should be listed in NonVecOp.
+static bool hasSameNumEltsOnAllVectorOperands(
+    GenericMachineInstr &MI, MachineRegisterInfo &MRI,
+    std::initializer_list<unsigned> NonVecOpIndices) {
+  if (MI.getNumMemOperands() != 0)
+    return false;
 
-  unsigned NumParts = LCMTy.getSizeInBits() / NarrowTy.getSizeInBits();
+  LLT VecTy = MRI.getType(MI.getReg(0));
+  if (!VecTy.isVector())
+    return false;
+  unsigned NumElts = VecTy.getNumElements();
 
-  auto NewUndef = MIRBuilder.buildUndef(NarrowTy);
-  SmallVector<Register, 8> Parts(NumParts, NewUndef.getReg(0));
+  for (unsigned OpIdx = 1; OpIdx < MI.getNumOperands(); ++OpIdx) {
+    MachineOperand &Op = MI.getOperand(OpIdx);
+    if (!Op.isReg()) {
+      if (!is_contained(NonVecOpIndices, OpIdx))
+        return false;
+      continue;
+    }
 
-  buildWidenedRemergeToDst(DstReg, LCMTy, Parts);
-  MI.eraseFromParent();
-  return Legalized;
+    LLT Ty = MRI.getType(Op.getReg());
+    if (!Ty.isVector()) {
+      if (!is_contained(NonVecOpIndices, OpIdx))
+        return false;
+      is_contained(NonVecOpIndices, OpIdx);
+      continue;
+    }
+
+    if (Ty.getNumElements() != NumElts)
+      return false;
+  }
+
+  return true;
+}
+#endif
+
+/// Fill \p DstOps with DstOps that have same number of elements combined as
+/// the Ty. These DstOps have either scalar type when \p NumElts = 1 or are
+/// vectors with \p NumElts elements. When Ty.getNumElements() is not multiple
+/// of \p NumElts last DstOp (leftover) has fewer then \p NumElts elements.
+static void makeDstOps(SmallVectorImpl<DstOp> &DstOps, LLT Ty,
+                       unsigned NumElts) {
+  LLT LeftoverTy;
+  assert(Ty.isVector() && "Expected vector type");
+  LLT EltTy = Ty.getElementType();
+  LLT NarrowTy = (NumElts == 1) ? EltTy : LLT::fixed_vector(NumElts, EltTy);
+  int NumParts, NumLeftover;
+  std::tie(NumParts, NumLeftover) =
+      getNarrowTypeBreakDown(Ty, NarrowTy, LeftoverTy);
+
+  assert(NumParts > 0 && "Error in getNarrowTypeBreakDown");
+  for (int i = 0; i < NumParts; ++i) {
+    DstOps.push_back(NarrowTy);
+  }
+
+  if (LeftoverTy.isValid()) {
+    assert(NumLeftover == 1 && "expected exactly one leftover");
+    DstOps.push_back(LeftoverTy);
+  }
+}
+
+/// Operand \p Op is used on \p N sub-instructions. Fill \p Ops with \p N SrcOps
+/// made from \p Op depending on operand type.
+static void broadcastSrcOp(SmallVectorImpl<SrcOp> &Ops, unsigned N,
+                           MachineOperand &Op) {
+  for (unsigned i = 0; i < N; ++i) {
+    if (Op.isReg())
+      Ops.push_back(Op.getReg());
+    else if (Op.isImm())
+      Ops.push_back(Op.getImm());
+    else if (Op.isPredicate())
+      Ops.push_back(static_cast<CmpInst::Predicate>(Op.getPredicate()));
+    else
+      llvm_unreachable("Unsupported type");
+  }
 }
 
 // Handle splitting vector operations which need to have the same number of
@@ -3588,335 +3722,116 @@ LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorImplicitDef(
 //             s64 = G_SHL s64, s32
 LegalizerHelper::LegalizeResult
 LegalizerHelper::fewerElementsVectorMultiEltType(
-  MachineInstr &MI, unsigned TypeIdx, LLT NarrowTyArg) {
-  if (TypeIdx != 0)
-    return UnableToLegalize;
-
-  const LLT NarrowTy0 = NarrowTyArg;
-  const Register DstReg = MI.getOperand(0).getReg();
-  LLT DstTy = MRI.getType(DstReg);
-  LLT LeftoverTy0;
-
-  // All of the operands need to have the same number of elements, so if we can
-  // determine a type breakdown for the result type, we can for all of the
-  // source types.
-  int NumParts = getNarrowTypeBreakDown(DstTy, NarrowTy0, LeftoverTy0).first;
-  if (NumParts < 0)
-    return UnableToLegalize;
-
-  SmallVector<MachineInstrBuilder, 4> NewInsts;
-
-  SmallVector<Register, 4> DstRegs, LeftoverDstRegs;
-  SmallVector<Register, 4> PartRegs, LeftoverRegs;
-
-  for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) {
-    Register SrcReg = MI.getOperand(I).getReg();
-    LLT SrcTyI = MRI.getType(SrcReg);
-    const auto NewEC = NarrowTy0.isVector() ? NarrowTy0.getElementCount()
-                                            : ElementCount::getFixed(1);
-    LLT NarrowTyI = LLT::scalarOrVector(NewEC, SrcTyI.getScalarType());
-    LLT LeftoverTyI;
-
-    // Split this operand into the requested typed registers, and any leftover
-    // required to reproduce the original type.
-    if (!extractParts(SrcReg, SrcTyI, NarrowTyI, LeftoverTyI, PartRegs,
-                      LeftoverRegs))
-      return UnableToLegalize;
-
-    if (I == 1) {
-      // For the first operand, create an instruction for each part and setup
-      // the result.
-      for (Register PartReg : PartRegs) {
-        Register PartDstReg = MRI.createGenericVirtualRegister(NarrowTy0);
-        NewInsts.push_back(MIRBuilder.buildInstrNoInsert(MI.getOpcode())
-                               .addDef(PartDstReg)
-                               .addUse(PartReg));
-        DstRegs.push_back(PartDstReg);
-      }
-
-      for (Register LeftoverReg : LeftoverRegs) {
-        Register PartDstReg = MRI.createGenericVirtualRegister(LeftoverTy0);
-        NewInsts.push_back(MIRBuilder.buildInstrNoInsert(MI.getOpcode())
-                               .addDef(PartDstReg)
-                               .addUse(LeftoverReg));
-        LeftoverDstRegs.push_back(PartDstReg);
-      }
+    GenericMachineInstr &MI, unsigned NumElts,
+    std::initializer_list<unsigned> NonVecOpIndices) {
+  assert(hasSameNumEltsOnAllVectorOperands(MI, MRI, NonVecOpIndices) &&
+         "Non-compatible opcode or not specified non-vector operands");
+  unsigned OrigNumElts = MRI.getType(MI.getReg(0)).getNumElements();
+
+  unsigned NumInputs = MI.getNumOperands() - MI.getNumDefs();
+  unsigned NumDefs = MI.getNumDefs();
+
+  // Create DstOps (sub-vectors with NumElts elts + Leftover) for each output.
+  // Build instructions with DstOps to use instruction found by CSE directly.
+  // CSE copies found instruction into given vreg when building with vreg dest.
+  SmallVector<SmallVector<DstOp, 8>, 2> OutputOpsPieces(NumDefs);
+  // Output registers will be taken from created instructions.
+  SmallVector<SmallVector<Register, 8>, 2> OutputRegs(NumDefs);
+  for (unsigned i = 0; i < NumDefs; ++i) {
+    makeDstOps(OutputOpsPieces[i], MRI.getType(MI.getReg(i)), NumElts);
+  }
+
+  // Split vector input operands into sub-vectors with NumElts elts + Leftover.
+  // Operands listed in NonVecOpIndices will be used as is without splitting;
+  // examples: compare predicate in icmp and fcmp (op 1), vector select with i1
+  // scalar condition (op 1), immediate in sext_inreg (op 2).
+  SmallVector<SmallVector<SrcOp, 8>, 3> InputOpsPieces(NumInputs);
+  for (unsigned UseIdx = NumDefs, UseNo = 0; UseIdx < MI.getNumOperands();
+       ++UseIdx, ++UseNo) {
+    if (is_contained(NonVecOpIndices, UseIdx)) {
+      broadcastSrcOp(InputOpsPieces[UseNo], OutputOpsPieces[0].size(),
+                     MI.getOperand(UseIdx));
     } else {
-      assert(NewInsts.size() == PartRegs.size() + LeftoverRegs.size());
-
-      // Add the newly created operand splits to the existing instructions. The
-      // odd-sized pieces are ordered after the requested NarrowTyArg sized
-      // pieces.
-      unsigned InstCount = 0;
-      for (unsigned J = 0, JE = PartRegs.size(); J != JE; ++J)
-        NewInsts[InstCount++].addUse(PartRegs[J]);
-      for (unsigned J = 0, JE = LeftoverRegs.size(); J != JE; ++J)
-        NewInsts[InstCount++].addUse(LeftoverRegs[J]);
+      SmallVector<Register, 8> SplitPieces;
+      extractVectorParts(MI.getReg(UseIdx), NumElts, SplitPieces);
+      for (auto Reg : SplitPieces)
+        InputOpsPieces[UseNo].push_back(Reg);
     }
-
-    PartRegs.clear();
-    LeftoverRegs.clear();
   }
 
-  // Insert the newly built operations and rebuild the result register.
-  for (auto &MIB : NewInsts)
-    MIRBuilder.insertInstr(MIB);
+  unsigned NumLeftovers = OrigNumElts % NumElts ? 1 : 0;
 
-  insertParts(DstReg, DstTy, NarrowTy0, DstRegs, LeftoverTy0, LeftoverDstRegs);
+  // Take i-th piece of each input operand split and build sub-vector/scalar
+  // instruction. Set i-th DstOp(s) from OutputOpsPieces as destination(s).
+  for (unsigned i = 0; i < OrigNumElts / NumElts + NumLeftovers; ++i) {
+    SmallVector<DstOp, 2> Defs;
+    for (unsigned DstNo = 0; DstNo < NumDefs; ++DstNo)
+      Defs.push_back(OutputOpsPieces[DstNo][i]);
 
-  MI.eraseFromParent();
-  return Legalized;
-}
+    SmallVector<SrcOp, 3> Uses;
+    for (unsigned InputNo = 0; InputNo < NumInputs; ++InputNo)
+      Uses.push_back(InputOpsPieces[InputNo][i]);
 
-LegalizerHelper::LegalizeResult
-LegalizerHelper::fewerElementsVectorCasts(MachineInstr &MI, unsigned TypeIdx,
-                                          LLT NarrowTy) {
-  if (TypeIdx != 0)
-    return UnableToLegalize;
-
-  Register DstReg = MI.getOperand(0).getReg();
-  Register SrcReg = MI.getOperand(1).getReg();
-  LLT DstTy = MRI.getType(DstReg);
-  LLT SrcTy = MRI.getType(SrcReg);
-
-  LLT NarrowTy0 = NarrowTy;
-  LLT NarrowTy1;
-  unsigned NumParts;
-
-  if (NarrowTy.isVector()) {
-    // Uneven breakdown not handled.
-    NumParts = DstTy.getNumElements() / NarrowTy.getNumElements();
-    if (NumParts * NarrowTy.getNumElements() != DstTy.getNumElements())
-      return UnableToLegalize;
-
-    NarrowTy1 = LLT::vector(NarrowTy.getElementCount(), SrcTy.getElementType());
-  } else {
-    NumParts = DstTy.getNumElements();
-    NarrowTy1 = SrcTy.getElementType();
+    auto I = MIRBuilder.buildInstr(MI.getOpcode(), Defs, Uses, MI.getFlags());
+    for (unsigned DstNo = 0; DstNo < NumDefs; ++DstNo)
+      OutputRegs[DstNo].push_back(I.getReg(DstNo));
   }
 
-  SmallVector<Register, 4> SrcRegs, DstRegs;
-  extractParts(SrcReg, NarrowTy1, NumParts, SrcRegs);
-
-  for (unsigned I = 0; I < NumParts; ++I) {
-    Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
-    MachineInstr *NewInst =
-        MIRBuilder.buildInstr(MI.getOpcode(), {DstReg}, {SrcRegs[I]});
-
-    NewInst->setFlags(MI.getFlags());
-    DstRegs.push_back(DstReg);
+  // Merge small outputs into MI's output for each def operand.
+  if (NumLeftovers) {
+    for (unsigned i = 0; i < NumDefs; ++i)
+      mergeMixedSubvectors(MI.getReg(i), OutputRegs[i]);
+  } else {
+    for (unsigned i = 0; i < NumDefs; ++i)
+      MIRBuilder.buildMerge(MI.getReg(i), OutputRegs[i]);
   }
 
-  if (NarrowTy.isVector())
-    MIRBuilder.buildConcatVectors(DstReg, DstRegs);
-  else
-    MIRBuilder.buildBuildVector(DstReg, DstRegs);
-
   MI.eraseFromParent();
   return Legalized;
 }
 
 LegalizerHelper::LegalizeResult
-LegalizerHelper::fewerElementsVectorCmp(MachineInstr &MI, unsigned TypeIdx,
-                                        LLT NarrowTy) {
-  Register DstReg = MI.getOperand(0).getReg();
-  Register Src0Reg = MI.getOperand(2).getReg();
-  LLT DstTy = MRI.getType(DstReg);
-  LLT SrcTy = MRI.getType(Src0Reg);
-
-  unsigned NumParts;
-  LLT NarrowTy0, NarrowTy1;
-
-  if (TypeIdx == 0) {
-    unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
-    unsigned OldElts = DstTy.getNumElements();
-
-    NarrowTy0 = NarrowTy;
-    NumParts = NarrowTy.isVector() ? (OldElts / NewElts) : DstTy.getNumElements();
-    NarrowTy1 = NarrowTy.isVector() ? LLT::vector(NarrowTy.getElementCount(),
-                                                  SrcTy.getScalarSizeInBits())
-                                    : SrcTy.getElementType();
-
-  } else {
-    unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
-    unsigned OldElts = SrcTy.getNumElements();
-
-    NumParts = NarrowTy.isVector() ? (OldElts / NewElts) :
-      NarrowTy.getNumElements();
-    NarrowTy0 =
-        LLT::vector(NarrowTy.getElementCount(), DstTy.getScalarSizeInBits());
-    NarrowTy1 = NarrowTy;
+LegalizerHelper::fewerElementsVectorPhi(GenericMachineInstr &MI,
+                                        unsigned NumElts) {
+  unsigned OrigNumElts = MRI.getType(MI.getReg(0)).getNumElements();
+
+  unsigned NumInputs = MI.getNumOperands() - MI.getNumDefs();
+  unsigned NumDefs = MI.getNumDefs();
+
+  SmallVector<DstOp, 8> OutputOpsPieces;
+  SmallVector<Register, 8> OutputRegs;
+  makeDstOps(OutputOpsPieces, MRI.getType(MI.getReg(0)), NumElts);
+
+  // Instructions that perform register split will be inserted in basic block
+  // where register is defined (basic block is in the next operand).
+  SmallVector<SmallVector<Register, 8>, 3> InputOpsPieces(NumInputs / 2);
+  for (unsigned UseIdx = NumDefs, UseNo = 0; UseIdx < MI.getNumOperands();
+       UseIdx += 2, ++UseNo) {
+    MachineBasicBlock &OpMBB = *MI.getOperand(UseIdx + 1).getMBB();
+    MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
+    extractVectorParts(MI.getReg(UseIdx), NumElts, InputOpsPieces[UseNo]);
   }
 
-  // FIXME: Don't know how to handle the situation where the small vectors
-  // aren't all the same size yet.
-  if (NarrowTy1.isVector() &&
-      NarrowTy1.getNumElements() * NumParts != DstTy.getNumElements())
-    return UnableToLegalize;
-
-  CmpInst::Predicate Pred
-    = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
-
-  SmallVector<Register, 2> Src1Regs, Src2Regs, DstRegs;
-  extractParts(MI.getOperand(2).getReg(), NarrowTy1, NumParts, Src1Regs);
-  extractParts(MI.getOperand(3).getReg(), NarrowTy1, NumParts, Src2Regs);
+  // Build PHIs with fewer elements.
+  unsigned NumLeftovers = OrigNumElts % NumElts ? 1 : 0;
+  MIRBuilder.setInsertPt(*MI.getParent(), MI);
+  for (unsigned i = 0; i < OrigNumElts / NumElts + NumLeftovers; ++i) {
+    auto Phi = MIRBuilder.buildInstr(TargetOpcode::G_PHI);
+    Phi.addDef(
+        MRI.createGenericVirtualRegister(OutputOpsPieces[i].getLLTTy(MRI)));
+    OutputRegs.push_back(Phi.getReg(0));
 
-  for (unsigned I = 0; I < NumParts; ++I) {
-    Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
-    DstRegs.push_back(DstReg);
-
-    if (MI.getOpcode() == TargetOpcode::G_ICMP)
-      MIRBuilder.buildICmp(Pred, DstReg, Src1Regs[I], Src2Regs[I]);
-    else {
-      MachineInstr *NewCmp
-        = MIRBuilder.buildFCmp(Pred, DstReg, Src1Regs[I], Src2Regs[I]);
-      NewCmp->setFlags(MI.getFlags());
+    for (unsigned j = 0; j < NumInputs / 2; ++j) {
+      Phi.addUse(InputOpsPieces[j][i]);
+      Phi.add(MI.getOperand(1 + j * 2 + 1));
     }
   }
 
-  if (NarrowTy1.isVector())
-    MIRBuilder.buildConcatVectors(DstReg, DstRegs);
-  else
-    MIRBuilder.buildBuildVector(DstReg, DstRegs);
-
-  MI.eraseFromParent();
-  return Legalized;
-}
-
-LegalizerHelper::LegalizeResult
-LegalizerHelper::fewerElementsVectorSelect(MachineInstr &MI, unsigned TypeIdx,
-                                           LLT NarrowTy) {
-  Register DstReg = MI.getOperand(0).getReg();
-  Register CondReg = MI.getOperand(1).getReg();
-
-  unsigned NumParts = 0;
-  LLT NarrowTy0, NarrowTy1;
-
-  LLT DstTy = MRI.getType(DstReg);
-  LLT CondTy = MRI.getType(CondReg);
-  unsigned Size = DstTy.getSizeInBits();
-
-  assert(TypeIdx == 0 || CondTy.isVector());
-
-  if (TypeIdx == 0) {
-    NarrowTy0 = NarrowTy;
-    NarrowTy1 = CondTy;
-
-    unsigned NarrowSize = NarrowTy0.getSizeInBits();
-    // FIXME: Don't know how to handle the situation where the small vectors
-    // aren't all the same size yet.
-    if (Size % NarrowSize != 0)
-      return UnableToLegalize;
-
-    NumParts = Size / NarrowSize;
-
-    // Need to break down the condition type
-    if (CondTy.isVector()) {
-      if (CondTy.getNumElements() == NumParts)
-        NarrowTy1 = CondTy.getElementType();
-      else
-        NarrowTy1 =
-            LLT::vector(CondTy.getElementCount().divideCoefficientBy(NumParts),
-                        CondTy.getScalarSizeInBits());
-    }
+  // Merge small outputs into MI's def.
+  if (NumLeftovers) {
+    mergeMixedSubvectors(MI.getReg(0), OutputRegs);
   } else {
-    NumParts = CondTy.getNumElements();
-    if (NarrowTy.isVector()) {
-      // TODO: Handle uneven breakdown.
-      if (NumParts * NarrowTy.getNumElements() != CondTy.getNumElements())
-        return UnableToLegalize;
-
-      return UnableToLegalize;
-    } else {
-      NarrowTy0 = DstTy.getElementType();
-      NarrowTy1 = NarrowTy;
-    }
-  }
-
-  SmallVector<Register, 2> DstRegs, Src0Regs, Src1Regs, Src2Regs;
-  if (CondTy.isVector())
-    extractParts(MI.getOperand(1).getReg(), NarrowTy1, NumParts, Src0Regs);
-
-  extractParts(MI.getOperand(2).getReg(), NarrowTy0, NumParts, Src1Regs);
-  extractParts(MI.getOperand(3).getReg(), NarrowTy0, NumParts, Src2Regs);
-
-  for (unsigned i = 0; i < NumParts; ++i) {
-    Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
-    MIRBuilder.buildSelect(DstReg, CondTy.isVector() ? Src0Regs[i] : CondReg,
-                           Src1Regs[i], Src2Regs[i]);
-    DstRegs.push_back(DstReg);
-  }
-
-  if (NarrowTy0.isVector())
-    MIRBuilder.buildConcatVectors(DstReg, DstRegs);
-  else
-    MIRBuilder.buildBuildVector(DstReg, DstRegs);
-
-  MI.eraseFromParent();
-  return Legalized;
-}
-
-LegalizerHelper::LegalizeResult
-LegalizerHelper::fewerElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx,
-                                        LLT NarrowTy) {
-  const Register DstReg = MI.getOperand(0).getReg();
-  LLT PhiTy = MRI.getType(DstReg);
-  LLT LeftoverTy;
-
-  // All of the operands need to have the same number of elements, so if we can
-  // determine a type breakdown for the result type, we can for all of the
-  // source types.
-  int NumParts, NumLeftover;
-  std::tie(NumParts, NumLeftover)
-    = getNarrowTypeBreakDown(PhiTy, NarrowTy, LeftoverTy);
-  if (NumParts < 0)
-    return UnableToLegalize;
-
-  SmallVector<Register, 4> DstRegs, LeftoverDstRegs;
-  SmallVector<MachineInstrBuilder, 4> NewInsts;
-
-  const int TotalNumParts = NumParts + NumLeftover;
-
-  // Insert the new phis in the result block first.
-  for (int I = 0; I != TotalNumParts; ++I) {
-    LLT Ty = I < NumParts ? NarrowTy : LeftoverTy;
-    Register PartDstReg = MRI.createGenericVirtualRegister(Ty);
-    NewInsts.push_back(MIRBuilder.buildInstr(TargetOpcode::G_PHI)
-                       .addDef(PartDstReg));
-    if (I < NumParts)
-      DstRegs.push_back(PartDstReg);
-    else
-      LeftoverDstRegs.push_back(PartDstReg);
-  }
-
-  MachineBasicBlock *MBB = MI.getParent();
-  MIRBuilder.setInsertPt(*MBB, MBB->getFirstNonPHI());
-  insertParts(DstReg, PhiTy, NarrowTy, DstRegs, LeftoverTy, LeftoverDstRegs);
-
-  SmallVector<Register, 4> PartRegs, LeftoverRegs;
-
-  // Insert code to extract the incoming values in each predecessor block.
-  for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
-    PartRegs.clear();
-    LeftoverRegs.clear();
-
-    Register SrcReg = MI.getOperand(I).getReg();
-    MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB();
-    MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
-
-    LLT Unused;
-    if (!extractParts(SrcReg, PhiTy, NarrowTy, Unused, PartRegs,
-                      LeftoverRegs))
-      return UnableToLegalize;
-
-    // Add the newly created operand splits to the existing instructions. The
-    // odd-sized pieces are ordered after the requested NarrowTyArg sized
-    // pieces.
-    for (int J = 0; J != TotalNumParts; ++J) {
-      MachineInstrBuilder MIB = NewInsts[J];
-      MIB.addUse(J < NumParts ? PartRegs[J] : LeftoverRegs[J - NumParts]);
-      MIB.addMBB(&OpMBB);
-    }
+    MIRBuilder.buildMerge(MI.getReg(0), OutputRegs);
   }
 
   MI.eraseFromParent();
@@ -3927,27 +3842,36 @@ LegalizerHelper::LegalizeResult
 LegalizerHelper::fewerElementsVectorUnmergeValues(MachineInstr &MI,
                                                   unsigned TypeIdx,
                                                   LLT NarrowTy) {
-  if (TypeIdx != 1)
-    return UnableToLegalize;
-
   const int NumDst = MI.getNumOperands() - 1;
   const Register SrcReg = MI.getOperand(NumDst).getReg();
-  LLT SrcTy = MRI.getType(SrcReg);
-
   LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
+  LLT SrcTy = MRI.getType(SrcReg);
 
-  // TODO: Create sequence of extracts.
-  if (DstTy == NarrowTy)
+  if (TypeIdx != 1 || NarrowTy == DstTy)
     return UnableToLegalize;
 
-  LLT GCDTy = getGCDType(SrcTy, NarrowTy);
-  if (DstTy == GCDTy) {
-    // This would just be a copy of the same unmerge.
-    // TODO: Create extracts, pad with undef and create intermediate merges.
+  // Requires compatible types. Otherwise SrcReg should have been defined by
+  // merge-like instruction that would get artifact combined. Most likely
+  // instruction that defines SrcReg has to perform more/fewer elements
+  // legalization compatible with NarrowTy.
+  assert(SrcTy.isVector() && NarrowTy.isVector() && "Expected vector types");
+  assert((SrcTy.getScalarType() == NarrowTy.getScalarType()) && "bad type");
+
+  if ((SrcTy.getSizeInBits() % NarrowTy.getSizeInBits() != 0) ||
+      (NarrowTy.getSizeInBits() % DstTy.getSizeInBits() != 0))
     return UnableToLegalize;
-  }
 
-  auto Unmerge = MIRBuilder.buildUnmerge(GCDTy, SrcReg);
+  // This is most likely DstTy (smaller then register size) packed in SrcTy
+  // (larger then register size) and since unmerge was not combined it will be
+  // lowered to bit sequence extracts from register. Unpack SrcTy to NarrowTy
+  // (register size) pieces first. Then unpack each of NarrowTy pieces to DstTy.
+
+  // %1:_(DstTy), %2, %3, %4 = G_UNMERGE_VALUES %0:_(SrcTy)
+  //
+  // %5:_(NarrowTy), %6 = G_UNMERGE_VALUES %0:_(SrcTy) - reg sequence
+  // %1:_(DstTy), %2 = G_UNMERGE_VALUES %5:_(NarrowTy) - sequence of bits in reg
+  // %3:_(DstTy), %4 = G_UNMERGE_VALUES %6:_(NarrowTy)
+  auto Unmerge = MIRBuilder.buildUnmerge(NarrowTy, SrcReg);
   const int NumUnmerge = Unmerge->getNumOperands() - 1;
   const int PartsPerUnmerge = NumDst / NumUnmerge;
 
@@ -3964,89 +3888,87 @@ LegalizerHelper::fewerElementsVectorUnmergeValues(MachineInstr &MI,
 }
 
 LegalizerHelper::LegalizeResult
-LegalizerHelper::fewerElementsVectorMulo(MachineInstr &MI, unsigned TypeIdx,
-                                         LLT NarrowTy) {
-  Register Result = MI.getOperand(0).getReg();
-  Register Overflow = MI.getOperand(1).getReg();
-  Register LHS = MI.getOperand(2).getReg();
-  Register RHS = MI.getOperand(3).getReg();
-
-  LLT SrcTy = MRI.getType(LHS);
-  if (!SrcTy.isVector())
-    return UnableToLegalize;
-
-  LLT ElementType = SrcTy.getElementType();
-  LLT OverflowElementTy = MRI.getType(Overflow).getElementType();
-  const ElementCount NumResult = SrcTy.getElementCount();
-  LLT GCDTy = getGCDType(SrcTy, NarrowTy);
-
-  // Unmerge the operands to smaller parts of GCD type.
-  auto UnmergeLHS = MIRBuilder.buildUnmerge(GCDTy, LHS);
-  auto UnmergeRHS = MIRBuilder.buildUnmerge(GCDTy, RHS);
-
-  const int NumOps = UnmergeLHS->getNumOperands() - 1;
-  const ElementCount PartsPerUnmerge = NumResult.divideCoefficientBy(NumOps);
-  LLT OverflowTy = LLT::scalarOrVector(PartsPerUnmerge, OverflowElementTy);
-  LLT ResultTy = LLT::scalarOrVector(PartsPerUnmerge, ElementType);
-
-  // Perform the operation over unmerged parts.
-  SmallVector<Register, 8> ResultParts;
-  SmallVector<Register, 8> OverflowParts;
-  for (int I = 0; I != NumOps; ++I) {
-    Register Operand1 = UnmergeLHS->getOperand(I).getReg();
-    Register Operand2 = UnmergeRHS->getOperand(I).getReg();
-    auto PartMul = MIRBuilder.buildInstr(MI.getOpcode(), {ResultTy, OverflowTy},
-                                         {Operand1, Operand2});
-    ResultParts.push_back(PartMul->getOperand(0).getReg());
-    OverflowParts.push_back(PartMul->getOperand(1).getReg());
-  }
-
-  LLT ResultLCMTy = buildLCMMergePieces(SrcTy, NarrowTy, GCDTy, ResultParts);
-  LLT OverflowLCMTy =
-      LLT::scalarOrVector(ResultLCMTy.getElementCount(), OverflowElementTy);
-
-  // Recombine the pieces to the original result and overflow registers.
-  buildWidenedRemergeToDst(Result, ResultLCMTy, ResultParts);
-  buildWidenedRemergeToDst(Overflow, OverflowLCMTy, OverflowParts);
-  MI.eraseFromParent();
-  return Legalized;
-}
-
-// Handle FewerElementsVector a G_BUILD_VECTOR or G_CONCAT_VECTORS that produces
-// a vector
-//
-// Create a G_BUILD_VECTOR or G_CONCAT_VECTORS of NarrowTy pieces, padding with
-// undef as necessary.
-//
-// %3:_(<3 x s16>) = G_BUILD_VECTOR %0, %1, %2
-//   -> <2 x s16>
-//
-// %4:_(s16) = G_IMPLICIT_DEF
-// %5:_(<2 x s16>) = G_BUILD_VECTOR %0, %1
-// %6:_(<2 x s16>) = G_BUILD_VECTOR %2, %4
-// %7:_(<2 x s16>) = G_IMPLICIT_DEF
-// %8:_(<6 x s16>) = G_CONCAT_VECTORS %5, %6, %7
-// %3:_(<3 x s16>), %8:_(<3 x s16>) = G_UNMERGE_VALUES %8
-LegalizerHelper::LegalizeResult
 LegalizerHelper::fewerElementsVectorMerge(MachineInstr &MI, unsigned TypeIdx,
                                           LLT NarrowTy) {
   Register DstReg = MI.getOperand(0).getReg();
   LLT DstTy = MRI.getType(DstReg);
   LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());
-  LLT GCDTy = getGCDType(getGCDType(SrcTy, NarrowTy), DstTy);
+  // Requires compatible types. Otherwise user of DstReg did not perform unmerge
+  // that should have been artifact combined. Most likely instruction that uses
+  // DstReg has to do more/fewer elements legalization compatible with NarrowTy.
+  assert(DstTy.isVector() && NarrowTy.isVector() && "Expected vector types");
+  assert((DstTy.getScalarType() == NarrowTy.getScalarType()) && "bad type");
+  if (NarrowTy == SrcTy)
+    return UnableToLegalize;
 
-  // Break into a common type
-  SmallVector<Register, 16> Parts;
-  for (const MachineOperand &MO : llvm::drop_begin(MI.operands()))
-    extractGCDType(Parts, GCDTy, MO.getReg());
+  // This attempts to lower part of LCMTy merge/unmerge sequence. Intended use
+  // is for old mir tests. Since the changes to more/fewer elements it should no
+  // longer be possible to generate MIR like this when starting from llvm-ir
+  // because LCMTy approach was replaced with merge/unmerge to vector elements.
+  if (TypeIdx == 1) {
+    assert(SrcTy.isVector() && "Expected vector types");
+    assert((SrcTy.getScalarType() == NarrowTy.getScalarType()) && "bad type");
+    if ((DstTy.getSizeInBits() % NarrowTy.getSizeInBits() != 0) ||
+        (NarrowTy.getNumElements() >= SrcTy.getNumElements()))
+      return UnableToLegalize;
+    // %2:_(DstTy) = G_CONCAT_VECTORS %0:_(SrcTy), %1:_(SrcTy)
+    //
+    // %3:_(EltTy), %4, %5 = G_UNMERGE_VALUES %0:_(SrcTy)
+    // %6:_(EltTy), %7, %8 = G_UNMERGE_VALUES %1:_(SrcTy)
+    // %9:_(NarrowTy) = G_BUILD_VECTOR %3:_(EltTy), %4
+    // %10:_(NarrowTy) = G_BUILD_VECTOR %5:_(EltTy), %6
+    // %11:_(NarrowTy) = G_BUILD_VECTOR %7:_(EltTy), %8
+    // %2:_(DstTy) = G_CONCAT_VECTORS %9:_(NarrowTy), %10, %11
+
+    SmallVector<Register, 8> Elts;
+    LLT EltTy = MRI.getType(MI.getOperand(1).getReg()).getScalarType();
+    for (unsigned i = 1; i < MI.getNumOperands(); ++i) {
+      auto Unmerge = MIRBuilder.buildUnmerge(EltTy, MI.getOperand(i).getReg());
+      for (unsigned j = 0; j < Unmerge->getNumDefs(); ++j)
+        Elts.push_back(Unmerge.getReg(j));
+    }
 
-  // Build the requested new merge, padding with undef.
-  LLT LCMTy = buildLCMMergePieces(DstTy, NarrowTy, GCDTy, Parts,
-                                  TargetOpcode::G_ANYEXT);
+    SmallVector<Register, 8> NarrowTyElts;
+    unsigned NumNarrowTyElts = NarrowTy.getNumElements();
+    unsigned NumNarrowTyPieces = DstTy.getNumElements() / NumNarrowTyElts;
+    for (unsigned i = 0, Offset = 0; i < NumNarrowTyPieces;
+         ++i, Offset += NumNarrowTyElts) {
+      ArrayRef<Register> Pieces(&Elts[Offset], NumNarrowTyElts);
+      NarrowTyElts.push_back(MIRBuilder.buildMerge(NarrowTy, Pieces).getReg(0));
+    }
 
-  // Pack into the original result register.
-  buildWidenedRemergeToDst(DstReg, LCMTy, Parts);
+    MIRBuilder.buildMerge(DstReg, NarrowTyElts);
+    MI.eraseFromParent();
+    return Legalized;
+  }
+
+  assert(TypeIdx == 0 && "Bad type index");
+  if ((NarrowTy.getSizeInBits() % SrcTy.getSizeInBits() != 0) ||
+      (DstTy.getSizeInBits() % NarrowTy.getSizeInBits() != 0))
+    return UnableToLegalize;
 
+  // This is most likely SrcTy (smaller then register size) packed in DstTy
+  // (larger then register size) and since merge was not combined it will be
+  // lowered to bit sequence packing into register. Merge SrcTy to NarrowTy
+  // (register size) pieces first. Then merge each of NarrowTy pieces to DstTy.
+
+  // %0:_(DstTy) = G_MERGE_VALUES %1:_(SrcTy), %2, %3, %4
+  //
+  // %5:_(NarrowTy) = G_MERGE_VALUES %1:_(SrcTy), %2 - sequence of bits in reg
+  // %6:_(NarrowTy) = G_MERGE_VALUES %3:_(SrcTy), %4
+  // %0:_(DstTy)  = G_MERGE_VALUES %5:_(NarrowTy), %6 - reg sequence
+  SmallVector<Register, 8> NarrowTyElts;
+  unsigned NumParts = DstTy.getNumElements() / NarrowTy.getNumElements();
+  unsigned NumSrcElts = SrcTy.isVector() ? SrcTy.getNumElements() : 1;
+  unsigned NumElts = NarrowTy.getNumElements() / NumSrcElts;
+  for (unsigned i = 0; i < NumParts; ++i) {
+    SmallVector<Register, 8> Sources;
+    for (unsigned j = 0; j < NumElts; ++j)
+      Sources.push_back(MI.getOperand(1 + i * NumElts + j).getReg());
+    NarrowTyElts.push_back(MIRBuilder.buildMerge(NarrowTy, Sources).getReg(0));
+  }
+
+  MIRBuilder.buildMerge(DstReg, NarrowTyElts);
   MI.eraseFromParent();
   return Legalized;
 }
@@ -4218,163 +4140,14 @@ LegalizerHelper::reduceLoadStoreWidth(GLoadStore &LdStMI, unsigned TypeIdx,
 }
 
 LegalizerHelper::LegalizeResult
-LegalizerHelper::reduceOperationWidth(MachineInstr &MI, unsigned int TypeIdx,
-                                      LLT NarrowTy) {
-  assert(TypeIdx == 0 && "only one type index expected");
-
-  const unsigned Opc = MI.getOpcode();
-  const int NumDefOps = MI.getNumExplicitDefs();
-  const int NumSrcOps = MI.getNumOperands() - NumDefOps;
-  const unsigned Flags = MI.getFlags();
-  const unsigned NarrowSize = NarrowTy.getSizeInBits();
-  const LLT NarrowScalarTy = LLT::scalar(NarrowSize);
-
-  assert(MI.getNumOperands() <= 4 && "expected instruction with either 1 "
-                                     "result and 1-3 sources or 2 results and "
-                                     "1-2 sources");
-
-  SmallVector<Register, 2> DstRegs;
-  for (int I = 0; I < NumDefOps; ++I)
-    DstRegs.push_back(MI.getOperand(I).getReg());
-
-  // First of all check whether we are narrowing (changing the element type)
-  // or reducing the vector elements
-  const LLT DstTy = MRI.getType(DstRegs[0]);
-  const bool IsNarrow = NarrowTy.getScalarType() != DstTy.getScalarType();
-
-  SmallVector<Register, 8> ExtractedRegs[3];
-  SmallVector<Register, 8> Parts;
-
-  // Break down all the sources into NarrowTy pieces we can operate on. This may
-  // involve creating merges to a wider type, padded with undef.
-  for (int I = 0; I != NumSrcOps; ++I) {
-    Register SrcReg = MI.getOperand(I + NumDefOps).getReg();
-    LLT SrcTy = MRI.getType(SrcReg);
-
-    // The type to narrow SrcReg to. For narrowing, this is a smaller scalar.
-    // For fewerElements, this is a smaller vector with the same element type.
-    LLT OpNarrowTy;
-    if (IsNarrow) {
-      OpNarrowTy = NarrowScalarTy;
-
-      // In case of narrowing, we need to cast vectors to scalars for this to
-      // work properly
-      // FIXME: Can we do without the bitcast here if we're narrowing?
-      if (SrcTy.isVector()) {
-        SrcTy = LLT::scalar(SrcTy.getSizeInBits());
-        SrcReg = MIRBuilder.buildBitcast(SrcTy, SrcReg).getReg(0);
-      }
-    } else {
-      auto NarrowEC = NarrowTy.isVector() ? NarrowTy.getElementCount()
-                                          : ElementCount::getFixed(1);
-      OpNarrowTy = LLT::scalarOrVector(NarrowEC, SrcTy.getScalarType());
-    }
-
-    LLT GCDTy = extractGCDType(ExtractedRegs[I], SrcTy, OpNarrowTy, SrcReg);
-
-    // Build a sequence of NarrowTy pieces in ExtractedRegs for this operand.
-    buildLCMMergePieces(SrcTy, OpNarrowTy, GCDTy, ExtractedRegs[I],
-                        TargetOpcode::G_ANYEXT);
-  }
-
-  SmallVector<Register, 8> ResultRegs[2];
-
-  // Input operands for each sub-instruction.
-  SmallVector<SrcOp, 4> InputRegs(NumSrcOps, Register());
-
-  int NumParts = ExtractedRegs[0].size();
-  const unsigned DstSize = DstTy.getSizeInBits();
-  const LLT DstScalarTy = LLT::scalar(DstSize);
-
-  // Narrowing needs to use scalar types
-  LLT DstLCMTy, NarrowDstTy;
-  if (IsNarrow) {
-    DstLCMTy = getLCMType(DstScalarTy, NarrowScalarTy);
-    NarrowDstTy = NarrowScalarTy;
-  } else {
-    DstLCMTy = getLCMType(DstTy, NarrowTy);
-    NarrowDstTy = NarrowTy;
-  }
-
-  // We widened the source registers to satisfy merge/unmerge size
-  // constraints. We'll have some extra fully undef parts.
-  const int NumRealParts = (DstSize + NarrowSize - 1) / NarrowSize;
-
-  for (int I = 0; I != NumRealParts; ++I) {
-    // Emit this instruction on each of the split pieces.
-    for (int J = 0; J != NumSrcOps; ++J)
-      InputRegs[J] = ExtractedRegs[J][I];
-
-    MachineInstrBuilder Inst;
-    if (NumDefOps == 1)
-      Inst = MIRBuilder.buildInstr(Opc, {NarrowDstTy}, InputRegs, Flags);
-    else
-      Inst = MIRBuilder.buildInstr(Opc, {NarrowDstTy, NarrowDstTy}, InputRegs,
-                                   Flags);
-
-    for (int J = 0; J != NumDefOps; ++J)
-      ResultRegs[J].push_back(Inst.getReg(J));
-  }
-
-  // Fill out the widened result with undef instead of creating instructions
-  // with undef inputs.
-  int NumUndefParts = NumParts - NumRealParts;
-  if (NumUndefParts != 0) {
-    Register Undef = MIRBuilder.buildUndef(NarrowDstTy).getReg(0);
-    for (int I = 0; I != NumDefOps; ++I)
-      ResultRegs[I].append(NumUndefParts, Undef);
-  }
-
-  // Extract the possibly padded result. Use a scratch register if we need to do
-  // a final bitcast, otherwise use the original result register.
-  Register MergeDstReg;
-  for (int I = 0; I != NumDefOps; ++I) {
-    if (IsNarrow && DstTy.isVector())
-      MergeDstReg = MRI.createGenericVirtualRegister(DstScalarTy);
-    else
-      MergeDstReg = DstRegs[I];
-
-    buildWidenedRemergeToDst(MergeDstReg, DstLCMTy, ResultRegs[I]);
-
-    // Recast to vector if we narrowed a vector
-    if (IsNarrow && DstTy.isVector())
-      MIRBuilder.buildBitcast(DstRegs[I], MergeDstReg);
-  }
-
-  MI.eraseFromParent();
-  return Legalized;
-}
-
-LegalizerHelper::LegalizeResult
-LegalizerHelper::fewerElementsVectorSextInReg(MachineInstr &MI, unsigned TypeIdx,
-                                              LLT NarrowTy) {
-  Register DstReg = MI.getOperand(0).getReg();
-  Register SrcReg = MI.getOperand(1).getReg();
-  int64_t Imm = MI.getOperand(2).getImm();
-
-  LLT DstTy = MRI.getType(DstReg);
-
-  SmallVector<Register, 8> Parts;
-  LLT GCDTy = extractGCDType(Parts, DstTy, NarrowTy, SrcReg);
-  LLT LCMTy = buildLCMMergePieces(DstTy, NarrowTy, GCDTy, Parts);
-
-  for (Register &R : Parts)
-    R = MIRBuilder.buildSExtInReg(NarrowTy, R, Imm).getReg(0);
-
-  buildWidenedRemergeToDst(DstReg, LCMTy, Parts);
-
-  MI.eraseFromParent();
-  return Legalized;
-}
-
-LegalizerHelper::LegalizeResult
 LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
                                      LLT NarrowTy) {
   using namespace TargetOpcode;
+  GenericMachineInstr &GMI = cast<GenericMachineInstr>(MI);
+  unsigned NumElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
 
   switch (MI.getOpcode()) {
   case G_IMPLICIT_DEF:
-    return fewerElementsVectorImplicitDef(MI, TypeIdx, NarrowTy);
   case G_TRUNC:
   case G_AND:
   case G_OR:
@@ -4439,10 +4212,8 @@ LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
   case G_SSUBSAT:
   case G_UADDSAT:
   case G_USUBSAT:
-    return reduceOperationWidth(MI, TypeIdx, NarrowTy);
   case G_UMULO:
   case G_SMULO:
-    return fewerElementsVectorMulo(MI, TypeIdx, NarrowTy);
   case G_SHL:
   case G_LSHR:
   case G_ASHR:
@@ -4454,7 +4225,6 @@ LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
   case G_CTTZ_ZERO_UNDEF:
   case G_CTPOP:
   case G_FCOPYSIGN:
-    return fewerElementsVectorMultiEltType(MI, TypeIdx, NarrowTy);
   case G_ZEXT:
   case G_SEXT:
   case G_ANYEXT:
@@ -4467,14 +4237,16 @@ LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
   case G_INTTOPTR:
   case G_PTRTOINT:
   case G_ADDRSPACE_CAST:
-    return fewerElementsVectorCasts(MI, TypeIdx, NarrowTy);
+    return fewerElementsVectorMultiEltType(GMI, NumElts);
   case G_ICMP:
   case G_FCMP:
-    return fewerElementsVectorCmp(MI, TypeIdx, NarrowTy);
+    return fewerElementsVectorMultiEltType(GMI, NumElts, {1 /*cpm predicate*/});
   case G_SELECT:
-    return fewerElementsVectorSelect(MI, TypeIdx, NarrowTy);
+    if (MRI.getType(MI.getOperand(1).getReg()).isVector())
+      return fewerElementsVectorMultiEltType(GMI, NumElts);
+    return fewerElementsVectorMultiEltType(GMI, NumElts, {1 /*scalar cond*/});
   case G_PHI:
-    return fewerElementsVectorPhi(MI, TypeIdx, NarrowTy);
+    return fewerElementsVectorPhi(GMI, NumElts);
   case G_UNMERGE_VALUES:
     return fewerElementsVectorUnmergeValues(MI, TypeIdx, NarrowTy);
   case G_BUILD_VECTOR:
@@ -4491,7 +4263,7 @@ LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
   case G_STORE:
     return reduceLoadStoreWidth(cast<GLoadStore>(MI), TypeIdx, NarrowTy);
   case G_SEXT_INREG:
-    return fewerElementsVectorSextInReg(MI, TypeIdx, NarrowTy);
+    return fewerElementsVectorMultiEltType(GMI, NumElts, {2 /*imm*/});
   GISEL_VECREDUCE_CASES_NONSEQ
     return fewerElementsVectorReductions(MI, TypeIdx, NarrowTy);
   case G_SHUFFLE_VECTOR:
@@ -5053,6 +4825,15 @@ LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx,
   case TargetOpcode::G_AND:
   case TargetOpcode::G_OR:
   case TargetOpcode::G_XOR:
+  case TargetOpcode::G_ADD:
+  case TargetOpcode::G_SUB:
+  case TargetOpcode::G_MUL:
+  case TargetOpcode::G_FADD:
+  case TargetOpcode::G_FMUL:
+  case TargetOpcode::G_UADDSAT:
+  case TargetOpcode::G_USUBSAT:
+  case TargetOpcode::G_SADDSAT:
+  case TargetOpcode::G_SSUBSAT:
   case TargetOpcode::G_SMIN:
   case TargetOpcode::G_SMAX:
   case TargetOpcode::G_UMIN:
@@ -5070,6 +4851,17 @@ LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx,
     Observer.changedInstr(MI);
     return Legalized;
   }
+  case TargetOpcode::G_FMA:
+  case TargetOpcode::G_FSHR:
+  case TargetOpcode::G_FSHL: {
+    Observer.changingInstr(MI);
+    moreElementsVectorSrc(MI, MoreTy, 1);
+    moreElementsVectorSrc(MI, MoreTy, 2);
+    moreElementsVectorSrc(MI, MoreTy, 3);
+    moreElementsVectorDst(MI, MoreTy, 0);
+    Observer.changedInstr(MI);
+    return Legalized;
+  }
   case TargetOpcode::G_EXTRACT:
     if (TypeIdx != 1)
       return UnableToLegalize;
@@ -5079,6 +4871,11 @@ LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx,
     return Legalized;
   case TargetOpcode::G_INSERT:
   case TargetOpcode::G_FREEZE:
+  case TargetOpcode::G_FNEG:
+  case TargetOpcode::G_FABS:
+  case TargetOpcode::G_BSWAP:
+  case TargetOpcode::G_FCANONICALIZE:
+  case TargetOpcode::G_SEXT_INREG:
     if (TypeIdx != 0)
       return UnableToLegalize;
     Observer.changingInstr(MI);
@@ -5098,30 +4895,34 @@ LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx,
     moreElementsVectorDst(MI, MoreTy, 0);
     Observer.changedInstr(MI);
     return Legalized;
-  case TargetOpcode::G_UNMERGE_VALUES: {
-    if (TypeIdx != 1)
-      return UnableToLegalize;
-
-    LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
-    int NumDst = MI.getNumOperands() - 1;
-    moreElementsVectorSrc(MI, MoreTy, NumDst);
-
-    auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES);
-    for (int I = 0; I != NumDst; ++I)
-      MIB.addDef(MI.getOperand(I).getReg());
+  case TargetOpcode::G_UNMERGE_VALUES:
+    return UnableToLegalize;
+  case TargetOpcode::G_PHI:
+    return moreElementsVectorPhi(MI, TypeIdx, MoreTy);
+  case TargetOpcode::G_SHUFFLE_VECTOR:
+    return moreElementsVectorShuffle(MI, TypeIdx, MoreTy);
+  case TargetOpcode::G_BUILD_VECTOR: {
+    SmallVector<SrcOp, 8> Elts;
+    for (auto Op : MI.uses()) {
+      Elts.push_back(Op.getReg());
+    }
 
-    int NewNumDst = MoreTy.getSizeInBits() / DstTy.getSizeInBits();
-    for (int I = NumDst; I != NewNumDst; ++I)
-      MIB.addDef(MRI.createGenericVirtualRegister(DstTy));
+    for (unsigned i = Elts.size(); i < MoreTy.getNumElements(); ++i) {
+      Elts.push_back(MIRBuilder.buildUndef(MoreTy.getScalarType()));
+    }
 
-    MIB.addUse(MI.getOperand(NumDst).getReg());
+    MIRBuilder.buildDeleteTrailingVectorElements(
+        MI.getOperand(0).getReg(), MIRBuilder.buildInstr(Opc, {MoreTy}, Elts));
     MI.eraseFromParent();
     return Legalized;
   }
-  case TargetOpcode::G_PHI:
-    return moreElementsVectorPhi(MI, TypeIdx, MoreTy);
-  case TargetOpcode::G_SHUFFLE_VECTOR:
-    return moreElementsVectorShuffle(MI, TypeIdx, MoreTy);
+  case TargetOpcode::G_TRUNC: {
+    Observer.changingInstr(MI);
+    moreElementsVectorSrc(MI, MoreTy, 1);
+    moreElementsVectorDst(MI, MoreTy, 0);
+    Observer.changedInstr(MI);
+    return Legalized;
+  }
   default:
     return UnableToLegalize;
   }
@@ -6778,6 +6579,24 @@ LegalizerHelper::lowerExtractInsertVectorElt(MachineInstr &MI) {
 
   LLT VecTy = MRI.getType(SrcVec);
   LLT EltTy = VecTy.getElementType();
+  unsigned NumElts = VecTy.getNumElements();
+
+  int64_t IdxVal;
+  if (mi_match(Idx, MRI, m_ICst(IdxVal)) && IdxVal <= NumElts) {
+    SmallVector<Register, 8> SrcRegs;
+    extractParts(SrcVec, EltTy, NumElts, SrcRegs);
+
+    if (InsertVal) {
+      SrcRegs[IdxVal] = MI.getOperand(2).getReg();
+      MIRBuilder.buildMerge(DstReg, SrcRegs);
+    } else {
+      MIRBuilder.buildCopy(DstReg, SrcRegs[IdxVal]);
+    }
+
+    MI.eraseFromParent();
+    return Legalized;
+  }
+
   if (!EltTy.isByteSized()) { // Not implemented.
     LLVM_DEBUG(dbgs() << "Can't handle non-byte element vectors yet\n");
     return UnableToLegalize;
@@ -6796,7 +6615,6 @@ LegalizerHelper::lowerExtractInsertVectorElt(MachineInstr &MI) {
   // if the index is out of bounds.
   Register EltPtr = getVectorElementPointer(StackTemp.getReg(0), VecTy, Idx);
 
-  int64_t IdxVal;
   if (mi_match(Idx, MRI, m_ICst(IdxVal))) {
     int64_t Offset = IdxVal * EltBytes;
     PtrInfo = PtrInfo.getWithOffset(Offset);
@@ -6923,6 +6741,32 @@ LegalizerHelper::lowerExtract(MachineInstr &MI) {
   LLT DstTy = MRI.getType(Dst);
   LLT SrcTy = MRI.getType(Src);
 
+  // Extract sub-vector or one element
+  if (SrcTy.isVector()) {
+    unsigned SrcEltSize = SrcTy.getElementType().getSizeInBits();
+    unsigned DstSize = DstTy.getSizeInBits();
+
+    if ((Offset % SrcEltSize == 0) && (DstSize % SrcEltSize == 0) &&
+        (Offset + DstSize <= SrcTy.getSizeInBits())) {
+      // Unmerge and allow access to each Src element for the artifact combiner.
+      auto Unmerge = MIRBuilder.buildUnmerge(SrcTy.getElementType(), Src);
+
+      // Take element(s) we need to extract and copy it (merge them).
+      SmallVector<Register, 8> SubVectorElts;
+      for (unsigned Idx = Offset / SrcEltSize;
+           Idx < (Offset + DstSize) / SrcEltSize; ++Idx) {
+        SubVectorElts.push_back(Unmerge.getReg(Idx));
+      }
+      if (SubVectorElts.size() == 1)
+        MIRBuilder.buildCopy(Dst, SubVectorElts[0]);
+      else
+        MIRBuilder.buildMerge(Dst, SubVectorElts);
+
+      MI.eraseFromParent();
+      return Legalized;
+    }
+  }
+
   if (DstTy.isScalar() &&
       (SrcTy.isScalar() ||
        (SrcTy.isVector() && DstTy == SrcTy.getElementType()))) {
@@ -6956,6 +6800,45 @@ LegalizerHelper::LegalizeResult LegalizerHelper::lowerInsert(MachineInstr &MI) {
   LLT DstTy = MRI.getType(Src);
   LLT InsertTy = MRI.getType(InsertSrc);
 
+  // Insert sub-vector or one element
+  if (DstTy.isVector() && !InsertTy.isPointer()) {
+    LLT EltTy = DstTy.getElementType();
+    unsigned EltSize = EltTy.getSizeInBits();
+    unsigned InsertSize = InsertTy.getSizeInBits();
+
+    if ((Offset % EltSize == 0) && (InsertSize % EltSize == 0) &&
+        (Offset + InsertSize <= DstTy.getSizeInBits())) {
+      auto UnmergeSrc = MIRBuilder.buildUnmerge(EltTy, Src);
+      SmallVector<Register, 8> DstElts;
+      unsigned Idx = 0;
+      // Elements from Src before insert start Offset
+      for (; Idx < Offset / EltSize; ++Idx) {
+        DstElts.push_back(UnmergeSrc.getReg(Idx));
+      }
+
+      // Replace elements in Src with elements from InsertSrc
+      if (InsertTy.getSizeInBits() > EltSize) {
+        auto UnmergeInsertSrc = MIRBuilder.buildUnmerge(EltTy, InsertSrc);
+        for (unsigned i = 0; Idx < (Offset + InsertSize) / EltSize;
+             ++Idx, ++i) {
+          DstElts.push_back(UnmergeInsertSrc.getReg(i));
+        }
+      } else {
+        DstElts.push_back(InsertSrc);
+        ++Idx;
+      }
+
+      // Remaining elements from Src after insert
+      for (; Idx < DstTy.getNumElements(); ++Idx) {
+        DstElts.push_back(UnmergeSrc.getReg(Idx));
+      }
+
+      MIRBuilder.buildMerge(Dst, DstElts);
+      MI.eraseFromParent();
+      return Legalized;
+    }
+  }
+
   if (InsertTy.isVector() ||
       (DstTy.isVector() && DstTy.getElementType() != InsertTy))
     return UnableToLegalize;
diff --git a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
index 03dda806cb1e..de8dbd456901 100644
--- a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
@@ -554,12 +554,11 @@ bool LoadStoreOpt::mergeBlockStores(MachineBasicBlock &MBB) {
   bool Changed = false;
   // Walk through the block bottom-up, looking for merging candidates.
   StoreMergeCandidate Candidate;
-  for (auto II = MBB.rbegin(), IE = MBB.rend(); II != IE; ++II) {
-    MachineInstr &MI = *II;
+  for (MachineInstr &MI : llvm::reverse(MBB)) {
     if (InstsToErase.contains(&MI))
       continue;
 
-    if (auto StoreMI = dyn_cast<GStore>(&*II)) {
+    if (auto *StoreMI = dyn_cast<GStore>(&MI)) {
       // We have a G_STORE. Add it to the candidate if it writes to an adjacent
       // address.
       if (!addStoreToCandidate(*StoreMI, Candidate)) {
diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
index fb5ed35c1f72..391251886fbb 100644
--- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
@@ -215,6 +215,48 @@ MachineInstrBuilder MachineIRBuilder::buildMaskLowPtrBits(const DstOp &Res,
   return buildPtrMask(Res, Op0, MaskReg);
 }
 
+MachineInstrBuilder
+MachineIRBuilder::buildPadVectorWithUndefElements(const DstOp &Res,
+                                                  const SrcOp &Op0) {
+  LLT ResTy = Res.getLLTTy(*getMRI());
+  LLT Op0Ty = Op0.getLLTTy(*getMRI());
+
+  assert((ResTy.isVector() && Op0Ty.isVector()) && "Non vector type");
+  assert((ResTy.getElementType() == Op0Ty.getElementType()) &&
+         "Different vector element types");
+  assert((ResTy.getNumElements() > Op0Ty.getNumElements()) &&
+         "Op0 has more elements");
+
+  auto Unmerge = buildUnmerge(Op0Ty.getElementType(), Op0);
+  SmallVector<Register, 8> Regs;
+  for (auto Op : Unmerge.getInstr()->defs())
+    Regs.push_back(Op.getReg());
+  Register Undef = buildUndef(Op0Ty.getElementType()).getReg(0);
+  unsigned NumberOfPadElts = ResTy.getNumElements() - Regs.size();
+  for (unsigned i = 0; i < NumberOfPadElts; ++i)
+    Regs.push_back(Undef);
+  return buildMerge(Res, Regs);
+}
+
+MachineInstrBuilder
+MachineIRBuilder::buildDeleteTrailingVectorElements(const DstOp &Res,
+                                                    const SrcOp &Op0) {
+  LLT ResTy = Res.getLLTTy(*getMRI());
+  LLT Op0Ty = Op0.getLLTTy(*getMRI());
+
+  assert((ResTy.isVector() && Op0Ty.isVector()) && "Non vector type");
+  assert((ResTy.getElementType() == Op0Ty.getElementType()) &&
+         "Different vector element types");
+  assert((ResTy.getNumElements() < Op0Ty.getNumElements()) &&
+         "Op0 has fewer elements");
+
+  SmallVector<Register, 8> Regs;
+  auto Unmerge = buildUnmerge(Op0Ty.getElementType(), Op0);
+  for (unsigned i = 0; i < ResTy.getNumElements(); ++i)
+    Regs.push_back(Unmerge.getReg(i));
+  return buildMerge(Res, Regs);
+}
+
 MachineInstrBuilder MachineIRBuilder::buildBr(MachineBasicBlock &Dest) {
   return buildInstr(TargetOpcode::G_BR).addMBB(&Dest);
 }
@@ -613,10 +655,8 @@ MachineInstrBuilder MachineIRBuilder::buildUnmerge(ArrayRef<LLT> Res,
 MachineInstrBuilder MachineIRBuilder::buildUnmerge(LLT Res,
                                                    const SrcOp &Op) {
   unsigned NumReg = Op.getLLTTy(*getMRI()).getSizeInBits() / Res.getSizeInBits();
-  SmallVector<Register, 8> TmpVec;
-  for (unsigned I = 0; I != NumReg; ++I)
-    TmpVec.push_back(getMRI()->createGenericVirtualRegister(Res));
-  return buildUnmerge(TmpVec, Op);
+  SmallVector<DstOp, 8> TmpVec(NumReg, Res);
+  return buildInstr(TargetOpcode::G_UNMERGE_VALUES, TmpVec, Op);
 }
 
 MachineInstrBuilder MachineIRBuilder::buildUnmerge(ArrayRef<Register> Res,
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index b0b84763e922..4981a537dc7c 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -923,6 +923,21 @@ LLT llvm::getLCMType(LLT OrigTy, LLT TargetTy) {
   return LLT::scalar(LCMSize);
 }
 
+LLT llvm::getCoverTy(LLT OrigTy, LLT TargetTy) {
+  if (!OrigTy.isVector() || !TargetTy.isVector() || OrigTy == TargetTy ||
+      (OrigTy.getScalarSizeInBits() != TargetTy.getScalarSizeInBits()))
+    return getLCMType(OrigTy, TargetTy);
+
+  unsigned OrigTyNumElts = OrigTy.getNumElements();
+  unsigned TargetTyNumElts = TargetTy.getNumElements();
+  if (OrigTyNumElts % TargetTyNumElts == 0)
+    return OrigTy;
+
+  unsigned NumElts = alignTo(OrigTyNumElts, TargetTyNumElts);
+  return LLT::scalarOrVector(ElementCount::getFixed(NumElts),
+                             OrigTy.getElementType());
+}
+
 LLT llvm::getGCDType(LLT OrigTy, LLT TargetTy) {
   const unsigned OrigSize = OrigTy.getSizeInBits();
   const unsigned TargetSize = TargetTy.getSizeInBits();
@@ -1184,25 +1199,6 @@ bool llvm::shouldOptForSize(const MachineBasicBlock &MBB,
          llvm::shouldOptimizeForSize(MBB.getBasicBlock(), PSI, BFI);
 }
 
-/// These artifacts generally don't have any debug users because they don't
-/// directly originate from IR instructions, but instead usually from
-/// legalization. Avoiding checking for debug users improves compile time.
-/// Note that truncates or extends aren't included because they have IR
-/// counterparts which can have debug users after translation.
-static bool shouldSkipDbgValueFor(MachineInstr &MI) {
-  switch (MI.getOpcode()) {
-  case TargetOpcode::G_UNMERGE_VALUES:
-  case TargetOpcode::G_MERGE_VALUES:
-  case TargetOpcode::G_CONCAT_VECTORS:
-  case TargetOpcode::G_BUILD_VECTOR:
-  case TargetOpcode::G_EXTRACT:
-  case TargetOpcode::G_INSERT:
-    return true;
-  default:
-    return false;
-  }
-}
-
 void llvm::saveUsesAndErase(MachineInstr &MI, MachineRegisterInfo &MRI,
                             LostDebugLocObserver *LocObserver,
                             SmallInstListTy &DeadInstChain) {
@@ -1212,10 +1208,7 @@ void llvm::saveUsesAndErase(MachineInstr &MI, MachineRegisterInfo &MRI,
   }
   LLVM_DEBUG(dbgs() << MI << "Is dead; erasing.\n");
   DeadInstChain.remove(&MI);
-  if (shouldSkipDbgValueFor(MI))
-    MI.eraseFromParent();
-  else
-    MI.eraseFromParentAndMarkDBGValuesForRemoval();
+  MI.eraseFromParent();
   if (LocObserver)
     LocObserver->checkpoint(false);
 }
diff --git a/llvm/lib/CodeGen/ImplicitNullChecks.cpp b/llvm/lib/CodeGen/ImplicitNullChecks.cpp
index 0882ce366c9c..fc97938ccd3e 100644
--- a/llvm/lib/CodeGen/ImplicitNullChecks.cpp
+++ b/llvm/lib/CodeGen/ImplicitNullChecks.cpp
@@ -242,7 +242,7 @@ bool ImplicitNullChecks::canHandle(const MachineInstr *MI) {
   auto IsRegMask = [](const MachineOperand &MO) { return MO.isRegMask(); };
   (void)IsRegMask;
 
-  assert(!llvm::any_of(MI->operands(), IsRegMask) &&
+  assert(llvm::none_of(MI->operands(), IsRegMask) &&
          "Calls were filtered out above!");
 
   auto IsUnordered = [](MachineMemOperand *MMO) { return MMO->isUnordered(); };
diff --git a/llvm/lib/CodeGen/InlineSpiller.cpp b/llvm/lib/CodeGen/InlineSpiller.cpp
index fc5ac45752ca..c975013db8c8 100644
--- a/llvm/lib/CodeGen/InlineSpiller.cpp
+++ b/llvm/lib/CodeGen/InlineSpiller.cpp
@@ -686,9 +686,7 @@ void InlineSpiller::reMaterializeAll() {
   // Remove any values that were completely rematted.
   for (Register Reg : RegsToSpill) {
     LiveInterval &LI = LIS.getInterval(Reg);
-    for (LiveInterval::vni_iterator I = LI.vni_begin(), E = LI.vni_end();
-         I != E; ++I) {
-      VNInfo *VNI = *I;
+    for (VNInfo *VNI : llvm::make_range(LI.vni_begin(), LI.vni_end())) {
       if (VNI->isUnused() || VNI->isPHIDef() || UsedValues.count(VNI))
         continue;
       MachineInstr *MI = LIS.getInstructionFromIndex(VNI->def);
diff --git a/llvm/lib/CodeGen/InterferenceCache.cpp b/llvm/lib/CodeGen/InterferenceCache.cpp
index a56485cdbc67..3cab9e5734ee 100644
--- a/llvm/lib/CodeGen/InterferenceCache.cpp
+++ b/llvm/lib/CodeGen/InterferenceCache.cpp
@@ -56,8 +56,8 @@ void InterferenceCache::init(MachineFunction *mf,
   LIUArray = liuarray;
   TRI = tri;
   reinitPhysRegEntries();
-  for (unsigned i = 0; i != CacheEntries; ++i)
-    Entries[i].clear(mf, indexes, lis);
+  for (Entry &E : Entries)
+    E.clear(mf, indexes, lis);
 }
 
 InterferenceCache::Entry *InterferenceCache::get(MCRegister PhysReg) {
diff --git a/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp b/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp
index cf62b0e5d7e8..e97dcca201e8 100644
--- a/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp
+++ b/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp
@@ -1249,8 +1249,8 @@ bool InstrRefBasedLDV::transferDebugPHI(MachineInstr &MI) {
     std::array<unsigned, 4> CandidateSizes = {64, 32, 16, 8};
     Optional<ValueIDNum> Result = None;
     Optional<LocIdx> SpillLoc = None;
-    for (unsigned int I = 0; I < CandidateSizes.size(); ++I) {
-      unsigned SpillID = MTracker->getLocID(SpillNo, {CandidateSizes[I], 0});
+    for (unsigned CS : CandidateSizes) {
+      unsigned SpillID = MTracker->getLocID(SpillNo, {CS, 0});
       SpillLoc = MTracker->getSpillMLoc(SpillID);
       ValueIDNum Val = MTracker->readMLoc(*SpillLoc);
       // If this value was defined in it's own position, then it was probably
diff --git a/llvm/lib/CodeGen/LiveDebugValues/VarLocBasedImpl.cpp b/llvm/lib/CodeGen/LiveDebugValues/VarLocBasedImpl.cpp
index a632d3d9ce76..b4dd41bbb810 100644
--- a/llvm/lib/CodeGen/LiveDebugValues/VarLocBasedImpl.cpp
+++ b/llvm/lib/CodeGen/LiveDebugValues/VarLocBasedImpl.cpp
@@ -492,10 +492,10 @@ private:
     static VarLoc CreateCopyLoc(const VarLoc &OldVL, const MachineLoc &OldML,
                                 Register NewReg) {
       VarLoc VL = OldVL;
-      for (size_t I = 0, E = VL.Locs.size(); I < E; ++I)
-        if (VL.Locs[I] == OldML) {
-          VL.Locs[I].Kind = MachineLocKind::RegisterKind;
-          VL.Locs[I].Value.RegNo = NewReg;
+      for (MachineLoc &ML : VL.Locs)
+        if (ML == OldML) {
+          ML.Kind = MachineLocKind::RegisterKind;
+          ML.Value.RegNo = NewReg;
           return VL;
         }
       llvm_unreachable("Should have found OldML in new VarLoc.");
@@ -506,10 +506,10 @@ private:
     static VarLoc CreateSpillLoc(const VarLoc &OldVL, const MachineLoc &OldML,
                                  unsigned SpillBase, StackOffset SpillOffset) {
       VarLoc VL = OldVL;
-      for (int I = 0, E = VL.Locs.size(); I < E; ++I)
-        if (VL.Locs[I] == OldML) {
-          VL.Locs[I].Kind = MachineLocKind::SpillLocKind;
-          VL.Locs[I].Value.SpillLocation = {SpillBase, SpillOffset};
+      for (MachineLoc &ML : VL.Locs)
+        if (ML == OldML) {
+          ML.Kind = MachineLocKind::SpillLocKind;
+          ML.Value.SpillLocation = {SpillBase, SpillOffset};
           return VL;
         }
       llvm_unreachable("Should have found OldML in new VarLoc.");
diff --git a/llvm/lib/CodeGen/LiveDebugVariables.cpp b/llvm/lib/CodeGen/LiveDebugVariables.cpp
index 5f976bf43c5b..e6661e5135c3 100644
--- a/llvm/lib/CodeGen/LiveDebugVariables.cpp
+++ b/llvm/lib/CodeGen/LiveDebugVariables.cpp
@@ -822,9 +822,6 @@ bool LDVImpl::handleDebugValue(MachineInstr &MI, SlotIndex Idx) {
   // register that hasn't been defined yet. If we do not remove those here, then
   // the re-insertion of the DBG_VALUE instruction after register allocation
   // will be incorrect.
-  // TODO: If earlier passes are corrected to generate sane debug information
-  // (and if the machine verifier is improved to catch this), then these checks
-  // could be removed or replaced by asserts.
   bool Discard = false;
   for (const MachineOperand &Op : MI.debug_operands()) {
     if (Op.isReg() && Register::isVirtualRegister(Op.getReg())) {
@@ -1341,8 +1338,8 @@ UserValue::splitLocation(unsigned OldLocNo, ArrayRef<Register> NewRegs,
   bool DidChange = false;
   LocMap::iterator LocMapI;
   LocMapI.setMap(locInts);
-  for (unsigned i = 0; i != NewRegs.size(); ++i) {
-    LiveInterval *LI = &LIS.getInterval(NewRegs[i]);
+  for (Register NewReg : NewRegs) {
+    LiveInterval *LI = &LIS.getInterval(NewReg);
     if (LI->empty())
       continue;
 
@@ -1500,8 +1497,8 @@ void LDVImpl::splitRegister(Register OldReg, ArrayRef<Register> NewRegs) {
 
   // Map all of the new virtual registers.
   UserValue *UV = lookupVirtReg(OldReg);
-  for (unsigned i = 0; i != NewRegs.size(); ++i)
-    mapVirtReg(NewRegs[i], UV);
+  for (Register NewReg : NewRegs)
+    mapVirtReg(NewReg, UV);
 }
 
 void LiveDebugVariables::
diff --git a/llvm/lib/CodeGen/LiveDebugVariables.h b/llvm/lib/CodeGen/LiveDebugVariables.h
index 07dd3a83866f..9998ce9e8dad 100644
--- a/llvm/lib/CodeGen/LiveDebugVariables.h
+++ b/llvm/lib/CodeGen/LiveDebugVariables.h
@@ -56,6 +56,11 @@ private:
   bool runOnMachineFunction(MachineFunction &) override;
   void releaseMemory() override;
   void getAnalysisUsage(AnalysisUsage &) const override;
+
+  MachineFunctionProperties getSetProperties() const override {
+    return MachineFunctionProperties().set(
+        MachineFunctionProperties::Property::TracksDebugUserValues);
+  }
 };
 
 } // end namespace llvm
diff --git a/llvm/lib/CodeGen/LiveRangeEdit.cpp b/llvm/lib/CodeGen/LiveRangeEdit.cpp
index 6380c4bfd6e6..05768140cbdf 100644
--- a/llvm/lib/CodeGen/LiveRangeEdit.cpp
+++ b/llvm/lib/CodeGen/LiveRangeEdit.cpp
@@ -133,6 +133,22 @@ bool LiveRangeEdit::allUsesAvailableAt(const MachineInstr *OrigMI,
 
     if (OVNI != li.getVNInfoAt(UseIdx))
       return false;
+
+    // Check that subrange is live at UseIdx.
+    if (MO.getSubReg()) {
+      const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
+      LaneBitmask LM = TRI->getSubRegIndexLaneMask(MO.getSubReg());
+      for (LiveInterval::SubRange &SR : li.subranges()) {
+        if ((SR.LaneMask & LM).none())
+          continue;
+        if (!SR.liveAt(UseIdx))
+          return false;
+        // Early exit if all used lanes are checked. No need to continue.
+        LM &= ~SR.LaneMask;
+        if (LM.none())
+          break;
+      }
+    }
   }
   return true;
 }
diff --git a/llvm/lib/CodeGen/LiveVariables.cpp b/llvm/lib/CodeGen/LiveVariables.cpp
index e8744797707b..94bdfab5e5e0 100644
--- a/llvm/lib/CodeGen/LiveVariables.cpp
+++ b/llvm/lib/CodeGen/LiveVariables.cpp
@@ -141,8 +141,8 @@ void LiveVariables::HandleVirtRegUse(Register Reg, MachineBasicBlock *MBB,
   }
 
 #ifndef NDEBUG
-  for (unsigned i = 0, e = VRInfo.Kills.size(); i != e; ++i)
-    assert(VRInfo.Kills[i]->getParent() != MBB && "entry should be at end!");
+  for (MachineInstr *Kill : VRInfo.Kills)
+    assert(Kill->getParent() != MBB && "entry should be at end!");
 #endif
 
   // This situation can occur:
@@ -534,8 +534,7 @@ void LiveVariables::runOnInstr(MachineInstr &MI,
 
   MachineBasicBlock *MBB = MI.getParent();
   // Process all uses.
-  for (unsigned i = 0, e = UseRegs.size(); i != e; ++i) {
-    unsigned MOReg = UseRegs[i];
+  for (unsigned MOReg : UseRegs) {
     if (Register::isVirtualRegister(MOReg))
       HandleVirtRegUse(MOReg, MBB, MI);
     else if (!MRI->isReserved(MOReg))
@@ -543,12 +542,11 @@ void LiveVariables::runOnInstr(MachineInstr &MI,
   }
 
   // Process all masked registers. (Call clobbers).
-  for (unsigned i = 0, e = RegMasks.size(); i != e; ++i)
-    HandleRegMask(MI.getOperand(RegMasks[i]));
+  for (unsigned Mask : RegMasks)
+    HandleRegMask(MI.getOperand(Mask));
 
   // Process all defs.
-  for (unsigned i = 0, e = DefRegs.size(); i != e; ++i) {
-    unsigned MOReg = DefRegs[i];
+  for (unsigned MOReg : DefRegs) {
     if (Register::isVirtualRegister(MOReg))
       HandleVirtRegDef(MOReg, MI);
     else if (!MRI->isReserved(MOReg))
diff --git a/llvm/lib/CodeGen/LocalStackSlotAllocation.cpp b/llvm/lib/CodeGen/LocalStackSlotAllocation.cpp
index ee2387d1e8e6..37fd3e4853ac 100644
--- a/llvm/lib/CodeGen/LocalStackSlotAllocation.cpp
+++ b/llvm/lib/CodeGen/LocalStackSlotAllocation.cpp
@@ -210,7 +210,11 @@ void LocalStackSlotPass::calculateFrameObjectOffsets(MachineFunction &Fn) {
     StackObjSet SmallArrayObjs;
     StackObjSet AddrOfObjs;
 
-    AdjustStackOffset(MFI, StackProtectorFI, Offset, StackGrowsDown, MaxAlign);
+    // Only place the stack protector in the local stack area if the target
+    // allows it.
+    if (TFI.isStackIdSafeForLocalArea(MFI.getStackID(StackProtectorFI)))
+      AdjustStackOffset(MFI, StackProtectorFI, Offset, StackGrowsDown,
+                        MaxAlign);
 
     // Assign large stack objects first.
     for (unsigned i = 0, e = MFI.getObjectIndexEnd(); i != e; ++i) {
diff --git a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
index 6221b5929301..d0323eaf3d78 100644
--- a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
@@ -350,18 +350,33 @@ void MIRParserImpl::computeFunctionProperties(MachineFunction &MF) {
 
   bool HasPHI = false;
   bool HasInlineAsm = false;
+  bool AllTiedOpsRewritten = true, HasTiedOps = false;
   for (const MachineBasicBlock &MBB : MF) {
     for (const MachineInstr &MI : MBB) {
       if (MI.isPHI())
         HasPHI = true;
       if (MI.isInlineAsm())
         HasInlineAsm = true;
+      for (unsigned I = 0; I < MI.getNumOperands(); ++I) {
+        const MachineOperand &MO = MI.getOperand(I);
+        if (!MO.isReg() || !MO.getReg())
+          continue;
+        unsigned DefIdx;
+        if (MO.isUse() && MI.isRegTiedToDefOperand(I, &DefIdx)) {
+          HasTiedOps = true;
+          if (MO.getReg() != MI.getOperand(DefIdx).getReg())
+            AllTiedOpsRewritten = false;
+        }
+      }
     }
   }
   if (!HasPHI)
     Properties.set(MachineFunctionProperties::Property::NoPHIs);
   MF.setHasInlineAsm(HasInlineAsm);
 
+  if (HasTiedOps && AllTiedOpsRewritten)
+    Properties.set(MachineFunctionProperties::Property::TiedOpsRewritten);
+
   if (isSSA(MF))
     Properties.set(MachineFunctionProperties::Property::IsSSA);
   else
@@ -457,6 +472,9 @@ MIRParserImpl::initializeMachineFunction(const yaml::MachineFunction &YamlMF,
   if (YamlMF.FailsVerification)
     MF.getProperties().set(
         MachineFunctionProperties::Property::FailsVerification);
+  if (YamlMF.TracksDebugUserValues)
+    MF.getProperties().set(
+        MachineFunctionProperties::Property::TracksDebugUserValues);
 
   PerFunctionMIParsingState PFS(MF, SM, IRSlots, *Target);
   if (parseRegisterInfo(PFS, YamlMF))
diff --git a/llvm/lib/CodeGen/MIRPrinter.cpp b/llvm/lib/CodeGen/MIRPrinter.cpp
index f1369396e37f..dc72f83ad0e4 100644
--- a/llvm/lib/CodeGen/MIRPrinter.cpp
+++ b/llvm/lib/CodeGen/MIRPrinter.cpp
@@ -219,6 +219,8 @@ void MIRPrinter::print(const MachineFunction &MF) {
       MachineFunctionProperties::Property::FailedISel);
   YamlMF.FailsVerification = MF.getProperties().hasProperty(
       MachineFunctionProperties::Property::FailsVerification);
+  YamlMF.TracksDebugUserValues = MF.getProperties().hasProperty(
+      MachineFunctionProperties::Property::TracksDebugUserValues);
 
   convert(YamlMF, MF.getRegInfo(), MF.getSubtarget().getRegisterInfo());
   MachineModuleSlotTracker MST(&MF);
diff --git a/llvm/lib/CodeGen/MachineBasicBlock.cpp b/llvm/lib/CodeGen/MachineBasicBlock.cpp
index 23c511aaa056..8c9d00d08c6a 100644
--- a/llvm/lib/CodeGen/MachineBasicBlock.cpp
+++ b/llvm/lib/CodeGen/MachineBasicBlock.cpp
@@ -193,7 +193,7 @@ void ilist_traits<MachineInstr>::transferNodesFromList(ilist_traits &FromList,
 
 void ilist_traits<MachineInstr>::deleteNode(MachineInstr *MI) {
   assert(!MI->getParent() && "MI is still in a block!");
-  Parent->getParent()->DeleteMachineInstr(MI);
+  Parent->getParent()->deleteMachineInstr(MI);
 }
 
 MachineBasicBlock::iterator MachineBasicBlock::getFirstNonPHI() {
@@ -1038,16 +1038,15 @@ MachineBasicBlock *MachineBasicBlock::SplitCriticalEdge(
   // Collect a list of virtual registers killed by the terminators.
   SmallVector<Register, 4> KilledRegs;
   if (LV)
-    for (instr_iterator I = getFirstInstrTerminator(), E = instr_end();
-         I != E; ++I) {
-      MachineInstr *MI = &*I;
-      for (MachineOperand &MO : MI->operands()) {
+    for (MachineInstr &MI :
+         llvm::make_range(getFirstInstrTerminator(), instr_end())) {
+      for (MachineOperand &MO : MI.operands()) {
         if (!MO.isReg() || MO.getReg() == 0 || !MO.isUse() || !MO.isKill() ||
             MO.isUndef())
           continue;
         Register Reg = MO.getReg();
         if (Register::isPhysicalRegister(Reg) ||
-            LV->getVarInfo(Reg).removeKill(*MI)) {
+            LV->getVarInfo(Reg).removeKill(MI)) {
           KilledRegs.push_back(Reg);
           LLVM_DEBUG(dbgs() << "Removing terminator kill: " << MI);
           MO.setIsKill(false);
@@ -1057,11 +1056,9 @@ MachineBasicBlock *MachineBasicBlock::SplitCriticalEdge(
 
   SmallVector<Register, 4> UsedRegs;
   if (LIS) {
-    for (instr_iterator I = getFirstInstrTerminator(), E = instr_end();
-         I != E; ++I) {
-      MachineInstr *MI = &*I;
-
-      for (const MachineOperand &MO : MI->operands()) {
+    for (MachineInstr &MI :
+         llvm::make_range(getFirstInstrTerminator(), instr_end())) {
+      for (const MachineOperand &MO : MI.operands()) {
         if (!MO.isReg() || MO.getReg() == 0)
           continue;
 
@@ -1078,9 +1075,9 @@ MachineBasicBlock *MachineBasicBlock::SplitCriticalEdge(
   // SlotIndexes.
   SmallVector<MachineInstr*, 4> Terminators;
   if (Indexes) {
-    for (instr_iterator I = getFirstInstrTerminator(), E = instr_end();
-         I != E; ++I)
-      Terminators.push_back(&*I);
+    for (MachineInstr &MI :
+         llvm::make_range(getFirstInstrTerminator(), instr_end()))
+      Terminators.push_back(&MI);
   }
 
   // Since we replaced all uses of Succ with NMBB, that should also be treated
@@ -1091,9 +1088,9 @@ MachineBasicBlock *MachineBasicBlock::SplitCriticalEdge(
 
   if (Indexes) {
     SmallVector<MachineInstr*, 4> NewTerminators;
-    for (instr_iterator I = getFirstInstrTerminator(), E = instr_end();
-         I != E; ++I)
-      NewTerminators.push_back(&*I);
+    for (MachineInstr &MI :
+         llvm::make_range(getFirstInstrTerminator(), instr_end()))
+      NewTerminators.push_back(&MI);
 
     for (MachineInstr *Terminator : Terminators) {
       if (!is_contained(NewTerminators, Terminator))
diff --git a/llvm/lib/CodeGen/MachineBlockPlacement.cpp b/llvm/lib/CodeGen/MachineBlockPlacement.cpp
index 8a1b4031642d..692587cd58fa 100644
--- a/llvm/lib/CodeGen/MachineBlockPlacement.cpp
+++ b/llvm/lib/CodeGen/MachineBlockPlacement.cpp
@@ -61,6 +61,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetMachine.h"
+#include "llvm/Transforms/Utils/CodeLayout.h"
 #include <algorithm>
 #include <cassert>
 #include <cstdint>
@@ -193,6 +194,11 @@ static cl::opt<unsigned> TriangleChainCount(
     cl::init(2),
     cl::Hidden);
 
+static cl::opt<bool> EnableExtTspBlockPlacement(
+    "enable-ext-tsp-block-placement", cl::Hidden, cl::init(false),
+    cl::desc("Enable machine block placement based on the ext-tsp model, "
+             "optimizing I-cache utilization."));
+
 namespace llvm {
 extern cl::opt<unsigned> StaticLikelyProb;
 extern cl::opt<unsigned> ProfileLikelyProb;
@@ -557,6 +563,15 @@ class MachineBlockPlacement : public MachineFunctionPass {
   /// but a local analysis would not find them.
   void precomputeTriangleChains();
 
+  /// Apply a post-processing step optimizing block placement.
+  void applyExtTsp();
+
+  /// Modify the existing block placement in the function and adjust all jumps.
+  void assignBlockOrder(const std::vector<const MachineBasicBlock *> &NewOrder);
+
+  /// Create a single CFG chain from the current block order.
+  void createCFGChainExtTsp();
+
 public:
   static char ID; // Pass identification, replacement for typeid
 
@@ -3387,6 +3402,15 @@ bool MachineBlockPlacement::runOnMachineFunction(MachineFunction &MF) {
     }
   }
 
+  // Apply a post-processing optimizing block placement.
+  if (MF.size() >= 3 && EnableExtTspBlockPlacement) {
+    // Find a new placement and modify the layout of the blocks in the function.
+    applyExtTsp();
+
+    // Re-create CFG chain so that we can optimizeBranches and alignBlocks.
+    createCFGChainExtTsp();
+  }
+
   optimizeBranches();
   alignBlocks();
 
@@ -3413,12 +3437,147 @@ bool MachineBlockPlacement::runOnMachineFunction(MachineFunction &MF) {
     MBFI->view("MBP." + MF.getName(), false);
   }
 
-
   // We always return true as we have no way to track whether the final order
   // differs from the original order.
   return true;
 }
 
+void MachineBlockPlacement::applyExtTsp() {
+  // Prepare data; blocks are indexed by their index in the current ordering.
+  DenseMap<const MachineBasicBlock *, uint64_t> BlockIndex;
+  BlockIndex.reserve(F->size());
+  std::vector<const MachineBasicBlock *> CurrentBlockOrder;
+  CurrentBlockOrder.reserve(F->size());
+  size_t NumBlocks = 0;
+  for (const MachineBasicBlock &MBB : *F) {
+    BlockIndex[&MBB] = NumBlocks++;
+    CurrentBlockOrder.push_back(&MBB);
+  }
+
+  auto BlockSizes = std::vector<uint64_t>(F->size());
+  auto BlockCounts = std::vector<uint64_t>(F->size());
+  DenseMap<std::pair<uint64_t, uint64_t>, uint64_t> JumpCounts;
+  for (MachineBasicBlock &MBB : *F) {
+    // Getting the block frequency.
+    BlockFrequency BlockFreq = MBFI->getBlockFreq(&MBB);
+    BlockCounts[BlockIndex[&MBB]] = BlockFreq.getFrequency();
+    // Getting the block size:
+    // - approximate the size of an instruction by 4 bytes, and
+    // - ignore debug instructions.
+    // Note: getting the exact size of each block is target-dependent and can be
+    // done by extending the interface of MCCodeEmitter. Experimentally we do
+    // not see a perf improvement with the exact block sizes.
+    auto NonDbgInsts =
+        instructionsWithoutDebug(MBB.instr_begin(), MBB.instr_end());
+    int NumInsts = std::distance(NonDbgInsts.begin(), NonDbgInsts.end());
+    BlockSizes[BlockIndex[&MBB]] = 4 * NumInsts;
+    // Getting jump frequencies.
+    for (MachineBasicBlock *Succ : MBB.successors()) {
+      auto EP = MBPI->getEdgeProbability(&MBB, Succ);
+      BlockFrequency EdgeFreq = BlockFreq * EP;
+      auto Edge = std::make_pair(BlockIndex[&MBB], BlockIndex[Succ]);
+      JumpCounts[Edge] = EdgeFreq.getFrequency();
+    }
+  }
+
+  LLVM_DEBUG(dbgs() << "Applying ext-tsp layout for |V| = " << F->size()
+                    << " with profile = " << F->getFunction().hasProfileData()
+                    << " (" << F->getName().str() << ")"
+                    << "\n");
+  LLVM_DEBUG(
+      dbgs() << format("  original  layout score: %0.2f\n",
+                       calcExtTspScore(BlockSizes, BlockCounts, JumpCounts)));
+
+  // Run the layout algorithm.
+  auto NewOrder = applyExtTspLayout(BlockSizes, BlockCounts, JumpCounts);
+  std::vector<const MachineBasicBlock *> NewBlockOrder;
+  NewBlockOrder.reserve(F->size());
+  for (uint64_t Node : NewOrder) {
+    NewBlockOrder.push_back(CurrentBlockOrder[Node]);
+  }
+  LLVM_DEBUG(dbgs() << format("  optimized layout score: %0.2f\n",
+                              calcExtTspScore(NewOrder, BlockSizes, BlockCounts,
+                                              JumpCounts)));
+
+  // Assign new block order.
+  assignBlockOrder(NewBlockOrder);
+}
+
+void MachineBlockPlacement::assignBlockOrder(
+    const std::vector<const MachineBasicBlock *> &NewBlockOrder) {
+  assert(F->size() == NewBlockOrder.size() && "Incorrect size of block order");
+  F->RenumberBlocks();
+
+  bool HasChanges = false;
+  for (size_t I = 0; I < NewBlockOrder.size(); I++) {
+    if (NewBlockOrder[I] != F->getBlockNumbered(I)) {
+      HasChanges = true;
+      break;
+    }
+  }
+  // Stop early if the new block order is identical to the existing one.
+  if (!HasChanges)
+    return;
+
+  SmallVector<MachineBasicBlock *, 4> PrevFallThroughs(F->getNumBlockIDs());
+  for (auto &MBB : *F) {
+    PrevFallThroughs[MBB.getNumber()] = MBB.getFallThrough();
+  }
+
+  // Sort basic blocks in the function according to the computed order.
+  DenseMap<const MachineBasicBlock *, size_t> NewIndex;
+  for (const MachineBasicBlock *MBB : NewBlockOrder) {
+    NewIndex[MBB] = NewIndex.size();
+  }
+  F->sort([&](MachineBasicBlock &L, MachineBasicBlock &R) {
+    return NewIndex[&L] < NewIndex[&R];
+  });
+
+  // Update basic block branches by inserting explicit fallthrough branches
+  // when required and re-optimize branches when possible.
+  const TargetInstrInfo *TII = F->getSubtarget().getInstrInfo();
+  SmallVector<MachineOperand, 4> Cond;
+  for (auto &MBB : *F) {
+    MachineFunction::iterator NextMBB = std::next(MBB.getIterator());
+    MachineFunction::iterator EndIt = MBB.getParent()->end();
+    auto *FTMBB = PrevFallThroughs[MBB.getNumber()];
+    // If this block had a fallthrough before we need an explicit unconditional
+    // branch to that block if the fallthrough block is not adjacent to the
+    // block in the new order.
+    if (FTMBB && (NextMBB == EndIt || &*NextMBB != FTMBB)) {
+      TII->insertUnconditionalBranch(MBB, FTMBB, MBB.findBranchDebugLoc());
+    }
+
+    // It might be possible to optimize branches by flipping the condition.
+    Cond.clear();
+    MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
+    if (TII->analyzeBranch(MBB, TBB, FBB, Cond))
+      continue;
+    MBB.updateTerminator(FTMBB);
+  }
+
+#ifndef NDEBUG
+  // Make sure we correctly constructed all branches.
+  F->verify(this, "After optimized block reordering");
+#endif
+}
+
+void MachineBlockPlacement::createCFGChainExtTsp() {
+  BlockToChain.clear();
+  ComputedEdges.clear();
+  ChainAllocator.DestroyAll();
+
+  MachineBasicBlock *HeadBB = &F->front();
+  BlockChain *FunctionChain =
+      new (ChainAllocator.Allocate()) BlockChain(BlockToChain, HeadBB);
+
+  for (MachineBasicBlock &MBB : *F) {
+    if (HeadBB == &MBB)
+      continue; // Ignore head of the chain
+    FunctionChain->merge(&MBB, nullptr);
+  }
+}
+
 namespace {
 
 /// A pass to compute block placement statistics.
diff --git a/llvm/lib/CodeGen/MachineCombiner.cpp b/llvm/lib/CodeGen/MachineCombiner.cpp
index e2b6cfe55c16..72ab9ee4f388 100644
--- a/llvm/lib/CodeGen/MachineCombiner.cpp
+++ b/llvm/lib/CodeGen/MachineCombiner.cpp
@@ -485,7 +485,7 @@ static void insertDeleteInstructions(MachineBasicBlock *MBB, MachineInstr &MI,
     MBB->insert((MachineBasicBlock::iterator)&MI, InstrPtr);
 
   for (auto *InstrPtr : DelInstrs) {
-    InstrPtr->eraseFromParentAndMarkDBGValuesForRemoval();
+    InstrPtr->eraseFromParent();
     // Erase all LiveRegs defined by the removed instruction
     for (auto I = RegUnits.begin(); I != RegUnits.end(); ) {
       if (I->MI == InstrPtr)
@@ -693,7 +693,7 @@ bool MachineCombiner::combineInstructions(MachineBasicBlock *MBB) {
         // use for them.
         MachineFunction *MF = MBB->getParent();
         for (auto *InstrPtr : InsInstrs)
-          MF->DeleteMachineInstr(InstrPtr);
+          MF->deleteMachineInstr(InstrPtr);
       }
       InstrIdxForVirtReg.clear();
     }
diff --git a/llvm/lib/CodeGen/MachineCopyPropagation.cpp b/llvm/lib/CodeGen/MachineCopyPropagation.cpp
index 7c83bacd80d9..57fbe4112e47 100644
--- a/llvm/lib/CodeGen/MachineCopyPropagation.cpp
+++ b/llvm/lib/CodeGen/MachineCopyPropagation.cpp
@@ -847,31 +847,27 @@ void MachineCopyPropagation::BackwardCopyPropagateBlock(
   LLVM_DEBUG(dbgs() << "MCP: BackwardCopyPropagateBlock " << MBB.getName()
                     << "\n");
 
-  for (MachineBasicBlock::reverse_iterator I = MBB.rbegin(), E = MBB.rend();
-       I != E;) {
-    MachineInstr *MI = &*I;
-    ++I;
-
+  for (MachineInstr &MI : llvm::make_early_inc_range(llvm::reverse(MBB))) {
     // Ignore non-trivial COPYs.
-    if (MI->isCopy() && MI->getNumOperands() == 2 &&
-        !TRI->regsOverlap(MI->getOperand(0).getReg(),
-                          MI->getOperand(1).getReg())) {
+    if (MI.isCopy() && MI.getNumOperands() == 2 &&
+        !TRI->regsOverlap(MI.getOperand(0).getReg(),
+                          MI.getOperand(1).getReg())) {
 
-      MCRegister Def = MI->getOperand(0).getReg().asMCReg();
-      MCRegister Src = MI->getOperand(1).getReg().asMCReg();
+      MCRegister Def = MI.getOperand(0).getReg().asMCReg();
+      MCRegister Src = MI.getOperand(1).getReg().asMCReg();
 
       // Unlike forward cp, we don't invoke propagateDefs here,
       // just let forward cp do COPY-to-COPY propagation.
-      if (isBackwardPropagatableCopy(*MI, *MRI)) {
+      if (isBackwardPropagatableCopy(MI, *MRI)) {
         Tracker.invalidateRegister(Src, *TRI);
         Tracker.invalidateRegister(Def, *TRI);
-        Tracker.trackCopy(MI, *TRI);
+        Tracker.trackCopy(&MI, *TRI);
         continue;
       }
     }
 
     // Invalidate any earlyclobber regs first.
-    for (const MachineOperand &MO : MI->operands())
+    for (const MachineOperand &MO : MI.operands())
       if (MO.isReg() && MO.isEarlyClobber()) {
         MCRegister Reg = MO.getReg().asMCReg();
         if (!Reg)
@@ -879,8 +875,8 @@ void MachineCopyPropagation::BackwardCopyPropagateBlock(
         Tracker.invalidateRegister(Reg, *TRI);
       }
 
-    propagateDefs(*MI);
-    for (const MachineOperand &MO : MI->operands()) {
+    propagateDefs(MI);
+    for (const MachineOperand &MO : MI.operands()) {
       if (!MO.isReg())
         continue;
 
@@ -898,7 +894,7 @@ void MachineCopyPropagation::BackwardCopyPropagateBlock(
           for (MCRegUnitIterator RUI(MO.getReg().asMCReg(), TRI); RUI.isValid();
                ++RUI) {
             if (auto *Copy = Tracker.findCopyDefViaUnit(*RUI, *TRI)) {
-              CopyDbgUsers[Copy].insert(MI);
+              CopyDbgUsers[Copy].insert(&MI);
             }
           }
         } else {
diff --git a/llvm/lib/CodeGen/MachineCycleAnalysis.cpp b/llvm/lib/CodeGen/MachineCycleAnalysis.cpp
new file mode 100644
index 000000000000..42a5e2b7af01
--- /dev/null
+++ b/llvm/lib/CodeGen/MachineCycleAnalysis.cpp
@@ -0,0 +1,113 @@
+//===- MachineCycleAnalysis.cpp - Compute CycleInfo for Machine IR --------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/MachineCycleAnalysis.h"
+#include "llvm/ADT/GenericCycleImpl.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineSSAContext.h"
+#include "llvm/InitializePasses.h"
+
+using namespace llvm;
+
+template class llvm::GenericCycleInfo<llvm::MachineSSAContext>;
+template class llvm::GenericCycle<llvm::MachineSSAContext>;
+
+namespace {
+
+/// Legacy analysis pass which computes a \ref MachineCycleInfo.
+class MachineCycleInfoWrapperPass : public MachineFunctionPass {
+  MachineFunction *F = nullptr;
+  MachineCycleInfo CI;
+
+public:
+  static char ID;
+
+  MachineCycleInfoWrapperPass();
+
+  MachineCycleInfo &getCycleInfo() { return CI; }
+  const MachineCycleInfo &getCycleInfo() const { return CI; }
+
+  bool runOnMachineFunction(MachineFunction &F) override;
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
+  void releaseMemory() override;
+  void print(raw_ostream &OS, const Module *M = nullptr) const override;
+
+  // TODO: verify analysis
+};
+
+class MachineCycleInfoPrinterPass : public MachineFunctionPass {
+public:
+  static char ID;
+
+  MachineCycleInfoPrinterPass();
+
+  bool runOnMachineFunction(MachineFunction &F) override;
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
+};
+
+} // namespace
+
+char MachineCycleInfoWrapperPass::ID = 0;
+
+MachineCycleInfoWrapperPass::MachineCycleInfoWrapperPass()
+    : MachineFunctionPass(ID) {
+  initializeMachineCycleInfoWrapperPassPass(*PassRegistry::getPassRegistry());
+}
+
+INITIALIZE_PASS_BEGIN(MachineCycleInfoWrapperPass, "machine-cycles",
+                      "Machine Cycle Info Analysis", true, true)
+INITIALIZE_PASS_END(MachineCycleInfoWrapperPass, "machine-cycles",
+                    "Machine Cycle Info Analysis", true, true)
+
+void MachineCycleInfoWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const {
+  AU.setPreservesAll();
+  MachineFunctionPass::getAnalysisUsage(AU);
+}
+
+bool MachineCycleInfoWrapperPass::runOnMachineFunction(MachineFunction &Func) {
+  CI.clear();
+
+  F = &Func;
+  CI.compute(Func);
+  return false;
+}
+
+void MachineCycleInfoWrapperPass::print(raw_ostream &OS, const Module *) const {
+  OS << "MachineCycleInfo for function: " << F->getName() << "\n";
+  CI.print(OS);
+}
+
+void MachineCycleInfoWrapperPass::releaseMemory() {
+  CI.clear();
+  F = nullptr;
+}
+
+char MachineCycleInfoPrinterPass::ID = 0;
+
+MachineCycleInfoPrinterPass::MachineCycleInfoPrinterPass()
+    : MachineFunctionPass(ID) {
+  initializeMachineCycleInfoPrinterPassPass(*PassRegistry::getPassRegistry());
+}
+
+INITIALIZE_PASS_BEGIN(MachineCycleInfoPrinterPass, "print-machine-cycles",
+                      "Print Machine Cycle Info Analysis", true, true)
+INITIALIZE_PASS_DEPENDENCY(MachineCycleInfoWrapperPass)
+INITIALIZE_PASS_END(MachineCycleInfoPrinterPass, "print-machine-cycles",
+                    "Print Machine Cycle Info Analysis", true, true)
+
+void MachineCycleInfoPrinterPass::getAnalysisUsage(AnalysisUsage &AU) const {
+  AU.setPreservesAll();
+  AU.addRequired<MachineCycleInfoWrapperPass>();
+  MachineFunctionPass::getAnalysisUsage(AU);
+}
+
+bool MachineCycleInfoPrinterPass::runOnMachineFunction(MachineFunction &F) {
+  auto &CI = getAnalysis<MachineCycleInfoWrapperPass>();
+  CI.print(errs());
+  return false;
+}
diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp
index 310c2721c3bd..81ed3d0e93ff 100644
--- a/llvm/lib/CodeGen/MachineFunction.cpp
+++ b/llvm/lib/CodeGen/MachineFunction.cpp
@@ -89,6 +89,7 @@ static cl::opt<unsigned> AlignAllFunctions(
 static const char *getPropertyName(MachineFunctionProperties::Property Prop) {
   using P = MachineFunctionProperties::Property;
 
+  // clang-format off
   switch(Prop) {
   case P::FailedISel: return "FailedISel";
   case P::IsSSA: return "IsSSA";
@@ -100,7 +101,9 @@ static const char *getPropertyName(MachineFunctionProperties::Property Prop) {
   case P::TracksLiveness: return "TracksLiveness";
   case P::TiedOpsRewritten: return "TiedOpsRewritten";
   case P::FailsVerification: return "FailsVerification";
+  case P::TracksDebugUserValues: return "TracksDebugUserValues";
   }
+  // clang-format on
   llvm_unreachable("Invalid machine function property");
 }
 
@@ -125,7 +128,7 @@ void MachineFunctionProperties::print(raw_ostream &OS) const {
 MachineFunctionInfo::~MachineFunctionInfo() = default;
 
 void ilist_alloc_traits<MachineBasicBlock>::deleteNode(MachineBasicBlock *MBB) {
-  MBB->getParent()->DeleteMachineBasicBlock(MBB);
+  MBB->getParent()->deleteMachineBasicBlock(MBB);
 }
 
 static inline unsigned getFnStackAlignment(const TargetSubtargetInfo *STI,
@@ -347,10 +350,10 @@ void MachineFunction::assignBeginEndSections() {
 
 /// Allocate a new MachineInstr. Use this instead of `new MachineInstr'.
 MachineInstr *MachineFunction::CreateMachineInstr(const MCInstrDesc &MCID,
-                                                  const DebugLoc &DL,
+                                                  DebugLoc DL,
                                                   bool NoImplicit) {
   return new (InstructionRecycler.Allocate<MachineInstr>(Allocator))
-      MachineInstr(*this, MCID, DL, NoImplicit);
+      MachineInstr(*this, MCID, std::move(DL), NoImplicit);
 }
 
 /// Create a new MachineInstr which is a copy of the 'Orig' instruction,
@@ -361,8 +364,9 @@ MachineFunction::CloneMachineInstr(const MachineInstr *Orig) {
              MachineInstr(*this, *Orig);
 }
 
-MachineInstr &MachineFunction::CloneMachineInstrBundle(MachineBasicBlock &MBB,
-    MachineBasicBlock::iterator InsertBefore, const MachineInstr &Orig) {
+MachineInstr &MachineFunction::cloneMachineInstrBundle(
+    MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore,
+    const MachineInstr &Orig) {
   MachineInstr *FirstClone = nullptr;
   MachineBasicBlock::const_instr_iterator I = Orig.getIterator();
   while (true) {
@@ -390,8 +394,7 @@ MachineInstr &MachineFunction::CloneMachineInstrBundle(MachineBasicBlock &MBB,
 ///
 /// This function also serves as the MachineInstr destructor - the real
 /// ~MachineInstr() destructor must be empty.
-void
-MachineFunction::DeleteMachineInstr(MachineInstr *MI) {
+void MachineFunction::deleteMachineInstr(MachineInstr *MI) {
   // Verify that a call site info is at valid state. This assertion should
   // be triggered during the implementation of support for the
   // call site info of a new architecture. If the assertion is triggered,
@@ -418,8 +421,7 @@ MachineFunction::CreateMachineBasicBlock(const BasicBlock *bb) {
 }
 
 /// Delete the given MachineBasicBlock.
-void
-MachineFunction::DeleteMachineBasicBlock(MachineBasicBlock *MBB) {
+void MachineFunction::deleteMachineBasicBlock(MachineBasicBlock *MBB) {
   assert(MBB->getParent() == this && "MBB parent mismatch!");
   // Clean up any references to MBB in jump tables before deleting it.
   if (JumpTableInfo)
@@ -769,8 +771,8 @@ MCSymbol *MachineFunction::addLandingPad(MachineBasicBlock *LandingPad) {
 void MachineFunction::addCatchTypeInfo(MachineBasicBlock *LandingPad,
                                        ArrayRef<const GlobalValue *> TyInfo) {
   LandingPadInfo &LP = getOrCreateLandingPadInfo(LandingPad);
-  for (unsigned N = TyInfo.size(); N; --N)
-    LP.TypeIds.push_back(getTypeIDFor(TyInfo[N - 1]));
+  for (const GlobalValue *GV : llvm::reverse(TyInfo))
+    LP.TypeIds.push_back(getTypeIDFor(GV));
 }
 
 void MachineFunction::addFilterTypeInfo(MachineBasicBlock *LandingPad,
@@ -1404,10 +1406,10 @@ MachineConstantPool::~MachineConstantPool() {
   // A constant may be a member of both Constants and MachineCPVsSharingEntries,
   // so keep track of which we've deleted to avoid double deletions.
   DenseSet<MachineConstantPoolValue*> Deleted;
-  for (unsigned i = 0, e = Constants.size(); i != e; ++i)
-    if (Constants[i].isMachineConstantPoolEntry()) {
-      Deleted.insert(Constants[i].Val.MachineCPVal);
-      delete Constants[i].Val.MachineCPVal;
+  for (const MachineConstantPoolEntry &C : Constants)
+    if (C.isMachineConstantPoolEntry()) {
+      Deleted.insert(C.Val.MachineCPVal);
+      delete C.Val.MachineCPVal;
     }
   for (MachineConstantPoolValue *CPV : MachineCPVsSharingEntries) {
     if (Deleted.count(CPV) == 0)
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index aaa80432d2f2..85b266afceef 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -115,10 +115,10 @@ void MachineInstr::addImplicitDefUseOperands(MachineFunction &MF) {
 /// MachineInstr ctor - This constructor creates a MachineInstr and adds the
 /// implicit operands. It reserves space for the number of operands specified by
 /// the MCInstrDesc.
-MachineInstr::MachineInstr(MachineFunction &MF, const MCInstrDesc &tid,
-                           DebugLoc dl, bool NoImp)
-    : MCID(&tid), debugLoc(std::move(dl)), DebugInstrNum(0) {
-  assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor");
+MachineInstr::MachineInstr(MachineFunction &MF, const MCInstrDesc &TID,
+                           DebugLoc DL, bool NoImp)
+    : MCID(&TID), DbgLoc(std::move(DL)), DebugInstrNum(0) {
+  assert(DbgLoc.hasTrivialDestructor() && "Expected trivial destructor");
 
   // Reserve space for the expected number of operands.
   if (unsigned NumOps = MCID->getNumOperands() +
@@ -135,9 +135,9 @@ MachineInstr::MachineInstr(MachineFunction &MF, const MCInstrDesc &tid,
 /// Does not copy the number from debug instruction numbering, to preserve
 /// uniqueness.
 MachineInstr::MachineInstr(MachineFunction &MF, const MachineInstr &MI)
-    : MCID(&MI.getDesc()), Info(MI.Info), debugLoc(MI.getDebugLoc()),
+    : MCID(&MI.getDesc()), Info(MI.Info), DbgLoc(MI.getDebugLoc()),
       DebugInstrNum(0) {
-  assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor");
+  assert(DbgLoc.hasTrivialDestructor() && "Expected trivial destructor");
 
   CapOperands = OperandCapacity::get(MI.getNumOperands());
   Operands = MF.allocateOperandArray(CapOperands);
@@ -682,26 +682,6 @@ void MachineInstr::eraseFromParent() {
   getParent()->erase(this);
 }
 
-void MachineInstr::eraseFromParentAndMarkDBGValuesForRemoval() {
-  assert(getParent() && "Not embedded in a basic block!");
-  MachineBasicBlock *MBB = getParent();
-  MachineFunction *MF = MBB->getParent();
-  assert(MF && "Not embedded in a function!");
-
-  MachineInstr *MI = (MachineInstr *)this;
-  MachineRegisterInfo &MRI = MF->getRegInfo();
-
-  for (const MachineOperand &MO : MI->operands()) {
-    if (!MO.isReg() || !MO.isDef())
-      continue;
-    Register Reg = MO.getReg();
-    if (!Reg.isVirtual())
-      continue;
-    MRI.markUsesInDebugValueAsUndef(Reg);
-  }
-  MI->eraseFromParent();
-}
-
 void MachineInstr::eraseFromBundle() {
   assert(getParent() && "Not embedded in a basic block!");
   getParent()->erase_instr(this);
diff --git a/llvm/lib/CodeGen/MachinePipeliner.cpp b/llvm/lib/CodeGen/MachinePipeliner.cpp
index 8d6459a627fa..762395542b40 100644
--- a/llvm/lib/CodeGen/MachinePipeliner.cpp
+++ b/llvm/lib/CodeGen/MachinePipeliner.cpp
@@ -649,7 +649,7 @@ void SwingSchedulerDAG::schedule() {
 /// Clean up after the software pipeliner runs.
 void SwingSchedulerDAG::finishBlock() {
   for (auto &KV : NewMIs)
-    MF.DeleteMachineInstr(KV.second);
+    MF.deleteMachineInstr(KV.second);
   NewMIs.clear();
 
   // Call the superclass.
@@ -1101,17 +1101,15 @@ unsigned SwingSchedulerDAG::calculateResMII() {
   // Sort the instructions by the number of available choices for scheduling,
   // least to most. Use the number of critical resources as the tie breaker.
   FuncUnitSorter FUS = FuncUnitSorter(MF.getSubtarget());
-  for (MachineBasicBlock::iterator I = MBB->getFirstNonPHI(),
-                                   E = MBB->getFirstTerminator();
-       I != E; ++I)
-    FUS.calcCriticalResources(*I);
+  for (MachineInstr &MI :
+       llvm::make_range(MBB->getFirstNonPHI(), MBB->getFirstTerminator()))
+    FUS.calcCriticalResources(MI);
   PriorityQueue<MachineInstr *, std::vector<MachineInstr *>, FuncUnitSorter>
       FuncUnitOrder(FUS);
 
-  for (MachineBasicBlock::iterator I = MBB->getFirstNonPHI(),
-                                   E = MBB->getFirstTerminator();
-       I != E; ++I)
-    FuncUnitOrder.push(&*I);
+  for (MachineInstr &MI :
+       llvm::make_range(MBB->getFirstNonPHI(), MBB->getFirstTerminator()))
+    FuncUnitOrder.push(&MI);
 
   while (!FuncUnitOrder.empty()) {
     MachineInstr *MI = FuncUnitOrder.top();
@@ -1192,14 +1190,10 @@ unsigned SwingSchedulerDAG::calculateRecMII(NodeSetType &NodeSets) {
 /// but we do this to find the circuits, and then change them back.
 static void swapAntiDependences(std::vector<SUnit> &SUnits) {
   SmallVector<std::pair<SUnit *, SDep>, 8> DepsAdded;
-  for (unsigned i = 0, e = SUnits.size(); i != e; ++i) {
-    SUnit *SU = &SUnits[i];
-    for (SUnit::pred_iterator IP = SU->Preds.begin(), EP = SU->Preds.end();
-         IP != EP; ++IP) {
-      if (IP->getKind() != SDep::Anti)
-        continue;
-      DepsAdded.push_back(std::make_pair(SU, *IP));
-    }
+  for (SUnit &SU : SUnits) {
+    for (SDep &Pred : SU.Preds)
+      if (Pred.getKind() == SDep::Anti)
+        DepsAdded.push_back(std::make_pair(&SU, Pred));
   }
   for (std::pair<SUnit *, SDep> &P : DepsAdded) {
     // Remove this anti dependency and add one in the reverse direction.
@@ -1471,27 +1465,23 @@ void SwingSchedulerDAG::computeNodeFunctions(NodeSetType &NodeSets) {
   }
 
   // Compute ALAP, ZeroLatencyHeight, and MOV.
-  for (ScheduleDAGTopologicalSort::const_reverse_iterator I = Topo.rbegin(),
-                                                          E = Topo.rend();
-       I != E; ++I) {
+  for (int I : llvm::reverse(Topo)) {
     int alap = maxASAP;
     int zeroLatencyHeight = 0;
-    SUnit *SU = &SUnits[*I];
-    for (SUnit::const_succ_iterator IS = SU->Succs.begin(),
-                                    ES = SU->Succs.end();
-         IS != ES; ++IS) {
-      SUnit *succ = IS->getSUnit();
-      if (IS->getLatency() == 0)
+    SUnit *SU = &SUnits[I];
+    for (const SDep &S : SU->Succs) {
+      SUnit *succ = S.getSUnit();
+      if (S.getLatency() == 0)
         zeroLatencyHeight =
             std::max(zeroLatencyHeight, getZeroLatencyHeight(succ) + 1);
-      if (ignoreDependence(*IS, true))
+      if (ignoreDependence(S, true))
         continue;
-      alap = std::min(alap, (int)(getALAP(succ) - IS->getLatency() +
-                                  getDistance(SU, succ, *IS) * MII));
+      alap = std::min(alap, (int)(getALAP(succ) - S.getLatency() +
+                                  getDistance(SU, succ, S) * MII));
     }
 
-    ScheduleInfo[*I].ALAP = alap;
-    ScheduleInfo[*I].ZeroLatencyHeight = zeroLatencyHeight;
+    ScheduleInfo[I].ALAP = alap;
+    ScheduleInfo[I].ZeroLatencyHeight = zeroLatencyHeight;
   }
 
   // After computing the node functions, compute the summary for each node set.
@@ -1548,9 +1538,8 @@ static bool succ_L(SetVector<SUnit *> &NodeOrder,
                    SmallSetVector<SUnit *, 8> &Succs,
                    const NodeSet *S = nullptr) {
   Succs.clear();
-  for (SetVector<SUnit *>::iterator I = NodeOrder.begin(), E = NodeOrder.end();
-       I != E; ++I) {
-    for (SDep &Succ : (*I)->Succs) {
+  for (const SUnit *SU : NodeOrder) {
+    for (const SDep &Succ : SU->Succs) {
       if (S && S->count(Succ.getSUnit()) == 0)
         continue;
       if (ignoreDependence(Succ, false))
@@ -1558,7 +1547,7 @@ static bool succ_L(SetVector<SUnit *> &NodeOrder,
       if (NodeOrder.count(Succ.getSUnit()) == 0)
         Succs.insert(Succ.getSUnit());
     }
-    for (SDep &Pred : (*I)->Preds) {
+    for (const SDep &Pred : SU->Preds) {
       if (Pred.getKind() != SDep::Anti)
         continue;
       if (S && S->count(Pred.getSUnit()) == 0)
@@ -2202,7 +2191,7 @@ bool SwingSchedulerDAG::canUseLastOffsetValue(MachineInstr *MI,
   MachineInstr *NewMI = MF.CloneMachineInstr(MI);
   NewMI->getOperand(OffsetPosLd).setImm(LoadOffset + StoreOffset);
   bool Disjoint = TII->areMemAccessesTriviallyDisjoint(*NewMI, *PrevDef);
-  MF.DeleteMachineInstr(NewMI);
+  MF.deleteMachineInstr(NewMI);
   if (!Disjoint)
     return false;
 
@@ -2885,10 +2874,8 @@ void SMSchedule::finalizeSchedule(SwingSchedulerDAG *SSD) {
          ++stage) {
       std::deque<SUnit *> &cycleInstrs =
           ScheduledInstrs[cycle + (stage * InitiationInterval)];
-      for (std::deque<SUnit *>::reverse_iterator I = cycleInstrs.rbegin(),
-                                                 E = cycleInstrs.rend();
-           I != E; ++I)
-        ScheduledInstrs[cycle].push_front(*I);
+      for (SUnit *SU : llvm::reverse(cycleInstrs))
+        ScheduledInstrs[cycle].push_front(SU);
     }
   }
 
@@ -2899,10 +2886,8 @@ void SMSchedule::finalizeSchedule(SwingSchedulerDAG *SSD) {
 
   // Change the registers in instruction as specified in the InstrChanges
   // map. We need to use the new registers to create the correct order.
-  for (int i = 0, e = SSD->SUnits.size(); i != e; ++i) {
-    SUnit *SU = &SSD->SUnits[i];
-    SSD->applyInstrChange(SU->getInstr(), *this);
-  }
+  for (const SUnit &SU : SSD->SUnits)
+    SSD->applyInstrChange(SU.getInstr(), *this);
 
   // Reorder the instructions in each cycle to fix and improve the
   // generated code.
diff --git a/llvm/lib/CodeGen/MachineSSAContext.cpp b/llvm/lib/CodeGen/MachineSSAContext.cpp
new file mode 100644
index 000000000000..8db893535daf
--- /dev/null
+++ b/llvm/lib/CodeGen/MachineSSAContext.cpp
@@ -0,0 +1,52 @@
+//===- MachineSSAContext.cpp ------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+///
+/// This file defines a specialization of the GenericSSAContext<X>
+/// template class for Machine IR.
+///
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/MachineSSAContext.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+
+MachineBasicBlock *MachineSSAContext::getEntryBlock(MachineFunction &F) {
+  return &F.front();
+}
+
+void MachineSSAContext::setFunction(MachineFunction &Fn) {
+  MF = &Fn;
+  RegInfo = &MF->getRegInfo();
+}
+
+Printable MachineSSAContext::print(MachineBasicBlock *Block) const {
+  return Printable([Block](raw_ostream &Out) { Block->printName(Out); });
+}
+
+Printable MachineSSAContext::print(MachineInstr *I) const {
+  return Printable([I](raw_ostream &Out) { I->print(Out); });
+}
+
+Printable MachineSSAContext::print(Register Value) const {
+  auto *MRI = RegInfo;
+  return Printable([MRI, Value](raw_ostream &Out) {
+    Out << printReg(Value, MRI->getTargetRegisterInfo(), 0, MRI);
+
+    if (Value) {
+      // Try to print the definition.
+      if (auto *Instr = MRI->getUniqueVRegDef(Value)) {
+        Out << ": ";
+        Instr->print(Out);
+      }
+    }
+  });
+}
diff --git a/llvm/lib/CodeGen/MachineSSAUpdater.cpp b/llvm/lib/CodeGen/MachineSSAUpdater.cpp
index 930677e4fd7d..48076663ddf5 100644
--- a/llvm/lib/CodeGen/MachineSSAUpdater.cpp
+++ b/llvm/lib/CodeGen/MachineSSAUpdater.cpp
@@ -126,7 +126,9 @@ MachineInstrBuilder InsertNewDef(unsigned Opcode,
 }
 
 /// GetValueInMiddleOfBlock - Construct SSA form, materializing a value that
-/// is live in the middle of the specified block.
+/// is live in the middle of the specified block. If ExistingValueOnly is
+/// true then this will only return an existing value or $noreg; otherwise new
+/// instructions may be inserted to materialize a value.
 ///
 /// GetValueInMiddleOfBlock is the same as GetValueAtEndOfBlock except in one
 /// important case: if there is a definition of the rewritten value after the
@@ -143,14 +145,18 @@ MachineInstrBuilder InsertNewDef(unsigned Opcode,
 /// their respective blocks.  However, the use of X happens in the *middle* of
 /// a block.  Because of this, we need to insert a new PHI node in SomeBB to
 /// merge the appropriate values, and this value isn't live out of the block.
-Register MachineSSAUpdater::GetValueInMiddleOfBlock(MachineBasicBlock *BB) {
+Register MachineSSAUpdater::GetValueInMiddleOfBlock(MachineBasicBlock *BB,
+                                                    bool ExistingValueOnly) {
   // If there is no definition of the renamed variable in this block, just use
   // GetValueAtEndOfBlock to do our work.
   if (!HasValueForBlock(BB))
-    return GetValueAtEndOfBlockInternal(BB);
+    return GetValueAtEndOfBlockInternal(BB, ExistingValueOnly);
 
   // If there are no predecessors, just return undef.
   if (BB->pred_empty()) {
+    // If we cannot insert new instructions, just return $noreg.
+    if (ExistingValueOnly)
+      return Register();
     // Insert an implicit_def to represent an undef value.
     MachineInstr *NewDef = InsertNewDef(TargetOpcode::IMPLICIT_DEF,
                                         BB, BB->getFirstTerminator(),
@@ -165,7 +171,7 @@ Register MachineSSAUpdater::GetValueInMiddleOfBlock(MachineBasicBlock *BB) {
 
   bool isFirstPred = true;
   for (MachineBasicBlock *PredBB : BB->predecessors()) {
-    Register PredVal = GetValueAtEndOfBlockInternal(PredBB);
+    Register PredVal = GetValueAtEndOfBlockInternal(PredBB, ExistingValueOnly);
     PredValues.push_back(std::make_pair(PredBB, PredVal));
 
     // Compute SingularValue.
@@ -185,6 +191,10 @@ Register MachineSSAUpdater::GetValueInMiddleOfBlock(MachineBasicBlock *BB) {
   if (DupPHI)
     return DupPHI;
 
+  // If we cannot create new instructions, return $noreg now.
+  if (ExistingValueOnly)
+    return Register();
+
   // Otherwise, we do need a PHI: insert one now.
   MachineBasicBlock::iterator Loc = BB->empty() ? BB->end() : BB->begin();
   MachineInstrBuilder InsertedPHI = InsertNewDef(TargetOpcode::PHI, BB,
@@ -350,10 +360,13 @@ public:
 /// for the specified BB and if so, return it.  If not, construct SSA form by
 /// first calculating the required placement of PHIs and then inserting new
 /// PHIs where needed.
-Register MachineSSAUpdater::GetValueAtEndOfBlockInternal(MachineBasicBlock *BB){
+Register
+MachineSSAUpdater::GetValueAtEndOfBlockInternal(MachineBasicBlock *BB,
+                                                bool ExistingValueOnly) {
   AvailableValsTy &AvailableVals = getAvailableVals(AV);
-  if (Register V = AvailableVals[BB])
-    return V;
+  Register ExistingVal = AvailableVals.lookup(BB);
+  if (ExistingVal || ExistingValueOnly)
+    return ExistingVal;
 
   SSAUpdaterImpl<MachineSSAUpdater> Impl(this, &AvailableVals, InsertedPHIs);
   return Impl.GetValue(BB);
diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp
index 47d40f0823c8..b043d4c1b0c1 100644
--- a/llvm/lib/CodeGen/MachineScheduler.cpp
+++ b/llvm/lib/CodeGen/MachineScheduler.cpp
@@ -90,12 +90,17 @@ cl::opt<bool> VerifyScheduling(
     "verify-misched", cl::Hidden,
     cl::desc("Verify machine instrs before and after machine scheduling"));
 
+#ifndef NDEBUG
+cl::opt<bool> ViewMISchedDAGs(
+    "view-misched-dags", cl::Hidden,
+    cl::desc("Pop up a window to show MISched dags after they are processed"));
+#else
+const bool ViewMISchedDAGs = false;
+#endif // NDEBUG
+
 } // end namespace llvm
 
 #ifndef NDEBUG
-static cl::opt<bool> ViewMISchedDAGs("view-misched-dags", cl::Hidden,
-  cl::desc("Pop up a window to show MISched dags after they are processed"));
-
 /// In some situations a few uninteresting nodes depend on nearly all other
 /// nodes in the graph, provide a cutoff to hide them.
 static cl::opt<unsigned> ViewMISchedCutoff("view-misched-cutoff", cl::Hidden,
@@ -111,7 +116,6 @@ static cl::opt<unsigned> SchedOnlyBlock("misched-only-block", cl::Hidden,
 static cl::opt<bool> PrintDAGs("misched-print-dags", cl::Hidden,
                               cl::desc("Print schedule DAGs"));
 #else
-static const bool ViewMISchedDAGs = false;
 static const bool PrintDAGs = false;
 #endif // NDEBUG
 
@@ -561,11 +565,10 @@ void MachineSchedulerBase::scheduleRegions(ScheduleDAGInstrs &Scheduler,
 
     MBBRegionsVector MBBRegions;
     getSchedRegions(&*MBB, MBBRegions, Scheduler.doMBBSchedRegionsTopDown());
-    for (MBBRegionsVector::iterator R = MBBRegions.begin();
-         R != MBBRegions.end(); ++R) {
-      MachineBasicBlock::iterator I = R->RegionBegin;
-      MachineBasicBlock::iterator RegionEnd = R->RegionEnd;
-      unsigned NumRegionInstrs = R->NumRegionInstrs;
+    for (const SchedRegion &R : MBBRegions) {
+      MachineBasicBlock::iterator I = R.RegionBegin;
+      MachineBasicBlock::iterator RegionEnd = R.RegionEnd;
+      unsigned NumRegionInstrs = R.NumRegionInstrs;
 
       // Notify the scheduler of the region, even if we may skip scheduling
       // it. Perhaps it still needs to be bundled.
diff --git a/llvm/lib/CodeGen/MachineTraceMetrics.cpp b/llvm/lib/CodeGen/MachineTraceMetrics.cpp
index 8df23b781ffd..0a5ff276fedc 100644
--- a/llvm/lib/CodeGen/MachineTraceMetrics.cpp
+++ b/llvm/lib/CodeGen/MachineTraceMetrics.cpp
@@ -80,9 +80,9 @@ bool MachineTraceMetrics::runOnMachineFunction(MachineFunction &Func) {
 void MachineTraceMetrics::releaseMemory() {
   MF = nullptr;
   BlockInfo.clear();
-  for (unsigned i = 0; i != TS_NumStrategies; ++i) {
-    delete Ensembles[i];
-    Ensembles[i] = nullptr;
+  for (Ensemble *&E : Ensembles) {
+    delete E;
+    E = nullptr;
   }
 }
 
@@ -398,9 +398,9 @@ void MachineTraceMetrics::invalidate(const MachineBasicBlock *MBB) {
   LLVM_DEBUG(dbgs() << "Invalidate traces through " << printMBBReference(*MBB)
                     << '\n');
   BlockInfo[MBB->getNumber()].invalidate();
-  for (unsigned i = 0; i != TS_NumStrategies; ++i)
-    if (Ensembles[i])
-      Ensembles[i]->invalidate(MBB);
+  for (Ensemble *E : Ensembles)
+    if (E)
+      E->invalidate(MBB);
 }
 
 void MachineTraceMetrics::verifyAnalysis() const {
@@ -408,9 +408,9 @@ void MachineTraceMetrics::verifyAnalysis() const {
     return;
 #ifndef NDEBUG
   assert(BlockInfo.size() == MF->getNumBlockIDs() && "Outdated BlockInfo size");
-  for (unsigned i = 0; i != TS_NumStrategies; ++i)
-    if (Ensembles[i])
-      Ensembles[i]->verify();
+  for (Ensemble *E : Ensembles)
+    if (E)
+      E->verify();
 #endif
 }
 
@@ -984,8 +984,7 @@ addLiveIns(const MachineInstr *DefMI, unsigned DefOp,
   const MachineBasicBlock *DefMBB = DefMI->getParent();
 
   // Reg is live-in to all blocks in Trace that follow DefMBB.
-  for (unsigned i = Trace.size(); i; --i) {
-    const MachineBasicBlock *MBB = Trace[i-1];
+  for (const MachineBasicBlock *MBB : llvm::reverse(Trace)) {
     if (MBB == DefMBB)
       return;
     TraceBlockInfo &TBI = BlockInfo[MBB->getNumber()];
@@ -1204,8 +1203,8 @@ unsigned MachineTraceMetrics::Trace::getResourceDepth(bool Bottom) const {
     for (unsigned K = 0; K != PRDepths.size(); ++K)
       PRMax = std::max(PRMax, PRDepths[K] + PRCycles[K]);
   } else {
-    for (unsigned K = 0; K != PRDepths.size(); ++K)
-      PRMax = std::max(PRMax, PRDepths[K]);
+    for (unsigned PRD : PRDepths)
+      PRMax = std::max(PRMax, PRD);
   }
   // Convert to cycle count.
   PRMax = TE.MTM.getCycles(PRMax);
diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp
index 32078db76cf3..005d4ad1a328 100644
--- a/llvm/lib/CodeGen/MachineVerifier.cpp
+++ b/llvm/lib/CodeGen/MachineVerifier.cpp
@@ -101,6 +101,7 @@ namespace {
     // Avoid querying the MachineFunctionProperties for each operand.
     bool isFunctionRegBankSelected;
     bool isFunctionSelected;
+    bool isFunctionTracksDebugUserValues;
 
     using RegVector = SmallVector<Register, 16>;
     using RegMaskVector = SmallVector<const uint32_t *, 4>;
@@ -384,6 +385,8 @@ unsigned MachineVerifier::verify(const MachineFunction &MF) {
       MachineFunctionProperties::Property::RegBankSelected);
   isFunctionSelected = MF.getProperties().hasProperty(
       MachineFunctionProperties::Property::Selected);
+  isFunctionTracksDebugUserValues = MF.getProperties().hasProperty(
+      MachineFunctionProperties::Property::TracksDebugUserValues);
 
   LiveVars = nullptr;
   LiveInts = nullptr;
@@ -1605,12 +1608,16 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) {
     }
     break;
   }
+  case TargetOpcode::G_SHL:
+  case TargetOpcode::G_LSHR:
+  case TargetOpcode::G_ASHR:
   case TargetOpcode::G_ROTR:
   case TargetOpcode::G_ROTL: {
     LLT Src1Ty = MRI->getType(MI->getOperand(1).getReg());
     LLT Src2Ty = MRI->getType(MI->getOperand(2).getReg());
     if (Src1Ty.isVector() != Src2Ty.isVector()) {
-      report("Rotate requires operands to be either all scalars or all vectors",
+      report("Shifts and rotates require operands to be either all scalars or "
+             "all vectors",
              MI);
       break;
     }
@@ -1980,41 +1987,50 @@ MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) {
         if (MO->isUndef())
           report("Generic virtual register use cannot be undef", MO, MONum);
 
-        // If we're post-Select, we can't have gvregs anymore.
-        if (isFunctionSelected) {
-          report("Generic virtual register invalid in a Selected function",
-                 MO, MONum);
-          return;
-        }
+        // Debug value instruction is permitted to use undefined vregs.
+        // This is a performance measure to skip the overhead of immediately
+        // pruning unused debug operands. The final undef substitution occurs
+        // when debug values are allocated in LDVImpl::handleDebugValue, so
+        // these verifications always apply after this pass.
+        if (isFunctionTracksDebugUserValues || !MO->isUse() ||
+            !MI->isDebugValue() || !MRI->def_empty(Reg)) {
+          // If we're post-Select, we can't have gvregs anymore.
+          if (isFunctionSelected) {
+            report("Generic virtual register invalid in a Selected function",
+                   MO, MONum);
+            return;
+          }
 
-        // The gvreg must have a type and it must not have a SubIdx.
-        LLT Ty = MRI->getType(Reg);
-        if (!Ty.isValid()) {
-          report("Generic virtual register must have a valid type", MO,
-                 MONum);
-          return;
-        }
+          // The gvreg must have a type and it must not have a SubIdx.
+          LLT Ty = MRI->getType(Reg);
+          if (!Ty.isValid()) {
+            report("Generic virtual register must have a valid type", MO,
+                   MONum);
+            return;
+          }
 
-        const RegisterBank *RegBank = MRI->getRegBankOrNull(Reg);
+          const RegisterBank *RegBank = MRI->getRegBankOrNull(Reg);
 
-        // If we're post-RegBankSelect, the gvreg must have a bank.
-        if (!RegBank && isFunctionRegBankSelected) {
-          report("Generic virtual register must have a bank in a "
-                 "RegBankSelected function",
-                 MO, MONum);
-          return;
-        }
+          // If we're post-RegBankSelect, the gvreg must have a bank.
+          if (!RegBank && isFunctionRegBankSelected) {
+            report("Generic virtual register must have a bank in a "
+                   "RegBankSelected function",
+                   MO, MONum);
+            return;
+          }
 
-        // Make sure the register fits into its register bank if any.
-        if (RegBank && Ty.isValid() &&
-            RegBank->getSize() < Ty.getSizeInBits()) {
-          report("Register bank is too small for virtual register", MO,
-                 MONum);
-          errs() << "Register bank " << RegBank->getName() << " too small("
-                 << RegBank->getSize() << ") to fit " << Ty.getSizeInBits()
-                 << "-bits\n";
-          return;
+          // Make sure the register fits into its register bank if any.
+          if (RegBank && Ty.isValid() &&
+              RegBank->getSize() < Ty.getSizeInBits()) {
+            report("Register bank is too small for virtual register", MO,
+                   MONum);
+            errs() << "Register bank " << RegBank->getName() << " too small("
+                   << RegBank->getSize() << ") to fit " << Ty.getSizeInBits()
+                   << "-bits\n";
+            return;
+          }
         }
+
         if (SubIdx)  {
           report("Generic virtual register does not allow subregister index", MO,
                  MONum);
@@ -2217,8 +2233,8 @@ void MachineVerifier::checkLiveness(const MachineOperand *MO, unsigned MONum) {
   if (LiveInts && Reg.isVirtual()) {
     if (LiveInts->hasInterval(Reg)) {
       LI = &LiveInts->getInterval(Reg);
-      if (SubRegIdx != 0 && !LI->empty() && !LI->hasSubRanges() &&
-          MRI->shouldTrackSubRegLiveness(Reg))
+      if (SubRegIdx != 0 && (MO->isDef() || !MO->isUndef()) && !LI->empty() &&
+          !LI->hasSubRanges() && MRI->shouldTrackSubRegLiveness(Reg))
         report("Live interval for subreg operand has no subranges", MO, MONum);
     } else {
       report("Virtual register has no live interval", MO, MONum);
diff --git a/llvm/lib/CodeGen/PHIElimination.cpp b/llvm/lib/CodeGen/PHIElimination.cpp
index 77a6c37e1362..7693ab417de9 100644
--- a/llvm/lib/CodeGen/PHIElimination.cpp
+++ b/llvm/lib/CodeGen/PHIElimination.cpp
@@ -213,7 +213,7 @@ bool PHIElimination::runOnMachineFunction(MachineFunction &MF) {
   for (auto &I : LoweredPHIs) {
     if (LIS)
       LIS->RemoveMachineInstrFromMaps(*I.first);
-    MF.DeleteMachineInstr(I.first);
+    MF.deleteMachineInstr(I.first);
   }
 
   // TODO: we should use the incremental DomTree updater here.
@@ -626,7 +626,7 @@ void PHIElimination::LowerPHINode(MachineBasicBlock &MBB,
   if (reusedIncoming || !IncomingReg) {
     if (LIS)
       LIS->RemoveMachineInstrFromMaps(*MPhi);
-    MF.DeleteMachineInstr(MPhi);
+    MF.deleteMachineInstr(MPhi);
   }
 }
 
diff --git a/llvm/lib/CodeGen/PostRASchedulerList.cpp b/llvm/lib/CodeGen/PostRASchedulerList.cpp
index b85f00a61eac..d7cd0a583cee 100644
--- a/llvm/lib/CodeGen/PostRASchedulerList.cpp
+++ b/llvm/lib/CodeGen/PostRASchedulerList.cpp
@@ -252,8 +252,8 @@ void SchedulePostRATDList::exitRegion() {
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 /// dumpSchedule - dump the scheduled Sequence.
 LLVM_DUMP_METHOD void SchedulePostRATDList::dumpSchedule() const {
-  for (unsigned i = 0, e = Sequence.size(); i != e; i++) {
-    if (SUnit *SU = Sequence[i])
+  for (const SUnit *SU : Sequence) {
+    if (SU)
       dumpNode(*SU);
     else
       dbgs() << "**** NOOP ****\n";
@@ -531,11 +531,11 @@ void SchedulePostRATDList::ListScheduleTopDown() {
   ReleaseSuccessors(&EntrySU);
 
   // Add all leaves to Available queue.
-  for (unsigned i = 0, e = SUnits.size(); i != e; ++i) {
+  for (SUnit &SUnit : SUnits) {
     // It is available if it has no predecessors.
-    if (!SUnits[i].NumPredsLeft && !SUnits[i].isAvailable) {
-      AvailableQueue.push(&SUnits[i]);
-      SUnits[i].isAvailable = true;
+    if (!SUnit.NumPredsLeft && !SUnit.isAvailable) {
+      AvailableQueue.push(&SUnit);
+      SUnit.isAvailable = true;
     }
   }
 
@@ -657,10 +657,7 @@ void SchedulePostRATDList::ListScheduleTopDown() {
 
 #ifndef NDEBUG
   unsigned ScheduledNodes = VerifyScheduledDAG(/*isBottomUp=*/false);
-  unsigned Noops = 0;
-  for (unsigned i = 0, e = Sequence.size(); i != e; ++i)
-    if (!Sequence[i])
-      ++Noops;
+  unsigned Noops = llvm::count(Sequence, nullptr);
   assert(Sequence.size() - Noops == ScheduledNodes &&
          "The number of nodes scheduled doesn't match the expected number!");
 #endif // NDEBUG
diff --git a/llvm/lib/CodeGen/PrologEpilogInserter.cpp b/llvm/lib/CodeGen/PrologEpilogInserter.cpp
index 29a88480fd9f..8d8a6126dad0 100644
--- a/llvm/lib/CodeGen/PrologEpilogInserter.cpp
+++ b/llvm/lib/CodeGen/PrologEpilogInserter.cpp
@@ -953,12 +953,22 @@ void PEI::calculateFrameObjectOffsets(MachineFunction &MF) {
     // LocalStackSlotPass didn't already allocate a slot for it.
     // If we are told to use the LocalStackAllocationBlock, the stack protector
     // is expected to be already pre-allocated.
-    if (!MFI.getUseLocalStackAllocationBlock())
+    if (MFI.getStackID(StackProtectorFI) != TargetStackID::Default) {
+      // If the stack protector isn't on the default stack then it's up to the
+      // target to set the stack offset.
+      assert(MFI.getObjectOffset(StackProtectorFI) != 0 &&
+             "Offset of stack protector on non-default stack expected to be "
+             "already set.");
+      assert(!MFI.isObjectPreAllocated(MFI.getStackProtectorIndex()) &&
+             "Stack protector on non-default stack expected to not be "
+             "pre-allocated by LocalStackSlotPass.");
+    } else if (!MFI.getUseLocalStackAllocationBlock()) {
       AdjustStackOffset(MFI, StackProtectorFI, StackGrowsDown, Offset, MaxAlign,
                         Skew);
-    else if (!MFI.isObjectPreAllocated(MFI.getStackProtectorIndex()))
+    } else if (!MFI.isObjectPreAllocated(MFI.getStackProtectorIndex())) {
       llvm_unreachable(
           "Stack protector not pre-allocated by LocalStackSlotPass.");
+    }
 
     // Assign large stack objects first.
     for (unsigned i = 0, e = MFI.getObjectIndexEnd(); i != e; ++i) {
diff --git a/llvm/lib/CodeGen/RDFGraph.cpp b/llvm/lib/CodeGen/RDFGraph.cpp
index f605068e076d..882f8e91bf1d 100644
--- a/llvm/lib/CodeGen/RDFGraph.cpp
+++ b/llvm/lib/CodeGen/RDFGraph.cpp
@@ -1500,8 +1500,8 @@ void DataFlowGraph::buildPhis(BlockRefsMap &PhiM, RegisterSet &AllRefs,
 
     // Erase from MaxRefs all elements in the closure.
     auto Begin = MaxRefs.begin();
-    for (unsigned i = ClosureIdx.size(); i != 0; --i)
-      MaxRefs.erase(Begin + ClosureIdx[i-1]);
+    for (unsigned Idx : llvm::reverse(ClosureIdx))
+      MaxRefs.erase(Begin + Idx);
   }
 }
 
diff --git a/llvm/lib/CodeGen/RegAllocEvictionAdvisor.cpp b/llvm/lib/CodeGen/RegAllocEvictionAdvisor.cpp
new file mode 100644
index 000000000000..9f1012c95964
--- /dev/null
+++ b/llvm/lib/CodeGen/RegAllocEvictionAdvisor.cpp
@@ -0,0 +1,121 @@
+//===- RegAllocEvictionAdvisor.cpp - eviction advisor ---------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implementation of the default eviction advisor and of the Analysis pass.
+//
+//===----------------------------------------------------------------------===//
+
+#include "RegAllocEvictionAdvisor.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/RegisterClassInfo.h"
+#include "llvm/CodeGen/VirtRegMap.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Pass.h"
+#include "llvm/PassRegistry.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Target/TargetMachine.h"
+
+using namespace llvm;
+
+static cl::opt<RegAllocEvictionAdvisorAnalysis::AdvisorMode> Mode(
+    "regalloc-enable-advisor", cl::Hidden,
+    cl::init(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Default),
+    cl::desc("Enable regalloc advisor mode"),
+    cl::values(
+        clEnumValN(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Default,
+                   "default", "Default"),
+        clEnumValN(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Release,
+                   "release", "precompiled"),
+        clEnumValN(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Development,
+                   "development", "for training")));
+
+static cl::opt<bool> EnableLocalReassignment(
+    "enable-local-reassign", cl::Hidden,
+    cl::desc("Local reassignment can yield better allocation decisions, but "
+             "may be compile time intensive"),
+    cl::init(false));
+
+#define DEBUG_TYPE "regalloc"
+
+char RegAllocEvictionAdvisorAnalysis::ID = 0;
+INITIALIZE_PASS(RegAllocEvictionAdvisorAnalysis, "regalloc-evict",
+                "Regalloc eviction policy", false, true)
+
+namespace {
+class DefaultEvictionAdvisorAnalysis final
+    : public RegAllocEvictionAdvisorAnalysis {
+public:
+  DefaultEvictionAdvisorAnalysis(bool NotAsRequested)
+      : RegAllocEvictionAdvisorAnalysis(AdvisorMode::Default),
+        NotAsRequested(NotAsRequested) {}
+
+  // support for isa<> and dyn_cast.
+  static bool classof(const RegAllocEvictionAdvisorAnalysis *R) {
+    return R->getAdvisorMode() == AdvisorMode::Default;
+  }
+
+private:
+  std::unique_ptr<RegAllocEvictionAdvisor>
+  getAdvisor(const MachineFunction &MF, LiveRegMatrix *Matrix,
+             LiveIntervals *LIS, VirtRegMap *VRM,
+             const RegisterClassInfo &RegClassInfo,
+             ExtraRegInfo *ExtraInfo) override {
+    return std::make_unique<DefaultEvictionAdvisor>(MF, Matrix, LIS, VRM,
+                                                    RegClassInfo, ExtraInfo);
+  }
+  bool doInitialization(Module &M) override {
+    if (NotAsRequested)
+      M.getContext().emitError("Requested regalloc eviction advisor analysis "
+                               "could be created. Using default");
+    return RegAllocEvictionAdvisorAnalysis::doInitialization(M);
+  }
+  const bool NotAsRequested;
+};
+} // namespace
+
+template <> Pass *llvm::callDefaultCtor<RegAllocEvictionAdvisorAnalysis>() {
+  Pass *Ret = nullptr;
+  switch (Mode) {
+  case RegAllocEvictionAdvisorAnalysis::AdvisorMode::Default:
+    Ret = new DefaultEvictionAdvisorAnalysis(/*NotAsRequested*/ false);
+    break;
+  case RegAllocEvictionAdvisorAnalysis::AdvisorMode::Development:
+    // TODO(mtrofin): add implementation
+    break;
+  case RegAllocEvictionAdvisorAnalysis::AdvisorMode::Release:
+    // TODO(mtrofin): add implementation
+    break;
+  }
+  if (Ret)
+    return Ret;
+  return new DefaultEvictionAdvisorAnalysis(/*NotAsRequested*/ true);
+}
+
+StringRef RegAllocEvictionAdvisorAnalysis::getPassName() const {
+  switch (getAdvisorMode()) {
+  case AdvisorMode::Default:
+    return "Default Regalloc Eviction Advisor";
+  case AdvisorMode::Release:
+    return "Release mode Regalloc Eviction Advisor";
+  case AdvisorMode::Development:
+    return "Development mode Regalloc Eviction Advisor";
+  }
+  llvm_unreachable("Unknown advisor kind");
+}
+
+RegAllocEvictionAdvisor::RegAllocEvictionAdvisor(
+    const MachineFunction &MF, LiveRegMatrix *Matrix, LiveIntervals *LIS,
+    VirtRegMap *VRM, const RegisterClassInfo &RegClassInfo,
+    ExtraRegInfo *ExtraInfo)
+    : MF(MF), Matrix(Matrix), LIS(LIS), VRM(VRM), MRI(&VRM->getRegInfo()),
+      TRI(MF.getSubtarget().getRegisterInfo()), RegClassInfo(RegClassInfo),
+      RegCosts(TRI->getRegisterCosts(MF)), ExtraInfo(ExtraInfo),
+      EnableLocalReassign(EnableLocalReassignment ||
+                          MF.getSubtarget().enableRALocalReassignment(
+                              MF.getTarget().getOptLevel())) {}
diff --git a/llvm/lib/CodeGen/RegAllocEvictionAdvisor.h b/llvm/lib/CodeGen/RegAllocEvictionAdvisor.h
index 85fd3207888b..debb75ed5020 100644
--- a/llvm/lib/CodeGen/RegAllocEvictionAdvisor.h
+++ b/llvm/lib/CodeGen/RegAllocEvictionAdvisor.h
@@ -18,6 +18,7 @@
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/Register.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
+#include "llvm/Config/llvm-config.h"
 #include "llvm/Pass.h"
 
 namespace llvm {
@@ -85,6 +86,215 @@ struct EvictionCost {
            std::tie(O.BrokenHints, O.MaxWeight);
   }
 };
+
+/// Track allocation stage and eviction loop prevention during allocation.
+// TODO(mtrofin): Consider exposing RAGreedy in a header instead, and folding
+// this back into it.
+class ExtraRegInfo final {
+  // RegInfo - Keep additional information about each live range.
+  struct RegInfo {
+    LiveRangeStage Stage = RS_New;
+
+    // Cascade - Eviction loop prevention. See
+    // canEvictInterferenceBasedOnCost().
+    unsigned Cascade = 0;
+
+    RegInfo() = default;
+  };
+
+  IndexedMap<RegInfo, VirtReg2IndexFunctor> Info;
+  unsigned NextCascade = 1;
+
+public:
+  ExtraRegInfo() = default;
+  ExtraRegInfo(const ExtraRegInfo &) = delete;
+
+  LiveRangeStage getStage(Register Reg) const { return Info[Reg].Stage; }
+
+  LiveRangeStage getStage(const LiveInterval &VirtReg) const {
+    return getStage(VirtReg.reg());
+  }
+
+  void setStage(Register Reg, LiveRangeStage Stage) {
+    Info.grow(Reg.id());
+    Info[Reg].Stage = Stage;
+  }
+
+  void setStage(const LiveInterval &VirtReg, LiveRangeStage Stage) {
+    setStage(VirtReg.reg(), Stage);
+  }
+
+  /// Return the current stage of the register, if present, otherwise initialize
+  /// it and return that.
+  LiveRangeStage getOrInitStage(Register Reg) {
+    Info.grow(Reg.id());
+    return getStage(Reg);
+  }
+
+  unsigned getCascade(Register Reg) const { return Info[Reg].Cascade; }
+
+  void setCascade(Register Reg, unsigned Cascade) {
+    Info.grow(Reg.id());
+    Info[Reg].Cascade = Cascade;
+  }
+
+  unsigned getOrAssignNewCascade(Register Reg) {
+    unsigned Cascade = getCascade(Reg);
+    if (!Cascade) {
+      Cascade = NextCascade++;
+      setCascade(Reg, Cascade);
+    }
+    return Cascade;
+  }
+
+  unsigned getCascadeOrCurrentNext(Register Reg) const {
+    unsigned Cascade = getCascade(Reg);
+    if (!Cascade)
+      Cascade = NextCascade;
+    return Cascade;
+  }
+
+  template <typename Iterator>
+  void setStage(Iterator Begin, Iterator End, LiveRangeStage NewStage) {
+    for (; Begin != End; ++Begin) {
+      Register Reg = *Begin;
+      Info.grow(Reg.id());
+      if (Info[Reg].Stage == RS_New)
+        Info[Reg].Stage = NewStage;
+    }
+  }
+  void LRE_DidCloneVirtReg(Register New, Register Old);
+};
+
+/// Interface to the eviction advisor, which is responsible for making a
+/// decision as to which live ranges should be evicted (if any).
+class RegAllocEvictionAdvisor {
+public:
+  RegAllocEvictionAdvisor(const RegAllocEvictionAdvisor &) = delete;
+  RegAllocEvictionAdvisor(RegAllocEvictionAdvisor &&) = delete;
+  virtual ~RegAllocEvictionAdvisor() = default;
+
+  /// Find a physical register that can be freed by evicting the FixedRegisters,
+  /// or return NoRegister. The eviction decision is assumed to be correct (i.e.
+  /// no fixed live ranges are evicted) and profitable.
+  virtual MCRegister
+  tryFindEvictionCandidate(LiveInterval &VirtReg, const AllocationOrder &Order,
+                           uint8_t CostPerUseLimit,
+                           const SmallVirtRegSet &FixedRegisters) const = 0;
+
+  /// Find out if we can evict the live ranges occupying the given PhysReg,
+  /// which is a hint (preferred register) for VirtReg.
+  virtual bool
+  canEvictHintInterference(LiveInterval &VirtReg, MCRegister PhysReg,
+                           const SmallVirtRegSet &FixedRegisters) const = 0;
+
+  /// Returns true if the given \p PhysReg is a callee saved register and has
+  /// not been used for allocation yet.
+  bool isUnusedCalleeSavedReg(MCRegister PhysReg) const;
+
+protected:
+  RegAllocEvictionAdvisor(const MachineFunction &MF, LiveRegMatrix *Matrix,
+                          LiveIntervals *LIS, VirtRegMap *VRM,
+                          const RegisterClassInfo &RegClassInfo,
+                          ExtraRegInfo *ExtraInfo);
+
+  Register canReassign(LiveInterval &VirtReg, Register PrevReg) const;
+
+  const MachineFunction &MF;
+  LiveRegMatrix *const Matrix;
+  LiveIntervals *const LIS;
+  VirtRegMap *const VRM;
+  MachineRegisterInfo *const MRI;
+  const TargetRegisterInfo *const TRI;
+  const RegisterClassInfo &RegClassInfo;
+  const ArrayRef<uint8_t> RegCosts;
+  ExtraRegInfo *const ExtraInfo;
+
+  /// Run or not the local reassignment heuristic. This information is
+  /// obtained from the TargetSubtargetInfo.
+  const bool EnableLocalReassign;
+
+private:
+  unsigned NextCascade = 1;
+};
+
+/// ImmutableAnalysis abstraction for fetching the Eviction Advisor. We model it
+/// as an analysis to decouple the user from the implementation insofar as
+/// dependencies on other analyses goes. The motivation for it being an
+/// immutable pass is twofold:
+/// - in the ML implementation case, the evaluator is stateless but (especially
+/// in the development mode) expensive to set up. With an immutable pass, we set
+/// it up once.
+/// - in the 'development' mode ML case, we want to capture the training log
+/// during allocation (this is a log of features encountered and decisions
+/// made), and then measure a score, potentially a few steps after allocation
+/// completes. So we need the properties of an immutable pass to keep the logger
+/// state around until we can make that measurement.
+///
+/// Because we need to offer additional services in 'development' mode, the
+/// implementations of this analysis need to implement RTTI support.
+class RegAllocEvictionAdvisorAnalysis : public ImmutablePass {
+public:
+  enum class AdvisorMode : int { Default, Release, Development };
+
+  RegAllocEvictionAdvisorAnalysis(AdvisorMode Mode)
+      : ImmutablePass(ID), Mode(Mode){};
+  static char ID;
+
+  /// Get an advisor for the given context (i.e. machine function, etc)
+  virtual std::unique_ptr<RegAllocEvictionAdvisor>
+  getAdvisor(const MachineFunction &MF, LiveRegMatrix *Matrix,
+             LiveIntervals *LIS, VirtRegMap *VRM,
+             const RegisterClassInfo &RegClassInfo,
+             ExtraRegInfo *ExtraInfo) = 0;
+  AdvisorMode getAdvisorMode() const { return Mode; }
+
+private:
+  // This analysis preserves everything, and subclasses may have additional
+  // requirements.
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesAll();
+  }
+
+  StringRef getPassName() const override;
+  const AdvisorMode Mode;
+};
+
+/// Specialization for the API used by the analysis infrastructure to create
+/// an instance of the eviction advisor.
+template <> Pass *callDefaultCtor<RegAllocEvictionAdvisorAnalysis>();
+
+// TODO(mtrofin): implement these.
+#ifdef LLVM_HAVE_TF_AOT
+RegAllocEvictionAdvisorAnalysis *createReleaseModeAdvisor();
+#endif
+
+#ifdef LLVM_HAVE_TF_API
+RegAllocEvictionAdvisorAnalysis *createDevelopmentModeAdvisor();
+#endif
+
+// TODO: move to RegAllocEvictionAdvisor.cpp when we move implementation
+// out of RegAllocGreedy.cpp
+class DefaultEvictionAdvisor : public RegAllocEvictionAdvisor {
+public:
+  DefaultEvictionAdvisor(const MachineFunction &MF, LiveRegMatrix *Matrix,
+                         LiveIntervals *LIS, VirtRegMap *VRM,
+                         const RegisterClassInfo &RegClassInfo,
+                         ExtraRegInfo *ExtraInfo)
+      : RegAllocEvictionAdvisor(MF, Matrix, LIS, VRM, RegClassInfo, ExtraInfo) {
+  }
+
+private:
+  MCRegister tryFindEvictionCandidate(LiveInterval &, const AllocationOrder &,
+                                      uint8_t,
+                                      const SmallVirtRegSet &) const override;
+  bool canEvictHintInterference(LiveInterval &, MCRegister,
+                                const SmallVirtRegSet &) const override;
+  bool canEvictInterferenceBasedOnCost(LiveInterval &, MCRegister, bool,
+                                       EvictionCost &,
+                                       const SmallVirtRegSet &) const;
+  bool shouldEvict(LiveInterval &A, bool, LiveInterval &B, bool) const;
+};
 } // namespace llvm
 
 #endif // LLVM_CODEGEN_REGALLOCEVICTIONADVISOR_H
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.cpp b/llvm/lib/CodeGen/RegAllocGreedy.cpp
index 50411c177007..ce3cf31dbd6b 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.cpp
+++ b/llvm/lib/CodeGen/RegAllocGreedy.cpp
@@ -112,12 +112,6 @@ static cl::opt<bool> ExhaustiveSearch(
              "and interference cutoffs of last chance recoloring"),
     cl::Hidden);
 
-static cl::opt<bool> EnableLocalReassignment(
-    "enable-local-reassign", cl::Hidden,
-    cl::desc("Local reassignment can yield better allocation decisions, but "
-             "may be compile time intensive"),
-    cl::init(false));
-
 static cl::opt<bool> EnableDeferredSpilling(
     "enable-deferred-spilling", cl::Hidden,
     cl::desc("Instead of spilling a variable right away, defer the actual "
@@ -172,8 +166,9 @@ class RAGreedy : public MachineFunctionPass,
   // state
   std::unique_ptr<Spiller> SpillerInstance;
   PQueue Queue;
-  unsigned NextCascade;
   std::unique_ptr<VirtRegAuxInfo> VRAI;
+  Optional<ExtraRegInfo> ExtraInfo;
+  std::unique_ptr<RegAllocEvictionAdvisor> EvictAdvisor;
 
   // Enum CutOffStage to keep a track whether the register allocation failed
   // because of the cutoffs encountered in last chance recoloring.
@@ -195,76 +190,6 @@ class RAGreedy : public MachineFunctionPass,
   static const char *const StageName[];
 #endif
 
-  // RegInfo - Keep additional information about each live range.
-  struct RegInfo {
-    LiveRangeStage Stage = RS_New;
-
-    // Cascade - Eviction loop prevention. See
-    // canEvictInterferenceBasedOnCost().
-    unsigned Cascade = 0;
-
-    RegInfo() = default;
-  };
-
-  IndexedMap<RegInfo, VirtReg2IndexFunctor> ExtraRegInfo;
-
-  LiveRangeStage getStage(Register Reg) const {
-    return ExtraRegInfo[Reg].Stage;
-  }
-
-  LiveRangeStage getStage(const LiveInterval &VirtReg) const {
-    return getStage(VirtReg.reg());
-  }
-
-  void setStage(Register Reg, LiveRangeStage Stage) {
-    ExtraRegInfo.resize(MRI->getNumVirtRegs());
-    ExtraRegInfo[Reg].Stage = Stage;
-  }
-
-  void setStage(const LiveInterval &VirtReg, LiveRangeStage Stage) {
-    setStage(VirtReg.reg(), Stage);
-  }
-
-  /// Return the current stage of the register, if present, otherwise initialize
-  /// it and return that.
-  LiveRangeStage getOrInitStage(Register Reg) {
-    ExtraRegInfo.grow(Reg);
-    return getStage(Reg);
-  }
-
-  unsigned getCascade(Register Reg) const { return ExtraRegInfo[Reg].Cascade; }
-
-  void setCascade(Register Reg, unsigned Cascade) {
-    ExtraRegInfo.resize(MRI->getNumVirtRegs());
-    ExtraRegInfo[Reg].Cascade = Cascade;
-  }
-
-  unsigned getOrAssignNewCascade(Register Reg) {
-    unsigned Cascade = getCascade(Reg);
-    if (!Cascade) {
-      Cascade = NextCascade++;
-      setCascade(Reg, Cascade);
-    }
-    return Cascade;
-  }
-
-  unsigned getCascadeOrCurrentNext(Register Reg) const {
-    unsigned Cascade = getCascade(Reg);
-    if (!Cascade)
-      Cascade = NextCascade;
-    return Cascade;
-  }
-
-  template<typename Iterator>
-  void setStage(Iterator Begin, Iterator End, LiveRangeStage NewStage) {
-    ExtraRegInfo.resize(MRI->getNumVirtRegs());
-    for (;Begin != End; ++Begin) {
-      Register Reg = *Begin;
-      if (ExtraRegInfo[Reg].Stage == RS_New)
-        ExtraRegInfo[Reg].Stage = NewStage;
-    }
-  }
-
   /// EvictionTrack - Keeps track of past evictions in order to optimize region
   /// split decision.
   class EvictionTrack {
@@ -375,10 +300,6 @@ class RAGreedy : public MachineFunctionPass,
   /// Callee-save register cost, calculated once per machine function.
   BlockFrequency CSRCost;
 
-  /// Run or not the local reassignment heuristic. This information is
-  /// obtained from the TargetSubtargetInfo.
-  bool EnableLocalReassign;
-
   /// Enable or not the consideration of the cost of local intervals created
   /// by a split candidate when choosing the best split candidate.
   bool EnableAdvancedRASplitCost;
@@ -447,13 +368,6 @@ private:
   bool calcCompactRegion(GlobalSplitCandidate&);
   void splitAroundRegion(LiveRangeEdit&, ArrayRef<unsigned>);
   void calcGapWeights(MCRegister, SmallVectorImpl<float> &);
-  Register canReassign(LiveInterval &VirtReg, Register PrevReg) const;
-  bool shouldEvict(LiveInterval &A, bool, LiveInterval &B, bool) const;
-  bool canEvictInterferenceBasedOnCost(LiveInterval &, MCRegister, bool,
-                                       EvictionCost &,
-                                       const SmallVirtRegSet &) const;
-  bool canEvictHintInterference(LiveInterval &, MCRegister,
-                                const SmallVirtRegSet &) const;
   bool canEvictInterferenceInRange(const LiveInterval &VirtReg,
                                    MCRegister PhysReg, SlotIndex Start,
                                    SlotIndex End, EvictionCost &MaxCost) const;
@@ -529,8 +443,6 @@ private:
   BlockFrequency getBrokenHintFreq(const HintsInfo &, MCRegister);
   void collectHintInfo(Register, HintsInfo &);
 
-  bool isUnusedCalleeSavedReg(MCRegister PhysReg) const;
-
   /// Greedy RA statistic to remark.
   struct RAGreedyStats {
     unsigned Reloads = 0;
@@ -597,6 +509,7 @@ INITIALIZE_PASS_DEPENDENCY(LiveRegMatrix)
 INITIALIZE_PASS_DEPENDENCY(EdgeBundles)
 INITIALIZE_PASS_DEPENDENCY(SpillPlacement)
 INITIALIZE_PASS_DEPENDENCY(MachineOptimizationRemarkEmitterPass)
+INITIALIZE_PASS_DEPENDENCY(RegAllocEvictionAdvisorAnalysis)
 INITIALIZE_PASS_END(RAGreedy, "greedy",
                 "Greedy Register Allocator", false, false)
 
@@ -663,6 +576,7 @@ void RAGreedy::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.addRequired<EdgeBundles>();
   AU.addRequired<SpillPlacement>();
   AU.addRequired<MachineOptimizationRemarkEmitterPass>();
+  AU.addRequired<RegAllocEvictionAdvisorAnalysis>();
   MachineFunctionPass::getAnalysisUsage(AU);
 }
 
@@ -696,22 +610,25 @@ void RAGreedy::LRE_WillShrinkVirtReg(Register VirtReg) {
 }
 
 void RAGreedy::LRE_DidCloneVirtReg(Register New, Register Old) {
+  ExtraInfo->LRE_DidCloneVirtReg(New, Old);
+}
+
+void ExtraRegInfo::LRE_DidCloneVirtReg(Register New, Register Old) {
   // Cloning a register we haven't even heard about yet?  Just ignore it.
-  if (!ExtraRegInfo.inBounds(Old))
+  if (!Info.inBounds(Old))
     return;
 
   // LRE may clone a virtual register because dead code elimination causes it to
   // be split into connected components. The new components are much smaller
   // than the original, so they should get a new chance at being assigned.
   // same stage as the parent.
-  ExtraRegInfo[Old].Stage = RS_Assign;
-  ExtraRegInfo.grow(New);
-  ExtraRegInfo[New] = ExtraRegInfo[Old];
+  Info[Old].Stage = RS_Assign;
+  Info.grow(New.id());
+  Info[New] = Info[Old];
 }
 
 void RAGreedy::releaseMemory() {
   SpillerInstance.reset();
-  ExtraRegInfo.clear();
   GlobalCand.clear();
 }
 
@@ -725,10 +642,10 @@ void RAGreedy::enqueue(PQueue &CurQueue, LiveInterval *LI) {
   assert(Reg.isVirtual() && "Can only enqueue virtual registers");
   unsigned Prio;
 
-  auto Stage = getOrInitStage(Reg);
+  auto Stage = ExtraInfo->getOrInitStage(Reg);
   if (Stage == RS_New) {
     Stage = RS_Assign;
-    setStage(Reg, Stage);
+    ExtraInfo->setStage(Reg, Stage);
   }
   if (Stage == RS_Split) {
     // Unsplit ranges that couldn't be allocated immediately are deferred until
@@ -824,7 +741,8 @@ MCRegister RAGreedy::tryAssign(LiveInterval &VirtReg,
       MCRegister PhysHint = Hint.asMCReg();
       LLVM_DEBUG(dbgs() << "missed hint " << printReg(PhysHint, TRI) << '\n');
 
-      if (canEvictHintInterference(VirtReg, PhysHint, FixedRegisters)) {
+      if (EvictAdvisor->canEvictHintInterference(VirtReg, PhysHint,
+                                                 FixedRegisters)) {
         evictInterference(VirtReg, PhysHint, NewVRegs);
         return PhysHint;
       }
@@ -850,7 +768,8 @@ MCRegister RAGreedy::tryAssign(LiveInterval &VirtReg,
 //                         Interference eviction
 //===----------------------------------------------------------------------===//
 
-Register RAGreedy::canReassign(LiveInterval &VirtReg, Register PrevReg) const {
+Register RegAllocEvictionAdvisor::canReassign(LiveInterval &VirtReg,
+                                              Register PrevReg) const {
   auto Order =
       AllocationOrder::create(VirtReg.reg(), *VRM, RegClassInfo, Matrix);
   MCRegister PhysReg;
@@ -889,9 +808,10 @@ Register RAGreedy::canReassign(LiveInterval &VirtReg, Register PrevReg) const {
 ///                   register.
 /// @param B          The live range to be evicted.
 /// @param BreaksHint True when B is already assigned to its preferred register.
-bool RAGreedy::shouldEvict(LiveInterval &A, bool IsHint,
-                           LiveInterval &B, bool BreaksHint) const {
-  bool CanSplit = getStage(B) < RS_Spill;
+bool DefaultEvictionAdvisor::shouldEvict(LiveInterval &A, bool IsHint,
+                                         LiveInterval &B,
+                                         bool BreaksHint) const {
+  bool CanSplit = ExtraInfo->getStage(B) < RS_Spill;
 
   // Be fairly aggressive about following hints as long as the evictee can be
   // split.
@@ -907,7 +827,7 @@ bool RAGreedy::shouldEvict(LiveInterval &A, bool IsHint,
 
 /// canEvictHintInterference - return true if the interference for VirtReg
 /// on the PhysReg, which is VirtReg's hint, can be evicted in favor of VirtReg.
-bool RAGreedy::canEvictHintInterference(
+bool DefaultEvictionAdvisor::canEvictHintInterference(
     LiveInterval &VirtReg, MCRegister PhysReg,
     const SmallVirtRegSet &FixedRegisters) const {
   EvictionCost MaxCost;
@@ -925,7 +845,7 @@ bool RAGreedy::canEvictHintInterference(
 /// @param MaxCost Only look for cheaper candidates and update with new cost
 ///                when returning true.
 /// @returns True when interference can be evicted cheaper than MaxCost.
-bool RAGreedy::canEvictInterferenceBasedOnCost(
+bool DefaultEvictionAdvisor::canEvictInterferenceBasedOnCost(
     LiveInterval &VirtReg, MCRegister PhysReg, bool IsHint,
     EvictionCost &MaxCost, const SmallVirtRegSet &FixedRegisters) const {
   // It is only possible to evict virtual register interference.
@@ -941,9 +861,7 @@ bool RAGreedy::canEvictInterferenceBasedOnCost(
   //
   // This works out so a register without a cascade number is allowed to evict
   // anything, and it can be evicted by anything.
-  unsigned Cascade = ExtraRegInfo[VirtReg.reg()].Cascade;
-  if (!Cascade)
-    Cascade = NextCascade;
+  unsigned Cascade = ExtraInfo->getCascadeOrCurrentNext(VirtReg.reg());
 
   EvictionCost Cost;
   for (MCRegUnitIterator Units(PhysReg, TRI); Units.isValid(); ++Units) {
@@ -965,7 +883,7 @@ bool RAGreedy::canEvictInterferenceBasedOnCost(
         return false;
 
       // Never evict spill products. They cannot split or spill.
-      if (getStage(*Intf) == RS_Done)
+      if (ExtraInfo->getStage(*Intf) == RS_Done)
         return false;
       // Once a live range becomes small enough, it is urgent that we find a
       // register for it. This is indicated by an infinite spill weight. These
@@ -980,7 +898,7 @@ bool RAGreedy::canEvictInterferenceBasedOnCost(
                RegClassInfo.getNumAllocatableRegs(
                    MRI->getRegClass(Intf->reg())));
       // Only evict older cascades or live ranges without a cascade.
-      unsigned IntfCascade = ExtraRegInfo[Intf->reg()].Cascade;
+      unsigned IntfCascade = ExtraInfo->getCascade(Intf->reg());
       if (Cascade <= IntfCascade) {
         if (!Urgent)
           return false;
@@ -1043,7 +961,7 @@ bool RAGreedy::canEvictInterferenceInRange(const LiveInterval &VirtReg,
       if (!Register::isVirtualRegister(Intf->reg()))
         return false;
       // Never evict spill products. They cannot split or spill.
-      if (getStage(*Intf) == RS_Done)
+      if (ExtraInfo->getStage(*Intf) == RS_Done)
         return false;
 
       // Would this break a satisfied hint?
@@ -1106,7 +1024,7 @@ void RAGreedy::evictInterference(LiveInterval &VirtReg, MCRegister PhysReg,
   // Make sure that VirtReg has a cascade number, and assign that cascade
   // number to every evicted register. These live ranges than then only be
   // evicted by a newer cascade, preventing infinite loops.
-  unsigned Cascade = getOrAssignNewCascade(VirtReg.reg());
+  unsigned Cascade = ExtraInfo->getOrAssignNewCascade(VirtReg.reg());
 
   LLVM_DEBUG(dbgs() << "evicting " << printReg(PhysReg, TRI)
                     << " interference: Cascade " << Cascade << '\n');
@@ -1132,10 +1050,10 @@ void RAGreedy::evictInterference(LiveInterval &VirtReg, MCRegister PhysReg,
     LastEvicted.addEviction(PhysReg, VirtReg.reg(), Intf->reg());
 
     Matrix->unassign(*Intf);
-    assert((getCascade(Intf->reg()) < Cascade ||
+    assert((ExtraInfo->getCascade(Intf->reg()) < Cascade ||
             VirtReg.isSpillable() < Intf->isSpillable()) &&
            "Cannot decrease cascade number, illegal eviction");
-    setCascade(Intf->reg(), Cascade);
+    ExtraInfo->setCascade(Intf->reg(), Cascade);
     ++NumEvicted;
     NewVRegs.push_back(Intf->reg());
   }
@@ -1143,7 +1061,7 @@ void RAGreedy::evictInterference(LiveInterval &VirtReg, MCRegister PhysReg,
 
 /// Returns true if the given \p PhysReg is a callee saved register and has not
 /// been used for allocation yet.
-bool RAGreedy::isUnusedCalleeSavedReg(MCRegister PhysReg) const {
+bool RegAllocEvictionAdvisor::isUnusedCalleeSavedReg(MCRegister PhysReg) const {
   MCRegister CSR = RegClassInfo.getLastCalleeSavedAlias(PhysReg);
   if (!CSR)
     return false;
@@ -1151,7 +1069,7 @@ bool RAGreedy::isUnusedCalleeSavedReg(MCRegister PhysReg) const {
   return !Matrix->isPhysRegUsed(PhysReg);
 }
 
-MCRegister RAGreedy::tryFindEvictionCandidate(
+MCRegister DefaultEvictionAdvisor::tryFindEvictionCandidate(
     LiveInterval &VirtReg, const AllocationOrder &Order,
     uint8_t CostPerUseLimit, const SmallVirtRegSet &FixedRegisters) const {
   // Keep track of the cheapest interference seen so far.
@@ -1225,8 +1143,8 @@ MCRegister RAGreedy::tryEvict(LiveInterval &VirtReg, AllocationOrder &Order,
   NamedRegionTimer T("evict", "Evict", TimerGroupName, TimerGroupDescription,
                      TimePassesIsEnabled);
 
-  MCRegister BestPhys =
-      tryFindEvictionCandidate(VirtReg, Order, CostPerUseLimit, FixedRegisters);
+  MCRegister BestPhys = EvictAdvisor->tryFindEvictionCandidate(
+      VirtReg, Order, CostPerUseLimit, FixedRegisters);
   if (BestPhys.isValid())
     evictInterference(VirtReg, BestPhys, NewVRegs);
   return BestPhys;
@@ -1769,8 +1687,8 @@ void RAGreedy::splitAroundRegion(LiveRangeEdit &LREdit,
   // the ActiveBlocks list with each candidate. We need to filter out
   // duplicates.
   BitVector Todo = SA->getThroughBlocks();
-  for (unsigned c = 0; c != UsedCands.size(); ++c) {
-    ArrayRef<unsigned> Blocks = GlobalCand[UsedCands[c]].ActiveBlocks;
+  for (unsigned UsedCand : UsedCands) {
+    ArrayRef<unsigned> Blocks = GlobalCand[UsedCand].ActiveBlocks;
     for (unsigned Number : Blocks) {
       if (!Todo.test(Number))
         continue;
@@ -1817,13 +1735,13 @@ void RAGreedy::splitAroundRegion(LiveRangeEdit &LREdit,
     const LiveInterval &Reg = LIS->getInterval(LREdit.get(I));
 
     // Ignore old intervals from DCE.
-    if (getOrInitStage(Reg.reg()) != RS_New)
+    if (ExtraInfo->getOrInitStage(Reg.reg()) != RS_New)
       continue;
 
     // Remainder interval. Don't try splitting again, spill if it doesn't
     // allocate.
     if (IntvMap[I] == 0) {
-      setStage(Reg, RS_Spill);
+      ExtraInfo->setStage(Reg, RS_Spill);
       continue;
     }
 
@@ -1834,7 +1752,7 @@ void RAGreedy::splitAroundRegion(LiveRangeEdit &LREdit,
         LLVM_DEBUG(dbgs() << "Main interval covers the same " << OrigBlocks
                           << " blocks as original.\n");
         // Don't allow repeated splitting as a safe guard against looping.
-        setStage(Reg, RS_Split2);
+        ExtraInfo->setStage(Reg, RS_Split2);
       }
       continue;
     }
@@ -1899,7 +1817,7 @@ unsigned RAGreedy::calculateRegionSplitCost(LiveInterval &VirtReg,
   unsigned BestCand = NoCand;
   for (MCPhysReg PhysReg : Order) {
     assert(PhysReg);
-    if (IgnoreCSR && isUnusedCalleeSavedReg(PhysReg))
+    if (IgnoreCSR && EvictAdvisor->isUnusedCalleeSavedReg(PhysReg))
       continue;
 
     // Discard bad candidates before we run out of interference cache cursors.
@@ -2065,8 +1983,8 @@ unsigned RAGreedy::tryBlockSplit(LiveInterval &VirtReg, AllocationOrder &Order,
   // goes straight to spilling, the new local ranges get to stay RS_New.
   for (unsigned I = 0, E = LREdit.size(); I != E; ++I) {
     const LiveInterval &LI = LIS->getInterval(LREdit.get(I));
-    if (getOrInitStage(LI.reg()) == RS_New && IntvMap[I] == 0)
-      setStage(LI, RS_Spill);
+    if (ExtraInfo->getOrInitStage(LI.reg()) == RS_New && IntvMap[I] == 0)
+      ExtraInfo->setStage(LI, RS_Spill);
   }
 
   if (VerifyEnabled)
@@ -2152,7 +2070,7 @@ RAGreedy::tryInstructionSplit(LiveInterval &VirtReg, AllocationOrder &Order,
   SE->finish(&IntvMap);
   DebugVars->splitRegister(VirtReg.reg(), LREdit.regs(), *LIS);
   // Assign all new registers to RS_Spill. This was the last chance.
-  setStage(LREdit.begin(), LREdit.end(), RS_Spill);
+  ExtraInfo->setStage(LREdit.begin(), LREdit.end(), RS_Spill);
   return 0;
 }
 
@@ -2320,7 +2238,7 @@ unsigned RAGreedy::tryLocalSplit(LiveInterval &VirtReg, AllocationOrder &Order,
   // These rules allow a 3 -> 2+3 split once, which we need. They also prevent
   // excessive splitting and infinite loops.
   //
-  bool ProgressRequired = getStage(VirtReg) >= RS_Split2;
+  bool ProgressRequired = ExtraInfo->getStage(VirtReg) >= RS_Split2;
 
   // Best split candidate.
   unsigned BestBefore = NumGaps;
@@ -2456,7 +2374,7 @@ unsigned RAGreedy::tryLocalSplit(LiveInterval &VirtReg, AllocationOrder &Order,
     assert(!ProgressRequired && "Didn't make progress when it was required.");
     for (unsigned I = 0, E = IntvMap.size(); I != E; ++I)
       if (IntvMap[I] == 1) {
-        setStage(LIS->getInterval(LREdit.get(I)), RS_Split2);
+        ExtraInfo->setStage(LIS->getInterval(LREdit.get(I)), RS_Split2);
         LLVM_DEBUG(dbgs() << ' ' << printReg(LREdit.get(I)));
       }
     LLVM_DEBUG(dbgs() << '\n');
@@ -2477,7 +2395,7 @@ unsigned RAGreedy::trySplit(LiveInterval &VirtReg, AllocationOrder &Order,
                             SmallVectorImpl<Register> &NewVRegs,
                             const SmallVirtRegSet &FixedRegisters) {
   // Ranges must be Split2 or less.
-  if (getStage(VirtReg) >= RS_Spill)
+  if (ExtraInfo->getStage(VirtReg) >= RS_Spill)
     return 0;
 
   // Local intervals are handled separately.
@@ -2499,7 +2417,7 @@ unsigned RAGreedy::trySplit(LiveInterval &VirtReg, AllocationOrder &Order,
   // First try to split around a region spanning multiple blocks. RS_Split2
   // ranges already made dubious progress with region splitting, so they go
   // straight to single block splitting.
-  if (getStage(VirtReg) < RS_Split2) {
+  if (ExtraInfo->getStage(VirtReg) < RS_Split2) {
     MCRegister PhysReg = tryRegionSplit(VirtReg, Order, NewVRegs);
     if (PhysReg || !NewVRegs.empty())
       return PhysReg;
@@ -2551,7 +2469,7 @@ bool RAGreedy::mayRecolorAllInterferences(
       // it would not be recolorable as it is in the same state as VirtReg.
       // However, if VirtReg has tied defs and Intf doesn't, then
       // there is still a point in examining if it can be recolorable.
-      if (((getStage(*Intf) == RS_Done &&
+      if (((ExtraInfo->getStage(*Intf) == RS_Done &&
             MRI->getRegClass(Intf->reg()) == CurRC) &&
            !(hasTiedDef(MRI, VirtReg.reg()) &&
              !hasTiedDef(MRI, Intf->reg()))) ||
@@ -2615,7 +2533,7 @@ unsigned RAGreedy::tryLastChanceRecoloring(LiveInterval &VirtReg,
 
   LLVM_DEBUG(dbgs() << "Try last chance recoloring for " << VirtReg << '\n');
   // Ranges must be Done.
-  assert((getStage(VirtReg) >= RS_Done || !VirtReg.isSpillable()) &&
+  assert((ExtraInfo->getStage(VirtReg) >= RS_Done || !VirtReg.isSpillable()) &&
          "Last chance recoloring should really be last chance");
   // Set the max depth to LastChanceRecoloringMaxDepth.
   // We may want to reconsider that if we end up with a too large search space
@@ -2806,7 +2724,7 @@ MCRegister
 RAGreedy::tryAssignCSRFirstTime(LiveInterval &VirtReg, AllocationOrder &Order,
                                 MCRegister PhysReg, uint8_t &CostPerUseLimit,
                                 SmallVectorImpl<Register> &NewVRegs) {
-  if (getStage(VirtReg) == RS_Spill && VirtReg.isSpillable()) {
+  if (ExtraInfo->getStage(VirtReg) == RS_Spill && VirtReg.isSpillable()) {
     // We choose spill over using the CSR for the first time if the spill cost
     // is lower than CSRCost.
     SA->analyze(&VirtReg);
@@ -2818,7 +2736,7 @@ RAGreedy::tryAssignCSRFirstTime(LiveInterval &VirtReg, AllocationOrder &Order,
     CostPerUseLimit = 1;
     return 0;
   }
-  if (getStage(VirtReg) < RS_Split) {
+  if (ExtraInfo->getStage(VirtReg) < RS_Split) {
     // We choose pre-splitting over using the CSR for the first time if
     // the cost of splitting is lower than CSRCost.
     SA->analyze(&VirtReg);
@@ -3051,8 +2969,8 @@ MCRegister RAGreedy::selectOrSplitImpl(LiveInterval &VirtReg,
     // When NewVRegs is not empty, we may have made decisions such as evicting
     // a virtual register, go with the earlier decisions and use the physical
     // register.
-    if (CSRCost.getFrequency() && isUnusedCalleeSavedReg(PhysReg) &&
-        NewVRegs.empty()) {
+    if (CSRCost.getFrequency() &&
+        EvictAdvisor->isUnusedCalleeSavedReg(PhysReg) && NewVRegs.empty()) {
       MCRegister CSRReg = tryAssignCSRFirstTime(VirtReg, Order, PhysReg,
                                                 CostPerUseLimit, NewVRegs);
       if (CSRReg || !NewVRegs.empty())
@@ -3063,9 +2981,9 @@ MCRegister RAGreedy::selectOrSplitImpl(LiveInterval &VirtReg,
       return PhysReg;
   }
 
-  LiveRangeStage Stage = getStage(VirtReg);
+  LiveRangeStage Stage = ExtraInfo->getStage(VirtReg);
   LLVM_DEBUG(dbgs() << StageName[Stage] << " Cascade "
-                    << getCascade(VirtReg.reg()) << '\n');
+                    << ExtraInfo->getCascade(VirtReg.reg()) << '\n');
 
   // Try to evict a less worthy live range, but only for ranges from the primary
   // queue. The RS_Split ranges already failed to do this, and they should not
@@ -3094,7 +3012,7 @@ MCRegister RAGreedy::selectOrSplitImpl(LiveInterval &VirtReg,
   // Wait until the second time, when all smaller ranges have been allocated.
   // This gives a better picture of the interference to split around.
   if (Stage < RS_Split) {
-    setStage(VirtReg, RS_Split);
+    ExtraInfo->setStage(VirtReg, RS_Split);
     LLVM_DEBUG(dbgs() << "wait for second round\n");
     NewVRegs.push_back(VirtReg.reg());
     return 0;
@@ -3120,12 +3038,12 @@ MCRegister RAGreedy::selectOrSplitImpl(LiveInterval &VirtReg,
   // Finally spill VirtReg itself.
   if ((EnableDeferredSpilling ||
        TRI->shouldUseDeferredSpillingForVirtReg(*MF, VirtReg)) &&
-      getStage(VirtReg) < RS_Memory) {
+      ExtraInfo->getStage(VirtReg) < RS_Memory) {
     // TODO: This is experimental and in particular, we do not model
     // the live range splitting done by spilling correctly.
     // We would need a deep integration with the spiller to do the
     // right thing here. Anyway, that is still good for early testing.
-    setStage(VirtReg, RS_Memory);
+    ExtraInfo->setStage(VirtReg, RS_Memory);
     LLVM_DEBUG(dbgs() << "Do as if this register is in memory\n");
     NewVRegs.push_back(VirtReg.reg());
   } else {
@@ -3133,7 +3051,7 @@ MCRegister RAGreedy::selectOrSplitImpl(LiveInterval &VirtReg,
                        TimerGroupDescription, TimePassesIsEnabled);
     LiveRangeEdit LRE(&VirtReg, NewVRegs, *MF, *LIS, VRM, this, &DeadRemats);
     spiller().spill(LRE);
-    setStage(NewVRegs.begin(), NewVRegs.end(), RS_Done);
+    ExtraInfo->setStage(NewVRegs.begin(), NewVRegs.end(), RS_Done);
 
     // Tell LiveDebugVariables about the new ranges. Ranges not being covered by
     // the new regs are kept in LDV (still mapping to the old register), until
@@ -3316,10 +3234,6 @@ bool RAGreedy::runOnMachineFunction(MachineFunction &mf) {
   TII = MF->getSubtarget().getInstrInfo();
   RCI.runOnMachineFunction(mf);
 
-  EnableLocalReassign = EnableLocalReassignment ||
-                        MF->getSubtarget().enableRALocalReassignment(
-                            MF->getTarget().getOptLevel());
-
   EnableAdvancedRASplitCost =
       ConsiderLocalIntervalCost.getNumOccurrences()
           ? ConsiderLocalIntervalCost
@@ -3354,8 +3268,9 @@ bool RAGreedy::runOnMachineFunction(MachineFunction &mf) {
 
   SA.reset(new SplitAnalysis(*VRM, *LIS, *Loops));
   SE.reset(new SplitEditor(*SA, *AA, *LIS, *VRM, *DomTree, *MBFI, *VRAI));
-  ExtraRegInfo.clear();
-  NextCascade = 1;
+  ExtraInfo.emplace();
+  EvictAdvisor = getAnalysis<RegAllocEvictionAdvisorAnalysis>().getAdvisor(
+      *MF, Matrix, LIS, VRM, RegClassInfo, &*ExtraInfo);
   IntfCache.init(MF, Matrix->getLiveUnions(), Indexes, LIS, TRI);
   GlobalCand.resize(32);  // This will grow as needed.
   SetOfBrokenHints.clear();
diff --git a/llvm/lib/CodeGen/RegAllocPBQP.cpp b/llvm/lib/CodeGen/RegAllocPBQP.cpp
index b22eb080791e..93be8f689d57 100644
--- a/llvm/lib/CodeGen/RegAllocPBQP.cpp
+++ b/llvm/lib/CodeGen/RegAllocPBQP.cpp
@@ -623,8 +623,8 @@ void RegAllocPBQP::initializeGraph(PBQPRAGraph &G, VirtRegMap &VRM,
     // Compute an initial allowed set for the current vreg.
     std::vector<MCRegister> VRegAllowed;
     ArrayRef<MCPhysReg> RawPRegOrder = TRC->getRawAllocationOrder(MF);
-    for (unsigned I = 0; I != RawPRegOrder.size(); ++I) {
-      MCRegister PReg(RawPRegOrder[I]);
+    for (MCPhysReg R : RawPRegOrder) {
+      MCRegister PReg(R);
       if (MRI.isReserved(PReg))
         continue;
 
diff --git a/llvm/lib/CodeGen/RegAllocScore.cpp b/llvm/lib/CodeGen/RegAllocScore.cpp
new file mode 100644
index 000000000000..740890831617
--- /dev/null
+++ b/llvm/lib/CodeGen/RegAllocScore.cpp
@@ -0,0 +1,124 @@
+//===- RegAllocScore.cpp - evaluate regalloc policy quality ---------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// Calculate a measure of the register allocation policy quality. This is used
+/// to construct a reward for the training of the ML-driven allocation policy.
+/// Currently, the score is the sum of the machine basic block frequency-weighed
+/// number of loads, stores, copies, and remat instructions, each factored with
+/// a relative weight.
+//===----------------------------------------------------------------------===//
+
+#include "RegAllocScore.h"
+#include "llvm/ADT/SetVector.h"
+#include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/TargetInstrInfo.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/Format.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Target/TargetMachine.h"
+#include <cassert>
+#include <cstdint>
+#include <numeric>
+#include <vector>
+
+using namespace llvm;
+cl::opt<double> CopyWeight("regalloc-copy-weight", cl::init(0.2), cl::Hidden);
+cl::opt<double> LoadWeight("regalloc-load-weight", cl::init(4.0), cl::Hidden);
+cl::opt<double> StoreWeight("regalloc-store-weight", cl::init(1.0), cl::Hidden);
+cl::opt<double> CheapRematWeight("regalloc-cheap-remat-weight", cl::init(0.2),
+                                 cl::Hidden);
+cl::opt<double> ExpensiveRematWeight("regalloc-expensive-remat-weight",
+                                     cl::init(1.0), cl::Hidden);
+#define DEBUG_TYPE "regalloc-score"
+
+RegAllocScore &RegAllocScore::operator+=(const RegAllocScore &Other) {
+  CopyCounts += Other.copyCounts();
+  LoadCounts += Other.loadCounts();
+  StoreCounts += Other.storeCounts();
+  LoadStoreCounts += Other.loadStoreCounts();
+  CheapRematCounts += Other.cheapRematCounts();
+  ExpensiveRematCounts += Other.expensiveRematCounts();
+  return *this;
+}
+
+bool RegAllocScore::operator==(const RegAllocScore &Other) const {
+  return copyCounts() == Other.copyCounts() &&
+         loadCounts() == Other.loadCounts() &&
+         storeCounts() == Other.storeCounts() &&
+         loadStoreCounts() == Other.loadStoreCounts() &&
+         cheapRematCounts() == Other.cheapRematCounts() &&
+         expensiveRematCounts() == Other.expensiveRematCounts();
+}
+
+bool RegAllocScore::operator!=(const RegAllocScore &Other) const {
+  return !(*this == Other);
+}
+
+double RegAllocScore::getScore() const {
+  double Ret = 0.0;
+  Ret += CopyWeight * copyCounts();
+  Ret += LoadWeight * loadCounts();
+  Ret += StoreWeight * storeCounts();
+  Ret += (LoadWeight + StoreWeight) * loadStoreCounts();
+  Ret += CheapRematWeight * cheapRematCounts();
+  Ret += ExpensiveRematWeight * expensiveRematCounts();
+
+  return Ret;
+}
+
+RegAllocScore
+llvm::calculateRegAllocScore(const MachineFunction &MF,
+                             const MachineBlockFrequencyInfo &MBFI,
+                             AAResults &AAResults) {
+  return calculateRegAllocScore(
+      MF,
+      [&](const MachineBasicBlock &MBB) {
+        return MBFI.getBlockFreqRelativeToEntryBlock(&MBB);
+      },
+      [&](const MachineInstr &MI) {
+        return MF.getSubtarget().getInstrInfo()->isTriviallyReMaterializable(
+            MI, &AAResults);
+      });
+}
+
+RegAllocScore llvm::calculateRegAllocScore(
+    const MachineFunction &MF,
+    llvm::function_ref<double(const MachineBasicBlock &)> GetBBFreq,
+    llvm::function_ref<bool(const MachineInstr &)>
+        IsTriviallyRematerializable) {
+  RegAllocScore Total;
+
+  for (const MachineBasicBlock &MBB : MF) {
+    double BlockFreqRelativeToEntrypoint = GetBBFreq(MBB);
+    RegAllocScore MBBScore;
+
+    for (const MachineInstr &MI : MBB) {
+      if (MI.isDebugInstr() || MI.isKill() || MI.isInlineAsm()) {
+        continue;
+      }
+      if (MI.isCopy()) {
+        MBBScore.onCopy(BlockFreqRelativeToEntrypoint);
+      } else if (IsTriviallyRematerializable(MI)) {
+        if (MI.getDesc().isAsCheapAsAMove()) {
+          MBBScore.onCheapRemat(BlockFreqRelativeToEntrypoint);
+        } else {
+          MBBScore.onExpensiveRemat(BlockFreqRelativeToEntrypoint);
+        }
+      } else if (MI.mayLoad() && MI.mayStore()) {
+        MBBScore.onLoadStore(BlockFreqRelativeToEntrypoint);
+      } else if (MI.mayLoad()) {
+        MBBScore.onLoad(BlockFreqRelativeToEntrypoint);
+      } else if (MI.mayStore()) {
+        MBBScore.onStore(BlockFreqRelativeToEntrypoint);
+      }
+    }
+    Total += MBBScore;
+  }
+  return Total;
+}
diff --git a/llvm/lib/CodeGen/RegAllocScore.h b/llvm/lib/CodeGen/RegAllocScore.h
new file mode 100644
index 000000000000..3c28bb61189d
--- /dev/null
+++ b/llvm/lib/CodeGen/RegAllocScore.h
@@ -0,0 +1,80 @@
+//==- RegAllocScore.h - evaluate regalloc policy quality  ----------*-C++-*-==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// Calculate a measure of the register allocation policy quality. This is used
+/// to construct a reward for the training of the ML-driven allocation policy.
+/// Currently, the score is the sum of the machine basic block frequency-weighed
+/// number of loads, stores, copies, and remat instructions, each factored with
+/// a relative weight.
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_REGALLOCSCORE_H_
+#define LLVM_CODEGEN_REGALLOCSCORE_H_
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SetVector.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/Analysis/ProfileSummaryInfo.h"
+#include "llvm/Analysis/Utils/TFUtils.h"
+#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/SelectionDAGNodes.h"
+#include "llvm/IR/Module.h"
+#include <cassert>
+#include <cstdint>
+#include <limits>
+
+namespace llvm {
+
+/// Regalloc score.
+class RegAllocScore final {
+  double CopyCounts = 0.0;
+  double LoadCounts = 0.0;
+  double StoreCounts = 0.0;
+  double CheapRematCounts = 0.0;
+  double LoadStoreCounts = 0.0;
+  double ExpensiveRematCounts = 0.0;
+
+public:
+  RegAllocScore() = default;
+  RegAllocScore(const RegAllocScore &) = default;
+
+  double copyCounts() const { return CopyCounts; }
+  double loadCounts() const { return LoadCounts; }
+  double storeCounts() const { return StoreCounts; }
+  double loadStoreCounts() const { return LoadStoreCounts; }
+  double expensiveRematCounts() const { return ExpensiveRematCounts; }
+  double cheapRematCounts() const { return CheapRematCounts; }
+
+  void onCopy(double Freq) { CopyCounts += Freq; }
+  void onLoad(double Freq) { LoadCounts += Freq; }
+  void onStore(double Freq) { StoreCounts += Freq; }
+  void onLoadStore(double Freq) { LoadStoreCounts += Freq; }
+  void onExpensiveRemat(double Freq) { ExpensiveRematCounts += Freq; }
+  void onCheapRemat(double Freq) { CheapRematCounts += Freq; }
+
+  RegAllocScore &operator+=(const RegAllocScore &Other);
+  bool operator==(const RegAllocScore &Other) const;
+  bool operator!=(const RegAllocScore &Other) const;
+  double getScore() const;
+};
+
+/// Calculate a score. When comparing 2 scores for the same function but
+/// different policies, the better policy would have a smaller score.
+/// The implementation is the overload below (which is also easily unittestable)
+RegAllocScore calculateRegAllocScore(const MachineFunction &MF,
+                                     const MachineBlockFrequencyInfo &MBFI,
+                                     AAResults &AAResults);
+
+/// Implementation of the above, which is also more easily unittestable.
+RegAllocScore calculateRegAllocScore(
+    const MachineFunction &MF,
+    llvm::function_ref<double(const MachineBasicBlock &)> GetBBFreq,
+    llvm::function_ref<bool(const MachineInstr &)> IsTriviallyRematerializable);
+} // end namespace llvm
+
+#endif // LLVM_CODEGEN_REGALLOCSCORE_H_
diff --git a/llvm/lib/CodeGen/RegisterClassInfo.cpp b/llvm/lib/CodeGen/RegisterClassInfo.cpp
index 797899fb5b86..65a65b9cae95 100644
--- a/llvm/lib/CodeGen/RegisterClassInfo.cpp
+++ b/llvm/lib/CodeGen/RegisterClassInfo.cpp
@@ -109,8 +109,7 @@ void RegisterClassInfo::compute(const TargetRegisterClass *RC) const {
   // FIXME: Once targets reserve registers instead of removing them from the
   // allocation order, we can simply use begin/end here.
   ArrayRef<MCPhysReg> RawOrder = RC->getRawAllocationOrder(*MF);
-  for (unsigned i = 0; i != RawOrder.size(); ++i) {
-    unsigned PhysReg = RawOrder[i];
+  for (unsigned PhysReg : RawOrder) {
     // Remove reserved registers from the allocation order.
     if (Reserved.test(PhysReg))
       continue;
diff --git a/llvm/lib/CodeGen/RegisterCoalescer.cpp b/llvm/lib/CodeGen/RegisterCoalescer.cpp
index 4c8534cf2d01..a917b0d27d4a 100644
--- a/llvm/lib/CodeGen/RegisterCoalescer.cpp
+++ b/llvm/lib/CodeGen/RegisterCoalescer.cpp
@@ -4067,13 +4067,13 @@ void RegisterCoalescer::joinAllIntervals() {
 
   // Coalesce intervals in MBB priority order.
   unsigned CurrDepth = std::numeric_limits<unsigned>::max();
-  for (unsigned i = 0, e = MBBs.size(); i != e; ++i) {
+  for (MBBPriorityInfo &MBB : MBBs) {
     // Try coalescing the collected local copies for deeper loops.
-    if (JoinGlobalCopies && MBBs[i].Depth < CurrDepth) {
+    if (JoinGlobalCopies && MBB.Depth < CurrDepth) {
       coalesceLocals();
-      CurrDepth = MBBs[i].Depth;
+      CurrDepth = MBB.Depth;
     }
-    copyCoalesceInMBB(MBBs[i].MBB);
+    copyCoalesceInMBB(MBB.MBB);
   }
   lateLiveIntervalUpdate();
   coalesceLocals();
diff --git a/llvm/lib/CodeGen/RemoveRedundantDebugValues.cpp b/llvm/lib/CodeGen/RemoveRedundantDebugValues.cpp
index de6129a912d3..49859aeec78b 100644
--- a/llvm/lib/CodeGen/RemoveRedundantDebugValues.cpp
+++ b/llvm/lib/CodeGen/RemoveRedundantDebugValues.cpp
@@ -159,20 +159,17 @@ static bool reduceDbgValsBackwardScan(MachineBasicBlock &MBB) {
   SmallVector<MachineInstr *, 8> DbgValsToBeRemoved;
   SmallDenseSet<DebugVariable> VariableSet;
 
-  for (MachineBasicBlock::reverse_iterator I = MBB.rbegin(), E = MBB.rend();
-       I != E; ++I) {
-    MachineInstr *MI = &*I;
-
-    if (MI->isDebugValue()) {
-      DebugVariable Var(MI->getDebugVariable(), MI->getDebugExpression(),
-                        MI->getDebugLoc()->getInlinedAt());
+  for (MachineInstr &MI : llvm::reverse(MBB)) {
+    if (MI.isDebugValue()) {
+      DebugVariable Var(MI.getDebugVariable(), MI.getDebugExpression(),
+                        MI.getDebugLoc()->getInlinedAt());
       auto R = VariableSet.insert(Var);
       // If it is a DBG_VALUE describing a constant as:
       //   DBG_VALUE 0, ...
       // we just don't consider such instructions as candidates
       // for redundant removal.
-      if (MI->isNonListDebugValue()) {
-        MachineOperand &Loc = MI->getDebugOperand(0);
+      if (MI.isNonListDebugValue()) {
+        MachineOperand &Loc = MI.getDebugOperand(0);
         if (!Loc.isReg()) {
           // If we have already encountered this variable, just stop
           // tracking it.
@@ -185,7 +182,7 @@ static bool reduceDbgValsBackwardScan(MachineBasicBlock &MBB) {
       // We have already encountered the value for this variable,
       // so this one can be deleted.
       if (!R.second)
-        DbgValsToBeRemoved.push_back(MI);
+        DbgValsToBeRemoved.push_back(&MI);
       continue;
     }
 
diff --git a/llvm/lib/CodeGen/SafeStack.cpp b/llvm/lib/CodeGen/SafeStack.cpp
index 50d9d64bfcfd..3d8a7eecce18 100644
--- a/llvm/lib/CodeGen/SafeStack.cpp
+++ b/llvm/lib/CodeGen/SafeStack.cpp
@@ -521,8 +521,7 @@ Value *SafeStack::moveStaticAllocasToUnsafeStack(
   StackLayout SSL(StackAlignment);
   if (StackGuardSlot) {
     Type *Ty = StackGuardSlot->getAllocatedType();
-    uint64_t Align =
-        std::max(DL.getPrefTypeAlignment(Ty), StackGuardSlot->getAlignment());
+    Align Align = std::max(DL.getPrefTypeAlign(Ty), StackGuardSlot->getAlign());
     SSL.addObject(StackGuardSlot, getStaticAllocaAllocationSize(StackGuardSlot),
                   Align, SSC.getFullLiveRange());
   }
@@ -534,8 +533,9 @@ Value *SafeStack::moveStaticAllocasToUnsafeStack(
       Size = 1; // Don't create zero-sized stack objects.
 
     // Ensure the object is properly aligned.
-    uint64_t Align =
-        std::max(DL.getPrefTypeAlignment(Ty), Arg->getParamAlignment());
+    Align Align = DL.getPrefTypeAlign(Ty);
+    if (auto A = Arg->getParamAlign())
+      Align = std::max(Align, *A);
     SSL.addObject(Arg, Size, Align, SSC.getFullLiveRange());
   }
 
@@ -546,24 +546,24 @@ Value *SafeStack::moveStaticAllocasToUnsafeStack(
       Size = 1; // Don't create zero-sized stack objects.
 
     // Ensure the object is properly aligned.
-    uint64_t Align = std::max(DL.getPrefTypeAlignment(Ty), AI->getAlignment());
+    Align Align = std::max(DL.getPrefTypeAlign(Ty), AI->getAlign());
 
     SSL.addObject(AI, Size, Align,
                   ClColoring ? SSC.getLiveRange(AI) : NoColoringRange);
   }
 
   SSL.computeLayout();
-  uint64_t FrameAlignment = SSL.getFrameAlignment();
+  Align FrameAlignment = SSL.getFrameAlignment();
 
   // FIXME: tell SSL that we start at a less-then-MaxAlignment aligned location
   // (AlignmentSkew).
   if (FrameAlignment > StackAlignment) {
     // Re-align the base pointer according to the max requested alignment.
-    assert(isPowerOf2_64(FrameAlignment));
     IRB.SetInsertPoint(BasePointer->getNextNode());
     BasePointer = cast<Instruction>(IRB.CreateIntToPtr(
-        IRB.CreateAnd(IRB.CreatePtrToInt(BasePointer, IntPtrTy),
-                      ConstantInt::get(IntPtrTy, ~uint64_t(FrameAlignment - 1))),
+        IRB.CreateAnd(
+            IRB.CreatePtrToInt(BasePointer, IntPtrTy),
+            ConstantInt::get(IntPtrTy, ~(FrameAlignment.value() - 1))),
         StackPtrTy));
   }
 
diff --git a/llvm/lib/CodeGen/SafeStackLayout.cpp b/llvm/lib/CodeGen/SafeStackLayout.cpp
index 7cdda7743c16..602afcfa9001 100644
--- a/llvm/lib/CodeGen/SafeStackLayout.cpp
+++ b/llvm/lib/CodeGen/SafeStackLayout.cpp
@@ -37,7 +37,7 @@ LLVM_DUMP_METHOD void StackLayout::print(raw_ostream &OS) {
   }
 }
 
-void StackLayout::addObject(const Value *V, unsigned Size, uint64_t Alignment,
+void StackLayout::addObject(const Value *V, unsigned Size, Align Alignment,
                             const StackLifetime::LiveRange &Range) {
   StackObjects.push_back({V, Size, Alignment, Range});
   ObjectAlignments[V] = Alignment;
@@ -45,7 +45,7 @@ void StackLayout::addObject(const Value *V, unsigned Size, uint64_t Alignment,
 }
 
 static unsigned AdjustStackOffset(unsigned Offset, unsigned Size,
-                                  uint64_t Alignment) {
+                                  Align Alignment) {
   return alignTo(Offset + Size, Alignment) - Size;
 }
 
@@ -62,7 +62,8 @@ void StackLayout::layoutObject(StackObject &Obj) {
   }
 
   LLVM_DEBUG(dbgs() << "Layout: size " << Obj.Size << ", align "
-                    << Obj.Alignment << ", range " << Obj.Range << "\n");
+                    << Obj.Alignment.value() << ", range " << Obj.Range
+                    << "\n");
   assert(Obj.Alignment <= MaxAlignment);
   unsigned Start = AdjustStackOffset(0, Obj.Size, Obj.Alignment);
   unsigned End = Start + Obj.Size;
diff --git a/llvm/lib/CodeGen/SafeStackLayout.h b/llvm/lib/CodeGen/SafeStackLayout.h
index b72450e57080..4ac7af2059f5 100644
--- a/llvm/lib/CodeGen/SafeStackLayout.h
+++ b/llvm/lib/CodeGen/SafeStackLayout.h
@@ -22,7 +22,7 @@ namespace safestack {
 
 /// Compute the layout of an unsafe stack frame.
 class StackLayout {
-  uint64_t MaxAlignment;
+  Align MaxAlignment;
 
   struct StackRegion {
     unsigned Start;
@@ -40,14 +40,14 @@ class StackLayout {
   struct StackObject {
     const Value *Handle;
     unsigned Size;
-    uint64_t Alignment;
+    Align Alignment;
     StackLifetime::LiveRange Range;
   };
 
   SmallVector<StackObject, 8> StackObjects;
 
   DenseMap<const Value *, unsigned> ObjectOffsets;
-  DenseMap<const Value *, uint64_t> ObjectAlignments;
+  DenseMap<const Value *, Align> ObjectAlignments;
 
   void layoutObject(StackObject &Obj);
 
@@ -56,7 +56,7 @@ public:
 
   /// Add an object to the stack frame. Value pointer is opaque and used as a
   /// handle to retrieve the object's offset in the frame later.
-  void addObject(const Value *V, unsigned Size, uint64_t Alignment,
+  void addObject(const Value *V, unsigned Size, Align Alignment,
                  const StackLifetime::LiveRange &Range);
 
   /// Run the layout computation for all previously added objects.
@@ -66,13 +66,13 @@ public:
   unsigned getObjectOffset(const Value *V) { return ObjectOffsets[V]; }
 
   /// Returns the alignment of the object
-  uint64_t getObjectAlignment(const Value *V) { return ObjectAlignments[V]; }
+  Align getObjectAlignment(const Value *V) { return ObjectAlignments[V]; }
 
   /// Returns the size of the entire frame.
   unsigned getFrameSize() { return Regions.empty() ? 0 : Regions.back().End; }
 
   /// Returns the alignment of the frame.
-  uint64_t getFrameAlignment() { return MaxAlignment; }
+  Align getFrameAlignment() { return MaxAlignment; }
 
   void print(raw_ostream &OS);
 };
diff --git a/llvm/lib/CodeGen/ScheduleDAG.cpp b/llvm/lib/CodeGen/ScheduleDAG.cpp
index ef3afab2b730..696b29018ae6 100644
--- a/llvm/lib/CodeGen/ScheduleDAG.cpp
+++ b/llvm/lib/CodeGen/ScheduleDAG.cpp
@@ -618,8 +618,8 @@ std::vector<int> ScheduleDAGTopologicalSort::GetSubGraph(const SUnit &StartSU,
   do {
     const SUnit *SU = WorkList.back();
     WorkList.pop_back();
-    for (int I = SU->Succs.size()-1; I >= 0; --I) {
-      const SUnit *Succ = SU->Succs[I].getSUnit();
+    for (const SDep &SD : llvm::reverse(SU->Succs)) {
+      const SUnit *Succ = SD.getSUnit();
       unsigned s = Succ->NodeNum;
       // Edges to non-SUnits are allowed but ignored (e.g. ExitSU).
       if (Succ->isBoundaryNode())
@@ -652,8 +652,8 @@ std::vector<int> ScheduleDAGTopologicalSort::GetSubGraph(const SUnit &StartSU,
   do {
     const SUnit *SU = WorkList.back();
     WorkList.pop_back();
-    for (int I = SU->Preds.size()-1; I >= 0; --I) {
-      const SUnit *Pred = SU->Preds[I].getSUnit();
+    for (const SDep &SD : llvm::reverse(SU->Preds)) {
+      const SUnit *Pred = SD.getSUnit();
       unsigned s = Pred->NodeNum;
       // Edges to non-SUnits are allowed but ignored (e.g. EntrySU).
       if (Pred->isBoundaryNode())
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index df5a041b87cd..067ad819e0d2 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -487,10 +487,7 @@ namespace {
     SDValue visitFCEIL(SDNode *N);
     SDValue visitFTRUNC(SDNode *N);
     SDValue visitFFLOOR(SDNode *N);
-    SDValue visitFMINNUM(SDNode *N);
-    SDValue visitFMAXNUM(SDNode *N);
-    SDValue visitFMINIMUM(SDNode *N);
-    SDValue visitFMAXIMUM(SDNode *N);
+    SDValue visitFMinMax(SDNode *N);
     SDValue visitBRCOND(SDNode *N);
     SDValue visitBR_CC(SDNode *N);
     SDValue visitLOAD(SDNode *N);
@@ -1701,10 +1698,10 @@ SDValue DAGCombiner::visit(SDNode *N) {
   case ISD::FNEG:               return visitFNEG(N);
   case ISD::FABS:               return visitFABS(N);
   case ISD::FFLOOR:             return visitFFLOOR(N);
-  case ISD::FMINNUM:            return visitFMINNUM(N);
-  case ISD::FMAXNUM:            return visitFMAXNUM(N);
-  case ISD::FMINIMUM:           return visitFMINIMUM(N);
-  case ISD::FMAXIMUM:           return visitFMAXIMUM(N);
+  case ISD::FMINNUM:
+  case ISD::FMAXNUM:
+  case ISD::FMINIMUM:
+  case ISD::FMAXIMUM:           return visitFMinMax(N);
   case ISD::FCEIL:              return visitFCEIL(N);
   case ISD::FTRUNC:             return visitFTRUNC(N);
   case ISD::BRCOND:             return visitBRCOND(N);
@@ -2260,6 +2257,21 @@ SDValue DAGCombiner::visitADDLike(SDNode *N) {
   EVT VT = N0.getValueType();
   SDLoc DL(N);
 
+  // fold (add x, undef) -> undef
+  if (N0.isUndef())
+    return N0;
+  if (N1.isUndef())
+    return N1;
+
+  // fold (add c1, c2) -> c1+c2
+  if (SDValue C = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N0, N1}))
+    return C;
+
+  // canonicalize constant to RHS
+  if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
+      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
+    return DAG.getNode(ISD::ADD, DL, VT, N1, N0);
+
   // fold vector ops
   if (VT.isVector()) {
     if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
@@ -2268,23 +2280,6 @@ SDValue DAGCombiner::visitADDLike(SDNode *N) {
     // fold (add x, 0) -> x, vector edition
     if (ISD::isConstantSplatVectorAllZeros(N1.getNode()))
       return N0;
-    if (ISD::isConstantSplatVectorAllZeros(N0.getNode()))
-      return N1;
-  }
-
-  // fold (add x, undef) -> undef
-  if (N0.isUndef())
-    return N0;
-
-  if (N1.isUndef())
-    return N1;
-
-  if (DAG.isConstantIntBuildVectorOrConstantInt(N0)) {
-    // canonicalize constant to RHS
-    if (!DAG.isConstantIntBuildVectorOrConstantInt(N1))
-      return DAG.getNode(ISD::ADD, DL, VT, N1, N0);
-    // fold (add c1, c2) -> c1+c2
-    return DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N0, N1});
   }
 
   // fold (add x, 0) -> x
@@ -2554,6 +2549,19 @@ SDValue DAGCombiner::visitADDSAT(SDNode *N) {
   EVT VT = N0.getValueType();
   SDLoc DL(N);
 
+  // fold (add_sat x, undef) -> -1
+  if (N0.isUndef() || N1.isUndef())
+    return DAG.getAllOnesConstant(DL, VT);
+
+  // fold (add_sat c1, c2) -> c3
+  if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1}))
+    return C;
+
+  // canonicalize constant to RHS
+  if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
+      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
+    return DAG.getNode(Opcode, DL, VT, N1, N0);
+
   // fold vector ops
   if (VT.isVector()) {
     // TODO SimplifyVBinOp
@@ -2561,20 +2569,6 @@ SDValue DAGCombiner::visitADDSAT(SDNode *N) {
     // fold (add_sat x, 0) -> x, vector edition
     if (ISD::isConstantSplatVectorAllZeros(N1.getNode()))
       return N0;
-    if (ISD::isConstantSplatVectorAllZeros(N0.getNode()))
-      return N1;
-  }
-
-  // fold (add_sat x, undef) -> -1
-  if (N0.isUndef() || N1.isUndef())
-    return DAG.getAllOnesConstant(DL, VT);
-
-  if (DAG.isConstantIntBuildVectorOrConstantInt(N0)) {
-    // canonicalize constant to RHS
-    if (!DAG.isConstantIntBuildVectorOrConstantInt(N1))
-      return DAG.getNode(Opcode, DL, VT, N1, N0);
-    // fold (add_sat c1, c2) -> c3
-    return DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1});
   }
 
   // fold (add_sat x, 0) -> x
@@ -3260,6 +3254,15 @@ SDValue DAGCombiner::visitSUB(SDNode *N) {
   EVT VT = N0.getValueType();
   SDLoc DL(N);
 
+  // fold (sub x, x) -> 0
+  // FIXME: Refactor this and xor and other similar operations together.
+  if (N0 == N1)
+    return tryFoldToZero(DL, TLI, VT, DAG, LegalOperations);
+
+  // fold (sub c1, c2) -> c3
+  if (SDValue C = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N0, N1}))
+    return C;
+
   // fold vector ops
   if (VT.isVector()) {
     if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
@@ -3270,15 +3273,6 @@ SDValue DAGCombiner::visitSUB(SDNode *N) {
       return N0;
   }
 
-  // fold (sub x, x) -> 0
-  // FIXME: Refactor this and xor and other similar operations together.
-  if (N0 == N1)
-    return tryFoldToZero(DL, TLI, VT, DAG, LegalOperations);
-
-  // fold (sub c1, c2) -> c3
-  if (SDValue C = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N0, N1}))
-    return C;
-
   if (SDValue NewSel = foldBinOpIntoSelect(N))
     return NewSel;
 
@@ -3611,15 +3605,6 @@ SDValue DAGCombiner::visitSUBSAT(SDNode *N) {
   EVT VT = N0.getValueType();
   SDLoc DL(N);
 
-  // fold vector ops
-  if (VT.isVector()) {
-    // TODO SimplifyVBinOp
-
-    // fold (sub_sat x, 0) -> x, vector edition
-    if (ISD::isConstantSplatVectorAllZeros(N1.getNode()))
-      return N0;
-  }
-
   // fold (sub_sat x, undef) -> 0
   if (N0.isUndef() || N1.isUndef())
     return DAG.getConstant(0, DL, VT);
@@ -3632,6 +3617,15 @@ SDValue DAGCombiner::visitSUBSAT(SDNode *N) {
   if (SDValue C = DAG.FoldConstantArithmetic(N->getOpcode(), DL, VT, {N0, N1}))
     return C;
 
+  // fold vector ops
+  if (VT.isVector()) {
+    // TODO SimplifyVBinOp
+
+    // fold (sub_sat x, 0) -> x, vector edition
+    if (ISD::isConstantSplatVectorAllZeros(N1.getNode()))
+      return N0;
+  }
+
   // fold (sub_sat x, 0) -> x
   if (isNullConstant(N1))
     return N0;
@@ -3781,6 +3775,15 @@ SDValue DAGCombiner::visitMUL(SDNode *N) {
   if (N0.isUndef() || N1.isUndef())
     return DAG.getConstant(0, SDLoc(N), VT);
 
+  // fold (mul c1, c2) -> c1*c2
+  if (SDValue C = DAG.FoldConstantArithmetic(ISD::MUL, SDLoc(N), VT, {N0, N1}))
+    return C;
+
+  // canonicalize constant to RHS (vector doesn't have to splat)
+  if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
+      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
+    return DAG.getNode(ISD::MUL, SDLoc(N), VT, N1, N0);
+
   bool N1IsConst = false;
   bool N1IsOpaqueConst = false;
   APInt ConstValue1;
@@ -3802,15 +3805,6 @@ SDValue DAGCombiner::visitMUL(SDNode *N) {
     }
   }
 
-  // fold (mul c1, c2) -> c1*c2
-  if (SDValue C = DAG.FoldConstantArithmetic(ISD::MUL, SDLoc(N), VT, {N0, N1}))
-    return C;
-
-  // canonicalize constant to RHS (vector doesn't have to splat)
-  if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
-     !DAG.isConstantIntBuildVectorOrConstantInt(N1))
-    return DAG.getNode(ISD::MUL, SDLoc(N), VT, N1, N0);
-
   // fold (mul x, 0) -> 0
   if (N1IsConst && ConstValue1.isZero())
     return N1;
@@ -4140,17 +4134,17 @@ SDValue DAGCombiner::visitSDIV(SDNode *N) {
   EVT CCVT = getSetCCResultType(VT);
   SDLoc DL(N);
 
+  // fold (sdiv c1, c2) -> c1/c2
+  if (SDValue C = DAG.FoldConstantArithmetic(ISD::SDIV, DL, VT, {N0, N1}))
+    return C;
+
   // fold vector ops
   if (VT.isVector())
     if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
       return FoldedVOp;
 
-  // fold (sdiv c1, c2) -> c1/c2
-  ConstantSDNode *N1C = isConstOrConstSplat(N1);
-  if (SDValue C = DAG.FoldConstantArithmetic(ISD::SDIV, DL, VT, {N0, N1}))
-    return C;
-
   // fold (sdiv X, -1) -> 0-X
+  ConstantSDNode *N1C = isConstOrConstSplat(N1);
   if (N1C && N1C->isAllOnes())
     return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), N0);
 
@@ -4284,17 +4278,17 @@ SDValue DAGCombiner::visitUDIV(SDNode *N) {
   EVT CCVT = getSetCCResultType(VT);
   SDLoc DL(N);
 
+  // fold (udiv c1, c2) -> c1/c2
+  if (SDValue C = DAG.FoldConstantArithmetic(ISD::UDIV, DL, VT, {N0, N1}))
+    return C;
+
   // fold vector ops
   if (VT.isVector())
     if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
       return FoldedVOp;
 
-  // fold (udiv c1, c2) -> c1/c2
-  ConstantSDNode *N1C = isConstOrConstSplat(N1);
-  if (SDValue C = DAG.FoldConstantArithmetic(ISD::UDIV, DL, VT, {N0, N1}))
-    return C;
-
   // fold (udiv X, -1) -> select(X == -1, 1, 0)
+  ConstantSDNode *N1C = isConstOrConstSplat(N1);
   if (N1C && N1C->isAllOnes())
     return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ),
                          DAG.getConstant(1, DL, VT),
@@ -4463,6 +4457,15 @@ SDValue DAGCombiner::visitMULHS(SDNode *N) {
   EVT VT = N->getValueType(0);
   SDLoc DL(N);
 
+  // fold (mulhs c1, c2)
+  if (SDValue C = DAG.FoldConstantArithmetic(ISD::MULHS, DL, VT, {N0, N1}))
+    return C;
+
+  // canonicalize constant to RHS.
+  if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
+      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
+    return DAG.getNode(ISD::MULHS, DL, N->getVTList(), N1, N0);
+
   if (VT.isVector()) {
     if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
       return FoldedVOp;
@@ -4474,15 +4477,6 @@ SDValue DAGCombiner::visitMULHS(SDNode *N) {
       return DAG.getConstant(0, DL, VT);
   }
 
-  // fold (mulhs c1, c2)
-  if (SDValue C = DAG.FoldConstantArithmetic(ISD::MULHS, DL, VT, {N0, N1}))
-    return C;
-
-  // canonicalize constant to RHS.
-  if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
-      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
-    return DAG.getNode(ISD::MULHS, DL, N->getVTList(), N1, N0);
-
   // fold (mulhs x, 0) -> 0
   if (isNullConstant(N1))
     return N1;
@@ -4523,6 +4517,15 @@ SDValue DAGCombiner::visitMULHU(SDNode *N) {
   EVT VT = N->getValueType(0);
   SDLoc DL(N);
 
+  // fold (mulhu c1, c2)
+  if (SDValue C = DAG.FoldConstantArithmetic(ISD::MULHU, DL, VT, {N0, N1}))
+    return C;
+
+  // canonicalize constant to RHS.
+  if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
+      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
+    return DAG.getNode(ISD::MULHU, DL, N->getVTList(), N1, N0);
+
   if (VT.isVector()) {
     if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
       return FoldedVOp;
@@ -4534,15 +4537,6 @@ SDValue DAGCombiner::visitMULHU(SDNode *N) {
       return DAG.getConstant(0, DL, VT);
   }
 
-  // fold (mulhu c1, c2)
-  if (SDValue C = DAG.FoldConstantArithmetic(ISD::MULHU, DL, VT, {N0, N1}))
-    return C;
-
-  // canonicalize constant to RHS.
-  if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
-      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
-    return DAG.getNode(ISD::MULHU, DL, N->getVTList(), N1, N0);
-
   // fold (mulhu x, 0) -> 0
   if (isNullConstant(N1))
     return N1;
@@ -4786,12 +4780,14 @@ SDValue DAGCombiner::visitMULO(SDNode *N) {
 }
 
 // Function to calculate whether the Min/Max pair of SDNodes (potentially
-// swapped around) make a signed saturate pattern, clamping to between -2^(BW-1)
-// and 2^(BW-1)-1. Returns the node being clamped and the bitwidth of the clamp
-// in BW. Should work with both SMIN/SMAX nodes and setcc/select combo. The
-// operands are the same as SimplifySelectCC. N0<N1 ? N2 : N3
+// swapped around) make a signed saturate pattern, clamping to between a signed
+// saturate of -2^(BW-1) and 2^(BW-1)-1, or an unsigned saturate of 0 and 2^BW.
+// Returns the node being clamped and the bitwidth of the clamp in BW. Should
+// work with both SMIN/SMAX nodes and setcc/select combo. The operands are the
+// same as SimplifySelectCC. N0<N1 ? N2 : N3.
 static SDValue isSaturatingMinMax(SDValue N0, SDValue N1, SDValue N2,
-                                  SDValue N3, ISD::CondCode CC, unsigned &BW) {
+                                  SDValue N3, ISD::CondCode CC, unsigned &BW,
+                                  bool &Unsigned) {
   auto isSignedMinMax = [&](SDValue N0, SDValue N1, SDValue N2, SDValue N3,
                             ISD::CondCode CC) {
     // The compare and select operand should be the same or the select operands
@@ -4858,17 +4854,27 @@ static SDValue isSaturatingMinMax(SDValue N0, SDValue N1, SDValue N2,
   const APInt &MinC = MinCOp->getAPIntValue();
   const APInt &MaxC = MaxCOp->getAPIntValue();
   APInt MinCPlus1 = MinC + 1;
-  if (-MaxC != MinCPlus1 || !MinCPlus1.isPowerOf2())
-    return SDValue();
-  BW = MinCPlus1.exactLogBase2() + 1;
-  return N02;
+  if (-MaxC == MinCPlus1 && MinCPlus1.isPowerOf2()) {
+    BW = MinCPlus1.exactLogBase2() + 1;
+    Unsigned = false;
+    return N02;
+  }
+
+  if (MaxC == 0 && MinCPlus1.isPowerOf2()) {
+    BW = MinCPlus1.exactLogBase2();
+    Unsigned = true;
+    return N02;
+  }
+
+  return SDValue();
 }
 
 static SDValue PerformMinMaxFpToSatCombine(SDValue N0, SDValue N1, SDValue N2,
                                            SDValue N3, ISD::CondCode CC,
                                            SelectionDAG &DAG) {
   unsigned BW;
-  SDValue Fp = isSaturatingMinMax(N0, N1, N2, N3, CC, BW);
+  bool Unsigned;
+  SDValue Fp = isSaturatingMinMax(N0, N1, N2, N3, CC, BW, Unsigned);
   if (!Fp || Fp.getOpcode() != ISD::FP_TO_SINT)
     return SDValue();
   EVT FPVT = Fp.getOperand(0).getValueType();
@@ -4876,13 +4882,14 @@ static SDValue PerformMinMaxFpToSatCombine(SDValue N0, SDValue N1, SDValue N2,
   if (FPVT.isVector())
     NewVT = EVT::getVectorVT(*DAG.getContext(), NewVT,
                              FPVT.getVectorElementCount());
-  if (!DAG.getTargetLoweringInfo().shouldConvertFpToSat(
-          ISD::FP_TO_SINT_SAT, Fp.getOperand(0).getValueType(), NewVT))
+  unsigned NewOpc = Unsigned ? ISD::FP_TO_UINT_SAT : ISD::FP_TO_SINT_SAT;
+  if (!DAG.getTargetLoweringInfo().shouldConvertFpToSat(NewOpc, FPVT, NewVT))
     return SDValue();
   SDLoc DL(Fp);
-  SDValue Sat = DAG.getNode(ISD::FP_TO_SINT_SAT, DL, NewVT, Fp.getOperand(0),
+  SDValue Sat = DAG.getNode(NewOpc, DL, NewVT, Fp.getOperand(0),
                             DAG.getValueType(NewVT.getScalarType()));
-  return DAG.getSExtOrTrunc(Sat, DL, N2->getValueType(0));
+  return Unsigned ? DAG.getZExtOrTrunc(Sat, DL, N2->getValueType(0))
+                  : DAG.getSExtOrTrunc(Sat, DL, N2->getValueType(0));
 }
 
 SDValue DAGCombiner::visitIMINMAX(SDNode *N) {
@@ -4892,11 +4899,6 @@ SDValue DAGCombiner::visitIMINMAX(SDNode *N) {
   unsigned Opcode = N->getOpcode();
   SDLoc DL(N);
 
-  // fold vector ops
-  if (VT.isVector())
-    if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
-      return FoldedVOp;
-
   // fold operation with constant operands.
   if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1}))
     return C;
@@ -4904,7 +4906,12 @@ SDValue DAGCombiner::visitIMINMAX(SDNode *N) {
   // canonicalize constant to RHS
   if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
       !DAG.isConstantIntBuildVectorOrConstantInt(N1))
-    return DAG.getNode(N->getOpcode(), DL, VT, N1, N0);
+    return DAG.getNode(Opcode, DL, VT, N1, N0);
+
+  // fold vector ops
+  if (VT.isVector())
+    if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
+      return FoldedVOp;
 
   // Is sign bits are zero, flip between UMIN/UMAX and SMIN/SMAX.
   // Only do this if the current op isn't legal and the flipped is.
@@ -5777,6 +5784,15 @@ SDValue DAGCombiner::visitAND(SDNode *N) {
   if (N0 == N1)
     return N0;
 
+  // fold (and c1, c2) -> c1&c2
+  if (SDValue C = DAG.FoldConstantArithmetic(ISD::AND, SDLoc(N), VT, {N0, N1}))
+    return C;
+
+  // canonicalize constant to RHS
+  if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
+      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
+    return DAG.getNode(ISD::AND, SDLoc(N), VT, N1, N0);
+
   // fold vector ops
   if (VT.isVector()) {
     if (SDValue FoldedVOp = SimplifyVBinOp(N, SDLoc(N)))
@@ -5824,22 +5840,13 @@ SDValue DAGCombiner::visitAND(SDNode *N) {
     }
   }
 
-  // fold (and c1, c2) -> c1&c2
-  ConstantSDNode *N1C = isConstOrConstSplat(N1);
-  if (SDValue C = DAG.FoldConstantArithmetic(ISD::AND, SDLoc(N), VT, {N0, N1}))
-    return C;
-
-  // canonicalize constant to RHS
-  if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
-      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
-    return DAG.getNode(ISD::AND, SDLoc(N), VT, N1, N0);
-
   // fold (and x, -1) -> x
   if (isAllOnesConstant(N1))
     return N0;
 
   // if (and x, c) is known to be zero, return 0
   unsigned BitWidth = VT.getScalarSizeInBits();
+  ConstantSDNode *N1C = isConstOrConstSplat(N1);
   if (N1C && DAG.MaskedValueIsZero(SDValue(N, 0), APInt::getAllOnes(BitWidth)))
     return DAG.getConstant(0, SDLoc(N), VT);
 
@@ -6546,21 +6553,25 @@ SDValue DAGCombiner::visitOR(SDNode *N) {
   if (N0 == N1)
     return N0;
 
+  // fold (or c1, c2) -> c1|c2
+  if (SDValue C = DAG.FoldConstantArithmetic(ISD::OR, SDLoc(N), VT, {N0, N1}))
+    return C;
+
+  // canonicalize constant to RHS
+  if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
+      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
+    return DAG.getNode(ISD::OR, SDLoc(N), VT, N1, N0);
+
   // fold vector ops
   if (VT.isVector()) {
     if (SDValue FoldedVOp = SimplifyVBinOp(N, SDLoc(N)))
       return FoldedVOp;
 
     // fold (or x, 0) -> x, vector edition
-    if (ISD::isConstantSplatVectorAllZeros(N0.getNode()))
-      return N1;
     if (ISD::isConstantSplatVectorAllZeros(N1.getNode()))
       return N0;
 
     // fold (or x, -1) -> -1, vector edition
-    if (ISD::isConstantSplatVectorAllOnes(N0.getNode()))
-      // do not return N0, because undef node may exist in N0
-      return DAG.getAllOnesConstant(SDLoc(N), N0.getValueType());
     if (ISD::isConstantSplatVectorAllOnes(N1.getNode()))
       // do not return N1, because undef node may exist in N1
       return DAG.getAllOnesConstant(SDLoc(N), N1.getValueType());
@@ -6629,16 +6640,6 @@ SDValue DAGCombiner::visitOR(SDNode *N) {
     }
   }
 
-  // fold (or c1, c2) -> c1|c2
-  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
-  if (SDValue C = DAG.FoldConstantArithmetic(ISD::OR, SDLoc(N), VT, {N0, N1}))
-    return C;
-
-  // canonicalize constant to RHS
-  if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
-     !DAG.isConstantIntBuildVectorOrConstantInt(N1))
-    return DAG.getNode(ISD::OR, SDLoc(N), VT, N1, N0);
-
   // fold (or x, 0) -> x
   if (isNullConstant(N1))
     return N0;
@@ -6651,6 +6652,7 @@ SDValue DAGCombiner::visitOR(SDNode *N) {
     return NewSel;
 
   // fold (or x, c) -> c iff (x & ~c) == 0
+  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
   if (N1C && DAG.MaskedValueIsZero(N0, ~N1C->getAPIntValue()))
     return N1;
 
@@ -7941,18 +7943,6 @@ SDValue DAGCombiner::visitXOR(SDNode *N) {
   EVT VT = N0.getValueType();
   SDLoc DL(N);
 
-  // fold vector ops
-  if (VT.isVector()) {
-    if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
-      return FoldedVOp;
-
-    // fold (xor x, 0) -> x, vector edition
-    if (ISD::isConstantSplatVectorAllZeros(N0.getNode()))
-      return N1;
-    if (ISD::isConstantSplatVectorAllZeros(N1.getNode()))
-      return N0;
-  }
-
   // fold (xor undef, undef) -> 0. This is a common idiom (misuse).
   if (N0.isUndef() && N1.isUndef())
     return DAG.getConstant(0, DL, VT);
@@ -7969,9 +7959,19 @@ SDValue DAGCombiner::visitXOR(SDNode *N) {
 
   // canonicalize constant to RHS
   if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
-     !DAG.isConstantIntBuildVectorOrConstantInt(N1))
+      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
     return DAG.getNode(ISD::XOR, DL, VT, N1, N0);
 
+  // fold vector ops
+  if (VT.isVector()) {
+    if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
+      return FoldedVOp;
+
+    // fold (xor x, 0) -> x, vector edition
+    if (ISD::isConstantSplatVectorAllZeros(N1.getNode()))
+      return N0;
+  }
+
   // fold (xor x, 0) -> x
   if (isNullConstant(N1))
     return N0;
@@ -8409,6 +8409,10 @@ SDValue DAGCombiner::visitSHL(SDNode *N) {
   EVT ShiftVT = N1.getValueType();
   unsigned OpSizeInBits = VT.getScalarSizeInBits();
 
+  // fold (shl c1, c2) -> c1<<c2
+  if (SDValue C = DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT, {N0, N1}))
+    return C;
+
   // fold vector ops
   if (VT.isVector()) {
     if (SDValue FoldedVOp = SimplifyVBinOp(N, SDLoc(N)))
@@ -8434,12 +8438,6 @@ SDValue DAGCombiner::visitSHL(SDNode *N) {
     }
   }
 
-  ConstantSDNode *N1C = isConstOrConstSplat(N1);
-
-  // fold (shl c1, c2) -> c1<<c2
-  if (SDValue C = DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT, {N0, N1}))
-    return C;
-
   if (SDValue NewSel = foldBinOpIntoSelect(N))
     return NewSel;
 
@@ -8558,6 +8556,7 @@ SDValue DAGCombiner::visitSHL(SDNode *N) {
   // fold (shl (sr[la] exact X,  C1), C2) -> (shl    X, (C2-C1)) if C1 <= C2
   // fold (shl (sr[la] exact X,  C1), C2) -> (sr[la] X, (C2-C1)) if C1  > C2
   // TODO - support non-uniform vector shift amounts.
+  ConstantSDNode *N1C = isConstOrConstSplat(N1);
   if (N1C && (N0.getOpcode() == ISD::SRL || N0.getOpcode() == ISD::SRA) &&
       N0->getFlags().hasExact()) {
     if (ConstantSDNode *N0C1 = isConstOrConstSplat(N0.getOperand(1))) {
@@ -8758,6 +8757,10 @@ SDValue DAGCombiner::visitSRA(SDNode *N) {
   EVT VT = N0.getValueType();
   unsigned OpSizeInBits = VT.getScalarSizeInBits();
 
+  // fold (sra c1, c2) -> (sra c1, c2)
+  if (SDValue C = DAG.FoldConstantArithmetic(ISD::SRA, SDLoc(N), VT, {N0, N1}))
+    return C;
+
   // Arithmetic shifting an all-sign-bit value is a no-op.
   // fold (sra 0, x) -> 0
   // fold (sra -1, x) -> -1
@@ -8769,17 +8772,12 @@ SDValue DAGCombiner::visitSRA(SDNode *N) {
     if (SDValue FoldedVOp = SimplifyVBinOp(N, SDLoc(N)))
       return FoldedVOp;
 
-  ConstantSDNode *N1C = isConstOrConstSplat(N1);
-
-  // fold (sra c1, c2) -> (sra c1, c2)
-  if (SDValue C = DAG.FoldConstantArithmetic(ISD::SRA, SDLoc(N), VT, {N0, N1}))
-    return C;
-
   if (SDValue NewSel = foldBinOpIntoSelect(N))
     return NewSel;
 
   // fold (sra (shl x, c1), c1) -> sext_inreg for some c1 and target supports
   // sext_inreg.
+  ConstantSDNode *N1C = isConstOrConstSplat(N1);
   if (N1C && N0.getOpcode() == ISD::SHL && N1 == N0.getOperand(1)) {
     unsigned LowBits = OpSizeInBits - (unsigned)N1C->getZExtValue();
     EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), LowBits);
@@ -8962,21 +8960,20 @@ SDValue DAGCombiner::visitSRL(SDNode *N) {
   EVT VT = N0.getValueType();
   unsigned OpSizeInBits = VT.getScalarSizeInBits();
 
+  // fold (srl c1, c2) -> c1 >>u c2
+  if (SDValue C = DAG.FoldConstantArithmetic(ISD::SRL, SDLoc(N), VT, {N0, N1}))
+    return C;
+
   // fold vector ops
   if (VT.isVector())
     if (SDValue FoldedVOp = SimplifyVBinOp(N, SDLoc(N)))
       return FoldedVOp;
 
-  ConstantSDNode *N1C = isConstOrConstSplat(N1);
-
-  // fold (srl c1, c2) -> c1 >>u c2
-  if (SDValue C = DAG.FoldConstantArithmetic(ISD::SRL, SDLoc(N), VT, {N0, N1}))
-    return C;
-
   if (SDValue NewSel = foldBinOpIntoSelect(N))
     return NewSel;
 
   // if (srl x, c) is known to be zero, return 0
+  ConstantSDNode *N1C = isConstOrConstSplat(N1);
   if (N1C &&
       DAG.MaskedValueIsZero(SDValue(N, 0), APInt::getAllOnes(OpSizeInBits)))
     return DAG.getConstant(0, SDLoc(N), VT);
@@ -10043,6 +10040,8 @@ SDValue DAGCombiner::visitMSTORE(SDNode *N) {
   MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
   SDValue Mask = MST->getMask();
   SDValue Chain = MST->getChain();
+  SDValue Value = MST->getValue();
+  SDValue Ptr = MST->getBasePtr();
   SDLoc DL(N);
 
   // Zap masked stores with a zero mask.
@@ -10054,12 +10053,50 @@ SDValue DAGCombiner::visitMSTORE(SDNode *N) {
   if (ISD::isConstantSplatVectorAllOnes(Mask.getNode()) && MST->isUnindexed() &&
       !MST->isCompressingStore() && !MST->isTruncatingStore())
     return DAG.getStore(MST->getChain(), SDLoc(N), MST->getValue(),
-                        MST->getBasePtr(), MST->getMemOperand());
+                        MST->getBasePtr(), MST->getPointerInfo(),
+                        MST->getOriginalAlign(), MachineMemOperand::MOStore,
+                        MST->getAAInfo());
 
   // Try transforming N to an indexed store.
   if (CombineToPreIndexedLoadStore(N) || CombineToPostIndexedLoadStore(N))
     return SDValue(N, 0);
 
+  if (MST->isTruncatingStore() && MST->isUnindexed() &&
+      Value.getValueType().isInteger() &&
+      (!isa<ConstantSDNode>(Value) ||
+       !cast<ConstantSDNode>(Value)->isOpaque())) {
+    APInt TruncDemandedBits =
+        APInt::getLowBitsSet(Value.getScalarValueSizeInBits(),
+                             MST->getMemoryVT().getScalarSizeInBits());
+
+    // See if we can simplify the operation with
+    // SimplifyDemandedBits, which only works if the value has a single use.
+    if (SimplifyDemandedBits(Value, TruncDemandedBits)) {
+      // Re-visit the store if anything changed and the store hasn't been merged
+      // with another node (N is deleted) SimplifyDemandedBits will add Value's
+      // node back to the worklist if necessary, but we also need to re-visit
+      // the Store node itself.
+      if (N->getOpcode() != ISD::DELETED_NODE)
+        AddToWorklist(N);
+      return SDValue(N, 0);
+    }
+  }
+
+  // If this is a TRUNC followed by a masked store, fold this into a masked
+  // truncating store.  We can do this even if this is already a masked
+  // truncstore.
+  if ((Value.getOpcode() == ISD::TRUNCATE) && Value.getNode()->hasOneUse() &&
+      MST->isUnindexed() &&
+      TLI.canCombineTruncStore(Value.getOperand(0).getValueType(),
+                               MST->getMemoryVT(), LegalOperations)) {
+    auto Mask = TLI.promoteTargetBoolean(DAG, MST->getMask(),
+                                         Value.getOperand(0).getValueType());
+    return DAG.getMaskedStore(Chain, SDLoc(N), Value.getOperand(0), Ptr,
+                              MST->getOffset(), Mask, MST->getMemoryVT(),
+                              MST->getMemOperand(), MST->getAddressingMode(),
+                              /*IsTruncating=*/true);
+  }
+
   return SDValue();
 }
 
@@ -10109,8 +10146,10 @@ SDValue DAGCombiner::visitMLOAD(SDNode *N) {
   // FIXME: Can we do this for indexed, expanding, or extending loads?
   if (ISD::isConstantSplatVectorAllOnes(Mask.getNode()) && MLD->isUnindexed() &&
       !MLD->isExpandingLoad() && MLD->getExtensionType() == ISD::NON_EXTLOAD) {
-    SDValue NewLd = DAG.getLoad(N->getValueType(0), SDLoc(N), MLD->getChain(),
-                                MLD->getBasePtr(), MLD->getMemOperand());
+    SDValue NewLd = DAG.getLoad(
+        N->getValueType(0), SDLoc(N), MLD->getChain(), MLD->getBasePtr(),
+        MLD->getPointerInfo(), MLD->getOriginalAlign(),
+        MachineMemOperand::MOLoad, MLD->getAAInfo(), MLD->getRanges());
     return CombineTo(N, NewLd, NewLd.getValue(1));
   }
 
@@ -13876,19 +13915,19 @@ SDValue DAGCombiner::visitFADD(SDNode *N) {
   if (SDValue R = DAG.simplifyFPBinop(N->getOpcode(), N0, N1, Flags))
     return R;
 
-  // fold vector ops
-  if (VT.isVector())
-    if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
-      return FoldedVOp;
-
   // fold (fadd c1, c2) -> c1 + c2
-  if (N0CFP && N1CFP)
-    return DAG.getNode(ISD::FADD, DL, VT, N0, N1);
+  if (SDValue C = DAG.FoldConstantArithmetic(ISD::FADD, DL, VT, {N0, N1}))
+    return C;
 
   // canonicalize constant to RHS
   if (N0CFP && !N1CFP)
     return DAG.getNode(ISD::FADD, DL, VT, N1, N0);
 
+  // fold vector ops
+  if (VT.isVector())
+    if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
+      return FoldedVOp;
+
   // N0 + -0.0 --> N0 (also allowed with +0.0 and fast-math)
   ConstantFPSDNode *N1C = isConstOrConstSplatFP(N1, true);
   if (N1C && N1C->isZero())
@@ -14084,15 +14123,15 @@ SDValue DAGCombiner::visitFSUB(SDNode *N) {
   if (SDValue R = DAG.simplifyFPBinop(N->getOpcode(), N0, N1, Flags))
     return R;
 
+  // fold (fsub c1, c2) -> c1-c2
+  if (SDValue C = DAG.FoldConstantArithmetic(ISD::FSUB, DL, VT, {N0, N1}))
+    return C;
+
   // fold vector ops
   if (VT.isVector())
     if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
       return FoldedVOp;
 
-  // fold (fsub c1, c2) -> c1-c2
-  if (N0CFP && N1CFP)
-    return DAG.getNode(ISD::FSUB, DL, VT, N0, N1);
-
   if (SDValue NewSel = foldBinOpIntoSelect(N))
     return NewSel;
 
@@ -14157,7 +14196,6 @@ SDValue DAGCombiner::visitFSUB(SDNode *N) {
 SDValue DAGCombiner::visitFMUL(SDNode *N) {
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
-  ConstantFPSDNode *N0CFP = isConstOrConstSplatFP(N0, true);
   ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N1, true);
   EVT VT = N->getValueType(0);
   SDLoc DL(N);
@@ -14168,22 +14206,20 @@ SDValue DAGCombiner::visitFMUL(SDNode *N) {
   if (SDValue R = DAG.simplifyFPBinop(N->getOpcode(), N0, N1, Flags))
     return R;
 
-  // fold vector ops
-  if (VT.isVector()) {
-    // This just handles C1 * C2 for vectors. Other vector folds are below.
-    if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
-      return FoldedVOp;
-  }
-
   // fold (fmul c1, c2) -> c1*c2
-  if (N0CFP && N1CFP)
-    return DAG.getNode(ISD::FMUL, DL, VT, N0, N1);
+  if (SDValue C = DAG.FoldConstantArithmetic(ISD::FMUL, DL, VT, {N0, N1}))
+    return C;
 
   // canonicalize constant to RHS
   if (DAG.isConstantFPBuildVectorOrConstantFP(N0) &&
      !DAG.isConstantFPBuildVectorOrConstantFP(N1))
     return DAG.getNode(ISD::FMUL, DL, VT, N1, N0);
 
+  // fold vector ops
+  if (VT.isVector())
+    if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
+      return FoldedVOp;
+
   if (SDValue NewSel = foldBinOpIntoSelect(N))
     return NewSel;
 
@@ -14495,8 +14531,6 @@ SDValue DAGCombiner::combineRepeatedFPDivisors(SDNode *N) {
 SDValue DAGCombiner::visitFDIV(SDNode *N) {
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
-  ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
-  ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
   EVT VT = N->getValueType(0);
   SDLoc DL(N);
   const TargetOptions &Options = DAG.getTarget().Options;
@@ -14506,15 +14540,15 @@ SDValue DAGCombiner::visitFDIV(SDNode *N) {
   if (SDValue R = DAG.simplifyFPBinop(N->getOpcode(), N0, N1, Flags))
     return R;
 
+  // fold (fdiv c1, c2) -> c1/c2
+  if (SDValue C = DAG.FoldConstantArithmetic(ISD::FDIV, DL, VT, {N0, N1}))
+    return C;
+
   // fold vector ops
   if (VT.isVector())
     if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
       return FoldedVOp;
 
-  // fold (fdiv c1, c2) -> c1/c2
-  if (N0CFP && N1CFP)
-    return DAG.getNode(ISD::FDIV, SDLoc(N), VT, N0, N1);
-
   if (SDValue NewSel = foldBinOpIntoSelect(N))
     return NewSel;
 
@@ -14523,7 +14557,7 @@ SDValue DAGCombiner::visitFDIV(SDNode *N) {
 
   if (Options.UnsafeFPMath || Flags.hasAllowReciprocal()) {
     // fold (fdiv X, c2) -> fmul X, 1/c2 if losing precision is acceptable.
-    if (N1CFP) {
+    if (auto *N1CFP = dyn_cast<ConstantFPSDNode>(N1)) {
       // Compute the reciprocal 1.0 / c2.
       const APFloat &N1APF = N1CFP->getValueAPF();
       APFloat Recip(N1APF.getSemantics(), 1); // 1.0
@@ -14639,8 +14673,6 @@ SDValue DAGCombiner::visitFDIV(SDNode *N) {
 SDValue DAGCombiner::visitFREM(SDNode *N) {
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
-  ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
-  ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
   EVT VT = N->getValueType(0);
   SDNodeFlags Flags = N->getFlags();
   SelectionDAG::FlagInserter FlagsInserter(DAG, N);
@@ -14649,9 +14681,9 @@ SDValue DAGCombiner::visitFREM(SDNode *N) {
     return R;
 
   // fold (frem c1, c2) -> fmod(c1,c2)
-  if (N0CFP && N1CFP)
-    return DAG.getNode(ISD::FREM, SDLoc(N), VT, N0, N1);
-
+  if (SDValue C = DAG.FoldConstantArithmetic(ISD::FREM, SDLoc(N), VT, {N0, N1}))
+    return C;
+  
   if (SDValue NewSel = foldBinOpIntoSelect(N))
     return NewSel;
 
@@ -14712,12 +14744,12 @@ static inline bool CanCombineFCOPYSIGN_EXTEND_ROUND(SDNode *N) {
 SDValue DAGCombiner::visitFCOPYSIGN(SDNode *N) {
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
-  bool N0CFP = DAG.isConstantFPBuildVectorOrConstantFP(N0);
-  bool N1CFP = DAG.isConstantFPBuildVectorOrConstantFP(N1);
   EVT VT = N->getValueType(0);
 
-  if (N0CFP && N1CFP) // Constant fold
-    return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, N0, N1);
+  // fold (fcopysign c1, c2) -> fcopysign(c1,c2)
+  if (SDValue C =
+          DAG.FoldConstantArithmetic(ISD::FCOPYSIGN, SDLoc(N), VT, {N0, N1}))
+    return C;
 
   if (ConstantFPSDNode *N1C = isConstOrConstSplatFP(N->getOperand(1))) {
     const APFloat &V = N1C->getValueAPF();
@@ -14835,14 +14867,6 @@ SDValue DAGCombiner::visitFPOW(SDNode *N) {
 
 static SDValue foldFPToIntToFP(SDNode *N, SelectionDAG &DAG,
                                const TargetLowering &TLI) {
-  // This optimization is guarded by a function attribute because it may produce
-  // unexpected results. Ie, programs may be relying on the platform-specific
-  // undefined behavior when the float-to-int conversion overflows.
-  const Function &F = DAG.getMachineFunction().getFunction();
-  Attribute StrictOverflow = F.getFnAttribute("strict-float-cast-overflow");
-  if (StrictOverflow.getValueAsString().equals("false"))
-    return SDValue();
-
   // We only do this if the target has legal ftrunc. Otherwise, we'd likely be
   // replacing casts with a libcall. We also must be allowed to ignore -0.0
   // because FTRUNC will return -0.0 for (-1.0, -0.0), but using integer
@@ -15216,31 +15240,26 @@ SDValue DAGCombiner::visitFNEG(SDNode *N) {
   return SDValue();
 }
 
-static SDValue visitFMinMax(SelectionDAG &DAG, SDNode *N,
-                            APFloat (*Op)(const APFloat &, const APFloat &)) {
+SDValue DAGCombiner::visitFMinMax(SDNode *N) {
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
   EVT VT = N->getValueType(0);
-  const ConstantFPSDNode *N0CFP = isConstOrConstSplatFP(N0);
-  const ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N1);
   const SDNodeFlags Flags = N->getFlags();
   unsigned Opc = N->getOpcode();
   bool PropagatesNaN = Opc == ISD::FMINIMUM || Opc == ISD::FMAXIMUM;
   bool IsMin = Opc == ISD::FMINNUM || Opc == ISD::FMINIMUM;
   SelectionDAG::FlagInserter FlagsInserter(DAG, N);
 
-  if (N0CFP && N1CFP) {
-    const APFloat &C0 = N0CFP->getValueAPF();
-    const APFloat &C1 = N1CFP->getValueAPF();
-    return DAG.getConstantFP(Op(C0, C1), SDLoc(N), VT);
-  }
+  // Constant fold.
+  if (SDValue C = DAG.FoldConstantArithmetic(Opc, SDLoc(N), VT, {N0, N1}))
+    return C;
 
   // Canonicalize to constant on RHS.
   if (DAG.isConstantFPBuildVectorOrConstantFP(N0) &&
       !DAG.isConstantFPBuildVectorOrConstantFP(N1))
     return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N1, N0);
 
-  if (N1CFP) {
+  if (const ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N1)) {
     const APFloat &AF = N1CFP->getValueAPF();
 
     // minnum(X, nan) -> X
@@ -15272,22 +15291,6 @@ static SDValue visitFMinMax(SelectionDAG &DAG, SDNode *N,
   return SDValue();
 }
 
-SDValue DAGCombiner::visitFMINNUM(SDNode *N) {
-  return visitFMinMax(DAG, N, minnum);
-}
-
-SDValue DAGCombiner::visitFMAXNUM(SDNode *N) {
-  return visitFMinMax(DAG, N, maxnum);
-}
-
-SDValue DAGCombiner::visitFMINIMUM(SDNode *N) {
-  return visitFMinMax(DAG, N, minimum);
-}
-
-SDValue DAGCombiner::visitFMAXIMUM(SDNode *N) {
-  return visitFMinMax(DAG, N, maximum);
-}
-
 SDValue DAGCombiner::visitFABS(SDNode *N) {
   SDValue N0 = N->getOperand(0);
   EVT VT = N->getValueType(0);
@@ -18392,8 +18395,8 @@ SDValue DAGCombiner::visitSTORE(SDNode *N) {
   if (StoreSDNode *ST1 = dyn_cast<StoreSDNode>(Chain)) {
     if (ST->isUnindexed() && ST->isSimple() &&
         ST1->isUnindexed() && ST1->isSimple()) {
-      if (ST1->getBasePtr() == Ptr && ST1->getValue() == Value &&
-          ST->getMemoryVT() == ST1->getMemoryVT() &&
+      if (OptLevel != CodeGenOpt::None && ST1->getBasePtr() == Ptr &&
+          ST1->getValue() == Value && ST->getMemoryVT() == ST1->getMemoryVT() &&
           ST->getAddressSpace() == ST1->getAddressSpace()) {
         // If this is a store followed by a store with the same value to the
         // same location, then the store is dead/noop.
@@ -20727,6 +20730,156 @@ static SDValue narrowExtractedVectorLoad(SDNode *Extract, SelectionDAG &DAG) {
   return NewLd;
 }
 
+/// Given  EXTRACT_SUBVECTOR(VECTOR_SHUFFLE(Op0, Op1, Mask)),
+/// try to produce  VECTOR_SHUFFLE(EXTRACT_SUBVECTOR(Op?, ?),
+///                                EXTRACT_SUBVECTOR(Op?, ?),
+///                                Mask'))
+/// iff it is legal and profitable to do so. Notably, the trimmed mask
+/// (containing only the elements that are extracted)
+/// must reference at most two subvectors.
+static SDValue foldExtractSubvectorFromShuffleVector(SDNode *N,
+                                                     SelectionDAG &DAG,
+                                                     const TargetLowering &TLI,
+                                                     bool LegalOperations) {
+  assert(N->getOpcode() == ISD::EXTRACT_SUBVECTOR &&
+         "Must only be called on EXTRACT_SUBVECTOR's");
+
+  SDValue N0 = N->getOperand(0);
+
+  // Only deal with non-scalable vectors.
+  EVT NarrowVT = N->getValueType(0);
+  EVT WideVT = N0.getValueType();
+  if (!NarrowVT.isFixedLengthVector() || !WideVT.isFixedLengthVector())
+    return SDValue();
+
+  // The operand must be a shufflevector.
+  auto *WideShuffleVector = dyn_cast<ShuffleVectorSDNode>(N0);
+  if (!WideShuffleVector)
+    return SDValue();
+
+  // The old shuffleneeds to go away.
+  if (!WideShuffleVector->hasOneUse())
+    return SDValue();
+
+  // And the narrow shufflevector that we'll form must be legal.
+  if (LegalOperations &&
+      !TLI.isOperationLegalOrCustom(ISD::VECTOR_SHUFFLE, NarrowVT))
+    return SDValue();
+
+  uint64_t FirstExtractedEltIdx = N->getConstantOperandVal(1);
+  int NumEltsExtracted = NarrowVT.getVectorNumElements();
+  assert((FirstExtractedEltIdx % NumEltsExtracted) == 0 &&
+         "Extract index is not a multiple of the output vector length.");
+
+  int WideNumElts = WideVT.getVectorNumElements();
+
+  SmallVector<int, 16> NewMask;
+  NewMask.reserve(NumEltsExtracted);
+  SmallSetVector<std::pair<SDValue /*Op*/, int /*SubvectorIndex*/>, 2>
+      DemandedSubvectors;
+
+  // Try to decode the wide mask into narrow mask from at most two subvectors.
+  for (int M : WideShuffleVector->getMask().slice(FirstExtractedEltIdx,
+                                                  NumEltsExtracted)) {
+    assert((M >= -1) && (M < (2 * WideNumElts)) &&
+           "Out-of-bounds shuffle mask?");
+
+    if (M < 0) {
+      // Does not depend on operands, does not require adjustment.
+      NewMask.emplace_back(M);
+      continue;
+    }
+
+    // From which operand of the shuffle does this shuffle mask element pick?
+    int WideShufOpIdx = M / WideNumElts;
+    // Which element of that operand is picked?
+    int OpEltIdx = M % WideNumElts;
+
+    assert((OpEltIdx + WideShufOpIdx * WideNumElts) == M &&
+           "Shuffle mask vector decomposition failure.");
+
+    // And which NumEltsExtracted-sized subvector of that operand is that?
+    int OpSubvecIdx = OpEltIdx / NumEltsExtracted;
+    // And which element within that subvector of that operand is that?
+    int OpEltIdxInSubvec = OpEltIdx % NumEltsExtracted;
+
+    assert((OpEltIdxInSubvec + OpSubvecIdx * NumEltsExtracted) == OpEltIdx &&
+           "Shuffle mask subvector decomposition failure.");
+
+    assert((OpEltIdxInSubvec + OpSubvecIdx * NumEltsExtracted +
+            WideShufOpIdx * WideNumElts) == M &&
+           "Shuffle mask full decomposition failure.");
+
+    SDValue Op = WideShuffleVector->getOperand(WideShufOpIdx);
+
+    if (Op.isUndef()) {
+      // Picking from an undef operand. Let's adjust mask instead.
+      NewMask.emplace_back(-1);
+      continue;
+    }
+
+    // Profitability check: only deal with extractions from the first subvector.
+    if (OpSubvecIdx != 0)
+      return SDValue();
+
+    const std::pair<SDValue, int> DemandedSubvector =
+        std::make_pair(Op, OpSubvecIdx);
+
+    if (DemandedSubvectors.insert(DemandedSubvector)) {
+      if (DemandedSubvectors.size() > 2)
+        return SDValue(); // We can't handle more than two subvectors.
+      // How many elements into the WideVT does this subvector start?
+      int Index = NumEltsExtracted * OpSubvecIdx;
+      // Bail out if the extraction isn't going to be cheap.
+      if (!TLI.isExtractSubvectorCheap(NarrowVT, WideVT, Index))
+        return SDValue();
+    }
+
+    // Ok, but from which operand of the new shuffle will this element pick?
+    int NewOpIdx =
+        getFirstIndexOf(DemandedSubvectors.getArrayRef(), DemandedSubvector);
+    assert((NewOpIdx == 0 || NewOpIdx == 1) && "Unexpected operand index.");
+
+    int AdjM = OpEltIdxInSubvec + NewOpIdx * NumEltsExtracted;
+    NewMask.emplace_back(AdjM);
+  }
+  assert(NewMask.size() == (unsigned)NumEltsExtracted && "Produced bad mask.");
+  assert(DemandedSubvectors.size() <= 2 &&
+         "Should have ended up demanding at most two subvectors.");
+
+  // Did we discover that the shuffle does not actually depend on operands?
+  if (DemandedSubvectors.empty())
+    return DAG.getUNDEF(NarrowVT);
+
+  // We still perform the exact same EXTRACT_SUBVECTOR,  just on different
+  // operand[s]/index[es], so there is no point in checking for it's legality.
+
+  // Do not turn a legal shuffle into an illegal one.
+  if (TLI.isShuffleMaskLegal(WideShuffleVector->getMask(), WideVT) &&
+      !TLI.isShuffleMaskLegal(NewMask, NarrowVT))
+    return SDValue();
+
+  SDLoc DL(N);
+
+  SmallVector<SDValue, 2> NewOps;
+  for (const std::pair<SDValue /*Op*/, int /*SubvectorIndex*/>
+           &DemandedSubvector : DemandedSubvectors) {
+    // How many elements into the WideVT does this subvector start?
+    int Index = NumEltsExtracted * DemandedSubvector.second;
+    SDValue IndexC = DAG.getVectorIdxConstant(Index, DL);
+    NewOps.emplace_back(DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NarrowVT,
+                                    DemandedSubvector.first, IndexC));
+  }
+  assert((NewOps.size() == 1 || NewOps.size() == 2) &&
+         "Should end up with either one or two ops");
+
+  // If we ended up with only one operand, pad with an undef.
+  if (NewOps.size() == 1)
+    NewOps.emplace_back(DAG.getUNDEF(NarrowVT));
+
+  return DAG.getVectorShuffle(NarrowVT, DL, NewOps[0], NewOps[1], NewMask);
+}
+
 SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode *N) {
   EVT NVT = N->getValueType(0);
   SDValue V = N->getOperand(0);
@@ -20840,6 +20993,10 @@ SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode *N) {
     }
   }
 
+  if (SDValue V =
+          foldExtractSubvectorFromShuffleVector(N, DAG, TLI, LegalOperations))
+    return V;
+
   V = peekThroughBitcasts(V);
 
   // If the input is a build vector. Try to make a smaller build vector.
@@ -22424,15 +22581,9 @@ SDValue DAGCombiner::SimplifyVBinOp(SDNode *N, const SDLoc &DL) {
 
   SDValue LHS = N->getOperand(0);
   SDValue RHS = N->getOperand(1);
-  SDValue Ops[] = {LHS, RHS};
   unsigned Opcode = N->getOpcode();
   SDNodeFlags Flags = N->getFlags();
 
-  // See if we can constant fold the vector operation.
-  if (SDValue Fold = DAG.FoldConstantArithmetic(Opcode, SDLoc(LHS),
-                                                LHS.getValueType(), Ops))
-    return Fold;
-
   // Move unary shuffles with identical masks after a vector binop:
   // VBinOp (shuffle A, Undef, Mask), (shuffle B, Undef, Mask))
   //   --> shuffle (VBinOp A, B), Undef, Mask
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 08598eeded7a..5dfb65ef131a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -3367,13 +3367,13 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
   }
   case ISD::FSHL:
   case ISD::FSHR:
-    if (TLI.expandFunnelShift(Node, Tmp1, DAG))
-      Results.push_back(Tmp1);
+    if (SDValue Expanded = TLI.expandFunnelShift(Node, DAG))
+      Results.push_back(Expanded);
     break;
   case ISD::ROTL:
   case ISD::ROTR:
-    if (TLI.expandROT(Node, true /*AllowVectorOps*/, Tmp1, DAG))
-      Results.push_back(Tmp1);
+    if (SDValue Expanded = TLI.expandROT(Node, true /*AllowVectorOps*/, DAG))
+      Results.push_back(Expanded);
     break;
   case ISD::SADDSAT:
   case ISD::UADDSAT:
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index 1fa4d88fcb4a..518e525e13d0 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -1277,8 +1277,7 @@ SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N, bool IsVP) {
 
 SDValue DAGTypeLegalizer::PromoteIntRes_Rotate(SDNode *N) {
   // Lower the rotate to shifts and ORs which can be promoted.
-  SDValue Res;
-  TLI.expandROT(N, true /*AllowVectorOps*/, Res, DAG);
+  SDValue Res = TLI.expandROT(N, true /*AllowVectorOps*/, DAG);
   ReplaceValueWith(SDValue(N, 0), Res);
   return SDValue();
 }
@@ -1286,7 +1285,7 @@ SDValue DAGTypeLegalizer::PromoteIntRes_Rotate(SDNode *N) {
 SDValue DAGTypeLegalizer::PromoteIntRes_FunnelShift(SDNode *N) {
   SDValue Hi = GetPromotedInteger(N->getOperand(0));
   SDValue Lo = GetPromotedInteger(N->getOperand(1));
-  SDValue Amount = GetPromotedInteger(N->getOperand(2));
+  SDValue Amt = GetPromotedInteger(N->getOperand(2));
 
   SDLoc DL(N);
   EVT OldVT = N->getOperand(0).getValueType();
@@ -1297,21 +1296,20 @@ SDValue DAGTypeLegalizer::PromoteIntRes_FunnelShift(SDNode *N) {
   unsigned NewBits = VT.getScalarSizeInBits();
 
   // Amount has to be interpreted modulo the old bit width.
-  Amount =
-      DAG.getNode(ISD::UREM, DL, VT, Amount, DAG.getConstant(OldBits, DL, VT));
+  Amt = DAG.getNode(ISD::UREM, DL, VT, Amt, DAG.getConstant(OldBits, DL, VT));
 
   // If the promoted type is twice the size (or more), then we use the
   // traditional funnel 'double' shift codegen. This isn't necessary if the
   // shift amount is constant.
   // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z % bw)) >> bw.
   // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z % bw)).
-  if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amount) &&
+  if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) &&
       !TLI.isOperationLegalOrCustom(Opcode, VT)) {
     SDValue HiShift = DAG.getConstant(OldBits, DL, VT);
     Hi = DAG.getNode(ISD::SHL, DL, VT, Hi, HiShift);
     Lo = DAG.getZeroExtendInReg(Lo, DL, OldVT);
     SDValue Res = DAG.getNode(ISD::OR, DL, VT, Hi, Lo);
-    Res = DAG.getNode(IsFSHR ? ISD::SRL : ISD::SHL, DL, VT, Res, Amount);
+    Res = DAG.getNode(IsFSHR ? ISD::SRL : ISD::SHL, DL, VT, Res, Amt);
     if (!IsFSHR)
       Res = DAG.getNode(ISD::SRL, DL, VT, Res, HiShift);
     return Res;
@@ -1324,9 +1322,9 @@ SDValue DAGTypeLegalizer::PromoteIntRes_FunnelShift(SDNode *N) {
   // Increase Amount to shift the result into the lower bits of the promoted
   // type.
   if (IsFSHR)
-    Amount = DAG.getNode(ISD::ADD, DL, VT, Amount, ShiftOffset);
+    Amt = DAG.getNode(ISD::ADD, DL, VT, Amt, ShiftOffset);
 
-  return DAG.getNode(Opcode, DL, VT, Hi, Lo, Amount);
+  return DAG.getNode(Opcode, DL, VT, Hi, Lo, Amt);
 }
 
 SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
index 98312f91d8c0..03dcd0f6d2c9 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
@@ -83,7 +83,7 @@ void DAGTypeLegalizer::PerformExpensiveChecks() {
       SDValue Res(&Node, i);
       bool Failed = false;
       // Don't create a value in map.
-      auto ResId = (ValueToIdMap.count(Res)) ? ValueToIdMap[Res] : 0;
+      auto ResId = ValueToIdMap.lookup(Res);
 
       unsigned Mapped = 0;
       if (ResId && (ReplacedValues.find(ResId) != ReplacedValues.end())) {
@@ -301,7 +301,7 @@ ScanOperands:
       if (IgnoreNodeResults(N->getOperand(i).getNode()))
         continue;
 
-      const auto Op = N->getOperand(i);
+      const auto &Op = N->getOperand(i);
       LLVM_DEBUG(dbgs() << "Analyzing operand: "; Op.dump(&DAG));
       EVT OpVT = Op.getValueType();
       switch (getTypeAction(OpVT)) {
@@ -1007,11 +1007,7 @@ SDValue DAGTypeLegalizer::JoinIntegers(SDValue Lo, SDValue Hi) {
 ///
 /// ValVT is the type of values that produced the boolean.
 SDValue DAGTypeLegalizer::PromoteTargetBoolean(SDValue Bool, EVT ValVT) {
-  SDLoc dl(Bool);
-  EVT BoolVT = getSetCCResultType(ValVT);
-  ISD::NodeType ExtendCode =
-      TargetLowering::getExtendForContent(TLI.getBooleanContents(ValVT));
-  return DAG.getNode(ExtendCode, dl, BoolVT, Bool);
+  return TLI.promoteTargetBoolean(DAG, Bool, ValVT);
 }
 
 /// Return the lower LoVT bits of Op in Lo and the upper HiVT bits in Hi.
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index 88a28a3be53e..1493f36fcd3e 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -254,69 +254,6 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
 
   SDNode *Node = DAG.UpdateNodeOperands(Op.getNode(), Ops);
 
-  if (Op.getOpcode() == ISD::LOAD) {
-    LoadSDNode *LD = cast<LoadSDNode>(Node);
-    ISD::LoadExtType ExtType = LD->getExtensionType();
-    if (LD->getMemoryVT().isVector() && ExtType != ISD::NON_EXTLOAD) {
-      LLVM_DEBUG(dbgs() << "\nLegalizing extending vector load: ";
-                 Node->dump(&DAG));
-      switch (TLI.getLoadExtAction(LD->getExtensionType(), LD->getValueType(0),
-                                   LD->getMemoryVT())) {
-      default: llvm_unreachable("This action is not supported yet!");
-      case TargetLowering::Legal:
-        return TranslateLegalizeResults(Op, Node);
-      case TargetLowering::Custom: {
-        SmallVector<SDValue, 2> ResultVals;
-        if (LowerOperationWrapper(Node, ResultVals)) {
-          if (ResultVals.empty())
-            return TranslateLegalizeResults(Op, Node);
-
-          Changed = true;
-          return RecursivelyLegalizeResults(Op, ResultVals);
-        }
-        LLVM_FALLTHROUGH;
-      }
-      case TargetLowering::Expand: {
-        Changed = true;
-        std::pair<SDValue, SDValue> Tmp = ExpandLoad(Node);
-        AddLegalizedOperand(Op.getValue(0), Tmp.first);
-        AddLegalizedOperand(Op.getValue(1), Tmp.second);
-        return Op.getResNo() ? Tmp.first : Tmp.second;
-      }
-      }
-    }
-  } else if (Op.getOpcode() == ISD::STORE) {
-    StoreSDNode *ST = cast<StoreSDNode>(Node);
-    EVT StVT = ST->getMemoryVT();
-    MVT ValVT = ST->getValue().getSimpleValueType();
-    if (StVT.isVector() && ST->isTruncatingStore()) {
-      LLVM_DEBUG(dbgs() << "\nLegalizing truncating vector store: ";
-                 Node->dump(&DAG));
-      switch (TLI.getTruncStoreAction(ValVT, StVT)) {
-      default: llvm_unreachable("This action is not supported yet!");
-      case TargetLowering::Legal:
-        return TranslateLegalizeResults(Op, Node);
-      case TargetLowering::Custom: {
-        SmallVector<SDValue, 1> ResultVals;
-        if (LowerOperationWrapper(Node, ResultVals)) {
-          if (ResultVals.empty())
-            return TranslateLegalizeResults(Op, Node);
-
-          Changed = true;
-          return RecursivelyLegalizeResults(Op, ResultVals);
-        }
-        LLVM_FALLTHROUGH;
-      }
-      case TargetLowering::Expand: {
-        Changed = true;
-        SDValue Chain = ExpandStore(Node);
-        AddLegalizedOperand(Op, Chain);
-        return Chain;
-      }
-      }
-    }
-  }
-
   bool HasVectorValueOrOp =
       llvm::any_of(Node->values(), [](EVT T) { return T.isVector(); }) ||
       llvm::any_of(Node->op_values(),
@@ -329,6 +266,22 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
   switch (Op.getOpcode()) {
   default:
     return TranslateLegalizeResults(Op, Node);
+  case ISD::LOAD: {
+    LoadSDNode *LD = cast<LoadSDNode>(Node);
+    ISD::LoadExtType ExtType = LD->getExtensionType();
+    EVT LoadedVT = LD->getMemoryVT();
+    if (LoadedVT.isVector() && ExtType != ISD::NON_EXTLOAD)
+      Action = TLI.getLoadExtAction(ExtType, LD->getValueType(0), LoadedVT);
+    break;
+  }
+  case ISD::STORE: {
+    StoreSDNode *ST = cast<StoreSDNode>(Node);
+    EVT StVT = ST->getMemoryVT();
+    MVT ValVT = ST->getValue().getSimpleValueType();
+    if (StVT.isVector() && ST->isTruncatingStore())
+      Action = TLI.getTruncStoreAction(ValVT, StVT);
+    break;
+  }
   case ISD::MERGE_VALUES:
     Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
     // This operation lies about being legal: when it claims to be legal,
@@ -512,6 +465,8 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
   switch (Action) {
   default: llvm_unreachable("This action is not supported yet!");
   case TargetLowering::Promote:
+    assert((Op.getOpcode() != ISD::LOAD && Op.getOpcode() != ISD::STORE) &&
+           "This action is not supported yet!");
     LLVM_DEBUG(dbgs() << "Promoting\n");
     Promote(Node, ResultVals);
     assert(!ResultVals.empty() && "No results for promotion?");
@@ -731,8 +686,16 @@ SDValue VectorLegalizer::ExpandStore(SDNode *N) {
 }
 
 void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
-  SDValue Tmp;
   switch (Node->getOpcode()) {
+  case ISD::LOAD: {
+    std::pair<SDValue, SDValue> Tmp = ExpandLoad(Node);
+    Results.push_back(Tmp.first);
+    Results.push_back(Tmp.second);
+    return;
+  }
+  case ISD::STORE:
+    Results.push_back(ExpandStore(Node));
+    return;
   case ISD::MERGE_VALUES:
     for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
       Results.push_back(Node->getOperand(i));
@@ -804,15 +767,15 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
     break;
   case ISD::FSHL:
   case ISD::FSHR:
-    if (TLI.expandFunnelShift(Node, Tmp, DAG)) {
-      Results.push_back(Tmp);
+    if (SDValue Expanded = TLI.expandFunnelShift(Node, DAG)) {
+      Results.push_back(Expanded);
       return;
     }
     break;
   case ISD::ROTL:
   case ISD::ROTR:
-    if (TLI.expandROT(Node, false /*AllowVectorOps*/, Tmp, DAG)) {
-      Results.push_back(Tmp);
+    if (SDValue Expanded = TLI.expandROT(Node, false /*AllowVectorOps*/, DAG)) {
+      Results.push_back(Expanded);
       return;
     }
     break;
diff --git a/llvm/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp b/llvm/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp
index 2695ed36991c..3d5c4c5b1cae 100644
--- a/llvm/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp
@@ -168,10 +168,9 @@ void ResourcePriorityQueue::initNodes(std::vector<SUnit> &sunits) {
   SUnits = &sunits;
   NumNodesSolelyBlocking.resize(SUnits->size(), 0);
 
-  for (unsigned i = 0, e = SUnits->size(); i != e; ++i) {
-    SUnit *SU = &(*SUnits)[i];
-    initNumRegDefsLeft(SU);
-    SU->NodeQueueId = 0;
+  for (SUnit &SU : *SUnits) {
+    initNumRegDefsLeft(&SU);
+    SU.NodeQueueId = 0;
   }
 }
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
index 84e6d2a16422..aec2cf38b400 100644
--- a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
@@ -442,33 +442,32 @@ void ScheduleDAGSDNodes::AddSchedEdges() {
   bool UnitLatencies = forceUnitLatencies();
 
   // Pass 2: add the preds, succs, etc.
-  for (unsigned su = 0, e = SUnits.size(); su != e; ++su) {
-    SUnit *SU = &SUnits[su];
-    SDNode *MainNode = SU->getNode();
+  for (SUnit &SU : SUnits) {
+    SDNode *MainNode = SU.getNode();
 
     if (MainNode->isMachineOpcode()) {
       unsigned Opc = MainNode->getMachineOpcode();
       const MCInstrDesc &MCID = TII->get(Opc);
       for (unsigned i = 0; i != MCID.getNumOperands(); ++i) {
         if (MCID.getOperandConstraint(i, MCOI::TIED_TO) != -1) {
-          SU->isTwoAddress = true;
+          SU.isTwoAddress = true;
           break;
         }
       }
       if (MCID.isCommutable())
-        SU->isCommutable = true;
+        SU.isCommutable = true;
     }
 
     // Find all predecessors and successors of the group.
-    for (SDNode *N = SU->getNode(); N; N = N->getGluedNode()) {
+    for (SDNode *N = SU.getNode(); N; N = N->getGluedNode()) {
       if (N->isMachineOpcode() &&
           TII->get(N->getMachineOpcode()).getImplicitDefs()) {
-        SU->hasPhysRegClobbers = true;
+        SU.hasPhysRegClobbers = true;
         unsigned NumUsed = InstrEmitter::CountResults(N);
         while (NumUsed != 0 && !N->hasAnyUseOfValue(NumUsed - 1))
           --NumUsed;    // Skip over unused values at the end.
         if (NumUsed > TII->get(N->getMachineOpcode()).getNumDefs())
-          SU->hasPhysRegDefs = true;
+          SU.hasPhysRegDefs = true;
       }
 
       for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
@@ -477,7 +476,8 @@ void ScheduleDAGSDNodes::AddSchedEdges() {
         if (isPassiveNode(OpN)) continue;   // Not scheduled.
         SUnit *OpSU = &SUnits[OpN->getNodeId()];
         assert(OpSU && "Node has no SUnit!");
-        if (OpSU == SU) continue;           // In the same group.
+        if (OpSU == &SU)
+          continue; // In the same group.
 
         EVT OpVT = N->getOperand(i).getValueType();
         assert(OpVT != MVT::Glue && "Glued nodes should be in same sunit!");
@@ -508,10 +508,10 @@ void ScheduleDAGSDNodes::AddSchedEdges() {
         Dep.setLatency(OpLatency);
         if (!isChain && !UnitLatencies) {
           computeOperandLatency(OpN, N, i, Dep);
-          ST.adjustSchedDependency(OpSU, DefIdx, SU, i, Dep);
+          ST.adjustSchedDependency(OpSU, DefIdx, &SU, i, Dep);
         }
 
-        if (!SU->addPred(Dep) && !Dep.isCtrl() && OpSU->NumRegDefsLeft > 1) {
+        if (!SU.addPred(Dep) && !Dep.isCtrl() && OpSU->NumRegDefsLeft > 1) {
           // Multiple register uses are combined in the same SUnit. For example,
           // we could have a set of glued nodes with all their defs consumed by
           // another set of glued nodes. Register pressure tracking sees this as
@@ -721,10 +721,7 @@ void ScheduleDAGSDNodes::dumpSchedule() const {
 ///
 void ScheduleDAGSDNodes::VerifyScheduledSequence(bool isBottomUp) {
   unsigned ScheduledNodes = ScheduleDAG::VerifyScheduledDAG(isBottomUp);
-  unsigned Noops = 0;
-  for (unsigned i = 0, e = Sequence.size(); i != e; ++i)
-    if (!Sequence[i])
-      ++Noops;
+  unsigned Noops = llvm::count(Sequence, nullptr);
   assert(Sequence.size() - Noops == ScheduledNodes &&
          "The number of nodes scheduled doesn't match the expected number!");
 }
@@ -911,8 +908,7 @@ EmitSchedule(MachineBasicBlock::iterator &InsertPos) {
     }
   }
 
-  for (unsigned i = 0, e = Sequence.size(); i != e; i++) {
-    SUnit *SU = Sequence[i];
+  for (SUnit *SU : Sequence) {
     if (!SU) {
       // Null SUnit* is a noop.
       TII->insertNoop(*Emitter.getBlock(), InsertPos);
diff --git a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp
index 540a6e3efbe1..10940478010e 100644
--- a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp
@@ -169,11 +169,11 @@ void ScheduleDAGVLIW::listScheduleTopDown() {
   releaseSuccessors(&EntrySU);
 
   // All leaves to AvailableQueue.
-  for (unsigned i = 0, e = SUnits.size(); i != e; ++i) {
+  for (SUnit &SU : SUnits) {
     // It is available if it has no predecessors.
-    if (SUnits[i].Preds.empty()) {
-      AvailableQueue->push(&SUnits[i]);
-      SUnits[i].isAvailable = true;
+    if (SU.Preds.empty()) {
+      AvailableQueue->push(&SU);
+      SU.isAvailable = true;
     }
   }
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index c282e03387dd..2ae0d4df7b77 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -2499,7 +2499,8 @@ bool SelectionDAG::MaskedValueIsAllOnes(SDValue V, const APInt &Mask,
 /// sense to specify which elements are demanded or undefined, therefore
 /// they are simply ignored.
 bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
-                                APInt &UndefElts, unsigned Depth) {
+                                APInt &UndefElts, unsigned Depth) const {
+  unsigned Opcode = V.getOpcode();
   EVT VT = V.getValueType();
   assert(VT.isVector() && "Vector type expected");
 
@@ -2511,7 +2512,7 @@ bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
 
   // Deal with some common cases here that work for both fixed and scalable
   // vector types.
-  switch (V.getOpcode()) {
+  switch (Opcode) {
   case ISD::SPLAT_VECTOR:
     UndefElts = V.getOperand(0).isUndef()
                     ? APInt::getAllOnes(DemandedElts.getBitWidth())
@@ -2537,7 +2538,12 @@ bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
   case ISD::SIGN_EXTEND:
   case ISD::ZERO_EXTEND:
     return isSplatValue(V.getOperand(0), DemandedElts, UndefElts, Depth + 1);
-  }
+  default:
+    if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
+        Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
+      return TLI->isSplatValueForTargetNode(V, DemandedElts, UndefElts, Depth);
+    break;
+}
 
   // We don't support other cases than those above for scalable vectors at
   // the moment.
@@ -2548,7 +2554,7 @@ bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
   assert(NumElts == DemandedElts.getBitWidth() && "Vector size mismatch");
   UndefElts = APInt::getZero(NumElts);
 
-  switch (V.getOpcode()) {
+  switch (Opcode) {
   case ISD::BUILD_VECTOR: {
     SDValue Scl;
     for (unsigned i = 0; i != NumElts; ++i) {
@@ -2600,13 +2606,30 @@ bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
     }
     break;
   }
+  case ISD::ANY_EXTEND_VECTOR_INREG:
+  case ISD::SIGN_EXTEND_VECTOR_INREG:
+  case ISD::ZERO_EXTEND_VECTOR_INREG: {
+    // Widen the demanded elts by the src element count.
+    SDValue Src = V.getOperand(0);
+    // We don't support scalable vectors at the moment.
+    if (Src.getValueType().isScalableVector())
+      return false;
+    unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
+    APInt UndefSrcElts;
+    APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts);
+    if (isSplatValue(Src, DemandedSrcElts, UndefSrcElts, Depth + 1)) {
+      UndefElts = UndefSrcElts.truncOrSelf(NumElts);
+      return true;
+    }
+    break;
+  }
   }
 
   return false;
 }
 
 /// Helper wrapper to main isSplatValue function.
-bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) {
+bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) const {
   EVT VT = V.getValueType();
   assert(VT.isVector() && "Vector type expected");
 
@@ -5291,9 +5314,10 @@ SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
   if (isUndef(Opcode, Ops))
     return getUNDEF(VT);
 
-  // Handle the case of two scalars.
+  // Handle binops special cases.
   if (NumOps == 2) {
-    // TODO: Move foldConstantFPMath here?
+    if (SDValue CFP = foldConstantFPMath(Opcode, DL, VT, Ops[0], Ops[1]))
+      return CFP;
 
     if (auto *C1 = dyn_cast<ConstantSDNode>(Ops[0])) {
       if (auto *C2 = dyn_cast<ConstantSDNode>(Ops[1])) {
@@ -5463,10 +5487,11 @@ SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL,
   //       should. That will require dealing with a potentially non-default
   //       rounding mode, checking the "opStatus" return value from the APFloat
   //       math calculations, and possibly other variations.
-  auto *N1CFP = dyn_cast<ConstantFPSDNode>(N1.getNode());
-  auto *N2CFP = dyn_cast<ConstantFPSDNode>(N2.getNode());
+  ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N1, /*AllowUndefs*/ false);
+  ConstantFPSDNode *N2CFP = isConstOrConstSplatFP(N2, /*AllowUndefs*/ false);
   if (N1CFP && N2CFP) {
-    APFloat C1 = N1CFP->getValueAPF(), C2 = N2CFP->getValueAPF();
+    APFloat C1 = N1CFP->getValueAPF(); // make copy
+    const APFloat &C2 = N2CFP->getValueAPF();
     switch (Opcode) {
     case ISD::FADD:
       C1.add(C2, APFloat::rmNearestTiesToEven);
@@ -5486,6 +5511,14 @@ SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL,
     case ISD::FCOPYSIGN:
       C1.copySign(C2);
       return getConstantFP(C1, DL, VT);
+    case ISD::FMINNUM:
+      return getConstantFP(minnum(C1, C2), DL, VT);
+    case ISD::FMAXNUM:
+      return getConstantFP(maxnum(C1, C2), DL, VT);
+    case ISD::FMINIMUM:
+      return getConstantFP(minimum(C1, C2), DL, VT);
+    case ISD::FMAXIMUM:
+      return getConstantFP(maximum(C1, C2), DL, VT);
     default: break;
     }
   }
@@ -5502,8 +5535,9 @@ SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL,
   switch (Opcode) {
   case ISD::FSUB:
     // -0.0 - undef --> undef (consistent with "fneg undef")
-    if (N1CFP && N1CFP->getValueAPF().isNegZero() && N2.isUndef())
-      return getUNDEF(VT);
+    if (ConstantFPSDNode *N1C = isConstOrConstSplatFP(N1, /*AllowUndefs*/ true))
+      if (N1C && N1C->getValueAPF().isNegZero() && N2.isUndef())
+        return getUNDEF(VT);
     LLVM_FALLTHROUGH;
 
   case ISD::FADD:
@@ -5962,9 +5996,6 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
   if (SDValue SV = FoldConstantArithmetic(Opcode, DL, VT, {N1, N2}))
     return SV;
 
-  if (SDValue V = foldConstantFPMath(Opcode, DL, VT, N1, N2))
-    return V;
-
   // Canonicalize an UNDEF to the RHS, even over a constant.
   if (N1.isUndef()) {
     if (TLI->isCommutativeBinOp(Opcode)) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 7726a0007e44..63cd723cf6da 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -1036,7 +1036,6 @@ void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis *aa,
   AA = aa;
   GFI = gfi;
   LibInfo = li;
-  DL = &DAG.getDataLayout();
   Context = DAG.getContext();
   LPadToCallSiteMap.clear();
   SL->init(DAG.getTargetLoweringInfo(), TM, DAG.getDataLayout());
@@ -1626,6 +1625,9 @@ SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
     if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(C))
       return getValue(Equiv->getGlobalValue());
 
+    if (const auto *NC = dyn_cast<NoCFIValue>(C))
+      return getValue(NC->getGlobalValue());
+
     VectorType *VecTy = cast<VectorType>(V->getType());
 
     // Now that we know the number and type of the elements, get that number of
@@ -1921,8 +1923,8 @@ void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
                         DAG.getDataLayout().getAllocaAddrSpace()),
                     PtrValueVTs);
 
-    SDValue RetPtr = DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
-                                        DemoteReg, PtrValueVTs[0]);
+    SDValue RetPtr =
+        DAG.getCopyFromReg(Chain, getCurSDLoc(), DemoteReg, PtrValueVTs[0]);
     SDValue RetOp = getValue(I.getOperand(0));
 
     SmallVector<EVT, 4> ValueVTs, MemVTs;
@@ -2657,7 +2659,8 @@ void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
   SDLoc dl = getCurSDLoc();
   SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
   const Module &M = *ParentBB->getParent()->getFunction().getParent();
-  Align Align = DL->getPrefTypeAlign(Type::getInt8PtrTy(M.getContext()));
+  Align Align =
+      DAG.getDataLayout().getPrefTypeAlign(Type::getInt8PtrTy(M.getContext()));
 
   // Generate code to load the content of the guard slot.
   SDValue GuardVal = DAG.getLoad(
@@ -3058,14 +3061,14 @@ void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
 void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,
                                            MachineBasicBlock *Last) {
   // Update JTCases.
-  for (unsigned i = 0, e = SL->JTCases.size(); i != e; ++i)
-    if (SL->JTCases[i].first.HeaderBB == First)
-      SL->JTCases[i].first.HeaderBB = Last;
+  for (JumpTableBlock &JTB : SL->JTCases)
+    if (JTB.first.HeaderBB == First)
+      JTB.first.HeaderBB = Last;
 
   // Update BitTestCases.
-  for (unsigned i = 0, e = SL->BitTestCases.size(); i != e; ++i)
-    if (SL->BitTestCases[i].Parent == First)
-      SL->BitTestCases[i].Parent = Last;
+  for (BitTestBlock &BTB : SL->BitTestCases)
+    if (BTB.Parent == First)
+      BTB.Parent = Last;
 }
 
 void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
@@ -3111,6 +3114,8 @@ void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
 
 void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {
   SDNodeFlags Flags;
+  if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
+    Flags.copyFMF(*FPOp);
 
   SDValue Op = getValue(I.getOperand(0));
   SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(),
@@ -3881,7 +3886,8 @@ void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
       unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
       if (Field) {
         // N = N + Offset
-        uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field);
+        uint64_t Offset =
+            DAG.getDataLayout().getStructLayout(StTy)->getElementOffset(Field);
 
         // In an inbounds GEP with an offset that is nonnegative even when
         // interpreted as signed, assume there is no unsigned overflow.
@@ -3898,7 +3904,8 @@ void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
       // (and fix up the result later).
       unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);
       MVT IdxTy = MVT::getIntegerVT(IdxSize);
-      TypeSize ElementSize = DL->getTypeAllocSize(GTI.getIndexedType());
+      TypeSize ElementSize =
+          DAG.getDataLayout().getTypeAllocSize(GTI.getIndexedType());
       // We intentionally mask away the high bits here; ElementSize may not
       // fit in IdxTy.
       APInt ElementMul(IdxSize, ElementSize.getKnownMinSize());
@@ -4788,7 +4795,7 @@ void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
     }
 
     // Use TargetConstant instead of a regular constant for immarg.
-    EVT VT = TLI.getValueType(*DL, Arg->getType(), true);
+    EVT VT = TLI.getValueType(DAG.getDataLayout(), Arg->getType(), true);
     if (const ConstantInt *CI = dyn_cast<ConstantInt>(Arg)) {
       assert(CI->getBitWidth() <= 64 &&
              "large intrinsic immediates not handled");
@@ -6571,7 +6578,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
     } else {
       EVT PtrTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
       const Value *Global = TLI.getSDagStackGuard(M);
-      Align Align = DL->getPrefTypeAlign(Global->getType());
+      Align Align = DAG.getDataLayout().getPrefTypeAlign(Global->getType());
       Res = DAG.getLoad(PtrTy, sdl, Chain, getValue(Global),
                         MachinePointerInfo(Global, 0), Align,
                         MachineMemOperand::MOVolatile);
@@ -7127,12 +7134,10 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
     }
     SDValue VectorStep = DAG.getStepVector(sdl, VecTy);
     SDValue VectorInduction = DAG.getNode(
-        ISD::UADDO, sdl, DAG.getVTList(VecTy, CCVT), VectorIndex, VectorStep);
-    SDValue SetCC = DAG.getSetCC(sdl, CCVT, VectorInduction.getValue(0),
+        ISD::UADDSAT, sdl, VecTy, VectorIndex, VectorStep);
+    SDValue SetCC = DAG.getSetCC(sdl, CCVT, VectorInduction,
                                  VectorTripCount, ISD::CondCode::SETULT);
-    setValue(&I, DAG.getNode(ISD::AND, sdl, CCVT,
-                             DAG.getNOT(sdl, VectorInduction.getValue(1), CCVT),
-                             SetCC));
+    setValue(&I, SetCC);
     return;
   }
   case Intrinsic::experimental_vector_insert: {
@@ -7317,32 +7322,26 @@ static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin) {
 
 void SelectionDAGBuilder::visitVPLoadGather(const VPIntrinsic &VPIntrin, EVT VT,
                                             SmallVector<SDValue, 7> &OpValues,
-                                            bool isGather) {
+                                            bool IsGather) {
   SDLoc DL = getCurSDLoc();
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   Value *PtrOperand = VPIntrin.getArgOperand(0);
-  MaybeAlign Alignment = DAG.getEVTAlign(VT);
+  MaybeAlign Alignment = VPIntrin.getPointerAlignment();
+  if (!Alignment)
+    Alignment = DAG.getEVTAlign(VT);
   AAMDNodes AAInfo = VPIntrin.getAAMetadata();
   const MDNode *Ranges = VPIntrin.getMetadata(LLVMContext::MD_range);
   SDValue LD;
   bool AddToChain = true;
-  if (!isGather) {
+  if (!IsGather) {
     // Do not serialize variable-length loads of constant memory with
     // anything.
-    MemoryLocation ML;
-    if (VT.isScalableVector())
-      ML = MemoryLocation::getAfter(PtrOperand);
-    else
-      ML = MemoryLocation(
-          PtrOperand,
-          LocationSize::precise(
-              DAG.getDataLayout().getTypeStoreSize(VPIntrin.getType())),
-          AAInfo);
+    MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
     AddToChain = !AA || !AA->pointsToConstantMemory(ML);
     SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
     MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
         MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,
-        VT.getStoreSize().getKnownMinSize(), *Alignment, AAInfo, Ranges);
+        MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);
     LD = DAG.getLoadVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2],
                        MMO, false /*IsExpanding */);
   } else {
@@ -7380,18 +7379,20 @@ void SelectionDAGBuilder::visitVPLoadGather(const VPIntrinsic &VPIntrin, EVT VT,
 
 void SelectionDAGBuilder::visitVPStoreScatter(const VPIntrinsic &VPIntrin,
                                               SmallVector<SDValue, 7> &OpValues,
-                                              bool isScatter) {
+                                              bool IsScatter) {
   SDLoc DL = getCurSDLoc();
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   Value *PtrOperand = VPIntrin.getArgOperand(1);
   EVT VT = OpValues[0].getValueType();
-  MaybeAlign Alignment = DAG.getEVTAlign(VT);
+  MaybeAlign Alignment = VPIntrin.getPointerAlignment();
+  if (!Alignment)
+    Alignment = DAG.getEVTAlign(VT);
   AAMDNodes AAInfo = VPIntrin.getAAMetadata();
   SDValue ST;
-  if (!isScatter) {
+  if (!IsScatter) {
     MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
         MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore,
-        VT.getStoreSize().getKnownMinSize(), *Alignment, AAInfo);
+        MemoryLocation::UnknownSize, *Alignment, AAInfo);
     ST =
         DAG.getStoreVP(getMemoryRoot(), DL, OpValues[0], OpValues[1],
                        OpValues[2], OpValues[3], MMO, false /* IsTruncating */);
@@ -7690,8 +7691,9 @@ static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
     LoadInput = ConstantExpr::getBitCast(const_cast<Constant *>(LoadInput),
                                          PointerType::getUnqual(LoadTy));
 
-    if (const Constant *LoadCst = ConstantFoldLoadFromConstPtr(
-            const_cast<Constant *>(LoadInput), LoadTy, *Builder.DL))
+    if (const Constant *LoadCst =
+            ConstantFoldLoadFromConstPtr(const_cast<Constant *>(LoadInput),
+                                         LoadTy, Builder.DAG.getDataLayout()))
       return Builder.getValue(LoadCst);
   }
 
@@ -9646,8 +9648,8 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
   // We push in swifterror return as the last element of CLI.Ins.
   ArgListTy &Args = CLI.getArgs();
   if (supportSwiftError()) {
-    for (unsigned i = 0, e = Args.size(); i != e; ++i) {
-      if (Args[i].IsSwiftError) {
+    for (const ArgListEntry &Arg : Args) {
+      if (Arg.IsSwiftError) {
         ISD::InputArg MyFlags;
         MyFlags.VT = getPointerTy(DL);
         MyFlags.ArgVT = EVT(getPointerTy(DL));
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
index d6122aa0a739..ea48042a5dcf 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
@@ -190,7 +190,6 @@ public:
   static const unsigned LowestSDNodeOrder = 1;
 
   SelectionDAG &DAG;
-  const DataLayout *DL = nullptr;
   AAResults *AA = nullptr;
   const TargetLibraryInfo *LibInfo;
 
@@ -568,9 +567,9 @@ private:
   void visitTargetIntrinsic(const CallInst &I, unsigned Intrinsic);
   void visitConstrainedFPIntrinsic(const ConstrainedFPIntrinsic &FPI);
   void visitVPLoadGather(const VPIntrinsic &VPIntrin, EVT VT,
-                         SmallVector<SDValue, 7> &OpValues, bool isGather);
+                         SmallVector<SDValue, 7> &OpValues, bool IsGather);
   void visitVPStoreScatter(const VPIntrinsic &VPIntrin,
-                           SmallVector<SDValue, 7> &OpValues, bool isScatter);
+                           SmallVector<SDValue, 7> &OpValues, bool IsScatter);
   void visitVectorPredicationIntrinsic(const VPIntrinsic &VPIntrin);
 
   void visitVAStart(const CallInst &I);
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index c7e37cf8ca14..77e11b364588 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -297,7 +297,7 @@ TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
 #ifndef NDEBUG
   dbgs() << "If a target marks an instruction with "
           "'usesCustomInserter', it must implement "
-          "TargetLowering::EmitInstrWithCustomInserter!";
+          "TargetLowering::EmitInstrWithCustomInserter!\n";
 #endif
   llvm_unreachable(nullptr);
 }
@@ -1784,27 +1784,25 @@ SelectionDAGISel::FinishBasicBlock() {
     }
 
     // Update PHI Nodes
-    for (unsigned pi = 0, pe = FuncInfo->PHINodesToUpdate.size();
-         pi != pe; ++pi) {
-      MachineInstrBuilder PHI(*MF, FuncInfo->PHINodesToUpdate[pi].first);
+    for (const std::pair<MachineInstr *, unsigned> &P :
+         FuncInfo->PHINodesToUpdate) {
+      MachineInstrBuilder PHI(*MF, P.first);
       MachineBasicBlock *PHIBB = PHI->getParent();
       assert(PHI->isPHI() &&
              "This is not a machine PHI node that we are updating!");
       // This is "default" BB. We have two jumps to it. From "header" BB and
       // from last "case" BB, unless the latter was skipped.
       if (PHIBB == BTB.Default) {
-        PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second).addMBB(BTB.Parent);
+        PHI.addReg(P.second).addMBB(BTB.Parent);
         if (!BTB.ContiguousRange) {
-          PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second)
-              .addMBB(BTB.Cases.back().ThisBB);
+          PHI.addReg(P.second).addMBB(BTB.Cases.back().ThisBB);
          }
       }
       // One of "cases" BB.
-      for (unsigned j = 0, ej = BTB.Cases.size();
-           j != ej; ++j) {
-        MachineBasicBlock* cBB = BTB.Cases[j].ThisBB;
+      for (const SwitchCG::BitTestCase &BT : BTB.Cases) {
+        MachineBasicBlock* cBB = BT.ThisBB;
         if (cBB->isSuccessor(PHIBB))
-          PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second).addMBB(cBB);
+          PHI.addReg(P.second).addMBB(cBB);
       }
     }
   }
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 737695b5eabe..e6b06ab93d6b 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -3136,6 +3136,19 @@ bool TargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
   return false;
 }
 
+bool TargetLowering::isSplatValueForTargetNode(SDValue Op,
+                                               const APInt &DemandedElts,
+                                               APInt &UndefElts,
+                                               unsigned Depth) const {
+  assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
+          Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
+          Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
+          Op.getOpcode() == ISD::INTRINSIC_VOID) &&
+         "Should use isSplatValue if you don't know whether Op"
+         " is a target node!");
+  return false;
+}
+
 // FIXME: Ideally, this would use ISD::isConstantSplatVector(), but that must
 // work with truncating build vectors and vectors with elements of less than
 // 8 bits.
@@ -4853,13 +4866,9 @@ TargetLowering::ParseConstraints(const DataLayout &DL,
       }
 
       // Now select chosen alternative in each constraint.
-      for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
-           cIndex != eIndex; ++cIndex) {
-        AsmOperandInfo &cInfo = ConstraintOperands[cIndex];
-        if (cInfo.Type == InlineAsm::isClobber)
-          continue;
-        cInfo.selectAlternative(bestMAIndex);
-      }
+      for (AsmOperandInfo &cInfo : ConstraintOperands)
+        if (cInfo.Type != InlineAsm::isClobber)
+          cInfo.selectAlternative(bestMAIndex);
     }
   }
 
@@ -4927,9 +4936,9 @@ TargetLowering::ConstraintWeight
   ConstraintWeight BestWeight = CW_Invalid;
 
   // Loop over the options, keeping track of the most general one.
-  for (unsigned i = 0, e = rCodes->size(); i != e; ++i) {
+  for (const std::string &rCode : *rCodes) {
     ConstraintWeight weight =
-      getSingleConstraintMatchWeight(info, (*rCodes)[i].c_str());
+        getSingleConstraintMatchWeight(info, rCode.c_str());
     if (weight > BestWeight)
       BestWeight = weight;
   }
@@ -6550,15 +6559,15 @@ static bool isNonZeroModBitWidthOrUndef(SDValue Z, unsigned BW) {
       true);
 }
 
-bool TargetLowering::expandFunnelShift(SDNode *Node, SDValue &Result,
-                                       SelectionDAG &DAG) const {
+SDValue TargetLowering::expandFunnelShift(SDNode *Node,
+                                          SelectionDAG &DAG) const {
   EVT VT = Node->getValueType(0);
 
   if (VT.isVector() && (!isOperationLegalOrCustom(ISD::SHL, VT) ||
                         !isOperationLegalOrCustom(ISD::SRL, VT) ||
                         !isOperationLegalOrCustom(ISD::SUB, VT) ||
                         !isOperationLegalOrCustomOrPromote(ISD::OR, VT)))
-    return false;
+    return SDValue();
 
   SDValue X = Node->getOperand(0);
   SDValue Y = Node->getOperand(1);
@@ -6592,8 +6601,7 @@ bool TargetLowering::expandFunnelShift(SDNode *Node, SDValue &Result,
       }
       Z = DAG.getNOT(DL, Z, ShVT);
     }
-    Result = DAG.getNode(RevOpcode, DL, VT, X, Y, Z);
-    return true;
+    return DAG.getNode(RevOpcode, DL, VT, X, Y, Z);
   }
 
   SDValue ShX, ShY;
@@ -6633,13 +6641,12 @@ bool TargetLowering::expandFunnelShift(SDNode *Node, SDValue &Result,
       ShY = DAG.getNode(ISD::SRL, DL, VT, Y, ShAmt);
     }
   }
-  Result = DAG.getNode(ISD::OR, DL, VT, ShX, ShY);
-  return true;
+  return DAG.getNode(ISD::OR, DL, VT, ShX, ShY);
 }
 
 // TODO: Merge with expandFunnelShift.
-bool TargetLowering::expandROT(SDNode *Node, bool AllowVectorOps,
-                               SDValue &Result, SelectionDAG &DAG) const {
+SDValue TargetLowering::expandROT(SDNode *Node, bool AllowVectorOps,
+                                  SelectionDAG &DAG) const {
   EVT VT = Node->getValueType(0);
   unsigned EltSizeInBits = VT.getScalarSizeInBits();
   bool IsLeft = Node->getOpcode() == ISD::ROTL;
@@ -6650,12 +6657,12 @@ bool TargetLowering::expandROT(SDNode *Node, bool AllowVectorOps,
   EVT ShVT = Op1.getValueType();
   SDValue Zero = DAG.getConstant(0, DL, ShVT);
 
-  // If a rotate in the other direction is supported, use it.
+  // If a rotate in the other direction is more supported, use it.
   unsigned RevRot = IsLeft ? ISD::ROTR : ISD::ROTL;
-  if (isOperationLegalOrCustom(RevRot, VT) && isPowerOf2_32(EltSizeInBits)) {
+  if (!isOperationLegalOrCustom(Node->getOpcode(), VT) &&
+      isOperationLegalOrCustom(RevRot, VT) && isPowerOf2_32(EltSizeInBits)) {
     SDValue Sub = DAG.getNode(ISD::SUB, DL, ShVT, Zero, Op1);
-    Result = DAG.getNode(RevRot, DL, VT, Op0, Sub);
-    return true;
+    return DAG.getNode(RevRot, DL, VT, Op0, Sub);
   }
 
   if (!AllowVectorOps && VT.isVector() &&
@@ -6664,7 +6671,7 @@ bool TargetLowering::expandROT(SDNode *Node, bool AllowVectorOps,
        !isOperationLegalOrCustom(ISD::SUB, VT) ||
        !isOperationLegalOrCustomOrPromote(ISD::OR, VT) ||
        !isOperationLegalOrCustomOrPromote(ISD::AND, VT)))
-    return false;
+    return SDValue();
 
   unsigned ShOpc = IsLeft ? ISD::SHL : ISD::SRL;
   unsigned HsOpc = IsLeft ? ISD::SRL : ISD::SHL;
@@ -6690,8 +6697,7 @@ bool TargetLowering::expandROT(SDNode *Node, bool AllowVectorOps,
     HsVal =
         DAG.getNode(HsOpc, DL, VT, DAG.getNode(HsOpc, DL, VT, Op0, One), HsAmt);
   }
-  Result = DAG.getNode(ISD::OR, DL, VT, ShVal, HsVal);
-  return true;
+  return DAG.getNode(ISD::OR, DL, VT, ShVal, HsVal);
 }
 
 void TargetLowering::expandShiftParts(SDNode *Node, SDValue &Lo, SDValue &Hi,
@@ -8048,7 +8054,8 @@ SDValue TargetLowering::expandIntMINMAX(SDNode *Node, SelectionDAG &DAG) const {
   if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT))
     return DAG.UnrollVectorOp(Node);
 
-  SDValue Cond = DAG.getSetCC(DL, VT, Op0, Op1, CC);
+  EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
+  SDValue Cond = DAG.getSetCC(DL, BoolVT, Op0, Op1, CC);
   return DAG.getSelect(DL, VT, Cond, Op0, Op1);
 }
 
diff --git a/llvm/lib/CodeGen/ShadowStackGCLowering.cpp b/llvm/lib/CodeGen/ShadowStackGCLowering.cpp
index 86b559fd6413..43a54ce33bf0 100644
--- a/llvm/lib/CodeGen/ShadowStackGCLowering.cpp
+++ b/llvm/lib/CodeGen/ShadowStackGCLowering.cpp
@@ -162,8 +162,8 @@ Type *ShadowStackGCLowering::GetConcreteStackEntryType(Function &F) {
   // doInitialization creates the generic version of this type.
   std::vector<Type *> EltTys;
   EltTys.push_back(StackEntryTy);
-  for (size_t I = 0; I != Roots.size(); I++)
-    EltTys.push_back(Roots[I].second->getAllocatedType());
+  for (const std::pair<CallInst *, AllocaInst *> &Root : Roots)
+    EltTys.push_back(Root.second->getAllocatedType());
 
   return StructType::create(EltTys, ("gc_stackentry." + F.getName()).str());
 }
@@ -240,8 +240,8 @@ void ShadowStackGCLowering::CollectRoots(Function &F) {
   SmallVector<std::pair<CallInst *, AllocaInst *>, 16> MetaRoots;
 
   for (BasicBlock &BB : F)
-    for (BasicBlock::iterator II = BB.begin(), E = BB.end(); II != E;)
-      if (IntrinsicInst *CI = dyn_cast<IntrinsicInst>(II++))
+    for (Instruction &I : BB)
+      if (IntrinsicInst *CI = dyn_cast<IntrinsicInst>(&I))
         if (Function *F = CI->getCalledFunction())
           if (F->getIntrinsicID() == Intrinsic::gcroot) {
             std::pair<CallInst *, AllocaInst *> Pair = std::make_pair(
@@ -377,9 +377,9 @@ bool ShadowStackGCLowering::runOnFunction(Function &F) {
   // Delete the original allocas (which are no longer used) and the intrinsic
   // calls (which are no longer valid). Doing this last avoids invalidating
   // iterators.
-  for (unsigned I = 0, E = Roots.size(); I != E; ++I) {
-    Roots[I].first->eraseFromParent();
-    Roots[I].second->eraseFromParent();
+  for (std::pair<CallInst *, AllocaInst *> &Root : Roots) {
+    Root.first->eraseFromParent();
+    Root.second->eraseFromParent();
   }
 
   Roots.clear();
diff --git a/llvm/lib/CodeGen/StackMapLivenessAnalysis.cpp b/llvm/lib/CodeGen/StackMapLivenessAnalysis.cpp
index 5ccfacfc26dc..3640296adbca 100644
--- a/llvm/lib/CodeGen/StackMapLivenessAnalysis.cpp
+++ b/llvm/lib/CodeGen/StackMapLivenessAnalysis.cpp
@@ -131,15 +131,15 @@ bool StackMapLiveness::calculateLiveness(MachineFunction &MF) {
     bool HasStackMap = false;
     // Reverse iterate over all instructions and add the current live register
     // set to an instruction if we encounter a patchpoint instruction.
-    for (auto I = MBB.rbegin(), E = MBB.rend(); I != E; ++I) {
-      if (I->getOpcode() == TargetOpcode::PATCHPOINT) {
-        addLiveOutSetToMI(MF, *I);
+    for (MachineInstr &MI : llvm::reverse(MBB)) {
+      if (MI.getOpcode() == TargetOpcode::PATCHPOINT) {
+        addLiveOutSetToMI(MF, MI);
         HasChanged = true;
         HasStackMap = true;
         ++NumStackMaps;
       }
-      LLVM_DEBUG(dbgs() << "   " << LiveRegs << "   " << *I);
-      LiveRegs.stepBackward(*I);
+      LLVM_DEBUG(dbgs() << "   " << LiveRegs << "   " << MI);
+      LiveRegs.stepBackward(MI);
     }
     ++NumBBsVisited;
     if (!HasStackMap)
diff --git a/llvm/lib/CodeGen/StackProtector.cpp b/llvm/lib/CodeGen/StackProtector.cpp
index 7445f77c955d..6765fd274686 100644
--- a/llvm/lib/CodeGen/StackProtector.cpp
+++ b/llvm/lib/CodeGen/StackProtector.cpp
@@ -162,7 +162,7 @@ bool StackProtector::ContainsProtectableArray(Type *Ty, bool &IsLarge,
 }
 
 bool StackProtector::HasAddressTaken(const Instruction *AI,
-                                     uint64_t AllocSize) {
+                                     TypeSize AllocSize) {
   const DataLayout &DL = M->getDataLayout();
   for (const User *U : AI->users()) {
     const auto *I = cast<Instruction>(U);
@@ -170,7 +170,8 @@ bool StackProtector::HasAddressTaken(const Instruction *AI,
     // the bounds of the allocated object.
     Optional<MemoryLocation> MemLoc = MemoryLocation::getOrNone(I);
     if (MemLoc.hasValue() && MemLoc->Size.hasValue() &&
-        MemLoc->Size.getValue() > AllocSize)
+        !TypeSize::isKnownGE(AllocSize,
+                             TypeSize::getFixed(MemLoc->Size.getValue())))
       return true;
     switch (I->getOpcode()) {
     case Instruction::Store:
@@ -203,13 +204,19 @@ bool StackProtector::HasAddressTaken(const Instruction *AI,
       // would use it could also be out-of-bounds meaning stack protection is
       // required.
       const GetElementPtrInst *GEP = cast<GetElementPtrInst>(I);
-      unsigned TypeSize = DL.getIndexTypeSizeInBits(I->getType());
-      APInt Offset(TypeSize, 0);
-      APInt MaxOffset(TypeSize, AllocSize);
-      if (!GEP->accumulateConstantOffset(DL, Offset) || Offset.ugt(MaxOffset))
+      unsigned IndexSize = DL.getIndexTypeSizeInBits(I->getType());
+      APInt Offset(IndexSize, 0);
+      if (!GEP->accumulateConstantOffset(DL, Offset))
+        return true;
+      TypeSize OffsetSize = TypeSize::Fixed(Offset.getLimitedValue());
+      if (!TypeSize::isKnownGT(AllocSize, OffsetSize))
         return true;
       // Adjust AllocSize to be the space remaining after this offset.
-      if (HasAddressTaken(I, AllocSize - Offset.getLimitedValue()))
+      // We can't subtract a fixed size from a scalable one, so in that case
+      // assume the scalable value is of minimum size.
+      TypeSize NewAllocSize =
+          TypeSize::Fixed(AllocSize.getKnownMinValue()) - OffsetSize;
+      if (HasAddressTaken(I, NewAllocSize))
         return true;
       break;
     }
diff --git a/llvm/lib/CodeGen/StackSlotColoring.cpp b/llvm/lib/CodeGen/StackSlotColoring.cpp
index f49ba5ccd447..17e6f51d0899 100644
--- a/llvm/lib/CodeGen/StackSlotColoring.cpp
+++ b/llvm/lib/CodeGen/StackSlotColoring.cpp
@@ -325,8 +325,7 @@ bool StackSlotColoring::ColorSlots(MachineFunction &MF) {
 
   LLVM_DEBUG(dbgs() << "Color spill slot intervals:\n");
   bool Changed = false;
-  for (unsigned i = 0, e = SSIntervals.size(); i != e; ++i) {
-    LiveInterval *li = SSIntervals[i];
+  for (LiveInterval *li : SSIntervals) {
     int SS = Register::stackSlot2Index(li->reg());
     int NewSS = ColorSlot(li);
     assert(NewSS >= 0 && "Stack coloring failed?");
@@ -338,8 +337,7 @@ bool StackSlotColoring::ColorSlots(MachineFunction &MF) {
   }
 
   LLVM_DEBUG(dbgs() << "\nSpill slots after coloring:\n");
-  for (unsigned i = 0, e = SSIntervals.size(); i != e; ++i) {
-    LiveInterval *li = SSIntervals[i];
+  for (LiveInterval *li : SSIntervals) {
     int SS = Register::stackSlot2Index(li->reg());
     li->setWeight(SlotWeights[SS]);
   }
@@ -347,8 +345,8 @@ bool StackSlotColoring::ColorSlots(MachineFunction &MF) {
   llvm::stable_sort(SSIntervals, IntervalSorter());
 
 #ifndef NDEBUG
-  for (unsigned i = 0, e = SSIntervals.size(); i != e; ++i)
-    LLVM_DEBUG(SSIntervals[i]->dump());
+  for (LiveInterval *li : SSIntervals)
+    LLVM_DEBUG(li->dump());
   LLVM_DEBUG(dbgs() << '\n');
 #endif
 
diff --git a/llvm/lib/CodeGen/TailDuplicator.cpp b/llvm/lib/CodeGen/TailDuplicator.cpp
index 54fc6ee45d00..68a7b80d6146 100644
--- a/llvm/lib/CodeGen/TailDuplicator.cpp
+++ b/llvm/lib/CodeGen/TailDuplicator.cpp
@@ -207,35 +207,34 @@ bool TailDuplicator::tailDuplicateAndUpdate(
       // Add the new vregs as available values.
       DenseMap<Register, AvailableValsTy>::iterator LI =
           SSAUpdateVals.find(VReg);
-      for (unsigned j = 0, ee = LI->second.size(); j != ee; ++j) {
-        MachineBasicBlock *SrcBB = LI->second[j].first;
-        Register SrcReg = LI->second[j].second;
+      for (std::pair<MachineBasicBlock *, Register> &J : LI->second) {
+        MachineBasicBlock *SrcBB = J.first;
+        Register SrcReg = J.second;
         SSAUpdate.AddAvailableValue(SrcBB, SrcReg);
       }
 
+      SmallVector<MachineOperand *> DebugUses;
       // Rewrite uses that are outside of the original def's block.
-      MachineRegisterInfo::use_iterator UI = MRI->use_begin(VReg);
-      // Only remove instructions after loop, as DBG_VALUE_LISTs with multiple
-      // uses of VReg may invalidate the use iterator when erased.
-      SmallPtrSet<MachineInstr *, 4> InstrsToRemove;
-      while (UI != MRI->use_end()) {
-        MachineOperand &UseMO = *UI;
+      for (MachineOperand &UseMO :
+           llvm::make_early_inc_range(MRI->use_operands(VReg))) {
         MachineInstr *UseMI = UseMO.getParent();
-        ++UI;
+        // Rewrite debug uses last so that they can take advantage of any
+        // register mappings introduced by other users in its BB, since we
+        // cannot create new register definitions specifically for the debug
+        // instruction (as debug instructions should not affect CodeGen).
         if (UseMI->isDebugValue()) {
-          // SSAUpdate can replace the use with an undef. That creates
-          // a debug instruction that is a kill.
-          // FIXME: Should it SSAUpdate job to delete debug instructions
-          // instead of replacing the use with undef?
-          InstrsToRemove.insert(UseMI);
+          DebugUses.push_back(&UseMO);
           continue;
         }
         if (UseMI->getParent() == DefBB && !UseMI->isPHI())
           continue;
         SSAUpdate.RewriteUse(UseMO);
       }
-      for (auto *MI : InstrsToRemove)
-        MI->eraseFromParent();
+      for (auto *UseMO : DebugUses) {
+        MachineInstr *UseMI = UseMO->getParent();
+        UseMO->setReg(
+            SSAUpdate.GetValueInMiddleOfBlock(UseMI->getParent(), true));
+      }
     }
 
     SSAUpdateVRs.clear();
@@ -511,8 +510,8 @@ void TailDuplicator::updateSuccessorsPHIs(
           SSAUpdateVals.find(Reg);
       if (LI != SSAUpdateVals.end()) {
         // This register is defined in the tail block.
-        for (unsigned j = 0, ee = LI->second.size(); j != ee; ++j) {
-          MachineBasicBlock *SrcBB = LI->second[j].first;
+        for (const std::pair<MachineBasicBlock *, Register> &J : LI->second) {
+          MachineBasicBlock *SrcBB = J.first;
           // If we didn't duplicate a bb into a particular predecessor, we
           // might still have added an entry to SSAUpdateVals to correcly
           // recompute SSA. If that case, avoid adding a dummy extra argument
@@ -520,7 +519,7 @@ void TailDuplicator::updateSuccessorsPHIs(
           if (!SrcBB->isSuccessor(SuccBB))
             continue;
 
-          Register SrcReg = LI->second[j].second;
+          Register SrcReg = J.second;
           if (Idx != 0) {
             MI.getOperand(Idx).setReg(SrcReg);
             MI.getOperand(Idx + 1).setMBB(SrcBB);
@@ -531,8 +530,7 @@ void TailDuplicator::updateSuccessorsPHIs(
         }
       } else {
         // Live in tail block, must also be live in predecessors.
-        for (unsigned j = 0, ee = TDBBs.size(); j != ee; ++j) {
-          MachineBasicBlock *SrcBB = TDBBs[j];
+        for (MachineBasicBlock *SrcBB : TDBBs) {
           if (Idx != 0) {
             MI.getOperand(Idx).setReg(Reg);
             MI.getOperand(Idx + 1).setMBB(SrcBB);
diff --git a/llvm/lib/CodeGen/TargetInstrInfo.cpp b/llvm/lib/CodeGen/TargetInstrInfo.cpp
index 5119dac36713..3f22cc4289f2 100644
--- a/llvm/lib/CodeGen/TargetInstrInfo.cpp
+++ b/llvm/lib/CodeGen/TargetInstrInfo.cpp
@@ -436,7 +436,7 @@ MachineInstr &TargetInstrInfo::duplicate(MachineBasicBlock &MBB,
     MachineBasicBlock::iterator InsertBefore, const MachineInstr &Orig) const {
   assert(!Orig.isNotDuplicable() && "Instruction cannot be duplicated");
   MachineFunction &MF = *MBB.getParent();
-  return MF.CloneMachineInstrBundle(MBB, InsertBefore, Orig);
+  return MF.cloneMachineInstrBundle(MBB, InsertBefore, Orig);
 }
 
 // If the COPY instruction in MI can be folded to a stack operation, return
@@ -1418,3 +1418,16 @@ void TargetInstrInfo::mergeOutliningCandidateAttributes(
       }))
     F.addFnAttr(Attribute::NoUnwind);
 }
+
+bool TargetInstrInfo::isMBBSafeToOutlineFrom(MachineBasicBlock &MBB,
+                                             unsigned &Flags) const {
+  // Some instrumentations create special TargetOpcode at the start which
+  // expands to special code sequences which must be present.
+  auto First = MBB.getFirstNonDebugInstr();
+  if (First != MBB.end() &&
+      (First->getOpcode() == TargetOpcode::FENTRY_CALL ||
+       First->getOpcode() == TargetOpcode::PATCHABLE_FUNCTION_ENTER))
+    return false;
+
+  return true;
+}
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index c0a7efff9e98..6fc6881f8736 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -1187,7 +1187,7 @@ TargetLoweringBase::emitPatchPoint(MachineInstr &InitialMI,
   // all stack slots), but we need to handle the different type of stackmap
   // operands and memory effects here.
 
-  if (!llvm::any_of(MI->operands(),
+  if (llvm::none_of(MI->operands(),
                     [](MachineOperand &Operand) { return Operand.isFI(); }))
     return MBB;
 
diff --git a/llvm/lib/CodeGen/TargetRegisterInfo.cpp b/llvm/lib/CodeGen/TargetRegisterInfo.cpp
index f4bb71535f7f..f5cb518fce3e 100644
--- a/llvm/lib/CodeGen/TargetRegisterInfo.cpp
+++ b/llvm/lib/CodeGen/TargetRegisterInfo.cpp
@@ -248,8 +248,8 @@ static void getAllocatableSetForRC(const MachineFunction &MF,
                                    const TargetRegisterClass *RC, BitVector &R){
   assert(RC->isAllocatable() && "invalid for nonallocatable sets");
   ArrayRef<MCPhysReg> Order = RC->getRawAllocationOrder(MF);
-  for (unsigned i = 0; i != Order.size(); ++i)
-    R.set(Order[i]);
+  for (MCPhysReg PR : Order)
+    R.set(PR);
 }
 
 BitVector TargetRegisterInfo::getAllocatableSet(const MachineFunction &MF,
diff --git a/llvm/lib/CodeGen/UnreachableBlockElim.cpp b/llvm/lib/CodeGen/UnreachableBlockElim.cpp
index c9a19948ff2f..3426a03b6083 100644
--- a/llvm/lib/CodeGen/UnreachableBlockElim.cpp
+++ b/llvm/lib/CodeGen/UnreachableBlockElim.cpp
@@ -144,23 +144,22 @@ bool UnreachableMachineBlockElim::runOnMachineFunction(MachineFunction &F) {
   }
 
   // Actually remove the blocks now.
-  for (unsigned i = 0, e = DeadBlocks.size(); i != e; ++i) {
+  for (MachineBasicBlock *BB : DeadBlocks) {
     // Remove any call site information for calls in the block.
-    for (auto &I : DeadBlocks[i]->instrs())
+    for (auto &I : BB->instrs())
       if (I.shouldUpdateCallSiteInfo())
-        DeadBlocks[i]->getParent()->eraseCallSiteInfo(&I);
+        BB->getParent()->eraseCallSiteInfo(&I);
 
-    DeadBlocks[i]->eraseFromParent();
+    BB->eraseFromParent();
   }
 
   // Cleanup PHI nodes.
-  for (MachineFunction::iterator I = F.begin(), E = F.end(); I != E; ++I) {
-    MachineBasicBlock *BB = &*I;
+  for (MachineBasicBlock &BB : F) {
     // Prune unneeded PHI entries.
-    SmallPtrSet<MachineBasicBlock*, 8> preds(BB->pred_begin(),
-                                             BB->pred_end());
-    MachineBasicBlock::iterator phi = BB->begin();
-    while (phi != BB->end() && phi->isPHI()) {
+    SmallPtrSet<MachineBasicBlock*, 8> preds(BB.pred_begin(),
+                                             BB.pred_end());
+    MachineBasicBlock::iterator phi = BB.begin();
+    while (phi != BB.end() && phi->isPHI()) {
       for (unsigned i = phi->getNumOperands() - 1; i >= 2; i-=2)
         if (!preds.count(phi->getOperand(i).getMBB())) {
           phi->RemoveOperand(i);
@@ -189,7 +188,7 @@ bool UnreachableMachineBlockElim::runOnMachineFunction(MachineFunction &F) {
             // insert a COPY instead of simply replacing the output
             // with the input.
             const TargetInstrInfo *TII = F.getSubtarget().getInstrInfo();
-            BuildMI(*BB, BB->getFirstNonPHI(), phi->getDebugLoc(),
+            BuildMI(BB, BB.getFirstNonPHI(), phi->getDebugLoc(),
                     TII->get(TargetOpcode::COPY), OutputReg)
                 .addReg(InputReg, getRegState(Input), InputSub);
           }
diff --git a/llvm/lib/CodeGen/VLIWMachineScheduler.cpp b/llvm/lib/CodeGen/VLIWMachineScheduler.cpp
new file mode 100644
index 000000000000..cbc5d9ec169b
--- /dev/null
+++ b/llvm/lib/CodeGen/VLIWMachineScheduler.cpp
@@ -0,0 +1,1009 @@
+//===- VLIWMachineScheduler.cpp - VLIW-Focused Scheduling Pass ------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// MachineScheduler schedules machine instructions after phi elimination. It
+// preserves LiveIntervals so it can be invoked before register allocation.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/VLIWMachineScheduler.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/CodeGen/DFAPacketizer.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/MachineLoopInfo.h"
+#include "llvm/CodeGen/RegisterClassInfo.h"
+#include "llvm/CodeGen/RegisterPressure.h"
+#include "llvm/CodeGen/ScheduleDAG.h"
+#include "llvm/CodeGen/ScheduleHazardRecognizer.h"
+#include "llvm/CodeGen/TargetInstrInfo.h"
+#include "llvm/CodeGen/TargetOpcodes.h"
+#include "llvm/CodeGen/TargetRegisterInfo.h"
+#include "llvm/CodeGen/TargetSchedule.h"
+#include "llvm/CodeGen/TargetSubtargetInfo.h"
+#include "llvm/IR/Function.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+#include <algorithm>
+#include <cassert>
+#include <iomanip>
+#include <limits>
+#include <memory>
+#include <sstream>
+
+using namespace llvm;
+
+#define DEBUG_TYPE "machine-scheduler"
+
+static cl::opt<bool> IgnoreBBRegPressure("ignore-bb-reg-pressure", cl::Hidden,
+                                         cl::ZeroOrMore, cl::init(false));
+
+static cl::opt<bool> UseNewerCandidate("use-newer-candidate", cl::Hidden,
+                                       cl::ZeroOrMore, cl::init(true));
+
+static cl::opt<unsigned> SchedDebugVerboseLevel("misched-verbose-level",
+                                                cl::Hidden, cl::ZeroOrMore,
+                                                cl::init(1));
+
+// Check if the scheduler should penalize instructions that are available to
+// early due to a zero-latency dependence.
+static cl::opt<bool> CheckEarlyAvail("check-early-avail", cl::Hidden,
+                                     cl::ZeroOrMore, cl::init(true));
+
+// This value is used to determine if a register class is a high pressure set.
+// We compute the maximum number of registers needed and divided by the total
+// available. Then, we compare the result to this value.
+static cl::opt<float> RPThreshold("vliw-misched-reg-pressure", cl::Hidden,
+                                  cl::init(0.75f),
+                                  cl::desc("High register pressure threhold."));
+
+VLIWResourceModel::VLIWResourceModel(const TargetSubtargetInfo &STI,
+                                     const TargetSchedModel *SM)
+    : TII(STI.getInstrInfo()), SchedModel(SM) {
+  ResourcesModel = createPacketizer(STI);
+
+  // This hard requirement could be relaxed,
+  // but for now do not let it proceed.
+  assert(ResourcesModel && "Unimplemented CreateTargetScheduleState.");
+
+  Packet.reserve(SchedModel->getIssueWidth());
+  Packet.clear();
+  ResourcesModel->clearResources();
+}
+
+void VLIWResourceModel::reset() {
+  Packet.clear();
+  ResourcesModel->clearResources();
+}
+
+VLIWResourceModel::~VLIWResourceModel() { delete ResourcesModel; }
+
+/// Return true if there is a dependence between SUd and SUu.
+bool VLIWResourceModel::hasDependence(const SUnit *SUd, const SUnit *SUu) {
+  if (SUd->Succs.size() == 0)
+    return false;
+
+  for (const auto &S : SUd->Succs) {
+    // Since we do not add pseudos to packets, might as well
+    // ignore order dependencies.
+    if (S.isCtrl())
+      continue;
+
+    if (S.getSUnit() == SUu && S.getLatency() > 0)
+      return true;
+  }
+  return false;
+}
+
+/// Check if scheduling of this SU is possible
+/// in the current packet.
+/// It is _not_ precise (statefull), it is more like
+/// another heuristic. Many corner cases are figured
+/// empirically.
+bool VLIWResourceModel::isResourceAvailable(SUnit *SU, bool IsTop) {
+  if (!SU || !SU->getInstr())
+    return false;
+
+  // First see if the pipeline could receive this instruction
+  // in the current cycle.
+  switch (SU->getInstr()->getOpcode()) {
+  default:
+    if (!ResourcesModel->canReserveResources(*SU->getInstr()))
+      return false;
+    break;
+  case TargetOpcode::EXTRACT_SUBREG:
+  case TargetOpcode::INSERT_SUBREG:
+  case TargetOpcode::SUBREG_TO_REG:
+  case TargetOpcode::REG_SEQUENCE:
+  case TargetOpcode::IMPLICIT_DEF:
+  case TargetOpcode::COPY:
+  case TargetOpcode::INLINEASM:
+  case TargetOpcode::INLINEASM_BR:
+    break;
+  }
+
+  // Now see if there are no other dependencies to instructions already
+  // in the packet.
+  if (IsTop) {
+    for (unsigned i = 0, e = Packet.size(); i != e; ++i)
+      if (hasDependence(Packet[i], SU))
+        return false;
+  } else {
+    for (unsigned i = 0, e = Packet.size(); i != e; ++i)
+      if (hasDependence(SU, Packet[i]))
+        return false;
+  }
+  return true;
+}
+
+/// Keep track of available resources.
+bool VLIWResourceModel::reserveResources(SUnit *SU, bool IsTop) {
+  bool startNewCycle = false;
+  // Artificially reset state.
+  if (!SU) {
+    reset();
+    TotalPackets++;
+    return false;
+  }
+  // If this SU does not fit in the packet or the packet is now full
+  // start a new one.
+  if (!isResourceAvailable(SU, IsTop) ||
+      Packet.size() >= SchedModel->getIssueWidth()) {
+    reset();
+    TotalPackets++;
+    startNewCycle = true;
+  }
+
+  switch (SU->getInstr()->getOpcode()) {
+  default:
+    ResourcesModel->reserveResources(*SU->getInstr());
+    break;
+  case TargetOpcode::EXTRACT_SUBREG:
+  case TargetOpcode::INSERT_SUBREG:
+  case TargetOpcode::SUBREG_TO_REG:
+  case TargetOpcode::REG_SEQUENCE:
+  case TargetOpcode::IMPLICIT_DEF:
+  case TargetOpcode::KILL:
+  case TargetOpcode::CFI_INSTRUCTION:
+  case TargetOpcode::EH_LABEL:
+  case TargetOpcode::COPY:
+  case TargetOpcode::INLINEASM:
+  case TargetOpcode::INLINEASM_BR:
+    break;
+  }
+  Packet.push_back(SU);
+
+#ifndef NDEBUG
+  LLVM_DEBUG(dbgs() << "Packet[" << TotalPackets << "]:\n");
+  for (unsigned i = 0, e = Packet.size(); i != e; ++i) {
+    LLVM_DEBUG(dbgs() << "\t[" << i << "] SU(");
+    LLVM_DEBUG(dbgs() << Packet[i]->NodeNum << ")\t");
+    LLVM_DEBUG(Packet[i]->getInstr()->dump());
+  }
+#endif
+
+  return startNewCycle;
+}
+
+DFAPacketizer *
+VLIWResourceModel::createPacketizer(const TargetSubtargetInfo &STI) const {
+  return STI.getInstrInfo()->CreateTargetScheduleState(STI);
+}
+
+/// schedule - Called back from MachineScheduler::runOnMachineFunction
+/// after setting up the current scheduling region. [RegionBegin, RegionEnd)
+/// only includes instructions that have DAG nodes, not scheduling boundaries.
+void VLIWMachineScheduler::schedule() {
+  LLVM_DEBUG(dbgs() << "********** MI Converging Scheduling VLIW "
+                    << printMBBReference(*BB) << " " << BB->getName()
+                    << " in_func " << BB->getParent()->getName()
+                    << " at loop depth " << MLI->getLoopDepth(BB) << " \n");
+
+  buildDAGWithRegPressure();
+
+  Topo.InitDAGTopologicalSorting();
+
+  // Postprocess the DAG to add platform-specific artificial dependencies.
+  postprocessDAG();
+
+  SmallVector<SUnit *, 8> TopRoots, BotRoots;
+  findRootsAndBiasEdges(TopRoots, BotRoots);
+
+  // Initialize the strategy before modifying the DAG.
+  SchedImpl->initialize(this);
+
+  LLVM_DEBUG({
+    unsigned maxH = 0;
+    for (const SUnit &SU : SUnits)
+      if (SU.getHeight() > maxH)
+        maxH = SU.getHeight();
+    dbgs() << "Max Height " << maxH << "\n";
+  });
+  LLVM_DEBUG({
+    unsigned maxD = 0;
+    for (const SUnit &SU : SUnits)
+      if (SU.getDepth() > maxD)
+        maxD = SU.getDepth();
+    dbgs() << "Max Depth " << maxD << "\n";
+  });
+  LLVM_DEBUG(dump());
+  if (ViewMISchedDAGs)
+    viewGraph();
+
+  initQueues(TopRoots, BotRoots);
+
+  bool IsTopNode = false;
+  while (true) {
+    LLVM_DEBUG(
+        dbgs() << "** VLIWMachineScheduler::schedule picking next node\n");
+    SUnit *SU = SchedImpl->pickNode(IsTopNode);
+    if (!SU)
+      break;
+
+    if (!checkSchedLimit())
+      break;
+
+    scheduleMI(SU, IsTopNode);
+
+    // Notify the scheduling strategy after updating the DAG.
+    SchedImpl->schedNode(SU, IsTopNode);
+
+    updateQueues(SU, IsTopNode);
+  }
+  assert(CurrentTop == CurrentBottom && "Nonempty unscheduled zone.");
+
+  placeDebugValues();
+
+  LLVM_DEBUG({
+    dbgs() << "*** Final schedule for "
+           << printMBBReference(*begin()->getParent()) << " ***\n";
+    dumpSchedule();
+    dbgs() << '\n';
+  });
+}
+
+void ConvergingVLIWScheduler::initialize(ScheduleDAGMI *dag) {
+  DAG = static_cast<VLIWMachineScheduler *>(dag);
+  SchedModel = DAG->getSchedModel();
+
+  Top.init(DAG, SchedModel);
+  Bot.init(DAG, SchedModel);
+
+  // Initialize the HazardRecognizers. If itineraries don't exist, are empty, or
+  // are disabled, then these HazardRecs will be disabled.
+  const InstrItineraryData *Itin = DAG->getSchedModel()->getInstrItineraries();
+  const TargetSubtargetInfo &STI = DAG->MF.getSubtarget();
+  const TargetInstrInfo *TII = STI.getInstrInfo();
+  delete Top.HazardRec;
+  delete Bot.HazardRec;
+  Top.HazardRec = TII->CreateTargetMIHazardRecognizer(Itin, DAG);
+  Bot.HazardRec = TII->CreateTargetMIHazardRecognizer(Itin, DAG);
+
+  delete Top.ResourceModel;
+  delete Bot.ResourceModel;
+  Top.ResourceModel = createVLIWResourceModel(STI, DAG->getSchedModel());
+  Bot.ResourceModel = createVLIWResourceModel(STI, DAG->getSchedModel());
+
+  const std::vector<unsigned> &MaxPressure =
+      DAG->getRegPressure().MaxSetPressure;
+  HighPressureSets.assign(MaxPressure.size(), 0);
+  for (unsigned i = 0, e = MaxPressure.size(); i < e; ++i) {
+    unsigned Limit = DAG->getRegClassInfo()->getRegPressureSetLimit(i);
+    HighPressureSets[i] =
+        ((float)MaxPressure[i] > ((float)Limit * RPThreshold));
+  }
+
+  assert((!ForceTopDown || !ForceBottomUp) &&
+         "-misched-topdown incompatible with -misched-bottomup");
+}
+
+VLIWResourceModel *ConvergingVLIWScheduler::createVLIWResourceModel(
+    const TargetSubtargetInfo &STI, const TargetSchedModel *SchedModel) const {
+  return new VLIWResourceModel(STI, SchedModel);
+}
+
+void ConvergingVLIWScheduler::releaseTopNode(SUnit *SU) {
+  for (const SDep &PI : SU->Preds) {
+    unsigned PredReadyCycle = PI.getSUnit()->TopReadyCycle;
+    unsigned MinLatency = PI.getLatency();
+#ifndef NDEBUG
+    Top.MaxMinLatency = std::max(MinLatency, Top.MaxMinLatency);
+#endif
+    if (SU->TopReadyCycle < PredReadyCycle + MinLatency)
+      SU->TopReadyCycle = PredReadyCycle + MinLatency;
+  }
+
+  if (!SU->isScheduled)
+    Top.releaseNode(SU, SU->TopReadyCycle);
+}
+
+void ConvergingVLIWScheduler::releaseBottomNode(SUnit *SU) {
+  assert(SU->getInstr() && "Scheduled SUnit must have instr");
+
+  for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); I != E;
+       ++I) {
+    unsigned SuccReadyCycle = I->getSUnit()->BotReadyCycle;
+    unsigned MinLatency = I->getLatency();
+#ifndef NDEBUG
+    Bot.MaxMinLatency = std::max(MinLatency, Bot.MaxMinLatency);
+#endif
+    if (SU->BotReadyCycle < SuccReadyCycle + MinLatency)
+      SU->BotReadyCycle = SuccReadyCycle + MinLatency;
+  }
+
+  if (!SU->isScheduled)
+    Bot.releaseNode(SU, SU->BotReadyCycle);
+}
+
+ConvergingVLIWScheduler::VLIWSchedBoundary::~VLIWSchedBoundary() {
+  delete ResourceModel;
+  delete HazardRec;
+}
+
+/// Does this SU have a hazard within the current instruction group.
+///
+/// The scheduler supports two modes of hazard recognition. The first is the
+/// ScheduleHazardRecognizer API. It is a fully general hazard recognizer that
+/// supports highly complicated in-order reservation tables
+/// (ScoreboardHazardRecognizer) and arbitrary target-specific logic.
+///
+/// The second is a streamlined mechanism that checks for hazards based on
+/// simple counters that the scheduler itself maintains. It explicitly checks
+/// for instruction dispatch limitations, including the number of micro-ops that
+/// can dispatch per cycle.
+///
+/// TODO: Also check whether the SU must start a new group.
+bool ConvergingVLIWScheduler::VLIWSchedBoundary::checkHazard(SUnit *SU) {
+  if (HazardRec->isEnabled())
+    return HazardRec->getHazardType(SU) != ScheduleHazardRecognizer::NoHazard;
+
+  unsigned uops = SchedModel->getNumMicroOps(SU->getInstr());
+  if (IssueCount + uops > SchedModel->getIssueWidth())
+    return true;
+
+  return false;
+}
+
+void ConvergingVLIWScheduler::VLIWSchedBoundary::releaseNode(
+    SUnit *SU, unsigned ReadyCycle) {
+  if (ReadyCycle < MinReadyCycle)
+    MinReadyCycle = ReadyCycle;
+
+  // Check for interlocks first. For the purpose of other heuristics, an
+  // instruction that cannot issue appears as if it's not in the ReadyQueue.
+  if (ReadyCycle > CurrCycle || checkHazard(SU))
+
+    Pending.push(SU);
+  else
+    Available.push(SU);
+}
+
+/// Move the boundary of scheduled code by one cycle.
+void ConvergingVLIWScheduler::VLIWSchedBoundary::bumpCycle() {
+  unsigned Width = SchedModel->getIssueWidth();
+  IssueCount = (IssueCount <= Width) ? 0 : IssueCount - Width;
+
+  assert(MinReadyCycle < std::numeric_limits<unsigned>::max() &&
+         "MinReadyCycle uninitialized");
+  unsigned NextCycle = std::max(CurrCycle + 1, MinReadyCycle);
+
+  if (!HazardRec->isEnabled()) {
+    // Bypass HazardRec virtual calls.
+    CurrCycle = NextCycle;
+  } else {
+    // Bypass getHazardType calls in case of long latency.
+    for (; CurrCycle != NextCycle; ++CurrCycle) {
+      if (isTop())
+        HazardRec->AdvanceCycle();
+      else
+        HazardRec->RecedeCycle();
+    }
+  }
+  CheckPending = true;
+
+  LLVM_DEBUG(dbgs() << "*** Next cycle " << Available.getName() << " cycle "
+                    << CurrCycle << '\n');
+}
+
+/// Move the boundary of scheduled code by one SUnit.
+void ConvergingVLIWScheduler::VLIWSchedBoundary::bumpNode(SUnit *SU) {
+  bool startNewCycle = false;
+
+  // Update the reservation table.
+  if (HazardRec->isEnabled()) {
+    if (!isTop() && SU->isCall) {
+      // Calls are scheduled with their preceding instructions. For bottom-up
+      // scheduling, clear the pipeline state before emitting.
+      HazardRec->Reset();
+    }
+    HazardRec->EmitInstruction(SU);
+  }
+
+  // Update DFA model.
+  startNewCycle = ResourceModel->reserveResources(SU, isTop());
+
+  // Check the instruction group dispatch limit.
+  // TODO: Check if this SU must end a dispatch group.
+  IssueCount += SchedModel->getNumMicroOps(SU->getInstr());
+  if (startNewCycle) {
+    LLVM_DEBUG(dbgs() << "*** Max instrs at cycle " << CurrCycle << '\n');
+    bumpCycle();
+  } else
+    LLVM_DEBUG(dbgs() << "*** IssueCount " << IssueCount << " at cycle "
+                      << CurrCycle << '\n');
+}
+
+/// Release pending ready nodes in to the available queue. This makes them
+/// visible to heuristics.
+void ConvergingVLIWScheduler::VLIWSchedBoundary::releasePending() {
+  // If the available queue is empty, it is safe to reset MinReadyCycle.
+  if (Available.empty())
+    MinReadyCycle = std::numeric_limits<unsigned>::max();
+
+  // Check to see if any of the pending instructions are ready to issue.  If
+  // so, add them to the available queue.
+  for (unsigned i = 0, e = Pending.size(); i != e; ++i) {
+    SUnit *SU = *(Pending.begin() + i);
+    unsigned ReadyCycle = isTop() ? SU->TopReadyCycle : SU->BotReadyCycle;
+
+    if (ReadyCycle < MinReadyCycle)
+      MinReadyCycle = ReadyCycle;
+
+    if (ReadyCycle > CurrCycle)
+      continue;
+
+    if (checkHazard(SU))
+      continue;
+
+    Available.push(SU);
+    Pending.remove(Pending.begin() + i);
+    --i;
+    --e;
+  }
+  CheckPending = false;
+}
+
+/// Remove SU from the ready set for this boundary.
+void ConvergingVLIWScheduler::VLIWSchedBoundary::removeReady(SUnit *SU) {
+  if (Available.isInQueue(SU))
+    Available.remove(Available.find(SU));
+  else {
+    assert(Pending.isInQueue(SU) && "bad ready count");
+    Pending.remove(Pending.find(SU));
+  }
+}
+
+/// If this queue only has one ready candidate, return it. As a side effect,
+/// advance the cycle until at least one node is ready. If multiple instructions
+/// are ready, return NULL.
+SUnit *ConvergingVLIWScheduler::VLIWSchedBoundary::pickOnlyChoice() {
+  if (CheckPending)
+    releasePending();
+
+  auto AdvanceCycle = [this]() {
+    if (Available.empty())
+      return true;
+    if (Available.size() == 1 && Pending.size() > 0)
+      return !ResourceModel->isResourceAvailable(*Available.begin(), isTop()) ||
+             getWeakLeft(*Available.begin(), isTop()) != 0;
+    return false;
+  };
+  for (unsigned i = 0; AdvanceCycle(); ++i) {
+    assert(i <= (HazardRec->getMaxLookAhead() + MaxMinLatency) &&
+           "permanent hazard");
+    (void)i;
+    ResourceModel->reserveResources(nullptr, isTop());
+    bumpCycle();
+    releasePending();
+  }
+  if (Available.size() == 1)
+    return *Available.begin();
+  return nullptr;
+}
+
+#ifndef NDEBUG
+void ConvergingVLIWScheduler::traceCandidate(const char *Label,
+                                             const ReadyQueue &Q, SUnit *SU,
+                                             int Cost, PressureChange P) {
+  dbgs() << Label << " " << Q.getName() << " ";
+  if (P.isValid())
+    dbgs() << DAG->TRI->getRegPressureSetName(P.getPSet()) << ":"
+           << P.getUnitInc() << " ";
+  else
+    dbgs() << "     ";
+  dbgs() << "cost(" << Cost << ")\t";
+  DAG->dumpNode(*SU);
+}
+
+// Very detailed queue dump, to be used with higher verbosity levels.
+void ConvergingVLIWScheduler::readyQueueVerboseDump(
+    const RegPressureTracker &RPTracker, SchedCandidate &Candidate,
+    ReadyQueue &Q) {
+  RegPressureTracker &TempTracker = const_cast<RegPressureTracker &>(RPTracker);
+
+  dbgs() << ">>> " << Q.getName() << "\n";
+  for (ReadyQueue::iterator I = Q.begin(), E = Q.end(); I != E; ++I) {
+    RegPressureDelta RPDelta;
+    TempTracker.getMaxPressureDelta((*I)->getInstr(), RPDelta,
+                                    DAG->getRegionCriticalPSets(),
+                                    DAG->getRegPressure().MaxSetPressure);
+    std::stringstream dbgstr;
+    dbgstr << "SU(" << std::setw(3) << (*I)->NodeNum << ")";
+    dbgs() << dbgstr.str();
+    SchedulingCost(Q, *I, Candidate, RPDelta, true);
+    dbgs() << "\t";
+    (*I)->getInstr()->dump();
+  }
+  dbgs() << "\n";
+}
+#endif
+
+/// isSingleUnscheduledPred - If SU2 is the only unscheduled predecessor
+/// of SU, return true (we may have duplicates)
+static inline bool isSingleUnscheduledPred(SUnit *SU, SUnit *SU2) {
+  if (SU->NumPredsLeft == 0)
+    return false;
+
+  for (auto &Pred : SU->Preds) {
+    // We found an available, but not scheduled, predecessor.
+    if (!Pred.getSUnit()->isScheduled && (Pred.getSUnit() != SU2))
+      return false;
+  }
+
+  return true;
+}
+
+/// isSingleUnscheduledSucc - If SU2 is the only unscheduled successor
+/// of SU, return true (we may have duplicates)
+static inline bool isSingleUnscheduledSucc(SUnit *SU, SUnit *SU2) {
+  if (SU->NumSuccsLeft == 0)
+    return false;
+
+  for (auto &Succ : SU->Succs) {
+    // We found an available, but not scheduled, successor.
+    if (!Succ.getSUnit()->isScheduled && (Succ.getSUnit() != SU2))
+      return false;
+  }
+  return true;
+}
+
+/// Check if the instruction changes the register pressure of a register in the
+/// high pressure set. The function returns a negative value if the pressure
+/// decreases and a positive value is the pressure increases. If the instruction
+/// doesn't use a high pressure register or doesn't change the register
+/// pressure, then return 0.
+int ConvergingVLIWScheduler::pressureChange(const SUnit *SU, bool isBotUp) {
+  PressureDiff &PD = DAG->getPressureDiff(SU);
+  for (auto &P : PD) {
+    if (!P.isValid())
+      continue;
+    // The pressure differences are computed bottom-up, so the comparision for
+    // an increase is positive in the bottom direction, but negative in the
+    //  top-down direction.
+    if (HighPressureSets[P.getPSet()])
+      return (isBotUp ? P.getUnitInc() : -P.getUnitInc());
+  }
+  return 0;
+}
+
+/// Single point to compute overall scheduling cost.
+/// TODO: More heuristics will be used soon.
+int ConvergingVLIWScheduler::SchedulingCost(ReadyQueue &Q, SUnit *SU,
+                                            SchedCandidate &Candidate,
+                                            RegPressureDelta &Delta,
+                                            bool verbose) {
+  // Initial trivial priority.
+  int ResCount = 1;
+
+  // Do not waste time on a node that is already scheduled.
+  if (!SU || SU->isScheduled)
+    return ResCount;
+
+  LLVM_DEBUG(if (verbose) dbgs()
+             << ((Q.getID() == TopQID) ? "(top|" : "(bot|"));
+  // Forced priority is high.
+  if (SU->isScheduleHigh) {
+    ResCount += PriorityOne;
+    LLVM_DEBUG(dbgs() << "H|");
+  }
+
+  unsigned IsAvailableAmt = 0;
+  // Critical path first.
+  if (Q.getID() == TopQID) {
+    if (Top.isLatencyBound(SU)) {
+      LLVM_DEBUG(if (verbose) dbgs() << "LB|");
+      ResCount += (SU->getHeight() * ScaleTwo);
+    }
+
+    LLVM_DEBUG(if (verbose) {
+      std::stringstream dbgstr;
+      dbgstr << "h" << std::setw(3) << SU->getHeight() << "|";
+      dbgs() << dbgstr.str();
+    });
+
+    // If resources are available for it, multiply the
+    // chance of scheduling.
+    if (Top.ResourceModel->isResourceAvailable(SU, true)) {
+      IsAvailableAmt = (PriorityTwo + PriorityThree);
+      ResCount += IsAvailableAmt;
+      LLVM_DEBUG(if (verbose) dbgs() << "A|");
+    } else
+      LLVM_DEBUG(if (verbose) dbgs() << " |");
+  } else {
+    if (Bot.isLatencyBound(SU)) {
+      LLVM_DEBUG(if (verbose) dbgs() << "LB|");
+      ResCount += (SU->getDepth() * ScaleTwo);
+    }
+
+    LLVM_DEBUG(if (verbose) {
+      std::stringstream dbgstr;
+      dbgstr << "d" << std::setw(3) << SU->getDepth() << "|";
+      dbgs() << dbgstr.str();
+    });
+
+    // If resources are available for it, multiply the
+    // chance of scheduling.
+    if (Bot.ResourceModel->isResourceAvailable(SU, false)) {
+      IsAvailableAmt = (PriorityTwo + PriorityThree);
+      ResCount += IsAvailableAmt;
+      LLVM_DEBUG(if (verbose) dbgs() << "A|");
+    } else
+      LLVM_DEBUG(if (verbose) dbgs() << " |");
+  }
+
+  unsigned NumNodesBlocking = 0;
+  if (Q.getID() == TopQID) {
+    // How many SUs does it block from scheduling?
+    // Look at all of the successors of this node.
+    // Count the number of nodes that
+    // this node is the sole unscheduled node for.
+    if (Top.isLatencyBound(SU))
+      for (const SDep &SI : SU->Succs)
+        if (isSingleUnscheduledPred(SI.getSUnit(), SU))
+          ++NumNodesBlocking;
+  } else {
+    // How many unscheduled predecessors block this node?
+    if (Bot.isLatencyBound(SU))
+      for (const SDep &PI : SU->Preds)
+        if (isSingleUnscheduledSucc(PI.getSUnit(), SU))
+          ++NumNodesBlocking;
+  }
+  ResCount += (NumNodesBlocking * ScaleTwo);
+
+  LLVM_DEBUG(if (verbose) {
+    std::stringstream dbgstr;
+    dbgstr << "blk " << std::setw(2) << NumNodesBlocking << ")|";
+    dbgs() << dbgstr.str();
+  });
+
+  // Factor in reg pressure as a heuristic.
+  if (!IgnoreBBRegPressure) {
+    // Decrease priority by the amount that register pressure exceeds the limit.
+    ResCount -= (Delta.Excess.getUnitInc() * PriorityOne);
+    // Decrease priority if register pressure exceeds the limit.
+    ResCount -= (Delta.CriticalMax.getUnitInc() * PriorityOne);
+    // Decrease priority slightly if register pressure would increase over the
+    // current maximum.
+    ResCount -= (Delta.CurrentMax.getUnitInc() * PriorityTwo);
+    // If there are register pressure issues, then we remove the value added for
+    // the instruction being available. The rationale is that we really don't
+    // want to schedule an instruction that causes a spill.
+    if (IsAvailableAmt && pressureChange(SU, Q.getID() != TopQID) > 0 &&
+        (Delta.Excess.getUnitInc() || Delta.CriticalMax.getUnitInc() ||
+         Delta.CurrentMax.getUnitInc()))
+      ResCount -= IsAvailableAmt;
+    LLVM_DEBUG(if (verbose) {
+      dbgs() << "RP " << Delta.Excess.getUnitInc() << "/"
+             << Delta.CriticalMax.getUnitInc() << "/"
+             << Delta.CurrentMax.getUnitInc() << ")|";
+    });
+  }
+
+  // Give preference to a zero latency instruction if the dependent
+  // instruction is in the current packet.
+  if (Q.getID() == TopQID && getWeakLeft(SU, true) == 0) {
+    for (const SDep &PI : SU->Preds) {
+      if (!PI.getSUnit()->getInstr()->isPseudo() && PI.isAssignedRegDep() &&
+          PI.getLatency() == 0 &&
+          Top.ResourceModel->isInPacket(PI.getSUnit())) {
+        ResCount += PriorityThree;
+        LLVM_DEBUG(if (verbose) dbgs() << "Z|");
+      }
+    }
+  } else if (Q.getID() == BotQID && getWeakLeft(SU, false) == 0) {
+    for (const SDep &SI : SU->Succs) {
+      if (!SI.getSUnit()->getInstr()->isPseudo() && SI.isAssignedRegDep() &&
+          SI.getLatency() == 0 &&
+          Bot.ResourceModel->isInPacket(SI.getSUnit())) {
+        ResCount += PriorityThree;
+        LLVM_DEBUG(if (verbose) dbgs() << "Z|");
+      }
+    }
+  }
+
+  // If the instruction has a non-zero latency dependence with an instruction in
+  // the current packet, then it should not be scheduled yet. The case occurs
+  // when the dependent instruction is scheduled in a new packet, so the
+  // scheduler updates the current cycle and pending instructions become
+  // available.
+  if (CheckEarlyAvail) {
+    if (Q.getID() == TopQID) {
+      for (const auto &PI : SU->Preds) {
+        if (PI.getLatency() > 0 &&
+            Top.ResourceModel->isInPacket(PI.getSUnit())) {
+          ResCount -= PriorityOne;
+          LLVM_DEBUG(if (verbose) dbgs() << "D|");
+        }
+      }
+    } else {
+      for (const auto &SI : SU->Succs) {
+        if (SI.getLatency() > 0 &&
+            Bot.ResourceModel->isInPacket(SI.getSUnit())) {
+          ResCount -= PriorityOne;
+          LLVM_DEBUG(if (verbose) dbgs() << "D|");
+        }
+      }
+    }
+  }
+
+  LLVM_DEBUG(if (verbose) {
+    std::stringstream dbgstr;
+    dbgstr << "Total " << std::setw(4) << ResCount << ")";
+    dbgs() << dbgstr.str();
+  });
+
+  return ResCount;
+}
+
+/// Pick the best candidate from the top queue.
+///
+/// TODO: getMaxPressureDelta results can be mostly cached for each SUnit during
+/// DAG building. To adjust for the current scheduling location we need to
+/// maintain the number of vreg uses remaining to be top-scheduled.
+ConvergingVLIWScheduler::CandResult
+ConvergingVLIWScheduler::pickNodeFromQueue(VLIWSchedBoundary &Zone,
+                                           const RegPressureTracker &RPTracker,
+                                           SchedCandidate &Candidate) {
+  ReadyQueue &Q = Zone.Available;
+  LLVM_DEBUG(if (SchedDebugVerboseLevel > 1)
+                 readyQueueVerboseDump(RPTracker, Candidate, Q);
+             else Q.dump(););
+
+  // getMaxPressureDelta temporarily modifies the tracker.
+  RegPressureTracker &TempTracker = const_cast<RegPressureTracker &>(RPTracker);
+
+  // BestSU remains NULL if no top candidates beat the best existing candidate.
+  CandResult FoundCandidate = NoCand;
+  for (ReadyQueue::iterator I = Q.begin(), E = Q.end(); I != E; ++I) {
+    RegPressureDelta RPDelta;
+    TempTracker.getMaxPressureDelta((*I)->getInstr(), RPDelta,
+                                    DAG->getRegionCriticalPSets(),
+                                    DAG->getRegPressure().MaxSetPressure);
+
+    int CurrentCost = SchedulingCost(Q, *I, Candidate, RPDelta, false);
+
+    // Initialize the candidate if needed.
+    if (!Candidate.SU) {
+      LLVM_DEBUG(traceCandidate("DCAND", Q, *I, CurrentCost));
+      Candidate.SU = *I;
+      Candidate.RPDelta = RPDelta;
+      Candidate.SCost = CurrentCost;
+      FoundCandidate = NodeOrder;
+      continue;
+    }
+
+    // Choose node order for negative cost candidates. There is no good
+    // candidate in this case.
+    if (CurrentCost < 0 && Candidate.SCost < 0) {
+      if ((Q.getID() == TopQID && (*I)->NodeNum < Candidate.SU->NodeNum) ||
+          (Q.getID() == BotQID && (*I)->NodeNum > Candidate.SU->NodeNum)) {
+        LLVM_DEBUG(traceCandidate("NCAND", Q, *I, CurrentCost));
+        Candidate.SU = *I;
+        Candidate.RPDelta = RPDelta;
+        Candidate.SCost = CurrentCost;
+        FoundCandidate = NodeOrder;
+      }
+      continue;
+    }
+
+    // Best cost.
+    if (CurrentCost > Candidate.SCost) {
+      LLVM_DEBUG(traceCandidate("CCAND", Q, *I, CurrentCost));
+      Candidate.SU = *I;
+      Candidate.RPDelta = RPDelta;
+      Candidate.SCost = CurrentCost;
+      FoundCandidate = BestCost;
+      continue;
+    }
+
+    // Choose an instruction that does not depend on an artificial edge.
+    unsigned CurrWeak = getWeakLeft(*I, (Q.getID() == TopQID));
+    unsigned CandWeak = getWeakLeft(Candidate.SU, (Q.getID() == TopQID));
+    if (CurrWeak != CandWeak) {
+      if (CurrWeak < CandWeak) {
+        LLVM_DEBUG(traceCandidate("WCAND", Q, *I, CurrentCost));
+        Candidate.SU = *I;
+        Candidate.RPDelta = RPDelta;
+        Candidate.SCost = CurrentCost;
+        FoundCandidate = Weak;
+      }
+      continue;
+    }
+
+    if (CurrentCost == Candidate.SCost && Zone.isLatencyBound(*I)) {
+      unsigned CurrSize, CandSize;
+      if (Q.getID() == TopQID) {
+        CurrSize = (*I)->Succs.size();
+        CandSize = Candidate.SU->Succs.size();
+      } else {
+        CurrSize = (*I)->Preds.size();
+        CandSize = Candidate.SU->Preds.size();
+      }
+      if (CurrSize > CandSize) {
+        LLVM_DEBUG(traceCandidate("SPCAND", Q, *I, CurrentCost));
+        Candidate.SU = *I;
+        Candidate.RPDelta = RPDelta;
+        Candidate.SCost = CurrentCost;
+        FoundCandidate = BestCost;
+      }
+      // Keep the old candidate if it's a better candidate. That is, don't use
+      // the subsequent tie breaker.
+      if (CurrSize != CandSize)
+        continue;
+    }
+
+    // Tie breaker.
+    // To avoid scheduling indeterminism, we need a tie breaker
+    // for the case when cost is identical for two nodes.
+    if (UseNewerCandidate && CurrentCost == Candidate.SCost) {
+      if ((Q.getID() == TopQID && (*I)->NodeNum < Candidate.SU->NodeNum) ||
+          (Q.getID() == BotQID && (*I)->NodeNum > Candidate.SU->NodeNum)) {
+        LLVM_DEBUG(traceCandidate("TCAND", Q, *I, CurrentCost));
+        Candidate.SU = *I;
+        Candidate.RPDelta = RPDelta;
+        Candidate.SCost = CurrentCost;
+        FoundCandidate = NodeOrder;
+        continue;
+      }
+    }
+
+    // Fall through to original instruction order.
+    // Only consider node order if Candidate was chosen from this Q.
+    if (FoundCandidate == NoCand)
+      continue;
+  }
+  return FoundCandidate;
+}
+
+/// Pick the best candidate node from either the top or bottom queue.
+SUnit *ConvergingVLIWScheduler::pickNodeBidrectional(bool &IsTopNode) {
+  // Schedule as far as possible in the direction of no choice. This is most
+  // efficient, but also provides the best heuristics for CriticalPSets.
+  if (SUnit *SU = Bot.pickOnlyChoice()) {
+    LLVM_DEBUG(dbgs() << "Picked only Bottom\n");
+    IsTopNode = false;
+    return SU;
+  }
+  if (SUnit *SU = Top.pickOnlyChoice()) {
+    LLVM_DEBUG(dbgs() << "Picked only Top\n");
+    IsTopNode = true;
+    return SU;
+  }
+  SchedCandidate BotCand;
+  // Prefer bottom scheduling when heuristics are silent.
+  CandResult BotResult =
+      pickNodeFromQueue(Bot, DAG->getBotRPTracker(), BotCand);
+  assert(BotResult != NoCand && "failed to find the first candidate");
+
+  // If either Q has a single candidate that provides the least increase in
+  // Excess pressure, we can immediately schedule from that Q.
+  //
+  // RegionCriticalPSets summarizes the pressure within the scheduled region and
+  // affects picking from either Q. If scheduling in one direction must
+  // increase pressure for one of the excess PSets, then schedule in that
+  // direction first to provide more freedom in the other direction.
+  if (BotResult == SingleExcess || BotResult == SingleCritical) {
+    LLVM_DEBUG(dbgs() << "Prefered Bottom Node\n");
+    IsTopNode = false;
+    return BotCand.SU;
+  }
+  // Check if the top Q has a better candidate.
+  SchedCandidate TopCand;
+  CandResult TopResult =
+      pickNodeFromQueue(Top, DAG->getTopRPTracker(), TopCand);
+  assert(TopResult != NoCand && "failed to find the first candidate");
+
+  if (TopResult == SingleExcess || TopResult == SingleCritical) {
+    LLVM_DEBUG(dbgs() << "Prefered Top Node\n");
+    IsTopNode = true;
+    return TopCand.SU;
+  }
+  // If either Q has a single candidate that minimizes pressure above the
+  // original region's pressure pick it.
+  if (BotResult == SingleMax) {
+    LLVM_DEBUG(dbgs() << "Prefered Bottom Node SingleMax\n");
+    IsTopNode = false;
+    return BotCand.SU;
+  }
+  if (TopResult == SingleMax) {
+    LLVM_DEBUG(dbgs() << "Prefered Top Node SingleMax\n");
+    IsTopNode = true;
+    return TopCand.SU;
+  }
+  if (TopCand.SCost > BotCand.SCost) {
+    LLVM_DEBUG(dbgs() << "Prefered Top Node Cost\n");
+    IsTopNode = true;
+    return TopCand.SU;
+  }
+  // Otherwise prefer the bottom candidate in node order.
+  LLVM_DEBUG(dbgs() << "Prefered Bottom in Node order\n");
+  IsTopNode = false;
+  return BotCand.SU;
+}
+
+/// Pick the best node to balance the schedule. Implements MachineSchedStrategy.
+SUnit *ConvergingVLIWScheduler::pickNode(bool &IsTopNode) {
+  if (DAG->top() == DAG->bottom()) {
+    assert(Top.Available.empty() && Top.Pending.empty() &&
+           Bot.Available.empty() && Bot.Pending.empty() && "ReadyQ garbage");
+    return nullptr;
+  }
+  SUnit *SU;
+  if (ForceTopDown) {
+    SU = Top.pickOnlyChoice();
+    if (!SU) {
+      SchedCandidate TopCand;
+      CandResult TopResult =
+          pickNodeFromQueue(Top, DAG->getTopRPTracker(), TopCand);
+      assert(TopResult != NoCand && "failed to find the first candidate");
+      (void)TopResult;
+      SU = TopCand.SU;
+    }
+    IsTopNode = true;
+  } else if (ForceBottomUp) {
+    SU = Bot.pickOnlyChoice();
+    if (!SU) {
+      SchedCandidate BotCand;
+      CandResult BotResult =
+          pickNodeFromQueue(Bot, DAG->getBotRPTracker(), BotCand);
+      assert(BotResult != NoCand && "failed to find the first candidate");
+      (void)BotResult;
+      SU = BotCand.SU;
+    }
+    IsTopNode = false;
+  } else {
+    SU = pickNodeBidrectional(IsTopNode);
+  }
+  if (SU->isTopReady())
+    Top.removeReady(SU);
+  if (SU->isBottomReady())
+    Bot.removeReady(SU);
+
+  LLVM_DEBUG(dbgs() << "*** " << (IsTopNode ? "Top" : "Bottom")
+                    << " Scheduling instruction in cycle "
+                    << (IsTopNode ? Top.CurrCycle : Bot.CurrCycle) << " ("
+                    << reportPackets() << ")\n";
+             DAG->dumpNode(*SU));
+  return SU;
+}
+
+/// Update the scheduler's state after scheduling a node. This is the same node
+/// that was just returned by pickNode(). However, VLIWMachineScheduler needs
+/// to update it's state based on the current cycle before MachineSchedStrategy
+/// does.
+void ConvergingVLIWScheduler::schedNode(SUnit *SU, bool IsTopNode) {
+  if (IsTopNode) {
+    Top.bumpNode(SU);
+    SU->TopReadyCycle = Top.CurrCycle;
+  } else {
+    Bot.bumpNode(SU);
+    SU->BotReadyCycle = Bot.CurrCycle;
+  }
+}
diff --git a/llvm/lib/CodeGen/ValueTypes.cpp b/llvm/lib/CodeGen/ValueTypes.cpp
index 4876b9e23717..0c42bef82005 100644
--- a/llvm/lib/CodeGen/ValueTypes.cpp
+++ b/llvm/lib/CodeGen/ValueTypes.cpp
@@ -201,9 +201,11 @@ Type *EVT::getTypeForEVT(LLVMContext &Context) const {
   case MVT::x86amx:  return Type::getX86_AMXTy(Context);
   case MVT::i64x8:   return IntegerType::get(Context, 512);
   case MVT::externref:
+    // pointer to opaque struct in addrspace(10)
     return PointerType::get(StructType::create(Context), 10);
   case MVT::funcref:
-    return PointerType::get(StructType::create(Context), 20);
+    // pointer to i8 addrspace(20)
+    return PointerType::get(Type::getInt8Ty(Context), 20);
   case MVT::v1i1:
     return FixedVectorType::get(Type::getInt1Ty(Context), 1);
   case MVT::v2i1:
diff --git a/llvm/lib/CodeGen/WinEHPrepare.cpp b/llvm/lib/CodeGen/WinEHPrepare.cpp
index 4564aa1c1278..d31183e46d65 100644
--- a/llvm/lib/CodeGen/WinEHPrepare.cpp
+++ b/llvm/lib/CodeGen/WinEHPrepare.cpp
@@ -573,9 +573,7 @@ void llvm::calculateClrEHStateNumbers(const Function *Fn,
       const auto *CatchSwitch = cast<CatchSwitchInst>(Pad);
       int CatchState = -1, FollowerState = -1;
       SmallVector<const BasicBlock *, 4> CatchBlocks(CatchSwitch->handlers());
-      for (auto CBI = CatchBlocks.rbegin(), CBE = CatchBlocks.rend();
-           CBI != CBE; ++CBI, FollowerState = CatchState) {
-        const BasicBlock *CatchBlock = *CBI;
+      for (const BasicBlock *CatchBlock : llvm::reverse(CatchBlocks)) {
         // Create the entry for this catch with the appropriate handler
         // properties.
         const auto *Catch = cast<CatchPadInst>(CatchBlock->getFirstNonPHI());
@@ -591,6 +589,7 @@ void llvm::calculateClrEHStateNumbers(const Function *Fn,
               Worklist.emplace_back(I, CatchState);
         // Remember this catch's state.
         FuncInfo.EHPadStateMap[Catch] = CatchState;
+        FollowerState = CatchState;
       }
       // Associate the catchswitch with the state of its first catch.
       assert(CatchSwitch->getNumHandlers());
@@ -601,11 +600,9 @@ void llvm::calculateClrEHStateNumbers(const Function *Fn,
   // Step two: record the TryParentState of each state.  For cleanuppads that
   // don't have cleanuprets, we may need to infer this from their child pads,
   // so visit pads in descendant-most to ancestor-most order.
-  for (auto Entry = FuncInfo.ClrEHUnwindMap.rbegin(),
-            End = FuncInfo.ClrEHUnwindMap.rend();
-       Entry != End; ++Entry) {
+  for (ClrEHUnwindMapEntry &Entry : llvm::reverse(FuncInfo.ClrEHUnwindMap)) {
     const Instruction *Pad =
-        Entry->Handler.get<const BasicBlock *>()->getFirstNonPHI();
+        Entry.Handler.get<const BasicBlock *>()->getFirstNonPHI();
     // For most pads, the TryParentState is the state associated with the
     // unwind dest of exceptional exits from it.
     const BasicBlock *UnwindDest;
@@ -615,7 +612,7 @@ void llvm::calculateClrEHStateNumbers(const Function *Fn,
       // that's not the unwind dest of exceptions escaping the catch.  Those
       // cases were already assigned a TryParentState in the first pass, so
       // skip them.
-      if (Entry->TryParentState != -1)
+      if (Entry.TryParentState != -1)
         continue;
       // Otherwise, get the unwind dest from the catchswitch.
       UnwindDest = Catch->getCatchSwitch()->getUnwindDest();
@@ -692,7 +689,7 @@ void llvm::calculateClrEHStateNumbers(const Function *Fn,
       UnwindDestState = FuncInfo.EHPadStateMap[UnwindDest->getFirstNonPHI()];
     }
 
-    Entry->TryParentState = UnwindDestState;
+    Entry.TryParentState = UnwindDestState;
   }
 
   // Step three: transfer information from pads to invokes.
diff --git a/llvm/lib/CodeGen/XRayInstrumentation.cpp b/llvm/lib/CodeGen/XRayInstrumentation.cpp
index 11d1b309aa64..b66429d8a5bf 100644
--- a/llvm/lib/CodeGen/XRayInstrumentation.cpp
+++ b/llvm/lib/CodeGen/XRayInstrumentation.cpp
@@ -226,6 +226,7 @@ bool XRayInstrumentation::runOnMachineFunction(MachineFunction &MF) {
     case Triple::ArchType::arm:
     case Triple::ArchType::thumb:
     case Triple::ArchType::aarch64:
+    case Triple::ArchType::hexagon:
     case Triple::ArchType::mips:
     case Triple::ArchType::mipsel:
     case Triple::ArchType::mips64:
diff --git a/llvm/lib/DWARFLinker/DWARFLinker.cpp b/llvm/lib/DWARFLinker/DWARFLinker.cpp
index a3dec6c25e44..ae0859e1ecfd 100644
--- a/llvm/lib/DWARFLinker/DWARFLinker.cpp
+++ b/llvm/lib/DWARFLinker/DWARFLinker.cpp
@@ -223,22 +223,21 @@ static void analyzeImportedModule(
     SysRoot = CU.getSysRoot();
   if (!SysRoot.empty() && Path.startswith(SysRoot))
     return;
-  if (Optional<DWARFFormValue> Val = DIE.find(dwarf::DW_AT_name))
-    if (Optional<const char *> Name = Val->getAsCString()) {
-      auto &Entry = (*ParseableSwiftInterfaces)[*Name];
-      // The prepend path is applied later when copying.
-      DWARFDie CUDie = CU.getOrigUnit().getUnitDIE();
-      SmallString<128> ResolvedPath;
-      if (sys::path::is_relative(Path))
-        resolveRelativeObjectPath(ResolvedPath, CUDie);
-      sys::path::append(ResolvedPath, Path);
-      if (!Entry.empty() && Entry != ResolvedPath)
-        ReportWarning(
-            Twine("Conflicting parseable interfaces for Swift Module ") +
-                *Name + ": " + Entry + " and " + Path,
-            DIE);
-      Entry = std::string(ResolvedPath.str());
-    }
+  Optional<const char*> Name = dwarf::toString(DIE.find(dwarf::DW_AT_name));
+  if (!Name)
+    return;
+  auto &Entry = (*ParseableSwiftInterfaces)[*Name];
+  // The prepend path is applied later when copying.
+  DWARFDie CUDie = CU.getOrigUnit().getUnitDIE();
+  SmallString<128> ResolvedPath;
+  if (sys::path::is_relative(Path))
+    resolveRelativeObjectPath(ResolvedPath, CUDie);
+  sys::path::append(ResolvedPath, Path);
+  if (!Entry.empty() && Entry != ResolvedPath)
+    ReportWarning(Twine("Conflicting parseable interfaces for Swift Module ") +
+                      *Name + ": " + Entry + " and " + Path,
+                  DIE);
+  Entry = std::string(ResolvedPath.str());
 }
 
 /// The distinct types of work performed by the work loop in
@@ -409,10 +408,10 @@ static bool dieNeedsChildrenToBeMeaningful(uint32_t Tag) {
 void DWARFLinker::cleanupAuxiliarryData(LinkContext &Context) {
   Context.clear();
 
-  for (auto I = DIEBlocks.begin(), E = DIEBlocks.end(); I != E; ++I)
-    (*I)->~DIEBlock();
-  for (auto I = DIELocs.begin(), E = DIELocs.end(); I != E; ++I)
-    (*I)->~DIELoc();
+  for (DIEBlock *I : DIEBlocks)
+    I->~DIEBlock();
+  for (DIELoc *I : DIELocs)
+    I->~DIELoc();
 
   DIEBlocks.clear();
   DIELocs.clear();
@@ -846,7 +845,7 @@ void DWARFLinker::assignAbbrev(DIEAbbrev &Abbrev) {
 unsigned DWARFLinker::DIECloner::cloneStringAttribute(
     DIE &Die, AttributeSpec AttrSpec, const DWARFFormValue &Val,
     const DWARFUnit &U, OffsetsStringPool &StringPool, AttributesInfo &Info) {
-  Optional<const char *> String = Val.getAsCString();
+  Optional<const char *> String = dwarf::toString(Val);
   if (!String)
     return 0;
 
@@ -1423,6 +1422,11 @@ DIE *DWARFLinker::DIECloner::cloneDIE(const DWARFDie &InputDIE,
     Flags |= TF_InFunctionScope;
     if (!Info.InDebugMap && LLVM_LIKELY(!Update))
       Flags |= TF_SkipPC;
+  } else if (Abbrev->getTag() == dwarf::DW_TAG_variable) {
+    // Function-local globals could be in the debug map even when the function
+    // is not, e.g., inlined functions.
+    if ((Flags & TF_InFunctionScope) && Info.InDebugMap)
+      Flags &= ~TF_SkipPC;
   }
 
   for (const auto &AttrSpec : Abbrev->attributes()) {
diff --git a/llvm/lib/DWARFLinker/DWARFStreamer.cpp b/llvm/lib/DWARFLinker/DWARFStreamer.cpp
index 46e7457f2368..1ab6ead3b5f6 100644
--- a/llvm/lib/DWARFLinker/DWARFStreamer.cpp
+++ b/llvm/lib/DWARFLinker/DWARFStreamer.cpp
@@ -531,9 +531,7 @@ void DwarfStreamer::emitLineTableForUnit(MCDwarfLineTableParams Params,
 
   unsigned RowsSinceLastSequence = 0;
 
-  for (unsigned Idx = 0; Idx < Rows.size(); ++Idx) {
-    auto &Row = Rows[Idx];
-
+  for (DWARFDebugLine::Row &Row : Rows) {
     int64_t AddressDelta;
     if (Address == -1ULL) {
       MS->emitIntValue(dwarf::DW_LNS_extended_op, 1);
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFContext.cpp b/llvm/lib/DebugInfo/DWARF/DWARFContext.cpp
index c8331487f282..95135c95e8d2 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFContext.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFContext.cpp
@@ -1195,7 +1195,7 @@ void DWARFContext::addLocalsForDie(DWARFCompileUnit *CU, DWARFDie Subprogram,
             Die.getAttributeValueAsReferencedDie(DW_AT_abstract_origin))
       Die = Origin;
     if (auto NameAttr = Die.find(DW_AT_name))
-      if (Optional<const char *> Name = NameAttr->getAsCString())
+      if (Optional<const char *> Name = dwarf::toString(*NameAttr))
         Local.Name = *Name;
     if (auto Type = Die.getAttributeValueAsReferencedDie(DW_AT_type))
       Local.Size = getTypeSize(Type, getCUAddrSize());
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFDebugLine.cpp b/llvm/lib/DebugInfo/DWARF/DWARFDebugLine.cpp
index bda41b1f34e9..f36d3f87257a 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFDebugLine.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFDebugLine.cpp
@@ -1331,8 +1331,8 @@ Optional<StringRef> DWARFDebugLine::LineTable::getSourceByIndex(uint64_t FileInd
   if (Kind == FileLineInfoKind::None || !Prologue.hasFileAtIndex(FileIndex))
     return None;
   const FileNameEntry &Entry = Prologue.getFileNameEntry(FileIndex);
-  if (Optional<const char *> source = Entry.Source.getAsCString())
-    return StringRef(*source);
+  if (auto E = dwarf::toString(Entry.Source))
+    return StringRef(*E);
   return None;
 }
 
@@ -1350,10 +1350,10 @@ bool DWARFDebugLine::Prologue::getFileNameByIndex(
   if (Kind == FileLineInfoKind::None || !hasFileAtIndex(FileIndex))
     return false;
   const FileNameEntry &Entry = getFileNameEntry(FileIndex);
-  Optional<const char *> Name = Entry.Name.getAsCString();
-  if (!Name)
+  auto E = dwarf::toString(Entry.Name);
+  if (!E)
     return false;
-  StringRef FileName = *Name;
+  StringRef FileName = *E;
   if (Kind == FileLineInfoKind::RawValue ||
       isPathAbsoluteOnWindowsOrPosix(FileName)) {
     Result = std::string(FileName);
@@ -1372,11 +1372,10 @@ bool DWARFDebugLine::Prologue::getFileNameByIndex(
     // relative names.
     if ((Entry.DirIdx != 0 || Kind != FileLineInfoKind::RelativeFilePath) &&
         Entry.DirIdx < IncludeDirectories.size())
-      IncludeDir = IncludeDirectories[Entry.DirIdx].getAsCString().getValue();
+      IncludeDir = dwarf::toStringRef(IncludeDirectories[Entry.DirIdx]);
   } else {
     if (0 < Entry.DirIdx && Entry.DirIdx <= IncludeDirectories.size())
-      IncludeDir =
-          IncludeDirectories[Entry.DirIdx - 1].getAsCString().getValue();
+      IncludeDir = dwarf::toStringRef(IncludeDirectories[Entry.DirIdx - 1]);
   }
 
   // For absolute paths only, include the compilation directory of compile unit.
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFDebugLoc.cpp b/llvm/lib/DebugInfo/DWARF/DWARFDebugLoc.cpp
index cdffb36741c8..f39c7871d603 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFDebugLoc.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFDebugLoc.cpp
@@ -41,9 +41,7 @@ public:
 } // namespace
 
 static Error createResolverError(uint32_t Index, unsigned Kind) {
-  return createStringError(errc::invalid_argument,
-                           "Unable to resolve indirect address %u for: %s",
-                           Index, dwarf::LocListEncodingString(Kind).data());
+  return make_error<ResolverError>(Index, (dwarf::LoclistEntries)Kind);
 }
 
 Expected<Optional<DWARFLocationExpression>>
@@ -404,3 +402,10 @@ void DWARFDebugLoclists::dumpRange(uint64_t StartOffset, uint64_t Size,
     OS << '\n';
   }
 }
+
+void llvm::ResolverError::log(raw_ostream &OS) const {
+  OS << format("unable to resolve indirect address %u for: %s", Index,
+               dwarf::LocListEncodingString(Kind).data());
+}
+
+char llvm::ResolverError::ID;
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFDebugMacro.cpp b/llvm/lib/DebugInfo/DWARF/DWARFDebugMacro.cpp
index 80ffd81b3403..7a81d7ff064b 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFDebugMacro.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFDebugMacro.cpp
@@ -194,13 +194,11 @@ Error DWARFDebugMacro::parseImpl(
       if (MacroContributionOffset == MacroToUnits.end())
         return createStringError(errc::invalid_argument,
                                  "Macro contribution of the unit not found");
-      Optional<uint64_t> StrOffset =
+      Expected<uint64_t> StrOffset =
           MacroContributionOffset->second->getStringOffsetSectionItem(
               Data.getULEB128(&Offset));
       if (!StrOffset)
-        return createStringError(
-            errc::invalid_argument,
-            "String offsets contribution of the unit not found");
+        return StrOffset.takeError();
       E.MacroStr =
           MacroContributionOffset->second->getStringExtractor().getCStr(
               &*StrOffset);
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFDie.cpp b/llvm/lib/DebugInfo/DWARF/DWARFDie.cpp
index ed50f2635738..5421b2d59a1b 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFDie.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFDie.cpp
@@ -89,7 +89,6 @@ static void dumpLocationList(raw_ostream &OS, const DWARFFormValue &FormValue,
   U->getLocationTable().dumpLocationList(&Offset, OS, U->getBaseAddress(), MRI,
                                          Ctx.getDWARFObj(), U, DumpOpts,
                                          Indent);
-  return;
 }
 
 static void dumpLocationExpr(raw_ostream &OS, const DWARFFormValue &FormValue,
@@ -105,7 +104,6 @@ static void dumpLocationExpr(raw_ostream &OS, const DWARFFormValue &FormValue,
                      Ctx.isLittleEndian(), 0);
   DWARFExpression(Data, U->getAddressByteSize(), U->getFormParams().Format)
       .print(OS, DumpOpts, MRI, U);
-  return;
 }
 
 static DWARFDie resolveReferencedType(DWARFDie D,
@@ -672,6 +670,8 @@ struct DWARFTypePrinter {
       return;
     if (D.getTag() == DW_TAG_subprogram)
       return;
+    if (D.getTag() == DW_TAG_lexical_block)
+      return;
     D = D.resolveTypeUnitReference();
     if (DWARFDie P = D.getParent())
       appendScopes(P);
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFExpression.cpp b/llvm/lib/DebugInfo/DWARF/DWARFExpression.cpp
index d0fbd702e831..e19f5b8138fa 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFExpression.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFExpression.cpp
@@ -217,8 +217,8 @@ static void prettyPrintBaseTypeRef(DWARFUnit *U, raw_ostream &OS,
     if (DumpOpts.Verbose)
       OS << format("0x%08" PRIx64 " -> ", Operands[Operand]);
     OS << format("0x%08" PRIx64 ")", U->getOffset() + Operands[Operand]);
-    if (auto Name = Die.find(dwarf::DW_AT_name))
-      OS << " \"" << Name->getAsCString() << "\"";
+    if (auto Name = dwarf::toString(Die.find(dwarf::DW_AT_name)))
+      OS << " \"" << *Name << "\"";
   } else {
     OS << format(" <invalid base_type ref: 0x%" PRIx64 ">",
                  Operands[Operand]);
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFFormValue.cpp b/llvm/lib/DebugInfo/DWARF/DWARFFormValue.cpp
index cea0f63bbf81..86991a3949dd 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFFormValue.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFFormValue.cpp
@@ -613,50 +613,53 @@ void DWARFFormValue::dump(raw_ostream &OS, DIDumpOptions DumpOpts) const {
 }
 
 void DWARFFormValue::dumpString(raw_ostream &OS) const {
-  Optional<const char *> DbgStr = getAsCString();
-  if (DbgStr.hasValue()) {
+  if (auto DbgStr = dwarf::toString(*this)) {
     auto COS = WithColor(OS, HighlightColor::String);
     COS.get() << '"';
-    COS.get().write_escaped(DbgStr.getValue());
+    COS.get().write_escaped(*DbgStr);
     COS.get() << '"';
   }
 }
 
-Optional<const char *> DWARFFormValue::getAsCString() const {
+Expected<const char *> DWARFFormValue::getAsCString() const {
   if (!isFormClass(FC_String))
-    return None;
+    return make_error<StringError>("Invalid form for string attribute",
+                                   inconvertibleErrorCode());
   if (Form == DW_FORM_string)
     return Value.cstr;
   // FIXME: Add support for DW_FORM_GNU_strp_alt
   if (Form == DW_FORM_GNU_strp_alt || C == nullptr)
-    return None;
+    return make_error<StringError>("Unsupported form for string attribute",
+                                   inconvertibleErrorCode());
   uint64_t Offset = Value.uval;
-  if (Form == DW_FORM_line_strp) {
-    // .debug_line_str is tracked in the Context.
-    if (const char *Str = C->getLineStringExtractor().getCStr(&Offset))
-      return Str;
-    return None;
-  }
+  Optional<uint32_t> Index;
   if (Form == DW_FORM_GNU_str_index || Form == DW_FORM_strx ||
       Form == DW_FORM_strx1 || Form == DW_FORM_strx2 || Form == DW_FORM_strx3 ||
       Form == DW_FORM_strx4) {
     if (!U)
-      return None;
-    Optional<uint64_t> StrOffset = U->getStringOffsetSectionItem(Offset);
+      return make_error<StringError>("API limitation - string extraction not "
+                                     "available without a DWARFUnit",
+                                     inconvertibleErrorCode());
+    Expected<uint64_t> StrOffset = U->getStringOffsetSectionItem(Offset);
+    Index = Offset;
     if (!StrOffset)
-      return None;
+      return StrOffset.takeError();
     Offset = *StrOffset;
   }
   // Prefer the Unit's string extractor, because for .dwo it will point to
   // .debug_str.dwo, while the Context's extractor always uses .debug_str.
-  if (U) {
-    if (const char *Str = U->getStringExtractor().getCStr(&Offset))
-      return Str;
-    return None;
-  }
-  if (const char *Str = C->getStringExtractor().getCStr(&Offset))
+  DataExtractor StrData = Form == DW_FORM_line_strp
+                              ? C->getLineStringExtractor()
+                          : U ? U->getStringExtractor()
+                              : C->getStringExtractor();
+  if (const char *Str = StrData.getCStr(&Offset))
     return Str;
-  return None;
+  std::string Msg = FormEncodingString(Form).str();
+  if (Index)
+    Msg += (" uses index " + Twine(*Index) + ", but the referenced string").str();
+  Msg += (" offset " + Twine(Offset) + " is beyond .debug_str bounds").str();
+  return make_error<StringError>(Msg,
+      inconvertibleErrorCode());
 }
 
 Optional<uint64_t> DWARFFormValue::getAsAddress() const {
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFUnit.cpp b/llvm/lib/DebugInfo/DWARF/DWARFUnit.cpp
index 82c34f537036..eed0a60ec75e 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFUnit.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFUnit.cpp
@@ -214,13 +214,17 @@ DWARFUnit::getAddrOffsetSectionItem(uint32_t Index) const {
   return {{Address, Section}};
 }
 
-Optional<uint64_t> DWARFUnit::getStringOffsetSectionItem(uint32_t Index) const {
+Expected<uint64_t> DWARFUnit::getStringOffsetSectionItem(uint32_t Index) const {
   if (!StringOffsetsTableContribution)
-    return None;
+    return make_error<StringError>(
+        "DW_FORM_strx used without a valid string offsets table",
+        inconvertibleErrorCode());
   unsigned ItemSize = getDwarfStringOffsetsByteSize();
   uint64_t Offset = getStringOffsetsBase() + Index * ItemSize;
   if (StringOffsetSection.Data.size() < Offset + ItemSize)
-    return None;
+    return make_error<StringError>("DW_FORM_strx uses index " + Twine(Index) +
+                                       ", which is too large",
+                                   inconvertibleErrorCode());
   DWARFDataExtractor DA(Context.getDWARFObj(), StringOffsetSection,
                         isLittleEndian, 0);
   return DA.getRelocatedValue(ItemSize, &Offset);
@@ -603,7 +607,7 @@ bool DWARFUnit::parseDWO() {
     DWO->setAddrOffsetSection(AddrOffsetSection, *AddrOffsetSectionBase);
   if (getVersion() == 4) {
     auto DWORangesBase = UnitDie.getRangesBaseAttribute();
-    DWO->setRangesSection(RangeSection, DWORangesBase ? *DWORangesBase : 0);
+    DWO->setRangesSection(RangeSection, DWORangesBase.getValueOr(0));
   }
 
   return true;
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp b/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp
index 7673a721c4ea..6424c2f59844 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp
@@ -390,6 +390,9 @@ bool DWARFVerifier::handleDebugInfo() {
 
   OS << "Verifying non-dwo Units...\n";
   NumErrors += verifyUnits(DCtx.getNormalUnitsVector());
+
+  OS << "Verifying dwo Units...\n";
+  NumErrors += verifyUnits(DCtx.getDWOUnitsVector());
   return NumErrors == 0;
 }
 
@@ -400,10 +403,13 @@ unsigned DWARFVerifier::verifyDieRanges(const DWARFDie &Die,
   if (!Die.isValid())
     return NumErrors;
 
+  DWARFUnit *Unit = Die.getDwarfUnit();
+
   auto RangesOrError = Die.getAddressRanges();
   if (!RangesOrError) {
     // FIXME: Report the error.
-    ++NumErrors;
+    if (!Unit->isDWOUnit())
+      ++NumErrors;
     llvm::consumeError(RangesOrError.takeError());
     return NumErrors;
   }
@@ -496,15 +502,18 @@ unsigned DWARFVerifier::verifyDebugInfoAttribute(const DWARFDie &Die,
   };
 
   const DWARFObject &DObj = DCtx.getDWARFObj();
+  DWARFUnit *U = Die.getDwarfUnit();
   const auto Attr = AttrValue.Attr;
   switch (Attr) {
   case DW_AT_ranges:
     // Make sure the offset in the DW_AT_ranges attribute is valid.
     if (auto SectionOffset = AttrValue.Value.getAsSectionOffset()) {
-      unsigned DwarfVersion = Die.getDwarfUnit()->getVersion();
+      unsigned DwarfVersion = U->getVersion();
       const DWARFSection &RangeSection = DwarfVersion < 5
                                              ? DObj.getRangesSection()
                                              : DObj.getRnglistsSection();
+      if (U->isDWOUnit() && RangeSection.Data.empty())
+        break;
       if (*SectionOffset >= RangeSection.Data.size())
         ReportError(
             "DW_AT_ranges offset is beyond " +
@@ -517,7 +526,7 @@ unsigned DWARFVerifier::verifyDebugInfoAttribute(const DWARFDie &Die,
   case DW_AT_stmt_list:
     // Make sure the offset in the DW_AT_stmt_list attribute is valid.
     if (auto SectionOffset = AttrValue.Value.getAsSectionOffset()) {
-      if (*SectionOffset >= DObj.getLineSection().Data.size())
+      if (*SectionOffset >= U->getLineSection().Data.size())
         ReportError("DW_AT_stmt_list offset is beyond .debug_line bounds: " +
                     llvm::formatv("{0:x8}", *SectionOffset));
       break;
@@ -525,9 +534,18 @@ unsigned DWARFVerifier::verifyDebugInfoAttribute(const DWARFDie &Die,
     ReportError("DIE has invalid DW_AT_stmt_list encoding:");
     break;
   case DW_AT_location: {
+    // FIXME: It might be nice if there's a way to walk location expressions
+    // without trying to resolve the address ranges - it'd be a more efficient
+    // API (since the API is currently unnecessarily resolving addresses for
+    // this use case which only wants to validate the expressions themselves) &
+    // then the expressions could be validated even if the addresses can't be
+    // resolved.
+    // That sort of API would probably look like a callback "for each
+    // expression" with some way to lazily resolve the address ranges when
+    // needed (& then the existing API used here could be built on top of that -
+    // using the callback API to build the data structure and return it).
     if (Expected<std::vector<DWARFLocationExpression>> Loc =
             Die.getLocations(DW_AT_location)) {
-      DWARFUnit *U = Die.getDwarfUnit();
       for (const auto &Entry : *Loc) {
         DataExtractor Data(toStringRef(Entry.Expr), DCtx.isLittleEndian(), 0);
         DWARFExpression Expression(Data, U->getAddressByteSize(),
@@ -539,8 +557,12 @@ unsigned DWARFVerifier::verifyDebugInfoAttribute(const DWARFDie &Die,
         if (Error || !Expression.verify(U))
           ReportError("DIE contains invalid DWARF expression:");
       }
-    } else
-      ReportError(toString(Loc.takeError()));
+    } else if (Error Err = handleErrors(
+                   Loc.takeError(), [&](std::unique_ptr<ResolverError> E) {
+                     return U->isDWOUnit() ? Error::success()
+                                           : Error(std::move(E));
+                   }))
+      ReportError(toString(std::move(Err)));
     break;
   }
   case DW_AT_specification:
@@ -576,7 +598,8 @@ unsigned DWARFVerifier::verifyDebugInfoAttribute(const DWARFDie &Die,
   case DW_AT_call_file:
   case DW_AT_decl_file: {
     if (auto FileIdx = AttrValue.Value.getAsUnsignedConstant()) {
-      DWARFUnit *U = Die.getDwarfUnit();
+      if (U->isDWOUnit() && !U->isTypeUnit())
+        break;
       const auto *LT = U->getContext().getLineTableForUnit(U);
       if (LT) {
         if (!LT->hasFileAtIndex(*FileIdx)) {
@@ -616,7 +639,6 @@ unsigned DWARFVerifier::verifyDebugInfoForm(const DWARFDie &Die,
                                             DWARFAttribute &AttrValue,
                                             ReferenceMap &LocalReferences,
                                             ReferenceMap &CrossUnitReferences) {
-  const DWARFObject &DObj = DCtx.getDWARFObj();
   auto DieCU = Die.getDwarfUnit();
   unsigned NumErrors = 0;
   const auto Form = AttrValue.Value.getForm();
@@ -667,51 +689,15 @@ unsigned DWARFVerifier::verifyDebugInfoForm(const DWARFDie &Die,
     }
     break;
   }
-  case DW_FORM_strp: {
-    auto SecOffset = AttrValue.Value.getAsSectionOffset();
-    assert(SecOffset); // DW_FORM_strp is a section offset.
-    if (SecOffset && *SecOffset >= DObj.getStrSection().size()) {
-      ++NumErrors;
-      error() << "DW_FORM_strp offset beyond .debug_str bounds:\n";
-      dump(Die) << '\n';
-    }
-    break;
-  }
+  case DW_FORM_strp:
   case DW_FORM_strx:
   case DW_FORM_strx1:
   case DW_FORM_strx2:
   case DW_FORM_strx3:
   case DW_FORM_strx4: {
-    auto Index = AttrValue.Value.getRawUValue();
-    auto DieCU = Die.getDwarfUnit();
-    // Check that we have a valid DWARF v5 string offsets table.
-    if (!DieCU->getStringOffsetsTableContribution()) {
-      ++NumErrors;
-      error() << FormEncodingString(Form)
-              << " used without a valid string offsets table:\n";
-      dump(Die) << '\n';
-      break;
-    }
-    // Check that the index is within the bounds of the section.
-    unsigned ItemSize = DieCU->getDwarfStringOffsetsByteSize();
-    // Use a 64-bit type to calculate the offset to guard against overflow.
-    uint64_t Offset =
-        (uint64_t)DieCU->getStringOffsetsBase() + Index * ItemSize;
-    if (DObj.getStrOffsetsSection().Data.size() < Offset + ItemSize) {
-      ++NumErrors;
-      error() << FormEncodingString(Form) << " uses index "
-              << format("%" PRIu64, Index) << ", which is too large:\n";
-      dump(Die) << '\n';
-      break;
-    }
-    // Check that the string offset is valid.
-    uint64_t StringOffset = *DieCU->getStringOffsetSectionItem(Index);
-    if (StringOffset >= DObj.getStrSection().size()) {
+    if (Error E = AttrValue.Value.getAsCString().takeError()) {
       ++NumErrors;
-      error() << FormEncodingString(Form) << " uses index "
-              << format("%" PRIu64, Index)
-              << ", but the referenced string"
-                 " offset is beyond .debug_str bounds:\n";
+      error() << toString(std::move(E)) << ":\n";
       dump(Die) << '\n';
     }
     break;
diff --git a/llvm/lib/DebugInfo/GSYM/DwarfTransformer.cpp b/llvm/lib/DebugInfo/GSYM/DwarfTransformer.cpp
index b2c43b893cd3..6eef6f84ab40 100644
--- a/llvm/lib/DebugInfo/GSYM/DwarfTransformer.cpp
+++ b/llvm/lib/DebugInfo/GSYM/DwarfTransformer.cpp
@@ -531,7 +531,7 @@ llvm::Error DwarfTransformer::verify(StringRef GsymPath) {
             << LR->Locations.size() << "\n";
         Log << "    " << NumDwarfInlineInfos << " DWARF frames:\n";
         for (size_t Idx = 0; Idx < NumDwarfInlineInfos; ++Idx) {
-          const auto dii = DwarfInlineInfos.getFrame(Idx);
+          const auto &dii = DwarfInlineInfos.getFrame(Idx);
           Log << "    [" << Idx << "]: " << dii.FunctionName << " @ "
               << dii.FileName << ':' << dii.Line << '\n';
         }
@@ -551,7 +551,7 @@ llvm::Error DwarfTransformer::verify(StringRef GsymPath) {
             ++Idx) {
         const auto &gii = LR->Locations[Idx];
         if (Idx < NumDwarfInlineInfos) {
-          const auto dii = DwarfInlineInfos.getFrame(Idx);
+          const auto &dii = DwarfInlineInfos.getFrame(Idx);
           gsymFilename = LR->getSourceFile(Idx);
           // Verify function name
           if (dii.FunctionName.find(gii.Name.str()) != 0)
diff --git a/llvm/lib/DebugInfo/MSF/MSFBuilder.cpp b/llvm/lib/DebugInfo/MSF/MSFBuilder.cpp
index 1a92e2cb7754..f9a763d724a8 100644
--- a/llvm/lib/DebugInfo/MSF/MSFBuilder.cpp
+++ b/llvm/lib/DebugInfo/MSF/MSFBuilder.cpp
@@ -343,15 +343,25 @@ Expected<FileBufferByteStream> MSFBuilder::commit(StringRef Path,
   Layout = std::move(*L);
 
   uint64_t FileSize = uint64_t(Layout.SB->BlockSize) * Layout.SB->NumBlocks;
-  if (FileSize > UINT32_MAX) {
-    // FIXME: Changing the BinaryStream classes to use 64-bit numbers lets
-    // us create PDBs larger than 4 GiB successfully. The file format is
-    // block-based and as long as each stream is small enough, PDBs larger than
-    // 4 GiB might work. Check if tools can handle these large PDBs, and if so
-    // add support for writing them.
+  // Ensure that the file size is under the limit for the specified block size.
+  if (FileSize > getMaxFileSizeFromBlockSize(Layout.SB->BlockSize)) {
+    msf_error_code error_code = [](uint32_t BlockSize) {
+      switch (BlockSize) {
+      case 8192:
+        return msf_error_code::size_overflow_8192;
+      case 16384:
+        return msf_error_code::size_overflow_16384;
+      case 32768:
+        return msf_error_code::size_overflow_32768;
+      default:
+        return msf_error_code::size_overflow_4096;
+      }
+    }(Layout.SB->BlockSize);
+
     return make_error<MSFError>(
-        msf_error_code::size_overflow,
-        formatv("File size would have been {0,1:N}", FileSize));
+        error_code,
+        formatv("File size {0,1:N} too large for current PDB page size {1}",
+                FileSize, Layout.SB->BlockSize));
   }
 
   auto OutFileOrError = FileOutputBuffer::create(Path, FileSize);
diff --git a/llvm/lib/DebugInfo/MSF/MSFError.cpp b/llvm/lib/DebugInfo/MSF/MSFError.cpp
index e42157e9d48e..9df2158423a4 100644
--- a/llvm/lib/DebugInfo/MSF/MSFError.cpp
+++ b/llvm/lib/DebugInfo/MSF/MSFError.cpp
@@ -28,8 +28,14 @@ public:
     case msf_error_code::insufficient_buffer:
       return "The buffer is not large enough to read the requested number of "
              "bytes.";
-    case msf_error_code::size_overflow:
+    case msf_error_code::size_overflow_4096:
       return "Output data is larger than 4 GiB.";
+    case msf_error_code::size_overflow_8192:
+      return "Output data is larger than 8 GiB.";
+    case msf_error_code::size_overflow_16384:
+      return "Output data is larger than 16 GiB.";
+    case msf_error_code::size_overflow_32768:
+      return "Output data is larger than 32 GiB.";
     case msf_error_code::not_writable:
       return "The specified stream is not writable.";
     case msf_error_code::no_stream:
diff --git a/llvm/lib/DebugInfo/PDB/Native/PDBFile.cpp b/llvm/lib/DebugInfo/PDB/Native/PDBFile.cpp
index cde645236851..5c61530c470d 100644
--- a/llvm/lib/DebugInfo/PDB/Native/PDBFile.cpp
+++ b/llvm/lib/DebugInfo/PDB/Native/PDBFile.cpp
@@ -100,7 +100,7 @@ PDBFile::getStreamBlockList(uint32_t StreamIndex) const {
   return ContainerLayout.StreamMap[StreamIndex];
 }
 
-uint32_t PDBFile::getFileSize() const { return Buffer->getLength(); }
+uint64_t PDBFile::getFileSize() const { return Buffer->getLength(); }
 
 Expected<ArrayRef<uint8_t>> PDBFile::getBlockData(uint32_t BlockIndex,
                                                   uint32_t NumBytes) const {
diff --git a/llvm/lib/DebugInfo/PDB/Native/SymbolCache.cpp b/llvm/lib/DebugInfo/PDB/Native/SymbolCache.cpp
index fd9a0deb54d6..f9e67014477e 100644
--- a/llvm/lib/DebugInfo/PDB/Native/SymbolCache.cpp
+++ b/llvm/lib/DebugInfo/PDB/Native/SymbolCache.cpp
@@ -518,8 +518,8 @@ SymbolCache::findLineTable(uint16_t Modi) const {
                            const std::vector<LineTableEntry> &RHS) {
     return LHS[0].Addr < RHS[0].Addr;
   });
-  for (size_t I = 0; I < EntryList.size(); ++I)
-    llvm::append_range(ModuleLineTable, EntryList[I]);
+  for (std::vector<LineTableEntry> &I : EntryList)
+    llvm::append_range(ModuleLineTable, I);
 
   return ModuleLineTable;
 }
diff --git a/llvm/lib/DebugInfo/Symbolize/Symbolize.cpp b/llvm/lib/DebugInfo/Symbolize/Symbolize.cpp
index f3f09584fdc9..5ec79df17fed 100644
--- a/llvm/lib/DebugInfo/Symbolize/Symbolize.cpp
+++ b/llvm/lib/DebugInfo/Symbolize/Symbolize.cpp
@@ -20,6 +20,7 @@
 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
 #include "llvm/DebugInfo/PDB/PDB.h"
 #include "llvm/DebugInfo/PDB/PDBContext.h"
+#include "llvm/Debuginfod/Debuginfod.h"
 #include "llvm/Demangle/Demangle.h"
 #include "llvm/Object/COFF.h"
 #include "llvm/Object/MachO.h"
@@ -384,7 +385,14 @@ bool findDebugBinary(const std::vector<std::string> &DebugFileDirectory,
       }
     }
   }
-  return false;
+  // Try debuginfod client cache and known servers.
+  Expected<std::string> PathOrErr = getCachedOrDownloadDebuginfo(BuildID);
+  if (!PathOrErr) {
+    consumeError(PathOrErr.takeError());
+    return false;
+  }
+  Result = *PathOrErr;
+  return true;
 }
 
 } // end anonymous namespace
diff --git a/llvm/lib/Debuginfod/Debuginfod.cpp b/llvm/lib/Debuginfod/Debuginfod.cpp
new file mode 100644
index 000000000000..389b18fd62ac
--- /dev/null
+++ b/llvm/lib/Debuginfod/Debuginfod.cpp
@@ -0,0 +1,183 @@
+//===-- llvm/Debuginfod/Debuginfod.cpp - Debuginfod client library --------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+///
+/// This file defines the fetchInfo function, which retrieves
+/// any of the three supported artifact types: (executable, debuginfo, source
+/// file) associated with a build-id from debuginfod servers. If a source file
+/// is to be fetched, its absolute path must be specified in the Description
+/// argument to fetchInfo.
+///
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Debuginfod/Debuginfod.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Debuginfod/HTTPClient.h"
+#include "llvm/Support/CachePruning.h"
+#include "llvm/Support/Caching.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/FileUtilities.h"
+#include "llvm/Support/xxhash.h"
+
+namespace llvm {
+static std::string uniqueKey(llvm::StringRef S) { return utostr(xxHash64(S)); }
+
+// Returns a binary BuildID as a normalized hex string.
+// Uses lowercase for compatibility with common debuginfod servers.
+static std::string buildIDToString(BuildIDRef ID) {
+  return llvm::toHex(ID, /*LowerCase=*/true);
+}
+
+Expected<SmallVector<StringRef>> getDefaultDebuginfodUrls() {
+  const char *DebuginfodUrlsEnv = std::getenv("DEBUGINFOD_URLS");
+  if (DebuginfodUrlsEnv == NULL)
+    return SmallVector<StringRef>();
+
+  SmallVector<StringRef> DebuginfodUrls;
+  StringRef(DebuginfodUrlsEnv).split(DebuginfodUrls, " ");
+  return DebuginfodUrls;
+}
+
+Expected<std::string> getDefaultDebuginfodCacheDirectory() {
+  if (const char *CacheDirectoryEnv = std::getenv("DEBUGINFOD_CACHE_PATH"))
+    return CacheDirectoryEnv;
+
+  SmallString<64> CacheDirectory;
+  if (!sys::path::cache_directory(CacheDirectory))
+    return createStringError(
+        errc::io_error, "Unable to determine appropriate cache directory.");
+  return std::string(CacheDirectory);
+}
+
+std::chrono::milliseconds getDefaultDebuginfodTimeout() {
+  long Timeout;
+  const char *DebuginfodTimeoutEnv = std::getenv("DEBUGINFOD_TIMEOUT");
+  if (DebuginfodTimeoutEnv &&
+      to_integer(StringRef(DebuginfodTimeoutEnv).trim(), Timeout, 10))
+    return std::chrono::milliseconds(Timeout * 1000);
+
+  return std::chrono::milliseconds(90 * 1000);
+}
+
+/// The following functions fetch a debuginfod artifact to a file in a local
+/// cache and return the cached file path. They first search the local cache,
+/// followed by the debuginfod servers.
+
+Expected<std::string> getCachedOrDownloadSource(BuildIDRef ID,
+                                                StringRef SourceFilePath) {
+  SmallString<64> UrlPath;
+  sys::path::append(UrlPath, sys::path::Style::posix, "buildid",
+                    buildIDToString(ID), "source",
+                    sys::path::convert_to_slash(SourceFilePath));
+  return getCachedOrDownloadArtifact(uniqueKey(UrlPath), UrlPath);
+}
+
+Expected<std::string> getCachedOrDownloadExecutable(BuildIDRef ID) {
+  SmallString<64> UrlPath;
+  sys::path::append(UrlPath, sys::path::Style::posix, "buildid",
+                    buildIDToString(ID), "executable");
+  return getCachedOrDownloadArtifact(uniqueKey(UrlPath), UrlPath);
+}
+
+Expected<std::string> getCachedOrDownloadDebuginfo(BuildIDRef ID) {
+  SmallString<64> UrlPath;
+  sys::path::append(UrlPath, sys::path::Style::posix, "buildid",
+                    buildIDToString(ID), "debuginfo");
+  return getCachedOrDownloadArtifact(uniqueKey(UrlPath), UrlPath);
+}
+
+// General fetching function.
+Expected<std::string> getCachedOrDownloadArtifact(StringRef UniqueKey,
+                                                  StringRef UrlPath) {
+  SmallString<10> CacheDir;
+
+  Expected<std::string> CacheDirOrErr = getDefaultDebuginfodCacheDirectory();
+  if (!CacheDirOrErr)
+    return CacheDirOrErr.takeError();
+  CacheDir = *CacheDirOrErr;
+
+  Expected<SmallVector<StringRef>> DebuginfodUrlsOrErr =
+      getDefaultDebuginfodUrls();
+  if (!DebuginfodUrlsOrErr)
+    return DebuginfodUrlsOrErr.takeError();
+  SmallVector<StringRef> &DebuginfodUrls = *DebuginfodUrlsOrErr;
+  return getCachedOrDownloadArtifact(UniqueKey, UrlPath, CacheDir,
+                                     DebuginfodUrls,
+                                     getDefaultDebuginfodTimeout());
+}
+
+Expected<std::string> getCachedOrDownloadArtifact(
+    StringRef UniqueKey, StringRef UrlPath, StringRef CacheDirectoryPath,
+    ArrayRef<StringRef> DebuginfodUrls, std::chrono::milliseconds Timeout) {
+  SmallString<64> AbsCachedArtifactPath;
+  sys::path::append(AbsCachedArtifactPath, CacheDirectoryPath,
+                    "llvmcache-" + UniqueKey);
+
+  Expected<FileCache> CacheOrErr =
+      localCache("Debuginfod-client", ".debuginfod-client", CacheDirectoryPath);
+  if (!CacheOrErr)
+    return CacheOrErr.takeError();
+
+  FileCache Cache = *CacheOrErr;
+  // We choose an arbitrary Task parameter as we do not make use of it.
+  unsigned Task = 0;
+  Expected<AddStreamFn> CacheAddStreamOrErr = Cache(Task, UniqueKey);
+  if (!CacheAddStreamOrErr)
+    return CacheAddStreamOrErr.takeError();
+  AddStreamFn &CacheAddStream = *CacheAddStreamOrErr;
+  if (!CacheAddStream)
+    return std::string(AbsCachedArtifactPath);
+  // The artifact was not found in the local cache, query the debuginfod
+  // servers.
+  if (!HTTPClient::isAvailable())
+    return createStringError(errc::io_error,
+                             "No working HTTP client is available.");
+
+  if (!HTTPClient::IsInitialized)
+    return createStringError(
+        errc::io_error,
+        "A working HTTP client is available, but it is not initialized. To "
+        "allow Debuginfod to make HTTP requests, call HTTPClient::initialize() "
+        "at the beginning of main.");
+
+  HTTPClient Client;
+  Client.setTimeout(Timeout);
+  for (StringRef ServerUrl : DebuginfodUrls) {
+    SmallString<64> ArtifactUrl;
+    sys::path::append(ArtifactUrl, sys::path::Style::posix, ServerUrl, UrlPath);
+
+    Expected<HTTPResponseBuffer> ResponseOrErr = Client.get(ArtifactUrl);
+    if (!ResponseOrErr)
+      return ResponseOrErr.takeError();
+
+    HTTPResponseBuffer &Response = *ResponseOrErr;
+    if (Response.Code != 200)
+      continue;
+
+    // We have retrieved the artifact from this server, and now add it to the
+    // file cache.
+    Expected<std::unique_ptr<CachedFileStream>> FileStreamOrErr =
+        CacheAddStream(Task);
+    if (!FileStreamOrErr)
+      return FileStreamOrErr.takeError();
+    std::unique_ptr<CachedFileStream> &FileStream = *FileStreamOrErr;
+    if (!Response.Body)
+      return createStringError(
+          errc::io_error, "Unallocated MemoryBuffer in HTTPResponseBuffer.");
+
+    *FileStream->OS << StringRef(Response.Body->getBufferStart(),
+                                 Response.Body->getBufferSize());
+
+    // Return the path to the artifact on disk.
+    return std::string(AbsCachedArtifactPath);
+  }
+
+  return createStringError(errc::argument_out_of_domain, "build id not found");
+}
+} // namespace llvm
diff --git a/llvm/lib/Debuginfod/HTTPClient.cpp b/llvm/lib/Debuginfod/HTTPClient.cpp
new file mode 100644
index 000000000000..65f457933b92
--- /dev/null
+++ b/llvm/lib/Debuginfod/HTTPClient.cpp
@@ -0,0 +1,216 @@
+//===-- llvm/Debuginfod/HTTPClient.cpp - HTTP client library ----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+///
+/// This file defines the methods of the HTTPRequest, HTTPClient, and
+/// BufferedHTTPResponseHandler classes.
+///
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Debuginfod/HTTPClient.h"
+#include "llvm/ADT/APInt.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Errc.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/MemoryBuffer.h"
+#ifdef LLVM_ENABLE_CURL
+#include <curl/curl.h>
+#endif
+
+using namespace llvm;
+
+HTTPRequest::HTTPRequest(StringRef Url) { this->Url = Url.str(); }
+
+bool operator==(const HTTPRequest &A, const HTTPRequest &B) {
+  return A.Url == B.Url && A.Method == B.Method &&
+         A.FollowRedirects == B.FollowRedirects;
+}
+
+HTTPResponseHandler::~HTTPResponseHandler() = default;
+
+static inline bool parseContentLengthHeader(StringRef LineRef,
+                                            size_t &ContentLength) {
+  // Content-Length is a mandatory header, and the only one we handle.
+  return LineRef.consume_front("Content-Length: ") &&
+         to_integer(LineRef.trim(), ContentLength, 10);
+}
+
+Error BufferedHTTPResponseHandler::handleHeaderLine(StringRef HeaderLine) {
+  if (ResponseBuffer.Body)
+    return Error::success();
+
+  size_t ContentLength;
+  if (parseContentLengthHeader(HeaderLine, ContentLength))
+    ResponseBuffer.Body =
+        WritableMemoryBuffer::getNewUninitMemBuffer(ContentLength);
+
+  return Error::success();
+}
+
+Error BufferedHTTPResponseHandler::handleBodyChunk(StringRef BodyChunk) {
+  if (!ResponseBuffer.Body)
+    return createStringError(errc::io_error,
+                             "Unallocated response buffer. HTTP Body data "
+                             "received before Content-Length header.");
+  if (Offset + BodyChunk.size() > ResponseBuffer.Body->getBufferSize())
+    return createStringError(errc::io_error,
+                             "Content size exceeds buffer size.");
+  memcpy(ResponseBuffer.Body->getBufferStart() + Offset, BodyChunk.data(),
+         BodyChunk.size());
+  Offset += BodyChunk.size();
+  return Error::success();
+}
+
+Error BufferedHTTPResponseHandler::handleStatusCode(unsigned Code) {
+  ResponseBuffer.Code = Code;
+  return Error::success();
+}
+
+bool HTTPClient::IsInitialized = false;
+
+class HTTPClientCleanup {
+public:
+  ~HTTPClientCleanup() { HTTPClient::cleanup(); }
+};
+static const HTTPClientCleanup Cleanup;
+
+Expected<HTTPResponseBuffer> HTTPClient::perform(const HTTPRequest &Request) {
+  BufferedHTTPResponseHandler Handler;
+  if (Error Err = perform(Request, Handler))
+    return std::move(Err);
+  return std::move(Handler.ResponseBuffer);
+}
+
+Expected<HTTPResponseBuffer> HTTPClient::get(StringRef Url) {
+  HTTPRequest Request(Url);
+  return perform(Request);
+}
+
+#ifdef LLVM_ENABLE_CURL
+
+bool HTTPClient::isAvailable() { return true; }
+
+void HTTPClient::initialize() {
+  if (!IsInitialized) {
+    curl_global_init(CURL_GLOBAL_ALL);
+    IsInitialized = true;
+  }
+}
+
+void HTTPClient::cleanup() {
+  if (IsInitialized) {
+    curl_global_cleanup();
+    IsInitialized = false;
+  }
+}
+
+void HTTPClient::setTimeout(std::chrono::milliseconds Timeout) {
+  if (Timeout < std::chrono::milliseconds(0))
+    Timeout = std::chrono::milliseconds(0);
+  curl_easy_setopt(Curl, CURLOPT_TIMEOUT_MS, Timeout.count());
+}
+
+/// CurlHTTPRequest and the curl{Header,Write}Function are implementation
+/// details used to work with Curl. Curl makes callbacks with a single
+/// customizable pointer parameter.
+struct CurlHTTPRequest {
+  CurlHTTPRequest(HTTPResponseHandler &Handler) : Handler(Handler) {}
+  void storeError(Error Err) {
+    ErrorState = joinErrors(std::move(Err), std::move(ErrorState));
+  }
+  HTTPResponseHandler &Handler;
+  llvm::Error ErrorState = Error::success();
+};
+
+static size_t curlHeaderFunction(char *Contents, size_t Size, size_t NMemb,
+                                 CurlHTTPRequest *CurlRequest) {
+  assert(Size == 1 && "The Size passed by libCURL to CURLOPT_HEADERFUNCTION "
+                      "should always be 1.");
+  if (Error Err =
+          CurlRequest->Handler.handleHeaderLine(StringRef(Contents, NMemb))) {
+    CurlRequest->storeError(std::move(Err));
+    return 0;
+  }
+  return NMemb;
+}
+
+static size_t curlWriteFunction(char *Contents, size_t Size, size_t NMemb,
+                                CurlHTTPRequest *CurlRequest) {
+  Size *= NMemb;
+  if (Error Err =
+          CurlRequest->Handler.handleBodyChunk(StringRef(Contents, Size))) {
+    CurlRequest->storeError(std::move(Err));
+    return 0;
+  }
+  return Size;
+}
+
+HTTPClient::HTTPClient() {
+  assert(IsInitialized &&
+         "Must call HTTPClient::initialize() at the beginning of main().");
+  if (Curl)
+    return;
+  assert((Curl = curl_easy_init()) && "Curl could not be initialized.");
+  // Set the callback hooks.
+  curl_easy_setopt(Curl, CURLOPT_WRITEFUNCTION, curlWriteFunction);
+  curl_easy_setopt(Curl, CURLOPT_HEADERFUNCTION, curlHeaderFunction);
+}
+
+HTTPClient::~HTTPClient() { curl_easy_cleanup(Curl); }
+
+Error HTTPClient::perform(const HTTPRequest &Request,
+                          HTTPResponseHandler &Handler) {
+  if (Request.Method != HTTPMethod::GET)
+    return createStringError(errc::invalid_argument,
+                             "Unsupported CURL request method.");
+
+  SmallString<128> Url = Request.Url;
+  curl_easy_setopt(Curl, CURLOPT_URL, Url.c_str());
+  curl_easy_setopt(Curl, CURLOPT_FOLLOWLOCATION, Request.FollowRedirects);
+
+  CurlHTTPRequest CurlRequest(Handler);
+  curl_easy_setopt(Curl, CURLOPT_WRITEDATA, &CurlRequest);
+  curl_easy_setopt(Curl, CURLOPT_HEADERDATA, &CurlRequest);
+  CURLcode CurlRes = curl_easy_perform(Curl);
+  if (CurlRes != CURLE_OK)
+    return joinErrors(std::move(CurlRequest.ErrorState),
+                      createStringError(errc::io_error,
+                                        "curl_easy_perform() failed: %s\n",
+                                        curl_easy_strerror(CurlRes)));
+  if (CurlRequest.ErrorState)
+    return std::move(CurlRequest.ErrorState);
+
+  unsigned Code;
+  curl_easy_getinfo(Curl, CURLINFO_RESPONSE_CODE, &Code);
+  if (Error Err = Handler.handleStatusCode(Code))
+    return joinErrors(std::move(CurlRequest.ErrorState), std::move(Err));
+
+  return std::move(CurlRequest.ErrorState);
+}
+
+#else
+
+HTTPClient::HTTPClient() = default;
+
+HTTPClient::~HTTPClient() = default;
+
+bool HTTPClient::isAvailable() { return false; }
+
+void HTTPClient::initialize() {}
+
+void HTTPClient::cleanup() {}
+
+void HTTPClient::setTimeout(std::chrono::milliseconds Timeout) {}
+
+Error HTTPClient::perform(const HTTPRequest &Request,
+                          HTTPResponseHandler &Handler) {
+  llvm_unreachable("No HTTP Client implementation available.");
+}
+
+#endif
diff --git a/llvm/lib/Demangle/DLangDemangle.cpp b/llvm/lib/Demangle/DLangDemangle.cpp
index f380aa90035e..0cefbd63a7ae 100644
--- a/llvm/lib/Demangle/DLangDemangle.cpp
+++ b/llvm/lib/Demangle/DLangDemangle.cpp
@@ -242,11 +242,77 @@ const char *Demangler::parseIdentifier(OutputBuffer *Demangled,
 
   // TODO: Parse template instances with a length prefix.
 
+  // There can be multiple different declarations in the same function that
+  // have the same mangled name.  To make the mangled names unique, a fake
+  // parent in the form `__Sddd' is added to the symbol.
+  if (Len >= 4 && Mangled[0] == '_' && Mangled[1] == '_' && Mangled[2] == 'S') {
+    const char *NumPtr = Mangled + 3;
+    while (NumPtr < (Mangled + Len) && std::isdigit(*NumPtr))
+      ++NumPtr;
+
+    if (Mangled + Len == NumPtr) {
+      // Skip over the fake parent.
+      Mangled += Len;
+      return parseIdentifier(Demangled, Mangled);
+    }
+
+    // Else demangle it as a plain identifier.
+  }
+
   return parseLName(Demangled, Mangled, Len);
 }
 
 const char *Demangler::parseLName(OutputBuffer *Demangled, const char *Mangled,
                                   unsigned long Len) {
+  switch (Len) {
+  case 6:
+    if (strncmp(Mangled, "__initZ", Len + 1) == 0) {
+      // The static initializer for a given symbol.
+      Demangled->prepend("initializer for ");
+      Demangled->setCurrentPosition(Demangled->getCurrentPosition() - 1);
+      Mangled += Len;
+      return Mangled;
+    }
+    if (strncmp(Mangled, "__vtblZ", Len + 1) == 0) {
+      // The vtable symbol for a given class.
+      Demangled->prepend("vtable for ");
+      Demangled->setCurrentPosition(Demangled->getCurrentPosition() - 1);
+      Mangled += Len;
+      return Mangled;
+    }
+    break;
+
+  case 7:
+    if (strncmp(Mangled, "__ClassZ", Len + 1) == 0) {
+      // The classinfo symbol for a given class.
+      Demangled->prepend("ClassInfo for ");
+      Demangled->setCurrentPosition(Demangled->getCurrentPosition() - 1);
+      Mangled += Len;
+      return Mangled;
+    }
+    break;
+
+  case 11:
+    if (strncmp(Mangled, "__InterfaceZ", Len + 1) == 0) {
+      // The interface symbol for a given class.
+      Demangled->prepend("Interface for ");
+      Demangled->setCurrentPosition(Demangled->getCurrentPosition() - 1);
+      Mangled += Len;
+      return Mangled;
+    }
+    break;
+
+  case 12:
+    if (strncmp(Mangled, "__ModuleInfoZ", Len + 1) == 0) {
+      // The ModuleInfo symbol for a given module.
+      Demangled->prepend("ModuleInfo for ");
+      Demangled->setCurrentPosition(Demangled->getCurrentPosition() - 1);
+      Mangled += Len;
+      return Mangled;
+    }
+    break;
+  }
+
   *Demangled << StringView(Mangled, Len);
   Mangled += Len;
 
diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp
index 3ea9ffee6554..27d8833ae19e 100644
--- a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp
@@ -241,7 +241,9 @@ private:
     }
     case Branch32: {
       Kind = x86_64::BranchPCRel32;
-      Addend = 0;
+      // BranchPCRel32 implicitly handles the '-4' PC adjustment, so we have to
+      // adjust the addend by '+4' to compensate.
+      Addend += 4;
       break;
     }
     }
@@ -252,7 +254,7 @@ private:
     Edge GE(Kind, Offset, *GraphSymbol, Addend);
     LLVM_DEBUG({
       dbgs() << "    ";
-      printEdge(dbgs(), *BlockToFix, GE, getELFX86RelocationKindName(Kind));
+      printEdge(dbgs(), *BlockToFix, GE, x86_64::getEdgeKindName(Kind));
       dbgs() << "\n";
     });
 
diff --git a/llvm/lib/ExecutionEngine/MCJIT/MCJIT.cpp b/llvm/lib/ExecutionEngine/MCJIT/MCJIT.cpp
index 200f42aec067..ed912280ac82 100644
--- a/llvm/lib/ExecutionEngine/MCJIT/MCJIT.cpp
+++ b/llvm/lib/ExecutionEngine/MCJIT/MCJIT.cpp
@@ -170,8 +170,8 @@ std::unique_ptr<MemoryBuffer> MCJIT::emitObject(Module *M) {
   PM.run(*M);
   // Flush the output buffer to get the generated code into memory
 
-  std::unique_ptr<MemoryBuffer> CompiledObjBuffer(
-      new SmallVectorMemoryBuffer(std::move(ObjBufferSV)));
+  auto CompiledObjBuffer = std::make_unique<SmallVectorMemoryBuffer>(
+      std::move(ObjBufferSV), /*RequiresNullTerminator=*/false);
 
   // If we have an object cache, tell it about the new object.
   // Note that we're using the compiled image, not the loaded image (as below).
diff --git a/llvm/lib/ExecutionEngine/MCJIT/MCJIT.h b/llvm/lib/ExecutionEngine/MCJIT/MCJIT.h
index 52e7eda90310..a5dd420c9132 100644
--- a/llvm/lib/ExecutionEngine/MCJIT/MCJIT.h
+++ b/llvm/lib/ExecutionEngine/MCJIT/MCJIT.h
@@ -151,12 +151,8 @@ class MCJIT : public ExecutionEngine {
     }
 
     void markAllLoadedModulesAsFinalized() {
-      for (ModulePtrSet::iterator I = LoadedModules.begin(),
-                                  E = LoadedModules.end();
-           I != E; ++I) {
-        Module *M = *I;
+      for (Module *M : LoadedModules)
         FinalizedModules.insert(M);
-      }
       LoadedModules.clear();
     }
 
@@ -167,10 +163,8 @@ class MCJIT : public ExecutionEngine {
 
     void freeModulePtrSet(ModulePtrSet& MPS) {
       // Go through the module set and delete everything.
-      for (ModulePtrSet::iterator I = MPS.begin(), E = MPS.end(); I != E; ++I) {
-        Module *M = *I;
+      for (Module *M : MPS)
         delete M;
-      }
       MPS.clear();
     }
   };
diff --git a/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp b/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp
index 9ff6cec8c6c5..e2a0cadb6348 100644
--- a/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp
@@ -78,11 +78,10 @@ public:
       : IRMaterializationUnit(ES, MO, std::move(TSM)), Parent(Parent) {}
 
   PartitioningIRMaterializationUnit(
-      ThreadSafeModule TSM, SymbolFlagsMap SymbolFlags,
-      SymbolStringPtr InitSymbol, SymbolNameToDefinitionMap SymbolToDefinition,
+      ThreadSafeModule TSM, Interface I,
+      SymbolNameToDefinitionMap SymbolToDefinition,
       CompileOnDemandLayer &Parent)
-      : IRMaterializationUnit(std::move(TSM), std::move(SymbolFlags),
-                              std::move(InitSymbol),
+      : IRMaterializationUnit(std::move(TSM), std::move(I),
                               std::move(SymbolToDefinition)),
         Parent(Parent) {}
 
@@ -298,7 +297,9 @@ void CompileOnDemandLayer::emitPartition(
   if (GVsToExtract->empty()) {
     if (auto Err =
             R->replace(std::make_unique<PartitioningIRMaterializationUnit>(
-                std::move(TSM), R->getSymbols(), R->getInitializerSymbol(),
+                std::move(TSM),
+                MaterializationUnit::Interface(R->getSymbols(),
+                                               R->getInitializerSymbol()),
                 std::move(Defs), *this))) {
       getExecutionSession().reportError(std::move(Err));
       R->failMaterialization();
diff --git a/llvm/lib/ExecutionEngine/Orc/CompileUtils.cpp b/llvm/lib/ExecutionEngine/Orc/CompileUtils.cpp
index f8efed15edea..f34247005258 100644
--- a/llvm/lib/ExecutionEngine/Orc/CompileUtils.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/CompileUtils.cpp
@@ -53,7 +53,8 @@ Expected<SimpleCompiler::CompileResult> SimpleCompiler::operator()(Module &M) {
   }
 
   auto ObjBuffer = std::make_unique<SmallVectorMemoryBuffer>(
-      std::move(ObjBufferSV), M.getModuleIdentifier() + "-jitted-objectbuffer");
+      std::move(ObjBufferSV), M.getModuleIdentifier() + "-jitted-objectbuffer",
+      /*RequiresNullTerminator=*/false);
 
   auto Obj = object::ObjectFile::createObjectFile(ObjBuffer->getMemBufferRef());
 
diff --git a/llvm/lib/ExecutionEngine/Orc/Core.cpp b/llvm/lib/ExecutionEngine/Orc/Core.cpp
index 56a97f83d915..aa82cf38c45d 100644
--- a/llvm/lib/ExecutionEngine/Orc/Core.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/Core.cpp
@@ -243,8 +243,7 @@ void AsynchronousSymbolQuery::detach() {
 
 AbsoluteSymbolsMaterializationUnit::AbsoluteSymbolsMaterializationUnit(
     SymbolMap Symbols)
-    : MaterializationUnit(extractFlags(Symbols), nullptr),
-      Symbols(std::move(Symbols)) {}
+    : MaterializationUnit(extractFlags(Symbols)), Symbols(std::move(Symbols)) {}
 
 StringRef AbsoluteSymbolsMaterializationUnit::getName() const {
   return "<Absolute Symbols>";
@@ -263,18 +262,18 @@ void AbsoluteSymbolsMaterializationUnit::discard(const JITDylib &JD,
   Symbols.erase(Name);
 }
 
-SymbolFlagsMap
+MaterializationUnit::Interface
 AbsoluteSymbolsMaterializationUnit::extractFlags(const SymbolMap &Symbols) {
   SymbolFlagsMap Flags;
   for (const auto &KV : Symbols)
     Flags[KV.first] = KV.second.getFlags();
-  return Flags;
+  return MaterializationUnit::Interface(std::move(Flags), nullptr);
 }
 
 ReExportsMaterializationUnit::ReExportsMaterializationUnit(
     JITDylib *SourceJD, JITDylibLookupFlags SourceJDLookupFlags,
     SymbolAliasMap Aliases)
-    : MaterializationUnit(extractFlags(Aliases), nullptr), SourceJD(SourceJD),
+    : MaterializationUnit(extractFlags(Aliases)), SourceJD(SourceJD),
       SourceJDLookupFlags(SourceJDLookupFlags), Aliases(std::move(Aliases)) {}
 
 StringRef ReExportsMaterializationUnit::getName() const {
@@ -456,13 +455,13 @@ void ReExportsMaterializationUnit::discard(const JITDylib &JD,
   Aliases.erase(Name);
 }
 
-SymbolFlagsMap
+MaterializationUnit::Interface
 ReExportsMaterializationUnit::extractFlags(const SymbolAliasMap &Aliases) {
   SymbolFlagsMap SymbolFlags;
   for (auto &KV : Aliases)
     SymbolFlags[KV.first] = KV.second.AliasFlags;
 
-  return SymbolFlags;
+  return MaterializationUnit::Interface(std::move(SymbolFlags), nullptr);
 }
 
 Expected<SymbolAliasMap> buildSimpleReexportsAliasMap(JITDylib &SourceJD,
@@ -2492,10 +2491,19 @@ void ExecutionSession::OL_applyQueryPhase1(
       }
     }
 
-    // If we get here then we've moved on to the next JITDylib.
-    LLVM_DEBUG(dbgs() << "Phase 1 moving to next JITDylib.\n");
-    ++IPLS->CurSearchOrderIndex;
-    IPLS->NewJITDylib = true;
+    if (IPLS->DefGeneratorCandidates.empty() &&
+        IPLS->DefGeneratorNonCandidates.empty()) {
+      // Early out if there are no remaining symbols.
+      LLVM_DEBUG(dbgs() << "All symbols matched.\n");
+      IPLS->CurSearchOrderIndex = IPLS->SearchOrder.size();
+      break;
+    } else {
+      // If we get here then we've moved on to the next JITDylib with candidates
+      // remaining.
+      LLVM_DEBUG(dbgs() << "Phase 1 moving to next JITDylib.\n");
+      ++IPLS->CurSearchOrderIndex;
+      IPLS->NewJITDylib = true;
+    }
   }
 
   // Remove any weakly referenced candidates that could not be found/generated.
diff --git a/llvm/lib/ExecutionEngine/Orc/DebuggerSupportPlugin.cpp b/llvm/lib/ExecutionEngine/Orc/DebuggerSupportPlugin.cpp
index 8479495623b8..fe62138c790c 100644
--- a/llvm/lib/ExecutionEngine/Orc/DebuggerSupportPlugin.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/DebuggerSupportPlugin.cpp
@@ -154,8 +154,24 @@ public:
         }
         DebugSecInfos.push_back({&Sec, Sec.getName().substr(0, SepPos),
                                  Sec.getName().substr(SepPos + 1), 0, 0});
-      } else
+      } else {
         NonDebugSections.push_back(&Sec);
+
+        // If the first block in the section has a non-zero alignment offset
+        // then we need to add a padding block, since the section command in
+        // the header doesn't allow for aligment offsets.
+        SectionRange R(Sec);
+        if (!R.empty()) {
+          auto &FB = *R.getFirstBlock();
+          if (FB.getAlignmentOffset() != 0) {
+            auto Padding = G.allocateBuffer(FB.getAlignmentOffset());
+            memset(Padding.data(), 0, Padding.size());
+            G.createContentBlock(Sec, Padding,
+                                 FB.getAddress() - FB.getAlignmentOffset(),
+                                 FB.getAlignment(), 0);
+          }
+        }
+      }
     }
 
     // Create container block.
diff --git a/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp
index b17d196f01b6..eded54f4bfb3 100644
--- a/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp
@@ -28,8 +28,8 @@ class DSOHandleMaterializationUnit : public MaterializationUnit {
 public:
   DSOHandleMaterializationUnit(ELFNixPlatform &ENP,
                                const SymbolStringPtr &DSOHandleSymbol)
-      : MaterializationUnit(createDSOHandleSectionSymbols(ENP, DSOHandleSymbol),
-                            DSOHandleSymbol),
+      : MaterializationUnit(
+            createDSOHandleSectionInterface(ENP, DSOHandleSymbol)),
         ENP(ENP) {}
 
   StringRef getName() const override { return "DSOHandleMU"; }
@@ -70,12 +70,13 @@ public:
   void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
 
 private:
-  static SymbolFlagsMap
-  createDSOHandleSectionSymbols(ELFNixPlatform &ENP,
-                                const SymbolStringPtr &DSOHandleSymbol) {
+  static MaterializationUnit::Interface
+  createDSOHandleSectionInterface(ELFNixPlatform &ENP,
+                                  const SymbolStringPtr &DSOHandleSymbol) {
     SymbolFlagsMap SymbolFlags;
     SymbolFlags[DSOHandleSymbol] = JITSymbolFlags::Exported;
-    return SymbolFlags;
+    return MaterializationUnit::Interface(std::move(SymbolFlags),
+                                          DSOHandleSymbol);
   }
 
   ArrayRef<char> getDSOHandleContent(size_t PointerSize) {
diff --git a/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp b/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
index 2ab9ed4f856b..ae2d47fb8c5e 100644
--- a/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
@@ -8,6 +8,7 @@
 
 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
 #include "llvm/ExecutionEngine/Orc/Layer.h"
+#include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/GlobalVariable.h"
@@ -269,25 +270,30 @@ Error DynamicLibrarySearchGenerator::tryToGenerate(
 }
 
 Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>>
-StaticLibraryDefinitionGenerator::Load(ObjectLayer &L, const char *FileName) {
+StaticLibraryDefinitionGenerator::Load(
+    ObjectLayer &L, const char *FileName,
+    GetObjectFileInterface GetObjFileInterface) {
   auto ArchiveBuffer = errorOrToExpected(MemoryBuffer::getFile(FileName));
 
   if (!ArchiveBuffer)
     return ArchiveBuffer.takeError();
 
-  return Create(L, std::move(*ArchiveBuffer));
+  return Create(L, std::move(*ArchiveBuffer), std::move(GetObjFileInterface));
 }
 
 Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>>
-StaticLibraryDefinitionGenerator::Load(ObjectLayer &L, const char *FileName,
-                                       const Triple &TT) {
+StaticLibraryDefinitionGenerator::Load(
+    ObjectLayer &L, const char *FileName, const Triple &TT,
+    GetObjectFileInterface GetObjFileInterface) {
+
   auto B = object::createBinary(FileName);
   if (!B)
     return B.takeError();
 
   // If this is a regular archive then create an instance from it.
   if (isa<object::Archive>(B->getBinary()))
-    return Create(L, std::move(B->takeBinary().second));
+    return Create(L, std::move(B->takeBinary().second),
+                  std::move(GetObjFileInterface));
 
   // If this is a universal binary then search for a slice matching the given
   // Triple.
@@ -309,7 +315,8 @@ StaticLibraryDefinitionGenerator::Load(ObjectLayer &L, const char *FileName,
                   " .. " + formatv("{0:x}", Obj.getOffset() + Obj.getSize()) +
                   ": " + SliceBuffer.getError().message(),
               SliceBuffer.getError());
-        return Create(L, std::move(*SliceBuffer));
+        return Create(L, std::move(*SliceBuffer),
+                      std::move(GetObjFileInterface));
       }
     }
 
@@ -326,11 +333,13 @@ StaticLibraryDefinitionGenerator::Load(ObjectLayer &L, const char *FileName,
 
 Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>>
 StaticLibraryDefinitionGenerator::Create(
-    ObjectLayer &L, std::unique_ptr<MemoryBuffer> ArchiveBuffer) {
+    ObjectLayer &L, std::unique_ptr<MemoryBuffer> ArchiveBuffer,
+    GetObjectFileInterface GetObjFileInterface) {
   Error Err = Error::success();
 
   std::unique_ptr<StaticLibraryDefinitionGenerator> ADG(
-      new StaticLibraryDefinitionGenerator(L, std::move(ArchiveBuffer), Err));
+      new StaticLibraryDefinitionGenerator(
+          L, std::move(ArchiveBuffer), std::move(GetObjFileInterface), Err));
 
   if (Err)
     return std::move(Err);
@@ -371,7 +380,12 @@ Error StaticLibraryDefinitionGenerator::tryToGenerate(
     MemoryBufferRef ChildBufferRef(ChildBufferInfo.first,
                                    ChildBufferInfo.second);
 
-    if (auto Err = L.add(JD, MemoryBuffer::getMemBuffer(ChildBufferRef, false)))
+    auto I = GetObjFileInterface(L.getExecutionSession(), ChildBufferRef);
+    if (!I)
+      return I.takeError();
+
+    if (auto Err = L.add(JD, MemoryBuffer::getMemBuffer(ChildBufferRef, false),
+                         std::move(*I)))
       return Err;
   }
 
@@ -379,9 +393,15 @@ Error StaticLibraryDefinitionGenerator::tryToGenerate(
 }
 
 StaticLibraryDefinitionGenerator::StaticLibraryDefinitionGenerator(
-    ObjectLayer &L, std::unique_ptr<MemoryBuffer> ArchiveBuffer, Error &Err)
-    : L(L), ArchiveBuffer(std::move(ArchiveBuffer)),
-      Archive(std::make_unique<object::Archive>(*this->ArchiveBuffer, Err)) {}
+    ObjectLayer &L, std::unique_ptr<MemoryBuffer> ArchiveBuffer,
+    GetObjectFileInterface GetObjFileInterface, Error &Err)
+    : L(L), GetObjFileInterface(std::move(GetObjFileInterface)),
+      ArchiveBuffer(std::move(ArchiveBuffer)),
+      Archive(std::make_unique<object::Archive>(*this->ArchiveBuffer, Err)) {
+
+  if (!this->GetObjFileInterface)
+    this->GetObjFileInterface = getObjectFileInterface;
+}
 
 } // End namespace orc.
 } // End namespace llvm.
diff --git a/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp b/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp
index ee1630a2ffa8..f427271bb45d 100644
--- a/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp
@@ -31,8 +31,8 @@ public:
 
   CompileCallbackMaterializationUnit(SymbolStringPtr Name,
                                      CompileFunction Compile)
-      : MaterializationUnit(SymbolFlagsMap({{Name, JITSymbolFlags::Exported}}),
-                            nullptr),
+      : MaterializationUnit(Interface(
+            SymbolFlagsMap({{Name, JITSymbolFlags::Exported}}), nullptr)),
         Name(std::move(Name)), Compile(std::move(Compile)) {}
 
   StringRef getName() const override { return "<Compile Callbacks>"; }
diff --git a/llvm/lib/ExecutionEngine/Orc/Layer.cpp b/llvm/lib/ExecutionEngine/Orc/Layer.cpp
index 20dfba23bf10..adb8861793b1 100644
--- a/llvm/lib/ExecutionEngine/Orc/Layer.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/Layer.cpp
@@ -10,9 +10,8 @@
 
 #include "llvm/ExecutionEngine/Orc/DebugUtils.h"
 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
+#include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h"
 #include "llvm/IR/Constants.h"
-#include "llvm/Object/MachO.h"
-#include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/Debug.h"
 
 #define DEBUG_TYPE "orc"
@@ -33,7 +32,7 @@ Error IRLayer::add(ResourceTrackerSP RT, ThreadSafeModule TSM) {
 IRMaterializationUnit::IRMaterializationUnit(
     ExecutionSession &ES, const IRSymbolMapper::ManglingOptions &MO,
     ThreadSafeModule TSM)
-    : MaterializationUnit(SymbolFlagsMap(), nullptr), TSM(std::move(TSM)) {
+    : MaterializationUnit(Interface()), TSM(std::move(TSM)) {
 
   assert(this->TSM && "Module must not be null");
 
@@ -98,10 +97,10 @@ IRMaterializationUnit::IRMaterializationUnit(
 }
 
 IRMaterializationUnit::IRMaterializationUnit(
-    ThreadSafeModule TSM, SymbolFlagsMap SymbolFlags,
-    SymbolStringPtr InitSymbol, SymbolNameToDefinitionMap SymbolToDefinition)
-    : MaterializationUnit(std::move(SymbolFlags), std::move(InitSymbol)),
-      TSM(std::move(TSM)), SymbolToDefinition(std::move(SymbolToDefinition)) {}
+    ThreadSafeModule TSM, Interface I,
+    SymbolNameToDefinitionMap SymbolToDefinition)
+    : MaterializationUnit(std::move(I)), TSM(std::move(TSM)),
+      SymbolToDefinition(std::move(SymbolToDefinition)) {}
 
 StringRef IRMaterializationUnit::getName() const {
   if (TSM)
@@ -161,37 +160,47 @@ ObjectLayer::ObjectLayer(ExecutionSession &ES) : ES(ES) {}
 
 ObjectLayer::~ObjectLayer() {}
 
-Error ObjectLayer::add(ResourceTrackerSP RT, std::unique_ptr<MemoryBuffer> O) {
+Error ObjectLayer::add(ResourceTrackerSP RT, std::unique_ptr<MemoryBuffer> O,
+                       MaterializationUnit::Interface I) {
   assert(RT && "RT can not be null");
-  auto ObjMU = BasicObjectLayerMaterializationUnit::Create(*this, std::move(O));
-  if (!ObjMU)
-    return ObjMU.takeError();
   auto &JD = RT->getJITDylib();
-  return JD.define(std::move(*ObjMU), std::move(RT));
+  return JD.define(std::make_unique<BasicObjectLayerMaterializationUnit>(
+                       *this, std::move(O), std::move(I)),
+                   std::move(RT));
+}
+
+Error ObjectLayer::add(ResourceTrackerSP RT, std::unique_ptr<MemoryBuffer> O) {
+  auto I = getObjectFileInterface(getExecutionSession(), O->getMemBufferRef());
+  if (!I)
+    return I.takeError();
+  return add(std::move(RT), std::move(O), std::move(*I));
+}
+
+Error ObjectLayer::add(JITDylib &JD, std::unique_ptr<MemoryBuffer> O) {
+  auto I = getObjectFileInterface(getExecutionSession(), O->getMemBufferRef());
+  if (!I)
+    return I.takeError();
+  return add(JD, std::move(O), std::move(*I));
 }
 
 Expected<std::unique_ptr<BasicObjectLayerMaterializationUnit>>
 BasicObjectLayerMaterializationUnit::Create(ObjectLayer &L,
                                             std::unique_ptr<MemoryBuffer> O) {
-  auto ObjSymInfo =
-      getObjectSymbolInfo(L.getExecutionSession(), O->getMemBufferRef());
 
-  if (!ObjSymInfo)
-    return ObjSymInfo.takeError();
+  auto ObjInterface =
+      getObjectFileInterface(L.getExecutionSession(), O->getMemBufferRef());
 
-  auto &SymbolFlags = ObjSymInfo->first;
-  auto &InitSymbol = ObjSymInfo->second;
+  if (!ObjInterface)
+    return ObjInterface.takeError();
 
   return std::unique_ptr<BasicObjectLayerMaterializationUnit>(
-      new BasicObjectLayerMaterializationUnit(
-          L, std::move(O), std::move(SymbolFlags), std::move(InitSymbol)));
+      new BasicObjectLayerMaterializationUnit(L, std::move(O),
+                                              std::move(*ObjInterface)));
 }
 
 BasicObjectLayerMaterializationUnit::BasicObjectLayerMaterializationUnit(
-    ObjectLayer &L, std::unique_ptr<MemoryBuffer> O, SymbolFlagsMap SymbolFlags,
-    SymbolStringPtr InitSymbol)
-    : MaterializationUnit(std::move(SymbolFlags), std::move(InitSymbol)), L(L),
-      O(std::move(O)) {}
+    ObjectLayer &L, std::unique_ptr<MemoryBuffer> O, Interface I)
+    : MaterializationUnit(std::move(I)), L(L), O(std::move(O)) {}
 
 StringRef BasicObjectLayerMaterializationUnit::getName() const {
   if (O)
diff --git a/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp b/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp
index e1f494415e86..66453e6a632f 100644
--- a/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp
@@ -144,7 +144,7 @@ createLocalLazyCallThroughManager(const Triple &T, ExecutionSession &ES,
 LazyReexportsMaterializationUnit::LazyReexportsMaterializationUnit(
     LazyCallThroughManager &LCTManager, IndirectStubsManager &ISManager,
     JITDylib &SourceJD, SymbolAliasMap CallableAliases, ImplSymbolMap *SrcJDLoc)
-    : MaterializationUnit(extractFlags(CallableAliases), nullptr),
+    : MaterializationUnit(extractFlags(CallableAliases)),
       LCTManager(LCTManager), ISManager(ISManager), SourceJD(SourceJD),
       CallableAliases(std::move(CallableAliases)), AliaseeTable(SrcJDLoc) {}
 
@@ -219,7 +219,7 @@ void LazyReexportsMaterializationUnit::discard(const JITDylib &JD,
   CallableAliases.erase(Name);
 }
 
-SymbolFlagsMap
+MaterializationUnit::Interface
 LazyReexportsMaterializationUnit::extractFlags(const SymbolAliasMap &Aliases) {
   SymbolFlagsMap SymbolFlags;
   for (auto &KV : Aliases) {
@@ -227,7 +227,7 @@ LazyReexportsMaterializationUnit::extractFlags(const SymbolAliasMap &Aliases) {
            "Lazy re-exports must be callable symbols");
     SymbolFlags[KV.first] = KV.second.AliasFlags;
   }
-  return SymbolFlags;
+  return MaterializationUnit::Interface(std::move(SymbolFlags), nullptr);
 }
 
 } // End namespace orc.
diff --git a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
index 46c915dfea9e..fb2e90e1c9c5 100644
--- a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
@@ -28,8 +28,7 @@ class MachOHeaderMaterializationUnit : public MaterializationUnit {
 public:
   MachOHeaderMaterializationUnit(MachOPlatform &MOP,
                                  const SymbolStringPtr &HeaderStartSymbol)
-      : MaterializationUnit(createHeaderSymbols(MOP, HeaderStartSymbol),
-                            HeaderStartSymbol),
+      : MaterializationUnit(createHeaderInterface(MOP, HeaderStartSymbol)),
         MOP(MOP) {}
 
   StringRef getName() const override { return "MachOHeaderMU"; }
@@ -110,9 +109,9 @@ private:
     return G.createContentBlock(HeaderSection, HeaderContent, 0, 8, 0);
   }
 
-  static SymbolFlagsMap
-  createHeaderSymbols(MachOPlatform &MOP,
-                      const SymbolStringPtr &HeaderStartSymbol) {
+  static MaterializationUnit::Interface
+  createHeaderInterface(MachOPlatform &MOP,
+                        const SymbolStringPtr &HeaderStartSymbol) {
     SymbolFlagsMap HeaderSymbolFlags;
 
     HeaderSymbolFlags[HeaderStartSymbol] = JITSymbolFlags::Exported;
@@ -120,7 +119,8 @@ private:
       HeaderSymbolFlags[MOP.getExecutionSession().intern(HS.Name)] =
           JITSymbolFlags::Exported;
 
-    return HeaderSymbolFlags;
+    return MaterializationUnit::Interface(std::move(HeaderSymbolFlags),
+                                          HeaderStartSymbol);
   }
 
   MachOPlatform &MOP;
diff --git a/llvm/lib/ExecutionEngine/Orc/Mangling.cpp b/llvm/lib/ExecutionEngine/Orc/Mangling.cpp
index 7b21e6a684ca..9c243c9bf1d2 100644
--- a/llvm/lib/ExecutionEngine/Orc/Mangling.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/Mangling.cpp
@@ -7,13 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ExecutionEngine/Orc/Mangling.h"
-#include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h"
-#include "llvm/ExecutionEngine/Orc/MachOPlatform.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/Mangler.h"
-#include "llvm/Object/ELFObjectFile.h"
-#include "llvm/Object/MachO.h"
-#include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/Debug.h"
 
 #define DEBUG_TYPE "orc"
@@ -85,188 +80,5 @@ void IRSymbolMapper::add(ExecutionSession &ES, const ManglingOptions &MO,
   }
 }
 
-static SymbolStringPtr addInitSymbol(SymbolFlagsMap &SymbolFlags,
-                                     ExecutionSession &ES,
-                                     StringRef ObjFileName) {
-  SymbolStringPtr InitSymbol;
-  size_t Counter = 0;
-
-  do {
-    std::string InitSymString;
-    raw_string_ostream(InitSymString)
-      << "$." << ObjFileName << ".__inits." << Counter++;
-    InitSymbol = ES.intern(InitSymString);
-  } while (SymbolFlags.count(InitSymbol));
-
-  SymbolFlags[InitSymbol] = JITSymbolFlags::MaterializationSideEffectsOnly;
-  return InitSymbol;
-}
-
-static Expected<std::pair<SymbolFlagsMap, SymbolStringPtr>>
-getMachOObjectFileSymbolInfo(ExecutionSession &ES,
-                             const object::MachOObjectFile &Obj) {
-  SymbolFlagsMap SymbolFlags;
-
-  for (auto &Sym : Obj.symbols()) {
-    Expected<uint32_t> SymFlagsOrErr = Sym.getFlags();
-    if (!SymFlagsOrErr)
-      // TODO: Test this error.
-      return SymFlagsOrErr.takeError();
-
-    // Skip symbols not defined in this object file.
-    if (*SymFlagsOrErr & object::BasicSymbolRef::SF_Undefined)
-      continue;
-
-    // Skip symbols that are not global.
-    if (!(*SymFlagsOrErr & object::BasicSymbolRef::SF_Global))
-      continue;
-
-    // Skip symbols that have type SF_File.
-    if (auto SymType = Sym.getType()) {
-      if (*SymType == object::SymbolRef::ST_File)
-        continue;
-    } else
-      return SymType.takeError();
-
-    auto Name = Sym.getName();
-    if (!Name)
-      return Name.takeError();
-    auto InternedName = ES.intern(*Name);
-    auto SymFlags = JITSymbolFlags::fromObjectSymbol(Sym);
-    if (!SymFlags)
-      return SymFlags.takeError();
-
-    // Strip the 'exported' flag from MachO linker-private symbols.
-    if (Name->startswith("l"))
-      *SymFlags &= ~JITSymbolFlags::Exported;
-
-    SymbolFlags[InternedName] = std::move(*SymFlags);
-  }
-
-  SymbolStringPtr InitSymbol;
-  for (auto &Sec : Obj.sections()) {
-    auto SecType = Obj.getSectionType(Sec);
-    if ((SecType & MachO::SECTION_TYPE) == MachO::S_MOD_INIT_FUNC_POINTERS) {
-      InitSymbol = addInitSymbol(SymbolFlags, ES, Obj.getFileName());
-      break;
-    }
-    auto SegName = Obj.getSectionFinalSegmentName(Sec.getRawDataRefImpl());
-    auto SecName = cantFail(Obj.getSectionName(Sec.getRawDataRefImpl()));
-    if (MachOPlatform::isInitializerSection(SegName, SecName)) {
-      InitSymbol = addInitSymbol(SymbolFlags, ES, Obj.getFileName());
-      break;
-    }
-  }
-
-  return std::make_pair(std::move(SymbolFlags), std::move(InitSymbol));
-}
-
-static Expected<std::pair<SymbolFlagsMap, SymbolStringPtr>>
-getELFObjectFileSymbolInfo(ExecutionSession &ES,
-                           const object::ELFObjectFileBase &Obj) {
-  SymbolFlagsMap SymbolFlags;
-  for (auto &Sym : Obj.symbols()) {
-    Expected<uint32_t> SymFlagsOrErr = Sym.getFlags();
-    if (!SymFlagsOrErr)
-      // TODO: Test this error.
-      return SymFlagsOrErr.takeError();
-
-    // Skip symbols not defined in this object file.
-    if (*SymFlagsOrErr & object::BasicSymbolRef::SF_Undefined)
-      continue;
-
-    // Skip symbols that are not global.
-    if (!(*SymFlagsOrErr & object::BasicSymbolRef::SF_Global))
-      continue;
-
-    // Skip symbols that have type SF_File.
-    if (auto SymType = Sym.getType()) {
-      if (*SymType == object::SymbolRef::ST_File)
-        continue;
-    } else
-      return SymType.takeError();
-
-    auto Name = Sym.getName();
-    if (!Name)
-      return Name.takeError();
-    auto InternedName = ES.intern(*Name);
-    auto SymFlags = JITSymbolFlags::fromObjectSymbol(Sym);
-    if (!SymFlags)
-      return SymFlags.takeError();
-
-    // ELF STB_GNU_UNIQUE should map to Weak for ORC.
-    if (Sym.getBinding() == ELF::STB_GNU_UNIQUE)
-      *SymFlags |= JITSymbolFlags::Weak;
-
-    SymbolFlags[InternedName] = std::move(*SymFlags);
-  }
-
-  SymbolStringPtr InitSymbol;
-  for (auto &Sec : Obj.sections()) {
-    if (auto SecName = Sec.getName()) {
-      if (ELFNixPlatform::isInitializerSection(*SecName)) {
-        InitSymbol = addInitSymbol(SymbolFlags, ES, Obj.getFileName());
-        break;
-      }
-    }
-  }
-
-  return std::make_pair(std::move(SymbolFlags), InitSymbol);
-}
-
-Expected<std::pair<SymbolFlagsMap, SymbolStringPtr>>
-getGenericObjectFileSymbolInfo(ExecutionSession &ES,
-                               const object::ObjectFile &Obj) {
-  SymbolFlagsMap SymbolFlags;
-  for (auto &Sym : Obj.symbols()) {
-    Expected<uint32_t> SymFlagsOrErr = Sym.getFlags();
-    if (!SymFlagsOrErr)
-      // TODO: Test this error.
-      return SymFlagsOrErr.takeError();
-
-    // Skip symbols not defined in this object file.
-    if (*SymFlagsOrErr & object::BasicSymbolRef::SF_Undefined)
-      continue;
-
-    // Skip symbols that are not global.
-    if (!(*SymFlagsOrErr & object::BasicSymbolRef::SF_Global))
-      continue;
-
-    // Skip symbols that have type SF_File.
-    if (auto SymType = Sym.getType()) {
-      if (*SymType == object::SymbolRef::ST_File)
-        continue;
-    } else
-      return SymType.takeError();
-
-    auto Name = Sym.getName();
-    if (!Name)
-      return Name.takeError();
-    auto InternedName = ES.intern(*Name);
-    auto SymFlags = JITSymbolFlags::fromObjectSymbol(Sym);
-    if (!SymFlags)
-      return SymFlags.takeError();
-
-    SymbolFlags[InternedName] = std::move(*SymFlags);
-  }
-
-  return std::make_pair(std::move(SymbolFlags), nullptr);
-}
-
-Expected<std::pair<SymbolFlagsMap, SymbolStringPtr>>
-getObjectSymbolInfo(ExecutionSession &ES, MemoryBufferRef ObjBuffer) {
-  auto Obj = object::ObjectFile::createObjectFile(ObjBuffer);
-
-  if (!Obj)
-    return Obj.takeError();
-
-  if (auto *MachOObj = dyn_cast<object::MachOObjectFile>(Obj->get()))
-    return getMachOObjectFileSymbolInfo(ES, *MachOObj);
-  else if (auto *ELFObj = dyn_cast<object::ELFObjectFileBase>(Obj->get()))
-    return getELFObjectFileSymbolInfo(ES, *ELFObj);
-
-  return getGenericObjectFileSymbolInfo(ES, **Obj);
-}
-
 } // End namespace orc.
 } // End namespace llvm.
diff --git a/llvm/lib/ExecutionEngine/Orc/ObjectFileInterface.cpp b/llvm/lib/ExecutionEngine/Orc/ObjectFileInterface.cpp
new file mode 100644
index 000000000000..c1ad569dd65d
--- /dev/null
+++ b/llvm/lib/ExecutionEngine/Orc/ObjectFileInterface.cpp
@@ -0,0 +1,205 @@
+//===------ ObjectFileInterface.cpp - MU interface utils for objects ------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h"
+#include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h"
+#include "llvm/ExecutionEngine/Orc/MachOPlatform.h"
+#include "llvm/Object/ELFObjectFile.h"
+#include "llvm/Object/MachO.h"
+#include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/Debug.h"
+
+#define DEBUG_TYPE "orc"
+
+namespace llvm {
+namespace orc {
+
+void addInitSymbol(MaterializationUnit::Interface &I, ExecutionSession &ES,
+                   StringRef ObjFileName) {
+  assert(!I.InitSymbol && "I already has an init symbol");
+  size_t Counter = 0;
+
+  do {
+    std::string InitSymString;
+    raw_string_ostream(InitSymString)
+        << "$." << ObjFileName << ".__inits." << Counter++;
+    I.InitSymbol = ES.intern(InitSymString);
+  } while (I.SymbolFlags.count(I.InitSymbol));
+
+  I.SymbolFlags[I.InitSymbol] = JITSymbolFlags::MaterializationSideEffectsOnly;
+}
+
+static Expected<MaterializationUnit::Interface>
+getMachOObjectFileSymbolInfo(ExecutionSession &ES,
+                             const object::MachOObjectFile &Obj) {
+  MaterializationUnit::Interface I;
+
+  for (auto &Sym : Obj.symbols()) {
+    Expected<uint32_t> SymFlagsOrErr = Sym.getFlags();
+    if (!SymFlagsOrErr)
+      // TODO: Test this error.
+      return SymFlagsOrErr.takeError();
+
+    // Skip symbols not defined in this object file.
+    if (*SymFlagsOrErr & object::BasicSymbolRef::SF_Undefined)
+      continue;
+
+    // Skip symbols that are not global.
+    if (!(*SymFlagsOrErr & object::BasicSymbolRef::SF_Global))
+      continue;
+
+    // Skip symbols that have type SF_File.
+    if (auto SymType = Sym.getType()) {
+      if (*SymType == object::SymbolRef::ST_File)
+        continue;
+    } else
+      return SymType.takeError();
+
+    auto Name = Sym.getName();
+    if (!Name)
+      return Name.takeError();
+    auto InternedName = ES.intern(*Name);
+    auto SymFlags = JITSymbolFlags::fromObjectSymbol(Sym);
+    if (!SymFlags)
+      return SymFlags.takeError();
+
+    // Strip the 'exported' flag from MachO linker-private symbols.
+    if (Name->startswith("l"))
+      *SymFlags &= ~JITSymbolFlags::Exported;
+
+    I.SymbolFlags[InternedName] = std::move(*SymFlags);
+  }
+
+  for (auto &Sec : Obj.sections()) {
+    auto SecType = Obj.getSectionType(Sec);
+    if ((SecType & MachO::SECTION_TYPE) == MachO::S_MOD_INIT_FUNC_POINTERS) {
+      addInitSymbol(I, ES, Obj.getFileName());
+      break;
+    }
+    auto SegName = Obj.getSectionFinalSegmentName(Sec.getRawDataRefImpl());
+    auto SecName = cantFail(Obj.getSectionName(Sec.getRawDataRefImpl()));
+    if (MachOPlatform::isInitializerSection(SegName, SecName)) {
+      addInitSymbol(I, ES, Obj.getFileName());
+      break;
+    }
+  }
+
+  return I;
+}
+
+static Expected<MaterializationUnit::Interface>
+getELFObjectFileSymbolInfo(ExecutionSession &ES,
+                           const object::ELFObjectFileBase &Obj) {
+  MaterializationUnit::Interface I;
+
+  for (auto &Sym : Obj.symbols()) {
+    Expected<uint32_t> SymFlagsOrErr = Sym.getFlags();
+    if (!SymFlagsOrErr)
+      // TODO: Test this error.
+      return SymFlagsOrErr.takeError();
+
+    // Skip symbols not defined in this object file.
+    if (*SymFlagsOrErr & object::BasicSymbolRef::SF_Undefined)
+      continue;
+
+    // Skip symbols that are not global.
+    if (!(*SymFlagsOrErr & object::BasicSymbolRef::SF_Global))
+      continue;
+
+    // Skip symbols that have type SF_File.
+    if (auto SymType = Sym.getType()) {
+      if (*SymType == object::SymbolRef::ST_File)
+        continue;
+    } else
+      return SymType.takeError();
+
+    auto Name = Sym.getName();
+    if (!Name)
+      return Name.takeError();
+    auto InternedName = ES.intern(*Name);
+    auto SymFlags = JITSymbolFlags::fromObjectSymbol(Sym);
+    if (!SymFlags)
+      return SymFlags.takeError();
+
+    // ELF STB_GNU_UNIQUE should map to Weak for ORC.
+    if (Sym.getBinding() == ELF::STB_GNU_UNIQUE)
+      *SymFlags |= JITSymbolFlags::Weak;
+
+    I.SymbolFlags[InternedName] = std::move(*SymFlags);
+  }
+
+  SymbolStringPtr InitSymbol;
+  for (auto &Sec : Obj.sections()) {
+    if (auto SecName = Sec.getName()) {
+      if (ELFNixPlatform::isInitializerSection(*SecName)) {
+        addInitSymbol(I, ES, Obj.getFileName());
+        break;
+      }
+    }
+  }
+
+  return I;
+}
+
+Expected<MaterializationUnit::Interface>
+getGenericObjectFileSymbolInfo(ExecutionSession &ES,
+                               const object::ObjectFile &Obj) {
+  MaterializationUnit::Interface I;
+
+  for (auto &Sym : Obj.symbols()) {
+    Expected<uint32_t> SymFlagsOrErr = Sym.getFlags();
+    if (!SymFlagsOrErr)
+      // TODO: Test this error.
+      return SymFlagsOrErr.takeError();
+
+    // Skip symbols not defined in this object file.
+    if (*SymFlagsOrErr & object::BasicSymbolRef::SF_Undefined)
+      continue;
+
+    // Skip symbols that are not global.
+    if (!(*SymFlagsOrErr & object::BasicSymbolRef::SF_Global))
+      continue;
+
+    // Skip symbols that have type SF_File.
+    if (auto SymType = Sym.getType()) {
+      if (*SymType == object::SymbolRef::ST_File)
+        continue;
+    } else
+      return SymType.takeError();
+
+    auto Name = Sym.getName();
+    if (!Name)
+      return Name.takeError();
+    auto InternedName = ES.intern(*Name);
+    auto SymFlags = JITSymbolFlags::fromObjectSymbol(Sym);
+    if (!SymFlags)
+      return SymFlags.takeError();
+
+    I.SymbolFlags[InternedName] = std::move(*SymFlags);
+  }
+
+  return I;
+}
+
+Expected<MaterializationUnit::Interface>
+getObjectFileInterface(ExecutionSession &ES, MemoryBufferRef ObjBuffer) {
+  auto Obj = object::ObjectFile::createObjectFile(ObjBuffer);
+
+  if (!Obj)
+    return Obj.takeError();
+
+  if (auto *MachOObj = dyn_cast<object::MachOObjectFile>(Obj->get()))
+    return getMachOObjectFileSymbolInfo(ES, *MachOObj);
+  else if (auto *ELFObj = dyn_cast<object::ELFObjectFileBase>(Obj->get()))
+    return getELFObjectFileSymbolInfo(ES, *ELFObj);
+
+  return getGenericObjectFileSymbolInfo(ES, **Obj);
+}
+
+} // End namespace orc.
+} // End namespace llvm.
diff --git a/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp b/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp
index 6f840a079dd1..0d6a33c5685e 100644
--- a/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp
@@ -23,12 +23,6 @@ using namespace llvm::orc;
 namespace {
 
 class LinkGraphMaterializationUnit : public MaterializationUnit {
-private:
-  struct LinkGraphInterface {
-    SymbolFlagsMap SymbolFlags;
-    SymbolStringPtr InitSymbol;
-  };
-
 public:
   static std::unique_ptr<LinkGraphMaterializationUnit>
   Create(ObjectLinkingLayer &ObjLinkingLayer, std::unique_ptr<LinkGraph> G) {
@@ -44,9 +38,9 @@ public:
   }
 
 private:
-  static LinkGraphInterface scanLinkGraph(ExecutionSession &ES, LinkGraph &G) {
+  static Interface scanLinkGraph(ExecutionSession &ES, LinkGraph &G) {
 
-    LinkGraphInterface LGI;
+    Interface LGI;
 
     for (auto *Sym : G.defined_symbols()) {
       // Skip local symbols.
@@ -98,11 +92,9 @@ private:
   }
 
   LinkGraphMaterializationUnit(ObjectLinkingLayer &ObjLinkingLayer,
-                               std::unique_ptr<LinkGraph> G,
-                               LinkGraphInterface LGI)
-      : MaterializationUnit(std::move(LGI.SymbolFlags),
-                            std::move(LGI.InitSymbol)),
-        ObjLinkingLayer(ObjLinkingLayer), G(std::move(G)) {}
+                               std::unique_ptr<LinkGraph> G, Interface LGI)
+      : MaterializationUnit(std::move(LGI)), ObjLinkingLayer(ObjLinkingLayer),
+        G(std::move(G)) {}
 
   void discard(const JITDylib &JD, const SymbolStringPtr &Name) override {
     for (auto *Sym : G->defined_symbols())
@@ -257,7 +249,8 @@ public:
 
     {
 
-      // Check that InternedResult matches up with MR->getSymbols().
+      // Check that InternedResult matches up with MR->getSymbols(), overriding
+      // flags if requested.
       // This guards against faulty transformations / compilers / object caches.
 
       // First check that there aren't any missing symbols.
@@ -266,16 +259,20 @@ public:
       SymbolNameVector MissingSymbols;
       for (auto &KV : MR->getSymbols()) {
 
+        auto I = InternedResult.find(KV.first);
+
         // If this is a materialization-side-effects only symbol then bump
         // the counter and make sure it's *not* defined, otherwise make
         // sure that it is defined.
         if (KV.second.hasMaterializationSideEffectsOnly()) {
           ++NumMaterializationSideEffectsOnlySymbols;
-          if (InternedResult.count(KV.first))
+          if (I != InternedResult.end())
             ExtraSymbols.push_back(KV.first);
           continue;
-        } else if (!InternedResult.count(KV.first))
+        } else if (I == InternedResult.end())
           MissingSymbols.push_back(KV.first);
+        else if (Layer.OverrideObjectFlags)
+          I->second.setFlags(KV.second);
       }
 
       // If there were missing symbols then report the error.
diff --git a/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp b/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp
index 673f7394450f..77a8f5af8ba0 100644
--- a/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp
@@ -192,8 +192,8 @@ public:
       LLVMOrcMaterializationUnitMaterializeFunction Materialize,
       LLVMOrcMaterializationUnitDiscardFunction Discard,
       LLVMOrcMaterializationUnitDestroyFunction Destroy)
-      : llvm::orc::MaterializationUnit(std::move(InitialSymbolFlags),
-                                       std::move(InitSymbol)),
+      : llvm::orc::MaterializationUnit(
+            Interface(std::move(InitialSymbolFlags), std::move(InitSymbol))),
         Name(std::move(Name)), Ctx(Ctx), Materialize(Materialize),
         Discard(Discard), Destroy(Destroy) {}
 
diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp
index f16c6bdbfa4f..3f38d26869d4 100644
--- a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp
+++ b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp
@@ -124,8 +124,10 @@ void RuntimeDyldImpl::resolveRelocations() {
   std::lock_guard<sys::Mutex> locked(lock);
 
   // Print out the sections prior to relocation.
-  LLVM_DEBUG(for (int i = 0, e = Sections.size(); i != e; ++i)
-                 dumpSectionMemory(Sections[i], "before relocations"););
+  LLVM_DEBUG({
+    for (SectionEntry &S : Sections)
+      dumpSectionMemory(S, "before relocations");
+  });
 
   // First, resolve relocations associated with external symbols.
   if (auto Err = resolveExternalSymbols()) {
@@ -136,21 +138,23 @@ void RuntimeDyldImpl::resolveRelocations() {
   resolveLocalRelocations();
 
   // Print out sections after relocation.
-  LLVM_DEBUG(for (int i = 0, e = Sections.size(); i != e; ++i)
-                 dumpSectionMemory(Sections[i], "after relocations"););
+  LLVM_DEBUG({
+    for (SectionEntry &S : Sections)
+      dumpSectionMemory(S, "after relocations");
+  });
 }
 
 void RuntimeDyldImpl::resolveLocalRelocations() {
   // Iterate over all outstanding relocations
-  for (auto it = Relocations.begin(), e = Relocations.end(); it != e; ++it) {
+  for (const auto &Rel : Relocations) {
     // The Section here (Sections[i]) refers to the section in which the
     // symbol for the relocation is located.  The SectionID in the relocation
     // entry provides the section to which the relocation will be applied.
-    unsigned Idx = it->first;
+    unsigned Idx = Rel.first;
     uint64_t Addr = getSectionLoadAddress(Idx);
     LLVM_DEBUG(dbgs() << "Resolving relocations Section #" << Idx << "\t"
                       << format("%p", (uintptr_t)Addr) << "\n");
-    resolveRelocationList(it->second, Addr);
+    resolveRelocationList(Rel.second, Addr);
   }
   Relocations.clear();
 }
@@ -457,9 +461,9 @@ static uint64_t
 computeAllocationSizeForSections(std::vector<uint64_t> &SectionSizes,
                                  uint64_t Alignment) {
   uint64_t TotalSize = 0;
-  for (size_t Idx = 0, Cnt = SectionSizes.size(); Idx < Cnt; Idx++) {
+  for (uint64_t SectionSize : SectionSizes) {
     uint64_t AlignedSize =
-        (SectionSizes[Idx] + Alignment - 1) / Alignment * Alignment;
+        (SectionSize + Alignment - 1) / Alignment * Alignment;
     TotalSize += AlignedSize;
   }
   return TotalSize;
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 18f1a2314853..5157d51fd18c 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -996,7 +996,7 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createSections(
   Builder.SetInsertPoint(AllocaIP.getBlock()->getTerminator());
   AllocaIP = Builder.saveIP();
   InsertPointTy AfterIP =
-      applyStaticWorkshareLoop(Loc.DL, LoopInfo, AllocaIP, true);
+      applyStaticWorkshareLoop(Loc.DL, LoopInfo, AllocaIP, !IsNowait);
   BasicBlock *LoopAfterBB = AfterIP.getBlock();
   Instruction *SplitPos = LoopAfterBB->getTerminator();
   if (!isa_and_nonnull<BranchInst>(SplitPos))
@@ -1156,7 +1156,7 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductions(
   Builder.SetInsertPoint(NonAtomicRedBlock);
   for (auto En : enumerate(ReductionInfos)) {
     const ReductionInfo &RI = En.value();
-    Type *ValueType = RI.getElementType();
+    Type *ValueType = RI.ElementType;
     Value *RedValue = Builder.CreateLoad(ValueType, RI.Variable,
                                          "red.value." + Twine(En.index()));
     Value *PrivateRedValue =
@@ -1181,8 +1181,8 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductions(
   Builder.SetInsertPoint(AtomicRedBlock);
   if (CanGenerateAtomic) {
     for (const ReductionInfo &RI : ReductionInfos) {
-      Builder.restoreIP(RI.AtomicReductionGen(Builder.saveIP(), RI.Variable,
-                                              RI.PrivateVariable));
+      Builder.restoreIP(RI.AtomicReductionGen(Builder.saveIP(), RI.ElementType,
+                                              RI.Variable, RI.PrivateVariable));
       if (!Builder.GetInsertBlock())
         return InsertPointTy();
     }
@@ -1207,13 +1207,13 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductions(
         RedArrayTy, LHSArrayPtr, 0, En.index());
     Value *LHSI8Ptr = Builder.CreateLoad(Builder.getInt8PtrTy(), LHSI8PtrPtr);
     Value *LHSPtr = Builder.CreateBitCast(LHSI8Ptr, RI.Variable->getType());
-    Value *LHS = Builder.CreateLoad(RI.getElementType(), LHSPtr);
+    Value *LHS = Builder.CreateLoad(RI.ElementType, LHSPtr);
     Value *RHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64(
         RedArrayTy, RHSArrayPtr, 0, En.index());
     Value *RHSI8Ptr = Builder.CreateLoad(Builder.getInt8PtrTy(), RHSI8PtrPtr);
     Value *RHSPtr =
         Builder.CreateBitCast(RHSI8Ptr, RI.PrivateVariable->getType());
-    Value *RHS = Builder.CreateLoad(RI.getElementType(), RHSPtr);
+    Value *RHS = Builder.CreateLoad(RI.ElementType, RHSPtr);
     Value *Reduced;
     Builder.restoreIP(RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced));
     if (!Builder.GetInsertBlock())
@@ -1329,13 +1329,10 @@ CanonicalLoopInfo *OpenMPIRBuilder::createLoopSkeleton(
   LoopInfos.emplace_front();
   CanonicalLoopInfo *CL = &LoopInfos.front();
 
-  CL->Preheader = Preheader;
   CL->Header = Header;
   CL->Cond = Cond;
-  CL->Body = Body;
   CL->Latch = Latch;
   CL->Exit = Exit;
-  CL->After = After;
 
 #ifndef NDEBUG
   CL->assertOK();
@@ -1359,7 +1356,7 @@ OpenMPIRBuilder::createCanonicalLoop(const LocationDescription &Loc,
     // Split the loop at the insertion point: Branch to the preheader and move
     // every following instruction to after the loop (the After BB). Also, the
     // new successor is the loop's after block.
-    Builder.CreateBr(CL->Preheader);
+    Builder.CreateBr(CL->getPreheader());
     After->getInstList().splice(After->begin(), BB->getInstList(),
                                 Builder.GetInsertPoint(), BB->end());
     After->replaceSuccessorsPhiUsesWith(BB, After);
@@ -1791,6 +1788,12 @@ OpenMPIRBuilder::collapseLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
   BasicBlock *OrigAfter = Outermost->getAfter();
   Function *F = OrigPreheader->getParent();
 
+  // Loop control blocks that may become orphaned later.
+  SmallVector<BasicBlock *, 12> OldControlBBs;
+  OldControlBBs.reserve(6 * Loops.size());
+  for (CanonicalLoopInfo *Loop : Loops)
+    Loop->collectControlBlocks(OldControlBBs);
+
   // Setup the IRBuilder for inserting the trip count computation.
   Builder.SetCurrentDebugLocation(DL);
   if (ComputeIP.isSet())
@@ -1828,7 +1831,7 @@ OpenMPIRBuilder::collapseLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
 
   Value *Leftover = Result->getIndVar();
   SmallVector<Value *> NewIndVars;
-  NewIndVars.set_size(NumLoops);
+  NewIndVars.resize(NumLoops);
   for (int i = NumLoops - 1; i >= 1; --i) {
     Value *OrigTripCount = Loops[i]->getTripCount();
 
@@ -1886,10 +1889,6 @@ OpenMPIRBuilder::collapseLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
     Loops[i]->getIndVar()->replaceAllUsesWith(NewIndVars[i]);
 
   // Remove unused parts of the input loops.
-  SmallVector<BasicBlock *, 12> OldControlBBs;
-  OldControlBBs.reserve(6 * Loops.size());
-  for (CanonicalLoopInfo *Loop : Loops)
-    Loop->collectControlBlocks(OldControlBBs);
   removeUnusedBlocksFromParent(OldControlBBs);
 
   for (CanonicalLoopInfo *L : Loops)
@@ -1915,6 +1914,12 @@ OpenMPIRBuilder::tileLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
   BasicBlock *InnerEnter = InnermostLoop->getBody();
   BasicBlock *InnerLatch = InnermostLoop->getLatch();
 
+  // Loop control blocks that may become orphaned later.
+  SmallVector<BasicBlock *, 12> OldControlBBs;
+  OldControlBBs.reserve(6 * Loops.size());
+  for (CanonicalLoopInfo *Loop : Loops)
+    Loop->collectControlBlocks(OldControlBBs);
+
   // Collect original trip counts and induction variable to be accessible by
   // index. Also, the structure of the original loops is not preserved during
   // the construction of the tiled loops, so do it before we scavenge the BBs of
@@ -2074,10 +2079,6 @@ OpenMPIRBuilder::tileLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
   }
 
   // Remove unused parts of the original loops.
-  SmallVector<BasicBlock *, 12> OldControlBBs;
-  OldControlBBs.reserve(6 * Loops.size());
-  for (CanonicalLoopInfo *Loop : Loops)
-    Loop->collectControlBlocks(OldControlBBs);
   removeUnusedBlocksFromParent(OldControlBBs);
 
   for (CanonicalLoopInfo *L : Loops)
@@ -3079,7 +3080,7 @@ OpenMPIRBuilder::createAtomicWrite(const LocationDescription &Loc,
 OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicUpdate(
     const LocationDescription &Loc, Instruction *AllocIP, AtomicOpValue &X,
     Value *Expr, AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
-    AtomicUpdateCallbackTy &UpdateOp, bool IsXLHSInRHSPart) {
+    AtomicUpdateCallbackTy &UpdateOp, bool IsXBinopExpr) {
   if (!updateToLocation(Loc))
     return Loc.IP;
 
@@ -3097,7 +3098,7 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicUpdate(
   });
 
   emitAtomicUpdate(AllocIP, X.Var, Expr, AO, RMWOp, UpdateOp, X.IsVolatile,
-                   IsXLHSInRHSPart);
+                   IsXBinopExpr);
   checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Update);
   return Builder.saveIP();
 }
@@ -3134,13 +3135,13 @@ std::pair<Value *, Value *>
 OpenMPIRBuilder::emitAtomicUpdate(Instruction *AllocIP, Value *X, Value *Expr,
                                   AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
                                   AtomicUpdateCallbackTy &UpdateOp,
-                                  bool VolatileX, bool IsXLHSInRHSPart) {
+                                  bool VolatileX, bool IsXBinopExpr) {
   Type *XElemTy = X->getType()->getPointerElementType();
 
   bool DoCmpExch =
       ((RMWOp == AtomicRMWInst::BAD_BINOP) || (RMWOp == AtomicRMWInst::FAdd)) ||
       (RMWOp == AtomicRMWInst::FSub) ||
-      (RMWOp == AtomicRMWInst::Sub && !IsXLHSInRHSPart);
+      (RMWOp == AtomicRMWInst::Sub && !IsXBinopExpr);
 
   std::pair<Value *, Value *> Res;
   if (XElemTy->isIntegerTy() && !DoCmpExch) {
@@ -3232,7 +3233,7 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCapture(
     const LocationDescription &Loc, Instruction *AllocIP, AtomicOpValue &X,
     AtomicOpValue &V, Value *Expr, AtomicOrdering AO,
     AtomicRMWInst::BinOp RMWOp, AtomicUpdateCallbackTy &UpdateOp,
-    bool UpdateExpr, bool IsPostfixUpdate, bool IsXLHSInRHSPart) {
+    bool UpdateExpr, bool IsPostfixUpdate, bool IsXBinopExpr) {
   if (!updateToLocation(Loc))
     return Loc.IP;
 
@@ -3251,9 +3252,8 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCapture(
   // If UpdateExpr is 'x' updated with some `expr` not based on 'x',
   // 'x' is simply atomically rewritten with 'expr'.
   AtomicRMWInst::BinOp AtomicOp = (UpdateExpr ? RMWOp : AtomicRMWInst::Xchg);
-  std::pair<Value *, Value *> Result =
-      emitAtomicUpdate(AllocIP, X.Var, Expr, AO, AtomicOp, UpdateOp,
-                       X.IsVolatile, IsXLHSInRHSPart);
+  std::pair<Value *, Value *> Result = emitAtomicUpdate(
+      AllocIP, X.Var, Expr, AO, AtomicOp, UpdateOp, X.IsVolatile, IsXBinopExpr);
 
   Value *CapturedVal = (IsPostfixUpdate ? Result.first : Result.second);
   Builder.CreateStore(CapturedVal, V.Var, V.IsVolatile);
@@ -3321,7 +3321,16 @@ void CanonicalLoopInfo::collectControlBlocks(
   // flow. For consistency, this also means we do not add the Body block, which
   // is just the entry to the body code.
   BBs.reserve(BBs.size() + 6);
-  BBs.append({Preheader, Header, Cond, Latch, Exit, After});
+  BBs.append({getPreheader(), Header, Cond, Latch, Exit, getAfter()});
+}
+
+BasicBlock *CanonicalLoopInfo::getPreheader() const {
+  assert(isValid() && "Requires a valid canonical loop");
+  for (BasicBlock *Pred : predecessors(Header)) {
+    if (Pred != Latch)
+      return Pred;
+  }
+  llvm_unreachable("Missing preheader");
 }
 
 void CanonicalLoopInfo::assertOK() const {
@@ -3330,6 +3339,10 @@ void CanonicalLoopInfo::assertOK() const {
   if (!isValid())
     return;
 
+  BasicBlock *Preheader = getPreheader();
+  BasicBlock *Body = getBody();
+  BasicBlock *After = getAfter();
+
   // Verify standard control-flow we use for OpenMP loops.
   assert(Preheader);
   assert(isa<BranchInst>(Preheader->getTerminator()) &&
@@ -3415,11 +3428,8 @@ void CanonicalLoopInfo::assertOK() const {
 }
 
 void CanonicalLoopInfo::invalidate() {
-  Preheader = nullptr;
   Header = nullptr;
   Cond = nullptr;
-  Body = nullptr;
   Latch = nullptr;
   Exit = nullptr;
-  After = nullptr;
 }
diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp
index c9748e1387eb..bbe0c97e60a2 100644
--- a/llvm/lib/IR/AsmWriter.cpp
+++ b/llvm/lib/IR/AsmWriter.cpp
@@ -512,10 +512,8 @@ void TypePrinting::incorporateTypes() {
   // the unnamed ones out to a numbering and remove the anonymous structs.
   unsigned NextNumber = 0;
 
-  std::vector<StructType*>::iterator NextToUse = NamedTypes.begin(), I, E;
-  for (I = NamedTypes.begin(), E = NamedTypes.end(); I != E; ++I) {
-    StructType *STy = *I;
-
+  std::vector<StructType *>::iterator NextToUse = NamedTypes.begin();
+  for (StructType *STy : NamedTypes) {
     // Ignore anonymous types.
     if (STy->isLiteral())
       continue;
@@ -1450,6 +1448,12 @@ static void WriteConstantInternal(raw_ostream &Out, const Constant *CV,
     return;
   }
 
+  if (const auto *NC = dyn_cast<NoCFIValue>(CV)) {
+    Out << "no_cfi ";
+    WriteAsOperandInternal(Out, NC->getGlobalValue(), WriterCtx);
+    return;
+  }
+
   if (const ConstantArray *CA = dyn_cast<ConstantArray>(CV)) {
     Type *ETy = CA->getType()->getElementType();
     Out << '[';
@@ -1583,11 +1587,9 @@ static void WriteConstantInternal(raw_ostream &Out, const Constant *CV,
         Out << ", ";
     }
 
-    if (CE->hasIndices()) {
-      ArrayRef<unsigned> Indices = CE->getIndices();
-      for (unsigned i = 0, e = Indices.size(); i != e; ++i)
-        Out << ", " << Indices[i];
-    }
+    if (CE->hasIndices())
+      for (unsigned I : CE->getIndices())
+        Out << ", " << I;
 
     if (CE->isCast()) {
       Out << " to ";
@@ -3528,8 +3530,8 @@ void AssemblyWriter::printGlobal(const GlobalVariable *GV) {
   }
 
   maybePrintComdat(Out, *GV);
-  if (GV->getAlignment())
-    Out << ", align " << GV->getAlignment();
+  if (MaybeAlign A = GV->getAlign())
+    Out << ", align " << A->value();
 
   SmallVector<std::pair<unsigned, MDNode *>, 4> MDs;
   GV->getAllMetadata(MDs);
@@ -3637,13 +3639,13 @@ void AssemblyWriter::printTypeIdentities() {
   }
 
   auto &NamedTypes = TypePrinter.getNamedTypes();
-  for (unsigned I = 0, E = NamedTypes.size(); I != E; ++I) {
-    PrintLLVMName(Out, NamedTypes[I]->getName(), LocalPrefix);
+  for (StructType *NamedType : NamedTypes) {
+    PrintLLVMName(Out, NamedType->getName(), LocalPrefix);
     Out << " = type ";
 
     // Make sure we print out at least one level of the type structure, so
     // that we do not get %FILE = type %FILE
-    TypePrinter.printStructBody(NamedTypes[I], Out);
+    TypePrinter.printStructBody(NamedType, Out);
     Out << '\n';
   }
 }
@@ -3757,8 +3759,8 @@ void AssemblyWriter::printFunction(const Function *F) {
     Out << '"';
   }
   maybePrintComdat(Out, *F);
-  if (F->getAlignment())
-    Out << " align " << F->getAlignment();
+  if (MaybeAlign A = F->getAlign())
+    Out << " align " << A->value();
   if (F->hasGC())
     Out << " gc \"" << F->getGC() << '"';
   if (F->hasPrefixData()) {
@@ -4239,8 +4241,8 @@ void AssemblyWriter::printInstruction(const Instruction &I) {
       Out << ", ";
       writeOperand(AI->getArraySize(), true);
     }
-    if (AI->getAlignment()) {
-      Out << ", align " << AI->getAlignment();
+    if (MaybeAlign A = AI->getAlign()) {
+      Out << ", align " << A->value();
     }
 
     unsigned AddrSpace = AI->getType()->getAddressSpace();
@@ -4310,13 +4312,13 @@ void AssemblyWriter::printInstruction(const Instruction &I) {
   if (const LoadInst *LI = dyn_cast<LoadInst>(&I)) {
     if (LI->isAtomic())
       writeAtomic(LI->getContext(), LI->getOrdering(), LI->getSyncScopeID());
-    if (LI->getAlignment())
-      Out << ", align " << LI->getAlignment();
+    if (MaybeAlign A = LI->getAlign())
+      Out << ", align " << A->value();
   } else if (const StoreInst *SI = dyn_cast<StoreInst>(&I)) {
     if (SI->isAtomic())
       writeAtomic(SI->getContext(), SI->getOrdering(), SI->getSyncScopeID());
-    if (SI->getAlignment())
-      Out << ", align " << SI->getAlignment();
+    if (MaybeAlign A = SI->getAlign())
+      Out << ", align " << A->value();
   } else if (const AtomicCmpXchgInst *CXI = dyn_cast<AtomicCmpXchgInst>(&I)) {
     writeAtomicCmpXchg(CXI->getContext(), CXI->getSuccessOrdering(),
                        CXI->getFailureOrdering(), CXI->getSyncScopeID());
diff --git a/llvm/lib/IR/AttributeImpl.h b/llvm/lib/IR/AttributeImpl.h
index c5bbe6571096..1153fb827b56 100644
--- a/llvm/lib/IR/AttributeImpl.h
+++ b/llvm/lib/IR/AttributeImpl.h
@@ -253,7 +253,8 @@ public:
   uint64_t getDereferenceableBytes() const;
   uint64_t getDereferenceableOrNullBytes() const;
   std::pair<unsigned, Optional<unsigned>> getAllocSizeArgs() const;
-  std::pair<unsigned, unsigned> getVScaleRangeArgs() const;
+  unsigned getVScaleRangeMin() const;
+  Optional<unsigned> getVScaleRangeMax() const;
   std::string getAsString(bool InAttrGrp) const;
   Type *getAttributeType(Attribute::AttrKind Kind) const;
 
diff --git a/llvm/lib/IR/Attributes.cpp b/llvm/lib/IR/Attributes.cpp
index f81a446d6e46..c899afae6cce 100644
--- a/llvm/lib/IR/Attributes.cpp
+++ b/llvm/lib/IR/Attributes.cpp
@@ -78,15 +78,18 @@ unpackAllocSizeArgs(uint64_t Num) {
   return std::make_pair(ElemSizeArg, NumElemsArg);
 }
 
-static uint64_t packVScaleRangeArgs(unsigned MinValue, unsigned MaxValue) {
-  return uint64_t(MinValue) << 32 | MaxValue;
+static uint64_t packVScaleRangeArgs(unsigned MinValue,
+                                    Optional<unsigned> MaxValue) {
+  return uint64_t(MinValue) << 32 | MaxValue.getValueOr(0);
 }
 
-static std::pair<unsigned, unsigned> unpackVScaleRangeArgs(uint64_t Value) {
+static std::pair<unsigned, Optional<unsigned>>
+unpackVScaleRangeArgs(uint64_t Value) {
   unsigned MaxValue = Value & std::numeric_limits<unsigned>::max();
   unsigned MinValue = Value >> 32;
 
-  return std::make_pair(MinValue, MaxValue);
+  return std::make_pair(MinValue,
+                        MaxValue > 0 ? MaxValue : Optional<unsigned>());
 }
 
 Attribute Attribute::get(LLVMContext &Context, Attribute::AttrKind Kind,
@@ -354,10 +357,16 @@ std::pair<unsigned, Optional<unsigned>> Attribute::getAllocSizeArgs() const {
   return unpackAllocSizeArgs(pImpl->getValueAsInt());
 }
 
-std::pair<unsigned, unsigned> Attribute::getVScaleRangeArgs() const {
+unsigned Attribute::getVScaleRangeMin() const {
+  assert(hasAttribute(Attribute::VScaleRange) &&
+         "Trying to get vscale args from non-vscale attribute");
+  return unpackVScaleRangeArgs(pImpl->getValueAsInt()).first;
+}
+
+Optional<unsigned> Attribute::getVScaleRangeMax() const {
   assert(hasAttribute(Attribute::VScaleRange) &&
          "Trying to get vscale args from non-vscale attribute");
-  return unpackVScaleRangeArgs(pImpl->getValueAsInt());
+  return unpackVScaleRangeArgs(pImpl->getValueAsInt()).second;
 }
 
 std::string Attribute::getAsString(bool InAttrGrp) const {
@@ -428,13 +437,13 @@ std::string Attribute::getAsString(bool InAttrGrp) const {
   }
 
   if (hasAttribute(Attribute::VScaleRange)) {
-    unsigned MinValue, MaxValue;
-    std::tie(MinValue, MaxValue) = getVScaleRangeArgs();
+    unsigned MinValue = getVScaleRangeMin();
+    Optional<unsigned> MaxValue = getVScaleRangeMax();
 
     std::string Result = "vscale_range(";
     Result += utostr(MinValue);
     Result += ',';
-    Result += utostr(MaxValue);
+    Result += utostr(MaxValue.getValueOr(0));
     Result += ')';
     return Result;
   }
@@ -717,9 +726,12 @@ std::pair<unsigned, Optional<unsigned>> AttributeSet::getAllocSizeArgs() const {
                  : std::pair<unsigned, Optional<unsigned>>(0, 0);
 }
 
-std::pair<unsigned, unsigned> AttributeSet::getVScaleRangeArgs() const {
-  return SetNode ? SetNode->getVScaleRangeArgs()
-                 : std::pair<unsigned, unsigned>(0, 0);
+unsigned AttributeSet::getVScaleRangeMin() const {
+  return SetNode ? SetNode->getVScaleRangeMin() : 1;
+}
+
+Optional<unsigned> AttributeSet::getVScaleRangeMax() const {
+  return SetNode ? SetNode->getVScaleRangeMax() : None;
 }
 
 std::string AttributeSet::getAsString(bool InAttrGrp) const {
@@ -897,10 +909,16 @@ AttributeSetNode::getAllocSizeArgs() const {
   return std::make_pair(0, 0);
 }
 
-std::pair<unsigned, unsigned> AttributeSetNode::getVScaleRangeArgs() const {
+unsigned AttributeSetNode::getVScaleRangeMin() const {
   if (auto A = findEnumAttribute(Attribute::VScaleRange))
-    return A->getVScaleRangeArgs();
-  return std::make_pair(0, 0);
+    return A->getVScaleRangeMin();
+  return 1;
+}
+
+Optional<unsigned> AttributeSetNode::getVScaleRangeMax() const {
+  if (auto A = findEnumAttribute(Attribute::VScaleRange))
+    return A->getVScaleRangeMax();
+  return None;
 }
 
 std::string AttributeSetNode::getAsString(bool InAttrGrp) const {
@@ -1118,16 +1136,21 @@ AttributeList AttributeList::get(LLVMContext &C, AttributeSet FnAttrs,
 }
 
 AttributeList AttributeList::get(LLVMContext &C, unsigned Index,
-                                 const AttrBuilder &B) {
-  if (!B.hasAttributes())
+                                 AttributeSet Attrs) {
+  if (!Attrs.hasAttributes())
     return {};
   Index = attrIdxToArrayIdx(Index);
   SmallVector<AttributeSet, 8> AttrSets(Index + 1);
-  AttrSets[Index] = AttributeSet::get(C, B);
+  AttrSets[Index] = Attrs;
   return getImpl(C, AttrSets);
 }
 
 AttributeList AttributeList::get(LLVMContext &C, unsigned Index,
+                                 const AttrBuilder &B) {
+  return get(C, Index, AttributeSet::get(C, B));
+}
+
+AttributeList AttributeList::get(LLVMContext &C, unsigned Index,
                                  ArrayRef<Attribute::AttrKind> Kinds) {
   SmallVector<std::pair<unsigned, Attribute>, 8> Attrs;
   for (const auto K : Kinds)
@@ -1623,8 +1646,12 @@ std::pair<unsigned, Optional<unsigned>> AttrBuilder::getAllocSizeArgs() const {
   return unpackAllocSizeArgs(getRawIntAttr(Attribute::AllocSize));
 }
 
-std::pair<unsigned, unsigned> AttrBuilder::getVScaleRangeArgs() const {
-  return unpackVScaleRangeArgs(getRawIntAttr(Attribute::VScaleRange));
+unsigned AttrBuilder::getVScaleRangeMin() const {
+  return unpackVScaleRangeArgs(getRawIntAttr(Attribute::VScaleRange)).first;
+}
+
+Optional<unsigned> AttrBuilder::getVScaleRangeMax() const {
+  return unpackVScaleRangeArgs(getRawIntAttr(Attribute::VScaleRange)).second;
 }
 
 AttrBuilder &AttrBuilder::addAlignmentAttr(MaybeAlign Align) {
@@ -1669,7 +1696,7 @@ AttrBuilder &AttrBuilder::addAllocSizeAttrFromRawRepr(uint64_t RawArgs) {
 }
 
 AttrBuilder &AttrBuilder::addVScaleRangeAttr(unsigned MinValue,
-                                             unsigned MaxValue) {
+                                             Optional<unsigned> MaxValue) {
   return addVScaleRangeAttrFromRawRepr(packVScaleRangeArgs(MinValue, MaxValue));
 }
 
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index d73d1e9c20b3..b8ad2b294b87 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -702,6 +702,31 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
       NewFn = Intrinsic::getDeclaration(F->getParent(), IID, Tys);
       return true;
     }
+
+    if (Name == "arm.mve.vctp64" &&
+        cast<FixedVectorType>(F->getReturnType())->getNumElements() == 4) {
+      // A vctp64 returning a v4i1 is converted to return a v2i1. Rename the
+      // function and deal with it below in UpgradeIntrinsicCall.
+      rename(F);
+      return true;
+    }
+    // These too are changed to accept a v2i1 insteead of the old v4i1.
+    if (Name == "arm.mve.mull.int.predicated.v2i64.v4i32.v4i1" ||
+        Name == "arm.mve.vqdmull.predicated.v2i64.v4i32.v4i1" ||
+        Name == "arm.mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" ||
+        Name == "arm.mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" ||
+        Name == "arm.mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" ||
+        Name == "arm.mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" ||
+        Name == "arm.mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" ||
+        Name == "arm.mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" ||
+        Name == "arm.cde.vcx1q.predicated.v2i64.v4i1" ||
+        Name == "arm.cde.vcx1qa.predicated.v2i64.v4i1" ||
+        Name == "arm.cde.vcx2q.predicated.v2i64.v4i1" ||
+        Name == "arm.cde.vcx2qa.predicated.v2i64.v4i1" ||
+        Name == "arm.cde.vcx3q.predicated.v2i64.v4i1" ||
+        Name == "arm.cde.vcx3qa.predicated.v2i64.v4i1")
+      return true;
+
     break;
   }
 
@@ -1803,6 +1828,96 @@ void llvm::UpgradeInlineAsmString(std::string *AsmStr) {
   }
 }
 
+static Value *UpgradeARMIntrinsicCall(StringRef Name, CallInst *CI, Function *F,
+                                      IRBuilder<> &Builder) {
+  if (Name == "mve.vctp64.old") {
+    // Replace the old v4i1 vctp64 with a v2i1 vctp and predicate-casts to the
+    // correct type.
+    Value *VCTP = Builder.CreateCall(
+        Intrinsic::getDeclaration(F->getParent(), Intrinsic::arm_mve_vctp64),
+        CI->getArgOperand(0), CI->getName());
+    Value *C1 = Builder.CreateCall(
+        Intrinsic::getDeclaration(
+            F->getParent(), Intrinsic::arm_mve_pred_v2i,
+            {VectorType::get(Builder.getInt1Ty(), 2, false)}),
+        VCTP);
+    return Builder.CreateCall(
+        Intrinsic::getDeclaration(
+            F->getParent(), Intrinsic::arm_mve_pred_i2v,
+            {VectorType::get(Builder.getInt1Ty(), 4, false)}),
+        C1);
+  } else if (Name == "mve.mull.int.predicated.v2i64.v4i32.v4i1" ||
+             Name == "mve.vqdmull.predicated.v2i64.v4i32.v4i1" ||
+             Name == "mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" ||
+             Name == "mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" ||
+             Name == "mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" ||
+             Name == "mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" ||
+             Name == "mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" ||
+             Name == "mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" ||
+             Name == "cde.vcx1q.predicated.v2i64.v4i1" ||
+             Name == "cde.vcx1qa.predicated.v2i64.v4i1" ||
+             Name == "cde.vcx2q.predicated.v2i64.v4i1" ||
+             Name == "cde.vcx2qa.predicated.v2i64.v4i1" ||
+             Name == "cde.vcx3q.predicated.v2i64.v4i1" ||
+             Name == "cde.vcx3qa.predicated.v2i64.v4i1") {
+    std::vector<Type *> Tys;
+    unsigned ID = CI->getIntrinsicID();
+    Type *V2I1Ty = FixedVectorType::get(Builder.getInt1Ty(), 2);
+    switch (ID) {
+    case Intrinsic::arm_mve_mull_int_predicated:
+    case Intrinsic::arm_mve_vqdmull_predicated:
+    case Intrinsic::arm_mve_vldr_gather_base_predicated:
+      Tys = {CI->getType(), CI->getOperand(0)->getType(), V2I1Ty};
+      break;
+    case Intrinsic::arm_mve_vldr_gather_base_wb_predicated:
+    case Intrinsic::arm_mve_vstr_scatter_base_predicated:
+    case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated:
+      Tys = {CI->getOperand(0)->getType(), CI->getOperand(0)->getType(),
+             V2I1Ty};
+      break;
+    case Intrinsic::arm_mve_vldr_gather_offset_predicated:
+      Tys = {CI->getType(), CI->getOperand(0)->getType(),
+             CI->getOperand(1)->getType(), V2I1Ty};
+      break;
+    case Intrinsic::arm_mve_vstr_scatter_offset_predicated:
+      Tys = {CI->getOperand(0)->getType(), CI->getOperand(1)->getType(),
+             CI->getOperand(2)->getType(), V2I1Ty};
+      break;
+    case Intrinsic::arm_cde_vcx1q_predicated:
+    case Intrinsic::arm_cde_vcx1qa_predicated:
+    case Intrinsic::arm_cde_vcx2q_predicated:
+    case Intrinsic::arm_cde_vcx2qa_predicated:
+    case Intrinsic::arm_cde_vcx3q_predicated:
+    case Intrinsic::arm_cde_vcx3qa_predicated:
+      Tys = {CI->getOperand(1)->getType(), V2I1Ty};
+      break;
+    default:
+      llvm_unreachable("Unhandled Intrinsic!");
+    }
+
+    std::vector<Value *> Ops;
+    for (Value *Op : CI->args()) {
+      Type *Ty = Op->getType();
+      if (Ty->getScalarSizeInBits() == 1) {
+        Value *C1 = Builder.CreateCall(
+            Intrinsic::getDeclaration(
+                F->getParent(), Intrinsic::arm_mve_pred_v2i,
+                {VectorType::get(Builder.getInt1Ty(), 4, false)}),
+            Op);
+        Op = Builder.CreateCall(
+            Intrinsic::getDeclaration(F->getParent(),
+                                      Intrinsic::arm_mve_pred_i2v, {V2I1Ty}),
+            C1);
+      }
+      Ops.push_back(Op);
+    }
+
+    Function *Fn = Intrinsic::getDeclaration(F->getParent(), ID, Tys);
+    return Builder.CreateCall(Fn, Ops, CI->getName());
+  }
+  llvm_unreachable("Unknown function for ARM CallInst upgrade.");
+}
+
 /// Upgrade a call to an old intrinsic. All argument and return casting must be
 /// provided to seamlessly integrate with existing context.
 void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
@@ -1826,6 +1941,9 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
     bool IsNVVM = Name.startswith("nvvm.");
     if (IsNVVM)
       Name = Name.substr(5);
+    bool IsARM = Name.startswith("arm.");
+    if (IsARM)
+      Name = Name.substr(4);
 
     if (IsX86 && Name.startswith("sse4a.movnt.")) {
       Module *M = F->getParent();
@@ -2289,14 +2407,12 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
       if (CI->arg_size() >= 3)
         Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
                             CI->getArgOperand(1));
-    } else if (IsX86 && (Name.startswith("avx512.mask.loadu."))) {
-      Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0),
-                              CI->getArgOperand(1), CI->getArgOperand(2),
-                              /*Aligned*/false);
-    } else if (IsX86 && (Name.startswith("avx512.mask.load."))) {
-      Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0),
-                              CI->getArgOperand(1),CI->getArgOperand(2),
-                              /*Aligned*/true);
+    } else if (IsX86 && Name.startswith("avx512.mask.load")) {
+      // "avx512.mask.loadu." or "avx512.mask.load."
+      bool Aligned = Name[16] != 'u'; // "avx512.mask.loadu".
+      Rep =
+          UpgradeMaskedLoad(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
+                            CI->getArgOperand(2), Aligned);
     } else if (IsX86 && Name.startswith("avx512.mask.expand.load.")) {
       auto *ResultTy = cast<FixedVectorType>(CI->getType());
       Type *PtrTy = ResultTy->getElementType();
@@ -3649,6 +3765,8 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
                                    F->getParent(), Intrinsic::convert_from_fp16,
                                    {Builder.getFloatTy()}),
                                CI->getArgOperand(0), "h2f");
+    } else if (IsARM) {
+      Rep = UpgradeARMIntrinsicCall(Name, CI, F, Builder);
     } else {
       llvm_unreachable("Unknown function for CallInst upgrade.");
     }
diff --git a/llvm/lib/IR/BasicBlock.cpp b/llvm/lib/IR/BasicBlock.cpp
index ed1956e0f7e9..7beafc485d09 100644
--- a/llvm/lib/IR/BasicBlock.cpp
+++ b/llvm/lib/IR/BasicBlock.cpp
@@ -450,8 +450,8 @@ BasicBlock *BasicBlock::splitBasicBlockBefore(iterator I, const Twine &BBName) {
 void BasicBlock::replacePhiUsesWith(BasicBlock *Old, BasicBlock *New) {
   // N.B. This might not be a complete BasicBlock, so don't assume
   // that it ends with a non-phi instruction.
-  for (iterator II = begin(), IE = end(); II != IE; ++II) {
-    PHINode *PN = dyn_cast<PHINode>(II);
+  for (Instruction &I : *this) {
+    PHINode *PN = dyn_cast<PHINode>(&I);
     if (!PN)
       break;
     PN->replaceIncomingBlockWith(Old, New);
diff --git a/llvm/lib/IR/ConstantFold.cpp b/llvm/lib/IR/ConstantFold.cpp
index 437fd0558447..8668fe82601c 100644
--- a/llvm/lib/IR/ConstantFold.cpp
+++ b/llvm/lib/IR/ConstantFold.cpp
@@ -1801,46 +1801,8 @@ Constant *llvm::ConstantFoldCompareInstruction(unsigned short pred,
   } else if (isa<ConstantFP>(C1) && isa<ConstantFP>(C2)) {
     const APFloat &C1V = cast<ConstantFP>(C1)->getValueAPF();
     const APFloat &C2V = cast<ConstantFP>(C2)->getValueAPF();
-    APFloat::cmpResult R = C1V.compare(C2V);
-    switch (pred) {
-    default: llvm_unreachable("Invalid FCmp Predicate");
-    case FCmpInst::FCMP_FALSE: return Constant::getNullValue(ResultTy);
-    case FCmpInst::FCMP_TRUE:  return Constant::getAllOnesValue(ResultTy);
-    case FCmpInst::FCMP_UNO:
-      return ConstantInt::get(ResultTy, R==APFloat::cmpUnordered);
-    case FCmpInst::FCMP_ORD:
-      return ConstantInt::get(ResultTy, R!=APFloat::cmpUnordered);
-    case FCmpInst::FCMP_UEQ:
-      return ConstantInt::get(ResultTy, R==APFloat::cmpUnordered ||
-                                        R==APFloat::cmpEqual);
-    case FCmpInst::FCMP_OEQ:
-      return ConstantInt::get(ResultTy, R==APFloat::cmpEqual);
-    case FCmpInst::FCMP_UNE:
-      return ConstantInt::get(ResultTy, R!=APFloat::cmpEqual);
-    case FCmpInst::FCMP_ONE:
-      return ConstantInt::get(ResultTy, R==APFloat::cmpLessThan ||
-                                        R==APFloat::cmpGreaterThan);
-    case FCmpInst::FCMP_ULT:
-      return ConstantInt::get(ResultTy, R==APFloat::cmpUnordered ||
-                                        R==APFloat::cmpLessThan);
-    case FCmpInst::FCMP_OLT:
-      return ConstantInt::get(ResultTy, R==APFloat::cmpLessThan);
-    case FCmpInst::FCMP_UGT:
-      return ConstantInt::get(ResultTy, R==APFloat::cmpUnordered ||
-                                        R==APFloat::cmpGreaterThan);
-    case FCmpInst::FCMP_OGT:
-      return ConstantInt::get(ResultTy, R==APFloat::cmpGreaterThan);
-    case FCmpInst::FCMP_ULE:
-      return ConstantInt::get(ResultTy, R!=APFloat::cmpGreaterThan);
-    case FCmpInst::FCMP_OLE:
-      return ConstantInt::get(ResultTy, R==APFloat::cmpLessThan ||
-                                        R==APFloat::cmpEqual);
-    case FCmpInst::FCMP_UGE:
-      return ConstantInt::get(ResultTy, R!=APFloat::cmpLessThan);
-    case FCmpInst::FCMP_OGE:
-      return ConstantInt::get(ResultTy, R==APFloat::cmpGreaterThan ||
-                                        R==APFloat::cmpEqual);
-    }
+    CmpInst::Predicate Predicate = CmpInst::Predicate(pred);
+    return ConstantInt::get(ResultTy, FCmpInst::compare(C1V, C2V, Predicate));
   } else if (auto *C1VTy = dyn_cast<VectorType>(C1->getType())) {
 
     // Fast path for splatted constants.
@@ -2215,9 +2177,8 @@ Constant *llvm::ConstantFoldGetElementPtr(Type *PointeeTy, Constant *C,
 
   if (C->isNullValue()) {
     bool isNull = true;
-    for (unsigned i = 0, e = Idxs.size(); i != e; ++i)
-      if (!isa<UndefValue>(Idxs[i]) &&
-          !cast<Constant>(Idxs[i])->isNullValue()) {
+    for (Value *Idx : Idxs)
+      if (!isa<UndefValue>(Idx) && !cast<Constant>(Idx)->isNullValue()) {
         isNull = false;
         break;
       }
@@ -2233,8 +2194,8 @@ Constant *llvm::ConstantFoldGetElementPtr(Type *PointeeTy, Constant *C,
 
       // The GEP returns a vector of pointers when one of more of
       // its arguments is a vector.
-      for (unsigned i = 0, e = Idxs.size(); i != e; ++i) {
-        if (auto *VT = dyn_cast<VectorType>(Idxs[i]->getType())) {
+      for (Value *Idx : Idxs) {
+        if (auto *VT = dyn_cast<VectorType>(Idx->getType())) {
           assert((!isa<VectorType>(GEPTy) || isa<ScalableVectorType>(GEPTy) ==
                                                  isa<ScalableVectorType>(VT)) &&
                  "Mismatched GEPTy vector types");
diff --git a/llvm/lib/IR/Constants.cpp b/llvm/lib/IR/Constants.cpp
index c66cfb6e9ac1..837be910f6d8 100644
--- a/llvm/lib/IR/Constants.cpp
+++ b/llvm/lib/IR/Constants.cpp
@@ -535,6 +535,9 @@ void llvm::deleteConstant(Constant *C) {
   case Constant::DSOLocalEquivalentVal:
     delete static_cast<DSOLocalEquivalent *>(C);
     break;
+  case Constant::NoCFIValueVal:
+    delete static_cast<NoCFIValue *>(C);
+    break;
   case Constant::UndefValueVal:
     delete static_cast<UndefValue *>(C);
     break;
@@ -1296,9 +1299,10 @@ Constant *ConstantArray::getImpl(ArrayType *Ty, ArrayRef<Constant*> V) {
   if (V.empty())
     return ConstantAggregateZero::get(Ty);
 
-  for (unsigned i = 0, e = V.size(); i != e; ++i) {
-    assert(V[i]->getType() == Ty->getElementType() &&
+  for (Constant *C : V) {
+    assert(C->getType() == Ty->getElementType() &&
            "Wrong type in array element initializer");
+    (void)C;
   }
 
   // If this is an all-zero array, return a ConstantAggregateZero object.  If
@@ -1364,12 +1368,12 @@ Constant *ConstantStruct::get(StructType *ST, ArrayRef<Constant*> V) {
     isZero = V[0]->isNullValue();
     // PoisonValue inherits UndefValue, so its check is not necessary.
     if (isUndef || isZero) {
-      for (unsigned i = 0, e = V.size(); i != e; ++i) {
-        if (!V[i]->isNullValue())
+      for (Constant *C : V) {
+        if (!C->isNullValue())
           isZero = false;
-        if (!isa<PoisonValue>(V[i]))
+        if (!isa<PoisonValue>(C))
           isPoison = false;
-        if (isa<PoisonValue>(V[i]) || !isa<UndefValue>(V[i]))
+        if (isa<PoisonValue>(C) || !isa<UndefValue>(C))
           isUndef = false;
       }
     }
@@ -1962,6 +1966,47 @@ Value *DSOLocalEquivalent::handleOperandChangeImpl(Value *From, Value *To) {
   return nullptr;
 }
 
+NoCFIValue *NoCFIValue::get(GlobalValue *GV) {
+  NoCFIValue *&NC = GV->getContext().pImpl->NoCFIValues[GV];
+  if (!NC)
+    NC = new NoCFIValue(GV);
+
+  assert(NC->getGlobalValue() == GV &&
+         "NoCFIValue does not match the expected global value");
+  return NC;
+}
+
+NoCFIValue::NoCFIValue(GlobalValue *GV)
+    : Constant(GV->getType(), Value::NoCFIValueVal, &Op<0>(), 1) {
+  setOperand(0, GV);
+}
+
+/// Remove the constant from the constant table.
+void NoCFIValue::destroyConstantImpl() {
+  const GlobalValue *GV = getGlobalValue();
+  GV->getContext().pImpl->NoCFIValues.erase(GV);
+}
+
+Value *NoCFIValue::handleOperandChangeImpl(Value *From, Value *To) {
+  assert(From == getGlobalValue() && "Changing value does not match operand.");
+
+  GlobalValue *GV = dyn_cast<GlobalValue>(To->stripPointerCasts());
+  assert(GV && "Can only replace the operands with a global value");
+
+  NoCFIValue *&NewNC = getContext().pImpl->NoCFIValues[GV];
+  if (NewNC)
+    return llvm::ConstantExpr::getBitCast(NewNC, getType());
+
+  getContext().pImpl->NoCFIValues.erase(getGlobalValue());
+  NewNC = this;
+  setOperand(0, GV);
+
+  if (GV->getType() != getType())
+    mutateType(GV->getType());
+
+  return nullptr;
+}
+
 //---- ConstantExpr::get() implementations.
 //
 
diff --git a/llvm/lib/IR/Core.cpp b/llvm/lib/IR/Core.cpp
index 2c396ae97499..a263d2536541 100644
--- a/llvm/lib/IR/Core.cpp
+++ b/llvm/lib/IR/Core.cpp
@@ -1696,6 +1696,14 @@ LLVMValueRef LLVMConstGEP(LLVMValueRef ConstantVal,
   return wrap(ConstantExpr::getGetElementPtr(Ty, Val, IdxList));
 }
 
+LLVMValueRef LLVMConstGEP2(LLVMTypeRef Ty, LLVMValueRef ConstantVal,
+                           LLVMValueRef *ConstantIndices, unsigned NumIndices) {
+  ArrayRef<Constant *> IdxList(unwrap<Constant>(ConstantIndices, NumIndices),
+                               NumIndices);
+  Constant *Val = unwrap<Constant>(ConstantVal);
+  return wrap(ConstantExpr::getGetElementPtr(unwrap(Ty), Val, IdxList));
+}
+
 LLVMValueRef LLVMConstInBoundsGEP(LLVMValueRef ConstantVal,
                                   LLVMValueRef *ConstantIndices,
                                   unsigned NumIndices) {
@@ -1707,6 +1715,15 @@ LLVMValueRef LLVMConstInBoundsGEP(LLVMValueRef ConstantVal,
   return wrap(ConstantExpr::getInBoundsGetElementPtr(Ty, Val, IdxList));
 }
 
+LLVMValueRef LLVMConstInBoundsGEP2(LLVMTypeRef Ty, LLVMValueRef ConstantVal,
+                                   LLVMValueRef *ConstantIndices,
+                                   unsigned NumIndices) {
+  ArrayRef<Constant *> IdxList(unwrap<Constant>(ConstantIndices, NumIndices),
+                               NumIndices);
+  Constant *Val = unwrap<Constant>(ConstantVal);
+  return wrap(ConstantExpr::getInBoundsGetElementPtr(unwrap(Ty), Val, IdxList));
+}
+
 LLVMValueRef LLVMConstTrunc(LLVMValueRef ConstantVal, LLVMTypeRef ToType) {
   return wrap(ConstantExpr::getTrunc(unwrap<Constant>(ConstantVal),
                                      unwrap(ToType)));
@@ -3007,13 +3024,17 @@ LLVMTypeRef LLVMGetAllocatedType(LLVMValueRef Alloca) {
 /*--.. Operations on gep instructions (only) ...............................--*/
 
 LLVMBool LLVMIsInBounds(LLVMValueRef GEP) {
-  return unwrap<GetElementPtrInst>(GEP)->isInBounds();
+  return unwrap<GEPOperator>(GEP)->isInBounds();
 }
 
 void LLVMSetIsInBounds(LLVMValueRef GEP, LLVMBool InBounds) {
   return unwrap<GetElementPtrInst>(GEP)->setIsInBounds(InBounds);
 }
 
+LLVMTypeRef LLVMGetGEPSourceElementType(LLVMValueRef GEP) {
+  return wrap(unwrap<GEPOperator>(GEP)->getSourceElementType());
+}
+
 /*--.. Operations on phi nodes .............................................--*/
 
 void LLVMAddIncoming(LLVMValueRef PhiNode, LLVMValueRef *IncomingValues,
@@ -3039,7 +3060,7 @@ LLVMBasicBlockRef LLVMGetIncomingBlock(LLVMValueRef PhiNode, unsigned Index) {
 
 unsigned LLVMGetNumIndices(LLVMValueRef Inst) {
   auto *I = unwrap(Inst);
-  if (auto *GEP = dyn_cast<GetElementPtrInst>(I))
+  if (auto *GEP = dyn_cast<GEPOperator>(I))
     return GEP->getNumIndices();
   if (auto *EV = dyn_cast<ExtractValueInst>(I))
     return EV->getNumIndices();
diff --git a/llvm/lib/IR/DIBuilder.cpp b/llvm/lib/IR/DIBuilder.cpp
index 548962bd6a98..35af22034a12 100644
--- a/llvm/lib/IR/DIBuilder.cpp
+++ b/llvm/lib/IR/DIBuilder.cpp
@@ -671,11 +671,11 @@ DIBuilder::getOrCreateMacroArray(ArrayRef<Metadata *> Elements) {
 
 DITypeRefArray DIBuilder::getOrCreateTypeArray(ArrayRef<Metadata *> Elements) {
   SmallVector<llvm::Metadata *, 16> Elts;
-  for (unsigned i = 0, e = Elements.size(); i != e; ++i) {
-    if (Elements[i] && isa<MDNode>(Elements[i]))
-      Elts.push_back(cast<DIType>(Elements[i]));
+  for (Metadata *E : Elements) {
+    if (isa_and_nonnull<MDNode>(E))
+      Elts.push_back(cast<DIType>(E));
     else
-      Elts.push_back(Elements[i]);
+      Elts.push_back(E);
   }
   return DITypeRefArray(MDNode::get(VMContext, Elts));
 }
diff --git a/llvm/lib/IR/DataLayout.cpp b/llvm/lib/IR/DataLayout.cpp
index 2ace18048262..61b2b13bfd03 100644
--- a/llvm/lib/IR/DataLayout.cpp
+++ b/llvm/lib/IR/DataLayout.cpp
@@ -124,26 +124,25 @@ LayoutAlignElem::operator==(const LayoutAlignElem &rhs) const {
 // PointerAlignElem, PointerAlign support
 //===----------------------------------------------------------------------===//
 
-PointerAlignElem PointerAlignElem::get(uint32_t AddressSpace, Align ABIAlign,
-                                       Align PrefAlign, uint32_t TypeByteWidth,
-                                       uint32_t IndexWidth) {
+PointerAlignElem PointerAlignElem::getInBits(uint32_t AddressSpace,
+                                             Align ABIAlign, Align PrefAlign,
+                                             uint32_t TypeBitWidth,
+                                             uint32_t IndexBitWidth) {
   assert(ABIAlign <= PrefAlign && "Preferred alignment worse than ABI!");
   PointerAlignElem retval;
   retval.AddressSpace = AddressSpace;
   retval.ABIAlign = ABIAlign;
   retval.PrefAlign = PrefAlign;
-  retval.TypeByteWidth = TypeByteWidth;
-  retval.IndexWidth = IndexWidth;
+  retval.TypeBitWidth = TypeBitWidth;
+  retval.IndexBitWidth = IndexBitWidth;
   return retval;
 }
 
 bool
 PointerAlignElem::operator==(const PointerAlignElem &rhs) const {
-  return (ABIAlign == rhs.ABIAlign
-          && AddressSpace == rhs.AddressSpace
-          && PrefAlign == rhs.PrefAlign
-          && TypeByteWidth == rhs.TypeByteWidth
-          && IndexWidth == rhs.IndexWidth);
+  return (ABIAlign == rhs.ABIAlign && AddressSpace == rhs.AddressSpace &&
+          PrefAlign == rhs.PrefAlign && TypeBitWidth == rhs.TypeBitWidth &&
+          IndexBitWidth == rhs.IndexBitWidth);
 }
 
 //===----------------------------------------------------------------------===//
@@ -197,7 +196,7 @@ void DataLayout::reset(StringRef Desc) {
                                  E.PrefAlign, E.TypeBitWidth))
       return report_fatal_error(std::move(Err));
   }
-  if (Error Err = setPointerAlignment(0, Align(8), Align(8), 8, 8))
+  if (Error Err = setPointerAlignmentInBits(0, Align(8), Align(8), 64, 64))
     return report_fatal_error(std::move(Err));
 
   if (Error Err = parseSpecifier(Desc))
@@ -318,7 +317,7 @@ Error DataLayout::parseSpecifier(StringRef Desc) {
       if (Error Err = ::split(Rest, ':', Split))
         return Err;
       unsigned PointerMemSize;
-      if (Error Err = getIntInBytes(Tok, PointerMemSize))
+      if (Error Err = getInt(Tok, PointerMemSize))
         return Err;
       if (!PointerMemSize)
         return reportError("Invalid pointer size of 0 bytes");
@@ -354,13 +353,13 @@ Error DataLayout::parseSpecifier(StringRef Desc) {
         if (!Rest.empty()) {
           if (Error Err = ::split(Rest, ':', Split))
             return Err;
-          if (Error Err = getIntInBytes(Tok, IndexSize))
+          if (Error Err = getInt(Tok, IndexSize))
             return Err;
           if (!IndexSize)
             return reportError("Invalid index size of 0 bytes");
         }
       }
-      if (Error Err = setPointerAlignment(
+      if (Error Err = setPointerAlignmentInBits(
               AddrSpace, assumeAligned(PointerABIAlign),
               assumeAligned(PointerPrefAlign), PointerMemSize, IndexSize))
         return Err;
@@ -603,9 +602,10 @@ DataLayout::getPointerAlignElem(uint32_t AddressSpace) const {
   return Pointers[0];
 }
 
-Error DataLayout::setPointerAlignment(uint32_t AddrSpace, Align ABIAlign,
-                                      Align PrefAlign, uint32_t TypeByteWidth,
-                                      uint32_t IndexWidth) {
+Error DataLayout::setPointerAlignmentInBits(uint32_t AddrSpace, Align ABIAlign,
+                                            Align PrefAlign,
+                                            uint32_t TypeBitWidth,
+                                            uint32_t IndexBitWidth) {
   if (PrefAlign < ABIAlign)
     return reportError(
         "Preferred alignment cannot be less than the ABI alignment");
@@ -615,13 +615,14 @@ Error DataLayout::setPointerAlignment(uint32_t AddrSpace, Align ABIAlign,
     return A.AddressSpace < AddressSpace;
   });
   if (I == Pointers.end() || I->AddressSpace != AddrSpace) {
-    Pointers.insert(I, PointerAlignElem::get(AddrSpace, ABIAlign, PrefAlign,
-                                             TypeByteWidth, IndexWidth));
+    Pointers.insert(I,
+                    PointerAlignElem::getInBits(AddrSpace, ABIAlign, PrefAlign,
+                                                TypeBitWidth, IndexBitWidth));
   } else {
     I->ABIAlign = ABIAlign;
     I->PrefAlign = PrefAlign;
-    I->TypeByteWidth = TypeByteWidth;
-    I->IndexWidth = IndexWidth;
+    I->TypeBitWidth = TypeBitWidth;
+    I->IndexBitWidth = IndexBitWidth;
   }
   return Error::success();
 }
@@ -704,13 +705,14 @@ Align DataLayout::getPointerPrefAlignment(unsigned AS) const {
 }
 
 unsigned DataLayout::getPointerSize(unsigned AS) const {
-  return getPointerAlignElem(AS).TypeByteWidth;
+  return divideCeil(getPointerAlignElem(AS).TypeBitWidth, 8);
 }
 
 unsigned DataLayout::getMaxIndexSize() const {
   unsigned MaxIndexSize = 0;
   for (auto &P : Pointers)
-    MaxIndexSize = std::max(MaxIndexSize, P.IndexWidth);
+    MaxIndexSize =
+        std::max(MaxIndexSize, (unsigned)divideCeil(P.TypeBitWidth, 8));
 
   return MaxIndexSize;
 }
@@ -723,7 +725,7 @@ unsigned DataLayout::getPointerTypeSizeInBits(Type *Ty) const {
 }
 
 unsigned DataLayout::getIndexSize(unsigned AS) const {
-  return getPointerAlignElem(AS).IndexWidth;
+  return divideCeil(getPointerAlignElem(AS).IndexBitWidth, 8);
 }
 
 unsigned DataLayout::getIndexTypeSizeInBits(Type *Ty) const {
@@ -901,16 +903,14 @@ int64_t DataLayout::getIndexedOffsetInType(Type *ElemTy,
   return Result;
 }
 
-static void addElementIndex(SmallVectorImpl<APInt> &Indices, TypeSize ElemSize,
-                            APInt &Offset) {
+static APInt getElementIndex(TypeSize ElemSize, APInt &Offset) {
   // Skip over scalable or zero size elements. Also skip element sizes larger
   // than the positive index space, because the arithmetic below may not be
   // correct in that case.
   unsigned BitWidth = Offset.getBitWidth();
   if (ElemSize.isScalable() || ElemSize == 0 ||
       !isUIntN(BitWidth - 1, ElemSize)) {
-    Indices.push_back(APInt::getZero(BitWidth));
-    return;
+    return APInt::getZero(BitWidth);
   }
 
   APInt Index = Offset.sdiv(ElemSize);
@@ -921,47 +921,52 @@ static void addElementIndex(SmallVectorImpl<APInt> &Indices, TypeSize ElemSize,
     Offset += ElemSize;
     assert(Offset.isNonNegative() && "Remaining offset shouldn't be negative");
   }
-  Indices.push_back(Index);
+  return Index;
 }
 
-SmallVector<APInt> DataLayout::getGEPIndicesForOffset(Type *&ElemTy,
-                                                      APInt &Offset) const {
-  assert(ElemTy->isSized() && "Element type must be sized");
-  SmallVector<APInt> Indices;
-  addElementIndex(Indices, getTypeAllocSize(ElemTy), Offset);
-  while (Offset != 0) {
-    if (auto *ArrTy = dyn_cast<ArrayType>(ElemTy)) {
-      ElemTy = ArrTy->getElementType();
-      addElementIndex(Indices, getTypeAllocSize(ElemTy), Offset);
-      continue;
-    }
+Optional<APInt> DataLayout::getGEPIndexForOffset(Type *&ElemTy,
+                                                 APInt &Offset) const {
+  if (auto *ArrTy = dyn_cast<ArrayType>(ElemTy)) {
+    ElemTy = ArrTy->getElementType();
+    return getElementIndex(getTypeAllocSize(ElemTy), Offset);
+  }
 
-    if (auto *VecTy = dyn_cast<VectorType>(ElemTy)) {
-      ElemTy = VecTy->getElementType();
-      unsigned ElemSizeInBits = getTypeSizeInBits(ElemTy).getFixedSize();
-      // GEPs over non-multiple of 8 size vector elements are invalid.
-      if (ElemSizeInBits % 8 != 0)
-        break;
+  if (auto *VecTy = dyn_cast<VectorType>(ElemTy)) {
+    ElemTy = VecTy->getElementType();
+    unsigned ElemSizeInBits = getTypeSizeInBits(ElemTy).getFixedSize();
+    // GEPs over non-multiple of 8 size vector elements are invalid.
+    if (ElemSizeInBits % 8 != 0)
+      return None;
 
-      addElementIndex(Indices, TypeSize::Fixed(ElemSizeInBits / 8), Offset);
-      continue;
-    }
+    return getElementIndex(TypeSize::Fixed(ElemSizeInBits / 8), Offset);
+  }
 
-    if (auto *STy = dyn_cast<StructType>(ElemTy)) {
-      const StructLayout *SL = getStructLayout(STy);
-      uint64_t IntOffset = Offset.getZExtValue();
-      if (IntOffset >= SL->getSizeInBytes())
-        break;
+  if (auto *STy = dyn_cast<StructType>(ElemTy)) {
+    const StructLayout *SL = getStructLayout(STy);
+    uint64_t IntOffset = Offset.getZExtValue();
+    if (IntOffset >= SL->getSizeInBytes())
+      return None;
 
-      unsigned Index = SL->getElementContainingOffset(IntOffset);
-      Offset -= SL->getElementOffset(Index);
-      ElemTy = STy->getElementType(Index);
-      Indices.push_back(APInt(32, Index));
-      continue;
-    }
+    unsigned Index = SL->getElementContainingOffset(IntOffset);
+    Offset -= SL->getElementOffset(Index);
+    ElemTy = STy->getElementType(Index);
+    return APInt(32, Index);
+  }
+
+  // Non-aggregate type.
+  return None;
+}
 
-    // Can't index into non-aggregate type.
-    break;
+SmallVector<APInt> DataLayout::getGEPIndicesForOffset(Type *&ElemTy,
+                                                      APInt &Offset) const {
+  assert(ElemTy->isSized() && "Element type must be sized");
+  SmallVector<APInt> Indices;
+  Indices.push_back(getElementIndex(getTypeAllocSize(ElemTy), Offset));
+  while (Offset != 0) {
+    Optional<APInt> Index = getGEPIndexForOffset(ElemTy, Offset);
+    if (!Index)
+      break;
+    Indices.push_back(*Index);
   }
 
   return Indices;
diff --git a/llvm/lib/IR/Function.cpp b/llvm/lib/IR/Function.cpp
index 82b20a8af91b..f1a6402fb11b 100644
--- a/llvm/lib/IR/Function.cpp
+++ b/llvm/lib/IR/Function.cpp
@@ -980,7 +980,10 @@ enum IIT_Info {
   IIT_STRUCT9 = 49,
   IIT_V256 = 50,
   IIT_AMX  = 51,
-  IIT_PPCF128 = 52
+  IIT_PPCF128 = 52,
+  IIT_V3 = 53,
+  IIT_EXTERNREF = 54,
+  IIT_FUNCREF = 55
 };
 
 static void DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos,
@@ -1056,6 +1059,10 @@ static void DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos,
     OutputTable.push_back(IITDescriptor::getVector(2, IsScalableVector));
     DecodeIITType(NextElt, Infos, Info, OutputTable);
     return;
+  case IIT_V3:
+    OutputTable.push_back(IITDescriptor::getVector(3, IsScalableVector));
+    DecodeIITType(NextElt, Infos, Info, OutputTable);
+    return;
   case IIT_V4:
     OutputTable.push_back(IITDescriptor::getVector(4, IsScalableVector));
     DecodeIITType(NextElt, Infos, Info, OutputTable);
@@ -1092,6 +1099,14 @@ static void DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos,
     OutputTable.push_back(IITDescriptor::getVector(1024, IsScalableVector));
     DecodeIITType(NextElt, Infos, Info, OutputTable);
     return;
+  case IIT_EXTERNREF:
+    OutputTable.push_back(IITDescriptor::get(IITDescriptor::Pointer, 10));
+    OutputTable.push_back(IITDescriptor::get(IITDescriptor::Struct, 0));
+    return;
+  case IIT_FUNCREF:
+    OutputTable.push_back(IITDescriptor::get(IITDescriptor::Pointer, 20));
+    OutputTable.push_back(IITDescriptor::get(IITDescriptor::Integer, 8));
+    return;
   case IIT_PTR:
     OutputTable.push_back(IITDescriptor::get(IITDescriptor::Pointer, 0));
     DecodeIITType(NextElt, Infos, Info, OutputTable);
diff --git a/llvm/lib/IR/Globals.cpp b/llvm/lib/IR/Globals.cpp
index 9f38288095e3..b6bd25aa1234 100644
--- a/llvm/lib/IR/Globals.cpp
+++ b/llvm/lib/IR/Globals.cpp
@@ -126,7 +126,7 @@ void GlobalObject::setAlignment(MaybeAlign Align) {
 
 void GlobalObject::copyAttributesFrom(const GlobalObject *Src) {
   GlobalValue::copyAttributesFrom(Src);
-  setAlignment(MaybeAlign(Src->getAlignment()));
+  setAlignment(Src->getAlign());
   setSection(Src->getSection());
 }
 
@@ -249,7 +249,7 @@ bool GlobalObject::canIncreaseAlignment() const {
   // alignment specified. (If it is assigned a section, the global
   // could be densely packed with other objects in the section, and
   // increasing the alignment could cause padding issues.)
-  if (hasSection() && getAlignment() > 0)
+  if (hasSection() && getAlign().hasValue())
     return false;
 
   // On ELF platforms, we're further restricted in that we can't
diff --git a/llvm/lib/IR/InlineAsm.cpp b/llvm/lib/IR/InlineAsm.cpp
index 56932b457225..a0c48781ced5 100644
--- a/llvm/lib/IR/InlineAsm.cpp
+++ b/llvm/lib/IR/InlineAsm.cpp
@@ -262,12 +262,12 @@ bool InlineAsm::Verify(FunctionType *Ty, StringRef ConstStr) {
   unsigned NumOutputs = 0, NumInputs = 0, NumClobbers = 0;
   unsigned NumIndirect = 0;
 
-  for (unsigned i = 0, e = Constraints.size(); i != e; ++i) {
-    switch (Constraints[i].Type) {
+  for (const ConstraintInfo &Constraint : Constraints) {
+    switch (Constraint.Type) {
     case InlineAsm::isOutput:
       if ((NumInputs-NumIndirect) != 0 || NumClobbers != 0)
         return false;  // outputs before inputs and clobbers.
-      if (!Constraints[i].isIndirect) {
+      if (!Constraint.isIndirect) {
         ++NumOutputs;
         break;
       }
diff --git a/llvm/lib/IR/Instruction.cpp b/llvm/lib/IR/Instruction.cpp
index a4659da7e807..4480ec799c35 100644
--- a/llvm/lib/IR/Instruction.cpp
+++ b/llvm/lib/IR/Instruction.cpp
@@ -166,7 +166,10 @@ void Instruction::dropPoisonGeneratingFlags() {
     cast<GetElementPtrInst>(this)->setIsInBounds(false);
     break;
   }
-  // TODO: FastMathFlags!
+  if (isa<FPMathOperator>(this)) {
+    setHasNoNaNs(false);
+    setHasNoInfs(false);
+  }
 
   assert(!hasPoisonGeneratingFlags() && "must be kept in sync");
 }
@@ -436,17 +439,17 @@ static bool haveSameSpecialState(const Instruction *I1, const Instruction *I2,
 
   if (const AllocaInst *AI = dyn_cast<AllocaInst>(I1))
     return AI->getAllocatedType() == cast<AllocaInst>(I2)->getAllocatedType() &&
-           (AI->getAlignment() == cast<AllocaInst>(I2)->getAlignment() ||
+           (AI->getAlign() == cast<AllocaInst>(I2)->getAlign() ||
             IgnoreAlignment);
   if (const LoadInst *LI = dyn_cast<LoadInst>(I1))
     return LI->isVolatile() == cast<LoadInst>(I2)->isVolatile() &&
-           (LI->getAlignment() == cast<LoadInst>(I2)->getAlignment() ||
+           (LI->getAlign() == cast<LoadInst>(I2)->getAlign() ||
             IgnoreAlignment) &&
            LI->getOrdering() == cast<LoadInst>(I2)->getOrdering() &&
            LI->getSyncScopeID() == cast<LoadInst>(I2)->getSyncScopeID();
   if (const StoreInst *SI = dyn_cast<StoreInst>(I1))
     return SI->isVolatile() == cast<StoreInst>(I2)->isVolatile() &&
-           (SI->getAlignment() == cast<StoreInst>(I2)->getAlignment() ||
+           (SI->getAlign() == cast<StoreInst>(I2)->getAlign() ||
             IgnoreAlignment) &&
            SI->getOrdering() == cast<StoreInst>(I2)->getOrdering() &&
            SI->getSyncScopeID() == cast<StoreInst>(I2)->getSyncScopeID();
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index ad27a6d8c08e..7798af3b19b9 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -1410,8 +1410,6 @@ bool AllocaInst::isStaticAlloca() const {
 void LoadInst::AssertOK() {
   assert(getOperand(0)->getType()->isPointerTy() &&
          "Ptr must have pointer type.");
-  assert(!(isAtomic() && getAlignment() == 0) &&
-         "Alignment required for atomic load");
 }
 
 static Align computeLoadStoreDefaultAlign(Type *Ty, BasicBlock *BB) {
@@ -1490,8 +1488,6 @@ void StoreInst::AssertOK() {
   assert(cast<PointerType>(getOperand(1)->getType())
              ->isOpaqueOrPointeeTypeMatches(getOperand(0)->getType()) &&
          "Ptr must be a pointer to Val type!");
-  assert(!(isAtomic() && getAlignment() == 0) &&
-         "Alignment required for atomic store");
 }
 
 StoreInst::StoreInst(Value *val, Value *addr, Instruction *InsertBefore)
@@ -2328,7 +2324,6 @@ bool ShuffleVectorInst::isInsertSubvectorMask(ArrayRef<int> Mask,
     }
     Src1Elts.setBit(i);
     Src1Identity &= (M == (i + NumSrcElts));
-    continue;
   }
   assert((Src0Elts | Src1Elts | UndefElts).isAllOnes() &&
          "unknown shuffle elements");
@@ -4165,6 +4160,47 @@ bool ICmpInst::compare(const APInt &LHS, const APInt &RHS,
   };
 }
 
+bool FCmpInst::compare(const APFloat &LHS, const APFloat &RHS,
+                       FCmpInst::Predicate Pred) {
+  APFloat::cmpResult R = LHS.compare(RHS);
+  switch (Pred) {
+  default:
+    llvm_unreachable("Invalid FCmp Predicate");
+  case FCmpInst::FCMP_FALSE:
+    return false;
+  case FCmpInst::FCMP_TRUE:
+    return true;
+  case FCmpInst::FCMP_UNO:
+    return R == APFloat::cmpUnordered;
+  case FCmpInst::FCMP_ORD:
+    return R != APFloat::cmpUnordered;
+  case FCmpInst::FCMP_UEQ:
+    return R == APFloat::cmpUnordered || R == APFloat::cmpEqual;
+  case FCmpInst::FCMP_OEQ:
+    return R == APFloat::cmpEqual;
+  case FCmpInst::FCMP_UNE:
+    return R != APFloat::cmpEqual;
+  case FCmpInst::FCMP_ONE:
+    return R == APFloat::cmpLessThan || R == APFloat::cmpGreaterThan;
+  case FCmpInst::FCMP_ULT:
+    return R == APFloat::cmpUnordered || R == APFloat::cmpLessThan;
+  case FCmpInst::FCMP_OLT:
+    return R == APFloat::cmpLessThan;
+  case FCmpInst::FCMP_UGT:
+    return R == APFloat::cmpUnordered || R == APFloat::cmpGreaterThan;
+  case FCmpInst::FCMP_OGT:
+    return R == APFloat::cmpGreaterThan;
+  case FCmpInst::FCMP_ULE:
+    return R != APFloat::cmpGreaterThan;
+  case FCmpInst::FCMP_OLE:
+    return R == APFloat::cmpLessThan || R == APFloat::cmpEqual;
+  case FCmpInst::FCMP_UGE:
+    return R != APFloat::cmpLessThan;
+  case FCmpInst::FCMP_OGE:
+    return R == APFloat::cmpGreaterThan || R == APFloat::cmpEqual;
+  }
+}
+
 CmpInst::Predicate CmpInst::getFlippedSignednessPredicate(Predicate pred) {
   assert(CmpInst::isRelational(pred) &&
          "Call only with non-equality predicates!");
@@ -4411,7 +4447,7 @@ void SwitchInstProfUpdateWrapper::addCase(
     Weights.getValue()[SI.getNumSuccessors() - 1] = *W;
   } else if (Weights) {
     Changed = true;
-    Weights.getValue().push_back(W ? *W : 0);
+    Weights.getValue().push_back(W.getValueOr(0));
   }
   if (Weights)
     assert(SI.getNumSuccessors() == Weights->size() &&
diff --git a/llvm/lib/IR/IntrinsicInst.cpp b/llvm/lib/IR/IntrinsicInst.cpp
index 9206cd37a6d1..8f7318665cfb 100644
--- a/llvm/lib/IR/IntrinsicInst.cpp
+++ b/llvm/lib/IR/IntrinsicInst.cpp
@@ -468,6 +468,7 @@ bool VPIntrinsic::canIgnoreVectorLengthParam() const {
 }
 
 Function *VPIntrinsic::getDeclarationForParams(Module *M, Intrinsic::ID VPID,
+                                               Type *ReturnType,
                                                ArrayRef<Value *> Params) {
   assert(isVPIntrinsic(VPID) && "not a VP intrinsic");
   Function *VPFunc;
@@ -486,22 +487,15 @@ Function *VPIntrinsic::getDeclarationForParams(Module *M, Intrinsic::ID VPID,
     break;
   case Intrinsic::vp_load:
     VPFunc = Intrinsic::getDeclaration(
-        M, VPID,
-        {Params[0]->getType()->getPointerElementType(), Params[0]->getType()});
+        M, VPID, {ReturnType, Params[0]->getType()});
     break;
   case Intrinsic::vp_gather:
     VPFunc = Intrinsic::getDeclaration(
-        M, VPID,
-        {VectorType::get(cast<VectorType>(Params[0]->getType())
-                             ->getElementType()
-                             ->getPointerElementType(),
-                         cast<VectorType>(Params[0]->getType())),
-         Params[0]->getType()});
+        M, VPID, {ReturnType, Params[0]->getType()});
     break;
   case Intrinsic::vp_store:
     VPFunc = Intrinsic::getDeclaration(
-        M, VPID,
-        {Params[1]->getType()->getPointerElementType(), Params[1]->getType()});
+        M, VPID, {Params[0]->getType(), Params[1]->getType()});
     break;
   case Intrinsic::vp_scatter:
     VPFunc = Intrinsic::getDeclaration(
diff --git a/llvm/lib/IR/LLVMContextImpl.h b/llvm/lib/IR/LLVMContextImpl.h
index b2909c425846..24c4a348f4da 100644
--- a/llvm/lib/IR/LLVMContextImpl.h
+++ b/llvm/lib/IR/LLVMContextImpl.h
@@ -386,8 +386,9 @@ template <> struct MDNodeKeyImpl<DIEnumerator> {
         IsUnsigned(N->isUnsigned()) {}
 
   bool isKeyOf(const DIEnumerator *RHS) const {
-    return APInt::isSameValue(Value, RHS->getValue()) &&
-           IsUnsigned == RHS->isUnsigned() && Name == RHS->getRawName();
+    return Value.getBitWidth() == RHS->getValue().getBitWidth() &&
+           Value == RHS->getValue() && IsUnsigned == RHS->isUnsigned() &&
+           Name == RHS->getRawName();
   }
 
   unsigned getHashValue() const { return hash_combine(Value, Name); }
@@ -1424,6 +1425,8 @@ public:
 
   DenseMap<const GlobalValue *, DSOLocalEquivalent *> DSOLocalEquivalents;
 
+  DenseMap<const GlobalValue *, NoCFIValue *> NoCFIValues;
+
   ConstantUniqueMap<ConstantExpr> ExprConstants;
 
   ConstantUniqueMap<InlineAsm> InlineAsms;
diff --git a/llvm/lib/IR/LegacyPassManager.cpp b/llvm/lib/IR/LegacyPassManager.cpp
index 7bccf09012ca..bb72bec93066 100644
--- a/llvm/lib/IR/LegacyPassManager.cpp
+++ b/llvm/lib/IR/LegacyPassManager.cpp
@@ -886,9 +886,8 @@ void PMDataManager::recordAvailableAnalysis(Pass *P) {
   // implements as well.
   const PassInfo *PInf = TPM->findAnalysisPassInfo(PI);
   if (!PInf) return;
-  const std::vector<const PassInfo*> &II = PInf->getInterfacesImplemented();
-  for (unsigned i = 0, e = II.size(); i != e; ++i)
-    AvailableAnalysis[II[i]->getTypeInfo()] = P;
+  for (const PassInfo *PI : PInf->getInterfacesImplemented())
+    AvailableAnalysis[PI->getTypeInfo()] = P;
 }
 
 // Return true if P preserves high level analysis used by other
@@ -1013,10 +1012,9 @@ void PMDataManager::freePass(Pass *P, StringRef Msg,
 
     // Remove all interfaces this pass implements, for which it is also
     // listed as the available implementation.
-    const std::vector<const PassInfo*> &II = PInf->getInterfacesImplemented();
-    for (unsigned i = 0, e = II.size(); i != e; ++i) {
-      DenseMap<AnalysisID, Pass*>::iterator Pos =
-        AvailableAnalysis.find(II[i]->getTypeInfo());
+    for (const PassInfo *PI : PInf->getInterfacesImplemented()) {
+      DenseMap<AnalysisID, Pass *>::iterator Pos =
+          AvailableAnalysis.find(PI->getTypeInfo());
       if (Pos != AvailableAnalysis.end() && Pos->second == P)
         AvailableAnalysis.erase(Pos);
     }
diff --git a/llvm/lib/IR/Module.cpp b/llvm/lib/IR/Module.cpp
index 63ea41fba89a..a0485a59d0e0 100644
--- a/llvm/lib/IR/Module.cpp
+++ b/llvm/lib/IR/Module.cpp
@@ -750,8 +750,8 @@ void Module::setSDKVersion(const VersionTuple &V) {
                 ConstantDataArray::get(Context, Entries));
 }
 
-VersionTuple Module::getSDKVersion() const {
-  auto *CM = dyn_cast_or_null<ConstantAsMetadata>(getModuleFlag("SDK Version"));
+static VersionTuple getSDKVersionMD(Metadata *MD) {
+  auto *CM = dyn_cast_or_null<ConstantAsMetadata>(MD);
   if (!CM)
     return {};
   auto *Arr = dyn_cast_or_null<ConstantDataArray>(CM->getValue());
@@ -775,6 +775,10 @@ VersionTuple Module::getSDKVersion() const {
   return Result;
 }
 
+VersionTuple Module::getSDKVersion() const {
+  return getSDKVersionMD(getModuleFlag("SDK Version"));
+}
+
 GlobalVariable *llvm::collectUsedGlobalVariables(
     const Module &M, SmallVectorImpl<GlobalValue *> &Vec, bool CompilerUsed) {
   const char *Name = CompilerUsed ? "llvm.compiler.used" : "llvm.used";
@@ -809,3 +813,13 @@ void Module::setPartialSampleProfileRatio(const ModuleSummaryIndex &Index) {
     }
   }
 }
+
+StringRef Module::getDarwinTargetVariantTriple() const {
+  if (const auto *MD = getModuleFlag("darwin.target_variant.triple"))
+    return cast<MDString>(MD)->getString();
+  return "";
+}
+
+VersionTuple Module::getDarwinTargetVariantSDKVersion() const {
+  return getSDKVersionMD(getModuleFlag("darwin.target_variant.SDK Version"));
+}
diff --git a/llvm/lib/IR/ModuleSummaryIndex.cpp b/llvm/lib/IR/ModuleSummaryIndex.cpp
index 31c5cd938d03..a0ac7d3ad7d3 100644
--- a/llvm/lib/IR/ModuleSummaryIndex.cpp
+++ b/llvm/lib/IR/ModuleSummaryIndex.cpp
@@ -447,11 +447,17 @@ static std::string linkageToString(GlobalValue::LinkageTypes LT) {
 
 static std::string fflagsToString(FunctionSummary::FFlags F) {
   auto FlagValue = [](unsigned V) { return V ? '1' : '0'; };
-  char FlagRep[] = {FlagValue(F.ReadNone),  FlagValue(F.ReadOnly),
-                    FlagValue(F.NoRecurse), FlagValue(F.ReturnDoesNotAlias),
-                    FlagValue(F.NoInline),  FlagValue(F.AlwaysInline),
-                    FlagValue(F.NoUnwind),  FlagValue(F.MayThrow), 
-                    FlagValue(F.HasUnknownCall), 0};
+  char FlagRep[] = {FlagValue(F.ReadNone),
+                    FlagValue(F.ReadOnly),
+                    FlagValue(F.NoRecurse),
+                    FlagValue(F.ReturnDoesNotAlias),
+                    FlagValue(F.NoInline),
+                    FlagValue(F.AlwaysInline),
+                    FlagValue(F.NoUnwind),
+                    FlagValue(F.MayThrow),
+                    FlagValue(F.HasUnknownCall),
+                    FlagValue(F.MustBeUnreachable),
+                    0};
 
   return FlagRep;
 }
diff --git a/llvm/lib/IR/Operator.cpp b/llvm/lib/IR/Operator.cpp
index d15fcfbc5b9f..08c1fc931e2e 100644
--- a/llvm/lib/IR/Operator.cpp
+++ b/llvm/lib/IR/Operator.cpp
@@ -39,9 +39,10 @@ bool Operator::hasPoisonGeneratingFlags() const {
     return GEP->isInBounds() || GEP->getInRangeIndex() != None;
   }
   default:
+    if (const auto *FP = dyn_cast<FPMathOperator>(this))
+      return FP->hasNoNaNs() || FP->hasNoInfs();
     return false;
   }
-  // TODO: FastMathFlags!  (On instructions, but not constexpr)
 }
 
 Type *GEPOperator::getSourceElementType() const {
@@ -89,7 +90,7 @@ bool GEPOperator::accumulateConstantOffset(
   assert(Offset.getBitWidth() ==
              DL.getIndexSizeInBits(getPointerAddressSpace()) &&
          "The offset bit width does not match DL specification.");
-  SmallVector<const Value *> Index(value_op_begin() + 1, value_op_end());
+  SmallVector<const Value *> Index(llvm::drop_begin(operand_values()));
   return GEPOperator::accumulateConstantOffset(getSourceElementType(), Index,
                                                DL, Offset, ExternalAnalysis);
 }
diff --git a/llvm/lib/IR/SSAContext.cpp b/llvm/lib/IR/SSAContext.cpp
new file mode 100644
index 000000000000..a96e39f32882
--- /dev/null
+++ b/llvm/lib/IR/SSAContext.cpp
@@ -0,0 +1,47 @@
+//===- SSAContext.cpp -------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+///
+/// This file defines a specialization of the GenericSSAContext<X>
+/// template class for LLVM IR.
+///
+//===----------------------------------------------------------------------===//
+
+#include "llvm/IR/SSAContext.h"
+#include "llvm/IR/Argument.h"
+#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/Instruction.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+
+BasicBlock *SSAContext::getEntryBlock(Function &F) {
+  return &F.getEntryBlock();
+}
+
+void SSAContext::setFunction(Function &Fn) { F = &Fn; }
+
+Printable SSAContext::print(Value *V) const {
+  return Printable([V](raw_ostream &Out) { V->print(Out); });
+}
+
+Printable SSAContext::print(Instruction *Inst) const {
+  return print(cast<Value>(Inst));
+}
+
+Printable SSAContext::print(BasicBlock *BB) const {
+  if (BB->hasName())
+    return Printable([BB](raw_ostream &Out) { Out << BB->getName(); });
+
+  return Printable([BB](raw_ostream &Out) {
+    ModuleSlotTracker MST{BB->getParent()->getParent(), false};
+    MST.incorporateFunction(*BB->getParent());
+    Out << MST.getLocalSlot(BB);
+  });
+}
diff --git a/llvm/lib/IR/Value.cpp b/llvm/lib/IR/Value.cpp
index b475c8327874..8741ed917f9f 100644
--- a/llvm/lib/IR/Value.cpp
+++ b/llvm/lib/IR/Value.cpp
@@ -928,7 +928,7 @@ Align Value::getPointerAlignment(const DataLayout &DL) const {
       }
       llvm_unreachable("Unhandled FunctionPtrAlignType");
     }
-    const MaybeAlign Alignment(GO->getAlignment());
+    const MaybeAlign Alignment(GO->getAlign());
     if (!Alignment) {
       if (auto *GVar = dyn_cast<GlobalVariable>(GO)) {
         Type *ObjectType = GVar->getValueType();
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 154b59835b01..fb7c423e54e2 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -543,7 +543,7 @@ private:
 
   void verifySwiftErrorCall(CallBase &Call, const Value *SwiftErrorVal);
   void verifySwiftErrorValue(const Value *SwiftErrorVal);
-  void verifyTailCCMustTailAttrs(AttrBuilder Attrs, StringRef Context);
+  void verifyTailCCMustTailAttrs(const AttrBuilder &Attrs, StringRef Context);
   void verifyMustTailCall(CallInst &CI);
   bool verifyAttributeCount(AttributeList Attrs, unsigned Params);
   void verifyAttributeTypes(AttributeSet Attrs, const Value *V);
@@ -553,8 +553,6 @@ private:
   void verifyFunctionAttrs(FunctionType *FT, AttributeList Attrs,
                            const Value *V, bool IsIntrinsic);
   void verifyFunctionMetadata(ArrayRef<std::pair<unsigned, MDNode *>> MDs);
-  template <typename T>
-  void verifyODRTypeAsScopeOperand(const MDNode &MD, T * = nullptr);
 
   void visitConstantExprsRecursively(const Constant *EntryC);
   void visitConstantExpr(const ConstantExpr *CE);
@@ -604,26 +602,35 @@ void Verifier::visit(Instruction &I) {
   InstVisitor<Verifier>::visit(I);
 }
 
-// Helper to recursively iterate over indirect users. By
-// returning false, the callback can ask to stop recursing
-// further.
+// Helper to iterate over indirect users. By returning false, the callback can ask to stop traversing further.
 static void forEachUser(const Value *User,
                         SmallPtrSet<const Value *, 32> &Visited,
                         llvm::function_ref<bool(const Value *)> Callback) {
   if (!Visited.insert(User).second)
     return;
-  for (const Value *TheNextUser : User->materialized_users())
-    if (Callback(TheNextUser))
-      forEachUser(TheNextUser, Visited, Callback);
+
+  SmallVector<const Value *> WorkList;
+  append_range(WorkList, User->materialized_users());
+  while (!WorkList.empty()) {
+   const Value *Cur = WorkList.pop_back_val();
+    if (!Visited.insert(Cur).second)
+      continue;
+    if (Callback(Cur))
+      append_range(WorkList, Cur->materialized_users());
+  }
 }
 
 void Verifier::visitGlobalValue(const GlobalValue &GV) {
   Assert(!GV.isDeclaration() || GV.hasValidDeclarationLinkage(),
          "Global is external, but doesn't have external or weak linkage!", &GV);
 
-  if (const GlobalObject *GO = dyn_cast<GlobalObject>(&GV))
-    Assert(GO->getAlignment() <= Value::MaximumAlignment,
-           "huge alignment values are unsupported", GO);
+  if (const GlobalObject *GO = dyn_cast<GlobalObject>(&GV)) {
+
+    if (MaybeAlign A = GO->getAlign()) {
+      Assert(A->value() <= Value::MaximumAlignment,
+             "huge alignment values are unsupported", GO);
+    }
+  }
   Assert(!GV.hasAppendingLinkage() || isa<GlobalVariable>(GV),
          "Only global variables can have appending linkage!", &GV);
 
@@ -733,8 +740,9 @@ void Verifier::visitGlobalVariable(const GlobalVariable &GV) {
           Value *V = Op->stripPointerCasts();
           Assert(isa<GlobalVariable>(V) || isa<Function>(V) ||
                      isa<GlobalAlias>(V),
-                 "invalid llvm.used member", V);
-          Assert(V->hasName(), "members of llvm.used must be named", V);
+                 Twine("invalid ") + GV.getName() + " member", V);
+          Assert(V->hasName(),
+                 Twine("members of ") + GV.getName() + " must be named", V);
         }
       }
     }
@@ -860,19 +868,6 @@ void Verifier::visitNamedMDNode(const NamedMDNode &NMD) {
   }
 }
 
-template <typename T>
-void Verifier::verifyODRTypeAsScopeOperand(const MDNode &MD, T *) {
-  if (isa<T>(MD)) {
-    if (auto *N = dyn_cast_or_null<DICompositeType>(cast<T>(MD).getScope()))
-      // Of all the supported tags for DICompositeType(see visitDICompositeType)
-      // we know that enum type cannot be a scope.
-      AssertDI(N->getTag() != dwarf::DW_TAG_enumeration_type,
-               "enum type is not a scope; check enum type ODR "
-               "violation",
-               N, &MD);
-  }
-}
-
 void Verifier::visitMDNode(const MDNode &MD, AreDebugLocsAllowed AllowLocs) {
   // Only visit each node once.  Metadata can be mutually recursive, so this
   // avoids infinite recursion here, as well as being an optimization.
@@ -882,12 +877,6 @@ void Verifier::visitMDNode(const MDNode &MD, AreDebugLocsAllowed AllowLocs) {
   Assert(&MD.getContext() == &Context,
          "MDNode context does not match Module context!", &MD);
 
-  // Makes sure when a scope operand is a ODR type, the ODR type uniquing does
-  // not create invalid debug metadata.
-  // TODO: check that the non-ODR-type scope operand is valid.
-  verifyODRTypeAsScopeOperand<DIType>(MD);
-  verifyODRTypeAsScopeOperand<DILocalScope>(MD);
-
   switch (MD.getMetadataID()) {
   default:
     llvm_unreachable("Invalid MDNode subclass");
@@ -2055,10 +2044,12 @@ void Verifier::verifyFunctionAttrs(FunctionType *FT, AttributeList Attrs,
   }
 
   if (Attrs.hasFnAttr(Attribute::VScaleRange)) {
-    std::pair<unsigned, unsigned> Args =
-        Attrs.getFnAttrs().getVScaleRangeArgs();
+    unsigned VScaleMin = Attrs.getFnAttrs().getVScaleRangeMin();
+    if (VScaleMin == 0)
+      CheckFailed("'vscale_range' minimum must be greater than 0", V);
 
-    if (Args.first > Args.second && Args.second != 0)
+    Optional<unsigned> VScaleMax = Attrs.getFnAttrs().getVScaleRangeMax();
+    if (VScaleMax && VScaleMin > VScaleMax)
       CheckFailed("'vscale_range' minimum cannot be greater than maximum", V);
   }
 
@@ -3328,7 +3319,7 @@ void Verifier::visitCallBase(CallBase &Call) {
   visitInstruction(Call);
 }
 
-void Verifier::verifyTailCCMustTailAttrs(AttrBuilder Attrs,
+void Verifier::verifyTailCCMustTailAttrs(const AttrBuilder &Attrs,
                                          StringRef Context) {
   Assert(!Attrs.contains(Attribute::InAlloca),
          Twine("inalloca attribute not allowed in ") + Context);
@@ -3733,15 +3724,15 @@ void Verifier::visitLoadInst(LoadInst &LI) {
   PointerType *PTy = dyn_cast<PointerType>(LI.getOperand(0)->getType());
   Assert(PTy, "Load operand must be a pointer.", &LI);
   Type *ElTy = LI.getType();
-  Assert(LI.getAlignment() <= Value::MaximumAlignment,
-         "huge alignment values are unsupported", &LI);
+  if (MaybeAlign A = LI.getAlign()) {
+    Assert(A->value() <= Value::MaximumAlignment,
+           "huge alignment values are unsupported", &LI);
+  }
   Assert(ElTy->isSized(), "loading unsized types is not allowed", &LI);
   if (LI.isAtomic()) {
     Assert(LI.getOrdering() != AtomicOrdering::Release &&
                LI.getOrdering() != AtomicOrdering::AcquireRelease,
            "Load cannot have Release ordering", &LI);
-    Assert(LI.getAlignment() != 0,
-           "Atomic load must specify explicit alignment", &LI);
     Assert(ElTy->isIntOrPtrTy() || ElTy->isFloatingPointTy(),
            "atomic load operand must have integer, pointer, or floating point "
            "type!",
@@ -3761,15 +3752,15 @@ void Verifier::visitStoreInst(StoreInst &SI) {
   Type *ElTy = SI.getOperand(0)->getType();
   Assert(PTy->isOpaqueOrPointeeTypeMatches(ElTy),
          "Stored value type does not match pointer operand type!", &SI, ElTy);
-  Assert(SI.getAlignment() <= Value::MaximumAlignment,
-         "huge alignment values are unsupported", &SI);
+  if (MaybeAlign A = SI.getAlign()) {
+    Assert(A->value() <= Value::MaximumAlignment,
+           "huge alignment values are unsupported", &SI);
+  }
   Assert(ElTy->isSized(), "storing unsized types is not allowed", &SI);
   if (SI.isAtomic()) {
     Assert(SI.getOrdering() != AtomicOrdering::Acquire &&
                SI.getOrdering() != AtomicOrdering::AcquireRelease,
            "Store cannot have Acquire ordering", &SI);
-    Assert(SI.getAlignment() != 0,
-           "Atomic store must specify explicit alignment", &SI);
     Assert(ElTy->isIntOrPtrTy() || ElTy->isFloatingPointTy(),
            "atomic store operand must have integer, pointer, or floating point "
            "type!",
@@ -3820,8 +3811,10 @@ void Verifier::visitAllocaInst(AllocaInst &AI) {
          "Cannot allocate unsized type", &AI);
   Assert(AI.getArraySize()->getType()->isIntegerTy(),
          "Alloca array size must have integer type", &AI);
-  Assert(AI.getAlignment() <= Value::MaximumAlignment,
-         "huge alignment values are unsupported", &AI);
+  if (MaybeAlign A = AI.getAlign()) {
+    Assert(A->value() <= Value::MaximumAlignment,
+           "huge alignment values are unsupported", &AI);
+  }
 
   if (AI.isSwiftError()) {
     verifySwiftErrorValue(&AI);
diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp
index 6ce2ed265739..f26ef4b21996 100644
--- a/llvm/lib/LTO/LTO.cpp
+++ b/llvm/lib/LTO/LTO.cpp
@@ -1106,7 +1106,7 @@ Error LTO::runRegularLTO(AddStreamFn AddStream) {
 
   if (Conf.PreOptModuleHook &&
       !Conf.PreOptModuleHook(0, *RegularLTO.CombinedModule))
-    return Error::success();
+    return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile));
 
   if (!Conf.CodeGenOnly) {
     for (const auto &R : GlobalResolutions) {
@@ -1132,7 +1132,7 @@ Error LTO::runRegularLTO(AddStreamFn AddStream) {
 
     if (Conf.PostInternalizeModuleHook &&
         !Conf.PostInternalizeModuleHook(0, *RegularLTO.CombinedModule))
-      return Error::success();
+      return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile));
   }
 
   if (!RegularLTO.EmptyCombinedModule || Conf.AlwaysEmitRegularLTOObj) {
diff --git a/llvm/lib/LTO/LTOBackend.cpp b/llvm/lib/LTO/LTOBackend.cpp
index be06556b0c3b..855d0fc8a8be 100644
--- a/llvm/lib/LTO/LTOBackend.cpp
+++ b/llvm/lib/LTO/LTOBackend.cpp
@@ -37,7 +37,6 @@
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Program.h"
-#include "llvm/Support/SmallVectorMemoryBuffer.h"
 #include "llvm/Support/ThreadPool.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetMachine.h"
@@ -413,6 +412,8 @@ static void codegen(const Config &Conf, TargetMachine *TM,
   if (Error Err = StreamOrErr.takeError())
     report_fatal_error(std::move(Err));
   std::unique_ptr<CachedFileStream> &Stream = *StreamOrErr;
+  TM->Options.ObjectFilenameForDebug = Stream->ObjectPathName;
+
   legacy::PassManager CodeGenPasses;
   CodeGenPasses.add(
       createImmutableModuleSummaryIndexWrapperPass(&CombinedIndex));
diff --git a/llvm/lib/LTO/LTOCodeGenerator.cpp b/llvm/lib/LTO/LTOCodeGenerator.cpp
index 088e45c9e8dc..fdc9896aca78 100644
--- a/llvm/lib/LTO/LTOCodeGenerator.cpp
+++ b/llvm/lib/LTO/LTOCodeGenerator.cpp
@@ -135,9 +135,8 @@ LTOCodeGenerator::LTOCodeGenerator(LLVMContext &Context)
 LTOCodeGenerator::~LTOCodeGenerator() {}
 
 void LTOCodeGenerator::setAsmUndefinedRefs(LTOModule *Mod) {
-  const std::vector<StringRef> &undefs = Mod->getAsmUndefinedRefs();
-  for (int i = 0, e = undefs.size(); i != e; ++i)
-    AsmUndefinedRefs.insert(undefs[i]);
+  for (const StringRef &Undef : Mod->getAsmUndefinedRefs())
+    AsmUndefinedRefs.insert(Undef);
 }
 
 bool LTOCodeGenerator::addModule(LTOModule *Mod) {
diff --git a/llvm/lib/LTO/ThinLTOCodeGenerator.cpp b/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
index 9474d8c9dafb..9aea27f0fdba 100644
--- a/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
+++ b/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
@@ -378,7 +378,8 @@ std::unique_ptr<MemoryBuffer> codegenModule(Module &TheModule,
     // Run codegen now. resulting binary is in OutputBuffer.
     PM.run(TheModule);
   }
-  return std::make_unique<SmallVectorMemoryBuffer>(std::move(OutputBuffer));
+  return std::make_unique<SmallVectorMemoryBuffer>(
+      std::move(OutputBuffer), /*RequiresNullTerminator=*/false);
 }
 
 /// Manage caching for a single Module.
@@ -541,7 +542,8 @@ ProcessThinLTOModule(Module &TheModule, ModuleSummaryIndex &Index,
       auto Index = buildModuleSummaryIndex(TheModule, nullptr, &PSI);
       WriteBitcodeToFile(TheModule, OS, true, &Index);
     }
-    return std::make_unique<SmallVectorMemoryBuffer>(std::move(OutputBuffer));
+    return std::make_unique<SmallVectorMemoryBuffer>(
+        std::move(OutputBuffer), /*RequiresNullTerminator=*/false);
   }
 
   return codegenModule(TheModule, TM);
diff --git a/llvm/lib/LineEditor/LineEditor.cpp b/llvm/lib/LineEditor/LineEditor.cpp
index 1aa3476eb357..37c4b79f8e29 100644
--- a/llvm/lib/LineEditor/LineEditor.cpp
+++ b/llvm/lib/LineEditor/LineEditor.cpp
@@ -69,9 +69,8 @@ LineEditor::ListCompleterConcept::complete(StringRef Buffer, size_t Pos) const {
   // common prefix will then be empty.
   if (CommonPrefix.empty()) {
     Action.Kind = CompletionAction::AK_ShowCompletions;
-    for (std::vector<Completion>::iterator I = Comps.begin(), E = Comps.end();
-         I != E; ++I)
-      Action.Completions.push_back(I->DisplayText);
+    for (const Completion &Comp : Comps)
+      Action.Completions.push_back(Comp.DisplayText);
   } else {
     Action.Kind = CompletionAction::AK_Insert;
     Action.Text = CommonPrefix;
diff --git a/llvm/lib/Linker/IRMover.cpp b/llvm/lib/Linker/IRMover.cpp
index bad483be197d..b475ea81d107 100644
--- a/llvm/lib/Linker/IRMover.cpp
+++ b/llvm/lib/Linker/IRMover.cpp
@@ -646,7 +646,7 @@ GlobalVariable *IRLinker::copyGlobalVariableProto(const GlobalVariable *SGVar) {
                          /*init*/ nullptr, SGVar->getName(),
                          /*insertbefore*/ nullptr, SGVar->getThreadLocalMode(),
                          SGVar->getAddressSpace());
-  NewDGV->setAlignment(MaybeAlign(SGVar->getAlignment()));
+  NewDGV->setAlignment(SGVar->getAlign());
   NewDGV->copyAttributesFrom(SGVar);
   return NewDGV;
 }
@@ -877,7 +877,7 @@ IRLinker::linkAppendingVarProto(GlobalVariable *DstGV,
     if (DstGV->isConstant() != SrcGV->isConstant())
       return stringErr("Appending variables linked with different const'ness!");
 
-    if (DstGV->getAlignment() != SrcGV->getAlignment())
+    if (DstGV->getAlign() != SrcGV->getAlign())
       return stringErr(
           "Appending variables with different alignment need to be linked!");
 
diff --git a/llvm/lib/MC/MCAsmStreamer.cpp b/llvm/lib/MC/MCAsmStreamer.cpp
index 2ca921017171..5c2aaddff4d1 100644
--- a/llvm/lib/MC/MCAsmStreamer.cpp
+++ b/llvm/lib/MC/MCAsmStreamer.cpp
@@ -168,9 +168,14 @@ public:
                       unsigned Update, VersionTuple SDKVersion) override;
   void emitBuildVersion(unsigned Platform, unsigned Major, unsigned Minor,
                         unsigned Update, VersionTuple SDKVersion) override;
+  void emitDarwinTargetVariantBuildVersion(unsigned Platform, unsigned Major,
+                                           unsigned Minor, unsigned Update,
+                                           VersionTuple SDKVersion) override;
   void emitThumbFunc(MCSymbol *Func) override;
 
   void emitAssignment(MCSymbol *Symbol, const MCExpr *Value) override;
+  void emitConditionalAssignment(MCSymbol *Symbol,
+                                 const MCExpr *Value) override;
   void emitWeakReference(MCSymbol *Alias, const MCSymbol *Symbol) override;
   bool emitSymbolAttribute(MCSymbol *Symbol, MCSymbolAttr Attribute) override;
 
@@ -640,6 +645,12 @@ void MCAsmStreamer::emitBuildVersion(unsigned Platform, unsigned Major,
   EmitEOL();
 }
 
+void MCAsmStreamer::emitDarwinTargetVariantBuildVersion(
+    unsigned Platform, unsigned Major, unsigned Minor, unsigned Update,
+    VersionTuple SDKVersion) {
+  emitBuildVersion(Platform, Major, Minor, Update, SDKVersion);
+}
+
 void MCAsmStreamer::emitThumbFunc(MCSymbol *Func) {
   // This needs to emit to a temporary string to get properly quoted
   // MCSymbols when they have spaces in them.
@@ -670,6 +681,15 @@ void MCAsmStreamer::emitAssignment(MCSymbol *Symbol, const MCExpr *Value) {
   MCStreamer::emitAssignment(Symbol, Value);
 }
 
+void MCAsmStreamer::emitConditionalAssignment(MCSymbol *Symbol,
+                                              const MCExpr *Value) {
+  OS << ".lto_set_conditional ";
+  Symbol->print(OS, MAI);
+  OS << ", ";
+  Value->print(OS, MAI);
+  EmitEOL();
+}
+
 void MCAsmStreamer::emitWeakReference(MCSymbol *Alias, const MCSymbol *Symbol) {
   OS << ".weakref ";
   Alias->print(OS, MAI);
diff --git a/llvm/lib/MC/MCAssembler.cpp b/llvm/lib/MC/MCAssembler.cpp
index d5e9f4fc66bc..a8837bbf57c7 100644
--- a/llvm/lib/MC/MCAssembler.cpp
+++ b/llvm/lib/MC/MCAssembler.cpp
@@ -89,6 +89,7 @@ MCAssembler::MCAssembler(MCContext &Context,
       BundleAlignSize(0), RelaxAll(false), SubsectionsViaSymbols(false),
       IncrementalLinkerCompatible(false), ELFHeaderEFlags(0) {
   VersionInfo.Major = 0; // Major version == 0 for "none specified"
+  DarwinTargetVariantVersionInfo.Major = 0;
 }
 
 MCAssembler::~MCAssembler() = default;
@@ -109,6 +110,8 @@ void MCAssembler::reset() {
   LOHContainer.reset();
   VersionInfo.Major = 0;
   VersionInfo.SDKVersion = VersionTuple();
+  DarwinTargetVariantVersionInfo.Major = 0;
+  DarwinTargetVariantVersionInfo.SDKVersion = VersionTuple();
 
   // reset objects owned by us
   if (getBackendPtr())
diff --git a/llvm/lib/MC/MCInstrAnalysis.cpp b/llvm/lib/MC/MCInstrAnalysis.cpp
index 52b59185c6fc..4ed1c6286a72 100644
--- a/llvm/lib/MC/MCInstrAnalysis.cpp
+++ b/llvm/lib/MC/MCInstrAnalysis.cpp
@@ -39,4 +39,4 @@ Optional<uint64_t>
 MCInstrAnalysis::getMemoryOperandRelocationOffset(const MCInst &Inst,
                                                   uint64_t Size) const {
   return None;
-}
\ No newline at end of file
+}
diff --git a/llvm/lib/MC/MCMachOStreamer.cpp b/llvm/lib/MC/MCMachOStreamer.cpp
index aa94b141d8be..3edf7a3f49e6 100644
--- a/llvm/lib/MC/MCMachOStreamer.cpp
+++ b/llvm/lib/MC/MCMachOStreamer.cpp
@@ -92,6 +92,9 @@ public:
                       unsigned Update, VersionTuple SDKVersion) override;
   void emitBuildVersion(unsigned Platform, unsigned Major, unsigned Minor,
                         unsigned Update, VersionTuple SDKVersion) override;
+  void emitDarwinTargetVariantBuildVersion(unsigned Platform, unsigned Major,
+                                           unsigned Minor, unsigned Update,
+                                           VersionTuple SDKVersion) override;
   void emitThumbFunc(MCSymbol *Func) override;
   bool emitSymbolAttribute(MCSymbol *Symbol, MCSymbolAttr Attribute) override;
   void emitSymbolDesc(MCSymbol *Symbol, unsigned DescValue) override;
@@ -283,6 +286,13 @@ void MCMachOStreamer::emitBuildVersion(unsigned Platform, unsigned Major,
                                  Update, SDKVersion);
 }
 
+void MCMachOStreamer::emitDarwinTargetVariantBuildVersion(
+    unsigned Platform, unsigned Major, unsigned Minor, unsigned Update,
+    VersionTuple SDKVersion) {
+  getAssembler().setDarwinTargetVariantBuildVersion(
+      (MachO::PlatformType)Platform, Major, Minor, Update, SDKVersion);
+}
+
 void MCMachOStreamer::emitThumbFunc(MCSymbol *Symbol) {
   // Remember that the function is a thumb function. Fixup and relocation
   // values will need adjusted.
@@ -516,7 +526,10 @@ MCStreamer *llvm::createMachOStreamer(MCContext &Context,
       new MCMachOStreamer(Context, std::move(MAB), std::move(OW), std::move(CE),
                           DWARFMustBeAtTheEnd, LabelSections);
   const Triple &Target = Context.getTargetTriple();
-  S->emitVersionForTarget(Target, Context.getObjectFileInfo()->getSDKVersion());
+  S->emitVersionForTarget(
+      Target, Context.getObjectFileInfo()->getSDKVersion(),
+      Context.getObjectFileInfo()->getDarwinTargetVariantTriple(),
+      Context.getObjectFileInfo()->getDarwinTargetVariantSDKVersion());
   if (RelaxAll)
     S->getAssembler().setRelaxAll(true);
   return S;
diff --git a/llvm/lib/MC/MCNullStreamer.cpp b/llvm/lib/MC/MCNullStreamer.cpp
index 291d840b4f4b..40b7eba58b03 100644
--- a/llvm/lib/MC/MCNullStreamer.cpp
+++ b/llvm/lib/MC/MCNullStreamer.cpp
@@ -40,6 +40,9 @@ namespace {
     void EmitCOFFSymbolStorageClass(int StorageClass) override {}
     void EmitCOFFSymbolType(int Type) override {}
     void EndCOFFSymbolDef() override {}
+    void
+    emitXCOFFSymbolLinkageWithVisibility(MCSymbol *Symbol, MCSymbolAttr Linkage,
+                                         MCSymbolAttr Visibility) override {}
   };
 
 }
diff --git a/llvm/lib/MC/MCObjectStreamer.cpp b/llvm/lib/MC/MCObjectStreamer.cpp
index 9c86fcc86bcb..6604d7988c4c 100644
--- a/llvm/lib/MC/MCObjectStreamer.cpp
+++ b/llvm/lib/MC/MCObjectStreamer.cpp
@@ -281,6 +281,18 @@ void MCObjectStreamer::emitLabel(MCSymbol *Symbol, SMLoc Loc) {
     Symbol->setOffset(0);
     addPendingLabel(Symbol);
   }
+
+  emitPendingAssignments(Symbol);
+}
+
+void MCObjectStreamer::emitPendingAssignments(MCSymbol *Symbol) {
+  auto Assignments = pendingAssignments.find(Symbol);
+  if (Assignments != pendingAssignments.end()) {
+    for (const PendingAssignment &A : Assignments->second)
+      emitAssignment(A.Symbol, A.Value);
+
+    pendingAssignments.erase(Assignments);
+  }
 }
 
 // Emit a label at a previously emitted fragment/offset position. This must be
@@ -353,6 +365,19 @@ bool MCObjectStreamer::changeSectionImpl(MCSection *Section,
 void MCObjectStreamer::emitAssignment(MCSymbol *Symbol, const MCExpr *Value) {
   getAssembler().registerSymbol(*Symbol);
   MCStreamer::emitAssignment(Symbol, Value);
+  emitPendingAssignments(Symbol);
+}
+
+void MCObjectStreamer::emitConditionalAssignment(MCSymbol *Symbol,
+                                                 const MCExpr *Value) {
+  const MCSymbol *Target = &cast<MCSymbolRefExpr>(*Value).getSymbol();
+
+  // If the symbol already exists, emit the assignment. Otherwise, emit it
+  // later only if the symbol is also emitted.
+  if (Target->isRegistered())
+    emitAssignment(Symbol, Value);
+  else
+    pendingAssignments[Target].push_back({Symbol, Value});
 }
 
 bool MCObjectStreamer::mayHaveInstructions(MCSection &Sec) const {
diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp
index ed9f2066dc20..705f7159d55b 100644
--- a/llvm/lib/MC/MCParser/AsmParser.cpp
+++ b/llvm/lib/MC/MCParser/AsmParser.cpp
@@ -356,8 +356,14 @@ private:
   /// return the contents from the current token up to the end or comma.
   StringRef parseStringToComma();
 
-  bool parseAssignment(StringRef Name, bool allow_redef,
-                       bool NoDeadStrip = false);
+  enum class AssignmentKind {
+    Set,
+    Equiv,
+    Equal,
+    LTOSetConditional,
+  };
+
+  bool parseAssignment(StringRef Name, AssignmentKind Kind);
 
   unsigned getBinOpPrecedence(AsmToken::TokenKind K,
                               MCBinaryExpr::Opcode &Kind);
@@ -534,6 +540,7 @@ private:
     DK_ADDRSIG_SYM,
     DK_PSEUDO_PROBE,
     DK_LTO_DISCARD,
+    DK_LTO_SET_CONDITIONAL,
     DK_END
   };
 
@@ -564,8 +571,8 @@ private:
                                const fltSemantics &); // ".single", ...
   bool parseDirectiveFill(); // ".fill"
   bool parseDirectiveZero(); // ".zero"
-  // ".set", ".equ", ".equiv"
-  bool parseDirectiveSet(StringRef IDVal, bool allow_redef);
+  // ".set", ".equ", ".equiv", ".lto_set_conditional"
+  bool parseDirectiveSet(StringRef IDVal, AssignmentKind Kind);
   bool parseDirectiveOrg(); // ".org"
   // ".align{,32}", ".p2align{,w,l}"
   bool parseDirectiveAlign(bool IsPow2, unsigned ValueSize);
@@ -1968,7 +1975,7 @@ bool AsmParser::parseStatement(ParseStatementInfo &Info,
     // identifier '=' ... -> assignment statement
     Lex();
 
-    return parseAssignment(IDVal, true);
+    return parseAssignment(IDVal, AssignmentKind::Equal);
 
   default: // Normal instruction or directive.
     break;
@@ -2027,9 +2034,11 @@ bool AsmParser::parseStatement(ParseStatementInfo &Info,
       break;
     case DK_SET:
     case DK_EQU:
-      return parseDirectiveSet(IDVal, true);
+      return parseDirectiveSet(IDVal, AssignmentKind::Set);
     case DK_EQUIV:
-      return parseDirectiveSet(IDVal, false);
+      return parseDirectiveSet(IDVal, AssignmentKind::Equiv);
+    case DK_LTO_SET_CONDITIONAL:
+      return parseDirectiveSet(IDVal, AssignmentKind::LTOSetConditional);
     case DK_ASCII:
       return parseDirectiveAscii(IDVal, false);
     case DK_ASCIZ:
@@ -2925,11 +2934,13 @@ void AsmParser::handleMacroExit() {
   ActiveMacros.pop_back();
 }
 
-bool AsmParser::parseAssignment(StringRef Name, bool allow_redef,
-                                bool NoDeadStrip) {
+bool AsmParser::parseAssignment(StringRef Name, AssignmentKind Kind) {
   MCSymbol *Sym;
   const MCExpr *Value;
-  if (MCParserUtils::parseAssignmentExpression(Name, allow_redef, *this, Sym,
+  SMLoc ExprLoc = getTok().getLoc();
+  bool AllowRedef =
+      Kind == AssignmentKind::Set || Kind == AssignmentKind::Equal;
+  if (MCParserUtils::parseAssignmentExpression(Name, AllowRedef, *this, Sym,
                                                Value))
     return true;
 
@@ -2944,9 +2955,22 @@ bool AsmParser::parseAssignment(StringRef Name, bool allow_redef,
     return false;
 
   // Do the assignment.
-  Out.emitAssignment(Sym, Value);
-  if (NoDeadStrip)
+  switch (Kind) {
+  case AssignmentKind::Equal:
+    Out.emitAssignment(Sym, Value);
+    break;
+  case AssignmentKind::Set:
+  case AssignmentKind::Equiv:
+    Out.emitAssignment(Sym, Value);
     Out.emitSymbolAttribute(Sym, MCSA_NoDeadStrip);
+    break;
+  case AssignmentKind::LTOSetConditional:
+    if (Value->getKind() != MCExpr::SymbolRef)
+      return Error(ExprLoc, "expected identifier");
+
+    Out.emitConditionalAssignment(Sym, Value);
+    break;
+  }
 
   return false;
 }
@@ -2998,10 +3022,11 @@ bool AsmParser::parseIdentifier(StringRef &Res) {
 ///   ::= .equ identifier ',' expression
 ///   ::= .equiv identifier ',' expression
 ///   ::= .set identifier ',' expression
-bool AsmParser::parseDirectiveSet(StringRef IDVal, bool allow_redef) {
+///   ::= .lto_set_conditional identifier ',' expression
+bool AsmParser::parseDirectiveSet(StringRef IDVal, AssignmentKind Kind) {
   StringRef Name;
   if (check(parseIdentifier(Name), "expected identifier") || parseComma() ||
-      parseAssignment(Name, allow_redef, true))
+      parseAssignment(Name, Kind))
     return true;
   return false;
 }
@@ -5581,6 +5606,7 @@ void AsmParser::initializeDirectiveKindMap() {
   DirectiveKindMap[".addrsig_sym"] = DK_ADDRSIG_SYM;
   DirectiveKindMap[".pseudoprobe"] = DK_PSEUDO_PROBE;
   DirectiveKindMap[".lto_discard"] = DK_LTO_DISCARD;
+  DirectiveKindMap[".lto_set_conditional"] = DK_LTO_SET_CONDITIONAL;
 }
 
 MCAsmMacro *AsmParser::parseMacroLikeBody(SMLoc DirectiveLoc) {
@@ -6012,12 +6038,13 @@ bool AsmParser::parseMSInlineAsm(
 
       bool isOutput = (i == 1) && Desc.mayStore();
       SMLoc Start = SMLoc::getFromPointer(SymName.data());
+      int64_t Size = Operand.isMemPlaceholder(Desc) ? 0 : SymName.size();
       if (isOutput) {
         ++InputIdx;
         OutputDecls.push_back(OpDecl);
         OutputDeclsAddressOf.push_back(Operand.needAddressOf());
         OutputConstraints.push_back(("=" + Constraint).str());
-        AsmStrRewrites.emplace_back(AOK_Output, Start, SymName.size());
+        AsmStrRewrites.emplace_back(AOK_Output, Start, Size);
       } else {
         InputDecls.push_back(OpDecl);
         InputDeclsAddressOf.push_back(Operand.needAddressOf());
@@ -6025,7 +6052,7 @@ bool AsmParser::parseMSInlineAsm(
         if (Desc.OpInfo[i - 1].isBranchTarget())
           AsmStrRewrites.emplace_back(AOK_CallInput, Start, SymName.size());
         else
-          AsmStrRewrites.emplace_back(AOK_Input, Start, SymName.size());
+          AsmStrRewrites.emplace_back(AOK_Input, Start, Size);
       }
     }
 
@@ -6140,13 +6167,17 @@ bool AsmParser::parseMSInlineAsm(
       OS << Ctx.getAsmInfo()->getPrivateLabelPrefix() << AR.Label;
       break;
     case AOK_Input:
-      OS << '$' << InputIdx++;
+      if (AR.Len)
+        OS << '$' << InputIdx;
+      ++InputIdx;
       break;
     case AOK_CallInput:
       OS << "${" << InputIdx++ << ":P}";
       break;
     case AOK_Output:
-      OS << '$' << OutputIdx++;
+      if (AR.Len)
+        OS << '$' << OutputIdx;
+      ++OutputIdx;
       break;
     case AOK_SizeDirective:
       switch (AR.Val) {
diff --git a/llvm/lib/MC/MCPseudoProbe.cpp b/llvm/lib/MC/MCPseudoProbe.cpp
index e35bcec8fe75..ebf38327f4dc 100644
--- a/llvm/lib/MC/MCPseudoProbe.cpp
+++ b/llvm/lib/MC/MCPseudoProbe.cpp
@@ -151,8 +151,8 @@ void MCPseudoProbeInlineTree::emit(MCObjectStreamer *MCOS,
   // InlineSite is unique for each pair,
   // so there will be no ordering of Inlinee based on MCPseudoProbeInlineTree*
   std::map<InlineSite, MCPseudoProbeInlineTree *> Inlinees;
-  for (auto Child = Children.begin(); Child != Children.end(); ++Child)
-    Inlinees[Child->first] = Child->second.get();
+  for (auto &Child : Children)
+    Inlinees[Child.first] = Child.second.get();
 
   for (const auto &Inlinee : Inlinees) {
     if (Guid) {
diff --git a/llvm/lib/MC/MCStreamer.cpp b/llvm/lib/MC/MCStreamer.cpp
index f4e64b42c817..9c37a7bebe2a 100644
--- a/llvm/lib/MC/MCStreamer.cpp
+++ b/llvm/lib/MC/MCStreamer.cpp
@@ -431,6 +431,9 @@ void MCStreamer::emitLabel(MCSymbol *Symbol, SMLoc Loc) {
     TS->emitLabel(Symbol);
 }
 
+void MCStreamer::emitConditionalAssignment(MCSymbol *Symbol,
+                                           const MCExpr *Value) {}
+
 void MCStreamer::emitCFISections(bool EH, bool Debug) {}
 
 void MCStreamer::emitCFIStartProc(bool IsSimple, SMLoc Loc) {
@@ -1308,45 +1311,78 @@ getMachoBuildVersionPlatformType(const Triple &Target) {
   llvm_unreachable("unexpected OS type");
 }
 
-void MCStreamer::emitVersionForTarget(const Triple &Target,
-                                      const VersionTuple &SDKVersion) {
+void MCStreamer::emitVersionForTarget(
+    const Triple &Target, const VersionTuple &SDKVersion,
+    const Triple *DarwinTargetVariantTriple,
+    const VersionTuple &DarwinTargetVariantSDKVersion) {
   if (!Target.isOSBinFormatMachO() || !Target.isOSDarwin())
     return;
   // Do we even know the version?
   if (Target.getOSMajorVersion() == 0)
     return;
 
-  unsigned Major = 0;
-  unsigned Minor = 0;
-  unsigned Update = 0;
+  VersionTuple Version;
   switch (Target.getOS()) {
   case Triple::MacOSX:
   case Triple::Darwin:
-    Target.getMacOSXVersion(Major, Minor, Update);
+    Target.getMacOSXVersion(Version);
     break;
   case Triple::IOS:
   case Triple::TvOS:
-    Target.getiOSVersion(Major, Minor, Update);
+    Version = Target.getiOSVersion();
     break;
   case Triple::WatchOS:
-    Target.getWatchOSVersion(Major, Minor, Update);
+    Version = Target.getWatchOSVersion();
     break;
   default:
     llvm_unreachable("unexpected OS type");
   }
-  assert(Major != 0 && "A non-zero major version is expected");
-  auto LinkedTargetVersion = targetVersionOrMinimumSupportedOSVersion(
-      Target, VersionTuple(Major, Minor, Update));
+  assert(Version.getMajor() != 0 && "A non-zero major version is expected");
+  auto LinkedTargetVersion =
+      targetVersionOrMinimumSupportedOSVersion(Target, Version);
   auto BuildVersionOSVersion = getMachoBuildVersionSupportedOS(Target);
+  bool ShouldEmitBuildVersion = false;
   if (BuildVersionOSVersion.empty() ||
-      LinkedTargetVersion >= BuildVersionOSVersion)
-    return emitBuildVersion(getMachoBuildVersionPlatformType(Target),
-                            LinkedTargetVersion.getMajor(),
-                            *LinkedTargetVersion.getMinor(),
-                            *LinkedTargetVersion.getSubminor(), SDKVersion);
+      LinkedTargetVersion >= BuildVersionOSVersion) {
+    if (Target.isMacCatalystEnvironment() && DarwinTargetVariantTriple &&
+        DarwinTargetVariantTriple->isMacOSX()) {
+      emitVersionForTarget(*DarwinTargetVariantTriple,
+                           DarwinTargetVariantSDKVersion,
+                           /*TargetVariantTriple=*/nullptr,
+                           /*TargetVariantSDKVersion=*/VersionTuple());
+      emitDarwinTargetVariantBuildVersion(
+          getMachoBuildVersionPlatformType(Target),
+          LinkedTargetVersion.getMajor(),
+          LinkedTargetVersion.getMinor().getValueOr(0),
+          LinkedTargetVersion.getSubminor().getValueOr(0), SDKVersion);
+      return;
+    }
+    emitBuildVersion(getMachoBuildVersionPlatformType(Target),
+                     LinkedTargetVersion.getMajor(),
+                     LinkedTargetVersion.getMinor().getValueOr(0),
+                     LinkedTargetVersion.getSubminor().getValueOr(0),
+                     SDKVersion);
+    ShouldEmitBuildVersion = true;
+  }
+
+  if (const Triple *TVT = DarwinTargetVariantTriple) {
+    if (Target.isMacOSX() && TVT->isMacCatalystEnvironment()) {
+      auto TVLinkedTargetVersion =
+          targetVersionOrMinimumSupportedOSVersion(*TVT, TVT->getiOSVersion());
+      emitDarwinTargetVariantBuildVersion(
+          getMachoBuildVersionPlatformType(*TVT),
+          TVLinkedTargetVersion.getMajor(),
+          TVLinkedTargetVersion.getMinor().getValueOr(0),
+          TVLinkedTargetVersion.getSubminor().getValueOr(0),
+          DarwinTargetVariantSDKVersion);
+    }
+  }
+
+  if (ShouldEmitBuildVersion)
+    return;
 
   emitVersionMin(getMachoVersionMinLoadCommandType(Target),
                  LinkedTargetVersion.getMajor(),
-                 *LinkedTargetVersion.getMinor(),
-                 *LinkedTargetVersion.getSubminor(), SDKVersion);
+                 LinkedTargetVersion.getMinor().getValueOr(0),
+                 LinkedTargetVersion.getSubminor().getValueOr(0), SDKVersion);
 }
diff --git a/llvm/lib/MC/MCWin64EH.cpp b/llvm/lib/MC/MCWin64EH.cpp
index 7773d8828931..2a93c352c68a 100644
--- a/llvm/lib/MC/MCWin64EH.cpp
+++ b/llvm/lib/MC/MCWin64EH.cpp
@@ -351,7 +351,7 @@ static uint32_t ARM64CountOfUnwindCodes(ArrayRef<WinEH::Instruction> Insns) {
 // Unwind opcode encodings and restrictions are documented at
 // https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling
 static void ARM64EmitUnwindCode(MCStreamer &streamer, const MCSymbol *begin,
-                                WinEH::Instruction &inst) {
+                                const WinEH::Instruction &inst) {
   uint8_t b, reg;
   switch (static_cast<Win64EH::UnwindOpcodes>(inst.Operation)) {
   default:
@@ -1050,10 +1050,8 @@ static void ARM64EmitUnwindInfo(MCStreamer &streamer, WinEH::FrameInfo *info,
   // Emit epilog unwind instructions
   for (auto &I : info->EpilogMap) {
     auto &EpilogInstrs = I.second;
-    for (uint32_t i = 0; i < EpilogInstrs.size(); i++) {
-      WinEH::Instruction inst = EpilogInstrs[i];
+    for (const WinEH::Instruction &inst : EpilogInstrs)
       ARM64EmitUnwindCode(streamer, info->Begin, inst);
-    }
   }
 
   int32_t BytesMod = CodeWords * 4 - TotalCodeBytes;
diff --git a/llvm/lib/MC/MachObjectWriter.cpp b/llvm/lib/MC/MachObjectWriter.cpp
index 277d88cf1cd2..16941b1cb727 100644
--- a/llvm/lib/MC/MachObjectWriter.cpp
+++ b/llvm/lib/MC/MachObjectWriter.cpp
@@ -484,15 +484,15 @@ void MachObjectWriter::bindIndirectSymbols(MCAssembler &Asm) {
 
   // Report errors for use of .indirect_symbol not in a symbol pointer section
   // or stub section.
-  for (MCAssembler::indirect_symbol_iterator it = Asm.indirect_symbol_begin(),
-         ie = Asm.indirect_symbol_end(); it != ie; ++it) {
-    const MCSectionMachO &Section = cast<MCSectionMachO>(*it->Section);
+  for (IndirectSymbolData &ISD : llvm::make_range(Asm.indirect_symbol_begin(),
+                                                  Asm.indirect_symbol_end())) {
+    const MCSectionMachO &Section = cast<MCSectionMachO>(*ISD.Section);
 
     if (Section.getType() != MachO::S_NON_LAZY_SYMBOL_POINTERS &&
         Section.getType() != MachO::S_LAZY_SYMBOL_POINTERS &&
         Section.getType() != MachO::S_THREAD_LOCAL_VARIABLE_POINTERS &&
         Section.getType() != MachO::S_SYMBOL_STUBS) {
-      MCSymbol &Symbol = *it->Symbol;
+      MCSymbol &Symbol = *ISD.Symbol;
       report_fatal_error("indirect symbol '" + Symbol.getName() +
                          "' not in a symbol pointer or stub section");
     }
@@ -779,6 +779,17 @@ uint64_t MachObjectWriter::writeObject(MCAssembler &Asm,
       LoadCommandsSize += sizeof(MachO::version_min_command);
   }
 
+  const MCAssembler::VersionInfoType &TargetVariantVersionInfo =
+      Layout.getAssembler().getDarwinTargetVariantVersionInfo();
+
+  // Add the target variant version info load command size, if used.
+  if (TargetVariantVersionInfo.Major != 0) {
+    ++NumLoadCommands;
+    assert(TargetVariantVersionInfo.EmitBuildVersion &&
+           "target variant should use build version");
+    LoadCommandsSize += sizeof(MachO::build_version_command);
+  }
+
   // Add the data-in-code load command size, if used.
   unsigned NumDataRegions = Asm.getDataRegions().size();
   if (NumDataRegions) {
@@ -862,38 +873,43 @@ uint64_t MachObjectWriter::writeObject(MCAssembler &Asm,
   }
 
   // Write out the deployment target information, if it's available.
-  if (VersionInfo.Major != 0) {
-    auto EncodeVersion = [](VersionTuple V) -> uint32_t {
-      assert(!V.empty() && "empty version");
-      unsigned Update = V.getSubminor() ? *V.getSubminor() : 0;
-      unsigned Minor = V.getMinor() ? *V.getMinor() : 0;
-      assert(Update < 256 && "unencodable update target version");
-      assert(Minor < 256 && "unencodable minor target version");
-      assert(V.getMajor() < 65536 && "unencodable major target version");
-      return Update | (Minor << 8) | (V.getMajor() << 16);
-    };
-    uint32_t EncodedVersion = EncodeVersion(
-        VersionTuple(VersionInfo.Major, VersionInfo.Minor, VersionInfo.Update));
-    uint32_t SDKVersion = !VersionInfo.SDKVersion.empty()
-                              ? EncodeVersion(VersionInfo.SDKVersion)
-                              : 0;
-    if (VersionInfo.EmitBuildVersion) {
-      // FIXME: Currently empty tools. Add clang version in the future.
-      W.write<uint32_t>(MachO::LC_BUILD_VERSION);
-      W.write<uint32_t>(sizeof(MachO::build_version_command));
-      W.write<uint32_t>(VersionInfo.TypeOrPlatform.Platform);
-      W.write<uint32_t>(EncodedVersion);
-      W.write<uint32_t>(SDKVersion);
-      W.write<uint32_t>(0);         // Empty tools list.
-    } else {
-      MachO::LoadCommandType LCType
-        = getLCFromMCVM(VersionInfo.TypeOrPlatform.Type);
-      W.write<uint32_t>(LCType);
-      W.write<uint32_t>(sizeof(MachO::version_min_command));
-      W.write<uint32_t>(EncodedVersion);
-      W.write<uint32_t>(SDKVersion);
-    }
-  }
+  auto EmitDeploymentTargetVersion =
+      [&](const MCAssembler::VersionInfoType &VersionInfo) {
+        auto EncodeVersion = [](VersionTuple V) -> uint32_t {
+          assert(!V.empty() && "empty version");
+          unsigned Update = V.getSubminor().getValueOr(0);
+          unsigned Minor = V.getMinor().getValueOr(0);
+          assert(Update < 256 && "unencodable update target version");
+          assert(Minor < 256 && "unencodable minor target version");
+          assert(V.getMajor() < 65536 && "unencodable major target version");
+          return Update | (Minor << 8) | (V.getMajor() << 16);
+        };
+        uint32_t EncodedVersion = EncodeVersion(VersionTuple(
+            VersionInfo.Major, VersionInfo.Minor, VersionInfo.Update));
+        uint32_t SDKVersion = !VersionInfo.SDKVersion.empty()
+                                  ? EncodeVersion(VersionInfo.SDKVersion)
+                                  : 0;
+        if (VersionInfo.EmitBuildVersion) {
+          // FIXME: Currently empty tools. Add clang version in the future.
+          W.write<uint32_t>(MachO::LC_BUILD_VERSION);
+          W.write<uint32_t>(sizeof(MachO::build_version_command));
+          W.write<uint32_t>(VersionInfo.TypeOrPlatform.Platform);
+          W.write<uint32_t>(EncodedVersion);
+          W.write<uint32_t>(SDKVersion);
+          W.write<uint32_t>(0); // Empty tools list.
+        } else {
+          MachO::LoadCommandType LCType =
+              getLCFromMCVM(VersionInfo.TypeOrPlatform.Type);
+          W.write<uint32_t>(LCType);
+          W.write<uint32_t>(sizeof(MachO::version_min_command));
+          W.write<uint32_t>(EncodedVersion);
+          W.write<uint32_t>(SDKVersion);
+        }
+      };
+  if (VersionInfo.Major != 0)
+    EmitDeploymentTargetVersion(VersionInfo);
+  if (TargetVariantVersionInfo.Major != 0)
+    EmitDeploymentTargetVersion(TargetVariantVersionInfo);
 
   // Write the data-in-code load command, if used.
   uint64_t DataInCodeTableEnd = RelocTableEnd + NumDataRegions * 8;
diff --git a/llvm/lib/MC/TargetRegistry.cpp b/llvm/lib/MC/TargetRegistry.cpp
index 0948a6b9f1a1..09684b1e5ad2 100644
--- a/llvm/lib/MC/TargetRegistry.cpp
+++ b/llvm/lib/MC/TargetRegistry.cpp
@@ -124,10 +124,10 @@ void TargetRegistry::printRegisteredTargetsForVersion(raw_ostream &OS) {
   array_pod_sort(Targets.begin(), Targets.end(), TargetArraySortFn);
 
   OS << "  Registered Targets:\n";
-  for (unsigned i = 0, e = Targets.size(); i != e; ++i) {
-    OS << "    " << Targets[i].first;
-    OS.indent(Width - Targets[i].first.size()) << " - "
-      << Targets[i].second->getShortDescription() << '\n';
+  for (const auto &Target : Targets) {
+    OS << "    " << Target.first;
+    OS.indent(Width - Target.first.size())
+        << " - " << Target.second->getShortDescription() << '\n';
   }
   if (Targets.empty())
     OS << "    (none)\n";
diff --git a/llvm/lib/Object/ArchiveWriter.cpp b/llvm/lib/Object/ArchiveWriter.cpp
index ce997464caa7..da8bcec7f3d4 100644
--- a/llvm/lib/Object/ArchiveWriter.cpp
+++ b/llvm/lib/Object/ArchiveWriter.cpp
@@ -696,7 +696,7 @@ writeArchiveToBuffer(ArrayRef<NewArchiveMember> NewMembers, bool WriteSymtab,
     return std::move(E);
 
   return std::make_unique<SmallVectorMemoryBuffer>(
-      std::move(ArchiveBufferVector));
+      std::move(ArchiveBufferVector), /*RequiresNullTerminator=*/false);
 }
 
 } // namespace llvm
diff --git a/llvm/lib/Object/ELF.cpp b/llvm/lib/Object/ELF.cpp
index 84181ae5e501..6e56da1a31f3 100644
--- a/llvm/lib/Object/ELF.cpp
+++ b/llvm/lib/Object/ELF.cpp
@@ -210,6 +210,8 @@ uint32_t llvm::object::getELFRelativeRelocationType(uint32_t Machine) {
     return ELF::R_SPARC_RELATIVE;
   case ELF::EM_CSKY:
     return ELF::R_CKCORE_RELATIVE;
+  case ELF::EM_VE:
+    return ELF::R_VE_RELATIVE;
   case ELF::EM_AMDGPU:
     break;
   case ELF::EM_BPF:
diff --git a/llvm/lib/Object/MachOObjectFile.cpp b/llvm/lib/Object/MachOObjectFile.cpp
index 7501661591f0..42e257516f4e 100644
--- a/llvm/lib/Object/MachOObjectFile.cpp
+++ b/llvm/lib/Object/MachOObjectFile.cpp
@@ -26,12 +26,15 @@
 #include "llvm/Object/SymbolicFile.h"
 #include "llvm/Support/DataExtractor.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/Errc.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Format.h"
 #include "llvm/Support/Host.h"
 #include "llvm/Support/LEB128.h"
 #include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Path.h"
 #include "llvm/Support/SwapByteOrder.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
@@ -4719,3 +4722,46 @@ StringRef MachOObjectFile::mapDebugSectionName(StringRef Name) const {
       .Case("debug_str_offs", "debug_str_offsets")
       .Default(Name);
 }
+
+Expected<std::vector<std::string>>
+MachOObjectFile::findDsymObjectMembers(StringRef Path) {
+  SmallString<256> BundlePath(Path);
+  // Normalize input path. This is necessary to accept `bundle.dSYM/`.
+  sys::path::remove_dots(BundlePath);
+  if (!sys::fs::is_directory(BundlePath) ||
+      sys::path::extension(BundlePath) != ".dSYM")
+    return std::vector<std::string>();
+  sys::path::append(BundlePath, "Contents", "Resources", "DWARF");
+  bool IsDir;
+  auto EC = sys::fs::is_directory(BundlePath, IsDir);
+  if (EC == errc::no_such_file_or_directory || (!EC && !IsDir))
+    return createStringError(
+        EC, "%s: expected directory 'Contents/Resources/DWARF' in dSYM bundle",
+        Path.str().c_str());
+  if (EC)
+    return createFileError(BundlePath, errorCodeToError(EC));
+
+  std::vector<std::string> ObjectPaths;
+  for (sys::fs::directory_iterator Dir(BundlePath, EC), DirEnd;
+       Dir != DirEnd && !EC; Dir.increment(EC)) {
+    StringRef ObjectPath = Dir->path();
+    sys::fs::file_status Status;
+    if (auto EC = sys::fs::status(ObjectPath, Status))
+      return createFileError(ObjectPath, errorCodeToError(EC));
+    switch (Status.type()) {
+    case sys::fs::file_type::regular_file:
+    case sys::fs::file_type::symlink_file:
+    case sys::fs::file_type::type_unknown:
+      ObjectPaths.push_back(ObjectPath.str());
+      break;
+    default: /*ignore*/;
+    }
+  }
+  if (EC)
+    return createFileError(BundlePath, errorCodeToError(EC));
+  if (ObjectPaths.empty())
+    return createStringError(std::error_code(),
+                             "%s: no objects found in dSYM bundle",
+                             Path.str().c_str());
+  return ObjectPaths;
+}
diff --git a/llvm/lib/Object/MachOUniversalWriter.cpp b/llvm/lib/Object/MachOUniversalWriter.cpp
index 9673c97a10f0..ae1ff09a4f8f 100644
--- a/llvm/lib/Object/MachOUniversalWriter.cpp
+++ b/llvm/lib/Object/MachOUniversalWriter.cpp
@@ -19,7 +19,6 @@
 #include "llvm/Object/IRObjectFile.h"
 #include "llvm/Object/MachO.h"
 #include "llvm/Object/MachOUniversal.h"
-#include "llvm/Support/SmallVectorMemoryBuffer.h"
 
 using namespace llvm;
 using namespace object;
diff --git a/llvm/lib/ObjectYAML/COFFEmitter.cpp b/llvm/lib/ObjectYAML/COFFEmitter.cpp
index 66ad16db1ba4..d884e2fd55cd 100644
--- a/llvm/lib/ObjectYAML/COFFEmitter.cpp
+++ b/llvm/lib/ObjectYAML/COFFEmitter.cpp
@@ -64,11 +64,7 @@ struct COFFParser {
   }
 
   bool parseSections() {
-    for (std::vector<COFFYAML::Section>::iterator i = Obj.Sections.begin(),
-                                                  e = Obj.Sections.end();
-         i != e; ++i) {
-      COFFYAML::Section &Sec = *i;
-
+    for (COFFYAML::Section &Sec : Obj.Sections) {
       // If the name is less than 8 bytes, store it in place, otherwise
       // store it in the string table.
       StringRef Name = Sec.Name;
@@ -103,11 +99,7 @@ struct COFFParser {
   }
 
   bool parseSymbols() {
-    for (std::vector<COFFYAML::Symbol>::iterator i = Obj.Symbols.begin(),
-                                                 e = Obj.Symbols.end();
-         i != e; ++i) {
-      COFFYAML::Symbol &Sym = *i;
-
+    for (COFFYAML::Symbol &Sym : Obj.Symbols) {
       // If the name is less than 8 bytes, store it in place, otherwise
       // store it in the string table.
       StringRef Name = Sym.Name;
diff --git a/llvm/lib/ObjectYAML/ELFYAML.cpp b/llvm/lib/ObjectYAML/ELFYAML.cpp
index e0dde4433d24..9b9266998ea6 100644
--- a/llvm/lib/ObjectYAML/ELFYAML.cpp
+++ b/llvm/lib/ObjectYAML/ELFYAML.cpp
@@ -464,29 +464,31 @@ void ScalarBitSetTraits<ELFYAML::ELF_EF>::bitset(IO &IO,
     BCaseMask(EF_MIPS_ARCH_64R6, EF_MIPS_ARCH);
     break;
   case ELF::EM_HEXAGON:
-    BCase(EF_HEXAGON_MACH_V2);
-    BCase(EF_HEXAGON_MACH_V3);
-    BCase(EF_HEXAGON_MACH_V4);
-    BCase(EF_HEXAGON_MACH_V5);
-    BCase(EF_HEXAGON_MACH_V55);
-    BCase(EF_HEXAGON_MACH_V60);
-    BCase(EF_HEXAGON_MACH_V62);
-    BCase(EF_HEXAGON_MACH_V65);
-    BCase(EF_HEXAGON_MACH_V66);
-    BCase(EF_HEXAGON_MACH_V67);
-    BCase(EF_HEXAGON_MACH_V67T);
-    BCase(EF_HEXAGON_MACH_V68);
-    BCase(EF_HEXAGON_ISA_V2);
-    BCase(EF_HEXAGON_ISA_V3);
-    BCase(EF_HEXAGON_ISA_V4);
-    BCase(EF_HEXAGON_ISA_V5);
-    BCase(EF_HEXAGON_ISA_V55);
-    BCase(EF_HEXAGON_ISA_V60);
-    BCase(EF_HEXAGON_ISA_V62);
-    BCase(EF_HEXAGON_ISA_V65);
-    BCase(EF_HEXAGON_ISA_V66);
-    BCase(EF_HEXAGON_ISA_V67);
-    BCase(EF_HEXAGON_ISA_V68);
+    BCaseMask(EF_HEXAGON_MACH_V2, EF_HEXAGON_MACH);
+    BCaseMask(EF_HEXAGON_MACH_V3, EF_HEXAGON_MACH);
+    BCaseMask(EF_HEXAGON_MACH_V4, EF_HEXAGON_MACH);
+    BCaseMask(EF_HEXAGON_MACH_V5, EF_HEXAGON_MACH);
+    BCaseMask(EF_HEXAGON_MACH_V55, EF_HEXAGON_MACH);
+    BCaseMask(EF_HEXAGON_MACH_V60, EF_HEXAGON_MACH);
+    BCaseMask(EF_HEXAGON_MACH_V62, EF_HEXAGON_MACH);
+    BCaseMask(EF_HEXAGON_MACH_V65, EF_HEXAGON_MACH);
+    BCaseMask(EF_HEXAGON_MACH_V66, EF_HEXAGON_MACH);
+    BCaseMask(EF_HEXAGON_MACH_V67, EF_HEXAGON_MACH);
+    BCaseMask(EF_HEXAGON_MACH_V67T, EF_HEXAGON_MACH);
+    BCaseMask(EF_HEXAGON_MACH_V68, EF_HEXAGON_MACH);
+    BCaseMask(EF_HEXAGON_MACH_V69, EF_HEXAGON_MACH);
+    BCaseMask(EF_HEXAGON_ISA_V2, EF_HEXAGON_ISA);
+    BCaseMask(EF_HEXAGON_ISA_V3, EF_HEXAGON_ISA);
+    BCaseMask(EF_HEXAGON_ISA_V4, EF_HEXAGON_ISA);
+    BCaseMask(EF_HEXAGON_ISA_V5, EF_HEXAGON_ISA);
+    BCaseMask(EF_HEXAGON_ISA_V55, EF_HEXAGON_ISA);
+    BCaseMask(EF_HEXAGON_ISA_V60, EF_HEXAGON_ISA);
+    BCaseMask(EF_HEXAGON_ISA_V62, EF_HEXAGON_ISA);
+    BCaseMask(EF_HEXAGON_ISA_V65, EF_HEXAGON_ISA);
+    BCaseMask(EF_HEXAGON_ISA_V66, EF_HEXAGON_ISA);
+    BCaseMask(EF_HEXAGON_ISA_V67, EF_HEXAGON_ISA);
+    BCaseMask(EF_HEXAGON_ISA_V68, EF_HEXAGON_ISA);
+    BCaseMask(EF_HEXAGON_ISA_V69, EF_HEXAGON_ISA);
     break;
   case ELF::EM_AVR:
     BCaseMask(EF_AVR_ARCH_AVR1, EF_AVR_ARCH_MASK);
diff --git a/llvm/lib/ObjectYAML/XCOFFEmitter.cpp b/llvm/lib/ObjectYAML/XCOFFEmitter.cpp
index 85d1f82bfafc..cf0d058c518c 100644
--- a/llvm/lib/ObjectYAML/XCOFFEmitter.cpp
+++ b/llvm/lib/ObjectYAML/XCOFFEmitter.cpp
@@ -86,13 +86,13 @@ bool XCOFFWriter::nameShouldBeInStringTable(StringRef SymbolName) {
 }
 
 bool XCOFFWriter::initRelocations(uint64_t &CurrentOffset) {
-  for (uint16_t I = 0, E = InitSections.size(); I < E; ++I) {
-    if (!InitSections[I].Relocations.empty()) {
-      InitSections[I].NumberOfRelocations = InitSections[I].Relocations.size();
-      InitSections[I].FileOffsetToRelocations = CurrentOffset;
+  for (XCOFFYAML::Section &InitSection : InitSections) {
+    if (!InitSection.Relocations.empty()) {
+      InitSection.NumberOfRelocations = InitSection.Relocations.size();
+      InitSection.FileOffsetToRelocations = CurrentOffset;
       uint64_t RelSize = Is64Bit ? XCOFF::RelocationSerializationSize64
                                  : XCOFF::RelocationSerializationSize32;
-      CurrentOffset += InitSections[I].NumberOfRelocations * RelSize;
+      CurrentOffset += InitSection.NumberOfRelocations * RelSize;
       if (CurrentOffset > MaxRawDataSize) {
         ErrHandler("maximum object size of" + Twine(MaxRawDataSize) +
                    "exceeded when writing relocation data");
diff --git a/llvm/lib/ObjectYAML/YAML.cpp b/llvm/lib/ObjectYAML/YAML.cpp
index 5dcb113d3395..54e8c627d5a1 100644
--- a/llvm/lib/ObjectYAML/YAML.cpp
+++ b/llvm/lib/ObjectYAML/YAML.cpp
@@ -30,9 +30,8 @@ StringRef yaml::ScalarTraits<yaml::BinaryRef>::input(StringRef Scalar, void *,
     return "BinaryRef hex string must contain an even number of nybbles.";
   // TODO: Can we improve YAMLIO to permit a more accurate diagnostic here?
   // (e.g. a caret pointing to the offending character).
-  for (unsigned I = 0, N = Scalar.size(); I != N; ++I)
-    if (!llvm::isHexDigit(Scalar[I]))
-      return "BinaryRef hex string must contain only hex digits.";
+  if (!llvm::all_of(Scalar, llvm::isHexDigit))
+    return "BinaryRef hex string must contain only hex digits.";
   Val = yaml::BinaryRef(Scalar);
   return {};
 }
diff --git a/llvm/lib/Option/OptTable.cpp b/llvm/lib/Option/OptTable.cpp
index 19e05b9272bb..c93b7ad7f5fa 100644
--- a/llvm/lib/Option/OptTable.cpp
+++ b/llvm/lib/Option/OptTable.cpp
@@ -591,16 +591,16 @@ static void PrintHelpOptionList(raw_ostream &OS, StringRef Title,
 
   // Find the maximum option length.
   unsigned OptionFieldWidth = 0;
-  for (unsigned i = 0, e = OptionHelp.size(); i != e; ++i) {
+  for (const OptionInfo &Opt : OptionHelp) {
     // Limit the amount of padding we are willing to give up for alignment.
-    unsigned Length = OptionHelp[i].Name.size();
+    unsigned Length = Opt.Name.size();
     if (Length <= 23)
       OptionFieldWidth = std::max(OptionFieldWidth, Length);
   }
 
   const unsigned InitialPad = 2;
-  for (unsigned i = 0, e = OptionHelp.size(); i != e; ++i) {
-    const std::string &Option = OptionHelp[i].Name;
+  for (const OptionInfo &Opt : OptionHelp) {
+    const std::string &Option = Opt.Name;
     int Pad = OptionFieldWidth - int(Option.size());
     OS.indent(InitialPad) << Option;
 
@@ -609,7 +609,7 @@ static void PrintHelpOptionList(raw_ostream &OS, StringRef Title,
       OS << "\n";
       Pad = OptionFieldWidth + InitialPad;
     }
-    OS.indent(Pad + 1) << OptionHelp[i].HelpText << '\n';
+    OS.indent(Pad + 1) << Opt.HelpText << '\n';
   }
 }
 
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 561a881bab0c..d7615ef4e9bf 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -28,6 +28,7 @@
 #include "llvm/Analysis/CGSCCPassManager.h"
 #include "llvm/Analysis/CallGraph.h"
 #include "llvm/Analysis/CostModel.h"
+#include "llvm/Analysis/CycleAnalysis.h"
 #include "llvm/Analysis/DDG.h"
 #include "llvm/Analysis/DDGPrinter.h"
 #include "llvm/Analysis/Delinearization.h"
@@ -151,6 +152,7 @@
 #include "llvm/Transforms/Scalar/DeadStoreElimination.h"
 #include "llvm/Transforms/Scalar/DivRemPairs.h"
 #include "llvm/Transforms/Scalar/EarlyCSE.h"
+#include "llvm/Transforms/Scalar/FlattenCFG.h"
 #include "llvm/Transforms/Scalar/Float2Int.h"
 #include "llvm/Transforms/Scalar/GVN.h"
 #include "llvm/Transforms/Scalar/GuardWidening.h"
diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp
index de1b0ace7876..a6a36ff25402 100644
--- a/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -178,6 +178,10 @@ static cl::opt<bool> EnableNoRerunSimplificationPipeline(
         "than once in the case that SCC mutations cause a function to be "
         "visited multiple times as long as the function has not been changed"));
 
+static cl::opt<bool> EnableMergeFunctions(
+    "enable-merge-functions", cl::init(false), cl::Hidden,
+    cl::desc("Enable function merging as part of the optimization pipeline"));
+
 PipelineTuningOptions::PipelineTuningOptions() {
   LoopInterleaving = true;
   LoopVectorization = true;
@@ -187,7 +191,7 @@ PipelineTuningOptions::PipelineTuningOptions() {
   LicmMssaOptCap = SetLicmMssaOptCap;
   LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap;
   CallGraphProfile = true;
-  MergeFunctions = false;
+  MergeFunctions = EnableMergeFunctions;
   EagerlyInvalidateAnalyses = EnableEagerlyInvalidateAnalyses;
 }
 
@@ -418,9 +422,9 @@ PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level,
   FPM.addPass(CorrelatedValuePropagationPass());
 
   FPM.addPass(SimplifyCFGPass());
+  FPM.addPass(InstCombinePass());
   if (Level == OptimizationLevel::O3)
     FPM.addPass(AggressiveInstCombinePass());
-  FPM.addPass(InstCombinePass());
 
   if (!Level.isOptimizingForSize())
     FPM.addPass(LibCallsShrinkWrapPass());
@@ -754,9 +758,11 @@ PassBuilder::buildInlinerPipeline(OptimizationLevel Level,
   return MIWP;
 }
 
-ModuleInlinerPass
+ModulePassManager
 PassBuilder::buildModuleInlinerPipeline(OptimizationLevel Level,
                                         ThinOrFullLTOPhase Phase) {
+  ModulePassManager MPM;
+
   InlineParams IP = getInlineParamsFromOptLevel(Level);
   if (Phase == ThinOrFullLTOPhase::ThinLTOPreLink && PGOOpt &&
       PGOOpt->Action == PGOOptions::SampleUse)
@@ -773,7 +779,16 @@ PassBuilder::buildModuleInlinerPipeline(OptimizationLevel Level,
   // inline deferral logic in module inliner.
   IP.EnableDeferral = false;
 
-  return ModuleInlinerPass(IP, UseInlineAdvisor);
+  MPM.addPass(ModuleInlinerPass(IP, UseInlineAdvisor));
+
+  MPM.addPass(createModuleToFunctionPassAdaptor(
+      buildFunctionSimplificationPipeline(Level, Phase),
+      PTO.EagerlyInvalidateAnalyses));
+
+  MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
+      CoroSplitPass(Level != OptimizationLevel::O0)));
+
+  return MPM;
 }
 
 ModulePassManager
@@ -980,26 +995,28 @@ void PassBuilder::addVectorPasses(OptimizationLevel Level,
   FPM.addPass(InstCombinePass());
 
   if (Level.getSpeedupLevel() > 1 && ExtraVectorizerPasses) {
+    ExtraVectorPassManager ExtraPasses;
     // At higher optimization levels, try to clean up any runtime overlap and
     // alignment checks inserted by the vectorizer. We want to track correlated
     // runtime checks for two inner loops in the same outer loop, fold any
     // common computations, hoist loop-invariant aspects out of any outer loop,
     // and unswitch the runtime checks if possible. Once hoisted, we may have
     // dead (or speculatable) control flows or more combining opportunities.
-    FPM.addPass(EarlyCSEPass());
-    FPM.addPass(CorrelatedValuePropagationPass());
-    FPM.addPass(InstCombinePass());
+    ExtraPasses.addPass(EarlyCSEPass());
+    ExtraPasses.addPass(CorrelatedValuePropagationPass());
+    ExtraPasses.addPass(InstCombinePass());
     LoopPassManager LPM;
     LPM.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap));
     LPM.addPass(SimpleLoopUnswitchPass(/* NonTrivial */ Level ==
                                        OptimizationLevel::O3));
-    FPM.addPass(
+    ExtraPasses.addPass(
         RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
-    FPM.addPass(
+    ExtraPasses.addPass(
         createFunctionToLoopPassAdaptor(std::move(LPM), /*UseMemorySSA=*/true,
                                         /*UseBlockFrequencyInfo=*/true));
-    FPM.addPass(SimplifyCFGPass());
-    FPM.addPass(InstCombinePass());
+    ExtraPasses.addPass(SimplifyCFGPass());
+    ExtraPasses.addPass(InstCombinePass());
+    FPM.addPass(std::move(ExtraPasses));
   }
 
   // Now that we've formed fast to execute loop structures, we do further
@@ -1149,8 +1166,9 @@ PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level,
   // Disable header duplication at -Oz.
   LPM.addPass(LoopRotatePass(Level != OptimizationLevel::Oz, LTOPreLink));
   // Some loops may have become dead by now. Try to delete them.
-  // FIXME: see disscussion in https://reviews.llvm.org/D112851
-  //        this may need to be revisited once GVN is more powerful.
+  // FIXME: see discussion in https://reviews.llvm.org/D112851,
+  //        this may need to be revisited once we run GVN before loop deletion
+  //        in the simplification pipeline.
   LPM.addPass(LoopDeletionPass());
   OptimizePM.addPass(createFunctionToLoopPassAdaptor(
       std::move(LPM), /*UseMemorySSA=*/false, /*UseBlockFrequencyInfo=*/false));
@@ -1167,23 +1185,6 @@ PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level,
 
   addVectorPasses(Level, OptimizePM, /* IsFullLTO */ false);
 
-  // Split out cold code. Splitting is done late to avoid hiding context from
-  // other optimizations and inadvertently regressing performance. The tradeoff
-  // is that this has a higher code size cost than splitting early.
-  if (EnableHotColdSplit && !LTOPreLink)
-    MPM.addPass(HotColdSplittingPass());
-
-  // Search the code for similar regions of code. If enough similar regions can
-  // be found where extracting the regions into their own function will decrease
-  // the size of the program, we extract the regions, a deduplicate the
-  // structurally similar regions.
-  if (EnableIROutliner)
-    MPM.addPass(IROutlinerPass());
-
-  // Merge functions if requested.
-  if (PTO.MergeFunctions)
-    MPM.addPass(MergeFunctionsPass());
-
   // LoopSink pass sinks instructions hoisted by LICM, which serves as a
   // canonicalization pass that enables other optimizations. As a result,
   // LoopSink pass needs to be a very late IR pass to avoid undoing LICM
@@ -1211,6 +1212,23 @@ PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level,
   for (auto &C : OptimizerLastEPCallbacks)
     C(MPM, Level);
 
+  // Split out cold code. Splitting is done late to avoid hiding context from
+  // other optimizations and inadvertently regressing performance. The tradeoff
+  // is that this has a higher code size cost than splitting early.
+  if (EnableHotColdSplit && !LTOPreLink)
+    MPM.addPass(HotColdSplittingPass());
+
+  // Search the code for similar regions of code. If enough similar regions can
+  // be found where extracting the regions into their own function will decrease
+  // the size of the program, we extract the regions, a deduplicate the
+  // structurally similar regions.
+  if (EnableIROutliner)
+    MPM.addPass(IROutlinerPass());
+
+  // Merge functions if requested.
+  if (PTO.MergeFunctions)
+    MPM.addPass(MergeFunctionsPass());
+
   if (PTO.CallGraphProfile)
     MPM.addPass(CGProfilePass());
 
@@ -1521,9 +1539,9 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level,
   // function pointers.  When this happens, we often have to resolve varargs
   // calls, etc, so let instcombine do this.
   FunctionPassManager PeepholeFPM;
+  PeepholeFPM.addPass(InstCombinePass());
   if (Level == OptimizationLevel::O3)
     PeepholeFPM.addPass(AggressiveInstCombinePass());
-  PeepholeFPM.addPass(InstCombinePass());
   invokePeepholeEPCallbacks(PeepholeFPM, Level);
 
   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(PeepholeFPM),
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index c2032b5b8276..74613a7fcce0 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -185,6 +185,7 @@ FUNCTION_ANALYSIS("aa", AAManager())
 FUNCTION_ANALYSIS("assumptions", AssumptionAnalysis())
 FUNCTION_ANALYSIS("block-freq", BlockFrequencyAnalysis())
 FUNCTION_ANALYSIS("branch-prob", BranchProbabilityAnalysis())
+FUNCTION_ANALYSIS("cycles", CycleAnalysis())
 FUNCTION_ANALYSIS("domtree", DominatorTreeAnalysis())
 FUNCTION_ANALYSIS("postdomtree", PostDominatorTreeAnalysis())
 FUNCTION_ANALYSIS("demanded-bits", DemandedBitsAnalysis())
@@ -202,6 +203,7 @@ FUNCTION_ANALYSIS("no-op-function", NoOpFunctionAnalysis())
 FUNCTION_ANALYSIS("opt-remark-emit", OptimizationRemarkEmitterAnalysis())
 FUNCTION_ANALYSIS("scalar-evolution", ScalarEvolutionAnalysis())
 FUNCTION_ANALYSIS("should-not-run-function-passes", ShouldNotRunFunctionPassesAnalysis())
+FUNCTION_ANALYSIS("should-run-extra-vector-passes", ShouldRunExtraVectorPasses())
 FUNCTION_ANALYSIS("stack-safety-local", StackSafetyAnalysis())
 FUNCTION_ANALYSIS("targetlibinfo", TargetLibraryAnalysis())
 FUNCTION_ANALYSIS("targetir",
@@ -253,6 +255,7 @@ FUNCTION_PASS("dse", DSEPass())
 FUNCTION_PASS("dot-cfg", CFGPrinterPass())
 FUNCTION_PASS("dot-cfg-only", CFGOnlyPrinterPass())
 FUNCTION_PASS("fix-irreducible", FixIrreduciblePass())
+FUNCTION_PASS("flattencfg", FlattenCFGPass())
 FUNCTION_PASS("make-guards-explicit", MakeGuardsExplicitPass())
 FUNCTION_PASS("gvn-hoist", GVNHoistPass())
 FUNCTION_PASS("gvn-sink", GVNSinkPass())
@@ -303,6 +306,7 @@ FUNCTION_PASS("print<assumptions>", AssumptionPrinterPass(dbgs()))
 FUNCTION_PASS("print<block-freq>", BlockFrequencyPrinterPass(dbgs()))
 FUNCTION_PASS("print<branch-prob>", BranchProbabilityPrinterPass(dbgs()))
 FUNCTION_PASS("print<cost-model>", CostModelPrinterPass(dbgs()))
+FUNCTION_PASS("print<cycles>", CycleInfoPrinterPass(dbgs()))
 FUNCTION_PASS("print<da>", DependenceAnalysisPrinterPass(dbgs()))
 FUNCTION_PASS("print<divergence>", DivergenceAnalysisPrinterPass(dbgs()))
 FUNCTION_PASS("print<domtree>", DominatorTreePrinterPass(dbgs()))
diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp
index 27a6c519ff82..23c825c78713 100644
--- a/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -1262,11 +1262,6 @@ void InLineChangePrinter::registerCallbacks(PassInstrumentationCallbacks &PIC) {
 
 namespace {
 
-enum IRChangeDiffType { InBefore, InAfter, IsCommon, NumIRChangeDiffTypes };
-
-// Describe where a given element exists.
-std::string Colours[NumIRChangeDiffTypes];
-
 class DisplayNode;
 class DotCfgDiffDisplayGraph;
 
@@ -1274,19 +1269,19 @@ class DotCfgDiffDisplayGraph;
 class DisplayElement {
 public:
   // Is this in before, after, or both?
-  IRChangeDiffType getType() const { return Type; }
+  StringRef getColour() const { return Colour; }
 
 protected:
-  DisplayElement(IRChangeDiffType T) : Type(T) {}
-  const IRChangeDiffType Type;
+  DisplayElement(StringRef Colour) : Colour(Colour) {}
+  const StringRef Colour;
 };
 
 // An edge representing a transition between basic blocks in the
 // dot-cfg-changes graph.
 class DisplayEdge : public DisplayElement {
 public:
-  DisplayEdge(std::string V, DisplayNode &Node, IRChangeDiffType T)
-      : DisplayElement(T), Value(V), Node(Node) {}
+  DisplayEdge(std::string Value, DisplayNode &Node, StringRef Colour)
+      : DisplayElement(Colour), Value(Value), Node(Node) {}
   // The value on which the transition is made.
   std::string getValue() const { return Value; }
   // The node (representing a basic block) reached by this transition.
@@ -1302,8 +1297,8 @@ class DisplayNode : public DisplayElement {
 public:
   // \p C is the content for the node, \p T indicates the colour for the
   // outline of the node
-  DisplayNode(std::string C, IRChangeDiffType T)
-      : DisplayElement(T), Content(C) {}
+  DisplayNode(std::string Content, StringRef Colour)
+      : DisplayElement(Colour), Content(Content) {}
 
   // Iterator to the child nodes.  Required by GraphWriter.
   using ChildIterator = std::unordered_set<DisplayNode *>::const_iterator;
@@ -1315,13 +1310,13 @@ public:
   EdgeIterator edges_begin() const { return EdgePtrs.cbegin(); }
   EdgeIterator edges_end() const { return EdgePtrs.cend(); }
 
-  // Create an edge to \p Node on value \p V, with type \p T.
-  void createEdge(StringRef V, DisplayNode &Node, IRChangeDiffType T);
+  // Create an edge to \p Node on value \p Value, with colour \p Colour.
+  void createEdge(StringRef Value, DisplayNode &Node, StringRef Colour);
 
   // Return the content of this node.
   std::string getContent() const { return Content; }
 
-  // Return the type of the edge to node \p S.
+  // Return the edge to node \p S.
   const DisplayEdge &getEdge(const DisplayNode &To) const {
     assert(EdgeMap.find(&To) != EdgeMap.end() && "Expected to find edge.");
     return *EdgeMap.find(&To)->second;
@@ -1383,9 +1378,9 @@ public:
   }
 
   // Create a node.
-  void createNode(std::string C, IRChangeDiffType T) {
+  void createNode(std::string C, StringRef Colour) {
     assert(!NodeGenerationComplete && "Unexpected node creation");
-    Nodes.emplace_back(C, T);
+    Nodes.emplace_back(C, Colour);
   }
   // Return the node at index \p N to avoid problems with vectors reallocating.
   DisplayNode &getNode(unsigned N) {
@@ -1408,13 +1403,13 @@ public:
 
   // Return a string with colour information for Dot.  Required by GraphWriter.
   std::string getNodeAttributes(const DisplayNode &Node) const {
-    return attribute(Node.getType());
+    return attribute(Node.getColour());
   }
 
   // Return a string with colour information for Dot.  Required by GraphWriter.
   std::string getEdgeColorAttr(const DisplayNode &From,
                                const DisplayNode &To) const {
-    return attribute(From.getEdge(To).getType());
+    return attribute(From.getEdge(To).getColour());
   }
 
   // Get the starting basic block.  Required by GraphWriter.
@@ -1425,7 +1420,9 @@ public:
 
 protected:
   // Return the string containing the colour to use as a Dot attribute.
-  std::string attribute(IRChangeDiffType T) const;
+  std::string attribute(StringRef Colour) const {
+    return "color=" + Colour.str();
+  }
 
   bool NodeGenerationComplete = false;
   const std::string GraphName;
@@ -1434,10 +1431,10 @@ protected:
   DisplayNode *EntryNode = nullptr;
 };
 
-void DisplayNode::createEdge(StringRef V, DisplayNode &Node,
-                             IRChangeDiffType T) {
+void DisplayNode::createEdge(StringRef Value, DisplayNode &Node,
+                             StringRef Colour) {
   assert(!AllEdgesCreated && "Expected to be able to still create edges.");
-  Edges.emplace_back(V.str(), Node, T);
+  Edges.emplace_back(Value.str(), Node, Colour);
   Children.insert(&Node);
 }
 
@@ -1458,13 +1455,14 @@ public:
   DotCfgDiffNode() = delete;
 
   // Create a node in Dot difference graph \p G representing the basic block
-  // represented by \p BD with type \p T (where it exists).
+  // represented by \p BD with colour \p Colour (where it exists).
   DotCfgDiffNode(DotCfgDiff &G, unsigned N, const BlockDataT<DCData> &BD,
-                 IRChangeDiffType T)
-      : Graph(G), N(N), Data{&BD, nullptr}, Type(T) {}
+                 StringRef Colour)
+      : Graph(G), N(N), Data{&BD, nullptr}, Colour(Colour) {}
   DotCfgDiffNode(const DotCfgDiffNode &DN)
-      : Graph(DN.Graph), N(DN.N), Data{DN.Data[0], DN.Data[1]}, Type(DN.Type),
-        EdgesMap(DN.EdgesMap), Children(DN.Children), Edges(DN.Edges) {}
+      : Graph(DN.Graph), N(DN.N), Data{DN.Data[0], DN.Data[1]},
+        Colour(DN.Colour), EdgesMap(DN.EdgesMap), Children(DN.Children),
+        Edges(DN.Edges) {}
 
   unsigned getIndex() const { return N; }
 
@@ -1473,29 +1471,29 @@ public:
     assert(Data[0] && "Expected Data[0] to be set.");
     return Data[0]->getLabel();
   }
-  // Return where this block exists.
-  IRChangeDiffType getType() const { return Type; }
+  // Return the colour for this block
+  StringRef getColour() const { return Colour; }
   // Change this basic block from being only in before to being common.
   // Save the pointer to \p Other.
   void setCommon(const BlockDataT<DCData> &Other) {
     assert(!Data[1] && "Expected only one block datum");
     Data[1] = &Other;
-    Type = IsCommon;
+    Colour = CommonColour;
   }
-  // Add an edge to \p E of type {\p Value, \p T}.
-  void addEdge(unsigned E, StringRef Value, IRChangeDiffType T) {
+  // Add an edge to \p E of colour {\p Value, \p Colour}.
+  void addEdge(unsigned E, StringRef Value, StringRef Colour) {
     // This is a new edge or it is an edge being made common.
-    assert((EdgesMap.count(E) == 0 || T == IsCommon) &&
-           "Unexpected edge count and type.");
-    EdgesMap[E] = {Value.str(), T};
+    assert((EdgesMap.count(E) == 0 || Colour == CommonColour) &&
+           "Unexpected edge count and color.");
+    EdgesMap[E] = {Value.str(), Colour};
   }
   // Record the children and create edges.
   void finalize(DotCfgDiff &G);
 
-  // Return the type of the edge to node \p S.
-  std::pair<std::string, IRChangeDiffType> getEdge(const unsigned S) const {
+  // Return the colour of the edge to node \p S.
+  StringRef getEdgeColour(const unsigned S) const {
     assert(EdgesMap.count(S) == 1 && "Expected to find edge.");
-    return EdgesMap.at(S);
+    return EdgesMap.at(S).second;
   }
 
   // Return the string representing the basic block.
@@ -1508,8 +1506,8 @@ protected:
   DotCfgDiff &Graph;
   const unsigned N;
   const BlockDataT<DCData> *Data[2];
-  IRChangeDiffType Type;
-  std::map<const unsigned, std::pair<std::string, IRChangeDiffType>> EdgesMap;
+  StringRef Colour;
+  std::map<const unsigned, std::pair<std::string, StringRef>> EdgesMap;
   std::vector<unsigned> Children;
   std::vector<unsigned> Edges;
 };
@@ -1552,12 +1550,11 @@ public:
 
 protected:
   // Return the string surrounded by HTML to make it the appropriate colour.
-  std::string colourize(std::string S, IRChangeDiffType T) const;
+  std::string colourize(std::string S, StringRef Colour) const;
 
-  void createNode(StringRef Label, const BlockDataT<DCData> &BD,
-                  IRChangeDiffType T) {
+  void createNode(StringRef Label, const BlockDataT<DCData> &BD, StringRef C) {
     unsigned Pos = Nodes.size();
-    Nodes.emplace_back(*this, Pos, BD, T);
+    Nodes.emplace_back(*this, Pos, BD, C);
     NodePosition.insert({Label, Pos});
   }
 
@@ -1572,7 +1569,7 @@ protected:
 };
 
 std::string DotCfgDiffNode::getBodyContent() const {
-  if (Type == IsCommon) {
+  if (Colour == CommonColour) {
     assert(Data[1] && "Expected Data[1] to be set.");
 
     StringRef SR[2];
@@ -1586,11 +1583,11 @@ std::string DotCfgDiffNode::getBodyContent() const {
     }
 
     SmallString<80> OldLineFormat = formatv(
-        "<FONT COLOR=\"{0}\">%l</FONT><BR align=\"left\"/>", Colours[InBefore]);
+        "<FONT COLOR=\"{0}\">%l</FONT><BR align=\"left\"/>", BeforeColour);
     SmallString<80> NewLineFormat = formatv(
-        "<FONT COLOR=\"{0}\">%l</FONT><BR align=\"left\"/>", Colours[InAfter]);
+        "<FONT COLOR=\"{0}\">%l</FONT><BR align=\"left\"/>", AfterColour);
     SmallString<80> UnchangedLineFormat = formatv(
-        "<FONT COLOR=\"{0}\">%l</FONT><BR align=\"left\"/>", Colours[IsCommon]);
+        "<FONT COLOR=\"{0}\">%l</FONT><BR align=\"left\"/>", CommonColour);
     std::string Diff = Data[0]->getLabel().str();
     Diff += ":\n<BR align=\"left\"/>" +
             doSystemDiff(makeHTMLReady(SR[0]), makeHTMLReady(SR[1]),
@@ -1625,7 +1622,7 @@ std::string DotCfgDiffNode::getBodyContent() const {
   // drop predecessors as they can be big and are redundant
   BS1 = BS1.drop_until([](char C) { return C == '\n'; }).drop_front();
 
-  std::string S = "<FONT COLOR=\"" + Colours[Type] + "\">" + Label.str() + ":";
+  std::string S = "<FONT COLOR=\"" + Colour.str() + "\">" + Label.str() + ":";
 
   // align each line to the left.
   while (BS1.size()) {
@@ -1638,26 +1635,22 @@ std::string DotCfgDiffNode::getBodyContent() const {
   return S;
 }
 
-std::string DotCfgDiff::colourize(std::string S, IRChangeDiffType T) const {
+std::string DotCfgDiff::colourize(std::string S, StringRef Colour) const {
   if (S.length() == 0)
     return S;
-  return "<FONT COLOR=\"" + Colours[T] + "\">" + S + "</FONT>";
-}
-
-std::string DotCfgDiffDisplayGraph::attribute(IRChangeDiffType T) const {
-  return "color=" + Colours[T];
+  return "<FONT COLOR=\"" + Colour.str() + "\">" + S + "</FONT>";
 }
 
 DotCfgDiff::DotCfgDiff(StringRef Title, const FuncDataT<DCData> &Before,
                        const FuncDataT<DCData> &After)
     : GraphName(Title.str()) {
-  StringMap<IRChangeDiffType> EdgesMap;
+  StringMap<StringRef> EdgesMap;
 
   // Handle each basic block in the before IR.
   for (auto &B : Before.getData()) {
     StringRef Label = B.getKey();
     const BlockDataT<DCData> &BD = B.getValue();
-    createNode(Label, BD, InBefore);
+    createNode(Label, BD, BeforeColour);
 
     // Create transitions with names made up of the from block label, the value
     // on which the transition is made and the to block label.
@@ -1666,7 +1659,7 @@ DotCfgDiff::DotCfgDiff(StringRef Title, const FuncDataT<DCData> &Before,
          Sink != E; ++Sink) {
       std::string Key = (Label + " " + Sink->getKey().str()).str() + " " +
                         BD.getData().getSuccessorLabel(Sink->getKey()).str();
-      EdgesMap.insert({Key, InBefore});
+      EdgesMap.insert({Key, BeforeColour});
     }
   }
 
@@ -1677,7 +1670,7 @@ DotCfgDiff::DotCfgDiff(StringRef Title, const FuncDataT<DCData> &Before,
     unsigned C = NodePosition.count(Label);
     if (C == 0)
       // This only exists in the after IR.  Create the node.
-      createNode(Label, BD, InAfter);
+      createNode(Label, BD, AfterColour);
     else {
       assert(C == 1 && "Unexpected multiple nodes.");
       Nodes[NodePosition[Label]].setCommon(BD);
@@ -1690,9 +1683,9 @@ DotCfgDiff::DotCfgDiff(StringRef Title, const FuncDataT<DCData> &Before,
                         BD.getData().getSuccessorLabel(Sink->getKey()).str();
       unsigned C = EdgesMap.count(Key);
       if (C == 0)
-        EdgesMap.insert({Key, InAfter});
+        EdgesMap.insert({Key, AfterColour});
       else {
-        EdgesMap[Key] = IsCommon;
+        EdgesMap[Key] = CommonColour;
       }
     }
   }
@@ -1712,18 +1705,18 @@ DotCfgDiff::DotCfgDiff(StringRef Title, const FuncDataT<DCData> &Before,
     DotCfgDiffNode &SourceNode = Nodes[NodePosition[Source]];
     assert(NodePosition.count(Sink) == 1 && "Expected to find node.");
     unsigned SinkNode = NodePosition[Sink];
-    IRChangeDiffType T = E.second;
+    StringRef Colour = E.second;
 
     // Look for an edge from Source to Sink
     if (EdgeLabels.count(SourceSink) == 0)
-      EdgeLabels.insert({SourceSink, colourize(Value.str(), T)});
+      EdgeLabels.insert({SourceSink, colourize(Value.str(), Colour)});
     else {
       StringRef V = EdgeLabels.find(SourceSink)->getValue();
-      std::string NV = colourize(V.str() + " " + Value.str(), T);
-      T = IsCommon;
+      std::string NV = colourize(V.str() + " " + Value.str(), Colour);
+      Colour = CommonColour;
       EdgeLabels[SourceSink] = NV;
     }
-    SourceNode.addEdge(SinkNode, Value, T);
+    SourceNode.addEdge(SinkNode, Value, Colour);
   }
   for (auto &I : Nodes)
     I.finalize(*this);
@@ -1744,7 +1737,7 @@ DotCfgDiffDisplayGraph DotCfgDiff::createDisplayGraph(StringRef Title,
   for (auto &I : Nodes) {
     if (I.getIndex() == Entry)
       EntryIndex = Index;
-    G.createNode(I.getBodyContent(), I.getType());
+    G.createNode(I.getBodyContent(), I.getColour());
     NodeMap.insert({I.getIndex(), Index++});
   }
   assert(EntryIndex >= 0 && "Expected entry node index to be set.");
@@ -1766,12 +1759,12 @@ void DotCfgDiffNode::createDisplayEdges(
 
   for (auto I : Edges) {
     unsigned SinkNodeIndex = I;
-    IRChangeDiffType Type = getEdge(SinkNodeIndex).second;
+    StringRef Colour = getEdgeColour(SinkNodeIndex);
     const DotCfgDiffNode *SinkNode = &Graph.getNode(SinkNodeIndex);
 
     StringRef Label = Graph.getEdgeSourceLabel(getIndex(), SinkNodeIndex);
     DisplayNode &SinkDisplayNode = DisplayGraph.getNode(SinkNode->getIndex());
-    SourceDisplayNode.createEdge(Label, SinkDisplayNode, Type);
+    SourceDisplayNode.createEdge(Label, SinkDisplayNode, Colour);
   }
   SourceDisplayNode.createEdgeMap();
 }
@@ -1891,12 +1884,7 @@ DCData::DCData(const BasicBlock &B) {
 }
 
 DotCfgChangeReporter::DotCfgChangeReporter(bool Verbose)
-    : ChangeReporter<IRDataT<DCData>>(Verbose) {
-  // Set up the colours based on the hidden options.
-  Colours[InBefore] = BeforeColour;
-  Colours[InAfter] = AfterColour;
-  Colours[IsCommon] = CommonColour;
-}
+    : ChangeReporter<IRDataT<DCData>>(Verbose) {}
 
 void DotCfgChangeReporter::handleFunctionCompare(
     StringRef Name, StringRef Prefix, StringRef PassID, StringRef Divider,
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index ab3487ecffe8..34e0c5ebcd58 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -110,6 +110,18 @@ static std::string getInstrProfErrString(instrprof_error Err,
   case instrprof_error::malformed:
     OS << "malformed instrumentation profile data";
     break;
+  case instrprof_error::missing_debug_info_for_correlation:
+    OS << "debug info for correlation is required";
+    break;
+  case instrprof_error::unexpected_debug_info_for_correlation:
+    OS << "debug info for correlation is not necessary";
+    break;
+  case instrprof_error::unable_to_correlate_profile:
+    OS << "unable to correlate profile";
+    break;
+  case instrprof_error::unsupported_debug_format:
+    OS << "unsupported debug info format (only DWARF is supported)";
+    break;
   case instrprof_error::invalid_prof:
     OS << "invalid profile created. Please file a bug "
           "at: " BUG_REPORT_URL
@@ -533,8 +545,8 @@ Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) {
 void InstrProfRecord::accumulateCounts(CountSumOrPercent &Sum) const {
   uint64_t FuncSum = 0;
   Sum.NumEntries += Counts.size();
-  for (size_t F = 0, E = Counts.size(); F < E; ++F)
-    FuncSum += Counts[F];
+  for (uint64_t Count : Counts)
+    FuncSum += Count;
   Sum.CountSum += FuncSum;
 
   for (uint32_t VK = IPVK_First; VK <= IPVK_Last; ++VK) {
@@ -674,9 +686,9 @@ void InstrProfValueSiteRecord::merge(InstrProfValueSiteRecord &Input,
 
 void InstrProfValueSiteRecord::scale(uint64_t N, uint64_t D,
                                      function_ref<void(instrprof_error)> Warn) {
-  for (auto I = ValueData.begin(), IE = ValueData.end(); I != IE; ++I) {
+  for (InstrProfValueData &I : ValueData) {
     bool Overflowed;
-    I->Count = SaturatingMultiply(I->Count, N, &Overflowed) / D;
+    I.Count = SaturatingMultiply(I.Count, N, &Overflowed) / D;
     if (Overflowed)
       Warn(instrprof_error::counter_overflow);
   }
@@ -1175,7 +1187,8 @@ bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken) {
 // Create a COMDAT variable INSTR_PROF_RAW_VERSION_VAR to make the runtime
 // aware this is an ir_level profile so it can set the version flag.
 GlobalVariable *createIRLevelProfileFlagVar(Module &M, bool IsCS,
-                                            bool InstrEntryBBEnabled) {
+                                            bool InstrEntryBBEnabled,
+                                            bool DebugInfoCorrelate) {
   const StringRef VarName(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR));
   Type *IntTy64 = Type::getInt64Ty(M.getContext());
   uint64_t ProfileVersion = (INSTR_PROF_RAW_VERSION | VARIANT_MASK_IR_PROF);
@@ -1183,6 +1196,8 @@ GlobalVariable *createIRLevelProfileFlagVar(Module &M, bool IsCS,
     ProfileVersion |= VARIANT_MASK_CSIR_PROF;
   if (InstrEntryBBEnabled)
     ProfileVersion |= VARIANT_MASK_INSTR_ENTRY;
+  if (DebugInfoCorrelate)
+    ProfileVersion |= VARIANT_MASK_DBG_CORRELATE;
   auto IRLevelVersionVariable = new GlobalVariable(
       M, IntTy64, true, GlobalValue::WeakAnyLinkage,
       Constant::getIntegerValue(IntTy64, APInt(64, ProfileVersion)), VarName);
diff --git a/llvm/lib/ProfileData/InstrProfCorrelator.cpp b/llvm/lib/ProfileData/InstrProfCorrelator.cpp
new file mode 100644
index 000000000000..f9c113027da2
--- /dev/null
+++ b/llvm/lib/ProfileData/InstrProfCorrelator.cpp
@@ -0,0 +1,264 @@
+//===-- InstrProfCorrelator.cpp -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ProfileData/InstrProfCorrelator.h"
+#include "llvm/Object/MachO.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+
+#define DEBUG_TYPE "correlator"
+
+using namespace llvm;
+
+/// Get the __llvm_prf_cnts section.
+Expected<object::SectionRef> getCountersSection(const object::ObjectFile &Obj) {
+  for (auto &Section : Obj.sections())
+    if (auto SectionName = Section.getName())
+      if (SectionName.get() == INSTR_PROF_CNTS_SECT_NAME)
+        return Section;
+  return make_error<InstrProfError>(
+      instrprof_error::unable_to_correlate_profile);
+}
+
+const char *InstrProfCorrelator::FunctionNameAttributeName = "Function Name";
+const char *InstrProfCorrelator::CFGHashAttributeName = "CFG Hash";
+const char *InstrProfCorrelator::NumCountersAttributeName = "Num Counters";
+
+llvm::Expected<std::unique_ptr<InstrProfCorrelator::Context>>
+InstrProfCorrelator::Context::get(std::unique_ptr<MemoryBuffer> Buffer,
+                                  const object::ObjectFile &Obj) {
+  auto CountersSection = getCountersSection(Obj);
+  if (auto Err = CountersSection.takeError())
+    return std::move(Err);
+  auto C = std::make_unique<Context>();
+  C->Buffer = std::move(Buffer);
+  C->CountersSectionStart = CountersSection->getAddress();
+  C->CountersSectionEnd = C->CountersSectionStart + CountersSection->getSize();
+  C->ShouldSwapBytes = Obj.isLittleEndian() != sys::IsLittleEndianHost;
+  return Expected<std::unique_ptr<Context>>(std::move(C));
+}
+
+llvm::Expected<std::unique_ptr<InstrProfCorrelator>>
+InstrProfCorrelator::get(StringRef DebugInfoFilename) {
+  auto DsymObjectsOrErr =
+      object::MachOObjectFile::findDsymObjectMembers(DebugInfoFilename);
+  if (auto Err = DsymObjectsOrErr.takeError())
+    return std::move(Err);
+  if (!DsymObjectsOrErr->empty()) {
+    // TODO: Enable profile correlation when there are multiple objects in a
+    // dSYM bundle.
+    if (DsymObjectsOrErr->size() > 1)
+      return createStringError(
+          std::error_code(),
+          "Profile correlation using multiple objects is not yet supported");
+    DebugInfoFilename = *DsymObjectsOrErr->begin();
+  }
+  auto BufferOrErr =
+      errorOrToExpected(MemoryBuffer::getFile(DebugInfoFilename));
+  if (auto Err = BufferOrErr.takeError())
+    return std::move(Err);
+
+  return get(std::move(*BufferOrErr));
+}
+
+llvm::Expected<std::unique_ptr<InstrProfCorrelator>>
+InstrProfCorrelator::get(std::unique_ptr<MemoryBuffer> Buffer) {
+  auto BinOrErr = object::createBinary(*Buffer);
+  if (auto Err = BinOrErr.takeError())
+    return std::move(Err);
+
+  if (auto *Obj = dyn_cast<object::ObjectFile>(BinOrErr->get())) {
+    auto CtxOrErr = Context::get(std::move(Buffer), *Obj);
+    if (auto Err = CtxOrErr.takeError())
+      return std::move(Err);
+    auto T = Obj->makeTriple();
+    if (T.isArch64Bit())
+      return InstrProfCorrelatorImpl<uint64_t>::get(std::move(*CtxOrErr), *Obj);
+    if (T.isArch32Bit())
+      return InstrProfCorrelatorImpl<uint32_t>::get(std::move(*CtxOrErr), *Obj);
+  }
+  return make_error<InstrProfError>(
+      instrprof_error::unable_to_correlate_profile);
+}
+
+namespace llvm {
+
+template <>
+InstrProfCorrelatorImpl<uint32_t>::InstrProfCorrelatorImpl(
+    std::unique_ptr<InstrProfCorrelator::Context> Ctx)
+    : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_32Bit,
+                              std::move(Ctx)) {}
+template <>
+InstrProfCorrelatorImpl<uint64_t>::InstrProfCorrelatorImpl(
+    std::unique_ptr<InstrProfCorrelator::Context> Ctx)
+    : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_64Bit,
+                              std::move(Ctx)) {}
+template <>
+bool InstrProfCorrelatorImpl<uint32_t>::classof(const InstrProfCorrelator *C) {
+  return C->getKind() == InstrProfCorrelatorKind::CK_32Bit;
+}
+template <>
+bool InstrProfCorrelatorImpl<uint64_t>::classof(const InstrProfCorrelator *C) {
+  return C->getKind() == InstrProfCorrelatorKind::CK_64Bit;
+}
+
+} // end namespace llvm
+
+template <class IntPtrT>
+llvm::Expected<std::unique_ptr<InstrProfCorrelatorImpl<IntPtrT>>>
+InstrProfCorrelatorImpl<IntPtrT>::get(
+    std::unique_ptr<InstrProfCorrelator::Context> Ctx,
+    const object::ObjectFile &Obj) {
+  if (Obj.isELF() || Obj.isMachO()) {
+    auto DICtx = DWARFContext::create(Obj);
+    return std::make_unique<DwarfInstrProfCorrelator<IntPtrT>>(std::move(DICtx),
+                                                               std::move(Ctx));
+  }
+  return make_error<InstrProfError>(instrprof_error::unsupported_debug_format);
+}
+
+template <class IntPtrT>
+Error InstrProfCorrelatorImpl<IntPtrT>::correlateProfileData() {
+  assert(Data.empty() && CompressedNames.empty() && Names.empty());
+  correlateProfileDataImpl();
+  auto Result =
+      collectPGOFuncNameStrings(Names, /*doCompression=*/true, CompressedNames);
+  Names.clear();
+  return Result;
+}
+
+template <class IntPtrT>
+void InstrProfCorrelatorImpl<IntPtrT>::addProbe(StringRef FunctionName,
+                                                uint64_t CFGHash,
+                                                IntPtrT CounterOffset,
+                                                IntPtrT FunctionPtr,
+                                                uint32_t NumCounters) {
+  Data.push_back({
+      maybeSwap<uint64_t>(IndexedInstrProf::ComputeHash(FunctionName)),
+      maybeSwap<uint64_t>(CFGHash),
+      // In this mode, CounterPtr actually stores the section relative address
+      // of the counter.
+      maybeSwap<IntPtrT>(CounterOffset),
+      maybeSwap<IntPtrT>(FunctionPtr),
+      // TODO: Value profiling is not yet supported.
+      /*ValuesPtr=*/maybeSwap<IntPtrT>(0),
+      maybeSwap<uint32_t>(NumCounters),
+      /*NumValueSites=*/{maybeSwap<uint16_t>(0), maybeSwap<uint16_t>(0)},
+  });
+  Names.push_back(FunctionName.str());
+}
+
+template <class IntPtrT>
+llvm::Optional<uint64_t>
+DwarfInstrProfCorrelator<IntPtrT>::getLocation(const DWARFDie &Die) const {
+  auto Locations = Die.getLocations(dwarf::DW_AT_location);
+  if (!Locations) {
+    consumeError(Locations.takeError());
+    return {};
+  }
+  auto &DU = *Die.getDwarfUnit();
+  for (auto &Location : *Locations) {
+    auto AddressSize = DU.getAddressByteSize();
+    DataExtractor Data(Location.Expr, DICtx->isLittleEndian(), AddressSize);
+    DWARFExpression Expr(Data, AddressSize);
+    for (auto &Op : Expr)
+      if (Op.getCode() == dwarf::DW_OP_addr)
+        return Op.getRawOperand(0);
+  }
+  return {};
+}
+
+template <class IntPtrT>
+bool DwarfInstrProfCorrelator<IntPtrT>::isDIEOfProbe(const DWARFDie &Die) {
+  const auto &ParentDie = Die.getParent();
+  if (!Die.isValid() || !ParentDie.isValid() || Die.isNULL())
+    return false;
+  if (Die.getTag() != dwarf::DW_TAG_variable)
+    return false;
+  if (!ParentDie.isSubprogramDIE())
+    return false;
+  if (!Die.hasChildren())
+    return false;
+  if (const char *Name = Die.getName(DINameKind::ShortName))
+    return StringRef(Name).startswith(getInstrProfCountersVarPrefix());
+  return false;
+}
+
+template <class IntPtrT>
+void DwarfInstrProfCorrelator<IntPtrT>::correlateProfileDataImpl() {
+  auto maybeAddProbe = [&](DWARFDie Die) {
+    if (!isDIEOfProbe(Die))
+      return;
+    Optional<const char *> FunctionName;
+    Optional<uint64_t> CFGHash;
+    Optional<uint64_t> CounterPtr = getLocation(Die);
+    auto FunctionPtr =
+        dwarf::toAddress(Die.getParent().find(dwarf::DW_AT_low_pc));
+    Optional<uint64_t> NumCounters;
+    for (const DWARFDie &Child : Die.children()) {
+      if (Child.getTag() != dwarf::DW_TAG_LLVM_annotation)
+        continue;
+      auto AnnotationFormName = Child.find(dwarf::DW_AT_name);
+      auto AnnotationFormValue = Child.find(dwarf::DW_AT_const_value);
+      if (!AnnotationFormName || !AnnotationFormValue)
+        continue;
+      auto AnnotationNameOrErr = AnnotationFormName->getAsCString();
+      if (auto Err = AnnotationNameOrErr.takeError()) {
+        consumeError(std::move(Err));
+        continue;
+      }
+      StringRef AnnotationName = *AnnotationNameOrErr;
+      if (AnnotationName.compare(
+              InstrProfCorrelator::FunctionNameAttributeName) == 0) {
+        if (auto EC =
+                AnnotationFormValue->getAsCString().moveInto(FunctionName))
+          consumeError(std::move(EC));
+      } else if (AnnotationName.compare(
+                     InstrProfCorrelator::CFGHashAttributeName) == 0) {
+        CFGHash = AnnotationFormValue->getAsUnsignedConstant();
+      } else if (AnnotationName.compare(
+                     InstrProfCorrelator::NumCountersAttributeName) == 0) {
+        NumCounters = AnnotationFormValue->getAsUnsignedConstant();
+      }
+    }
+    if (!FunctionName || !CFGHash || !CounterPtr || !NumCounters) {
+      LLVM_DEBUG(dbgs() << "Incomplete DIE for probe\n\tFunctionName: "
+                        << FunctionName << "\n\tCFGHash: " << CFGHash
+                        << "\n\tCounterPtr: " << CounterPtr
+                        << "\n\tNumCounters: " << NumCounters);
+      LLVM_DEBUG(Die.dump(dbgs()));
+      return;
+    }
+    uint64_t CountersStart = this->Ctx->CountersSectionStart;
+    uint64_t CountersEnd = this->Ctx->CountersSectionEnd;
+    if (*CounterPtr < CountersStart || *CounterPtr >= CountersEnd) {
+      LLVM_DEBUG(
+          dbgs() << "CounterPtr out of range for probe\n\tFunction Name: "
+                 << FunctionName << "\n\tExpected: [0x"
+                 << Twine::utohexstr(CountersStart) << ", 0x"
+                 << Twine::utohexstr(CountersEnd) << ")\n\tActual: 0x"
+                 << Twine::utohexstr(*CounterPtr));
+      LLVM_DEBUG(Die.dump(dbgs()));
+      return;
+    }
+    if (!FunctionPtr) {
+      LLVM_DEBUG(dbgs() << "Could not find address of " << *FunctionName
+                        << "\n");
+      LLVM_DEBUG(Die.dump(dbgs()));
+    }
+    this->addProbe(*FunctionName, *CFGHash, *CounterPtr - CountersStart,
+                   FunctionPtr.getValueOr(0), *NumCounters);
+  };
+  for (auto &CU : DICtx->normal_units())
+    for (const auto &Entry : CU->dies())
+      maybeAddProbe(DWARFDie(CU.get(), &Entry));
+  for (auto &CU : DICtx->dwo_units())
+    for (const auto &Entry : CU->dies())
+      maybeAddProbe(DWARFDie(CU.get(), &Entry));
+}
diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp
index 885c1fe49240..37cdf4dd1fe2 100644
--- a/llvm/lib/ProfileData/InstrProfReader.cpp
+++ b/llvm/lib/ProfileData/InstrProfReader.cpp
@@ -52,16 +52,19 @@ static Error initializeReader(InstrProfReader &Reader) {
 }
 
 Expected<std::unique_ptr<InstrProfReader>>
-InstrProfReader::create(const Twine &Path) {
+InstrProfReader::create(const Twine &Path,
+                        const InstrProfCorrelator *Correlator) {
   // Set up the buffer to read.
   auto BufferOrError = setupMemoryBuffer(Path);
   if (Error E = BufferOrError.takeError())
     return std::move(E);
-  return InstrProfReader::create(std::move(BufferOrError.get()));
+  return InstrProfReader::create(std::move(BufferOrError.get()), Correlator);
 }
 
 Expected<std::unique_ptr<InstrProfReader>>
-InstrProfReader::create(std::unique_ptr<MemoryBuffer> Buffer) {
+InstrProfReader::create(std::unique_ptr<MemoryBuffer> Buffer,
+                        const InstrProfCorrelator *Correlator) {
+  // Sanity check the buffer.
   if (uint64_t(Buffer->getBufferSize()) > std::numeric_limits<uint64_t>::max())
     return make_error<InstrProfError>(instrprof_error::too_large);
 
@@ -73,9 +76,9 @@ InstrProfReader::create(std::unique_ptr<MemoryBuffer> Buffer) {
   if (IndexedInstrProfReader::hasFormat(*Buffer))
     Result.reset(new IndexedInstrProfReader(std::move(Buffer)));
   else if (RawInstrProfReader64::hasFormat(*Buffer))
-    Result.reset(new RawInstrProfReader64(std::move(Buffer)));
+    Result.reset(new RawInstrProfReader64(std::move(Buffer), Correlator));
   else if (RawInstrProfReader32::hasFormat(*Buffer))
-    Result.reset(new RawInstrProfReader32(std::move(Buffer)));
+    Result.reset(new RawInstrProfReader32(std::move(Buffer), Correlator));
   else if (TextInstrProfReader::hasFormat(*Buffer))
     Result.reset(new TextInstrProfReader(std::move(Buffer)));
   else
@@ -352,7 +355,7 @@ Error RawInstrProfReader<IntPtrT>::readNextHeader(const char *CurrentPos) {
 
 template <class IntPtrT>
 Error RawInstrProfReader<IntPtrT>::createSymtab(InstrProfSymtab &Symtab) {
-  if (Error E = Symtab.create(StringRef(NamesStart, NamesSize)))
+  if (Error E = Symtab.create(StringRef(NamesStart, NamesEnd - NamesStart)))
     return error(std::move(E));
   for (const RawInstrProf::ProfileData<IntPtrT> *I = Data; I != DataEnd; ++I) {
     const IntPtrT FPtr = swap(I->FunctionPointer);
@@ -369,6 +372,10 @@ Error RawInstrProfReader<IntPtrT>::readHeader(
   Version = swap(Header.Version);
   if (GET_VERSION(Version) != RawInstrProf::Version)
     return error(instrprof_error::unsupported_version);
+  if (useDebugInfoCorrelate() && !Correlator)
+    return error(instrprof_error::missing_debug_info_for_correlation);
+  if (!useDebugInfoCorrelate() && Correlator)
+    return error(instrprof_error::unexpected_debug_info_for_correlation);
 
   BinaryIdsSize = swap(Header.BinaryIdsSize);
   if (BinaryIdsSize % sizeof(uint64_t))
@@ -380,7 +387,7 @@ Error RawInstrProfReader<IntPtrT>::readHeader(
   auto PaddingBytesBeforeCounters = swap(Header.PaddingBytesBeforeCounters);
   auto CountersSize = swap(Header.CountersSize);
   auto PaddingBytesAfterCounters = swap(Header.PaddingBytesAfterCounters);
-  NamesSize = swap(Header.NamesSize);
+  auto NamesSize = swap(Header.NamesSize);
   ValueKindLast = swap(Header.ValueKindLast);
 
   auto DataSizeInBytes = DataSize * sizeof(RawInstrProf::ProfileData<IntPtrT>);
@@ -398,15 +405,27 @@ Error RawInstrProfReader<IntPtrT>::readHeader(
   if (Start + ValueDataOffset > DataBuffer->getBufferEnd())
     return error(instrprof_error::bad_header);
 
-  Data = reinterpret_cast<const RawInstrProf::ProfileData<IntPtrT> *>(
-      Start + DataOffset);
-  DataEnd = Data + DataSize;
+  if (Correlator) {
+    // These sizes in the raw file are zero because we constructed them in the
+    // Correlator.
+    assert(DataSize == 0 && NamesSize == 0);
+    assert(CountersDelta == 0 && NamesDelta == 0);
+    Data = Correlator->getDataPointer();
+    DataEnd = Data + Correlator->getDataSize();
+    NamesStart = Correlator->getCompressedNamesPointer();
+    NamesEnd = NamesStart + Correlator->getCompressedNamesSize();
+  } else {
+    Data = reinterpret_cast<const RawInstrProf::ProfileData<IntPtrT> *>(
+        Start + DataOffset);
+    DataEnd = Data + DataSize;
+    NamesStart = Start + NamesOffset;
+    NamesEnd = NamesStart + NamesSize;
+  }
 
   // Binary ids start just after the header.
   BinaryIdsStart =
       reinterpret_cast<const uint8_t *>(&Header) + sizeof(RawInstrProf::Header);
   CountersStart = reinterpret_cast<const uint64_t *>(Start + CountersOffset);
-  NamesStart = Start + NamesOffset;
   ValueDataStart = reinterpret_cast<const uint8_t *>(Start + ValueDataOffset);
 
   const uint8_t *BufferEnd = (const uint8_t *)DataBuffer->getBufferEnd();
@@ -440,45 +459,50 @@ Error RawInstrProfReader<IntPtrT>::readRawCounts(
   if (NumCounters == 0)
     return error(instrprof_error::malformed, "number of counters is zero");
 
-  IntPtrT CounterPtr = Data->CounterPtr;
-  auto *NamesStartAsCounter = reinterpret_cast<const uint64_t *>(NamesStart);
-  ptrdiff_t MaxNumCounters = NamesStartAsCounter - CountersStart;
-
-  // Check bounds. Note that the counter pointer embedded in the data record
-  // may itself be corrupt.
-  if (MaxNumCounters < 0 || NumCounters > (uint32_t)MaxNumCounters)
-    return error(instrprof_error::malformed,
-                 "counter pointer is out of bounds");
-
-  // We need to compute the in-buffer counter offset from the in-memory address
-  // distance. The initial CountersDelta is the in-memory address difference
-  // start(__llvm_prf_cnts)-start(__llvm_prf_data), so SrcData->CounterPtr -
-  // CountersDelta computes the offset into the in-buffer counter section.
-  //
-  // CountersDelta decreases as we advance to the next data record.
-  ptrdiff_t CounterOffset = getCounterOffset(CounterPtr);
-  CountersDelta -= sizeof(*Data);
-  if (CounterOffset < 0)
-    return error(
-        instrprof_error::malformed,
-        ("counter offset " + Twine(CounterOffset) + " is negative").str());
-
-  if (CounterOffset > MaxNumCounters)
-    return error(instrprof_error::malformed,
-                 ("counter offset " + Twine(CounterOffset) +
-                  " is greater than the maximum number of counters " +
-                  Twine((uint32_t)MaxNumCounters))
-                     .str());
-
-  if (((uint32_t)CounterOffset + NumCounters) > (uint32_t)MaxNumCounters)
-    return error(instrprof_error::malformed,
-                 ("number of counters " +
-                  Twine(((uint32_t)CounterOffset + NumCounters)) +
-                  " is greater than the maximum number of counters " +
-                  Twine((uint32_t)MaxNumCounters))
-                     .str());
-
-  auto RawCounts = makeArrayRef(getCounter(CounterOffset), NumCounters);
+  ArrayRef<uint64_t> RawCounts;
+  if (Correlator) {
+    uint64_t CounterOffset = swap<IntPtrT>(Data->CounterPtr) / sizeof(uint64_t);
+    RawCounts =
+        makeArrayRef<uint64_t>(CountersStart + CounterOffset, NumCounters);
+  } else {
+    IntPtrT CounterPtr = Data->CounterPtr;
+    ptrdiff_t CounterOffset = getCounterOffset(CounterPtr);
+    if (CounterOffset < 0)
+      return error(
+          instrprof_error::malformed,
+          ("counter offset " + Twine(CounterOffset) + " is negative").str());
+
+    // Check bounds. Note that the counter pointer embedded in the data record
+    // may itself be corrupt.
+    auto *NamesStartAsCounter = reinterpret_cast<const uint64_t *>(NamesStart);
+    ptrdiff_t MaxNumCounters = NamesStartAsCounter - CountersStart;
+    if (MaxNumCounters < 0 || NumCounters > (uint32_t)MaxNumCounters)
+      return error(instrprof_error::malformed,
+                   "counter pointer is out of bounds");
+    // We need to compute the in-buffer counter offset from the in-memory
+    // address distance. The initial CountersDelta is the in-memory address
+    // difference start(__llvm_prf_cnts)-start(__llvm_prf_data), so
+    // SrcData->CounterPtr - CountersDelta computes the offset into the
+    // in-buffer counter section.
+    if (CounterOffset > MaxNumCounters)
+      return error(instrprof_error::malformed,
+                   ("counter offset " + Twine(CounterOffset) +
+                    " is greater than the maximum number of counters " +
+                    Twine((uint32_t)MaxNumCounters))
+                       .str());
+
+    if (((uint32_t)CounterOffset + NumCounters) > (uint32_t)MaxNumCounters)
+      return error(instrprof_error::malformed,
+                   ("number of counters " +
+                    Twine(((uint32_t)CounterOffset + NumCounters)) +
+                    " is greater than the maximum number of counters " +
+                    Twine((uint32_t)MaxNumCounters))
+                       .str());
+    // CountersDelta decreases as we advance to the next data record.
+    CountersDelta -= sizeof(*Data);
+
+    RawCounts = makeArrayRef(getCounter(CounterOffset), NumCounters);
+  }
 
   if (ShouldSwapBytes) {
     Record.Counts.clear();
@@ -977,11 +1001,10 @@ IndexedInstrProfReader::getInstrProfRecord(StringRef FuncName,
   if (Err)
     return std::move(Err);
   // Found it. Look for counters with the right hash.
-  for (unsigned I = 0, E = Data.size(); I < E; ++I) {
+  for (const NamedInstrProfRecord &I : Data) {
     // Check for a match and fill the vector if there is one.
-    if (Data[I].Hash == FuncHash) {
-      return std::move(Data[I]);
-    }
+    if (I.Hash == FuncHash)
+      return std::move(I);
   }
   return error(instrprof_error::hash_mismatch);
 }
diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp
index 492e3541cb5a..6628eea80640 100644
--- a/llvm/lib/ProfileData/InstrProfWriter.cpp
+++ b/llvm/lib/ProfileData/InstrProfWriter.cpp
@@ -32,6 +32,7 @@
 #include <vector>
 
 using namespace llvm;
+extern cl::opt<bool> DebugInfoCorrelate;
 
 // A struct to define how the data stream should be patched. For Indexed
 // profiling, only uint64_t data type is needed.
diff --git a/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp b/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp
index f54df7b295e3..bbb640cfaee8 100644
--- a/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp
+++ b/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp
@@ -194,7 +194,7 @@ SampleProfileSummaryBuilder::computeSummaryForProfiles(
   // more function profiles each with lower counts, which in turn leads to lower
   // hot thresholds. To compensate for that, by default we merge context
   // profiles before computing profile summary.
-  if (UseContextLessSummary || (sampleprof::FunctionSamples::ProfileIsCS &&
+  if (UseContextLessSummary || (sampleprof::FunctionSamples::ProfileIsCSFlat &&
                                 !UseContextLessSummary.getNumOccurrences())) {
     for (const auto &I : Profiles) {
       ContextLessProfiles[I.second.getName()].merge(I.second);
diff --git a/llvm/lib/ProfileData/SampleProf.cpp b/llvm/lib/ProfileData/SampleProf.cpp
index fd8fd3b675b7..9b01a386a360 100644
--- a/llvm/lib/ProfileData/SampleProf.cpp
+++ b/llvm/lib/ProfileData/SampleProf.cpp
@@ -35,11 +35,18 @@ static cl::opt<uint64_t> ProfileSymbolListCutOff(
     cl::desc("Cutoff value about how many symbols in profile symbol list "
              "will be used. This is very useful for performance debugging"));
 
+cl::opt<bool> GenerateMergedBaseProfiles(
+    "generate-merged-base-profiles", cl::init(true), cl::ZeroOrMore,
+    cl::desc("When generating nested context-sensitive profiles, always "
+             "generate extra base profile for function with all its context "
+             "profiles merged into it."));
+
 namespace llvm {
 namespace sampleprof {
 SampleProfileFormat FunctionSamples::Format;
 bool FunctionSamples::ProfileIsProbeBased = false;
-bool FunctionSamples::ProfileIsCS = false;
+bool FunctionSamples::ProfileIsCSFlat = false;
+bool FunctionSamples::ProfileIsCSNested = false;
 bool FunctionSamples::UseMD5 = false;
 bool FunctionSamples::HasUniqSuffix = true;
 bool FunctionSamples::ProfileIsFS = false;
@@ -218,8 +225,9 @@ unsigned FunctionSamples::getOffset(const DILocation *DIL) {
       0xffff;
 }
 
-LineLocation FunctionSamples::getCallSiteIdentifier(const DILocation *DIL) {
-  if (FunctionSamples::ProfileIsProbeBased)
+LineLocation FunctionSamples::getCallSiteIdentifier(const DILocation *DIL,
+                                                    bool ProfileIsFS) {
+  if (FunctionSamples::ProfileIsProbeBased) {
     // In a pseudo-probe based profile, a callsite is simply represented by the
     // ID of the probe associated with the call instruction. The probe ID is
     // encoded in the Discriminator field of the call instruction's debug
@@ -227,9 +235,19 @@ LineLocation FunctionSamples::getCallSiteIdentifier(const DILocation *DIL) {
     return LineLocation(PseudoProbeDwarfDiscriminator::extractProbeIndex(
                             DIL->getDiscriminator()),
                         0);
-  else
-    return LineLocation(FunctionSamples::getOffset(DIL),
-                        DIL->getBaseDiscriminator());
+  } else {
+    unsigned Discriminator =
+        ProfileIsFS ? DIL->getDiscriminator() : DIL->getBaseDiscriminator();
+    return LineLocation(FunctionSamples::getOffset(DIL), Discriminator);
+  }
+}
+
+uint64_t FunctionSamples::getCallSiteHash(StringRef CalleeName,
+                                          const LineLocation &Callsite) {
+  uint64_t NameHash = std::hash<std::string>{}(CalleeName.str());
+  uint64_t LocId =
+      (((uint64_t)Callsite.LineOffset) << 32) | Callsite.Discriminator;
+  return NameHash + (LocId << 5) + LocId;
 }
 
 const FunctionSamples *FunctionSamples::findFunctionSamples(
@@ -239,21 +257,16 @@ const FunctionSamples *FunctionSamples::findFunctionSamples(
 
   const DILocation *PrevDIL = DIL;
   for (DIL = DIL->getInlinedAt(); DIL; DIL = DIL->getInlinedAt()) {
-    unsigned Discriminator;
-    if (ProfileIsFS)
-      Discriminator = DIL->getDiscriminator();
-    else
-      Discriminator = DIL->getBaseDiscriminator();
-
     // Use C++ linkage name if possible.
     StringRef Name = PrevDIL->getScope()->getSubprogram()->getLinkageName();
     if (Name.empty())
       Name = PrevDIL->getScope()->getSubprogram()->getName();
-
-    S.push_back(
-        std::make_pair(LineLocation(getOffset(DIL), Discriminator), Name));
+    S.emplace_back(FunctionSamples::getCallSiteIdentifier(
+                       DIL, FunctionSamples::ProfileIsFS),
+                   Name);
     PrevDIL = DIL;
   }
+
   if (S.size() == 0)
     return this;
   const FunctionSamples *FS = this;
@@ -454,3 +467,81 @@ void ProfileSymbolList::dump(raw_ostream &OS) const {
   for (auto &Sym : SortedList)
     OS << Sym << "\n";
 }
+
+CSProfileConverter::FrameNode *
+CSProfileConverter::FrameNode::getOrCreateChildFrame(
+    const LineLocation &CallSite, StringRef CalleeName) {
+  uint64_t Hash = FunctionSamples::getCallSiteHash(CalleeName, CallSite);
+  auto It = AllChildFrames.find(Hash);
+  if (It != AllChildFrames.end()) {
+    assert(It->second.FuncName == CalleeName &&
+           "Hash collision for child context node");
+    return &It->second;
+  }
+
+  AllChildFrames[Hash] = FrameNode(CalleeName, nullptr, CallSite);
+  return &AllChildFrames[Hash];
+}
+
+CSProfileConverter::CSProfileConverter(SampleProfileMap &Profiles)
+    : ProfileMap(Profiles) {
+  for (auto &FuncSample : Profiles) {
+    FunctionSamples *FSamples = &FuncSample.second;
+    auto *NewNode = getOrCreateContextPath(FSamples->getContext());
+    assert(!NewNode->FuncSamples && "New node cannot have sample profile");
+    NewNode->FuncSamples = FSamples;
+  }
+}
+
+CSProfileConverter::FrameNode *
+CSProfileConverter::getOrCreateContextPath(const SampleContext &Context) {
+  auto Node = &RootFrame;
+  LineLocation CallSiteLoc(0, 0);
+  for (auto &Callsite : Context.getContextFrames()) {
+    Node = Node->getOrCreateChildFrame(CallSiteLoc, Callsite.FuncName);
+    CallSiteLoc = Callsite.Location;
+  }
+  return Node;
+}
+
+void CSProfileConverter::convertProfiles(CSProfileConverter::FrameNode &Node) {
+  // Process each child profile. Add each child profile to callsite profile map
+  // of the current node `Node` if `Node` comes with a profile. Otherwise
+  // promote the child profile to a standalone profile.
+  auto *NodeProfile = Node.FuncSamples;
+  for (auto &It : Node.AllChildFrames) {
+    auto &ChildNode = It.second;
+    convertProfiles(ChildNode);
+    auto *ChildProfile = ChildNode.FuncSamples;
+    if (!ChildProfile)
+      continue;
+    SampleContext OrigChildContext = ChildProfile->getContext();
+    // Reset the child context to be contextless.
+    ChildProfile->getContext().setName(OrigChildContext.getName());
+    if (NodeProfile) {
+      // Add child profile to the callsite profile map.
+      auto &SamplesMap = NodeProfile->functionSamplesAt(ChildNode.CallSiteLoc);
+      SamplesMap.emplace(OrigChildContext.getName().str(), *ChildProfile);
+      NodeProfile->addTotalSamples(ChildProfile->getTotalSamples());
+    }
+
+    // Separate child profile to be a standalone profile, if the current parent
+    // profile doesn't exist. This is a duplicating operation when the child
+    // profile is already incorporated into the parent which is still useful and
+    // thus done optionally. It is seen that duplicating context profiles into
+    // base profiles improves the code quality for thinlto build by allowing a
+    // profile in the prelink phase for to-be-fully-inlined functions.
+    if (!NodeProfile || GenerateMergedBaseProfiles)
+      ProfileMap[ChildProfile->getContext()].merge(*ChildProfile);
+
+    // Contexts coming with a `ContextShouldBeInlined` attribute indicate this
+    // is a preinliner-computed profile.
+    if (OrigChildContext.hasAttribute(ContextShouldBeInlined))
+      FunctionSamples::ProfileIsCSNested = true;
+
+    // Remove the original child profile.
+    ProfileMap.erase(OrigChildContext);
+  }
+}
+
+void CSProfileConverter::convertProfiles() { convertProfiles(RootFrame); }
diff --git a/llvm/lib/ProfileData/SampleProfReader.cpp b/llvm/lib/ProfileData/SampleProfReader.cpp
index eefb7c2ba627..da16309fb82c 100644
--- a/llvm/lib/ProfileData/SampleProfReader.cpp
+++ b/llvm/lib/ProfileData/SampleProfReader.cpp
@@ -146,7 +146,7 @@ static bool ParseLine(const StringRef &Input, LineType &LineTy, uint32_t &Depth,
   if (Depth == 0)
     return false;
 
-  if (Depth == 1 && Input[Depth] == '!') {
+  if (Input[Depth] == '!') {
     LineTy = LineType::Metadata;
     return parseMetadata(Input.substr(Depth), FunctionHash, Attributes);
   }
@@ -244,11 +244,11 @@ std::error_code SampleProfileReaderText::readImpl() {
   sampleprof_error Result = sampleprof_error::success;
 
   InlineCallStack InlineStack;
-  uint32_t ProbeProfileCount = 0;
+  uint32_t TopLevelProbeProfileCount = 0;
 
-  // SeenMetadata tracks whether we have processed metadata for the current
-  // top-level function profile.
-  bool SeenMetadata = false;
+  // DepthMetadata tracks whether we have processed metadata for the current
+  // top-level or nested function profile.
+  uint32_t DepthMetadata = 0;
 
   ProfileIsFS = ProfileIsFSDisciminator;
   FunctionSamples::ProfileIsFS = ProfileIsFS;
@@ -275,7 +275,7 @@ std::error_code SampleProfileReaderText::readImpl() {
                     "Expected 'mangled_name:NUM:NUM', found " + *LineIt);
         return sampleprof_error::malformed;
       }
-      SeenMetadata = false;
+      DepthMetadata = 0;
       SampleContext FContext(FName, CSNameTable);
       if (FContext.hasContext())
         ++CSProfileCount;
@@ -302,7 +302,7 @@ std::error_code SampleProfileReaderText::readImpl() {
                         *LineIt);
         return sampleprof_error::malformed;
       }
-      if (SeenMetadata && LineTy != LineType::Metadata) {
+      if (LineTy != LineType::Metadata && Depth == DepthMetadata) {
         // Metadata must be put at the end of a function profile.
         reportError(LineIt.line_number(),
                     "Found non-metadata after metadata: " + *LineIt);
@@ -322,6 +322,7 @@ std::error_code SampleProfileReaderText::readImpl() {
         FSamples.setName(FName);
         MergeResult(Result, FSamples.addTotalSamples(NumSamples));
         InlineStack.push_back(&FSamples);
+        DepthMetadata = 0;
         break;
       }
       case LineType::BodyProfile: {
@@ -342,11 +343,13 @@ std::error_code SampleProfileReaderText::readImpl() {
         FunctionSamples &FProfile = *InlineStack.back();
         if (FunctionHash) {
           FProfile.setFunctionHash(FunctionHash);
-          ++ProbeProfileCount;
+          if (Depth == 1)
+            ++TopLevelProbeProfileCount;
         }
-        if (Attributes)
-          FProfile.getContext().setAllAttributes(Attributes);
-        SeenMetadata = true;
+        FProfile.getContext().setAllAttributes(Attributes);
+        if (Attributes & (uint32_t)ContextShouldBeInlined)
+          ProfileIsCSNested = true;
+        DepthMetadata = Depth;
         break;
       }
       }
@@ -355,12 +358,14 @@ std::error_code SampleProfileReaderText::readImpl() {
 
   assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) &&
          "Cannot have both context-sensitive and regular profile");
-  ProfileIsCS = (CSProfileCount > 0);
-  assert((ProbeProfileCount == 0 || ProbeProfileCount == Profiles.size()) &&
+  ProfileIsCSFlat = (CSProfileCount > 0);
+  assert((TopLevelProbeProfileCount == 0 ||
+          TopLevelProbeProfileCount == Profiles.size()) &&
          "Cannot have both probe-based profiles and regular profiles");
-  ProfileIsProbeBased = (ProbeProfileCount > 0);
+  ProfileIsProbeBased = (TopLevelProbeProfileCount > 0);
   FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased;
-  FunctionSamples::ProfileIsCS = ProfileIsCS;
+  FunctionSamples::ProfileIsCSFlat = ProfileIsCSFlat;
+  FunctionSamples::ProfileIsCSNested = ProfileIsCSNested;
 
   if (Result == sampleprof_error::success)
     computeSummary();
@@ -625,7 +630,7 @@ SampleProfileReaderExtBinaryBase::readContextFromTable() {
 
 ErrorOr<SampleContext>
 SampleProfileReaderExtBinaryBase::readSampleContextFromTable() {
-  if (ProfileIsCS) {
+  if (ProfileIsCSFlat) {
     auto FContext(readContextFromTable());
     if (std::error_code EC = FContext.getError())
       return EC;
@@ -649,7 +654,7 @@ std::error_code SampleProfileReaderExtBinaryBase::readOneSection(
     if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagPartial))
       Summary->setPartialProfile(true);
     if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagFullContext))
-      FunctionSamples::ProfileIsCS = ProfileIsCS = true;
+      FunctionSamples::ProfileIsCSFlat = ProfileIsCSFlat = true;
     if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagFSDiscriminator))
       FunctionSamples::ProfileIsFS = ProfileIsFS = true;
     break;
@@ -683,6 +688,9 @@ std::error_code SampleProfileReaderExtBinaryBase::readOneSection(
     ProfileIsProbeBased =
         hasSecFlag(Entry, SecFuncMetadataFlags::SecFlagIsProbeBased);
     FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased;
+    ProfileIsCSNested =
+        hasSecFlag(Entry, SecFuncMetadataFlags::SecFlagIsCSNested);
+    FunctionSamples::ProfileIsCSNested = ProfileIsCSNested;
     bool HasAttribute =
         hasSecFlag(Entry, SecFuncMetadataFlags::SecFlagHasAttribute);
     if (std::error_code EC = readFuncMetadata(HasAttribute))
@@ -770,7 +778,7 @@ std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles() {
       }
     }
 
-    if (ProfileIsCS) {
+    if (ProfileIsCSFlat) {
       DenseSet<uint64_t> FuncGuidsToUse;
       if (useMD5()) {
         for (auto Name : FuncsToUse)
@@ -840,7 +848,7 @@ std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles() {
   }
   assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) &&
          "Cannot have both context-sensitive and regular profile");
-  assert((!CSProfileCount || ProfileIsCS) &&
+  assert((!CSProfileCount || ProfileIsCSFlat) &&
          "Section flag should be consistent with actual profile");
   return sampleprof_error::success;
 }
@@ -1078,30 +1086,77 @@ std::error_code SampleProfileReaderExtBinaryBase::readCSNameTableSec() {
 }
 
 std::error_code
-SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute) {
-  while (Data < End) {
-    auto FContext(readSampleContextFromTable());
-    if (std::error_code EC = FContext.getError())
-      return EC;
 
-    bool ProfileInMap = Profiles.count(*FContext);
+SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute,
+                                                   FunctionSamples *FProfile) {
+  if (Data < End) {
     if (ProfileIsProbeBased) {
       auto Checksum = readNumber<uint64_t>();
       if (std::error_code EC = Checksum.getError())
         return EC;
-      if (ProfileInMap)
-        Profiles[*FContext].setFunctionHash(*Checksum);
+      if (FProfile)
+        FProfile->setFunctionHash(*Checksum);
     }
 
     if (ProfileHasAttribute) {
       auto Attributes = readNumber<uint32_t>();
       if (std::error_code EC = Attributes.getError())
         return EC;
-      if (ProfileInMap)
-        Profiles[*FContext].getContext().setAllAttributes(*Attributes);
+      if (FProfile)
+        FProfile->getContext().setAllAttributes(*Attributes);
+    }
+
+    if (!ProfileIsCSFlat) {
+      // Read all the attributes for inlined function calls.
+      auto NumCallsites = readNumber<uint32_t>();
+      if (std::error_code EC = NumCallsites.getError())
+        return EC;
+
+      for (uint32_t J = 0; J < *NumCallsites; ++J) {
+        auto LineOffset = readNumber<uint64_t>();
+        if (std::error_code EC = LineOffset.getError())
+          return EC;
+
+        auto Discriminator = readNumber<uint64_t>();
+        if (std::error_code EC = Discriminator.getError())
+          return EC;
+
+        auto FContext(readSampleContextFromTable());
+        if (std::error_code EC = FContext.getError())
+          return EC;
+
+        FunctionSamples *CalleeProfile = nullptr;
+        if (FProfile) {
+          CalleeProfile = const_cast<FunctionSamples *>(
+              &FProfile->functionSamplesAt(LineLocation(
+                  *LineOffset,
+                  *Discriminator))[std::string(FContext.get().getName())]);
+        }
+        if (std::error_code EC =
+                readFuncMetadata(ProfileHasAttribute, CalleeProfile))
+          return EC;
+      }
     }
   }
 
+  return sampleprof_error::success;
+}
+
+std::error_code
+SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute) {
+  while (Data < End) {
+    auto FContext(readSampleContextFromTable());
+    if (std::error_code EC = FContext.getError())
+      return EC;
+    FunctionSamples *FProfile = nullptr;
+    auto It = Profiles.find(*FContext);
+    if (It != Profiles.end())
+      FProfile = &It->second;
+
+    if (std::error_code EC = readFuncMetadata(ProfileHasAttribute, FProfile))
+      return EC;
+  }
+
   assert(Data == End && "More data is read than expected");
   return sampleprof_error::success;
 }
@@ -1233,6 +1288,8 @@ static std::string getSecFlagsStr(const SecHdrTableEntry &Entry) {
       Flags.append("probe,");
     if (hasSecFlag(Entry, SecFuncMetadataFlags::SecFlagHasAttribute))
       Flags.append("attr,");
+    if (hasSecFlag(Entry, SecFuncMetadataFlags::SecFlagIsCSNested))
+      Flags.append("preinlined,");
     break;
   default:
     break;
diff --git a/llvm/lib/ProfileData/SampleProfWriter.cpp b/llvm/lib/ProfileData/SampleProfWriter.cpp
index 78006aab1541..6f02bd203a9f 100644
--- a/llvm/lib/ProfileData/SampleProfWriter.cpp
+++ b/llvm/lib/ProfileData/SampleProfWriter.cpp
@@ -172,7 +172,7 @@ std::error_code SampleProfileWriterExtBinaryBase::writeFuncOffsetTable() {
     return (std::error_code)sampleprof_error::success;
   };
 
-  if (FunctionSamples::ProfileIsCS) {
+  if (FunctionSamples::ProfileIsCSFlat) {
     // Sort the contexts before writing them out. This is to help fast load all
     // context profiles for a function as well as their callee contexts which
     // can help profile-guided importing for ThinLTO.
@@ -195,17 +195,45 @@ std::error_code SampleProfileWriterExtBinaryBase::writeFuncOffsetTable() {
 }
 
 std::error_code SampleProfileWriterExtBinaryBase::writeFuncMetadata(
+    const FunctionSamples &FunctionProfile) {
+  auto &OS = *OutputStream;
+  if (std::error_code EC = writeContextIdx(FunctionProfile.getContext()))
+    return EC;
+
+  if (FunctionSamples::ProfileIsProbeBased)
+    encodeULEB128(FunctionProfile.getFunctionHash(), OS);
+  if (FunctionSamples::ProfileIsCSFlat || FunctionSamples::ProfileIsCSNested) {
+    encodeULEB128(FunctionProfile.getContext().getAllAttributes(), OS);
+  }
+
+  if (!FunctionSamples::ProfileIsCSFlat) {
+    // Recursively emit attributes for all callee samples.
+    uint64_t NumCallsites = 0;
+    for (const auto &J : FunctionProfile.getCallsiteSamples())
+      NumCallsites += J.second.size();
+    encodeULEB128(NumCallsites, OS);
+    for (const auto &J : FunctionProfile.getCallsiteSamples()) {
+      for (const auto &FS : J.second) {
+        LineLocation Loc = J.first;
+        encodeULEB128(Loc.LineOffset, OS);
+        encodeULEB128(Loc.Discriminator, OS);
+        if (std::error_code EC = writeFuncMetadata(FS.second))
+          return EC;
+      }
+    }
+  }
+
+  return sampleprof_error::success;
+}
+
+std::error_code SampleProfileWriterExtBinaryBase::writeFuncMetadata(
     const SampleProfileMap &Profiles) {
-  if (!FunctionSamples::ProfileIsProbeBased && !FunctionSamples::ProfileIsCS)
+  if (!FunctionSamples::ProfileIsProbeBased &&
+      !FunctionSamples::ProfileIsCSFlat && !FunctionSamples::ProfileIsCSNested)
     return sampleprof_error::success;
-  auto &OS = *OutputStream;
   for (const auto &Entry : Profiles) {
-    if (std::error_code EC = writeContextIdx(Entry.second.getContext()))
+    if (std::error_code EC = writeFuncMetadata(Entry.second))
       return EC;
-    if (FunctionSamples::ProfileIsProbeBased)
-      encodeULEB128(Entry.second.getFunctionHash(), OS);
-    if (FunctionSamples::ProfileIsCS)
-      encodeULEB128(Entry.second.getContext().getAllAttributes(), OS);
   }
   return sampleprof_error::success;
 }
@@ -295,10 +323,13 @@ std::error_code SampleProfileWriterExtBinaryBase::writeOneSection(
     setToCompressSection(SecProfileSymbolList);
   if (Type == SecFuncMetadata && FunctionSamples::ProfileIsProbeBased)
     addSectionFlag(SecFuncMetadata, SecFuncMetadataFlags::SecFlagIsProbeBased);
-  if (Type == SecProfSummary && FunctionSamples::ProfileIsCS)
-    addSectionFlag(SecProfSummary, SecProfSummaryFlags::SecFlagFullContext);
-  if (Type == SecFuncMetadata && FunctionSamples::ProfileIsCS)
+  if (Type == SecFuncMetadata && FunctionSamples::ProfileIsCSNested)
+    addSectionFlag(SecFuncMetadata, SecFuncMetadataFlags::SecFlagIsCSNested);
+  if (Type == SecFuncMetadata &&
+      (FunctionSamples::ProfileIsCSFlat || FunctionSamples::ProfileIsCSNested))
     addSectionFlag(SecFuncMetadata, SecFuncMetadataFlags::SecFlagHasAttribute);
+  if (Type == SecProfSummary && FunctionSamples::ProfileIsCSFlat)
+    addSectionFlag(SecProfSummary, SecProfSummaryFlags::SecFlagFullContext);
   if (Type == SecProfSummary && FunctionSamples::ProfileIsFS)
     addSectionFlag(SecProfSummary, SecProfSummaryFlags::SecFlagFSDiscriminator);
 
@@ -440,7 +471,7 @@ SampleProfileWriterCompactBinary::write(const SampleProfileMap &ProfileMap) {
 /// it needs to be parsed by the SampleProfileReaderText class.
 std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) {
   auto &OS = *OutputStream;
-  if (FunctionSamples::ProfileIsCS)
+  if (FunctionSamples::ProfileIsCSFlat)
     OS << "[" << S.getContext().toString() << "]:" << S.getTotalSamples();
   else
     OS << S.getName() << ":" << S.getTotalSamples();
@@ -483,15 +514,14 @@ std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) {
     }
   Indent -= 1;
 
-  if (Indent == 0) {
-    if (FunctionSamples::ProfileIsProbeBased) {
-      OS.indent(Indent + 1);
-      OS << "!CFGChecksum: " << S.getFunctionHash() << "\n";
-    }
-    if (FunctionSamples::ProfileIsCS) {
-      OS.indent(Indent + 1);
-      OS << "!Attributes: " << S.getContext().getAllAttributes() << "\n";
-    }
+  if (FunctionSamples::ProfileIsProbeBased) {
+    OS.indent(Indent + 1);
+    OS << "!CFGChecksum: " << S.getFunctionHash() << "\n";
+  }
+
+  if (S.getContext().getAllAttributes()) {
+    OS.indent(Indent + 1);
+    OS << "!Attributes: " << S.getContext().getAllAttributes() << "\n";
   }
 
   return sampleprof_error::success;
@@ -841,7 +871,8 @@ SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS,
   std::unique_ptr<SampleProfileWriter> Writer;
 
   // Currently only Text and Extended Binary format are supported for CSSPGO.
-  if ((FunctionSamples::ProfileIsCS || FunctionSamples::ProfileIsProbeBased) &&
+  if ((FunctionSamples::ProfileIsCSFlat ||
+       FunctionSamples::ProfileIsProbeBased) &&
       (Format == SPF_Binary || Format == SPF_Compact_Binary))
     return sampleprof_error::unsupported_writing_format;
 
diff --git a/llvm/lib/Support/AArch64TargetParser.cpp b/llvm/lib/Support/AArch64TargetParser.cpp
index a3e41ccd199c..4bc9c8487131 100644
--- a/llvm/lib/Support/AArch64TargetParser.cpp
+++ b/llvm/lib/Support/AArch64TargetParser.cpp
@@ -240,4 +240,4 @@ AArch64::ArchKind AArch64::parseCPUArch(StringRef CPU) {
       return C.ArchID;
   }
   return ArchKind::INVALID;
-}
\ No newline at end of file
+}
diff --git a/llvm/lib/Support/Caching.cpp b/llvm/lib/Support/Caching.cpp
index a2fe37a26617..8c685640f791 100644
--- a/llvm/lib/Support/Caching.cpp
+++ b/llvm/lib/Support/Caching.cpp
@@ -79,14 +79,13 @@ Expected<FileCache> llvm::localCache(Twine CacheNameRef,
     struct CacheStream : CachedFileStream {
       AddBufferFn AddBuffer;
       sys::fs::TempFile TempFile;
-      std::string EntryPath;
       unsigned Task;
 
       CacheStream(std::unique_ptr<raw_pwrite_stream> OS, AddBufferFn AddBuffer,
                   sys::fs::TempFile TempFile, std::string EntryPath,
                   unsigned Task)
-          : CachedFileStream(std::move(OS)), AddBuffer(std::move(AddBuffer)),
-            TempFile(std::move(TempFile)), EntryPath(std::move(EntryPath)),
+          : CachedFileStream(std::move(OS), std::move(EntryPath)),
+            AddBuffer(std::move(AddBuffer)), TempFile(std::move(TempFile)),
             Task(Task) {}
 
       ~CacheStream() {
@@ -99,7 +98,7 @@ Expected<FileCache> llvm::localCache(Twine CacheNameRef,
         // Open the file first to avoid racing with a cache pruner.
         ErrorOr<std::unique_ptr<MemoryBuffer>> MBOrErr =
             MemoryBuffer::getOpenFile(
-                sys::fs::convertFDToNativeFile(TempFile.FD), TempFile.TmpName,
+                sys::fs::convertFDToNativeFile(TempFile.FD), ObjectPathName,
                 /*FileSize=*/-1, /*RequiresNullTerminator=*/false);
         if (!MBOrErr)
           report_fatal_error(Twine("Failed to open new cache file ") +
@@ -115,14 +114,14 @@ Expected<FileCache> llvm::localCache(Twine CacheNameRef,
         // AddBuffer a copy of the bytes we wrote in that case. We do this
         // instead of just using the existing file, because the pruner might
         // delete the file before we get a chance to use it.
-        Error E = TempFile.keep(EntryPath);
+        Error E = TempFile.keep(ObjectPathName);
         E = handleErrors(std::move(E), [&](const ECError &E) -> Error {
           std::error_code EC = E.convertToErrorCode();
           if (EC != errc::permission_denied)
             return errorCodeToError(EC);
 
           auto MBCopy = MemoryBuffer::getMemBufferCopy((*MBOrErr)->getBuffer(),
-                                                       EntryPath);
+                                                       ObjectPathName);
           MBOrErr = std::move(MBCopy);
 
           // FIXME: should we consume the discard error?
@@ -133,7 +132,7 @@ Expected<FileCache> llvm::localCache(Twine CacheNameRef,
 
         if (E)
           report_fatal_error(Twine("Failed to rename temporary file ") +
-                             TempFile.TmpName + " to " + EntryPath + ": " +
+                             TempFile.TmpName + " to " + ObjectPathName + ": " +
                              toString(std::move(E)) + "\n");
 
         AddBuffer(Task, std::move(*MBOrErr));
diff --git a/llvm/lib/Support/CommandLine.cpp b/llvm/lib/Support/CommandLine.cpp
index 5b7004c86f5a..4153a69abf5d 100644
--- a/llvm/lib/Support/CommandLine.cpp
+++ b/llvm/lib/Support/CommandLine.cpp
@@ -1538,10 +1538,8 @@ bool CommandLineParser::ParseCommandLineOptions(int argc,
 
         ErrorParsing = true;
       } else {
-        for (SmallVectorImpl<Option *>::iterator I = SinkOpts.begin(),
-                                                 E = SinkOpts.end();
-             I != E; ++I)
-          (*I)->addOccurrence(i, "", StringRef(argv[i]));
+        for (Option *SinkOpt : SinkOpts)
+          SinkOpt->addOccurrence(i, "", StringRef(argv[i]));
       }
       continue;
     }
@@ -2303,11 +2301,8 @@ protected:
 
     // Collect registered option categories into vector in preparation for
     // sorting.
-    for (auto I = GlobalParser->RegisteredOptionCategories.begin(),
-              E = GlobalParser->RegisteredOptionCategories.end();
-         I != E; ++I) {
-      SortedCategories.push_back(*I);
-    }
+    for (OptionCategory *Category : GlobalParser->RegisteredOptionCategories)
+      SortedCategories.push_back(Category);
 
     // Sort the different option categories alphabetically.
     assert(SortedCategories.size() > 0 && "No option categories registered!");
@@ -2315,11 +2310,8 @@ protected:
                    OptionCategoryCompare);
 
     // Create map to empty vectors.
-    for (std::vector<OptionCategory *>::const_iterator
-             I = SortedCategories.begin(),
-             E = SortedCategories.end();
-         I != E; ++I)
-      CategorizedOptions[*I] = std::vector<Option *>();
+    for (OptionCategory *Category : SortedCategories)
+      CategorizedOptions[Category] = std::vector<Option *>();
 
     // Walk through pre-sorted options and assign into categories.
     // Because the options are already alphabetically sorted the
@@ -2334,23 +2326,20 @@ protected:
     }
 
     // Now do printing.
-    for (std::vector<OptionCategory *>::const_iterator
-             Category = SortedCategories.begin(),
-             E = SortedCategories.end();
-         Category != E; ++Category) {
+    for (OptionCategory *Category : SortedCategories) {
       // Hide empty categories for --help, but show for --help-hidden.
-      const auto &CategoryOptions = CategorizedOptions[*Category];
+      const auto &CategoryOptions = CategorizedOptions[Category];
       bool IsEmptyCategory = CategoryOptions.empty();
       if (!ShowHidden && IsEmptyCategory)
         continue;
 
       // Print category information.
       outs() << "\n";
-      outs() << (*Category)->getName() << ":\n";
+      outs() << Category->getName() << ":\n";
 
       // Check if description is set.
-      if (!(*Category)->getDescription().empty())
-        outs() << (*Category)->getDescription() << "\n\n";
+      if (!Category->getDescription().empty())
+        outs() << Category->getDescription() << "\n\n";
       else
         outs() << "\n";
 
diff --git a/llvm/lib/Support/Compression.cpp b/llvm/lib/Support/Compression.cpp
index b8c77cf69b95..ccf6ef4bb662 100644
--- a/llvm/lib/Support/Compression.cpp
+++ b/llvm/lib/Support/Compression.cpp
@@ -49,14 +49,14 @@ bool zlib::isAvailable() { return true; }
 Error zlib::compress(StringRef InputBuffer,
                      SmallVectorImpl<char> &CompressedBuffer, int Level) {
   unsigned long CompressedSize = ::compressBound(InputBuffer.size());
-  CompressedBuffer.reserve(CompressedSize);
+  CompressedBuffer.resize_for_overwrite(CompressedSize);
   int Res =
       ::compress2((Bytef *)CompressedBuffer.data(), &CompressedSize,
                   (const Bytef *)InputBuffer.data(), InputBuffer.size(), Level);
   // Tell MemorySanitizer that zlib output buffer is fully initialized.
   // This avoids a false report when running LLVM with uninstrumented ZLib.
   __msan_unpoison(CompressedBuffer.data(), CompressedSize);
-  CompressedBuffer.set_size(CompressedSize);
+  CompressedBuffer.truncate(CompressedSize);
   return Res ? createError(convertZlibCodeToString(Res)) : Error::success();
 }
 
@@ -74,10 +74,10 @@ Error zlib::uncompress(StringRef InputBuffer, char *UncompressedBuffer,
 Error zlib::uncompress(StringRef InputBuffer,
                        SmallVectorImpl<char> &UncompressedBuffer,
                        size_t UncompressedSize) {
-  UncompressedBuffer.reserve(UncompressedSize);
+  UncompressedBuffer.resize_for_overwrite(UncompressedSize);
   Error E =
       uncompress(InputBuffer, UncompressedBuffer.data(), UncompressedSize);
-  UncompressedBuffer.set_size(UncompressedSize);
+  UncompressedBuffer.truncate(UncompressedSize);
   return E;
 }
 
diff --git a/llvm/lib/Support/ConvertUTFWrapper.cpp b/llvm/lib/Support/ConvertUTFWrapper.cpp
index d8d46712a593..392c4c4890e1 100644
--- a/llvm/lib/Support/ConvertUTFWrapper.cpp
+++ b/llvm/lib/Support/ConvertUTFWrapper.cpp
@@ -103,8 +103,8 @@ bool convertUTF16ToUTF8String(ArrayRef<char> SrcBytes, std::string &Out) {
   std::vector<UTF16> ByteSwapped;
   if (Src[0] == UNI_UTF16_BYTE_ORDER_MARK_SWAPPED) {
     ByteSwapped.insert(ByteSwapped.end(), Src, SrcEnd);
-    for (unsigned I = 0, E = ByteSwapped.size(); I != E; ++I)
-      ByteSwapped[I] = llvm::ByteSwap_16(ByteSwapped[I]);
+    for (UTF16 &I : ByteSwapped)
+      I = llvm::ByteSwap_16(I);
     Src = &ByteSwapped[0];
     SrcEnd = &ByteSwapped[ByteSwapped.size() - 1] + 1;
   }
diff --git a/llvm/lib/Support/DAGDeltaAlgorithm.cpp b/llvm/lib/Support/DAGDeltaAlgorithm.cpp
index e5e6301d41cc..a6daee00bd43 100644
--- a/llvm/lib/Support/DAGDeltaAlgorithm.cpp
+++ b/llvm/lib/Support/DAGDeltaAlgorithm.cpp
@@ -180,22 +180,19 @@ DAGDeltaAlgorithmImpl::DAGDeltaAlgorithmImpl(
     DAGDeltaAlgorithm &DDA, const changeset_ty &Changes,
     const std::vector<edge_ty> &Dependencies)
     : DDA(DDA) {
-  for (changeset_ty::const_iterator it = Changes.begin(),
-         ie = Changes.end(); it != ie; ++it) {
-    Predecessors.insert(std::make_pair(*it, std::vector<change_ty>()));
-    Successors.insert(std::make_pair(*it, std::vector<change_ty>()));
+  for (change_ty Change : Changes) {
+    Predecessors.insert(std::make_pair(Change, std::vector<change_ty>()));
+    Successors.insert(std::make_pair(Change, std::vector<change_ty>()));
   }
-  for (std::vector<edge_ty>::const_iterator it = Dependencies.begin(),
-         ie = Dependencies.end(); it != ie; ++it) {
-    Predecessors[it->second].push_back(it->first);
-    Successors[it->first].push_back(it->second);
+  for (const edge_ty &Dep : Dependencies) {
+    Predecessors[Dep.second].push_back(Dep.first);
+    Successors[Dep.first].push_back(Dep.second);
   }
 
   // Compute the roots.
-  for (changeset_ty::const_iterator it = Changes.begin(),
-         ie = Changes.end(); it != ie; ++it)
-    if (succ_begin(*it) == succ_end(*it))
-      Roots.push_back(*it);
+  for (change_ty Change : Changes)
+    if (succ_begin(Change) == succ_end(Change))
+      Roots.push_back(Change);
 
   // Pre-compute the closure of the successor relation.
   std::vector<change_ty> Worklist(Roots.begin(), Roots.end());
@@ -213,14 +210,13 @@ DAGDeltaAlgorithmImpl::DAGDeltaAlgorithmImpl(
   }
 
   // Invert to form the predecessor closure map.
-  for (changeset_ty::const_iterator it = Changes.begin(),
-         ie = Changes.end(); it != ie; ++it)
-    PredClosure.insert(std::make_pair(*it, std::set<change_ty>()));
-  for (changeset_ty::const_iterator it = Changes.begin(),
-         ie = Changes.end(); it != ie; ++it)
-    for (succ_closure_iterator_ty it2 = succ_closure_begin(*it),
-           ie2 = succ_closure_end(*it); it2 != ie2; ++it2)
-      PredClosure[*it2].insert(*it);
+  for (change_ty Change : Changes)
+    PredClosure.insert(std::make_pair(Change, std::set<change_ty>()));
+  for (change_ty Change : Changes)
+    for (succ_closure_iterator_ty it2 = succ_closure_begin(Change),
+                                  ie2 = succ_closure_end(Change);
+         it2 != ie2; ++it2)
+      PredClosure[*it2].insert(Change);
 
   // Dump useful debug info.
   LLVM_DEBUG({
@@ -256,13 +252,12 @@ DAGDeltaAlgorithmImpl::DAGDeltaAlgorithmImpl(
     llvm::errs() << "]\n";
 
     llvm::errs() << "Predecessor Closure:\n";
-    for (changeset_ty::const_iterator it = Changes.begin(), ie = Changes.end();
-         it != ie; ++it) {
-      llvm::errs() << format("  %-4d: [", *it);
-      for (pred_closure_iterator_ty it2 = pred_closure_begin(*it),
-                                    ie2 = pred_closure_end(*it);
+    for (change_ty Change : Changes) {
+      llvm::errs() << format("  %-4d: [", Change);
+      for (pred_closure_iterator_ty it2 = pred_closure_begin(Change),
+                                    ie2 = pred_closure_end(Change);
            it2 != ie2; ++it2) {
-        if (it2 != pred_closure_begin(*it))
+        if (it2 != pred_closure_begin(Change))
           llvm::errs() << ", ";
         llvm::errs() << *it2;
       }
@@ -270,13 +265,12 @@ DAGDeltaAlgorithmImpl::DAGDeltaAlgorithmImpl(
     }
 
     llvm::errs() << "Successor Closure:\n";
-    for (changeset_ty::const_iterator it = Changes.begin(), ie = Changes.end();
-         it != ie; ++it) {
-      llvm::errs() << format("  %-4d: [", *it);
-      for (succ_closure_iterator_ty it2 = succ_closure_begin(*it),
-                                    ie2 = succ_closure_end(*it);
+    for (change_ty Change : Changes) {
+      llvm::errs() << format("  %-4d: [", Change);
+      for (succ_closure_iterator_ty it2 = succ_closure_begin(Change),
+                                    ie2 = succ_closure_end(Change);
            it2 != ie2; ++it2) {
-        if (it2 != succ_closure_begin(*it))
+        if (it2 != succ_closure_begin(Change))
           llvm::errs() << ", ";
         llvm::errs() << *it2;
       }
@@ -291,9 +285,8 @@ bool DAGDeltaAlgorithmImpl::GetTestResult(const changeset_ty &Changes,
                                           const changeset_ty &Required) {
   changeset_ty Extended(Required);
   Extended.insert(Changes.begin(), Changes.end());
-  for (changeset_ty::const_iterator it = Changes.begin(),
-         ie = Changes.end(); it != ie; ++it)
-    Extended.insert(pred_closure_begin(*it), pred_closure_end(*it));
+  for (change_ty Change : Changes)
+    Extended.insert(pred_closure_begin(Change), pred_closure_end(Change));
 
   if (FailedTestsCache.count(Extended))
     return false;
@@ -340,9 +333,8 @@ DAGDeltaAlgorithmImpl::Run() {
     // Replace the current set with the predecssors of the minimized set of
     // active changes.
     CurrentSet.clear();
-    for (changeset_ty::const_iterator it = CurrentMinSet.begin(),
-           ie = CurrentMinSet.end(); it != ie; ++it)
-      CurrentSet.insert(pred_begin(*it), pred_end(*it));
+    for (change_ty CT : CurrentMinSet)
+      CurrentSet.insert(pred_begin(CT), pred_end(CT));
 
     // FIXME: We could enforce CurrentSet intersect Required == {} here if we
     // wanted to protect against cyclic graphs.
diff --git a/llvm/lib/Support/DeltaAlgorithm.cpp b/llvm/lib/Support/DeltaAlgorithm.cpp
index 6aee69f43405..a2017a10ab3f 100644
--- a/llvm/lib/Support/DeltaAlgorithm.cpp
+++ b/llvm/lib/Support/DeltaAlgorithm.cpp
@@ -57,9 +57,8 @@ DeltaAlgorithm::Delta(const changeset_ty &Changes,
 
   // Otherwise, partition the sets if possible; if not we are done.
   changesetlist_ty SplitSets;
-  for (changesetlist_ty::const_iterator it = Sets.begin(),
-         ie = Sets.end(); it != ie; ++it)
-    Split(*it, SplitSets);
+  for (const changeset_ty &Set : Sets)
+    Split(Set, SplitSets);
   if (SplitSets.size() == Sets.size())
     return Changes;
 
diff --git a/llvm/lib/Support/HTTPClient.cpp b/llvm/lib/Support/HTTPClient.cpp
deleted file mode 100644
index 68ba56d1fe50..000000000000
--- a/llvm/lib/Support/HTTPClient.cpp
+++ /dev/null
@@ -1,97 +0,0 @@
-//===-- llvm/Support/HTTPClient.cpp - HTTP client library -------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-///
-/// \file
-///
-/// This file defines the methods of the HTTPRequest, HTTPClient, and
-/// BufferedHTTPResponseHandler classes.
-///
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Support/HTTPClient.h"
-#include "llvm/ADT/APInt.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/Support/Errc.h"
-#include "llvm/Support/Error.h"
-#include "llvm/Support/MemoryBuffer.h"
-
-using namespace llvm;
-
-HTTPRequest::HTTPRequest(StringRef Url) { this->Url = Url.str(); }
-
-bool operator==(const HTTPRequest &A, const HTTPRequest &B) {
-  return A.Url == B.Url && A.Method == B.Method &&
-         A.FollowRedirects == B.FollowRedirects;
-}
-
-HTTPResponseHandler::~HTTPResponseHandler() = default;
-
-static inline bool parseContentLengthHeader(StringRef LineRef,
-                                            size_t &ContentLength) {
-  // Content-Length is a mandatory header, and the only one we handle.
-  return LineRef.consume_front("Content-Length: ") &&
-         to_integer(LineRef.trim(), ContentLength, 10);
-}
-
-Error BufferedHTTPResponseHandler::handleHeaderLine(StringRef HeaderLine) {
-  if (ResponseBuffer.Body)
-    return Error::success();
-
-  size_t ContentLength;
-  if (parseContentLengthHeader(HeaderLine, ContentLength))
-    ResponseBuffer.Body =
-        WritableMemoryBuffer::getNewUninitMemBuffer(ContentLength);
-
-  return Error::success();
-}
-
-Error BufferedHTTPResponseHandler::handleBodyChunk(StringRef BodyChunk) {
-  if (!ResponseBuffer.Body)
-    return createStringError(errc::io_error,
-                             "Unallocated response buffer. HTTP Body data "
-                             "received before Content-Length header.");
-  if (Offset + BodyChunk.size() > ResponseBuffer.Body->getBufferSize())
-    return createStringError(errc::io_error,
-                             "Content size exceeds buffer size.");
-  memcpy(ResponseBuffer.Body->getBufferStart() + Offset, BodyChunk.data(),
-         BodyChunk.size());
-  Offset += BodyChunk.size();
-  return Error::success();
-}
-
-Error BufferedHTTPResponseHandler::handleStatusCode(unsigned Code) {
-  ResponseBuffer.Code = Code;
-  return Error::success();
-}
-
-Expected<HTTPResponseBuffer> HTTPClient::perform(const HTTPRequest &Request) {
-  BufferedHTTPResponseHandler Handler;
-  if (Error Err = perform(Request, Handler))
-    return std::move(Err);
-  return std::move(Handler.ResponseBuffer);
-}
-
-Expected<HTTPResponseBuffer> HTTPClient::get(StringRef Url) {
-  HTTPRequest Request(Url);
-  return perform(Request);
-}
-
-HTTPClient::HTTPClient() = default;
-
-HTTPClient::~HTTPClient() = default;
-
-bool HTTPClient::isAvailable() { return false; }
-
-void HTTPClient::cleanup() {}
-
-void HTTPClient::setTimeout(std::chrono::milliseconds Timeout) {}
-
-Error HTTPClient::perform(const HTTPRequest &Request,
-                          HTTPResponseHandler &Handler) {
-  llvm_unreachable("No HTTP Client implementation available.");
-}
diff --git a/llvm/lib/Support/KnownBits.cpp b/llvm/lib/Support/KnownBits.cpp
index 554e3248524c..8e154067abc0 100644
--- a/llvm/lib/Support/KnownBits.cpp
+++ b/llvm/lib/Support/KnownBits.cpp
@@ -420,11 +420,19 @@ KnownBits KnownBits::mul(const KnownBits &LHS, const KnownBits &RHS,
   assert((!SelfMultiply || (LHS.One == RHS.One && LHS.Zero == RHS.Zero)) &&
          "Self multiplication knownbits mismatch");
 
-  // Compute a conservative estimate for high known-0 bits.
-  unsigned LHSLeadZ = LHS.countMinLeadingZeros();
-  unsigned RHSLeadZ = RHS.countMinLeadingZeros();
-  unsigned LeadZ = std::max(LHSLeadZ + RHSLeadZ, BitWidth) - BitWidth;
-  assert(LeadZ <= BitWidth && "More zeros than bits?");
+  // Compute the high known-0 bits by multiplying the unsigned max of each side.
+  // Conservatively, M active bits * N active bits results in M + N bits in the
+  // result. But if we know a value is a power-of-2 for example, then this
+  // computes one more leading zero.
+  // TODO: This could be generalized to number of sign bits (negative numbers).
+  APInt UMaxLHS = LHS.getMaxValue();
+  APInt UMaxRHS = RHS.getMaxValue();
+
+  // For leading zeros in the result to be valid, the unsigned max product must
+  // fit in the bitwidth (it must not overflow).
+  bool HasOverflow;
+  APInt UMaxResult = UMaxLHS.umul_ov(UMaxRHS, HasOverflow);
+  unsigned LeadZ = HasOverflow ? 0 : UMaxResult.countLeadingZeros();
 
   // The result of the bottom bits of an integer multiply can be
   // inferred by looking at the bottom bits of both operands and
diff --git a/llvm/lib/Support/MemoryBuffer.cpp b/llvm/lib/Support/MemoryBuffer.cpp
index bcf13d828a5d..d3fa3c6f065d 100644
--- a/llvm/lib/Support/MemoryBuffer.cpp
+++ b/llvm/lib/Support/MemoryBuffer.cpp
@@ -227,17 +227,20 @@ static ErrorOr<std::unique_ptr<WritableMemoryBuffer>>
 getMemoryBufferForStream(sys::fs::file_t FD, const Twine &BufferName) {
   const ssize_t ChunkSize = 4096*4;
   SmallString<ChunkSize> Buffer;
+
   // Read into Buffer until we hit EOF.
+  size_t Size = Buffer.size();
   for (;;) {
-    Buffer.reserve(Buffer.size() + ChunkSize);
+    Buffer.resize_for_overwrite(Size + ChunkSize);
     Expected<size_t> ReadBytes = sys::fs::readNativeFile(
-        FD, makeMutableArrayRef(Buffer.end(), ChunkSize));
+        FD, makeMutableArrayRef(Buffer.begin() + Size, ChunkSize));
     if (!ReadBytes)
       return errorToErrorCode(ReadBytes.takeError());
     if (*ReadBytes == 0)
       break;
-    Buffer.set_size(Buffer.size() + *ReadBytes);
+    Size += *ReadBytes;
   }
+  Buffer.truncate(Size);
 
   return getMemBufferCopyImpl(Buffer, BufferName);
 }
diff --git a/llvm/lib/Support/NativeFormatting.cpp b/llvm/lib/Support/NativeFormatting.cpp
index ae9f03745850..254d18d797b3 100644
--- a/llvm/lib/Support/NativeFormatting.cpp
+++ b/llvm/lib/Support/NativeFormatting.cpp
@@ -168,7 +168,7 @@ void llvm::write_double(raw_ostream &S, double N, FloatStyle Style,
     S << "nan";
     return;
   } else if (std::isinf(N)) {
-    S << "INF";
+    S << (std::signbit(N) ? "-INF" : "INF");
     return;
   }
 
diff --git a/llvm/lib/Support/Path.cpp b/llvm/lib/Support/Path.cpp
index 3957547dfaaa..7c99d088911c 100644
--- a/llvm/lib/Support/Path.cpp
+++ b/llvm/lib/Support/Path.cpp
@@ -474,7 +474,7 @@ StringRef parent_path(StringRef path, Style style) {
 void remove_filename(SmallVectorImpl<char> &path, Style style) {
   size_t end_pos = parent_path_end(StringRef(path.begin(), path.size()), style);
   if (end_pos != StringRef::npos)
-    path.set_size(end_pos);
+    path.truncate(end_pos);
 }
 
 void replace_extension(SmallVectorImpl<char> &path, const Twine &extension,
@@ -486,7 +486,7 @@ void replace_extension(SmallVectorImpl<char> &path, const Twine &extension,
   // Erase existing extension.
   size_t pos = p.find_last_of('.');
   if (pos != StringRef::npos && pos >= filename_pos(p, style))
-    path.set_size(pos);
+    path.truncate(pos);
 
   // Append '.' if needed.
   if (ext.size() > 0 && ext[0] != '.')
diff --git a/llvm/lib/Support/RISCVISAInfo.cpp b/llvm/lib/Support/RISCVISAInfo.cpp
index 8e984002f90d..e2e4340f44e9 100644
--- a/llvm/lib/Support/RISCVISAInfo.cpp
+++ b/llvm/lib/Support/RISCVISAInfo.cpp
@@ -61,7 +61,6 @@ static const RISCVSupportedExtension SupportedExperimentalExtensions[] = {
     {"zbs", RISCVExtensionVersion{1, 0}},
     {"zbt", RISCVExtensionVersion{0, 93}},
 
-    {"zvamo", RISCVExtensionVersion{0, 10}},
     {"zvlsseg", RISCVExtensionVersion{0, 10}},
 
     {"zfhmin", RISCVExtensionVersion{0, 1}},
@@ -72,6 +71,28 @@ static bool stripExperimentalPrefix(StringRef &Ext) {
   return Ext.consume_front("experimental-");
 }
 
+// This function finds the first character that doesn't belong to a version
+// (e.g. zbe0p93 is extension 'zbe' of version '0p93'). So the function will
+// consume [0-9]*p[0-9]* starting from the backward. An extension name will not
+// end with a digit or the letter 'p', so this function will parse correctly.
+// NOTE: This function is NOT able to take empty strings or strings that only
+// have version numbers and no extension name. It assumes the extension name
+// will be at least more than one character.
+static size_t findFirstNonVersionCharacter(const StringRef &Ext) {
+  if (Ext.size() == 0)
+    llvm_unreachable("Already guarded by if-statement in ::parseArchString");
+
+  int Pos = Ext.size() - 1;
+  while (Pos > 0 && isDigit(Ext[Pos]))
+    Pos--;
+  if (Pos > 0 && Ext[Pos] == 'p' && isDigit(Ext[Pos - 1])) {
+    Pos--;
+    while (Pos > 0 && isDigit(Ext[Pos]))
+      Pos--;
+  }
+  return Pos;
+}
+
 struct FindByName {
   FindByName(StringRef Ext) : Ext(Ext){};
   StringRef Ext;
@@ -264,10 +285,6 @@ void RISCVISAInfo::toFeatures(
     if (ExtName == "zvlsseg") {
       Features.push_back("+experimental-v");
       Features.push_back("+experimental-zvlsseg");
-    } else if (ExtName == "zvamo") {
-      Features.push_back("+experimental-v");
-      Features.push_back("+experimental-zvlsseg");
-      Features.push_back("+experimental-zvamo");
     } else if (isExperimentalExtension(ExtName)) {
       Features.push_back(StrAlloc("+experimental-" + ExtName));
     } else {
@@ -390,7 +407,6 @@ RISCVISAInfo::parseFeatures(unsigned XLen,
   assert(XLen == 32 || XLen == 64);
   std::unique_ptr<RISCVISAInfo> ISAInfo(new RISCVISAInfo(XLen));
 
-  bool HasE = false;
   for (auto &Feature : Features) {
     StringRef ExtName = Feature;
     bool Experimental = false;
@@ -409,29 +425,19 @@ RISCVISAInfo::parseFeatures(unsigned XLen,
     if (ExtensionInfoIterator == ExtensionInfos.end())
       continue;
 
-    if (Add) {
-      if (ExtName == "e") {
-        if (XLen != 32)
-          return createStringError(
-              errc::invalid_argument,
-              "standard user-level extension 'e' requires 'rv32'");
-        HasE = true;
-      }
-
+    if (Add)
       ISAInfo->addExtension(ExtName, ExtensionInfoIterator->Version.Major,
                             ExtensionInfoIterator->Version.Minor);
-    } else
-      ISAInfo->Exts.erase(ExtName.str());
-  }
-  if (!HasE) {
-    if (auto Version = findDefaultVersion("i"))
-      ISAInfo->addExtension("i", Version->Major, Version->Minor);
     else
-      llvm_unreachable("Default extension version for 'i' not found?");
+      ISAInfo->Exts.erase(ExtName.str());
   }
 
+  ISAInfo->updateImplication();
   ISAInfo->updateFLen();
 
+  if (Error Result = ISAInfo->checkDependency())
+    return std::move(Result);
+
   return std::move(ISAInfo);
 }
 
@@ -457,7 +463,6 @@ RISCVISAInfo::parseArchString(StringRef Arch, bool EnableExperimentalExtension,
   // The canonical order specified in ISA manual.
   // Ref: Table 22.1 in RISC-V User-Level ISA V2.2
   StringRef StdExts = AllStdExts;
-  bool HasF = false, HasD = false;
   char Baseline = Arch[4];
 
   // First letter should be 'e', 'i' or 'g'.
@@ -478,8 +483,6 @@ RISCVISAInfo::parseArchString(StringRef Arch, bool EnableExperimentalExtension,
   case 'g':
     // g = imafd
     StdExts = StdExts.drop_front(4);
-    HasF = true;
-    HasD = true;
     break;
   }
 
@@ -560,34 +563,14 @@ RISCVISAInfo::parseArchString(StringRef Arch, bool EnableExperimentalExtension,
 
     // The order is OK, then push it into features.
     // TODO: Use version number when setting target features
-    switch (C) {
-    default:
-      // Currently LLVM supports only "mafdcbv".
+    // Currently LLVM supports only "mafdcbv".
+    StringRef SupportedStandardExtension = "mafdcbv";
+    if (!SupportedStandardExtension.contains(C))
       return createStringError(errc::invalid_argument,
                                "unsupported standard user-level extension '%c'",
                                C);
-    case 'm':
-      ISAInfo->addExtension("m", Major, Minor);
-      break;
-    case 'a':
-      ISAInfo->addExtension("a", Major, Minor);
-      break;
-    case 'f':
-      ISAInfo->addExtension("f", Major, Minor);
-      HasF = true;
-      break;
-    case 'd':
-      ISAInfo->addExtension("d", Major, Minor);
-      HasD = true;
-      break;
-    case 'c':
-      ISAInfo->addExtension("c", Major, Minor);
-      break;
-    case 'v':
-      ISAInfo->addExtension("v", Major, Minor);
-      ISAInfo->addExtension("zvlsseg", Major, Minor);
-      break;
-    }
+    ISAInfo->addExtension(std::string(1, C), Major, Minor);
+
     // Consume full extension name and version, including any optional '_'
     // between this extension and the next
     ++I;
@@ -595,21 +578,6 @@ RISCVISAInfo::parseArchString(StringRef Arch, bool EnableExperimentalExtension,
     if (*I == '_')
       ++I;
   }
-  // Dependency check.
-  // It's illegal to specify the 'd' (double-precision floating point)
-  // extension without also specifying the 'f' (single precision
-  // floating-point) extension.
-  // TODO: This has been removed in later specs, which specify that D implies F
-  if (HasD && !HasF)
-    return createStringError(errc::invalid_argument,
-                             "d requires f extension to also be specified");
-
-  // Additional dependency checks.
-  // TODO: The 'q' extension requires rv64.
-  // TODO: It is illegal to specify 'e' extensions with 'f' and 'd'.
-
-  if (OtherExts.empty())
-    return std::move(ISAInfo);
 
   // Handle other types of extensions other than the standard
   // general purpose and standard user-level extensions.
@@ -630,52 +598,53 @@ RISCVISAInfo::parseArchString(StringRef Arch, bool EnableExperimentalExtension,
   std::array<StringRef, 4> Prefix{"z", "x", "s", "sx"};
   auto I = Prefix.begin();
   auto E = Prefix.end();
+  if (Split.size() > 1 || Split[0] != "") {
+    for (StringRef Ext : Split) {
+      if (Ext.empty())
+        return createStringError(errc::invalid_argument,
+                                 "extension name missing after separator '_'");
+
+      StringRef Type = getExtensionType(Ext);
+      StringRef Desc = getExtensionTypeDesc(Ext);
+      auto Pos = findFirstNonVersionCharacter(Ext) + 1;
+      StringRef Name(Ext.substr(0, Pos));
+      StringRef Vers(Ext.substr(Pos));
+
+      if (Type.empty())
+        return createStringError(errc::invalid_argument,
+                                 "invalid extension prefix '" + Ext + "'");
+
+      // Check ISA extensions are specified in the canonical order.
+      while (I != E && *I != Type)
+        ++I;
+
+      if (I == E)
+        return createStringError(errc::invalid_argument,
+                                 "%s not given in canonical order '%s'",
+                                 Desc.str().c_str(), Ext.str().c_str());
+
+      if (Name.size() == Type.size()) {
+        return createStringError(errc::invalid_argument,
+                                 "%s name missing after '%s'",
+                                 Desc.str().c_str(), Type.str().c_str());
+      }
 
-  for (StringRef Ext : Split) {
-    if (Ext.empty())
-      return createStringError(errc::invalid_argument,
-                               "extension name missing after separator '_'");
-
-    StringRef Type = getExtensionType(Ext);
-    StringRef Desc = getExtensionTypeDesc(Ext);
-    auto Pos = Ext.find_if(isDigit);
-    StringRef Name(Ext.substr(0, Pos));
-    StringRef Vers(Ext.substr(Pos));
-
-    if (Type.empty())
-      return createStringError(errc::invalid_argument,
-                               "invalid extension prefix '" + Ext + "'");
-
-    // Check ISA extensions are specified in the canonical order.
-    while (I != E && *I != Type)
-      ++I;
-
-    if (I == E)
-      return createStringError(errc::invalid_argument,
-                               "%s not given in canonical order '%s'",
-                               Desc.str().c_str(), Ext.str().c_str());
-
-    if (Name.size() == Type.size()) {
-      return createStringError(errc::invalid_argument,
-                               "%s name missing after '%s'", Desc.str().c_str(),
-                               Type.str().c_str());
+      unsigned Major, Minor, ConsumeLength;
+      if (auto E = getExtensionVersion(Name, Vers, Major, Minor, ConsumeLength,
+                                       EnableExperimentalExtension,
+                                       ExperimentalExtensionVersionCheck))
+        return std::move(E);
+
+      // Check if duplicated extension.
+      if (llvm::is_contained(AllExts, Name))
+        return createStringError(errc::invalid_argument, "duplicated %s '%s'",
+                                 Desc.str().c_str(), Name.str().c_str());
+
+      ISAInfo->addExtension(Name, Major, Minor);
+      // Extension format is correct, keep parsing the extensions.
+      // TODO: Save Type, Name, Major, Minor to avoid parsing them later.
+      AllExts.push_back(Name);
     }
-
-    unsigned Major, Minor, ConsumeLength;
-    if (auto E = getExtensionVersion(Name, Vers, Major, Minor, ConsumeLength,
-                                     EnableExperimentalExtension,
-                                     ExperimentalExtensionVersionCheck))
-      return std::move(E);
-
-    // Check if duplicated extension.
-    if (llvm::is_contained(AllExts, Name))
-      return createStringError(errc::invalid_argument, "duplicated %s '%s'",
-                               Desc.str().c_str(), Name.str().c_str());
-
-    ISAInfo->addExtension(Name, Major, Minor);
-    // Extension format is correct, keep parsing the extensions.
-    // TODO: Save Type, Name, Major, Minor to avoid parsing them later.
-    AllExts.push_back(Name);
   }
 
   for (auto Ext : AllExts) {
@@ -686,11 +655,83 @@ RISCVISAInfo::parseArchString(StringRef Arch, bool EnableExperimentalExtension,
     }
   }
 
+  ISAInfo->updateImplication();
   ISAInfo->updateFLen();
 
+  if (Error Result = ISAInfo->checkDependency())
+    return std::move(Result);
+
   return std::move(ISAInfo);
 }
 
+Error RISCVISAInfo::checkDependency() {
+  bool IsRv32 = XLen == 32;
+  bool HasE = Exts.count("e") == 1;
+  bool HasD = Exts.count("d") == 1;
+  bool HasF = Exts.count("f") == 1;
+
+  if (HasE && !IsRv32)
+    return createStringError(
+        errc::invalid_argument,
+        "standard user-level extension 'e' requires 'rv32'");
+
+  // It's illegal to specify the 'd' (double-precision floating point)
+  // extension without also specifying the 'f' (single precision
+  // floating-point) extension.
+  // TODO: This has been removed in later specs, which specify that D implies F
+  if (HasD && !HasF)
+    return createStringError(errc::invalid_argument,
+                             "d requires f extension to also be specified");
+
+  // Additional dependency checks.
+  // TODO: The 'q' extension requires rv64.
+  // TODO: It is illegal to specify 'e' extensions with 'f' and 'd'.
+
+  return Error::success();
+}
+
+static const char *ImpliedExtsV[] = {"zvlsseg"};
+static const char *ImpliedExtsZfh[] = {"zfhmin"};
+
+struct ImpliedExtsEntry {
+  StringLiteral Name;
+  ArrayRef<const char *> Exts;
+
+  bool operator<(const ImpliedExtsEntry &Other) const {
+    return Name < Other.Name;
+  }
+
+  bool operator<(StringRef Other) const { return Name < Other; }
+};
+
+static constexpr ImpliedExtsEntry ImpliedExts[] = {
+    {{"v"}, {ImpliedExtsV}},
+    {{"zfh"}, {ImpliedExtsZfh}},
+};
+
+void RISCVISAInfo::updateImplication() {
+  bool HasE = Exts.count("e") == 1;
+  bool HasI = Exts.count("i") == 1;
+
+  // If not in e extension and i extension does not exist, i extension is
+  // implied
+  if (!HasE && !HasI) {
+    auto Version = findDefaultVersion("i");
+    addExtension("i", Version->Major, Version->Minor);
+  }
+
+  assert(llvm::is_sorted(ImpliedExts) && "Table not sorted by Name");
+  for (auto &Ext : Exts) {
+    auto I = llvm::lower_bound(ImpliedExts, Ext.first);
+    if (I != std::end(ImpliedExts) && I->Name == Ext.first) {
+      for (auto &ImpliedExt : I->Exts) {
+        auto Version = findDefaultVersion(ImpliedExt);
+        addExtension(ImpliedExt, Version->Major, Version->Minor);
+      }
+    }
+  }
+}
+
 void RISCVISAInfo::updateFLen() {
   FLen = 0;
   // TODO: Handle q extension.
diff --git a/llvm/lib/Support/ScopedPrinter.cpp b/llvm/lib/Support/ScopedPrinter.cpp
index 779c6c45257d..ea90a24eaced 100644
--- a/llvm/lib/Support/ScopedPrinter.cpp
+++ b/llvm/lib/Support/ScopedPrinter.cpp
@@ -43,4 +43,14 @@ void ScopedPrinter::printBinaryImpl(StringRef Label, StringRef Str,
   }
 }
 
+JSONScopedPrinter::JSONScopedPrinter(
+    raw_ostream &OS, bool PrettyPrint,
+    std::unique_ptr<DelimitedScope> &&OuterScope)
+    : ScopedPrinter(OS, ScopedPrinter::ScopedPrinterKind::JSON),
+      JOS(OS, /*Indent=*/PrettyPrint ? 2 : 0),
+      OuterScope(std::move(OuterScope)) {
+  if (this->OuterScope)
+    this->OuterScope->setPrinter(*this);
+}
+
 } // namespace llvm
diff --git a/llvm/lib/Support/Signals.cpp b/llvm/lib/Support/Signals.cpp
index dd4dded4cd1d..c018dc92bf40 100644
--- a/llvm/lib/Support/Signals.cpp
+++ b/llvm/lib/Support/Signals.cpp
@@ -87,8 +87,7 @@ static CallbackAndCookie CallBacksToRun[MaxSignalHandlerCallbacks];
 
 // Signal-safe.
 void sys::RunSignalHandlers() {
-  for (size_t I = 0; I < MaxSignalHandlerCallbacks; ++I) {
-    auto &RunMe = CallBacksToRun[I];
+  for (CallbackAndCookie &RunMe : CallBacksToRun) {
     auto Expected = CallbackAndCookie::Status::Initialized;
     auto Desired = CallbackAndCookie::Status::Executing;
     if (!RunMe.Flag.compare_exchange_strong(Expected, Desired))
@@ -103,8 +102,7 @@ void sys::RunSignalHandlers() {
 // Signal-safe.
 static void insertSignalHandler(sys::SignalHandlerCallback FnPtr,
                                 void *Cookie) {
-  for (size_t I = 0; I < MaxSignalHandlerCallbacks; ++I) {
-    auto &SetMe = CallBacksToRun[I];
+  for (CallbackAndCookie &SetMe : CallBacksToRun) {
     auto Expected = CallbackAndCookie::Status::Empty;
     auto Desired = CallbackAndCookie::Status::Initializing;
     if (!SetMe.Flag.compare_exchange_strong(Expected, Desired))
diff --git a/llvm/lib/Support/SourceMgr.cpp b/llvm/lib/Support/SourceMgr.cpp
index 89b7dc939dfc..2eb2989b200b 100644
--- a/llvm/lib/Support/SourceMgr.cpp
+++ b/llvm/lib/Support/SourceMgr.cpp
@@ -292,8 +292,7 @@ SMDiagnostic SourceMgr::GetMessage(SMLoc Loc, SourceMgr::DiagKind Kind,
 
     // Convert any ranges to column ranges that only intersect the line of the
     // location.
-    for (unsigned i = 0, e = Ranges.size(); i != e; ++i) {
-      SMRange R = Ranges[i];
+    for (SMRange R : Ranges) {
       if (!R.isValid())
         continue;
 
diff --git a/llvm/lib/Support/Statistic.cpp b/llvm/lib/Support/Statistic.cpp
index d95c8642c16e..95ee885d2f8f 100644
--- a/llvm/lib/Support/Statistic.cpp
+++ b/llvm/lib/Support/Statistic.cpp
@@ -177,11 +177,10 @@ void llvm::PrintStatistics(raw_ostream &OS) {
 
   // Figure out how long the biggest Value and Name fields are.
   unsigned MaxDebugTypeLen = 0, MaxValLen = 0;
-  for (size_t i = 0, e = Stats.Stats.size(); i != e; ++i) {
-    MaxValLen = std::max(MaxValLen,
-                         (unsigned)utostr(Stats.Stats[i]->getValue()).size());
-    MaxDebugTypeLen = std::max(MaxDebugTypeLen,
-                         (unsigned)std::strlen(Stats.Stats[i]->getDebugType()));
+  for (TrackingStatistic *Stat : Stats.Stats) {
+    MaxValLen = std::max(MaxValLen, (unsigned)utostr(Stat->getValue()).size());
+    MaxDebugTypeLen =
+        std::max(MaxDebugTypeLen, (unsigned)std::strlen(Stat->getDebugType()));
   }
 
   Stats.sort();
@@ -192,11 +191,9 @@ void llvm::PrintStatistics(raw_ostream &OS) {
      << "===" << std::string(73, '-') << "===\n\n";
 
   // Print all of the statistics.
-  for (size_t i = 0, e = Stats.Stats.size(); i != e; ++i)
-    OS << format("%*u %-*s - %s\n",
-                 MaxValLen, Stats.Stats[i]->getValue(),
-                 MaxDebugTypeLen, Stats.Stats[i]->getDebugType(),
-                 Stats.Stats[i]->getDesc());
+  for (TrackingStatistic *Stat : Stats.Stats)
+    OS << format("%*u %-*s - %s\n", MaxValLen, Stat->getValue(),
+                 MaxDebugTypeLen, Stat->getDebugType(), Stat->getDesc());
 
   OS << '\n';  // Flush the output stream.
   OS.flush();
diff --git a/llvm/lib/Support/TargetParser.cpp b/llvm/lib/Support/TargetParser.cpp
index 4acc23dd455b..bc60bdea5f62 100644
--- a/llvm/lib/Support/TargetParser.cpp
+++ b/llvm/lib/Support/TargetParser.cpp
@@ -331,6 +331,21 @@ bool getCPUFeaturesExceptStdExt(CPUKind Kind,
   return true;
 }
 
+StringRef computeDefaultABIFromArch(const llvm::RISCVISAInfo &ISAInfo) {
+  if (ISAInfo.getXLen() == 32) {
+    if (ISAInfo.hasExtension("d"))
+      return "ilp32d";
+    if (ISAInfo.hasExtension("e"))
+      return "ilp32e";
+    return "ilp32";
+  } else if (ISAInfo.getXLen() == 64) {
+    if (ISAInfo.hasExtension("d"))
+      return "lp64d";
+    return "lp64";
+  }
+  llvm_unreachable("Invalid XLEN");
+}
+
 } // namespace RISCV
 } // namespace llvm
 
diff --git a/llvm/lib/Support/ThreadPool.cpp b/llvm/lib/Support/ThreadPool.cpp
index c11e16d3cf98..54ea84d4bd6d 100644
--- a/llvm/lib/Support/ThreadPool.cpp
+++ b/llvm/lib/Support/ThreadPool.cpp
@@ -21,13 +21,17 @@ using namespace llvm;
 #if LLVM_ENABLE_THREADS
 
 ThreadPool::ThreadPool(ThreadPoolStrategy S)
-    : ThreadCount(S.compute_thread_count()) {
-  // Create ThreadCount threads that will loop forever, wait on QueueCondition
-  // for tasks to be queued or the Pool to be destroyed.
-  Threads.reserve(ThreadCount);
-  for (unsigned ThreadID = 0; ThreadID < ThreadCount; ++ThreadID) {
-    Threads.emplace_back([S, ThreadID, this] {
-      S.apply_thread_strategy(ThreadID);
+    : Strategy(S), MaxThreadCount(S.compute_thread_count()) {}
+
+void ThreadPool::grow(int requested) {
+  std::unique_lock<std::mutex> LockGuard(ThreadsLock);
+  if (Threads.size() >= MaxThreadCount)
+    return; // Already hit the max thread pool size.
+  int newThreadCount = std::min<int>(requested, MaxThreadCount);
+  while (static_cast<int>(Threads.size()) < newThreadCount) {
+    int ThreadID = Threads.size();
+    Threads.emplace_back([this, ThreadID] {
+      Strategy.apply_thread_strategy(ThreadID);
       while (true) {
         std::function<void()> Task;
         {
@@ -73,6 +77,7 @@ void ThreadPool::wait() {
 }
 
 bool ThreadPool::isWorkerThread() const {
+  std::unique_lock<std::mutex> LockGuard(ThreadsLock);
   llvm::thread::id CurrentThreadId = llvm::this_thread::get_id();
   for (const llvm::thread &Thread : Threads)
     if (CurrentThreadId == Thread.get_id())
@@ -87,6 +92,7 @@ ThreadPool::~ThreadPool() {
     EnableFlag = false;
   }
   QueueCondition.notify_all();
+  std::unique_lock<std::mutex> LockGuard(ThreadsLock);
   for (auto &Worker : Threads)
     Worker.join();
 }
@@ -94,8 +100,8 @@ ThreadPool::~ThreadPool() {
 #else // LLVM_ENABLE_THREADS Disabled
 
 // No threads are launched, issue a warning if ThreadCount is not 0
-ThreadPool::ThreadPool(ThreadPoolStrategy S)
-    : ThreadCount(S.compute_thread_count()) {
+ThreadPool::ThreadPool(ThreadPoolStrategy S) : MaxThreadCount(1) {
+  int ThreadCount = S.compute_thread_count();
   if (ThreadCount != 1) {
     errs() << "Warning: request a ThreadPool with " << ThreadCount
            << " threads, but LLVM_ENABLE_THREADS has been turned off\n";
diff --git a/llvm/lib/Support/Triple.cpp b/llvm/lib/Support/Triple.cpp
index b9a92e280576..2819dc0c139a 100644
--- a/llvm/lib/Support/Triple.cpp
+++ b/llvm/lib/Support/Triple.cpp
@@ -989,10 +989,9 @@ std::string Triple::normalize(StringRef Str) {
   }
 
   // Replace empty components with "unknown" value.
-  for (unsigned i = 0, e = Components.size(); i < e; ++i) {
-    if (Components[i].empty())
-      Components[i] = "unknown";
-  }
+  for (StringRef &C : Components)
+    if (C.empty())
+      C = "unknown";
 
   // Special case logic goes here.  At this point Arch, Vendor and OS have the
   // correct values for the computed components.
@@ -1091,53 +1090,22 @@ StringRef Triple::getOSAndEnvironmentName() const {
   return Tmp.split('-').second;                      // Strip second component
 }
 
-static unsigned EatNumber(StringRef &Str) {
-  assert(!Str.empty() && isDigit(Str[0]) && "Not a number");
-  unsigned Result = 0;
-
-  do {
-    // Consume the leading digit.
-    Result = Result*10 + (Str[0] - '0');
-
-    // Eat the digit.
-    Str = Str.substr(1);
-  } while (!Str.empty() && isDigit(Str[0]));
-
-  return Result;
-}
-
-static void parseVersionFromName(StringRef Name, unsigned &Major,
-                                 unsigned &Minor, unsigned &Micro) {
-  // Any unset version defaults to 0.
-  Major = Minor = Micro = 0;
-
-  // Parse up to three components.
-  unsigned *Components[3] = {&Major, &Minor, &Micro};
-  for (unsigned i = 0; i != 3; ++i) {
-    if (Name.empty() || Name[0] < '0' || Name[0] > '9')
-      break;
-
-    // Consume the leading number.
-    *Components[i] = EatNumber(Name);
-
-    // Consume the separator, if present.
-    if (Name.startswith("."))
-      Name = Name.substr(1);
-  }
+static VersionTuple parseVersionFromName(StringRef Name) {
+  VersionTuple Version;
+  Version.tryParse(Name);
+  return Version.withoutBuild();
 }
 
-void Triple::getEnvironmentVersion(unsigned &Major, unsigned &Minor,
-                                   unsigned &Micro) const {
+VersionTuple Triple::getEnvironmentVersion() const {
   StringRef EnvironmentName = getEnvironmentName();
   StringRef EnvironmentTypeName = getEnvironmentTypeName(getEnvironment());
   if (EnvironmentName.startswith(EnvironmentTypeName))
     EnvironmentName = EnvironmentName.substr(EnvironmentTypeName.size());
 
-  parseVersionFromName(EnvironmentName, Major, Minor, Micro);
+  return parseVersionFromName(EnvironmentName);
 }
 
-void Triple::getOSVersion(unsigned &Major, unsigned &Minor,
-                          unsigned &Micro) const {
+VersionTuple Triple::getOSVersion() const {
   StringRef OSName = getOSName();
   // Assume that the OS portion of the triple starts with the canonical name.
   StringRef OSTypeName = getOSTypeName(getOS());
@@ -1146,40 +1114,36 @@ void Triple::getOSVersion(unsigned &Major, unsigned &Minor,
   else if (getOS() == MacOSX)
     OSName.consume_front("macos");
 
-  parseVersionFromName(OSName, Major, Minor, Micro);
+  return parseVersionFromName(OSName);
 }
 
-bool Triple::getMacOSXVersion(unsigned &Major, unsigned &Minor,
-                              unsigned &Micro) const {
-  getOSVersion(Major, Minor, Micro);
+bool Triple::getMacOSXVersion(VersionTuple &Version) const {
+  Version = getOSVersion();
 
   switch (getOS()) {
   default: llvm_unreachable("unexpected OS for Darwin triple");
   case Darwin:
     // Default to darwin8, i.e., MacOSX 10.4.
-    if (Major == 0)
-      Major = 8;
+    if (Version.getMajor() == 0)
+      Version = VersionTuple(8);
     // Darwin version numbers are skewed from OS X versions.
-    if (Major < 4)
+    if (Version.getMajor() < 4) {
       return false;
-    if (Major <= 19) {
-      Micro = 0;
-      Minor = Major - 4;
-      Major = 10;
+    }
+    if (Version.getMajor() <= 19) {
+      Version = VersionTuple(10, Version.getMajor() - 4);
     } else {
-      Micro = 0;
-      Minor = 0;
       // darwin20+ corresponds to macOS 11+.
-      Major = 11 + Major - 20;
+      Version = VersionTuple(11 + Version.getMajor() - 20);
     }
     break;
   case MacOSX:
     // Default to 10.4.
-    if (Major == 0) {
-      Major = 10;
-      Minor = 4;
-    } else if (Major < 10)
+    if (Version.getMajor() == 0) {
+      Version = VersionTuple(10, 4);
+    } else if (Version.getMajor() < 10) {
       return false;
+    }
     break;
   case IOS:
   case TvOS:
@@ -1188,16 +1152,13 @@ bool Triple::getMacOSXVersion(unsigned &Major, unsigned &Minor,
     // the clang driver combines OS X and IOS support into a common Darwin
     // toolchain that wants to know the OS X version number even when targeting
     // IOS.
-    Major = 10;
-    Minor = 4;
-    Micro = 0;
+    Version = VersionTuple(10, 4);
     break;
   }
   return true;
 }
 
-void Triple::getiOSVersion(unsigned &Major, unsigned &Minor,
-                           unsigned &Micro) const {
+VersionTuple Triple::getiOSVersion() const {
   switch (getOS()) {
   default: llvm_unreachable("unexpected OS for Darwin triple");
   case Darwin:
@@ -1206,24 +1167,21 @@ void Triple::getiOSVersion(unsigned &Major, unsigned &Minor,
     // the clang driver combines OS X and IOS support into a common Darwin
     // toolchain that wants to know the iOS version number even when targeting
     // OS X.
-    Major = 5;
-    Minor = 0;
-    Micro = 0;
-    break;
+    return VersionTuple(5);
   case IOS:
-  case TvOS:
-    getOSVersion(Major, Minor, Micro);
+  case TvOS: {
+    VersionTuple Version = getOSVersion();
     // Default to 5.0 (or 7.0 for arm64).
-    if (Major == 0)
-      Major = (getArch() == aarch64) ? 7 : 5;
-    break;
+    if (Version.getMajor() == 0)
+      return (getArch() == aarch64) ? VersionTuple(7) : VersionTuple(5);
+    return Version;
+  }
   case WatchOS:
     llvm_unreachable("conflicting triple info");
   }
 }
 
-void Triple::getWatchOSVersion(unsigned &Major, unsigned &Minor,
-                               unsigned &Micro) const {
+VersionTuple Triple::getWatchOSVersion() const {
   switch (getOS()) {
   default: llvm_unreachable("unexpected OS for Darwin triple");
   case Darwin:
@@ -1232,15 +1190,13 @@ void Triple::getWatchOSVersion(unsigned &Major, unsigned &Minor,
     // the clang driver combines OS X and IOS support into a common Darwin
     // toolchain that wants to know the iOS version number even when targeting
     // OS X.
-    Major = 2;
-    Minor = 0;
-    Micro = 0;
-    break;
-  case WatchOS:
-    getOSVersion(Major, Minor, Micro);
-    if (Major == 0)
-      Major = 2;
-    break;
+    return VersionTuple(2);
+  case WatchOS: {
+    VersionTuple Version = getOSVersion();
+    if (Version.getMajor() == 0)
+      return VersionTuple(2);
+    return Version;
+  }
   case IOS:
     llvm_unreachable("conflicting triple info");
   }
diff --git a/llvm/lib/Support/Unix/Path.inc b/llvm/lib/Support/Unix/Path.inc
index 19d89db55627..f5cb5895d95d 100644
--- a/llvm/lib/Support/Unix/Path.inc
+++ b/llvm/lib/Support/Unix/Path.inc
@@ -590,19 +590,6 @@ std::error_code rename(const Twine &from, const Twine &to) {
 }
 
 std::error_code resize_file(int FD, uint64_t Size) {
-#if defined(HAVE_POSIX_FALLOCATE)
-  // If we have posix_fallocate use it. Unlike ftruncate it always allocates
-  // space, so we get an error if the disk is full.
-  if (int Err = ::posix_fallocate(FD, 0, Size)) {
-#ifdef _AIX
-    constexpr int NotSupportedError = ENOTSUP;
-#else
-    constexpr int NotSupportedError = EOPNOTSUPP;
-#endif
-    if (Err != EINVAL && Err != NotSupportedError)
-      return std::error_code(Err, std::generic_category());
-  }
-#endif
   // Use ftruncate as a fallback. It may or may not allocate space. At least on
   // OS X with HFS+ it does.
   if (::ftruncate(FD, Size) == -1)
diff --git a/llvm/lib/Support/VirtualFileSystem.cpp b/llvm/lib/Support/VirtualFileSystem.cpp
index 9bf0384b5f1b..bec4e8dbe06c 100644
--- a/llvm/lib/Support/VirtualFileSystem.cpp
+++ b/llvm/lib/Support/VirtualFileSystem.cpp
@@ -75,6 +75,12 @@ Status::Status(const Twine &Name, UniqueID UID, sys::TimePoint<> MTime,
     : Name(Name.str()), UID(UID), MTime(MTime), User(User), Group(Group),
       Size(Size), Type(Type), Perms(Perms) {}
 
+Status Status::copyWithNewSize(const Status &In, uint64_t NewSize) {
+  return Status(In.getName(), In.getUniqueID(), In.getLastModificationTime(),
+                In.getUser(), In.getGroup(), NewSize, In.getType(),
+                In.getPermissions());
+}
+
 Status Status::copyWithNewName(const Status &In, const Twine &NewName) {
   return Status(NewName, In.getUniqueID(), In.getLastModificationTime(),
                 In.getUser(), In.getGroup(), In.getSize(), In.getType(),
diff --git a/llvm/lib/Support/YAMLParser.cpp b/llvm/lib/Support/YAMLParser.cpp
index f68ba0d065c1..2adf37a511d1 100644
--- a/llvm/lib/Support/YAMLParser.cpp
+++ b/llvm/lib/Support/YAMLParser.cpp
@@ -1876,8 +1876,8 @@ document_iterator Stream::end() {
 }
 
 void Stream::skip() {
-  for (document_iterator i = begin(), e = end(); i != e; ++i)
-    i->skip();
+  for (Document &Doc : *this)
+    Doc.skip();
 }
 
 Node::Node(unsigned int Type, std::unique_ptr<Document> &D, StringRef A,
diff --git a/llvm/lib/TableGen/StringMatcher.cpp b/llvm/lib/TableGen/StringMatcher.cpp
index 7f30c7b60752..7474c5dfe885 100644
--- a/llvm/lib/TableGen/StringMatcher.cpp
+++ b/llvm/lib/TableGen/StringMatcher.cpp
@@ -32,8 +32,8 @@ FindFirstNonCommonLetter(const std::vector<const
     // Check to see if letter i is the same across the set.
     char Letter = Matches[0]->first[i];
 
-    for (unsigned str = 0, e = Matches.size(); str != e; ++str)
-      if (Matches[str]->first[i] != Letter)
+    for (const StringMatcher::StringPair *Match : Matches)
+      if (Match->first[i] != Letter)
         return i;
   }
 
@@ -75,9 +75,8 @@ bool StringMatcher::EmitStringMatcherForChar(
   // Bucket the matches by the character we are comparing.
   std::map<char, std::vector<const StringPair*>> MatchesByLetter;
 
-  for (unsigned i = 0, e = Matches.size(); i != e; ++i)
-    MatchesByLetter[Matches[i]->first[CharNo]].push_back(Matches[i]);
-
+  for (const StringPair *Match : Matches)
+    MatchesByLetter[Match->first[CharNo]].push_back(Match);
 
   // If we have exactly one bucket to match, see how many characters are common
   // across the whole set and match all of them at once.
@@ -135,8 +134,8 @@ void StringMatcher::Emit(unsigned Indent, bool IgnoreDuplicates) const {
   // First level categorization: group strings by length.
   std::map<unsigned, std::vector<const StringPair*>> MatchesByLength;
 
-  for (unsigned i = 0, e = Matches.size(); i != e; ++i)
-    MatchesByLength[Matches[i].first.size()].push_back(&Matches[i]);
+  for (const StringPair &Match : Matches)
+    MatchesByLength[Match.first.size()].push_back(&Match);
 
   // Output a switch statement on length and categorize the elements within each
   // bin.
diff --git a/llvm/lib/Target/AArch64/AArch64.td b/llvm/lib/Target/AArch64/AArch64.td
index 548e4e0c9389..cb17fd94c335 100644
--- a/llvm/lib/Target/AArch64/AArch64.td
+++ b/llvm/lib/Target/AArch64/AArch64.td
@@ -455,6 +455,9 @@ def FeatureEL2VMSA : SubtargetFeature<"el2vmsa", "HasEL2VMSA", "true",
 def FeatureEL3 : SubtargetFeature<"el3", "HasEL3", "true",
   "Enable Exception Level 3">;
 
+def FeatureFixCortexA53_835769 : SubtargetFeature<"fix-cortex-a53-835769",
+  "FixCortexA53_835769", "true", "Mitigate Cortex-A53 Erratum 835769">;
+
 //===----------------------------------------------------------------------===//
 // Architectures.
 //
diff --git a/llvm/lib/Target/AArch64/AArch64A53Fix835769.cpp b/llvm/lib/Target/AArch64/AArch64A53Fix835769.cpp
index 7fd51a98ad94..4cdf5f144437 100644
--- a/llvm/lib/Target/AArch64/AArch64A53Fix835769.cpp
+++ b/llvm/lib/Target/AArch64/AArch64A53Fix835769.cpp
@@ -15,6 +15,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "AArch64.h"
+#include "AArch64Subtarget.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
@@ -116,8 +117,13 @@ INITIALIZE_PASS(AArch64A53Fix835769, "aarch64-fix-cortex-a53-835769-pass",
 bool
 AArch64A53Fix835769::runOnMachineFunction(MachineFunction &F) {
   LLVM_DEBUG(dbgs() << "***** AArch64A53Fix835769 *****\n");
+  auto &STI = F.getSubtarget<AArch64Subtarget>();
+  // Fix not requested, skip pass.
+  if (!STI.fixCortexA53_835769())
+    return false;
+
   bool Changed = false;
-  TII = F.getSubtarget().getInstrInfo();
+  TII = STI.getInstrInfo();
 
   for (auto &MBB : F) {
     Changed |= runOnBasicBlock(MBB);
diff --git a/llvm/lib/Target/AArch64/AArch64AdvSIMDScalarPass.cpp b/llvm/lib/Target/AArch64/AArch64AdvSIMDScalarPass.cpp
index cd67e058a9c1..9e31243cd696 100644
--- a/llvm/lib/Target/AArch64/AArch64AdvSIMDScalarPass.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AdvSIMDScalarPass.cpp
@@ -398,8 +398,8 @@ bool AArch64AdvSIMDScalar::runOnMachineFunction(MachineFunction &mf) {
   TII = mf.getSubtarget().getInstrInfo();
 
   // Just check things on a one-block-at-a-time basis.
-  for (MachineFunction::iterator I = mf.begin(), E = mf.end(); I != E; ++I)
-    if (processMachineBasicBlock(&*I))
+  for (MachineBasicBlock &MBB : mf)
+    if (processMachineBasicBlock(&MBB))
       Changed = true;
   return Changed;
 }
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index aeebb49675b2..85a9c04a3fef 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -73,6 +73,7 @@ class AArch64AsmPrinter : public AsmPrinter {
   StackMaps SM;
   FaultMaps FM;
   const AArch64Subtarget *STI;
+  bool ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = false;
 
 public:
   AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
@@ -186,6 +187,10 @@ private:
   using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>;
 
   MInstToMCSymbol LOHInstToLabel;
+
+  bool shouldEmitWeakSwiftAsyncExtendedFramePointerFlags() const override {
+    return ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags;
+  }
 };
 
 } // end anonymous namespace
@@ -1132,6 +1137,15 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
   if (emitPseudoExpansionLowering(*OutStreamer, MI))
     return;
 
+  if (MI->getOpcode() == AArch64::ADRP) {
+    for (auto &Opd : MI->operands()) {
+      if (Opd.isSymbol() && StringRef(Opd.getSymbolName()) ==
+                                "swift_async_extendedFramePointerFlags") {
+        ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = true;
+      }
+    }
+  }
+
   if (AArch64FI->getLOHRelated().count(MI)) {
     // Generate a label for LOH related instruction
     MCSymbol *LOHLabel = createTempSymbol("loh");
diff --git a/llvm/lib/Target/AArch64/AArch64Combine.td b/llvm/lib/Target/AArch64/AArch64Combine.td
index d2097f7e6ee3..1994e0eb7fb9 100644
--- a/llvm/lib/Target/AArch64/AArch64Combine.td
+++ b/llvm/lib/Target/AArch64/AArch64Combine.td
@@ -196,6 +196,13 @@ def mutate_anyext_to_zext : GICombineRule<
   (apply [{ applyMutateAnyExtToZExt(*${d}, MRI, B, Observer); }])
 >;
 
+def split_store_zero_128 : GICombineRule<
+  (defs root:$d),
+  (match (wip_match_opcode G_STORE):$d,
+          [{ return matchSplitStoreZero128(*${d}, MRI); }]),
+  (apply [{ applySplitStoreZero128(*${d}, MRI, B, Observer); }])
+>;
+
 // Post-legalization combines which should happen at all optimization levels.
 // (E.g. ones that facilitate matching for the selector) For example, matching
 // pseudos.
@@ -220,6 +227,7 @@ def AArch64PostLegalizerCombinerHelper
                         icmp_to_true_false_known_bits, merge_unmerge,
                         select_combines, fold_merge_to_zext,
                         constant_fold, identity_combines,
-                        ptr_add_immed_chain, overlapping_and]> {
+                        ptr_add_immed_chain, overlapping_and,
+                        split_store_zero_128]> {
   let DisableRuleOption = "aarch64postlegalizercombiner-disable-rule";
 }
diff --git a/llvm/lib/Target/AArch64/AArch64ExpandImm.cpp b/llvm/lib/Target/AArch64/AArch64ExpandImm.cpp
index d98a5cfd4f50..4f324198f3dc 100644
--- a/llvm/lib/Target/AArch64/AArch64ExpandImm.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ExpandImm.cpp
@@ -51,10 +51,9 @@ static bool tryToreplicateChunks(uint64_t UImm,
     ++Counts[getChunk(UImm, Idx)];
 
   // Traverse the chunks to find one which occurs more than once.
-  for (CountMap::const_iterator Chunk = Counts.begin(), End = Counts.end();
-       Chunk != End; ++Chunk) {
-    const uint64_t ChunkVal = Chunk->first;
-    const unsigned Count = Chunk->second;
+  for (const auto &Chunk : Counts) {
+    const uint64_t ChunkVal = Chunk.first;
+    const unsigned Count = Chunk.second;
 
     uint64_t Encoding = 0;
 
diff --git a/llvm/lib/Target/AArch64/AArch64FalkorHWPFFix.cpp b/llvm/lib/Target/AArch64/AArch64FalkorHWPFFix.cpp
index 209f9f7255a5..793663ef97d7 100644
--- a/llvm/lib/Target/AArch64/AArch64FalkorHWPFFix.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FalkorHWPFFix.cpp
@@ -138,8 +138,8 @@ bool FalkorMarkStridedAccesses::run() {
   bool MadeChange = false;
 
   for (Loop *L : LI)
-    for (auto LIt = df_begin(L), LE = df_end(L); LIt != LE; ++LIt)
-      MadeChange |= runOnLoop(**LIt);
+    for (Loop *LIt : depth_first(L))
+      MadeChange |= runOnLoop(*LIt);
 
   return MadeChange;
 }
@@ -828,10 +828,10 @@ bool FalkorHWPFFix::runOnMachineFunction(MachineFunction &Fn) {
   Modified = false;
 
   for (MachineLoop *I : LI)
-    for (auto L = df_begin(I), LE = df_end(I); L != LE; ++L)
+    for (MachineLoop *L : depth_first(I))
       // Only process inner-loops
       if (L->isInnermost())
-        runOnLoop(**L, Fn);
+        runOnLoop(*L, Fn);
 
   return Modified;
 }
diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index b630f4f0df5f..638e45b30d99 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -3041,10 +3041,21 @@ static int64_t determineSVEStackObjectOffsets(MachineFrameInfo &MFI,
 
   // Create a buffer of SVE objects to allocate and sort it.
   SmallVector<int, 8> ObjectsToAllocate;
+  // If we have a stack protector, and we've previously decided that we have SVE
+  // objects on the stack and thus need it to go in the SVE stack area, then it
+  // needs to go first.
+  int StackProtectorFI = -1;
+  if (MFI.hasStackProtectorIndex()) {
+    StackProtectorFI = MFI.getStackProtectorIndex();
+    if (MFI.getStackID(StackProtectorFI) == TargetStackID::ScalableVector)
+      ObjectsToAllocate.push_back(StackProtectorFI);
+  }
   for (int I = 0, E = MFI.getObjectIndexEnd(); I != E; ++I) {
     unsigned StackID = MFI.getStackID(I);
     if (StackID != TargetStackID::ScalableVector)
       continue;
+    if (I == StackProtectorFI)
+      continue;
     if (MaxCSFrameIndex >= I && I >= MinCSFrameIndex)
       continue;
     if (MFI.isDeadObjectIndex(I))
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 72461aa1f772..e141179fb5c8 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -33,6 +33,7 @@
 #include "llvm/Analysis/VectorUtils.h"
 #include "llvm/CodeGen/Analysis.h"
 #include "llvm/CodeGen/CallingConvLower.h"
+#include "llvm/CodeGen/ISDOpcodes.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
@@ -204,6 +205,8 @@ static bool isMergePassthruOpcode(unsigned Opc) {
     return false;
   case AArch64ISD::BITREVERSE_MERGE_PASSTHRU:
   case AArch64ISD::BSWAP_MERGE_PASSTHRU:
+  case AArch64ISD::REVH_MERGE_PASSTHRU:
+  case AArch64ISD::REVW_MERGE_PASSTHRU:
   case AArch64ISD::CTLZ_MERGE_PASSTHRU:
   case AArch64ISD::CTPOP_MERGE_PASSTHRU:
   case AArch64ISD::DUP_MERGE_PASSTHRU:
@@ -2227,6 +2230,8 @@ const char *AArch64TargetLowering::getTargetNodeName(unsigned Opcode) const {
     MAKE_CASE(AArch64ISD::STNP)
     MAKE_CASE(AArch64ISD::BITREVERSE_MERGE_PASSTHRU)
     MAKE_CASE(AArch64ISD::BSWAP_MERGE_PASSTHRU)
+    MAKE_CASE(AArch64ISD::REVH_MERGE_PASSTHRU)
+    MAKE_CASE(AArch64ISD::REVW_MERGE_PASSTHRU)
     MAKE_CASE(AArch64ISD::CTLZ_MERGE_PASSTHRU)
     MAKE_CASE(AArch64ISD::CTPOP_MERGE_PASSTHRU)
     MAKE_CASE(AArch64ISD::DUP_MERGE_PASSTHRU)
@@ -4213,6 +4218,12 @@ SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
   case Intrinsic::aarch64_sve_revb:
     return DAG.getNode(AArch64ISD::BSWAP_MERGE_PASSTHRU, dl, Op.getValueType(),
                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
+  case Intrinsic::aarch64_sve_revh:
+    return DAG.getNode(AArch64ISD::REVH_MERGE_PASSTHRU, dl, Op.getValueType(),
+                       Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
+  case Intrinsic::aarch64_sve_revw:
+    return DAG.getNode(AArch64ISD::REVW_MERGE_PASSTHRU, dl, Op.getValueType(),
+                       Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
   case Intrinsic::aarch64_sve_sxtb:
     return DAG.getNode(
         AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU, dl, Op.getValueType(),
@@ -10958,16 +10969,15 @@ SDValue AArch64TargetLowering::LowerINSERT_SUBVECTOR(SDValue Op,
   EVT InVT = Op.getOperand(1).getValueType();
   unsigned Idx = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue();
 
-  if (InVT.isScalableVector()) {
-    SDLoc DL(Op);
-    EVT VT = Op.getValueType();
+  SDValue Vec0 = Op.getOperand(0);
+  SDValue Vec1 = Op.getOperand(1);
+  SDLoc DL(Op);
+  EVT VT = Op.getValueType();
 
+  if (InVT.isScalableVector()) {
     if (!isTypeLegal(VT))
       return SDValue();
 
-    SDValue Vec0 = Op.getOperand(0);
-    SDValue Vec1 = Op.getOperand(1);
-
     // Ensure the subvector is half the size of the main vector.
     if (VT.getVectorElementCount() != (InVT.getVectorElementCount() * 2))
       return SDValue();
@@ -10997,9 +11007,18 @@ SDValue AArch64TargetLowering::LowerINSERT_SUBVECTOR(SDValue Op,
     return SDValue();
   }
 
-  // This will be matched by custom code during ISelDAGToDAG.
-  if (Idx == 0 && isPackedVectorType(InVT, DAG) && Op.getOperand(0).isUndef())
-    return Op;
+  if (Idx == 0 && isPackedVectorType(VT, DAG)) {
+    // This will be matched by custom code during ISelDAGToDAG.
+    if (Vec0.isUndef())
+      return Op;
+
+    unsigned int PredPattern =
+        getSVEPredPatternFromNumElements(InVT.getVectorNumElements());
+    auto PredTy = VT.changeVectorElementType(MVT::i1);
+    SDValue PTrue = getPTrue(DAG, DL, PredTy, PredPattern);
+    SDValue ScalableVec1 = convertToScalableVector(DAG, VT, Vec1);
+    return DAG.getNode(ISD::VSELECT, DL, VT, PTrue, ScalableVec1, Vec0);
+  }
 
   return SDValue();
 }
@@ -11794,6 +11813,9 @@ bool AArch64TargetLowering::shouldReduceLoadWidth(SDNode *Load,
       Base.getOperand(1).getOpcode() == ISD::SHL &&
       Base.getOperand(1).hasOneUse() &&
       Base.getOperand(1).getOperand(1).getOpcode() == ISD::Constant) {
+    // It's unknown whether a scalable vector has a power-of-2 bitwidth.
+    if (Mem->getMemoryVT().isScalableVector())
+      return false;
     // The shift can be combined if it matches the size of the value being
     // loaded (and so reducing the width would make it not match).
     uint64_t ShiftAmount = Base.getOperand(1).getConstantOperandVal(1);
@@ -15820,6 +15842,23 @@ static SDValue performVectorShiftCombine(SDNode *N,
   return SDValue();
 }
 
+static SDValue performSunpkloCombine(SDNode *N, SelectionDAG &DAG) {
+  // sunpklo(sext(pred)) -> sext(extract_low_half(pred))
+  // This transform works in partnership with performSetCCPunpkCombine to
+  // remove unnecessary transfer of predicates into standard registers and back
+  if (N->getOperand(0).getOpcode() == ISD::SIGN_EXTEND &&
+      N->getOperand(0)->getOperand(0)->getValueType(0).getScalarType() ==
+          MVT::i1) {
+    SDValue CC = N->getOperand(0)->getOperand(0);
+    auto VT = CC->getValueType(0).getHalfNumVectorElementsVT(*DAG.getContext());
+    SDValue Unpk = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), VT, CC,
+                               DAG.getVectorIdxConstant(0, SDLoc(N)));
+    return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), N->getValueType(0), Unpk);
+  }
+
+  return SDValue();
+}
+
 /// Target-specific DAG combine function for post-increment LD1 (lane) and
 /// post-increment LD1R.
 static SDValue performPostLD1Combine(SDNode *N,
@@ -15982,7 +16021,9 @@ static SDValue performSTORECombine(SDNode *N,
   if (DCI.isBeforeLegalizeOps() && Value.getOpcode() == ISD::FP_ROUND &&
       Value.getNode()->hasOneUse() && ST->isUnindexed() &&
       Subtarget->useSVEForFixedLengthVectors() &&
-      Value.getValueType().isFixedLengthVector())
+      Value.getValueType().isFixedLengthVector() &&
+      Value.getValueType().getFixedSizeInBits() >=
+          Subtarget->getMinSVEVectorSizeInBits())
     return DAG.getTruncStore(Chain, SDLoc(N), Value.getOperand(0), Ptr,
                              ST->getMemoryVT(), ST->getMemOperand());
 
@@ -16495,6 +16536,44 @@ static SDValue performSETCCCombine(SDNode *N, SelectionDAG &DAG) {
   return SDValue();
 }
 
+static SDValue performSetCCPunpkCombine(SDNode *N, SelectionDAG &DAG) {
+  // setcc_merge_zero pred
+  //   (sign_extend (extract_subvector (setcc_merge_zero ... pred ...))), 0, ne
+  //   => extract_subvector (inner setcc_merge_zero)
+  SDValue Pred = N->getOperand(0);
+  SDValue LHS = N->getOperand(1);
+  SDValue RHS = N->getOperand(2);
+  ISD::CondCode Cond = cast<CondCodeSDNode>(N->getOperand(3))->get();
+
+  if (Cond != ISD::SETNE || !isZerosVector(RHS.getNode()) ||
+      LHS->getOpcode() != ISD::SIGN_EXTEND)
+    return SDValue();
+
+  SDValue Extract = LHS->getOperand(0);
+  if (Extract->getOpcode() != ISD::EXTRACT_SUBVECTOR ||
+      Extract->getValueType(0) != N->getValueType(0) ||
+      Extract->getConstantOperandVal(1) != 0)
+    return SDValue();
+
+  SDValue InnerSetCC = Extract->getOperand(0);
+  if (InnerSetCC->getOpcode() != AArch64ISD::SETCC_MERGE_ZERO)
+    return SDValue();
+
+  // By this point we've effectively got
+  // zero_inactive_lanes_and_trunc_i1(sext_i1(A)). If we can prove A's inactive
+  // lanes are already zero then the trunc(sext()) sequence is redundant and we
+  // can operate on A directly.
+  SDValue InnerPred = InnerSetCC.getOperand(0);
+  if (Pred.getOpcode() == AArch64ISD::PTRUE &&
+      InnerPred.getOpcode() == AArch64ISD::PTRUE &&
+      Pred.getConstantOperandVal(0) == InnerPred.getConstantOperandVal(0) &&
+      Pred->getConstantOperandVal(0) >= AArch64SVEPredPattern::vl1 &&
+      Pred->getConstantOperandVal(0) <= AArch64SVEPredPattern::vl256)
+    return Extract;
+
+  return SDValue();
+}
+
 static SDValue performSetccMergeZeroCombine(SDNode *N, SelectionDAG &DAG) {
   assert(N->getOpcode() == AArch64ISD::SETCC_MERGE_ZERO &&
          "Unexpected opcode!");
@@ -16513,6 +16592,9 @@ static SDValue performSetccMergeZeroCombine(SDNode *N, SelectionDAG &DAG) {
       LHS->getOperand(0)->getOperand(0) == Pred)
     return LHS->getOperand(0);
 
+  if (SDValue V = performSetCCPunpkCombine(N, DAG))
+    return V;
+
   return SDValue();
 }
 
@@ -17343,7 +17425,8 @@ SDValue performFPExtendCombine(SDNode *N, SelectionDAG &DAG,
   // they can be split down into something legal.
   if (DCI.isBeforeLegalizeOps() && ISD::isNormalLoad(N0.getNode()) &&
       N0.hasOneUse() && Subtarget->useSVEForFixedLengthVectors() &&
-      VT.isFixedLengthVector()) {
+      VT.isFixedLengthVector() &&
+      VT.getFixedSizeInBits() >= Subtarget->getMinSVEVectorSizeInBits()) {
     LoadSDNode *LN0 = cast<LoadSDNode>(N0);
     SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, SDLoc(N), VT,
                                      LN0->getChain(), LN0->getBasePtr(),
@@ -17455,6 +17538,8 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
   case AArch64ISD::VASHR:
   case AArch64ISD::VLSHR:
     return performVectorShiftCombine(N, *this, DCI);
+  case AArch64ISD::SUNPKLO:
+    return performSunpkloCombine(N, DAG);
   case ISD::INSERT_VECTOR_ELT:
     return performInsertVectorEltCombine(N, DCI);
   case ISD::EXTRACT_VECTOR_ELT:
@@ -18570,7 +18655,25 @@ AArch64TargetLowering::getVaListSizeInBits(const DataLayout &DL) const {
 }
 
 void AArch64TargetLowering::finalizeLowering(MachineFunction &MF) const {
-  MF.getFrameInfo().computeMaxCallFrameSize(MF);
+  MachineFrameInfo &MFI = MF.getFrameInfo();
+  // If we have any vulnerable SVE stack objects then the stack protector
+  // needs to be placed at the top of the SVE stack area, as the SVE locals
+  // are placed above the other locals, so we allocate it as if it were a
+  // scalable vector.
+  // FIXME: It may be worthwhile having a specific interface for this rather
+  // than doing it here in finalizeLowering.
+  if (MFI.hasStackProtectorIndex()) {
+    for (unsigned int i = 0, e = MFI.getObjectIndexEnd(); i != e; ++i) {
+      if (MFI.getStackID(i) == TargetStackID::ScalableVector &&
+          MFI.getObjectSSPLayout(i) != MachineFrameInfo::SSPLK_None) {
+        MFI.setStackID(MFI.getStackProtectorIndex(),
+                       TargetStackID::ScalableVector);
+        MFI.setObjectAlignment(MFI.getStackProtectorIndex(), Align(16));
+        break;
+      }
+    }
+  }
+  MFI.computeMaxCallFrameSize(MF);
   TargetLoweringBase::finalizeLowering(MF);
 }
 
@@ -18855,10 +18958,7 @@ SDValue AArch64TargetLowering::LowerFixedLengthVectorStoreToSVE(
 
 SDValue AArch64TargetLowering::LowerFixedLengthVectorMStoreToSVE(
     SDValue Op, SelectionDAG &DAG) const {
-  auto Store = cast<MaskedStoreSDNode>(Op);
-
-  if (Store->isTruncatingStore())
-    return SDValue();
+  auto *Store = cast<MaskedStoreSDNode>(Op);
 
   SDLoc DL(Op);
   EVT VT = Store->getValue().getValueType();
@@ -19103,7 +19203,7 @@ SDValue AArch64TargetLowering::LowerToPredicatedOp(SDValue Op,
   if (isMergePassthruOpcode(NewOp))
     Operands.push_back(DAG.getUNDEF(VT));
 
-  return DAG.getNode(NewOp, DL, VT, Operands);
+  return DAG.getNode(NewOp, DL, VT, Operands, Op->getFlags());
 }
 
 // If a fixed length vector operation has no side effects when applied to
@@ -19498,6 +19598,94 @@ SDValue AArch64TargetLowering::LowerFixedLengthVECTOR_SHUFFLEToSVE(
     return convertFromScalableVector(DAG, VT, Op);
   }
 
+  for (unsigned LaneSize : {64U, 32U, 16U}) {
+    if (isREVMask(ShuffleMask, VT, LaneSize)) {
+      EVT NewVT =
+          getPackedSVEVectorVT(EVT::getIntegerVT(*DAG.getContext(), LaneSize));
+      unsigned RevOp;
+      unsigned EltSz = VT.getScalarSizeInBits();
+      if (EltSz == 8)
+        RevOp = AArch64ISD::BSWAP_MERGE_PASSTHRU;
+      else if (EltSz == 16)
+        RevOp = AArch64ISD::REVH_MERGE_PASSTHRU;
+      else
+        RevOp = AArch64ISD::REVW_MERGE_PASSTHRU;
+
+      Op = DAG.getNode(ISD::BITCAST, DL, NewVT, Op1);
+      Op = LowerToPredicatedOp(Op, DAG, RevOp);
+      Op = DAG.getNode(ISD::BITCAST, DL, ContainerVT, Op);
+      return convertFromScalableVector(DAG, VT, Op);
+    }
+  }
+
+  unsigned WhichResult;
+  if (isZIPMask(ShuffleMask, VT, WhichResult) && WhichResult == 0)
+    return convertFromScalableVector(
+        DAG, VT, DAG.getNode(AArch64ISD::ZIP1, DL, ContainerVT, Op1, Op2));
+
+  if (isTRNMask(ShuffleMask, VT, WhichResult)) {
+    unsigned Opc = (WhichResult == 0) ? AArch64ISD::TRN1 : AArch64ISD::TRN2;
+    return convertFromScalableVector(
+        DAG, VT, DAG.getNode(Opc, DL, ContainerVT, Op1, Op2));
+  }
+
+  if (isZIP_v_undef_Mask(ShuffleMask, VT, WhichResult) && WhichResult == 0)
+    return convertFromScalableVector(
+        DAG, VT, DAG.getNode(AArch64ISD::ZIP1, DL, ContainerVT, Op1, Op1));
+
+  if (isTRN_v_undef_Mask(ShuffleMask, VT, WhichResult)) {
+    unsigned Opc = (WhichResult == 0) ? AArch64ISD::TRN1 : AArch64ISD::TRN2;
+    return convertFromScalableVector(
+        DAG, VT, DAG.getNode(Opc, DL, ContainerVT, Op1, Op1));
+  }
+
+  // Functions like isZIPMask return true when a ISD::VECTOR_SHUFFLE's mask
+  // represents the same logical operation as performed by a ZIP instruction. In
+  // isolation these functions do not mean the ISD::VECTOR_SHUFFLE is exactly
+  // equivalent to an AArch64 instruction. There's the extra component of
+  // ISD::VECTOR_SHUFFLE's value type to consider. Prior to SVE these functions
+  // only operated on 64/128bit vector types that have a direct mapping to a
+  // target register and so an exact mapping is implied.
+  // However, when using SVE for fixed length vectors, most legal vector types
+  // are actually sub-vectors of a larger SVE register. When mapping
+  // ISD::VECTOR_SHUFFLE to an SVE instruction care must be taken to consider
+  // how the mask's indices translate. Specifically, when the mapping requires
+  // an exact meaning for a specific vector index (e.g. Index X is the last
+  // vector element in the register) then such mappings are often only safe when
+  // the exact SVE register size is know. The main exception to this is when
+  // indices are logically relative to the first element of either
+  // ISD::VECTOR_SHUFFLE operand because these relative indices don't change
+  // when converting from fixed-length to scalable vector types (i.e. the start
+  // of a fixed length vector is always the start of a scalable vector).
+  unsigned MinSVESize = Subtarget->getMinSVEVectorSizeInBits();
+  unsigned MaxSVESize = Subtarget->getMaxSVEVectorSizeInBits();
+  if (MinSVESize == MaxSVESize && MaxSVESize == VT.getSizeInBits()) {
+    if (ShuffleVectorInst::isReverseMask(ShuffleMask) && Op2.isUndef()) {
+      Op = DAG.getNode(ISD::VECTOR_REVERSE, DL, ContainerVT, Op1);
+      return convertFromScalableVector(DAG, VT, Op);
+    }
+
+    if (isZIPMask(ShuffleMask, VT, WhichResult) && WhichResult != 0)
+      return convertFromScalableVector(
+          DAG, VT, DAG.getNode(AArch64ISD::ZIP2, DL, ContainerVT, Op1, Op2));
+
+    if (isUZPMask(ShuffleMask, VT, WhichResult)) {
+      unsigned Opc = (WhichResult == 0) ? AArch64ISD::UZP1 : AArch64ISD::UZP2;
+      return convertFromScalableVector(
+          DAG, VT, DAG.getNode(Opc, DL, ContainerVT, Op1, Op2));
+    }
+
+    if (isZIP_v_undef_Mask(ShuffleMask, VT, WhichResult) && WhichResult != 0)
+      return convertFromScalableVector(
+          DAG, VT, DAG.getNode(AArch64ISD::ZIP2, DL, ContainerVT, Op1, Op1));
+
+    if (isUZP_v_undef_Mask(ShuffleMask, VT, WhichResult)) {
+      unsigned Opc = (WhichResult == 0) ? AArch64ISD::UZP1 : AArch64ISD::UZP2;
+      return convertFromScalableVector(
+          DAG, VT, DAG.getNode(Opc, DL, ContainerVT, Op1, Op1));
+    }
+  }
+
   return SDValue();
 }
 
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index ea884cdccd28..367ba3039a0c 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -324,6 +324,8 @@ enum NodeType : unsigned {
 
   BITREVERSE_MERGE_PASSTHRU,
   BSWAP_MERGE_PASSTHRU,
+  REVH_MERGE_PASSTHRU,
+  REVW_MERGE_PASSTHRU,
   CTLZ_MERGE_PASSTHRU,
   CTPOP_MERGE_PASSTHRU,
   DUP_MERGE_PASSTHRU,
diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
index cd4bc8a61a8a..f8d492188744 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
@@ -387,16 +387,16 @@ def simm7s16 : Operand<i32> {
   let PrintMethod = "printImmScale<16>";
 }
 
-def am_sve_fi : ComplexPattern<i64, 2, "SelectAddrModeFrameIndexSVE", []>;
+def am_sve_fi : ComplexPattern<iPTR, 2, "SelectAddrModeFrameIndexSVE", []>;
 
-def am_indexed7s8   : ComplexPattern<i64, 2, "SelectAddrModeIndexed7S8", []>;
-def am_indexed7s16  : ComplexPattern<i64, 2, "SelectAddrModeIndexed7S16", []>;
-def am_indexed7s32  : ComplexPattern<i64, 2, "SelectAddrModeIndexed7S32", []>;
-def am_indexed7s64  : ComplexPattern<i64, 2, "SelectAddrModeIndexed7S64", []>;
-def am_indexed7s128 : ComplexPattern<i64, 2, "SelectAddrModeIndexed7S128", []>;
+def am_indexed7s8   : ComplexPattern<iPTR, 2, "SelectAddrModeIndexed7S8", []>;
+def am_indexed7s16  : ComplexPattern<iPTR, 2, "SelectAddrModeIndexed7S16", []>;
+def am_indexed7s32  : ComplexPattern<iPTR, 2, "SelectAddrModeIndexed7S32", []>;
+def am_indexed7s64  : ComplexPattern<iPTR, 2, "SelectAddrModeIndexed7S64", []>;
+def am_indexed7s128 : ComplexPattern<iPTR, 2, "SelectAddrModeIndexed7S128", []>;
 
-def am_indexedu6s128 : ComplexPattern<i64, 2, "SelectAddrModeIndexedU6S128", []>;
-def am_indexeds9s128 : ComplexPattern<i64, 2, "SelectAddrModeIndexedS9S128", []>;
+def am_indexedu6s128 : ComplexPattern<iPTR, 2, "SelectAddrModeIndexedU6S128", []>;
+def am_indexeds9s128 : ComplexPattern<iPTR, 2, "SelectAddrModeIndexedS9S128", []>;
 
 def UImmS1XForm : SDNodeXForm<imm, [{
   return CurDAG->getTargetConstant(N->getZExtValue(), SDLoc(N), MVT::i64);
@@ -3177,18 +3177,18 @@ def maski16_or_more : Operand<i32>,
 
 // (unsigned immediate)
 // Indexed for 8-bit registers. offset is in range [0,4095].
-def am_indexed8 : ComplexPattern<i64, 2, "SelectAddrModeIndexed8", []>;
-def am_indexed16 : ComplexPattern<i64, 2, "SelectAddrModeIndexed16", []>;
-def am_indexed32 : ComplexPattern<i64, 2, "SelectAddrModeIndexed32", []>;
-def am_indexed64 : ComplexPattern<i64, 2, "SelectAddrModeIndexed64", []>;
-def am_indexed128 : ComplexPattern<i64, 2, "SelectAddrModeIndexed128", []>;
+def am_indexed8 : ComplexPattern<iPTR, 2, "SelectAddrModeIndexed8", []>;
+def am_indexed16 : ComplexPattern<iPTR, 2, "SelectAddrModeIndexed16", []>;
+def am_indexed32 : ComplexPattern<iPTR, 2, "SelectAddrModeIndexed32", []>;
+def am_indexed64 : ComplexPattern<iPTR, 2, "SelectAddrModeIndexed64", []>;
+def am_indexed128 : ComplexPattern<iPTR, 2, "SelectAddrModeIndexed128", []>;
 
 // (unsigned immediate)
 // Indexed for 8-bit registers. offset is in range [0,63].
-def am_indexed8_6b : ComplexPattern<i64, 2, "SelectAddrModeIndexedUImm<1,63>", []>;
-def am_indexed16_6b : ComplexPattern<i64, 2, "SelectAddrModeIndexedUImm<2,63>", []>;
-def am_indexed32_6b : ComplexPattern<i64, 2, "SelectAddrModeIndexedUImm<4,63>", []>;
-def am_indexed64_6b : ComplexPattern<i64, 2, "SelectAddrModeIndexedUImm<8,63>", []>;
+def am_indexed8_6b : ComplexPattern<iPTR, 2, "SelectAddrModeIndexedUImm<1,63>", []>;
+def am_indexed16_6b : ComplexPattern<iPTR, 2, "SelectAddrModeIndexedUImm<2,63>", []>;
+def am_indexed32_6b : ComplexPattern<iPTR, 2, "SelectAddrModeIndexedUImm<4,63>", []>;
+def am_indexed64_6b : ComplexPattern<iPTR, 2, "SelectAddrModeIndexedUImm<8,63>", []>;
 
 def gi_am_indexed8 :
     GIComplexOperandMatcher<s64, "selectAddrModeIndexed<8>">,
@@ -3358,11 +3358,11 @@ class PrefetchLiteral<bits<2> opc, bit V, string asm, list<dag> pat>
 // Load/store register offset
 //---
 
-def ro_Xindexed8 : ComplexPattern<i64, 4, "SelectAddrModeXRO<8>", []>;
-def ro_Xindexed16 : ComplexPattern<i64, 4, "SelectAddrModeXRO<16>", []>;
-def ro_Xindexed32 : ComplexPattern<i64, 4, "SelectAddrModeXRO<32>", []>;
-def ro_Xindexed64 : ComplexPattern<i64, 4, "SelectAddrModeXRO<64>", []>;
-def ro_Xindexed128 : ComplexPattern<i64, 4, "SelectAddrModeXRO<128>", []>;
+def ro_Xindexed8 : ComplexPattern<iPTR, 4, "SelectAddrModeXRO<8>", []>;
+def ro_Xindexed16 : ComplexPattern<iPTR, 4, "SelectAddrModeXRO<16>", []>;
+def ro_Xindexed32 : ComplexPattern<iPTR, 4, "SelectAddrModeXRO<32>", []>;
+def ro_Xindexed64 : ComplexPattern<iPTR, 4, "SelectAddrModeXRO<64>", []>;
+def ro_Xindexed128 : ComplexPattern<iPTR, 4, "SelectAddrModeXRO<128>", []>;
 
 def gi_ro_Xindexed8 :
     GIComplexOperandMatcher<s64, "selectAddrModeXRO<8>">,
@@ -3380,11 +3380,11 @@ def gi_ro_Xindexed128 :
     GIComplexOperandMatcher<s64, "selectAddrModeXRO<128>">,
     GIComplexPatternEquiv<ro_Xindexed128>;
 
-def ro_Windexed8 : ComplexPattern<i64, 4, "SelectAddrModeWRO<8>", []>;
-def ro_Windexed16 : ComplexPattern<i64, 4, "SelectAddrModeWRO<16>", []>;
-def ro_Windexed32 : ComplexPattern<i64, 4, "SelectAddrModeWRO<32>", []>;
-def ro_Windexed64 : ComplexPattern<i64, 4, "SelectAddrModeWRO<64>", []>;
-def ro_Windexed128 : ComplexPattern<i64, 4, "SelectAddrModeWRO<128>", []>;
+def ro_Windexed8 : ComplexPattern<iPTR, 4, "SelectAddrModeWRO<8>", []>;
+def ro_Windexed16 : ComplexPattern<iPTR, 4, "SelectAddrModeWRO<16>", []>;
+def ro_Windexed32 : ComplexPattern<iPTR, 4, "SelectAddrModeWRO<32>", []>;
+def ro_Windexed64 : ComplexPattern<iPTR, 4, "SelectAddrModeWRO<64>", []>;
+def ro_Windexed128 : ComplexPattern<iPTR, 4, "SelectAddrModeWRO<128>", []>;
 
 def gi_ro_Windexed8 :
     GIComplexOperandMatcher<s64, "selectAddrModeWRO<8>">,
@@ -3880,11 +3880,11 @@ multiclass PrefetchRO<bits<2> sz, bit V, bits<2> opc, string asm> {
 // Load/store unscaled immediate
 //---
 
-def am_unscaled8 :  ComplexPattern<i64, 2, "SelectAddrModeUnscaled8", []>;
-def am_unscaled16 : ComplexPattern<i64, 2, "SelectAddrModeUnscaled16", []>;
-def am_unscaled32 : ComplexPattern<i64, 2, "SelectAddrModeUnscaled32", []>;
-def am_unscaled64 : ComplexPattern<i64, 2, "SelectAddrModeUnscaled64", []>;
-def am_unscaled128 :ComplexPattern<i64, 2, "SelectAddrModeUnscaled128", []>;
+def am_unscaled8 :  ComplexPattern<iPTR, 2, "SelectAddrModeUnscaled8", []>;
+def am_unscaled16 : ComplexPattern<iPTR, 2, "SelectAddrModeUnscaled16", []>;
+def am_unscaled32 : ComplexPattern<iPTR, 2, "SelectAddrModeUnscaled32", []>;
+def am_unscaled64 : ComplexPattern<iPTR, 2, "SelectAddrModeUnscaled64", []>;
+def am_unscaled128 :ComplexPattern<iPTR, 2, "SelectAddrModeUnscaled128", []>;
 
 def gi_am_unscaled8 :
     GIComplexOperandMatcher<s64, "selectAddrModeUnscaled8">,
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index f8f8ee3f1e6c..5fc5e4e5eb35 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -7055,6 +7055,8 @@ bool AArch64InstrInfo::isFunctionSafeToOutlineFrom(
 
 bool AArch64InstrInfo::isMBBSafeToOutlineFrom(MachineBasicBlock &MBB,
                                               unsigned &Flags) const {
+  if (!TargetInstrInfo::isMBBSafeToOutlineFrom(MBB, Flags))
+    return false;
   // Check if LR is available through all of the MBB. If it's not, then set
   // a flag.
   assert(MBB.getParent()->getRegInfo().tracksLiveness() &&
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index decee117d2d5..ebccc07edc7a 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -4174,6 +4174,21 @@ defm CMLT   : SIMDCmpTwoVector<0, 0b01010, "cmlt", AArch64cmltz>;
 defm CNT    : SIMDTwoVectorB<0, 0b00, 0b00101, "cnt", ctpop>;
 defm FABS   : SIMDTwoVectorFP<0, 1, 0b01111, "fabs", fabs>;
 
+def : Pat<(v8i8 (AArch64vashr (v8i8 V64:$Rn), (i32 7))),
+          (CMLTv8i8rz V64:$Rn)>;
+def : Pat<(v4i16 (AArch64vashr (v4i16 V64:$Rn), (i32 15))),
+          (CMLTv4i16rz V64:$Rn)>;
+def : Pat<(v2i32 (AArch64vashr (v2i32 V64:$Rn), (i32 31))),
+          (CMLTv2i32rz V64:$Rn)>;
+def : Pat<(v16i8 (AArch64vashr (v16i8 V128:$Rn), (i32 7))),
+          (CMLTv16i8rz V128:$Rn)>;
+def : Pat<(v8i16 (AArch64vashr (v8i16 V128:$Rn), (i32 15))),
+          (CMLTv8i16rz V128:$Rn)>;
+def : Pat<(v4i32 (AArch64vashr (v4i32 V128:$Rn), (i32 31))),
+          (CMLTv4i32rz V128:$Rn)>;
+def : Pat<(v2i64 (AArch64vashr (v2i64 V128:$Rn), (i32 63))),
+          (CMLTv2i64rz V128:$Rn)>;
+
 defm FCMEQ  : SIMDFPCmpTwoVector<0, 1, 0b01101, "fcmeq", AArch64fcmeqz>;
 defm FCMGE  : SIMDFPCmpTwoVector<1, 1, 0b01100, "fcmge", AArch64fcmgez>;
 defm FCMGT  : SIMDFPCmpTwoVector<0, 1, 0b01100, "fcmgt", AArch64fcmgtz>;
@@ -4363,6 +4378,32 @@ def : Pat<(v4i16 (trunc (smax (smin (v4i32 V128:$Vn), (v4i32 VImm7FFF)),
                               (v4i32 VImm8000)))),
           (SQXTNv4i16 V128:$Vn)>;
 
+// concat_vectors(Vd, trunc(smin(smax Vm, -128), 127) ~> SQXTN2(Vd, Vn)
+// with reversed min/max
+def : Pat<(v16i8 (concat_vectors
+                 (v8i8 V64:$Vd),
+                 (v8i8 (trunc (smin (smax (v8i16 V128:$Vn), (v8i16 VImm80)),
+                                          (v8i16 VImm7F)))))),
+          (SQXTNv16i8 (INSERT_SUBREG (IMPLICIT_DEF), V64:$Vd, dsub), V128:$Vn)>;
+def : Pat<(v16i8 (concat_vectors
+                 (v8i8 V64:$Vd),
+                 (v8i8 (trunc (smax (smin (v8i16 V128:$Vn), (v8i16 VImm7F)),
+                                          (v8i16 VImm80)))))),
+          (SQXTNv16i8 (INSERT_SUBREG (IMPLICIT_DEF), V64:$Vd, dsub), V128:$Vn)>;
+
+// concat_vectors(Vd, trunc(smin(smax Vm, -32768), 32767) ~> SQXTN2(Vd, Vn)
+// with reversed min/max
+def : Pat<(v8i16 (concat_vectors
+                 (v4i16 V64:$Vd),
+                 (v4i16 (trunc (smin (smax (v4i32 V128:$Vn), (v4i32 VImm8000)),
+                                           (v4i32 VImm7FFF)))))),
+          (SQXTNv8i16 (INSERT_SUBREG (IMPLICIT_DEF), V64:$Vd, dsub), V128:$Vn)>;
+def : Pat<(v8i16 (concat_vectors
+                 (v4i16 V64:$Vd),
+                 (v4i16 (trunc (smax (smin (v4i32 V128:$Vn), (v4i32 VImm7FFF)),
+                                           (v4i32 VImm8000)))))),
+          (SQXTNv8i16 (INSERT_SUBREG (IMPLICIT_DEF), V64:$Vd, dsub), V128:$Vn)>;
+
 //===----------------------------------------------------------------------===//
 // Advanced SIMD three vector instructions.
 //===----------------------------------------------------------------------===//
@@ -4825,6 +4866,9 @@ defm UQXTN  : SIMDTwoScalarMixedBHS<1, 0b10100, "uqxtn", int_aarch64_neon_scalar
 defm USQADD : SIMDTwoScalarBHSDTied< 1, 0b00011, "usqadd",
                                     int_aarch64_neon_usqadd>;
 
+def : Pat<(v1i64 (AArch64vashr (v1i64 V64:$Rn), (i32 63))),
+          (CMLTv1i64rz V64:$Rn)>;
+
 def : Pat<(v1i64 (int_aarch64_neon_fcvtas (v1f64 FPR64:$Rn))),
           (FCVTASv1i64 FPR64:$Rn)>;
 def : Pat<(v1i64 (int_aarch64_neon_fcvtau (v1f64 FPR64:$Rn))),
@@ -5288,6 +5332,29 @@ defm UZP2 : SIMDZipVector<0b101, "uzp2", AArch64uzp2>;
 defm ZIP1 : SIMDZipVector<0b011, "zip1", AArch64zip1>;
 defm ZIP2 : SIMDZipVector<0b111, "zip2", AArch64zip2>;
 
+def : Pat<(v16i8 (concat_vectors (v8i8 (trunc (v8i16 V128:$Vn))),
+                                 (v8i8 (trunc (v8i16 V128:$Vm))))),
+          (UZP1v16i8 V128:$Vn, V128:$Vm)>;
+def : Pat<(v8i16 (concat_vectors (v4i16 (trunc (v4i32 V128:$Vn))),
+                                 (v4i16 (trunc (v4i32 V128:$Vm))))),
+          (UZP1v8i16 V128:$Vn, V128:$Vm)>;
+def : Pat<(v4i32 (concat_vectors (v2i32 (trunc (v2i64 V128:$Vn))),
+                                 (v2i32 (trunc (v2i64 V128:$Vm))))),
+          (UZP1v4i32 V128:$Vn, V128:$Vm)>;
+
+def : Pat<(v16i8 (concat_vectors
+                 (v8i8 (trunc (AArch64vlshr (v8i16 V128:$Vn), (i32 8)))),
+                 (v8i8 (trunc (AArch64vlshr (v8i16 V128:$Vm), (i32 8)))))),
+          (UZP2v16i8 V128:$Vn, V128:$Vm)>;
+def : Pat<(v8i16 (concat_vectors
+                 (v4i16 (trunc (AArch64vlshr (v4i32 V128:$Vn), (i32 16)))),
+                 (v4i16 (trunc (AArch64vlshr (v4i32 V128:$Vm), (i32 16)))))),
+          (UZP2v8i16 V128:$Vn, V128:$Vm)>;
+def : Pat<(v4i32 (concat_vectors
+                 (v2i32 (trunc (AArch64vlshr (v2i64 V128:$Vn), (i32 32)))),
+                 (v2i32 (trunc (AArch64vlshr (v2i64 V128:$Vm), (i32 32)))))),
+          (UZP2v4i32 V128:$Vn, V128:$Vm)>;
+
 //----------------------------------------------------------------------------
 // AdvSIMD TBL/TBX instructions
 //----------------------------------------------------------------------------
@@ -6536,6 +6603,34 @@ defm USHR    : SIMDVectorRShiftBHSD<1, 0b00000, "ushr", AArch64vlshr>;
 defm USRA    : SIMDVectorRShiftBHSDTied<1, 0b00010, "usra",
                 TriOpFrag<(add node:$LHS, (AArch64vlshr node:$MHS, node:$RHS))> >;
 
+// RADDHN patterns for when RSHRN shifts by half the size of the vector element
+def : Pat<(v8i8 (int_aarch64_neon_rshrn (v8i16 V128:$Vn), (i32 8))),
+          (RADDHNv8i16_v8i8 V128:$Vn, (v8i16 (MOVIv2d_ns (i32 0))))>;
+def : Pat<(v4i16 (int_aarch64_neon_rshrn (v4i32 V128:$Vn), (i32 16))),
+          (RADDHNv4i32_v4i16 V128:$Vn, (v4i32 (MOVIv2d_ns (i32 0))))>;
+def : Pat<(v2i32 (int_aarch64_neon_rshrn (v2i64 V128:$Vn), (i32 32))),
+          (RADDHNv2i64_v2i32 V128:$Vn, (v2i64 (MOVIv2d_ns (i32 0))))>;
+
+// RADDHN2 patterns for when RSHRN shifts by half the size of the vector element
+def : Pat<(v16i8 (concat_vectors
+                 (v8i8 V64:$Vd),
+                 (v8i8 (int_aarch64_neon_rshrn (v8i16 V128:$Vn), (i32 8))))),
+          (RADDHNv8i16_v16i8
+                 (INSERT_SUBREG (IMPLICIT_DEF), V64:$Vd, dsub), V128:$Vn,
+                 (v8i16 (MOVIv2d_ns (i32 0))))>;
+def : Pat<(v8i16 (concat_vectors
+                 (v4i16 V64:$Vd),
+                 (v4i16 (int_aarch64_neon_rshrn (v4i32 V128:$Vn), (i32 16))))),
+          (RADDHNv4i32_v8i16
+                 (INSERT_SUBREG (IMPLICIT_DEF), V64:$Vd, dsub), V128:$Vn,
+                 (v4i32 (MOVIv2d_ns (i32 0))))>;
+def : Pat<(v4i32 (concat_vectors
+                 (v2i32 V64:$Vd),
+                 (v2i32 (int_aarch64_neon_rshrn (v2i64 V128:$Vn), (i32 32))))),
+          (RADDHNv2i64_v4i32
+                 (INSERT_SUBREG (IMPLICIT_DEF), V64:$Vd, dsub), V128:$Vn,
+                 (v2i64 (MOVIv2d_ns (i32 0))))>;
+
 // SHRN patterns for when a logical right shift was used instead of arithmetic
 // (the immediate guarantees no sign bits actually end up in the result so it
 // doesn't matter).
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 25d53f4ab065..eb55a472a69a 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -136,15 +136,15 @@ def AArch64stnt1_scatter : SDNode<"AArch64ISD::SSTNT1_PRED", SDT_AArch64_SCATTER
 //
 
 // SVE CNT/INC/RDVL
-def sve_rdvl_imm : ComplexPattern<i32, 1, "SelectRDVLImm<-32, 31, 16>">;
-def sve_cnth_imm : ComplexPattern<i32, 1, "SelectRDVLImm<1, 16, 8>">;
-def sve_cntw_imm : ComplexPattern<i32, 1, "SelectRDVLImm<1, 16, 4>">;
-def sve_cntd_imm : ComplexPattern<i32, 1, "SelectRDVLImm<1, 16, 2>">;
+def sve_rdvl_imm : ComplexPattern<i64, 1, "SelectRDVLImm<-32, 31, 16>">;
+def sve_cnth_imm : ComplexPattern<i64, 1, "SelectRDVLImm<1, 16, 8>">;
+def sve_cntw_imm : ComplexPattern<i64, 1, "SelectRDVLImm<1, 16, 4>">;
+def sve_cntd_imm : ComplexPattern<i64, 1, "SelectRDVLImm<1, 16, 2>">;
 
 // SVE DEC
-def sve_cnth_imm_neg : ComplexPattern<i32, 1, "SelectRDVLImm<1, 16, -8>">;
-def sve_cntw_imm_neg : ComplexPattern<i32, 1, "SelectRDVLImm<1, 16, -4>">;
-def sve_cntd_imm_neg : ComplexPattern<i32, 1, "SelectRDVLImm<1, 16, -2>">;
+def sve_cnth_imm_neg : ComplexPattern<i64, 1, "SelectRDVLImm<1, 16, -8>">;
+def sve_cntw_imm_neg : ComplexPattern<i64, 1, "SelectRDVLImm<1, 16, -4>">;
+def sve_cntd_imm_neg : ComplexPattern<i64, 1, "SelectRDVLImm<1, 16, -2>">;
 
 def SDT_AArch64Reduce : SDTypeProfile<1, 2, [SDTCisVec<1>, SDTCisVec<2>]>;
 def AArch64faddv_p   : SDNode<"AArch64ISD::FADDV_PRED",   SDT_AArch64Reduce>;
@@ -231,6 +231,8 @@ def AArch64fsqrt_mt  : SDNode<"AArch64ISD::FSQRT_MERGE_PASSTHRU", SDT_AArch64Ari
 def AArch64frecpx_mt : SDNode<"AArch64ISD::FRECPX_MERGE_PASSTHRU", SDT_AArch64Arith>;
 def AArch64rbit_mt   : SDNode<"AArch64ISD::BITREVERSE_MERGE_PASSTHRU", SDT_AArch64Arith>;
 def AArch64revb_mt   : SDNode<"AArch64ISD::BSWAP_MERGE_PASSTHRU", SDT_AArch64Arith>;
+def AArch64revh_mt   : SDNode<"AArch64ISD::REVH_MERGE_PASSTHRU", SDT_AArch64Arith>;
+def AArch64revw_mt   : SDNode<"AArch64ISD::REVW_MERGE_PASSTHRU", SDT_AArch64Arith>;
 
 // These are like the above but we don't yet have need for ISD nodes. They allow
 // a single pattern to match intrinsic and ISD operand layouts.
@@ -275,6 +277,11 @@ def AArch64mul_p_oneuse : PatFrag<(ops node:$pred, node:$src1, node:$src2),
   return N->hasOneUse();
 }]>;
 
+def AArch64fneg_mt_nsz : PatFrag<(ops node:$pred, node:$op, node:$pt),
+                                  (AArch64fneg_mt node:$pred, node:$op, node:$pt), [{
+  return N->getFlags().hasNoSignedZeros();
+}]>;
+
 def SDT_AArch64Arith_Unpred : SDTypeProfile<1, 2, [
   SDTCisVec<0>, SDTCisVec<1>, SDTCisVec<2>,
   SDTCisSameAs<0,1>, SDTCisSameAs<1,2>
@@ -536,7 +543,8 @@ let Predicates = [HasSVEorStreamingSVE] in {
               (!cast<Instruction>("FNMLA_ZPZZZ_UNDEF_"#Suffix) $P, ZPR:$Za, ZPR:$Zn, ZPR:$Zm)>;
 
     // Zd = -(Za + Zn * Zm)
-    def : Pat<(AArch64fneg_mt PredTy:$P, (AArch64fma_p PredTy:$P, Ty:$Zn, Ty:$Zm, Ty:$Za), (Ty (undef))),
+    // (with nsz neg.)
+    def : Pat<(AArch64fneg_mt_nsz PredTy:$P, (AArch64fma_p PredTy:$P, Ty:$Zn, Ty:$Zm, Ty:$Za), (Ty (undef))),
               (!cast<Instruction>("FNMLA_ZPZZZ_UNDEF_"#Suffix) $P, ZPR:$Za, ZPR:$Zn, ZPR:$Zm)>;
 
     // Zda = Zda + Zn * Zm
@@ -624,13 +632,13 @@ let Predicates = [HasSVEorStreamingSVE] in {
   def : Pat<(nxv8bf16 (AArch64dup (bf16 fpimm0))), (DUP_ZI_H 0, 0)>;
 
   // Duplicate Int immediate into all vector elements
-  def : Pat<(nxv16i8 (AArch64dup (i32 (SVE8BitLslImm i32:$a, i32:$b)))),
+  def : Pat<(nxv16i8 (AArch64dup (i32 (SVE8BitLslImm32 i32:$a, i32:$b)))),
             (DUP_ZI_B $a, $b)>;
-  def : Pat<(nxv8i16 (AArch64dup (i32 (SVE8BitLslImm i32:$a, i32:$b)))),
+  def : Pat<(nxv8i16 (AArch64dup (i32 (SVE8BitLslImm32 i32:$a, i32:$b)))),
             (DUP_ZI_H $a, $b)>;
-  def : Pat<(nxv4i32 (AArch64dup (i32 (SVE8BitLslImm i32:$a, i32:$b)))),
+  def : Pat<(nxv4i32 (AArch64dup (i32 (SVE8BitLslImm32 i32:$a, i32:$b)))),
             (DUP_ZI_S $a, $b)>;
-  def : Pat<(nxv2i64 (AArch64dup (i64 (SVE8BitLslImm i32:$a, i32:$b)))),
+  def : Pat<(nxv2i64 (AArch64dup (i64 (SVE8BitLslImm64 i32:$a, i32:$b)))),
             (DUP_ZI_D $a, $b)>;
 
   // Duplicate immediate FP into all vector elements.
@@ -674,8 +682,8 @@ let Predicates = [HasSVEorStreamingSVE] in {
 
   defm RBIT_ZPmZ : sve_int_perm_rev_rbit<"rbit", AArch64rbit_mt>;
   defm REVB_ZPmZ : sve_int_perm_rev_revb<"revb", AArch64revb_mt>;
-  defm REVH_ZPmZ : sve_int_perm_rev_revh<"revh", int_aarch64_sve_revh>;
-  defm REVW_ZPmZ : sve_int_perm_rev_revw<"revw", int_aarch64_sve_revw>;
+  defm REVH_ZPmZ : sve_int_perm_rev_revh<"revh", AArch64revh_mt>;
+  defm REVW_ZPmZ : sve_int_perm_rev_revw<"revw", AArch64revw_mt>;
 
   defm REV_PP : sve_int_perm_reverse_p<"rev", vector_reverse>;
   defm REV_ZZ : sve_int_perm_reverse_z<"rev", vector_reverse>;
@@ -2686,13 +2694,13 @@ let Predicates = [HasSVEorStreamingSVE] in {
 
   // Splice with lane bigger or equal to 0
   def : Pat<(nxv16i8 (vector_splice (nxv16i8 ZPR:$Z1), (nxv16i8 ZPR:$Z2), (i64 (sve_ext_imm_0_255 i32:$index)))),
-            (EXT_ZZI  ZPR:$Z1, ZPR:$Z2, sve_ext_imm_0_255:$index)>;
+            (EXT_ZZI  ZPR:$Z1, ZPR:$Z2, imm0_255:$index)>;
   def : Pat<(nxv8i16 (vector_splice (nxv8i16 ZPR:$Z1), (nxv8i16 ZPR:$Z2), (i64 (sve_ext_imm_0_127 i32:$index)))),
-            (EXT_ZZI  ZPR:$Z1, ZPR:$Z2, sve_ext_imm_0_127:$index)>;
+            (EXT_ZZI  ZPR:$Z1, ZPR:$Z2, imm0_255:$index)>;
   def : Pat<(nxv4i32 (vector_splice (nxv4i32 ZPR:$Z1), (nxv4i32 ZPR:$Z2), (i64 (sve_ext_imm_0_63 i32:$index)))),
-            (EXT_ZZI  ZPR:$Z1, ZPR:$Z2, sve_ext_imm_0_63:$index)>;
+            (EXT_ZZI  ZPR:$Z1, ZPR:$Z2, imm0_255:$index)>;
   def : Pat<(nxv2i64 (vector_splice (nxv2i64 ZPR:$Z1), (nxv2i64 ZPR:$Z2), (i64 (sve_ext_imm_0_31 i32:$index)))),
-            (EXT_ZZI  ZPR:$Z1, ZPR:$Z2, sve_ext_imm_0_31:$index)>;
+            (EXT_ZZI  ZPR:$Z1, ZPR:$Z2, imm0_255:$index)>;
 
 } // End HasSVEorStreamingSVE
 
diff --git a/llvm/lib/Target/AArch64/AArch64StackTagging.cpp b/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
index 5cec4cb66339..566c7a16db23 100644
--- a/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
+++ b/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
@@ -488,7 +488,7 @@ Instruction *AArch64StackTagging::insertBaseTaggedPointer(
 
 void AArch64StackTagging::alignAndPadAlloca(AllocaInfo &Info) {
   const Align NewAlignment =
-      max(MaybeAlign(Info.AI->getAlignment()), kTagGranuleSize);
+      max(MaybeAlign(Info.AI->getAlign()), kTagGranuleSize);
   Info.AI->setAlignment(NewAlignment);
 
   uint64_t Size = Info.AI->getAllocationSizeInBits(*DL).getValue() / 8;
@@ -537,15 +537,14 @@ bool AArch64StackTagging::runOnFunction(Function &Fn) {
   SmallVector<Instruction *, 4> UnrecognizedLifetimes;
 
   for (auto &BB : *F) {
-    for (BasicBlock::iterator IT = BB.begin(); IT != BB.end(); ++IT) {
-      Instruction *I = &*IT;
-      if (auto *AI = dyn_cast<AllocaInst>(I)) {
+    for (Instruction &I : BB) {
+      if (auto *AI = dyn_cast<AllocaInst>(&I)) {
         Allocas[AI].AI = AI;
         Allocas[AI].OldAI = AI;
         continue;
       }
 
-      if (auto *DVI = dyn_cast<DbgVariableIntrinsic>(I)) {
+      if (auto *DVI = dyn_cast<DbgVariableIntrinsic>(&I)) {
         for (Value *V : DVI->location_ops())
           if (auto *AI = dyn_cast_or_null<AllocaInst>(V))
             if (Allocas[AI].DbgVariableIntrinsics.empty() ||
@@ -554,12 +553,12 @@ bool AArch64StackTagging::runOnFunction(Function &Fn) {
         continue;
       }
 
-      auto *II = dyn_cast<IntrinsicInst>(I);
+      auto *II = dyn_cast<IntrinsicInst>(&I);
       if (II && (II->getIntrinsicID() == Intrinsic::lifetime_start ||
                  II->getIntrinsicID() == Intrinsic::lifetime_end)) {
         AllocaInst *AI = findAllocaForValue(II->getArgOperand(1));
         if (!AI) {
-          UnrecognizedLifetimes.push_back(I);
+          UnrecognizedLifetimes.push_back(&I);
           continue;
         }
         if (II->getIntrinsicID() == Intrinsic::lifetime_start)
@@ -568,8 +567,8 @@ bool AArch64StackTagging::runOnFunction(Function &Fn) {
           Allocas[AI].LifetimeEnd.push_back(II);
       }
 
-      if (isa<ReturnInst>(I) || isa<ResumeInst>(I) || isa<CleanupReturnInst>(I))
-        RetVec.push_back(I);
+      if (isa<ReturnInst, ResumeInst, CleanupReturnInst>(&I))
+        RetVec.push_back(&I);
     }
   }
 
diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
index d782d6352cbe..f7d3dd0bc222 100644
--- a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
+++ b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
@@ -346,9 +346,7 @@ bool AArch64Subtarget::supportsAddressTopByteIgnored() const {
     return false;
 
   if (TargetTriple.isiOS()) {
-    unsigned Major, Minor, Micro;
-    TargetTriple.getiOSVersion(Major, Minor, Micro);
-    return Major >= 8;
+    return TargetTriple.getiOSVersion() >= VersionTuple(8);
   }
 
   return false;
diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.h b/llvm/lib/Target/AArch64/AArch64Subtarget.h
index 19db774ccd7b..b3cd5ebd5f65 100644
--- a/llvm/lib/Target/AArch64/AArch64Subtarget.h
+++ b/llvm/lib/Target/AArch64/AArch64Subtarget.h
@@ -116,6 +116,8 @@ protected:
   bool HasFP16FML = false;
   bool HasSPE = false;
 
+  bool FixCortexA53_835769 = false;
+
   // ARMv8.1 extensions
   bool HasVH = false;
   bool HasPAN = false;
@@ -571,6 +573,8 @@ public:
   bool hasEL2VMSA() const { return HasEL2VMSA; }
   bool hasEL3() const { return HasEL3; }
 
+  bool fixCortexA53_835769() const { return FixCortexA53_835769; }
+
   bool addrSinkUsingGEPs() const override {
     // Keeping GEPs inbounds is important for exploiting AArch64
     // addressing-modes in ILP32 mode.
@@ -632,8 +636,7 @@ public:
     // extended frames should be flagged as present.
     const Triple &TT = getTargetTriple();
 
-    unsigned Major, Minor, Micro;
-    TT.getOSVersion(Major, Minor, Micro);
+    unsigned Major = TT.getOSVersion().getMajor();
     switch(TT.getOS()) {
     default:
       return false;
diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
index ce26c62af61a..4af28fc070dd 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
@@ -117,11 +117,6 @@ static cl::opt<bool>
                   cl::init(true), cl::Hidden);
 
 static cl::opt<bool>
-EnableA53Fix835769("aarch64-fix-cortex-a53-835769", cl::Hidden,
-                cl::desc("Work around Cortex-A53 erratum 835769"),
-                cl::init(false));
-
-static cl::opt<bool>
     EnableGEPOpt("aarch64-enable-gep-opt", cl::Hidden,
                  cl::desc("Enable optimizations on complex GEPs"),
                  cl::init(false));
@@ -382,10 +377,9 @@ AArch64TargetMachine::getSubtargetImpl(const Function &F) const {
   unsigned MaxSVEVectorSize = 0;
   Attribute VScaleRangeAttr = F.getFnAttribute(Attribute::VScaleRange);
   if (VScaleRangeAttr.isValid()) {
-    std::tie(MinSVEVectorSize, MaxSVEVectorSize) =
-        VScaleRangeAttr.getVScaleRangeArgs();
-    MinSVEVectorSize *= 128;
-    MaxSVEVectorSize *= 128;
+    Optional<unsigned> VScaleMax = VScaleRangeAttr.getVScaleRangeMax();
+    MinSVEVectorSize = VScaleRangeAttr.getVScaleRangeMin() * 128;
+    MaxSVEVectorSize = VScaleMax ? VScaleMax.getValue() * 128 : 0;
   } else {
     MinSVEVectorSize = SVEVectorBitsMinOpt;
     MaxSVEVectorSize = SVEVectorBitsMaxOpt;
@@ -765,8 +759,7 @@ void AArch64PassConfig::addPreEmitPass() {
   if (TM->getOptLevel() >= CodeGenOpt::Aggressive && EnableLoadStoreOpt)
     addPass(createAArch64LoadStoreOptimizationPass());
 
-  if (EnableA53Fix835769)
-    addPass(createAArch64A53Fix835769());
+  addPass(createAArch64A53Fix835769());
 
   if (EnableBranchTargets)
     addPass(createAArch64BranchTargetsPass());
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 34015d2dbd49..d21854e38f5a 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -30,6 +30,12 @@ using namespace llvm::PatternMatch;
 static cl::opt<bool> EnableFalkorHWPFUnrollFix("enable-falkor-hwpf-unroll-fix",
                                                cl::init(true), cl::Hidden);
 
+static cl::opt<unsigned> SVEGatherOverhead("sve-gather-overhead", cl::init(10),
+                                           cl::Hidden);
+
+static cl::opt<unsigned> SVEScatterOverhead("sve-scatter-overhead",
+                                            cl::init(10), cl::Hidden);
+
 bool AArch64TTIImpl::areInlineCompatible(const Function *Caller,
                                          const Function *Callee) const {
   const TargetMachine &TM = getTLI()->getTargetMachine();
@@ -725,6 +731,22 @@ static Optional<Instruction *> instCombineSVEVectorFMLA(InstCombiner &IC,
   return IC.replaceInstUsesWith(II, FMLA);
 }
 
+static bool isAllActivePredicate(Value *Pred) {
+  // Look through convert.from.svbool(convert.to.svbool(...) chain.
+  Value *UncastedPred;
+  if (match(Pred, m_Intrinsic<Intrinsic::aarch64_sve_convert_from_svbool>(
+                      m_Intrinsic<Intrinsic::aarch64_sve_convert_to_svbool>(
+                          m_Value(UncastedPred)))))
+    // If the predicate has the same or less lanes than the uncasted
+    // predicate then we know the casting has no effect.
+    if (cast<ScalableVectorType>(Pred->getType())->getMinNumElements() <=
+        cast<ScalableVectorType>(UncastedPred->getType())->getMinNumElements())
+      Pred = UncastedPred;
+
+  return match(Pred, m_Intrinsic<Intrinsic::aarch64_sve_ptrue>(
+                         m_ConstantInt<AArch64SVEPredPattern::all>()));
+}
+
 static Optional<Instruction *>
 instCombineSVELD1(InstCombiner &IC, IntrinsicInst &II, const DataLayout &DL) {
   IRBuilder<> Builder(II.getContext());
@@ -735,8 +757,7 @@ instCombineSVELD1(InstCombiner &IC, IntrinsicInst &II, const DataLayout &DL) {
   Type *VecTy = II.getType();
   Value *VecPtr = Builder.CreateBitCast(PtrOp, VecTy->getPointerTo());
 
-  if (match(Pred, m_Intrinsic<Intrinsic::aarch64_sve_ptrue>(
-                      m_ConstantInt<AArch64SVEPredPattern::all>()))) {
+  if (isAllActivePredicate(Pred)) {
     LoadInst *Load = Builder.CreateLoad(VecTy, VecPtr);
     return IC.replaceInstUsesWith(II, Load);
   }
@@ -758,8 +779,7 @@ instCombineSVEST1(InstCombiner &IC, IntrinsicInst &II, const DataLayout &DL) {
   Value *VecPtr =
       Builder.CreateBitCast(PtrOp, VecOp->getType()->getPointerTo());
 
-  if (match(Pred, m_Intrinsic<Intrinsic::aarch64_sve_ptrue>(
-                      m_ConstantInt<AArch64SVEPredPattern::all>()))) {
+  if (isAllActivePredicate(Pred)) {
     Builder.CreateStore(VecOp, VecPtr);
     return IC.eraseInstFromFunction(II);
   }
@@ -1008,6 +1028,40 @@ static Optional<Instruction *> instCombineST1ScatterIndex(InstCombiner &IC,
   return None;
 }
 
+static Optional<Instruction *> instCombineSVESDIV(InstCombiner &IC,
+                                                  IntrinsicInst &II) {
+  IRBuilder<> Builder(II.getContext());
+  Builder.SetInsertPoint(&II);
+  Type *Int32Ty = Builder.getInt32Ty();
+  Value *Pred = II.getOperand(0);
+  Value *Vec = II.getOperand(1);
+  Value *DivVec = II.getOperand(2);
+
+  Value *SplatValue = getSplatValue(DivVec);
+  ConstantInt *SplatConstantInt = dyn_cast_or_null<ConstantInt>(SplatValue);
+  if (!SplatConstantInt)
+    return None;
+  APInt Divisor = SplatConstantInt->getValue();
+
+  if (Divisor.isPowerOf2()) {
+    Constant *DivisorLog2 = ConstantInt::get(Int32Ty, Divisor.logBase2());
+    auto ASRD = Builder.CreateIntrinsic(
+        Intrinsic::aarch64_sve_asrd, {II.getType()}, {Pred, Vec, DivisorLog2});
+    return IC.replaceInstUsesWith(II, ASRD);
+  }
+  if (Divisor.isNegatedPowerOf2()) {
+    Divisor.negate();
+    Constant *DivisorLog2 = ConstantInt::get(Int32Ty, Divisor.logBase2());
+    auto ASRD = Builder.CreateIntrinsic(
+        Intrinsic::aarch64_sve_asrd, {II.getType()}, {Pred, Vec, DivisorLog2});
+    auto NEG = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_neg,
+                                       {ASRD->getType()}, {ASRD, Pred, ASRD});
+    return IC.replaceInstUsesWith(II, NEG);
+  }
+
+  return None;
+}
+
 Optional<Instruction *>
 AArch64TTIImpl::instCombineIntrinsic(InstCombiner &IC,
                                      IntrinsicInst &II) const {
@@ -1068,6 +1122,8 @@ AArch64TTIImpl::instCombineIntrinsic(InstCombiner &IC,
     return instCombineSVELD1(IC, II, DL);
   case Intrinsic::aarch64_sve_st1:
     return instCombineSVEST1(IC, II, DL);
+  case Intrinsic::aarch64_sve_sdiv:
+    return instCombineSVESDIV(IC, II);
   }
 
   return None;
@@ -1746,7 +1802,7 @@ InstructionCost
 AArch64TTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *Src,
                                       Align Alignment, unsigned AddressSpace,
                                       TTI::TargetCostKind CostKind) {
-  if (!isa<ScalableVectorType>(Src))
+  if (useNeonVector(Src))
     return BaseT::getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
                                         CostKind);
   auto LT = TLI->getTypeLegalizationCost(DL, Src);
@@ -1763,6 +1819,10 @@ AArch64TTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *Src,
   return LT.first * 2;
 }
 
+static unsigned getSVEGatherScatterOverhead(unsigned Opcode) {
+  return Opcode == Instruction::Load ? SVEGatherOverhead : SVEScatterOverhead;
+}
+
 InstructionCost AArch64TTIImpl::getGatherScatterOpCost(
     unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
     Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) {
@@ -1785,6 +1845,10 @@ InstructionCost AArch64TTIImpl::getGatherScatterOpCost(
   ElementCount LegalVF = LT.second.getVectorElementCount();
   InstructionCost MemOpCost =
       getMemoryOpCost(Opcode, VT->getElementType(), Alignment, 0, CostKind, I);
+  // Add on an overhead cost for using gathers/scatters.
+  // TODO: At the moment this is applied unilaterally for all CPUs, but at some
+  // point we may want a per-CPU overhead.
+  MemOpCost *= getSVEGatherScatterOverhead(Opcode);
   return LT.first * MemOpCost * getMaxNumElements(LegalVF);
 }
 
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index d1e8cd204b3a..c3e1735cd4cd 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -309,6 +309,8 @@ public:
 
   bool supportsScalableVectors() const { return ST->hasSVE(); }
 
+  bool enableScalableVectorization() const { return ST->hasSVE(); }
+
   bool isLegalToVectorizeReduction(const RecurrenceDescriptor &RdxDesc,
                                    ElementCount VF) const;
 
diff --git a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
index 6d3aea2721de..62038b10fccd 100644
--- a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
+++ b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
@@ -1031,12 +1031,7 @@ public:
     if (DarwinRefKind != MCSymbolRefExpr::VK_None)
       return false;
 
-    for (unsigned i = 0; i != AllowedModifiers.size(); ++i) {
-      if (ELFRefKind == AllowedModifiers[i])
-        return true;
-    }
-
-    return false;
+    return llvm::is_contained(AllowedModifiers, ELFRefKind);
   }
 
   bool isMovWSymbolG3() const {
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 1524aa5eb0ec..e8894e7933d6 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -785,6 +785,11 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
       .libcallFor({s128})
       .minScalar(0, MinFPScalar);
 
+  // TODO: Vector types.
+  getActionDefinitionsBuilder({G_FMAXIMUM, G_FMINIMUM})
+      .legalFor({MinFPScalar, s32, s64})
+      .minScalar(0, MinFPScalar);
+
   // TODO: Libcall support for s128.
   // TODO: s16 should be legal with full FP16 support.
   getActionDefinitionsBuilder({G_LROUND, G_LLROUND})
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
index a9b3792e0118..3dec980a819a 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
@@ -289,6 +289,44 @@ static void applyMutateAnyExtToZExt(MachineInstr &MI, MachineRegisterInfo &MRI,
   Observer.changedInstr(MI);
 }
 
+/// Match a 128b store of zero and split it into two 64 bit stores, for
+/// size/performance reasons.
+static bool matchSplitStoreZero128(MachineInstr &MI, MachineRegisterInfo &MRI) {
+  GStore &Store = cast<GStore>(MI);
+  if (!Store.isSimple())
+    return false;
+  LLT ValTy = MRI.getType(Store.getValueReg());
+  if (!ValTy.isVector() || ValTy.getSizeInBits() != 128)
+    return false;
+  if (ValTy.getSizeInBits() != Store.getMemSizeInBits())
+    return false; // Don't split truncating stores.
+  if (!MRI.hasOneNonDBGUse(Store.getValueReg()))
+    return false;
+  auto MaybeCst = isConstantOrConstantSplatVector(
+      *MRI.getVRegDef(Store.getValueReg()), MRI);
+  return MaybeCst && MaybeCst->isZero();
+}
+
+static void applySplitStoreZero128(MachineInstr &MI, MachineRegisterInfo &MRI,
+                                   MachineIRBuilder &B,
+                                   GISelChangeObserver &Observer) {
+  B.setInstrAndDebugLoc(MI);
+  GStore &Store = cast<GStore>(MI);
+  assert(MRI.getType(Store.getValueReg()).isVector() &&
+         "Expected a vector store value");
+  LLT NewTy = LLT::scalar(64);
+  Register PtrReg = Store.getPointerReg();
+  auto Zero = B.buildConstant(NewTy, 0);
+  auto HighPtr = B.buildPtrAdd(MRI.getType(PtrReg), PtrReg,
+                               B.buildConstant(LLT::scalar(64), 8));
+  auto &MF = *MI.getMF();
+  auto *LowMMO = MF.getMachineMemOperand(&Store.getMMO(), 0, NewTy);
+  auto *HighMMO = MF.getMachineMemOperand(&Store.getMMO(), 8, NewTy);
+  B.buildStore(Zero, PtrReg, *LowMMO);
+  B.buildStore(Zero, HighPtr, *HighMMO);
+  Store.eraseFromParent();
+}
+
 #define AARCH64POSTLEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_DEPS
 #include "AArch64GenPostLegalizeGICombiner.inc"
 #undef AARCH64POSTLEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_DEPS
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp
index 40ddf6a94f73..515a5c63a559 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp
@@ -430,6 +430,8 @@ static bool isPreISelGenericFloatingPointOpcode(unsigned Opc) {
   case TargetOpcode::G_INTRINSIC_ROUND:
   case TargetOpcode::G_FMAXNUM:
   case TargetOpcode::G_FMINNUM:
+  case TargetOpcode::G_FMAXIMUM:
+  case TargetOpcode::G_FMINIMUM:
     return true;
   }
   return false;
@@ -600,6 +602,8 @@ AArch64RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
   case TargetOpcode::G_FSUB:
   case TargetOpcode::G_FMUL:
   case TargetOpcode::G_FDIV:
+  case TargetOpcode::G_FMAXIMUM:
+  case TargetOpcode::G_FMINIMUM:
     return getSameKindOfOperandsMapping(MI);
   case TargetOpcode::G_FPEXT: {
     LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp
index 90688f1a3e83..c1186ae804d2 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp
@@ -239,8 +239,8 @@ void AArch64_MC::initLLVMToCVRegMapping(MCRegisterInfo *MRI) {
       {codeview::RegisterId::ARM64_Q31, AArch64::Q31},
 
   };
-  for (unsigned I = 0; I < array_lengthof(RegMap); ++I)
-    MRI->mapLLVMRegToCVReg(RegMap[I].Reg, static_cast<int>(RegMap[I].CVReg));
+  for (const auto &I : RegMap)
+    MRI->mapLLVMRegToCVReg(I.Reg, static_cast<int>(I.CVReg));
 }
 
 static MCRegisterInfo *createAArch64MCRegisterInfo(const Triple &Triple) {
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index 010ffa1502de..bb488cd7da32 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -197,34 +197,42 @@ def addsub_imm8_opt_lsl_i64 : imm8_opt_lsl<64, "uint64_t", SVEAddSubImmOperand64
 def SVEAddSubImm8Pat  : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i8>", []>;
 def SVEAddSubImm16Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i16>", []>;
 def SVEAddSubImm32Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i32>", []>;
-def SVEAddSubImm64Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i64>", []>;
+def SVEAddSubImm64Pat : ComplexPattern<i64, 2, "SelectSVEAddSubImm<MVT::i64>", []>;
 
-def SVELogicalImm8Pat  : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i8>", []>;
-def SVELogicalImm16Pat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i16>", []>;
-def SVELogicalImm32Pat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i32>", []>;
+def SVELogicalImm8Pat  : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i8>", []>;
+def SVELogicalImm16Pat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i16>", []>;
+def SVELogicalImm32Pat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i32>", []>;
 def SVELogicalImm64Pat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i64>", []>;
 
-def SVELogicalImm8NotPat  : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i8, true>", []>;
-def SVELogicalImm16NotPat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i16, true>", []>;
-def SVELogicalImm32NotPat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i32, true>", []>;
+def SVELogicalImm8NotPat  : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i8, true>", []>;
+def SVELogicalImm16NotPat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i16, true>", []>;
+def SVELogicalImm32NotPat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i32, true>", []>;
 def SVELogicalImm64NotPat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i64, true>", []>;
 
-def SVE8BitLslImm : ComplexPattern<i32, 2, "SelectSVE8BitLslImm", [imm]>;
+def SVE8BitLslImm32 : ComplexPattern<i32, 2, "SelectSVE8BitLslImm", [imm]>;
+def SVE8BitLslImm64 : ComplexPattern<i64, 2, "SelectSVE8BitLslImm", [imm]>;
+class SVE8BitLslImm<ValueType ty> {
+  ComplexPattern Pat = !cond(
+    !eq(ty, i32): SVE8BitLslImm32,
+    !eq(ty, i64): SVE8BitLslImm64);
+}
 
 def SVEArithUImm8Pat  : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i8>", []>;
 def SVEArithUImm16Pat  : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i16>", []>;
 def SVEArithUImm32Pat  : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i32>", []>;
-def SVEArithUImm64Pat  : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i64>", []>;
-def SVEArithSImmPat  : ComplexPattern<i32, 1, "SelectSVESignedArithImm", []>;
+def SVEArithUImm64Pat  : ComplexPattern<i64, 1, "SelectSVEArithImm<MVT::i64>", []>;
+
+def SVEArithSImmPat32 : ComplexPattern<i32, 1, "SelectSVESignedArithImm", []>;
+def SVEArithSImmPat64 : ComplexPattern<i64, 1, "SelectSVESignedArithImm", []>;
 
 def SVEShiftImmL8  : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 7>",  []>;
 def SVEShiftImmL16 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 15>", []>;
 def SVEShiftImmL32 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 31>", []>;
-def SVEShiftImmL64 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 63>", []>;
+def SVEShiftImmL64 : ComplexPattern<i64, 1, "SelectSVEShiftImm<0, 63>", []>;
 def SVEShiftImmR8  : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 8,  true>", []>;
 def SVEShiftImmR16 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 16, true>", []>;
 def SVEShiftImmR32 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 32, true>", []>;
-def SVEShiftImmR64 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 64, true>", []>;
+def SVEShiftImmR64 : ComplexPattern<i64, 1, "SelectSVEShiftImm<1, 64, true>", []>;
 
 def SVEAllActive : ComplexPattern<untyped, 0, "SelectAllActivePredicate", []>;
 
@@ -260,14 +268,14 @@ def sve_incdec_imm : Operand<i32>, TImmLeaf<i32, [{
 }
 
 // This allows i32 immediate extraction from i64 based arithmetic.
-def sve_cnt_mul_imm : ComplexPattern<i32, 1, "SelectCntImm<1, 16, 1, false>">;
-def sve_cnt_shl_imm : ComplexPattern<i32, 1, "SelectCntImm<1, 16, 1, true>">;
-
+def sve_cnt_mul_imm_i32 : ComplexPattern<i32, 1, "SelectCntImm<1, 16, 1, false>">;
+def sve_cnt_mul_imm_i64 : ComplexPattern<i64, 1, "SelectCntImm<1, 16, 1, false>">;
+def sve_cnt_shl_imm     : ComplexPattern<i64, 1, "SelectCntImm<1, 16, 1, true>">;
 
-def sve_ext_imm_0_31  : ComplexPattern<i32, 1, "SelectEXTImm<31, 8>">;
-def sve_ext_imm_0_63  : ComplexPattern<i32, 1, "SelectEXTImm<63, 4>">;
-def sve_ext_imm_0_127 : ComplexPattern<i32, 1, "SelectEXTImm<127, 2>">;
-def sve_ext_imm_0_255 : ComplexPattern<i32, 1, "SelectEXTImm<255, 1>">;
+def sve_ext_imm_0_31  : ComplexPattern<i64, 1, "SelectEXTImm<31, 8>">;
+def sve_ext_imm_0_63  : ComplexPattern<i64, 1, "SelectEXTImm<63, 4>">;
+def sve_ext_imm_0_127 : ComplexPattern<i64, 1, "SelectEXTImm<127, 2>">;
+def sve_ext_imm_0_255 : ComplexPattern<i64, 1, "SelectEXTImm<255, 1>">;
 
 def int_aarch64_sve_cntp_oneuse : PatFrag<(ops node:$pred, node:$src2),
                                           (int_aarch64_sve_cntp node:$pred, node:$src2), [{
@@ -435,8 +443,8 @@ class SVE_4_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
 : Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3, (vt4 ImmTy:$Op4))),
       (inst $Op1, $Op2, $Op3, ImmTy:$Op4)>;
 
-def SVEDup0 : ComplexPattern<i64, 0, "SelectDupZero", []>;
-def SVEDup0Undef : ComplexPattern<i64, 0, "SelectDupZeroOrUndef", []>;
+def SVEDup0 : ComplexPattern<vAny, 0, "SelectDupZero", []>;
+def SVEDup0Undef : ComplexPattern<vAny, 0, "SelectDupZeroOrUndef", []>;
 
 let AddedComplexity = 1 in {
 class SVE_3_Op_Pat_SelZero<ValueType vtd, SDPatternOperator op, ValueType vt1,
@@ -868,10 +876,10 @@ multiclass sve_int_count<bits<3> opc, string asm, SDPatternOperator op> {
   def : InstAlias<asm # "\t$Rd",
                   (!cast<Instruction>(NAME) GPR64:$Rd, 0b11111, 1), 2>;
 
-  def : Pat<(i64 (mul (op sve_pred_enum:$pattern), (sve_cnt_mul_imm i32:$imm))),
+  def : Pat<(i64 (mul (op sve_pred_enum:$pattern), (sve_cnt_mul_imm_i64 i32:$imm))),
             (!cast<Instruction>(NAME) sve_pred_enum:$pattern, sve_incdec_imm:$imm)>;
 
-  def : Pat<(i64 (shl (op sve_pred_enum:$pattern), (i64 (sve_cnt_shl_imm i32:$imm)))),
+  def : Pat<(i64 (shl (op sve_pred_enum:$pattern), (sve_cnt_shl_imm i32:$imm))),
             (!cast<Instruction>(NAME) sve_pred_enum:$pattern, sve_incdec_imm:$imm)>;
 
   def : Pat<(i64 (op sve_pred_enum:$pattern)),
@@ -951,10 +959,10 @@ multiclass sve_int_pred_pattern_a<bits<3> opc, string asm,
     def : Pat<(i64 (op GPR64:$Rdn, (opcnt sve_pred_enum:$pattern))),
               (!cast<Instruction>(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, 1)>;
 
-    def : Pat<(i64 (op GPR64:$Rdn, (mul (opcnt sve_pred_enum:$pattern), (sve_cnt_mul_imm i32:$imm)))),
+    def : Pat<(i64 (op GPR64:$Rdn, (mul (opcnt sve_pred_enum:$pattern), (sve_cnt_mul_imm_i64 i32:$imm)))),
               (!cast<Instruction>(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, $imm)>;
 
-    def : Pat<(i64 (op GPR64:$Rdn, (shl (opcnt sve_pred_enum:$pattern), (i64 (sve_cnt_shl_imm i32:$imm))))),
+    def : Pat<(i64 (op GPR64:$Rdn, (shl (opcnt sve_pred_enum:$pattern), (sve_cnt_shl_imm i32:$imm)))),
               (!cast<Instruction>(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, $imm)>;
 
     def : Pat<(i32 (op GPR32:$Rdn, (i32 (trunc (opcnt (sve_pred_enum:$pattern)))))),
@@ -962,12 +970,12 @@ multiclass sve_int_pred_pattern_a<bits<3> opc, string asm,
                                                GPR32:$Rdn, sub_32), sve_pred_enum:$pattern, 1),
                                     sub_32))>;
 
-    def : Pat<(i32 (op GPR32:$Rdn, (mul (i32 (trunc (opcnt (sve_pred_enum:$pattern)))), (sve_cnt_mul_imm i32:$imm)))),
+    def : Pat<(i32 (op GPR32:$Rdn, (mul (i32 (trunc (opcnt (sve_pred_enum:$pattern)))), (sve_cnt_mul_imm_i32 i32:$imm)))),
               (i32 (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (i64 (IMPLICIT_DEF)),
                                                GPR32:$Rdn, sub_32), sve_pred_enum:$pattern, $imm),
                                     sub_32))>;
 
-    def : Pat<(i32 (op GPR32:$Rdn, (shl (i32 (trunc (opcnt (sve_pred_enum:$pattern)))), (i64 (sve_cnt_shl_imm i32:$imm))))),
+    def : Pat<(i32 (op GPR32:$Rdn, (shl (i32 (trunc (opcnt (sve_pred_enum:$pattern)))), (sve_cnt_shl_imm i32:$imm)))),
               (i32 (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (i64 (IMPLICIT_DEF)),
                                                GPR32:$Rdn, sub_32), sve_pred_enum:$pattern, $imm),
                                     sub_32))>;
@@ -4324,10 +4332,10 @@ multiclass sve_int_arith_imm1<bits<2> opc, string asm, SDPatternOperator op> {
   def _S : sve_int_arith_imm<0b10, { 0b1010, opc }, asm, ZPR32, simm8>;
   def _D : sve_int_arith_imm<0b11, { 0b1010, opc }, asm, ZPR64, simm8>;
 
-  def : SVE_1_Op_Imm_Arith_All_Active<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _B)>;
-  def : SVE_1_Op_Imm_Arith_All_Active<nxv8i16, nxv8i1,  op, ZPR16, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _H)>;
-  def : SVE_1_Op_Imm_Arith_All_Active<nxv4i32, nxv4i1,  op, ZPR32, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _S)>;
-  def : SVE_1_Op_Imm_Arith_All_Active<nxv2i64, nxv2i1,  op, ZPR64, i64, SVEArithSImmPat, !cast<Instruction>(NAME # _D)>;
+  def : SVE_1_Op_Imm_Arith_All_Active<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _B)>;
+  def : SVE_1_Op_Imm_Arith_All_Active<nxv8i16, nxv8i1,  op, ZPR16, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _H)>;
+  def : SVE_1_Op_Imm_Arith_All_Active<nxv4i32, nxv4i1,  op, ZPR32, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _S)>;
+  def : SVE_1_Op_Imm_Arith_All_Active<nxv2i64, nxv2i1,  op, ZPR64, i64, SVEArithSImmPat64, !cast<Instruction>(NAME # _D)>;
 }
 
 multiclass sve_int_arith_imm1_unsigned<bits<2> opc, string asm, SDPatternOperator op> {
@@ -4348,10 +4356,10 @@ multiclass sve_int_arith_imm2<string asm, SDPatternOperator op> {
   def _S : sve_int_arith_imm<0b10, 0b110000, asm, ZPR32, simm8>;
   def _D : sve_int_arith_imm<0b11, 0b110000, asm, ZPR64, simm8>;
 
-  def : SVE_1_Op_Imm_Arith_All_Active<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _B)>;
-  def : SVE_1_Op_Imm_Arith_All_Active<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _H)>;
-  def : SVE_1_Op_Imm_Arith_All_Active<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _S)>;
-  def : SVE_1_Op_Imm_Arith_All_Active<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithSImmPat, !cast<Instruction>(NAME # _D)>;
+  def : SVE_1_Op_Imm_Arith_All_Active<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _B)>;
+  def : SVE_1_Op_Imm_Arith_All_Active<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _H)>;
+  def : SVE_1_Op_Imm_Arith_All_Active<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _S)>;
+  def : SVE_1_Op_Imm_Arith_All_Active<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithSImmPat64, !cast<Instruction>(NAME # _D)>;
 }
 
 //===----------------------------------------------------------------------===//
@@ -4542,7 +4550,7 @@ multiclass sve_int_dup_imm_pred_merge_inst<
                   (!cast<Instruction>(NAME) zprty:$Zd, PPRAny:$Pg, cpyimm:$imm), 1>;
   def : Pat<(intty
               (vselect predty:$Pg,
-                (intty (AArch64dup (scalarty (SVE8BitLslImm i32:$imm, i32:$shift)))),
+                (intty (AArch64dup (scalarty (SVE8BitLslImm<scalarty>.Pat i32:$imm, i32:$shift)))),
                 intty:$Zd)),
             (!cast<Instruction>(NAME) zprty:$Zd, $Pg, i32:$imm, i32:$shift)>;
 }
@@ -4580,7 +4588,7 @@ multiclass sve_int_dup_imm_pred_zero_inst<
             (!cast<Instruction>(NAME) PPRAny:$Ps1, 1, 0)>;
   def : Pat<(intty
               (vselect predty:$Pg,
-                (intty (AArch64dup (scalarty (SVE8BitLslImm i32:$imm, i32:$shift)))),
+                (intty (AArch64dup (scalarty (SVE8BitLslImm<scalarty>.Pat i32:$imm, i32:$shift)))),
                 (intty (AArch64dup (scalarty 0))))),
             (!cast<Instruction>(NAME) $Pg, i32:$imm, i32:$shift)>;
 }
@@ -6476,14 +6484,14 @@ multiclass sve_int_perm_rev_revh<string asm, SDPatternOperator op> {
   def _S : sve_int_perm_rev<0b10, 0b01, asm, ZPR32>;
   def _D : sve_int_perm_rev<0b11, 0b01, asm, ZPR64>;
 
-  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1,  nxv4i32, !cast<Instruction>(NAME # _S)>;
-  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1,  nxv2i64, !cast<Instruction>(NAME # _D)>;
+  def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1,  nxv4i32, !cast<Instruction>(NAME # _S)>;
+  def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1,  nxv2i64, !cast<Instruction>(NAME # _D)>;
 }
 
 multiclass sve_int_perm_rev_revw<string asm, SDPatternOperator op> {
   def _D : sve_int_perm_rev<0b11, 0b10, asm, ZPR64>;
 
-  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1,  nxv2i64, !cast<Instruction>(NAME # _D)>;
+  def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1,  nxv2i64, !cast<Instruction>(NAME # _D)>;
 }
 
 class sve_int_perm_cpy_r<bits<2> sz8_64, string asm, ZPRRegOp zprty,
@@ -8377,13 +8385,13 @@ multiclass sve_int_perm_bin_perm_128_zz<bits<2> opc, bit P, string asm, SDPatter
 }
 
 /// Addressing modes
-def am_sve_indexed_s4 :ComplexPattern<i64, 2, "SelectAddrModeIndexedSVE<-8,7>", [], [SDNPWantRoot]>;
-def am_sve_indexed_s6 :ComplexPattern<i64, 2, "SelectAddrModeIndexedSVE<-32,31>", [], [SDNPWantRoot]>;
+def am_sve_indexed_s4 :ComplexPattern<iPTR, 2, "SelectAddrModeIndexedSVE<-8,7>", [], [SDNPWantRoot]>;
+def am_sve_indexed_s6 :ComplexPattern<iPTR, 2, "SelectAddrModeIndexedSVE<-32,31>", [], [SDNPWantRoot]>;
 
-def am_sve_regreg_lsl0 : ComplexPattern<i64, 2, "SelectSVERegRegAddrMode<0>", []>;
-def am_sve_regreg_lsl1 : ComplexPattern<i64, 2, "SelectSVERegRegAddrMode<1>", []>;
-def am_sve_regreg_lsl2 : ComplexPattern<i64, 2, "SelectSVERegRegAddrMode<2>", []>;
-def am_sve_regreg_lsl3 : ComplexPattern<i64, 2, "SelectSVERegRegAddrMode<3>", []>;
+def am_sve_regreg_lsl0 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<0>", []>;
+def am_sve_regreg_lsl1 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<1>", []>;
+def am_sve_regreg_lsl2 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<2>", []>;
+def am_sve_regreg_lsl3 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<3>", []>;
 
 // Predicated pseudo floating point two operand instructions.
 multiclass sve_fp_bin_pred_hfd<SDPatternOperator op> {
diff --git a/llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp b/llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp
index e72dccdc4b78..642080a0d40d 100644
--- a/llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp
+++ b/llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp
@@ -152,7 +152,7 @@ bool SVEIntrinsicOpts::coalescePTrueIntrinsicCalls(
   // Remove the most encompassing ptrue, as well as any promoted ptrues, leaving
   // behind only the ptrues to be coalesced.
   PTrues.remove(MostEncompassingPTrue);
-  PTrues.remove_if([](auto *PTrue) { return isPTruePromoted(PTrue); });
+  PTrues.remove_if(isPTruePromoted);
 
   // Hoist MostEncompassingPTrue to the start of the basic block. It is always
   // safe to do this, since ptrue intrinsic calls are guaranteed to have no
@@ -287,10 +287,10 @@ bool SVEIntrinsicOpts::optimizePredicateStore(Instruction *I) {
   if (!Attr.isValid())
     return false;
 
-  unsigned MinVScale, MaxVScale;
-  std::tie(MinVScale, MaxVScale) = Attr.getVScaleRangeArgs();
+  unsigned MinVScale = Attr.getVScaleRangeMin();
+  Optional<unsigned> MaxVScale = Attr.getVScaleRangeMax();
   // The transform needs to know the exact runtime length of scalable vectors
-  if (MinVScale != MaxVScale || MinVScale == 0)
+  if (!MaxVScale || MinVScale != MaxVScale)
     return false;
 
   auto *PredType =
@@ -351,10 +351,10 @@ bool SVEIntrinsicOpts::optimizePredicateLoad(Instruction *I) {
   if (!Attr.isValid())
     return false;
 
-  unsigned MinVScale, MaxVScale;
-  std::tie(MinVScale, MaxVScale) = Attr.getVScaleRangeArgs();
+  unsigned MinVScale = Attr.getVScaleRangeMin();
+  Optional<unsigned> MaxVScale = Attr.getVScaleRangeMax();
   // The transform needs to know the exact runtime length of scalable vectors
-  if (MinVScale != MaxVScale || MinVScale == 0)
+  if (!MaxVScale || MinVScale != MaxVScale)
     return false;
 
   auto *PredType =
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.cpp
index aab76d27ef11..d28f38e42430 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.cpp
@@ -173,14 +173,7 @@ constexpr AMDGPUFunctionArgInfo AMDGPUFunctionArgInfo::fixedABILayout() {
 const AMDGPUFunctionArgInfo &
 AMDGPUArgumentUsageInfo::lookupFuncArgInfo(const Function &F) const {
   auto I = ArgInfoMap.find(&F);
-  if (I == ArgInfoMap.end()) {
-    if (AMDGPUTargetMachine::EnableFixedFunctionABI)
-      return FixedABIFunctionInfo;
-
-    // Without the fixed ABI, we assume no function has special inputs.
-    assert(F.isDeclaration());
-    return ExternFunctionInfo;
-  }
-
+  if (I == ArgInfoMap.end())
+    return FixedABIFunctionInfo;
   return I->second;
 }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp
index f0aadab3302f..b4ebc7d7d75f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp
@@ -112,6 +112,17 @@ static bool isDSAddress(const Constant *C) {
   return AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS;
 }
 
+/// Returns true if the function requires the implicit argument be passed
+/// regardless of the function contents.
+static bool funcRequiresImplicitArgPtr(const Function &F) {
+  // Sanitizers require the hostcall buffer passed in the implicit arguments.
+  return F.hasFnAttribute(Attribute::SanitizeAddress) ||
+         F.hasFnAttribute(Attribute::SanitizeThread) ||
+         F.hasFnAttribute(Attribute::SanitizeMemory) ||
+         F.hasFnAttribute(Attribute::SanitizeHWAddress) ||
+         F.hasFnAttribute(Attribute::SanitizeMemTag);
+}
+
 namespace {
 class AMDGPUInformationCache : public InformationCache {
 public:
@@ -296,7 +307,7 @@ struct AAUniformWorkGroupSizeFunction : public AAUniformWorkGroupSize {
 
     bool AllCallSitesKnown = true;
     if (!A.checkForAllCallSites(CheckCallSite, *this, true, AllCallSitesKnown))
-      indicatePessimisticFixpoint();
+      return indicatePessimisticFixpoint();
 
     return Change;
   }
@@ -339,7 +350,17 @@ struct AAAMDAttributesFunction : public AAAMDAttributes {
 
   void initialize(Attributor &A) override {
     Function *F = getAssociatedFunction();
+
+    // If the function requires the implicit arg pointer due to sanitizers,
+    // assume it's needed even if explicitly marked as not requiring it.
+    const bool NeedsImplicit = funcRequiresImplicitArgPtr(*F);
+    if (NeedsImplicit)
+      removeAssumedBits(IMPLICIT_ARG_PTR);
+
     for (auto Attr : ImplicitAttrs) {
+      if (NeedsImplicit && Attr.first == IMPLICIT_ARG_PTR)
+        continue;
+
       if (F->hasFnAttribute(Attr.second))
         addKnownBits(Attr.first);
     }
@@ -500,6 +521,9 @@ struct AAAMDFlatWorkGroupSize
     std::tie(MinGroupSize, MaxGroupSize) = InfoCache.getFlatWorkGroupSizes(*F);
     intersectKnown(
         ConstantRange(APInt(32, MinGroupSize), APInt(32, MaxGroupSize + 1)));
+
+    if (AMDGPU::isEntryFunctionCC(F->getCallingConv()))
+      indicatePessimisticFixpoint();
   }
 
   ChangeStatus updateImpl(Attributor &A) override {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
index 43928d7c2a09..2f1e7823f65c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
@@ -652,8 +652,8 @@ bool AMDGPUCallLowering::lowerFormalArguments(
       ++PSInputNum;
 
       if (SkipArg) {
-        for (int I = 0, E = VRegs[Idx].size(); I != E; ++I)
-          B.buildUndef(VRegs[Idx][I]);
+        for (Register R : VRegs[Idx])
+          B.buildUndef(R);
 
         ++Idx;
         continue;
@@ -715,10 +715,9 @@ bool AMDGPUCallLowering::lowerFormalArguments(
   if (!MBB.empty())
     B.setInstr(*MBB.begin());
 
-  if (!IsEntryFunc) {
+  if (!IsEntryFunc && !IsGraphics) {
     // For the fixed ABI, pass workitem IDs in the last argument register.
-    if (AMDGPUTargetMachine::EnableFixedFunctionABI)
-      TLI.allocateSpecialInputVGPRsFixed(CCInfo, MF, *TRI, *Info);
+    TLI.allocateSpecialInputVGPRsFixed(CCInfo, MF, *TRI, *Info);
   }
 
   IncomingValueAssigner Assigner(AssignFn);
@@ -731,11 +730,6 @@ bool AMDGPUCallLowering::lowerFormalArguments(
 
   uint64_t StackOffset = Assigner.StackOffset;
 
-  if (!IsEntryFunc && !AMDGPUTargetMachine::EnableFixedFunctionABI) {
-    // Special inputs come after user arguments.
-    TLI.allocateSpecialInputVGPRs(CCInfo, MF, *TRI, *Info);
-  }
-
   // Start adding system SGPRs.
   if (IsEntryFunc) {
     TLI.allocateSystemSGPRs(CCInfo, MF, *Info, CC, IsGraphics);
@@ -829,9 +823,12 @@ bool AMDGPUCallLowering::passSpecialInputs(MachineIRBuilder &MIRBuilder,
 
     if (IncomingArg) {
       LI->loadInputValue(InputReg, MIRBuilder, IncomingArg, ArgRC, ArgTy);
-    } else {
-      assert(InputID == AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR);
+    } else if (InputID == AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR) {
       LI->getImplicitArgPtr(InputReg, MRI, MIRBuilder);
+    } else {
+      // We may have proven the input wasn't needed, although the ABI is
+      // requiring it. We just need to allocate the register appropriately.
+      MIRBuilder.buildUndef(InputReg);
     }
 
     if (OutgoingArg->isRegister()) {
@@ -1233,8 +1230,7 @@ bool AMDGPUCallLowering::lowerTailCall(
   // after the ordinary user argument registers.
   SmallVector<std::pair<MCRegister, Register>, 12> ImplicitArgRegs;
 
-  if (AMDGPUTargetMachine::EnableFixedFunctionABI &&
-      Info.CallConv != CallingConv::AMDGPU_Gfx) {
+  if (Info.CallConv != CallingConv::AMDGPU_Gfx) {
     // With a fixed ABI, allocate fixed registers before user arguments.
     if (!passSpecialInputs(MIRBuilder, CCInfo, ImplicitArgRegs, Info))
       return false;
@@ -1300,12 +1296,6 @@ bool AMDGPUCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
   const SITargetLowering &TLI = *getTLI<SITargetLowering>();
   const DataLayout &DL = F.getParent()->getDataLayout();
 
-  if (!AMDGPUTargetMachine::EnableFixedFunctionABI &&
-      Info.CallConv != CallingConv::AMDGPU_Gfx) {
-    LLVM_DEBUG(dbgs() << "Variable function ABI not implemented\n");
-    return false;
-  }
-
   SmallVector<ArgInfo, 8> OutArgs;
   for (auto &OrigArg : Info.OrigArgs)
     splitToValueTypes(OrigArg, OutArgs, DL, Info.CallConv);
@@ -1359,8 +1349,7 @@ bool AMDGPUCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
   // after the ordinary user argument registers.
   SmallVector<std::pair<MCRegister, Register>, 12> ImplicitArgRegs;
 
-  if (AMDGPUTargetMachine::EnableFixedFunctionABI &&
-      Info.CallConv != CallingConv::AMDGPU_Gfx) {
+  if (Info.CallConv != CallingConv::AMDGPU_Gfx) {
     // With a fixed ABI, allocate fixed registers before user arguments.
     if (!passSpecialInputs(MIRBuilder, CCInfo, ImplicitArgRegs, Info))
       return false;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCombine.td b/llvm/lib/Target/AMDGPU/AMDGPUCombine.td
index c7c5ff7bcbe7..2415fdfecaae 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCombine.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCombine.td
@@ -64,6 +64,30 @@ def int_minmax_to_med3 : GICombineRule<
          [{ return RegBankHelper.matchIntMinMaxToMed3(*${min_or_max}, ${matchinfo}); }]),
   (apply [{ RegBankHelper.applyMed3(*${min_or_max}, ${matchinfo}); }])>;
 
+def fp_minmax_to_med3 : GICombineRule<
+  (defs root:$min_or_max, med3_matchdata:$matchinfo),
+  (match (wip_match_opcode G_FMAXNUM,
+                           G_FMINNUM,
+                           G_FMAXNUM_IEEE,
+                           G_FMINNUM_IEEE):$min_or_max,
+         [{ return RegBankHelper.matchFPMinMaxToMed3(*${min_or_max}, ${matchinfo}); }]),
+  (apply [{ RegBankHelper.applyMed3(*${min_or_max}, ${matchinfo}); }])>;
+
+def fp_minmax_to_clamp : GICombineRule<
+  (defs root:$min_or_max, register_matchinfo:$matchinfo),
+  (match (wip_match_opcode G_FMAXNUM,
+                           G_FMINNUM,
+                           G_FMAXNUM_IEEE,
+                           G_FMINNUM_IEEE):$min_or_max,
+         [{ return RegBankHelper.matchFPMinMaxToClamp(*${min_or_max}, ${matchinfo}); }]),
+  (apply [{ RegBankHelper.applyClamp(*${min_or_max}, ${matchinfo}); }])>;
+
+def fmed3_intrinsic_to_clamp : GICombineRule<
+  (defs root:$fmed3, register_matchinfo:$matchinfo),
+  (match (wip_match_opcode G_INTRINSIC):$fmed3,
+         [{ return RegBankHelper.matchFPMed3ToClamp(*${fmed3}, ${matchinfo}); }]),
+  (apply [{ RegBankHelper.applyClamp(*${fmed3}, ${matchinfo}); }])>;
+
 def remove_fcanonicalize_matchinfo : GIDefMatchData<"Register">;
 
 def remove_fcanonicalize : GICombineRule<
@@ -102,7 +126,9 @@ def AMDGPUPostLegalizerCombinerHelper: GICombinerHelper<
 }
 
 def AMDGPURegBankCombinerHelper : GICombinerHelper<
-  "AMDGPUGenRegBankCombinerHelper", [zext_trunc_fold, int_minmax_to_med3, ptr_add_immed_chain]> {
+  "AMDGPUGenRegBankCombinerHelper",
+  [zext_trunc_fold, int_minmax_to_med3, ptr_add_immed_chain,
+   fp_minmax_to_clamp, fp_minmax_to_med3, fmed3_intrinsic_to_clamp]> {
   let DisableRuleOption = "amdgpuregbankcombiner-disable-rule";
   let StateClass = "AMDGPURegBankCombinerHelperState";
   let AdditionalArguments = [];
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.cpp
index 301e6f6d6f42..e79ff9b597c9 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.cpp
@@ -378,5 +378,4 @@ void AMDGPUCombinerHelper::applyFoldableFneg(MachineInstr &MI,
   }
 
   MI.eraseFromParent();
-  return;
 }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
index 12cef2774aaf..7fd94a977be7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
@@ -172,6 +172,8 @@ def : GINodeEquiv<G_AMDGPU_CVT_F32_UBYTE3, AMDGPUcvt_f32_ubyte3>;
 def : GINodeEquiv<G_AMDGPU_CVT_PK_I16_I32, AMDGPUpk_i16_i32_impl>;
 def : GINodeEquiv<G_AMDGPU_SMED3, AMDGPUsmed3>;
 def : GINodeEquiv<G_AMDGPU_UMED3, AMDGPUumed3>;
+def : GINodeEquiv<G_AMDGPU_FMED3, AMDGPUfmed3_impl>;
+def : GINodeEquiv<G_AMDGPU_CLAMP, AMDGPUclamp>;
 
 def : GINodeEquiv<G_AMDGPU_ATOMIC_CMPXCHG, AMDGPUatomic_cmp_swap>;
 def : GINodeEquiv<G_AMDGPU_BUFFER_LOAD, SIbuffer_load>;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp
index b9c59f4c615a..699c6c479455 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp
@@ -280,11 +280,12 @@ void MetadataStreamerV2::emitKernelAttrs(const Function &Func) {
   }
 }
 
-void MetadataStreamerV2::emitKernelArgs(const Function &Func) {
+void MetadataStreamerV2::emitKernelArgs(const Function &Func,
+                                        const GCNSubtarget &ST) {
   for (auto &Arg : Func.args())
     emitKernelArg(Arg);
 
-  emitHiddenKernelArgs(Func);
+  emitHiddenKernelArgs(Func, ST);
 }
 
 void MetadataStreamerV2::emitKernelArg(const Argument &Arg) {
@@ -381,10 +382,9 @@ void MetadataStreamerV2::emitKernelArg(const DataLayout &DL, Type *Ty,
   }
 }
 
-void MetadataStreamerV2::emitHiddenKernelArgs(const Function &Func) {
-  int HiddenArgNumBytes =
-      getIntegerAttribute(Func, "amdgpu-implicitarg-num-bytes", 0);
-
+void MetadataStreamerV2::emitHiddenKernelArgs(const Function &Func,
+                                              const GCNSubtarget &ST) {
+  unsigned HiddenArgNumBytes = ST.getImplicitArgNumBytes(Func);
   if (!HiddenArgNumBytes)
     return;
 
@@ -465,11 +465,12 @@ void MetadataStreamerV2::emitKernel(const MachineFunction &MF,
   HSAMetadata.mKernels.push_back(Kernel::Metadata());
   auto &Kernel = HSAMetadata.mKernels.back();
 
+  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
   Kernel.mName = std::string(Func.getName());
   Kernel.mSymbolName = (Twine(Func.getName()) + Twine("@kd")).str();
   emitKernelLanguage(Func);
   emitKernelAttrs(Func);
-  emitKernelArgs(Func);
+  emitKernelArgs(Func, ST);
   HSAMetadata.mKernels.back().mCodeProps = CodeProps;
   HSAMetadata.mKernels.back().mDebugProps = DebugProps;
 }
@@ -673,13 +674,14 @@ void MetadataStreamerV3::emitKernelAttrs(const Function &Func,
 }
 
 void MetadataStreamerV3::emitKernelArgs(const Function &Func,
+                                        const GCNSubtarget &ST,
                                         msgpack::MapDocNode Kern) {
   unsigned Offset = 0;
   auto Args = HSAMetadataDoc->getArrayNode();
   for (auto &Arg : Func.args())
     emitKernelArg(Arg, Offset, Args);
 
-  emitHiddenKernelArgs(Func, Offset, Args);
+  emitHiddenKernelArgs(Func, ST, Offset, Args);
 
   Kern[".args"] = Args;
 }
@@ -791,11 +793,10 @@ void MetadataStreamerV3::emitKernelArg(
 }
 
 void MetadataStreamerV3::emitHiddenKernelArgs(const Function &Func,
+                                              const GCNSubtarget &ST,
                                               unsigned &Offset,
                                               msgpack::ArrayDocNode Args) {
-  int HiddenArgNumBytes =
-      getIntegerAttribute(Func, "amdgpu-implicitarg-num-bytes", 0);
-
+  unsigned HiddenArgNumBytes = ST.getImplicitArgNumBytes(Func);
   if (!HiddenArgNumBytes)
     return;
 
@@ -912,6 +913,7 @@ void MetadataStreamerV3::emitKernel(const MachineFunction &MF,
                                     const SIProgramInfo &ProgramInfo) {
   auto &Func = MF.getFunction();
   auto Kern = getHSAKernelProps(MF, ProgramInfo);
+  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
 
   assert(Func.getCallingConv() == CallingConv::AMDGPU_KERNEL ||
          Func.getCallingConv() == CallingConv::SPIR_KERNEL);
@@ -925,7 +927,7 @@ void MetadataStreamerV3::emitKernel(const MachineFunction &MF,
         (Twine(Func.getName()) + Twine(".kd")).str(), /*Copy=*/true);
     emitKernelLanguage(Func, Kern);
     emitKernelAttrs(Func, Kern);
-    emitKernelArgs(Func, Kern);
+    emitKernelArgs(Func, ST, Kern);
   }
 
   Kernels.push_back(Kern);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h
index af5dae1cd8c0..54ed0afbba6d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h
@@ -30,6 +30,7 @@ class MDNode;
 class Module;
 struct SIProgramInfo;
 class Type;
+class GCNSubtarget;
 
 namespace AMDGPU {
 
@@ -86,7 +87,8 @@ protected:
 
   void emitKernelAttrs(const Function &Func, msgpack::MapDocNode Kern);
 
-  void emitKernelArgs(const Function &Func, msgpack::MapDocNode Kern);
+  void emitKernelArgs(const Function &Func, const GCNSubtarget &ST,
+                      msgpack::MapDocNode Kern);
 
   void emitKernelArg(const Argument &Arg, unsigned &Offset,
                      msgpack::ArrayDocNode Args);
@@ -98,8 +100,8 @@ protected:
                      StringRef BaseTypeName = "", StringRef AccQual = "",
                      StringRef TypeQual = "");
 
-  void emitHiddenKernelArgs(const Function &Func, unsigned &Offset,
-                            msgpack::ArrayDocNode Args);
+  void emitHiddenKernelArgs(const Function &Func, const GCNSubtarget &ST,
+                            unsigned &Offset, msgpack::ArrayDocNode Args);
 
   msgpack::DocNode &getRootMetadata(StringRef Key) {
     return HSAMetadataDoc->getRoot().getMap(/*Convert=*/true)[Key];
@@ -173,7 +175,7 @@ private:
 
   void emitKernelAttrs(const Function &Func);
 
-  void emitKernelArgs(const Function &Func);
+  void emitKernelArgs(const Function &Func, const GCNSubtarget &ST);
 
   void emitKernelArg(const Argument &Arg);
 
@@ -183,7 +185,7 @@ private:
                      StringRef BaseTypeName = "", StringRef AccQual = "",
                      StringRef TypeQual = "");
 
-  void emitHiddenKernelArgs(const Function &Func);
+  void emitHiddenKernelArgs(const Function &Func, const GCNSubtarget &ST);
 
   const Metadata &getHSAMetadata() const {
     return HSAMetadata;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index 88b4ec53a2a0..db84b8766924 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -892,6 +892,15 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
     }
     break;
   }
+  case Intrinsic::amdgcn_is_shared:
+  case Intrinsic::amdgcn_is_private: {
+    if (isa<UndefValue>(II.getArgOperand(0)))
+      return IC.replaceInstUsesWith(II, UndefValue::get(II.getType()));
+
+    if (isa<ConstantPointerNull>(II.getArgOperand(0)))
+      return IC.replaceInstUsesWith(II, ConstantInt::getFalse(II.getType()));
+    break;
+  }
   default: {
     if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
             AMDGPU::getImageDimIntrinsicInfo(II.getIntrinsicID())) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 1f898f2ba8b3..5046daaed977 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -533,7 +533,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
     getActionDefinitionsBuilder({G_ADD, G_SUB, G_MUL})
       .legalFor({S32, S16, V2S16})
       .minScalar(0, S16)
-      .clampMaxNumElements(0, S16, 2)
+      .clampMaxNumElementsStrict(0, S16, 2)
       .widenScalarToNextMultipleOf(0, 32)
       .maxScalar(0, S32)
       .scalarize(0);
@@ -541,7 +541,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
     getActionDefinitionsBuilder({G_UADDSAT, G_USUBSAT, G_SADDSAT, G_SSUBSAT})
       .legalFor({S32, S16, V2S16}) // Clamp modifier
       .minScalarOrElt(0, S16)
-      .clampMaxNumElements(0, S16, 2)
+      .clampMaxNumElementsStrict(0, S16, 2)
       .scalarize(0)
       .widenScalarToNextPow2(0, 32)
       .lower();
@@ -712,7 +712,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
   }
 
   if (ST.hasVOP3PInsts())
-    FPOpActions.clampMaxNumElements(0, S16, 2);
+    FPOpActions.clampMaxNumElementsStrict(0, S16, 2);
 
   FPOpActions
     .scalarize(0)
@@ -728,7 +728,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
 
   getActionDefinitionsBuilder({G_FNEG, G_FABS})
     .legalFor(FPTypesPK16)
-    .clampMaxNumElements(0, S16, 2)
+    .clampMaxNumElementsStrict(0, S16, 2)
     .scalarize(0)
     .clampScalar(0, S16, S64);
 
@@ -965,7 +965,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
   if (ST.has16BitInsts()) {
     getActionDefinitionsBuilder(G_BSWAP)
       .legalFor({S16, S32, V2S16})
-      .clampMaxNumElements(0, S16, 2)
+      .clampMaxNumElementsStrict(0, S16, 2)
       // FIXME: Fixing non-power-of-2 before clamp is workaround for
       // narrowScalar limitation.
       .widenScalarToNextPow2(0)
@@ -1052,10 +1052,6 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
 
     // Split vector extloads.
     unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
-    unsigned AlignBits = Query.MMODescrs[0].AlignInBits;
-
-    if (MemSize < DstTy.getSizeInBits())
-      MemSize = std::max(MemSize, AlignBits);
 
     if (DstTy.isVector() && DstTy.getSizeInBits() > MemSize)
       return true;
@@ -1077,12 +1073,6 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
         return true;
     }
 
-    if (AlignBits < MemSize) {
-      const SITargetLowering *TLI = ST.getTargetLowering();
-      return !TLI->allowsMisalignedMemoryAccessesImpl(MemSize, AS,
-                                                      Align(AlignBits / 8));
-    }
-
     return false;
   };
 
@@ -1176,20 +1166,6 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
               if (DstSize > MemSize)
                 return std::make_pair(0, LLT::scalar(MemSize));
 
-              if (!isPowerOf2_32(DstSize)) {
-                // We're probably decomposing an odd sized store. Try to split
-                // to the widest type. TODO: Account for alignment. As-is it
-                // should be OK, since the new parts will be further legalized.
-                unsigned FloorSize = PowerOf2Floor(DstSize);
-                return std::make_pair(0, LLT::scalar(FloorSize));
-              }
-
-              if (DstSize > 32 && (DstSize % 32 != 0)) {
-                // FIXME: Need a way to specify non-extload of larger size if
-                // suitably aligned.
-                return std::make_pair(0, LLT::scalar(32 * (DstSize / 32)));
-              }
-
               unsigned MaxSize = maxSizeForAddrSpace(ST,
                                                      PtrTy.getAddressSpace(),
                                                      Op == G_LOAD);
@@ -1257,14 +1233,6 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
                            ElementCount::getFixed(FloorSize / EltSize), EltTy));
               }
 
-              // Need to split because of alignment.
-              unsigned Align = Query.MMODescrs[0].AlignInBits;
-              if (EltSize > Align &&
-                  (EltSize / Align < DstTy.getNumElements())) {
-                return std::make_pair(
-                    0, LLT::fixed_vector(EltSize / Align, EltTy));
-              }
-
               // May need relegalization for the scalars.
               return std::make_pair(0, EltTy);
             })
@@ -1457,6 +1425,13 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
     // FIXME: Doesn't handle extract of illegal sizes.
     getActionDefinitionsBuilder(Op)
       .lowerIf(all(typeIs(LitTyIdx, S16), sizeIs(BigTyIdx, 32)))
+      .lowerIf([=](const LegalityQuery &Query) {
+          // Sub-vector(or single element) insert and extract.
+          // TODO: verify immediate offset here since lower only works with
+          // whole elements.
+          const LLT BigTy = Query.Types[BigTyIdx];
+          return BigTy.isVector();
+        })
       // FIXME: Multiples of 16 should not be legal.
       .legalIf([=](const LegalityQuery &Query) {
           const LLT BigTy = Query.Types[BigTyIdx];
@@ -1615,7 +1590,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
       // Prefer to reduce vector widths for 16-bit vectors before lowering, to
       // get more vector shift opportunities, since we'll get those when
       // expanded.
-      .fewerElementsIf(elementTypeIs(0, S16), changeTo(0, V2S16));
+      .clampMaxNumElementsStrict(0, S16, 2);
   } else if (ST.has16BitInsts()) {
     SextInReg.lowerFor({{S32}, {S64}, {S16}});
   } else {
@@ -1637,14 +1612,14 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
   getActionDefinitionsBuilder(G_FSHR)
     .legalFor({{S32, S32}})
     .lowerFor({{V2S16, V2S16}})
-    .fewerElementsIf(elementTypeIs(0, S16), changeTo(0, V2S16))
+    .clampMaxNumElementsStrict(0, S16, 2)
     .scalarize(0)
     .lower();
 
   if (ST.hasVOP3PInsts()) {
     getActionDefinitionsBuilder(G_FSHL)
       .lowerFor({{V2S16, V2S16}})
-      .fewerElementsIf(elementTypeIs(0, S16), changeTo(0, V2S16))
+      .clampMaxNumElementsStrict(0, S16, 2)
       .scalarize(0)
       .lower();
   } else {
@@ -2567,10 +2542,8 @@ bool AMDGPULegalizerInfo::legalizeLoad(LegalizerHelper &Helper,
       } else {
         // For cases where the widened type isn't a nice register value, unmerge
         // from a widened register (e.g. <3 x s16> -> <4 x s16>)
-        B.setInsertPt(B.getMBB(), ++B.getInsertPt());
-        WideLoad = Helper.widenWithUnmerge(WideTy, ValReg);
-        B.setInsertPt(B.getMBB(), MI.getIterator());
-        B.buildLoadFromOffset(WideLoad, PtrReg, *MMO, 0);
+        WideLoad = B.buildLoadFromOffset(WideTy, PtrReg, *MMO, 0).getReg(0);
+        B.buildDeleteTrailingVectorElements(ValReg, WideLoad);
       }
     }
 
@@ -3843,6 +3816,10 @@ Register AMDGPULegalizerInfo::handleD16VData(MachineIRBuilder &B,
     llvm_unreachable("invalid data type");
   }
 
+  if (StoreVT == LLT::fixed_vector(3, S16)) {
+    Reg = B.buildPadVectorWithUndefElements(LLT::fixed_vector(4, S16), Reg)
+              .getReg(0);
+  }
   return Reg;
 }
 
@@ -4237,8 +4214,17 @@ static void packImage16bitOpsToDwords(MachineIRBuilder &B, MachineInstr &MI,
         (I >= Intr->GradientStart && I < Intr->CoordStart && !IsG16) ||
         (I >= Intr->CoordStart && !IsA16)) {
       // Handle any gradient or coordinate operands that should not be packed
-      AddrReg = B.buildBitcast(V2S16, AddrReg).getReg(0);
-      PackedAddrs.push_back(AddrReg);
+      if ((I < Intr->GradientStart) && IsA16 &&
+          (B.getMRI()->getType(AddrReg) == S16)) {
+        // Special handling of bias when A16 is on. Bias is of type half but
+        // occupies full 32-bit.
+        PackedAddrs.push_back(
+            B.buildBuildVector(V2S16, {AddrReg, B.buildUndef(S16).getReg(0)})
+                .getReg(0));
+      } else {
+        AddrReg = B.buildBitcast(V2S16, AddrReg).getReg(0);
+        PackedAddrs.push_back(AddrReg);
+      }
     } else {
       // Dz/dh, dz/dv and the last odd coord are packed with undef. Also, in 1D,
       // derivatives dx/dh and dx/dv are packed with undef.
@@ -4676,9 +4662,23 @@ bool AMDGPULegalizerInfo::legalizeImageIntrinsic(
   // Deal with the one annoying legal case.
   const LLT V3S16 = LLT::fixed_vector(3, 16);
   if (Ty == V3S16) {
-    padWithUndef(ResTy, RegsToCover - ResultRegs.size() + 1);
-    auto Concat = B.buildConcatVectors(LLT::fixed_vector(6, 16), ResultRegs);
-    B.buildUnmerge({DstReg, MRI->createGenericVirtualRegister(V3S16)}, Concat);
+    if (IsTFE) {
+      if (ResultRegs.size() == 1) {
+        NewResultReg = ResultRegs[0];
+      } else if (ResultRegs.size() == 2) {
+        LLT V4S16 = LLT::fixed_vector(4, 16);
+        NewResultReg = B.buildConcatVectors(V4S16, ResultRegs).getReg(0);
+      } else {
+        return false;
+      }
+    }
+
+    if (MRI->getType(DstReg).getNumElements() <
+        MRI->getType(NewResultReg).getNumElements()) {
+      B.buildDeleteTrailingVectorElements(DstReg, NewResultReg);
+    } else {
+      B.buildPadVectorWithUndefElements(DstReg, NewResultReg);
+    }
     return true;
   }
 
@@ -4869,8 +4869,8 @@ bool AMDGPULegalizerInfo::legalizeBVHIntrinsic(MachineInstr &MI,
   }
   Ops.push_back(RayExtent);
 
-  auto packLanes = [&Ops, &S32, &B] (Register Src) {
-    auto Unmerge = B.buildUnmerge({S32, S32, S32, S32}, Src);
+  auto packLanes = [&Ops, &S32, &B](Register Src) {
+    auto Unmerge = B.buildUnmerge({S32, S32, S32}, Src);
     Ops.push_back(Unmerge.getReg(0));
     Ops.push_back(Unmerge.getReg(1));
     Ops.push_back(Unmerge.getReg(2));
@@ -4878,8 +4878,8 @@ bool AMDGPULegalizerInfo::legalizeBVHIntrinsic(MachineInstr &MI,
 
   packLanes(RayOrigin);
   if (IsA16) {
-    auto UnmergeRayDir = B.buildUnmerge({S16, S16, S16, S16}, RayDir);
-    auto UnmergeRayInvDir = B.buildUnmerge({S16, S16, S16, S16}, RayInvDir);
+    auto UnmergeRayDir = B.buildUnmerge({S16, S16, S16}, RayDir);
+    auto UnmergeRayInvDir = B.buildUnmerge({S16, S16, S16}, RayInvDir);
     Register R1 = MRI.createGenericVirtualRegister(S32);
     Register R2 = MRI.createGenericVirtualRegister(S32);
     Register R3 = MRI.createGenericVirtualRegister(S32);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp
index 12d6d35a6917..6e2b5dc471bc 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp
@@ -24,13 +24,6 @@
 // A possible future refinement is to specialise the structure per-kernel, so
 // that fields can be elided based on more expensive analysis.
 //
-// NOTE: Since this pass will directly pack LDS (assume large LDS) into a struct
-// type which would cause allocating huge memory for struct instance within
-// every kernel. Hence, before running this pass, it is advisable to run the
-// pass "amdgpu-replace-lds-use-with-pointer" which will replace LDS uses within
-// non-kernel functions by pointers and thereby minimizes the unnecessary per
-// kernel allocation of LDS memory.
-//
 //===----------------------------------------------------------------------===//
 
 #include "AMDGPU.h"
@@ -62,6 +55,20 @@ static cl::opt<bool> SuperAlignLDSGlobals(
 
 namespace {
 
+SmallPtrSet<GlobalValue *, 32> getUsedList(Module &M) {
+  SmallPtrSet<GlobalValue *, 32> UsedList;
+
+  SmallVector<GlobalValue *, 32> TmpVec;
+  collectUsedGlobalVariables(M, TmpVec, true);
+  UsedList.insert(TmpVec.begin(), TmpVec.end());
+
+  TmpVec.clear();
+  collectUsedGlobalVariables(M, TmpVec, false);
+  UsedList.insert(TmpVec.begin(), TmpVec.end());
+
+  return UsedList;
+}
+
 class AMDGPULowerModuleLDS : public ModulePass {
 
   static void removeFromUsedList(Module &M, StringRef Name,
@@ -105,11 +112,9 @@ class AMDGPULowerModuleLDS : public ModulePass {
   removeFromUsedLists(Module &M,
                       const std::vector<GlobalVariable *> &LocalVars) {
     SmallPtrSet<Constant *, 32> LocalVarsSet;
-    for (size_t I = 0; I < LocalVars.size(); I++) {
-      if (Constant *C = dyn_cast<Constant>(LocalVars[I]->stripPointerCasts())) {
+    for (GlobalVariable *LocalVar : LocalVars)
+      if (Constant *C = dyn_cast<Constant>(LocalVar->stripPointerCasts()))
         LocalVarsSet.insert(C);
-      }
-    }
     removeFromUsedList(M, "llvm.used", LocalVarsSet);
     removeFromUsedList(M, "llvm.compiler.used", LocalVarsSet);
   }
@@ -158,9 +163,9 @@ public:
   }
 
   bool runOnModule(Module &M) override {
-    UsedList = AMDGPU::getUsedList(M);
-
-    bool Changed = processUsedLDS(M);
+    UsedList = getUsedList(M);
+    bool Changed = superAlignLDSGlobals(M);
+    Changed |= processUsedLDS(M);
 
     for (Function &F : M.functions()) {
       if (F.isDeclaration())
@@ -177,6 +182,50 @@ public:
   }
 
 private:
+  // Increase the alignment of LDS globals if necessary to maximise the chance
+  // that we can use aligned LDS instructions to access them.
+  static bool superAlignLDSGlobals(Module &M) {
+    const DataLayout &DL = M.getDataLayout();
+    bool Changed = false;
+    if (!SuperAlignLDSGlobals) {
+      return Changed;
+    }
+
+    for (auto &GV : M.globals()) {
+      if (GV.getType()->getPointerAddressSpace() != AMDGPUAS::LOCAL_ADDRESS) {
+        // Only changing alignment of LDS variables
+        continue;
+      }
+      if (!GV.hasInitializer()) {
+        // cuda/hip extern __shared__ variable, leave alignment alone
+        continue;
+      }
+
+      Align Alignment = AMDGPU::getAlign(DL, &GV);
+      TypeSize GVSize = DL.getTypeAllocSize(GV.getValueType());
+
+      if (GVSize > 8) {
+        // We might want to use a b96 or b128 load/store
+        Alignment = std::max(Alignment, Align(16));
+      } else if (GVSize > 4) {
+        // We might want to use a b64 load/store
+        Alignment = std::max(Alignment, Align(8));
+      } else if (GVSize > 2) {
+        // We might want to use a b32 load/store
+        Alignment = std::max(Alignment, Align(4));
+      } else if (GVSize > 1) {
+        // We might want to use a b16 load/store
+        Alignment = std::max(Alignment, Align(2));
+      }
+
+      if (Alignment != AMDGPU::getAlign(DL, &GV)) {
+        Changed = true;
+        GV.setAlignment(Alignment);
+      }
+    }
+    return Changed;
+  }
+
   bool processUsedLDS(Module &M, Function *F = nullptr) {
     LLVMContext &Ctx = M.getContext();
     const DataLayout &DL = M.getDataLayout();
@@ -190,31 +239,6 @@ private:
       return false;
     }
 
-    // Increase the alignment of LDS globals if necessary to maximise the chance
-    // that we can use aligned LDS instructions to access them.
-    if (SuperAlignLDSGlobals) {
-      for (auto *GV : FoundLocalVars) {
-        Align Alignment = AMDGPU::getAlign(DL, GV);
-        TypeSize GVSize = DL.getTypeAllocSize(GV->getValueType());
-
-        if (GVSize > 8) {
-          // We might want to use a b96 or b128 load/store
-          Alignment = std::max(Alignment, Align(16));
-        } else if (GVSize > 4) {
-          // We might want to use a b64 load/store
-          Alignment = std::max(Alignment, Align(8));
-        } else if (GVSize > 2) {
-          // We might want to use a b32 load/store
-          Alignment = std::max(Alignment, Align(4));
-        } else if (GVSize > 1) {
-          // We might want to use a b16 load/store
-          Alignment = std::max(Alignment, Align(2));
-        }
-
-        GV->setAlignment(Alignment);
-      }
-    }
-
     SmallVector<OptimizedStructLayoutField, 8> LayoutFields;
     LayoutFields.reserve(FoundLocalVars.size());
     for (GlobalVariable *GV : FoundLocalVars) {
@@ -343,20 +367,14 @@ private:
       refineUsesAlignmentAndAA(GEP, A, DL, AliasScope, NoAlias);
     }
 
-    // Mark kernels with asm that reads the address of the allocated structure
-    // This is not necessary for lowering. This lets other passes, specifically
-    // PromoteAlloca, accurately calculate how much LDS will be used by the
-    // kernel after lowering.
+    // This ensures the variable is allocated when called functions access it.
+    // It also lets other passes, specifically PromoteAlloca, accurately
+    // calculate how much LDS will be used by the kernel after lowering.
     if (!F) {
       IRBuilder<> Builder(Ctx);
-      SmallPtrSet<Function *, 32> Kernels;
       for (Function &Func : M.functions()) {
-        if (Func.isDeclaration())
-          continue;
-
-        if (AMDGPU::isKernelCC(&Func) && !Kernels.contains(&Func)) {
+        if (!Func.isDeclaration() && AMDGPU::isKernelCC(&Func)) {
           markUsedByKernel(Builder, &Func, SGV);
-          Kernels.insert(&Func);
         }
       }
     }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMachineCFGStructurizer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMachineCFGStructurizer.cpp
index 5d4b007f11e6..4e2f98d2a5db 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMachineCFGStructurizer.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMachineCFGStructurizer.cpp
@@ -2786,12 +2786,8 @@ AMDGPUMachineCFGStructurizer::initializeSelectRegisters(MRT *MRT, unsigned Selec
     // Fixme: Move linearization creation to the original spot
     createLinearizedRegion(Region, SelectOut);
 
-    for (auto CI = Region->getChildren()->begin(),
-              CE = Region->getChildren()->end();
-         CI != CE; ++CI) {
-      InnerSelectOut =
-          initializeSelectRegisters((*CI), InnerSelectOut, MRI, TII);
-    }
+    for (auto *CI : *Region->getChildren())
+      InnerSelectOut = initializeSelectRegisters(CI, InnerSelectOut, MRI, TII);
     MRT->setBBSelectRegIn(InnerSelectOut);
     return InnerSelectOut;
   } else {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPerfHintAnalysis.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPerfHintAnalysis.cpp
index 2aa02299ecdc..8ad344816ad2 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPerfHintAnalysis.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPerfHintAnalysis.cpp
@@ -119,31 +119,27 @@ private:
   bool isConstantAddr(const Value *V) const;
 };
 
-static const Value *getMemoryInstrPtr(const Instruction *Inst) {
-  if (auto LI = dyn_cast<LoadInst>(Inst)) {
-    return LI->getPointerOperand();
-  }
-  if (auto SI = dyn_cast<StoreInst>(Inst)) {
-    return SI->getPointerOperand();
-  }
-  if (auto AI = dyn_cast<AtomicCmpXchgInst>(Inst)) {
-    return AI->getPointerOperand();
-  }
-  if (auto AI = dyn_cast<AtomicRMWInst>(Inst)) {
-    return AI->getPointerOperand();
-  }
-  if (auto MI = dyn_cast<AnyMemIntrinsic>(Inst)) {
-    return MI->getRawDest();
-  }
-
-  return nullptr;
+static std::pair<const Value *, const Type *> getMemoryInstrPtrAndType(
+    const Instruction *Inst) {
+  if (auto LI = dyn_cast<LoadInst>(Inst))
+    return {LI->getPointerOperand(), LI->getType()};
+  if (auto SI = dyn_cast<StoreInst>(Inst))
+    return {SI->getPointerOperand(), SI->getValueOperand()->getType()};
+  if (auto AI = dyn_cast<AtomicCmpXchgInst>(Inst))
+    return {AI->getPointerOperand(), AI->getCompareOperand()->getType()};
+  if (auto AI = dyn_cast<AtomicRMWInst>(Inst))
+    return {AI->getPointerOperand(), AI->getValOperand()->getType()};
+  if (auto MI = dyn_cast<AnyMemIntrinsic>(Inst))
+    return {MI->getRawDest(), Type::getInt8Ty(MI->getContext())};
+
+  return {nullptr, nullptr};
 }
 
 bool AMDGPUPerfHint::isIndirectAccess(const Instruction *Inst) const {
   LLVM_DEBUG(dbgs() << "[isIndirectAccess] " << *Inst << '\n');
   SmallSet<const Value *, 32> WorkSet;
   SmallSet<const Value *, 32> Visited;
-  if (const Value *MO = getMemoryInstrPtr(Inst)) {
+  if (const Value *MO = getMemoryInstrPtrAndType(Inst).first) {
     if (isGlobalAddr(MO))
       WorkSet.insert(MO);
   }
@@ -209,10 +205,8 @@ AMDGPUPerfHintAnalysis::FuncInfo *AMDGPUPerfHint::visit(const Function &F) {
   for (auto &B : F) {
     LastAccess = MemAccessInfo();
     for (auto &I : B) {
-      if (const Value *Ptr = getMemoryInstrPtr(&I)) {
-        unsigned Size = divideCeil(
-            Ptr->getType()->getPointerElementType()->getPrimitiveSizeInBits(),
-            32);
+      if (const Type *Ty = getMemoryInstrPtrAndType(&I).second) {
+        unsigned Size = divideCeil(Ty->getPrimitiveSizeInBits(), 32);
         if (isIndirectAccess(&I))
           FI.IAMInstCost += Size;
         if (isLargeStride(&I))
@@ -326,7 +320,7 @@ bool AMDGPUPerfHint::isLargeStride(const Instruction *Inst) {
 AMDGPUPerfHint::MemAccessInfo
 AMDGPUPerfHint::makeMemAccessInfo(Instruction *Inst) const {
   MemAccessInfo MAI;
-  const Value *MO = getMemoryInstrPtr(Inst);
+  const Value *MO = getMemoryInstrPtrAndType(Inst).first;
 
   LLVM_DEBUG(dbgs() << "[isLargeStride] MO: " << *MO << '\n');
   // Do not treat local-addr memory access as large stride.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
index 3ec5dd7e0eff..f9a9fe403ff6 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
@@ -939,7 +939,7 @@ bool AMDGPUPromoteAllocaImpl::handleAlloca(AllocaInst &I, bool SufficientLDS) {
       GlobalVariable::NotThreadLocal,
       AMDGPUAS::LOCAL_ADDRESS);
   GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
-  GV->setAlignment(MaybeAlign(I.getAlignment()));
+  GV->setAlignment(I.getAlign());
 
   Value *TCntY, *TCntZ;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankCombiner.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankCombiner.cpp
index 12b5830ef930..3ce67a733c10 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankCombiner.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankCombiner.cpp
@@ -16,6 +16,7 @@
 #include "AMDGPURegisterBankInfo.h"
 #include "GCNSubtarget.h"
 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
+#include "SIMachineFunctionInfo.h"
 #include "llvm/CodeGen/GlobalISel/Combiner.h"
 #include "llvm/CodeGen/GlobalISel/CombinerHelper.h"
 #include "llvm/CodeGen/GlobalISel/CombinerInfo.h"
@@ -23,6 +24,7 @@
 #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
 #include "llvm/CodeGen/MachineDominators.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/IR/IntrinsicsAMDGPU.h"
 #include "llvm/Target/TargetMachine.h"
 #define DEBUG_TYPE "amdgpu-regbank-combiner"
 
@@ -36,13 +38,15 @@ protected:
   MachineRegisterInfo &MRI;
   const RegisterBankInfo &RBI;
   const TargetRegisterInfo &TRI;
+  const SIInstrInfo &TII;
   CombinerHelper &Helper;
 
 public:
   AMDGPURegBankCombinerHelper(MachineIRBuilder &B, CombinerHelper &Helper)
       : B(B), MF(B.getMF()), MRI(*B.getMRI()),
         RBI(*MF.getSubtarget().getRegBankInfo()),
-        TRI(*MF.getSubtarget().getRegisterInfo()), Helper(Helper){};
+        TRI(*MF.getSubtarget().getRegisterInfo()),
+        TII(*MF.getSubtarget<GCNSubtarget>().getInstrInfo()), Helper(Helper){};
 
   bool isVgprRegBank(Register Reg);
   Register getAsVgpr(Register Reg);
@@ -63,7 +67,19 @@ public:
                 Register &Val, CstTy &K0, CstTy &K1);
 
   bool matchIntMinMaxToMed3(MachineInstr &MI, Med3MatchInfo &MatchInfo);
+  bool matchFPMinMaxToMed3(MachineInstr &MI, Med3MatchInfo &MatchInfo);
+  bool matchFPMinMaxToClamp(MachineInstr &MI, Register &Reg);
+  bool matchFPMed3ToClamp(MachineInstr &MI, Register &Reg);
   void applyMed3(MachineInstr &MI, Med3MatchInfo &MatchInfo);
+  void applyClamp(MachineInstr &MI, Register &Reg);
+
+private:
+  AMDGPU::SIModeRegisterDefaults getMode();
+  bool getIEEE();
+  bool getDX10Clamp();
+  bool isFminnumIeee(const MachineInstr &MI);
+  bool isFCst(MachineInstr *MI);
+  bool isClampZeroToOne(MachineInstr *K0, MachineInstr *K1);
 };
 
 bool AMDGPURegBankCombinerHelper::isVgprRegBank(Register Reg) {
@@ -98,6 +114,13 @@ AMDGPURegBankCombinerHelper::getMinMaxPair(unsigned Opc) {
   case AMDGPU::G_UMAX:
   case AMDGPU::G_UMIN:
     return {AMDGPU::G_UMIN, AMDGPU::G_UMAX, AMDGPU::G_AMDGPU_UMED3};
+  case AMDGPU::G_FMAXNUM:
+  case AMDGPU::G_FMINNUM:
+    return {AMDGPU::G_FMINNUM, AMDGPU::G_FMAXNUM, AMDGPU::G_AMDGPU_FMED3};
+  case AMDGPU::G_FMAXNUM_IEEE:
+  case AMDGPU::G_FMINNUM_IEEE:
+    return {AMDGPU::G_FMINNUM_IEEE, AMDGPU::G_FMAXNUM_IEEE,
+            AMDGPU::G_AMDGPU_FMED3};
   }
 }
 
@@ -148,6 +171,146 @@ bool AMDGPURegBankCombinerHelper::matchIntMinMaxToMed3(
   return true;
 }
 
+// fmed3(NaN, K0, K1) = min(min(NaN, K0), K1)
+// ieee = true  : min/max(SNaN, K) = QNaN, min/max(QNaN, K) = K
+// ieee = false : min/max(NaN, K) = K
+// clamp(NaN) = dx10_clamp ? 0.0 : NaN
+// Consider values of min(max(Val, K0), K1) and max(min(Val, K1), K0) as input.
+// Other operand commutes (see matchMed) give same result since min and max are
+// commutative.
+
+// Try to replace fp min(max(Val, K0), K1) or max(min(Val, K1), K0), KO<=K1
+// with fmed3(Val, K0, K1) or clamp(Val). Clamp requires K0 = 0.0 and K1 = 1.0.
+// Val = SNaN only for ieee = true
+// fmed3(SNaN, K0, K1) = min(min(SNaN, K0), K1) = min(QNaN, K1) = K1
+// min(max(SNaN, K0), K1) = min(QNaN, K1) = K1
+// max(min(SNaN, K1), K0) = max(K1, K0) = K1
+// Val = NaN,ieee = false or Val = QNaN,ieee = true
+// fmed3(NaN, K0, K1) = min(min(NaN, K0), K1) = min(K0, K1) = K0
+// min(max(NaN, K0), K1) = min(K0, K1) = K0 (can clamp when dx10_clamp = true)
+// max(min(NaN, K1), K0) = max(K1, K0) = K1 != K0
+bool AMDGPURegBankCombinerHelper::matchFPMinMaxToMed3(
+    MachineInstr &MI, Med3MatchInfo &MatchInfo) {
+  Register Dst = MI.getOperand(0).getReg();
+  LLT Ty = MRI.getType(Dst);
+  if (Ty != LLT::scalar(16) && Ty != LLT::scalar(32))
+    return false;
+
+  auto OpcodeTriple = getMinMaxPair(MI.getOpcode());
+
+  Register Val;
+  Optional<FPValueAndVReg> K0, K1;
+  // Match min(max(Val, K0), K1) or max(min(Val, K1), K0). Then see if K0 <= K1.
+  if (!matchMed<GFCstAndRegMatch>(MI, MRI, OpcodeTriple, Val, K0, K1))
+    return false;
+
+  if (K0->Value > K1->Value)
+    return false;
+
+  // For IEEE=false perform combine only when it's safe to assume that there are
+  // no NaN inputs. Most often MI is marked with nnan fast math flag.
+  // For IEEE=true consider NaN inputs. fmed3(NaN, K0, K1) is equivalent to
+  // min(min(NaN, K0), K1). Safe to fold for min(max(Val, K0), K1) since inner
+  // nodes(max/min) have same behavior when one input is NaN and other isn't.
+  // Don't consider max(min(SNaN, K1), K0) since there is no isKnownNeverQNaN,
+  // also post-legalizer inputs to min/max are fcanonicalized (never SNaN).
+  if ((getIEEE() && isFminnumIeee(MI)) || isKnownNeverNaN(Dst, MRI)) {
+    // Don't fold single use constant that can't be inlined.
+    if ((!MRI.hasOneNonDBGUse(K0->VReg) || TII.isInlineConstant(K0->Value)) &&
+        (!MRI.hasOneNonDBGUse(K1->VReg) || TII.isInlineConstant(K1->Value))) {
+      MatchInfo = {OpcodeTriple.Med, Val, K0->VReg, K1->VReg};
+      return true;
+    }
+  }
+
+  return false;
+}
+
+bool AMDGPURegBankCombinerHelper::matchFPMinMaxToClamp(MachineInstr &MI,
+                                                       Register &Reg) {
+  // Clamp is available on all types after regbankselect (f16, f32, f64, v2f16).
+  auto OpcodeTriple = getMinMaxPair(MI.getOpcode());
+  Register Val;
+  Optional<FPValueAndVReg> K0, K1;
+  // Match min(max(Val, K0), K1) or max(min(Val, K1), K0).
+  if (!matchMed<GFCstOrSplatGFCstMatch>(MI, MRI, OpcodeTriple, Val, K0, K1))
+    return false;
+
+  if (!K0->Value.isExactlyValue(0.0) || !K1->Value.isExactlyValue(1.0))
+    return false;
+
+  // For IEEE=false perform combine only when it's safe to assume that there are
+  // no NaN inputs. Most often MI is marked with nnan fast math flag.
+  // For IEEE=true consider NaN inputs. Only min(max(QNaN, 0.0), 1.0) evaluates
+  // to 0.0 requires dx10_clamp = true.
+  if ((getIEEE() && getDX10Clamp() && isFminnumIeee(MI) &&
+       isKnownNeverSNaN(Val, MRI)) ||
+      isKnownNeverNaN(MI.getOperand(0).getReg(), MRI)) {
+    Reg = Val;
+    return true;
+  }
+
+  return false;
+}
+
+// Replacing fmed3(NaN, 0.0, 1.0) with clamp. Requires dx10_clamp = true.
+// Val = SNaN only for ieee = true. It is important which operand is NaN.
+// min(min(SNaN, 0.0), 1.0) = min(QNaN, 1.0) = 1.0
+// min(min(SNaN, 1.0), 0.0) = min(QNaN, 0.0) = 0.0
+// min(min(0.0, 1.0), SNaN) = min(0.0, SNaN) = QNaN
+// Val = NaN,ieee = false or Val = QNaN,ieee = true
+// min(min(NaN, 0.0), 1.0) = min(0.0, 1.0) = 0.0
+// min(min(NaN, 1.0), 0.0) = min(1.0, 0.0) = 0.0
+// min(min(0.0, 1.0), NaN) = min(0.0, NaN) = 0.0
+bool AMDGPURegBankCombinerHelper::matchFPMed3ToClamp(MachineInstr &MI,
+                                                     Register &Reg) {
+  if (MI.getIntrinsicID() != Intrinsic::amdgcn_fmed3)
+    return false;
+
+  // In llvm-ir, clamp is often represented as an intrinsic call to
+  // @llvm.amdgcn.fmed3.f32(%Val, 0.0, 1.0). Check for other operand orders.
+  MachineInstr *Src0 = getDefIgnoringCopies(MI.getOperand(2).getReg(), MRI);
+  MachineInstr *Src1 = getDefIgnoringCopies(MI.getOperand(3).getReg(), MRI);
+  MachineInstr *Src2 = getDefIgnoringCopies(MI.getOperand(4).getReg(), MRI);
+
+  if (isFCst(Src0) && !isFCst(Src1))
+    std::swap(Src0, Src1);
+  if (isFCst(Src1) && !isFCst(Src2))
+    std::swap(Src1, Src2);
+  if (isFCst(Src0) && !isFCst(Src1))
+    std::swap(Src0, Src1);
+  if (!isClampZeroToOne(Src1, Src2))
+    return false;
+
+  Register Val = Src0->getOperand(0).getReg();
+
+  auto isOp3Zero = [&]() {
+    MachineInstr *Op3 = getDefIgnoringCopies(MI.getOperand(4).getReg(), MRI);
+    if (Op3->getOpcode() == TargetOpcode::G_FCONSTANT)
+      return Op3->getOperand(1).getFPImm()->isExactlyValue(0.0);
+    return false;
+  };
+  // For IEEE=false perform combine only when it's safe to assume that there are
+  // no NaN inputs. Most often MI is marked with nnan fast math flag.
+  // For IEEE=true consider NaN inputs. Requires dx10_clamp = true. Safe to fold
+  // when Val could be QNaN. If Val can also be SNaN third input should be 0.0.
+  if (isKnownNeverNaN(MI.getOperand(0).getReg(), MRI) ||
+      (getIEEE() && getDX10Clamp() &&
+       (isKnownNeverSNaN(Val, MRI) || isOp3Zero()))) {
+    Reg = Val;
+    return true;
+  }
+
+  return false;
+}
+
+void AMDGPURegBankCombinerHelper::applyClamp(MachineInstr &MI, Register &Reg) {
+  B.setInstrAndDebugLoc(MI);
+  B.buildInstr(AMDGPU::G_AMDGPU_CLAMP, {MI.getOperand(0)}, {Reg},
+               MI.getFlags());
+  MI.eraseFromParent();
+}
+
 void AMDGPURegBankCombinerHelper::applyMed3(MachineInstr &MI,
                                             Med3MatchInfo &MatchInfo) {
   B.setInstrAndDebugLoc(MI);
@@ -158,6 +321,33 @@ void AMDGPURegBankCombinerHelper::applyMed3(MachineInstr &MI,
   MI.eraseFromParent();
 }
 
+AMDGPU::SIModeRegisterDefaults AMDGPURegBankCombinerHelper::getMode() {
+  return MF.getInfo<SIMachineFunctionInfo>()->getMode();
+}
+
+bool AMDGPURegBankCombinerHelper::getIEEE() { return getMode().IEEE; }
+
+bool AMDGPURegBankCombinerHelper::getDX10Clamp() { return getMode().DX10Clamp; }
+
+bool AMDGPURegBankCombinerHelper::isFminnumIeee(const MachineInstr &MI) {
+  return MI.getOpcode() == AMDGPU::G_FMINNUM_IEEE;
+}
+
+bool AMDGPURegBankCombinerHelper::isFCst(MachineInstr *MI) {
+  return MI->getOpcode() == AMDGPU::G_FCONSTANT;
+}
+
+bool AMDGPURegBankCombinerHelper::isClampZeroToOne(MachineInstr *K0,
+                                                   MachineInstr *K1) {
+  if (isFCst(K0) && isFCst(K1)) {
+    const ConstantFP *KO_FPImm = K0->getOperand(1).getFPImm();
+    const ConstantFP *K1_FPImm = K1->getOperand(1).getFPImm();
+    return (KO_FPImm->isExactlyValue(0.0) && K1_FPImm->isExactlyValue(1.0)) ||
+           (KO_FPImm->isExactlyValue(1.0) && K1_FPImm->isExactlyValue(0.0));
+  }
+  return false;
+}
+
 class AMDGPURegBankCombinerHelperState {
 protected:
   CombinerHelper &Helper;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index 5988403c0a29..c60012bcfe2e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -707,9 +707,6 @@ bool AMDGPURegisterBankInfo::executeInWaterfallLoop(
   iterator_range<MachineBasicBlock::iterator> Range,
   SmallSet<Register, 4> &SGPROperandRegs,
   MachineRegisterInfo &MRI) const {
-  SmallVector<Register, 4> ResultRegs;
-  SmallVector<Register, 4> InitResultRegs;
-  SmallVector<Register, 4> PhiRegs;
 
   // Track use registers which have already been expanded with a readfirstlane
   // sequence. This may have multiple uses if moving a sequence.
@@ -774,15 +771,6 @@ bool AMDGPURegisterBankInfo::executeInWaterfallLoop(
     .addReg(NewExec)
     .addMBB(LoopBB);
 
-  for (auto Result : zip(InitResultRegs, ResultRegs, PhiRegs)) {
-    B.buildInstr(TargetOpcode::G_PHI)
-      .addDef(std::get<2>(Result))
-      .addReg(std::get<0>(Result)) // Initial value / implicit_def
-      .addMBB(&MBB)
-      .addReg(std::get<1>(Result)) // Mid-loop value.
-      .addMBB(LoopBB);
-  }
-
   const DebugLoc &DL = B.getDL();
 
   MachineInstr &FirstInst = *Range.begin();
@@ -1174,18 +1162,25 @@ bool AMDGPURegisterBankInfo::applyMappingLoad(MachineInstr &MI,
       // 96-bit loads are only available for vector loads. We need to split this
       // into a 64-bit part, and 32 (unless we can widen to a 128-bit load).
       if (MMO->getAlign() < Align(16)) {
+        MachineFunction *MF = MI.getParent()->getParent();
+        ApplyRegBankMapping ApplyBank(*this, MRI, DstBank);
+        MachineIRBuilder B(MI, ApplyBank);
+        LegalizerHelper Helper(*MF, ApplyBank, B);
         LLT Part64, Part32;
         std::tie(Part64, Part32) = splitUnequalType(LoadTy, 64);
-        auto Load0 = B.buildLoadFromOffset(Part64, PtrReg, *MMO, 0);
-        auto Load1 = B.buildLoadFromOffset(Part32, PtrReg, *MMO, 8);
-
-        auto Undef = B.buildUndef(LoadTy);
-        auto Ins0 = B.buildInsert(LoadTy, Undef, Load0, 0);
-        B.buildInsert(MI.getOperand(0), Ins0, Load1, 64);
+        if (Helper.reduceLoadStoreWidth(cast<GAnyLoad>(MI), 0, Part64) !=
+            LegalizerHelper::Legalized)
+          return false;
+        return true;
       } else {
         LLT WiderTy = widen96To128(LoadTy);
         auto WideLoad = B.buildLoadFromOffset(WiderTy, PtrReg, *MMO, 0);
-        B.buildExtract(MI.getOperand(0), WideLoad, 0);
+        if (WiderTy.isScalar())
+          B.buildTrunc(MI.getOperand(0), WideLoad);
+        else {
+          B.buildDeleteTrailingVectorElements(MI.getOperand(0).getReg(),
+                                              WideLoad);
+        }
       }
     }
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUReplaceLDSUseWithPointer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUReplaceLDSUseWithPointer.cpp
index d55bf3917e9c..2475b44b42a3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUReplaceLDSUseWithPointer.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUReplaceLDSUseWithPointer.cpp
@@ -87,6 +87,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetOperations.h"
+#include "llvm/Analysis/CallGraph.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DerivedTypes.h"
@@ -110,6 +111,18 @@ using namespace llvm;
 
 namespace {
 
+namespace AMDGPU {
+/// Collect all the instructions where user \p U belongs to. \p U could be
+/// instruction itself or it could be a constant expression which is used within
+/// an instruction. If \p CollectKernelInsts is true, collect instructions only
+/// from kernels, otherwise collect instructions only from non-kernel functions.
+DenseMap<Function *, SmallPtrSet<Instruction *, 8>>
+getFunctionToInstsMap(User *U, bool CollectKernelInsts);
+
+SmallPtrSet<Function *, 8> collectNonKernelAccessorsOfLDS(GlobalVariable *GV);
+
+} // namespace AMDGPU
+
 class ReplaceLDSUseImpl {
   Module &M;
   LLVMContext &Ctx;
@@ -127,7 +140,8 @@ class ReplaceLDSUseImpl {
   // Collect LDS which requires their uses to be replaced by pointer.
   std::vector<GlobalVariable *> collectLDSRequiringPointerReplace() {
     // Collect LDS which requires module lowering.
-    std::vector<GlobalVariable *> LDSGlobals = AMDGPU::findVariablesToLower(M);
+    std::vector<GlobalVariable *> LDSGlobals =
+        llvm::AMDGPU::findVariablesToLower(M);
 
     // Remove LDS which don't qualify for replacement.
     llvm::erase_if(LDSGlobals, [&](GlobalVariable *GV) {
@@ -172,7 +186,7 @@ class ReplaceLDSUseImpl {
         AMDGPUAS::LOCAL_ADDRESS);
 
     LDSPointer->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
-    LDSPointer->setAlignment(AMDGPU::getAlign(DL, LDSPointer));
+    LDSPointer->setAlignment(llvm::AMDGPU::getAlign(DL, LDSPointer));
 
     // Mark that an associated LDS pointer is created for LDS.
     LDSToPointer[GV] = LDSPointer;
@@ -245,10 +259,9 @@ class ReplaceLDSUseImpl {
       auto FunctionToInsts =
           AMDGPU::getFunctionToInstsMap(U, false /*=CollectKernelInsts*/);
 
-      for (auto FI = FunctionToInsts.begin(), FE = FunctionToInsts.end();
-           FI != FE; ++FI) {
-        Function *F = FI->first;
-        auto &Insts = FI->second;
+      for (const auto &FunctionToInst : FunctionToInsts) {
+        Function *F = FunctionToInst.first;
+        auto &Insts = FunctionToInst.second;
         for (auto *I : Insts) {
           // If `U` is a constant expression, then we need to break the
           // associated instruction into a set of separate instructions by
@@ -341,10 +354,9 @@ bool ReplaceLDSUseImpl::replaceLDSUse(GlobalVariable *GV) {
 
   // Traverse through each kernel K, check and if required, initialize the
   // LDS pointer to point to LDS within K.
-  for (auto KI = KernelToCallees.begin(), KE = KernelToCallees.end(); KI != KE;
-       ++KI) {
-    Function *K = KI->first;
-    SmallPtrSet<Function *, 8> Callees = KI->second;
+  for (const auto &KernelToCallee : KernelToCallees) {
+    Function *K = KernelToCallee.first;
+    SmallPtrSet<Function *, 8> Callees = KernelToCallee.second;
 
     // Compute reachable and LDS used callees for kernel K.
     set_intersect(Callees, LDSAccessors);
@@ -378,6 +390,184 @@ bool ReplaceLDSUseImpl::replaceLDSUse(GlobalVariable *GV) {
   return true;
 }
 
+namespace AMDGPU {
+
+// An helper class for collecting all reachable callees for each kernel defined
+// within the module.
+class CollectReachableCallees {
+  Module &M;
+  CallGraph CG;
+  SmallPtrSet<CallGraphNode *, 8> AddressTakenFunctions;
+
+  // Collect all address taken functions within the module.
+  void collectAddressTakenFunctions() {
+    auto *ECNode = CG.getExternalCallingNode();
+
+    for (const auto &GI : *ECNode) {
+      auto *CGN = GI.second;
+      auto *F = CGN->getFunction();
+      if (!F || F->isDeclaration() || llvm::AMDGPU::isKernelCC(F))
+        continue;
+      AddressTakenFunctions.insert(CGN);
+    }
+  }
+
+  // For given kernel, collect all its reachable non-kernel functions.
+  SmallPtrSet<Function *, 8> collectReachableCallees(Function *K) {
+    SmallPtrSet<Function *, 8> ReachableCallees;
+
+    // Call graph node which represents this kernel.
+    auto *KCGN = CG[K];
+
+    // Go through all call graph nodes reachable from the node representing this
+    // kernel, visit all their call sites, if the call site is direct, add
+    // corresponding callee to reachable callee set, if it is indirect, resolve
+    // the indirect call site to potential reachable callees, add them to
+    // reachable callee set, and repeat the process for the newly added
+    // potential callee nodes.
+    //
+    // FIXME: Need to handle bit-casted function pointers.
+    //
+    SmallVector<CallGraphNode *, 8> CGNStack(depth_first(KCGN));
+    SmallPtrSet<CallGraphNode *, 8> VisitedCGNodes;
+    while (!CGNStack.empty()) {
+      auto *CGN = CGNStack.pop_back_val();
+
+      if (!VisitedCGNodes.insert(CGN).second)
+        continue;
+
+      // Ignore call graph node which does not have associated function or
+      // associated function is not a definition.
+      if (!CGN->getFunction() || CGN->getFunction()->isDeclaration())
+        continue;
+
+      for (const auto &GI : *CGN) {
+        auto *RCB = cast<CallBase>(GI.first.getValue());
+        auto *RCGN = GI.second;
+
+        if (auto *DCallee = RCGN->getFunction()) {
+          ReachableCallees.insert(DCallee);
+        } else if (RCB->isIndirectCall()) {
+          auto *RCBFTy = RCB->getFunctionType();
+          for (auto *ACGN : AddressTakenFunctions) {
+            auto *ACallee = ACGN->getFunction();
+            if (ACallee->getFunctionType() == RCBFTy) {
+              ReachableCallees.insert(ACallee);
+              CGNStack.append(df_begin(ACGN), df_end(ACGN));
+            }
+          }
+        }
+      }
+    }
+
+    return ReachableCallees;
+  }
+
+public:
+  explicit CollectReachableCallees(Module &M) : M(M), CG(CallGraph(M)) {
+    // Collect address taken functions.
+    collectAddressTakenFunctions();
+  }
+
+  void collectReachableCallees(
+      DenseMap<Function *, SmallPtrSet<Function *, 8>> &KernelToCallees) {
+    // Collect reachable callee set for each kernel defined in the module.
+    for (Function &F : M.functions()) {
+      if (!llvm::AMDGPU::isKernelCC(&F))
+        continue;
+      Function *K = &F;
+      KernelToCallees[K] = collectReachableCallees(K);
+    }
+  }
+};
+
+/// Collect reachable callees for each kernel defined in the module \p M and
+/// return collected callees at \p KernelToCallees.
+void collectReachableCallees(
+    Module &M,
+    DenseMap<Function *, SmallPtrSet<Function *, 8>> &KernelToCallees) {
+  CollectReachableCallees CRC{M};
+  CRC.collectReachableCallees(KernelToCallees);
+}
+
+/// For the given LDS global \p GV, visit all its users and collect all
+/// non-kernel functions within which \p GV is used and return collected list of
+/// such non-kernel functions.
+SmallPtrSet<Function *, 8> collectNonKernelAccessorsOfLDS(GlobalVariable *GV) {
+  SmallPtrSet<Function *, 8> LDSAccessors;
+  SmallVector<User *, 8> UserStack(GV->users());
+  SmallPtrSet<User *, 8> VisitedUsers;
+
+  while (!UserStack.empty()) {
+    auto *U = UserStack.pop_back_val();
+
+    // `U` is already visited? continue to next one.
+    if (!VisitedUsers.insert(U).second)
+      continue;
+
+    // `U` is a global variable which is initialized with LDS. Ignore LDS.
+    if (isa<GlobalValue>(U))
+      return SmallPtrSet<Function *, 8>();
+
+    // Recursively explore constant users.
+    if (isa<Constant>(U)) {
+      append_range(UserStack, U->users());
+      continue;
+    }
+
+    // `U` should be an instruction, if it belongs to a non-kernel function F,
+    // then collect F.
+    Function *F = cast<Instruction>(U)->getFunction();
+    if (!llvm::AMDGPU::isKernelCC(F))
+      LDSAccessors.insert(F);
+  }
+
+  return LDSAccessors;
+}
+
+DenseMap<Function *, SmallPtrSet<Instruction *, 8>>
+getFunctionToInstsMap(User *U, bool CollectKernelInsts) {
+  DenseMap<Function *, SmallPtrSet<Instruction *, 8>> FunctionToInsts;
+  SmallVector<User *, 8> UserStack;
+  SmallPtrSet<User *, 8> VisitedUsers;
+
+  UserStack.push_back(U);
+
+  while (!UserStack.empty()) {
+    auto *UU = UserStack.pop_back_val();
+
+    if (!VisitedUsers.insert(UU).second)
+      continue;
+
+    if (isa<GlobalValue>(UU))
+      continue;
+
+    if (isa<Constant>(UU)) {
+      append_range(UserStack, UU->users());
+      continue;
+    }
+
+    auto *I = cast<Instruction>(UU);
+    Function *F = I->getFunction();
+    if (CollectKernelInsts) {
+      if (!llvm::AMDGPU::isKernelCC(F)) {
+        continue;
+      }
+    } else {
+      if (llvm::AMDGPU::isKernelCC(F)) {
+        continue;
+      }
+    }
+
+    FunctionToInsts.insert(std::make_pair(F, SmallPtrSet<Instruction *, 8>()));
+    FunctionToInsts[F].insert(I);
+  }
+
+  return FunctionToInsts;
+}
+
+} // namespace AMDGPU
+
 // Entry-point function which interface ReplaceLDSUseImpl with outside of the
 // class.
 bool ReplaceLDSUseImpl::replaceLDSUse() {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
index 0655b4342ba1..cd05797fdbdb 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
@@ -413,21 +413,21 @@ bool GCNSubtarget::zeroesHigh16BitsOfDest(unsigned Opcode) const {
   case AMDGPU::V_MAX_I16_e32:
   case AMDGPU::V_MIN_I16_e64:
   case AMDGPU::V_MIN_I16_e32:
+  case AMDGPU::V_MAD_F16_e64:
+  case AMDGPU::V_MAD_U16_e64:
+  case AMDGPU::V_MAD_I16_e64:
+  case AMDGPU::V_FMA_F16_e64:
+  case AMDGPU::V_DIV_FIXUP_F16_e64:
     // On gfx10, all 16-bit instructions preserve the high bits.
     return getGeneration() <= AMDGPUSubtarget::GFX9;
-  case AMDGPU::V_MAD_F16_e64:
   case AMDGPU::V_MADAK_F16:
   case AMDGPU::V_MADMK_F16:
   case AMDGPU::V_MAC_F16_e64:
   case AMDGPU::V_MAC_F16_e32:
   case AMDGPU::V_FMAMK_F16:
   case AMDGPU::V_FMAAK_F16:
-  case AMDGPU::V_MAD_U16_e64:
-  case AMDGPU::V_MAD_I16_e64:
-  case AMDGPU::V_FMA_F16_e64:
   case AMDGPU::V_FMAC_F16_e64:
   case AMDGPU::V_FMAC_F16_e32:
-  case AMDGPU::V_DIV_FIXUP_F16_e64:
     // In gfx9, the preferred handling of the unused high 16-bits changed. Most
     // instructions maintain the legacy behavior of 0ing. Some instructions
     // changed to preserving the high bits.
@@ -648,9 +648,18 @@ bool AMDGPUSubtarget::makeLIDRangeMetadata(Instruction *I) const {
 }
 
 unsigned AMDGPUSubtarget::getImplicitArgNumBytes(const Function &F) const {
+  assert(AMDGPU::isKernel(F.getCallingConv()));
+
+  // We don't allocate the segment if we know the implicit arguments weren't
+  // used, even if the ABI implies we need them.
+  if (F.hasFnAttribute("amdgpu-no-implicitarg-ptr"))
+    return 0;
+
   if (isMesaKernel(F))
     return 16;
-  return AMDGPU::getIntegerAttribute(F, "amdgpu-implicitarg-num-bytes", 0);
+
+  // Assume all implicit inputs are used by default
+  return AMDGPU::getIntegerAttribute(F, "amdgpu-implicitarg-num-bytes", 56);
 }
 
 uint64_t AMDGPUSubtarget::getExplicitKernArgSize(const Function &F,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index de11676279f2..a2c61f9da8da 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -231,13 +231,6 @@ static cl::opt<bool, true> LateCFGStructurize(
   cl::location(AMDGPUTargetMachine::EnableLateStructurizeCFG),
   cl::Hidden);
 
-static cl::opt<bool, true> EnableAMDGPUFixedFunctionABIOpt(
-  "amdgpu-fixed-function-abi",
-  cl::desc("Enable all implicit function arguments"),
-  cl::location(AMDGPUTargetMachine::EnableFixedFunctionABI),
-  cl::init(false),
-  cl::Hidden);
-
 // Enable lib calls simplifications
 static cl::opt<bool> EnableLibCallSimplify(
   "amdgpu-simplify-libcall",
@@ -505,7 +498,6 @@ AMDGPUTargetMachine::AMDGPUTargetMachine(const Target &T, const Triple &TT,
 
 bool AMDGPUTargetMachine::EnableLateStructurizeCFG = false;
 bool AMDGPUTargetMachine::EnableFunctionCalls = false;
-bool AMDGPUTargetMachine::EnableFixedFunctionABI = false;
 bool AMDGPUTargetMachine::EnableLowerModuleLDS = true;
 
 AMDGPUTargetMachine::~AMDGPUTargetMachine() = default;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
index 0ff2db2a52d9..226646a96953 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
@@ -37,7 +37,6 @@ protected:
 public:
   static bool EnableLateStructurizeCFG;
   static bool EnableFunctionCalls;
-  static bool EnableFixedFunctionABI;
   static bool EnableLowerModuleLDS;
 
   AMDGPUTargetMachine(const Target &T, const Triple &TT, StringRef CPU,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index ecdbdf613a53..09c5eb192e1f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -519,57 +519,6 @@ InstructionCost GCNTTIImpl::getArithmeticInstrCost(
     TTI::OperandValueProperties Opd1PropInfo,
     TTI::OperandValueProperties Opd2PropInfo, ArrayRef<const Value *> Args,
     const Instruction *CxtI) {
-  EVT OrigTy = TLI->getValueType(DL, Ty);
-  if (!OrigTy.isSimple()) {
-    // FIXME: We're having to query the throughput cost so that the basic
-    // implementation tries to generate legalize and scalarization costs. Maybe
-    // we could hoist the scalarization code here?
-    if (CostKind != TTI::TCK_CodeSize)
-      return BaseT::getArithmeticInstrCost(Opcode, Ty, TTI::TCK_RecipThroughput,
-                                           Opd1Info, Opd2Info, Opd1PropInfo,
-                                           Opd2PropInfo, Args, CxtI);
-    // Scalarization
-
-    // Check if any of the operands are vector operands.
-    int ISD = TLI->InstructionOpcodeToISD(Opcode);
-    assert(ISD && "Invalid opcode");
-
-    std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, Ty);
-
-    bool IsFloat = Ty->isFPOrFPVectorTy();
-    // Assume that floating point arithmetic operations cost twice as much as
-    // integer operations.
-    unsigned OpCost = (IsFloat ? 2 : 1);
-
-    if (TLI->isOperationLegalOrPromote(ISD, LT.second)) {
-      // The operation is legal. Assume it costs 1.
-      // TODO: Once we have extract/insert subvector cost we need to use them.
-      return LT.first * OpCost;
-    }
-
-    if (!TLI->isOperationExpand(ISD, LT.second)) {
-      // If the operation is custom lowered, then assume that the code is twice
-      // as expensive.
-      return LT.first * 2 * OpCost;
-    }
-
-    // Else, assume that we need to scalarize this op.
-    // TODO: If one of the types get legalized by splitting, handle this
-    // similarly to what getCastInstrCost() does.
-    if (auto *VTy = dyn_cast<VectorType>(Ty)) {
-      unsigned Num = cast<FixedVectorType>(VTy)->getNumElements();
-      InstructionCost Cost = getArithmeticInstrCost(
-          Opcode, VTy->getScalarType(), CostKind, Opd1Info, Opd2Info,
-          Opd1PropInfo, Opd2PropInfo, Args, CxtI);
-      // Return the cost of multiple scalar invocation plus the cost of
-      // inserting and extracting the values.
-      SmallVector<Type *> Tys(Args.size(), Ty);
-      return getScalarizationOverhead(VTy, Args, Tys) + Num * Cost;
-    }
-
-    // We don't know anything about this scalar instruction.
-    return OpCost;
-  }
 
   // Legalize the type.
   std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, Ty);
@@ -742,40 +691,6 @@ GCNTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
     return BaseT::getIntrinsicInstrCost(ICA, CostKind);
 
   Type *RetTy = ICA.getReturnType();
-  EVT OrigTy = TLI->getValueType(DL, RetTy);
-  if (!OrigTy.isSimple()) {
-    if (CostKind != TTI::TCK_CodeSize)
-      return BaseT::getIntrinsicInstrCost(ICA, CostKind);
-
-    // TODO: Combine these two logic paths.
-    if (ICA.isTypeBasedOnly())
-      return getTypeBasedIntrinsicInstrCost(ICA, CostKind);
-
-    unsigned RetVF =
-        (RetTy->isVectorTy() ? cast<FixedVectorType>(RetTy)->getNumElements()
-                             : 1);
-    const IntrinsicInst *I = ICA.getInst();
-    const SmallVectorImpl<const Value *> &Args = ICA.getArgs();
-    FastMathFlags FMF = ICA.getFlags();
-    // Assume that we need to scalarize this intrinsic.
-
-    // Compute the scalarization overhead based on Args for a vector
-    // intrinsic. A vectorizer will pass a scalar RetTy and VF > 1, while
-    // CostModel will pass a vector RetTy and VF is 1.
-    InstructionCost ScalarizationCost = InstructionCost::getInvalid();
-    if (RetVF > 1) {
-      ScalarizationCost = 0;
-      if (!RetTy->isVoidTy())
-        ScalarizationCost +=
-            getScalarizationOverhead(cast<VectorType>(RetTy), true, false);
-      ScalarizationCost +=
-          getOperandsScalarizationOverhead(Args, ICA.getArgTypes());
-    }
-
-    IntrinsicCostAttributes Attrs(ICA.getID(), RetTy, ICA.getArgTypes(), FMF, I,
-                                  ScalarizationCost);
-    return getIntrinsicInstrCost(Attrs, CostKind);
-  }
 
   // Legalize the type.
   std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, RetTy);
diff --git a/llvm/lib/Target/AMDGPU/AMDILCFGStructurizer.cpp b/llvm/lib/Target/AMDGPU/AMDILCFGStructurizer.cpp
index 712f6dece911..1736c078eb83 100644
--- a/llvm/lib/Target/AMDGPU/AMDILCFGStructurizer.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDILCFGStructurizer.cpp
@@ -173,10 +173,8 @@ protected:
   }
 
   static void PrintLoopinfo(const MachineLoopInfo &LoopInfo) {
-    for (MachineLoop::iterator iter = LoopInfo.begin(),
-         iterEnd = LoopInfo.end(); iter != iterEnd; ++iter) {
-      (*iter)->print(dbgs());
-    }
+    for (const MachineLoop *L : LoopInfo)
+      L->print(dbgs());
   }
 
   // UTILITY FUNCTIONS
@@ -691,9 +689,7 @@ bool AMDGPUCFGStructurizer::prepare() {
   SmallVector<MachineBasicBlock *, DEFAULT_VEC_SLOTS> RetBlks;
 
   // Add an ExitBlk to loop that don't have one
-  for (MachineLoopInfo::iterator It = MLI->begin(),
-       E = MLI->end(); It != E; ++It) {
-    MachineLoop *LoopRep = (*It);
+  for (MachineLoop *LoopRep : *MLI) {
     MBBVector ExitingMBBs;
     LoopRep->getExitingBlocks(ExitingMBBs);
 
@@ -827,14 +823,13 @@ bool AMDGPUCFGStructurizer::run() {
   wrapup(*GraphTraits<MachineFunction *>::nodes_begin(FuncRep));
 
   // Detach retired Block, release memory.
-  for (MBBInfoMap::iterator It = BlockInfoMap.begin(), E = BlockInfoMap.end();
-      It != E; ++It) {
-    if ((*It).second && (*It).second->IsRetired) {
-      assert(((*It).first)->getNumber() != -1);
-      LLVM_DEBUG(dbgs() << "Erase BB" << ((*It).first)->getNumber() << "\n";);
-      (*It).first->eraseFromParent();  //Remove from the parent Function.
+  for (auto &It : BlockInfoMap) {
+    if (It.second && It.second->IsRetired) {
+      assert((It.first)->getNumber() != -1);
+      LLVM_DEBUG(dbgs() << "Erase BB" << (It.first)->getNumber() << "\n";);
+      It.first->eraseFromParent(); // Remove from the parent Function.
     }
-    delete (*It).second;
+    delete It.second;
   }
   BlockInfoMap.clear();
   LLInfoMap.clear();
diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index 4acd77a9d5d2..2bb59086f391 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -246,8 +246,12 @@ public:
     return isRegKind() && !hasModifiers();
   }
 
+  bool isRegOrInline(unsigned RCID, MVT type) const {
+    return isRegClass(RCID) || isInlinableImm(type);
+  }
+
   bool isRegOrImmWithInputMods(unsigned RCID, MVT type) const {
-    return isRegClass(RCID) || isInlinableImm(type) || isLiteralImm(type);
+    return isRegOrInline(RCID, type) || isLiteralImm(type);
   }
 
   bool isRegOrImmWithInt16InputMods() const {
@@ -372,7 +376,7 @@ public:
   bool isInlineValue() const;
 
   bool isRegOrInlineNoMods(unsigned RCID, MVT type) const {
-    return (isRegClass(RCID) || isInlinableImm(type)) && !hasModifiers();
+    return isRegOrInline(RCID, type) && !hasModifiers();
   }
 
   bool isSCSrcB16() const {
diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td
index d3644db7cf8b..a535c8cc0918 100644
--- a/llvm/lib/Target/AMDGPU/BUFInstructions.td
+++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td
@@ -6,11 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-def MUBUFAddr64 : ComplexPattern<i64, 4, "SelectMUBUFAddr64">;
-def MUBUFOffset : ComplexPattern<i64, 3, "SelectMUBUFOffset">;
+def MUBUFAddr64 : ComplexPattern<iPTR, 4, "SelectMUBUFAddr64">;
+def MUBUFOffset : ComplexPattern<iPTR, 3, "SelectMUBUFOffset">;
 
-def MUBUFScratchOffen : ComplexPattern<i64, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>;
-def MUBUFScratchOffset : ComplexPattern<i64, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>;
+def MUBUFScratchOffen : ComplexPattern<iPTR, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>;
+def MUBUFScratchOffset : ComplexPattern<iPTR, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>;
 
 def BUFAddrKind {
   int Offset = 0;
diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td
index bb0aa648ff90..c7ec5308e6d0 100644
--- a/llvm/lib/Target/AMDGPU/FLATInstructions.td
+++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td
@@ -6,12 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-def FlatOffset : ComplexPattern<i64, 2, "SelectFlatOffset", [], [SDNPWantRoot], -10>;
-def GlobalOffset : ComplexPattern<i64, 2, "SelectGlobalOffset", [], [SDNPWantRoot], -10>;
-def ScratchOffset : ComplexPattern<i32, 2, "SelectScratchOffset", [], [SDNPWantRoot], -10>;
+def FlatOffset : ComplexPattern<iPTR, 2, "SelectFlatOffset", [], [SDNPWantRoot], -10>;
+def GlobalOffset : ComplexPattern<iPTR, 2, "SelectGlobalOffset", [], [SDNPWantRoot], -10>;
+def ScratchOffset : ComplexPattern<iPTR, 2, "SelectScratchOffset", [], [SDNPWantRoot], -10>;
 
-def GlobalSAddr : ComplexPattern<i64, 3, "SelectGlobalSAddr", [], [SDNPWantRoot], -10>;
-def ScratchSAddr : ComplexPattern<i32, 2, "SelectScratchSAddr", [], [SDNPWantRoot], -10>;
+def GlobalSAddr : ComplexPattern<iPTR, 3, "SelectGlobalSAddr", [], [SDNPWantRoot], -10>;
+def ScratchSAddr : ComplexPattern<iPTR, 2, "SelectScratchSAddr", [], [SDNPWantRoot], -10>;
 
 //===----------------------------------------------------------------------===//
 // FLAT classes
diff --git a/llvm/lib/Target/AMDGPU/MCA/AMDGPUCustomBehaviour.cpp b/llvm/lib/Target/AMDGPU/MCA/AMDGPUCustomBehaviour.cpp
index f3f664f7972a..912bcc792e4d 100644
--- a/llvm/lib/Target/AMDGPU/MCA/AMDGPUCustomBehaviour.cpp
+++ b/llvm/lib/Target/AMDGPU/MCA/AMDGPUCustomBehaviour.cpp
@@ -120,8 +120,7 @@ unsigned AMDGPUCustomBehaviour::handleWaitCnt(ArrayRef<InstRef> IssuedInst,
 
   // We will now look at each of the currently executing instructions
   // to find out if this wait instruction still needs to wait.
-  for (auto I = IssuedInst.begin(), E = IssuedInst.end(); I != E; I++) {
-    const InstRef &PrevIR = *I;
+  for (const InstRef &PrevIR : IssuedInst) {
     const Instruction &PrevInst = *PrevIR.getInstruction();
     const unsigned PrevInstIndex = PrevIR.getSourceIndex() % SrcMgr.size();
     const WaitCntInfo &PrevInstWaitInfo = InstrWaitCntInfo[PrevInstIndex];
diff --git a/llvm/lib/Target/AMDGPU/R600ControlFlowFinalizer.cpp b/llvm/lib/Target/AMDGPU/R600ControlFlowFinalizer.cpp
index 29c37c706138..8a48a67b829c 100644
--- a/llvm/lib/Target/AMDGPU/R600ControlFlowFinalizer.cpp
+++ b/llvm/lib/Target/AMDGPU/R600ControlFlowFinalizer.cpp
@@ -440,9 +440,8 @@ private:
     CounterPropagateAddr(*Clause.first, CfCount);
     MachineBasicBlock *BB = Clause.first->getParent();
     BuildMI(BB, DL, TII->get(R600::FETCH_CLAUSE)).addImm(CfCount);
-    for (unsigned i = 0, e = Clause.second.size(); i < e; ++i) {
-      BB->splice(InsertPos, BB, Clause.second[i]);
-    }
+    for (MachineInstr *MI : Clause.second)
+      BB->splice(InsertPos, BB, MI);
     CfCount += 2 * Clause.second.size();
   }
 
@@ -452,9 +451,8 @@ private:
     CounterPropagateAddr(*Clause.first, CfCount);
     MachineBasicBlock *BB = Clause.first->getParent();
     BuildMI(BB, DL, TII->get(R600::ALU_CLAUSE)).addImm(CfCount);
-    for (unsigned i = 0, e = Clause.second.size(); i < e; ++i) {
-      BB->splice(InsertPos, BB, Clause.second[i]);
-    }
+    for (MachineInstr *MI : Clause.second)
+      BB->splice(InsertPos, BB, MI);
     CfCount += Clause.second.size();
   }
 
@@ -635,10 +633,10 @@ public:
             CfCount++;
           }
           MI->eraseFromParent();
-          for (unsigned i = 0, e = FetchClauses.size(); i < e; i++)
-            EmitFetchClause(I, DL, FetchClauses[i], CfCount);
-          for (unsigned i = 0, e = AluClauses.size(); i < e; i++)
-            EmitALUClause(I, DL, AluClauses[i], CfCount);
+          for (ClauseFile &CF : FetchClauses)
+            EmitFetchClause(I, DL, CF, CfCount);
+          for (ClauseFile &CF : AluClauses)
+            EmitALUClause(I, DL, CF, CfCount);
           break;
         }
         default:
@@ -649,8 +647,7 @@ public:
           break;
         }
       }
-      for (unsigned i = 0, e = ToPopAfter.size(); i < e; ++i) {
-        MachineInstr *Alu = ToPopAfter[i];
+      for (MachineInstr *Alu : ToPopAfter) {
         BuildMI(MBB, Alu, MBB.findDebugLoc((MachineBasicBlock::iterator)Alu),
             TII->get(R600::CF_ALU_POP_AFTER))
             .addImm(Alu->getOperand(0).getImm())
diff --git a/llvm/lib/Target/AMDGPU/R600InstrInfo.cpp b/llvm/lib/Target/AMDGPU/R600InstrInfo.cpp
index a7ebf72315cb..aec8b1ae4837 100644
--- a/llvm/lib/Target/AMDGPU/R600InstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/R600InstrInfo.cpp
@@ -268,17 +268,15 @@ R600InstrInfo::getSrcs(MachineInstr &MI) const {
       {R600::OpName::src1_W, R600::OpName::src1_sel_W},
     };
 
-    for (unsigned j = 0; j < 8; j++) {
-      MachineOperand &MO =
-          MI.getOperand(getOperandIdx(MI.getOpcode(), OpTable[j][0]));
+    for (const auto &Op : OpTable) {
+      MachineOperand &MO = MI.getOperand(getOperandIdx(MI.getOpcode(), Op[0]));
       Register Reg = MO.getReg();
       if (Reg == R600::ALU_CONST) {
         MachineOperand &Sel =
-            MI.getOperand(getOperandIdx(MI.getOpcode(), OpTable[j][1]));
+            MI.getOperand(getOperandIdx(MI.getOpcode(), Op[1]));
         Result.push_back(std::make_pair(&MO, Sel.getImm()));
         continue;
       }
-
     }
     return Result;
   }
@@ -289,15 +287,14 @@ R600InstrInfo::getSrcs(MachineInstr &MI) const {
     {R600::OpName::src2, R600::OpName::src2_sel},
   };
 
-  for (unsigned j = 0; j < 3; j++) {
-    int SrcIdx = getOperandIdx(MI.getOpcode(), OpTable[j][0]);
+  for (const auto &Op : OpTable) {
+    int SrcIdx = getOperandIdx(MI.getOpcode(), Op[0]);
     if (SrcIdx < 0)
       break;
     MachineOperand &MO = MI.getOperand(SrcIdx);
     Register Reg = MO.getReg();
     if (Reg == R600::ALU_CONST) {
-      MachineOperand &Sel =
-          MI.getOperand(getOperandIdx(MI.getOpcode(), OpTable[j][1]));
+      MachineOperand &Sel = MI.getOperand(getOperandIdx(MI.getOpcode(), Op[1]));
       Result.push_back(std::make_pair(&MO, Sel.getImm()));
       continue;
     }
@@ -521,12 +518,11 @@ R600InstrInfo::fitsReadPortLimitations(const std::vector<MachineInstr *> &IG,
   ValidSwizzle.clear();
   unsigned ConstCount;
   BankSwizzle TransBS = ALU_VEC_012_SCL_210;
-  for (unsigned i = 0, e = IG.size(); i < e; ++i) {
-    IGSrcs.push_back(ExtractSrcs(*IG[i], PV, ConstCount));
-    unsigned Op = getOperandIdx(IG[i]->getOpcode(),
-        R600::OpName::bank_swizzle);
-    ValidSwizzle.push_back( (R600InstrInfo::BankSwizzle)
-        IG[i]->getOperand(Op).getImm());
+  for (MachineInstr *MI : IG) {
+    IGSrcs.push_back(ExtractSrcs(*MI, PV, ConstCount));
+    unsigned Op = getOperandIdx(MI->getOpcode(), R600::OpName::bank_swizzle);
+    ValidSwizzle.push_back(
+        (R600InstrInfo::BankSwizzle)MI->getOperand(Op).getImm());
   }
   std::vector<std::pair<int, unsigned>> TransOps;
   if (!isLastAluTrans)
@@ -542,8 +538,7 @@ R600InstrInfo::fitsReadPortLimitations(const std::vector<MachineInstr *> &IG,
     ALU_VEC_120_SCL_212,
     ALU_VEC_102_SCL_221
   };
-  for (unsigned i = 0; i < 4; i++) {
-    TransBS = TransSwz[i];
+  for (R600InstrInfo::BankSwizzle TransBS : TransSwz) {
     if (!isConstCompatible(TransBS, TransOps, ConstCount))
       continue;
     bool Result = FindSwizzleForVectorSlot(IGSrcs, ValidSwizzle, TransOps,
@@ -562,9 +557,9 @@ R600InstrInfo::fitsConstReadLimitations(const std::vector<unsigned> &Consts)
     const {
   assert (Consts.size() <= 12 && "Too many operands in instructions group");
   unsigned Pair1 = 0, Pair2 = 0;
-  for (unsigned i = 0, n = Consts.size(); i < n; ++i) {
-    unsigned ReadConstHalf = Consts[i] & 2;
-    unsigned ReadConstIndex = Consts[i] & (~3);
+  for (unsigned Const : Consts) {
+    unsigned ReadConstHalf = Const & 2;
+    unsigned ReadConstIndex = Const & (~3);
     unsigned ReadHalfConst = ReadConstIndex | ReadConstHalf;
     if (!Pair1) {
       Pair1 = ReadHalfConst;
@@ -587,12 +582,11 @@ R600InstrInfo::fitsConstReadLimitations(const std::vector<MachineInstr *> &MIs)
     const {
   std::vector<unsigned> Consts;
   SmallSet<int64_t, 4> Literals;
-  for (unsigned i = 0, n = MIs.size(); i < n; i++) {
-    MachineInstr &MI = *MIs[i];
-    if (!isALUInstr(MI.getOpcode()))
+  for (MachineInstr *MI : MIs) {
+    if (!isALUInstr(MI->getOpcode()))
       continue;
 
-    for (const auto &Src : getSrcs(MI)) {
+    for (const auto &Src : getSrcs(*MI)) {
       if (Src.first->getReg() == R600::ALU_LITERAL_X)
         Literals.insert(Src.second);
       if (Literals.size() > 4)
@@ -1330,11 +1324,11 @@ MachineInstr *R600InstrInfo::buildSlotOfVectorInstruction(
   MIB->getOperand(getOperandIdx(Opcode, R600::OpName::pred_sel))
       .setReg(MO.getReg());
 
-  for (unsigned i = 0; i < 14; i++) {
+  for (unsigned Operand : Operands) {
     MachineOperand &MO = MI->getOperand(
-        getOperandIdx(MI->getOpcode(), getSlotedOps(Operands[i], Slot)));
+        getOperandIdx(MI->getOpcode(), getSlotedOps(Operand, Slot)));
     assert (MO.isImm());
-    setImmOperand(*MIB, Operands[i], MO.getImm());
+    setImmOperand(*MIB, Operand, MO.getImm());
   }
   MIB->getOperand(20).setImm(0);
   return MIB;
diff --git a/llvm/lib/Target/AMDGPU/R600MachineScheduler.cpp b/llvm/lib/Target/AMDGPU/R600MachineScheduler.cpp
index 6aee2f591b56..d26879ed8d60 100644
--- a/llvm/lib/Target/AMDGPU/R600MachineScheduler.cpp
+++ b/llvm/lib/Target/AMDGPU/R600MachineScheduler.cpp
@@ -328,9 +328,9 @@ SUnit *R600SchedStrategy::PopInst(std::vector<SUnit *> &Q, bool AnyALU) {
 
 void R600SchedStrategy::LoadAlu() {
   std::vector<SUnit *> &QSrc = Pending[IDAlu];
-  for (unsigned i = 0, e = QSrc.size(); i < e; ++i) {
-    AluKind AK = getAluKind(QSrc[i]);
-    AvailableAlus[AK].push_back(QSrc[i]);
+  for (SUnit *SU : QSrc) {
+    AluKind AK = getAluKind(SU);
+    AvailableAlus[AK].push_back(SU);
   }
   QSrc.clear();
 }
diff --git a/llvm/lib/Target/AMDGPU/R600OpenCLImageTypeLoweringPass.cpp b/llvm/lib/Target/AMDGPU/R600OpenCLImageTypeLoweringPass.cpp
index ac6a3581e255..aa156190b7ae 100644
--- a/llvm/lib/Target/AMDGPU/R600OpenCLImageTypeLoweringPass.cpp
+++ b/llvm/lib/Target/AMDGPU/R600OpenCLImageTypeLoweringPass.cpp
@@ -307,8 +307,8 @@ class R600OpenCLImageTypeLoweringPass : public ModulePass {
     // Build new MDNode.
     SmallVector<Metadata *, 6> KernelMDArgs;
     KernelMDArgs.push_back(ConstantAsMetadata::get(NewF));
-    for (unsigned i = 0; i < NumKernelArgMDNodes; ++i)
-      KernelMDArgs.push_back(MDNode::get(*Context, NewArgMDs.ArgVector[i]));
+    for (const MDVector &MDV : NewArgMDs.ArgVector)
+      KernelMDArgs.push_back(MDNode::get(*Context, MDV));
     MDNode *NewMDNode = MDNode::get(*Context, KernelMDArgs);
 
     return std::make_tuple(NewF, NewMDNode);
diff --git a/llvm/lib/Target/AMDGPU/R600OptimizeVectorRegisters.cpp b/llvm/lib/Target/AMDGPU/R600OptimizeVectorRegisters.cpp
index 72cf48c04e7f..795bc898a7bf 100644
--- a/llvm/lib/Target/AMDGPU/R600OptimizeVectorRegisters.cpp
+++ b/llvm/lib/Target/AMDGPU/R600OptimizeVectorRegisters.cpp
@@ -150,19 +150,18 @@ bool R600VectorRegMerger::tryMergeVector(const RegSeqInfo *Untouched,
     RegSeqInfo *ToMerge, std::vector< std::pair<unsigned, unsigned>> &Remap)
     const {
   unsigned CurrentUndexIdx = 0;
-  for (DenseMap<Register, unsigned>::iterator It = ToMerge->RegToChan.begin(),
-      E = ToMerge->RegToChan.end(); It != E; ++It) {
+  for (auto &It : ToMerge->RegToChan) {
     DenseMap<Register, unsigned>::const_iterator PosInUntouched =
-        Untouched->RegToChan.find((*It).first);
+        Untouched->RegToChan.find(It.first);
     if (PosInUntouched != Untouched->RegToChan.end()) {
-      Remap.push_back(std::pair<unsigned, unsigned>
-          ((*It).second, (*PosInUntouched).second));
+      Remap.push_back(
+          std::pair<unsigned, unsigned>(It.second, (*PosInUntouched).second));
       continue;
     }
     if (CurrentUndexIdx >= Untouched->UndefReg.size())
       return false;
-    Remap.push_back(std::pair<unsigned, unsigned>
-        ((*It).second, Untouched->UndefReg[CurrentUndexIdx++]));
+    Remap.push_back(std::pair<unsigned, unsigned>(
+        It.second, Untouched->UndefReg[CurrentUndexIdx++]));
   }
 
   return true;
@@ -172,9 +171,9 @@ static
 unsigned getReassignedChan(
     const std::vector<std::pair<unsigned, unsigned>> &RemapChan,
     unsigned Chan) {
-  for (unsigned j = 0, je = RemapChan.size(); j < je; j++) {
-    if (RemapChan[j].first == Chan)
-      return RemapChan[j].second;
+  for (const auto &J : RemapChan) {
+    if (J.first == Chan)
+      return J.second;
   }
   llvm_unreachable("Chan wasn't reassigned");
 }
@@ -190,11 +189,10 @@ MachineInstr *R600VectorRegMerger::RebuildVector(
   Register SrcVec = BaseRSI->Instr->getOperand(0).getReg();
   DenseMap<Register, unsigned> UpdatedRegToChan = BaseRSI->RegToChan;
   std::vector<Register> UpdatedUndef = BaseRSI->UndefReg;
-  for (DenseMap<Register, unsigned>::iterator It = RSI->RegToChan.begin(),
-      E = RSI->RegToChan.end(); It != E; ++It) {
+  for (const auto &It : RSI->RegToChan) {
     Register DstReg = MRI->createVirtualRegister(&R600::R600_Reg128RegClass);
-    unsigned SubReg = (*It).first;
-    unsigned Swizzle = (*It).second;
+    unsigned SubReg = It.first;
+    unsigned Swizzle = It.second;
     unsigned Chan = getReassignedChan(RemapChan, Swizzle);
 
     MachineInstr *Tmp = BuildMI(MBB, Pos, DL, TII->get(R600::INSERT_SUBREG),
@@ -234,14 +232,12 @@ MachineInstr *R600VectorRegMerger::RebuildVector(
 }
 
 void R600VectorRegMerger::RemoveMI(MachineInstr *MI) {
-  for (InstructionSetMap::iterator It = PreviousRegSeqByReg.begin(),
-      E = PreviousRegSeqByReg.end(); It != E; ++It) {
-    std::vector<MachineInstr *> &MIs = (*It).second;
+  for (auto &It : PreviousRegSeqByReg) {
+    std::vector<MachineInstr *> &MIs = It.second;
     MIs.erase(llvm::find(MIs, MI), MIs.end());
   }
-  for (InstructionSetMap::iterator It = PreviousRegSeqByUndefCount.begin(),
-      E = PreviousRegSeqByUndefCount.end(); It != E; ++It) {
-    std::vector<MachineInstr *> &MIs = (*It).second;
+  for (auto &It : PreviousRegSeqByUndefCount) {
+    std::vector<MachineInstr *> &MIs = It.second;
     MIs.erase(llvm::find(MIs, MI), MIs.end());
   }
 }
@@ -255,9 +251,9 @@ void R600VectorRegMerger::SwizzleInput(MachineInstr &MI,
     Offset = 3;
   for (unsigned i = 0; i < 4; i++) {
     unsigned Swizzle = MI.getOperand(i + Offset).getImm() + 1;
-    for (unsigned j = 0, e = RemapChan.size(); j < e; j++) {
-      if (RemapChan[j].first == Swizzle) {
-        MI.getOperand(i + Offset).setImm(RemapChan[j].second - 1);
+    for (const auto &J : RemapChan) {
+      if (J.first == Swizzle) {
+        MI.getOperand(i + Offset).setImm(J.second - 1);
         break;
       }
     }
diff --git a/llvm/lib/Target/AMDGPU/R600Packetizer.cpp b/llvm/lib/Target/AMDGPU/R600Packetizer.cpp
index beb0aad86e89..fbe2a1cd9fba 100644
--- a/llvm/lib/Target/AMDGPU/R600Packetizer.cpp
+++ b/llvm/lib/Target/AMDGPU/R600Packetizer.cpp
@@ -127,8 +127,8 @@ private:
       R600::OpName::src1,
       R600::OpName::src2
     };
-    for (unsigned i = 0; i < 3; i++) {
-      int OperandIdx = TII->getOperandIdx(MI.getOpcode(), Ops[i]);
+    for (unsigned Op : Ops) {
+      int OperandIdx = TII->getOperandIdx(MI.getOpcode(), Op);
       if (OperandIdx < 0)
         continue;
       Register Src = MI.getOperand(OperandIdx).getReg();
diff --git a/llvm/lib/Target/AMDGPU/R600RegisterInfo.cpp b/llvm/lib/Target/AMDGPU/R600RegisterInfo.cpp
index 99a1a8e9871a..c329bae50f92 100644
--- a/llvm/lib/Target/AMDGPU/R600RegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/R600RegisterInfo.cpp
@@ -54,10 +54,8 @@ BitVector R600RegisterInfo::getReservedRegs(const MachineFunction &MF) const {
   reserveRegisterTuples(Reserved, R600::PRED_SEL_ONE);
   reserveRegisterTuples(Reserved, R600::INDIRECT_BASE_ADDR);
 
-  for (TargetRegisterClass::iterator I = R600::R600_AddrRegClass.begin(),
-                        E = R600::R600_AddrRegClass.end(); I != E; ++I) {
-    reserveRegisterTuples(Reserved, *I);
-  }
+  for (MCPhysReg R : R600::R600_AddrRegClass)
+    reserveRegisterTuples(Reserved, R);
 
   TII->reserveIndirectRegisters(Reserved, MF, *this);
 
diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index 200e00ee5521..1f93284fc7ee 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -1620,7 +1620,7 @@ bool SIFoldOperands::tryFoldRegSequence(MachineInstr &MI) {
   // Erase the REG_SEQUENCE eagerly, unless we followed a chain of COPY users,
   // in which case we can erase them all later in runOnMachineFunction.
   if (MRI->use_nodbg_empty(MI.getOperand(0).getReg()))
-    MI.eraseFromParentAndMarkDBGValuesForRemoval();
+    MI.eraseFromParent();
   return true;
 }
 
@@ -1821,7 +1821,7 @@ bool SIFoldOperands::runOnMachineFunction(MachineFunction &MF) {
       while (MRI->use_nodbg_empty(InstToErase->getOperand(0).getReg())) {
         auto &SrcOp = InstToErase->getOperand(1);
         auto SrcReg = SrcOp.isReg() ? SrcOp.getReg() : Register();
-        InstToErase->eraseFromParentAndMarkDBGValuesForRemoval();
+        InstToErase->eraseFromParent();
         InstToErase = nullptr;
         if (!SrcReg || SrcReg.isPhysical())
           break;
@@ -1831,7 +1831,7 @@ bool SIFoldOperands::runOnMachineFunction(MachineFunction &MF) {
       }
       if (InstToErase && InstToErase->isRegSequence() &&
           MRI->use_nodbg_empty(InstToErase->getOperand(0).getReg()))
-        InstToErase->eraseFromParentAndMarkDBGValuesForRemoval();
+        InstToErase->eraseFromParent();
     }
   }
   return true;
diff --git a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
index 4706c74be721..d4fe74ecb96e 100644
--- a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
@@ -1167,11 +1167,13 @@ void SIFrameLowering::processFunctionBeforeFrameFinalized(
   if (SpillVGPRToAGPR) {
     // To track the spill frame indices handled in this pass.
     BitVector SpillFIs(MFI.getObjectIndexEnd(), false);
+    BitVector NonVGPRSpillFIs(MFI.getObjectIndexEnd(), false);
 
     bool SeenDbgInstr = false;
 
     for (MachineBasicBlock &MBB : MF) {
       for (MachineInstr &MI : llvm::make_early_inc_range(MBB)) {
+        int FrameIndex;
         if (MI.isDebugInstr())
           SeenDbgInstr = true;
 
@@ -1191,10 +1193,18 @@ void SIFrameLowering::processFunctionBeforeFrameFinalized(
             SpillFIs.set(FI);
             continue;
           }
-        }
+        } else if (TII->isStoreToStackSlot(MI, FrameIndex) ||
+                   TII->isLoadFromStackSlot(MI, FrameIndex))
+          NonVGPRSpillFIs.set(FrameIndex);
       }
     }
 
+    // Stack slot coloring may assign different objets to the same stack slot.
+    // If not, then the VGPR to AGPR spill slot is dead.
+    for (unsigned FI : SpillFIs.set_bits())
+      if (!NonVGPRSpillFIs.test(FI))
+        FuncInfo->setVGPRToAGPRSpillDead(FI);
+
     for (MachineBasicBlock &MBB : MF) {
       for (MCPhysReg Reg : FuncInfo->getVGPRSpillAGPRs())
         MBB.addLiveIn(Reg);
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 35b72f5d201b..9f138136e6e9 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -24,6 +24,7 @@
 #include "llvm/CodeGen/Analysis.h"
 #include "llvm/CodeGen/FunctionLoweringInfo.h"
 #include "llvm/CodeGen/GlobalISel/GISelKnownBits.h"
+#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineLoopInfo.h"
 #include "llvm/IR/DiagnosticInfo.h"
@@ -2062,33 +2063,30 @@ void SITargetLowering::allocateSpecialInputSGPRs(
   SIMachineFunctionInfo &Info) const {
   auto &ArgInfo = Info.getArgInfo();
 
-  // We need to allocate these in place regardless of their use.
-  const bool IsFixed = AMDGPUTargetMachine::EnableFixedFunctionABI;
-
   // TODO: Unify handling with private memory pointers.
-  if (IsFixed || Info.hasDispatchPtr())
+  if (Info.hasDispatchPtr())
     allocateSGPR64Input(CCInfo, ArgInfo.DispatchPtr);
 
-  if (IsFixed || Info.hasQueuePtr())
+  if (Info.hasQueuePtr())
     allocateSGPR64Input(CCInfo, ArgInfo.QueuePtr);
 
   // Implicit arg ptr takes the place of the kernarg segment pointer. This is a
   // constant offset from the kernarg segment.
-  if (IsFixed || Info.hasImplicitArgPtr())
+  if (Info.hasImplicitArgPtr())
     allocateSGPR64Input(CCInfo, ArgInfo.ImplicitArgPtr);
 
-  if (IsFixed || Info.hasDispatchID())
+  if (Info.hasDispatchID())
     allocateSGPR64Input(CCInfo, ArgInfo.DispatchID);
 
   // flat_scratch_init is not applicable for non-kernel functions.
 
-  if (IsFixed || Info.hasWorkGroupIDX())
+  if (Info.hasWorkGroupIDX())
     allocateSGPR32Input(CCInfo, ArgInfo.WorkGroupIDX);
 
-  if (IsFixed || Info.hasWorkGroupIDY())
+  if (Info.hasWorkGroupIDY())
     allocateSGPR32Input(CCInfo, ArgInfo.WorkGroupIDY);
 
-  if (IsFixed || Info.hasWorkGroupIDZ())
+  if (Info.hasWorkGroupIDZ())
     allocateSGPR32Input(CCInfo, ArgInfo.WorkGroupIDZ);
 }
 
@@ -2419,10 +2417,9 @@ SDValue SITargetLowering::LowerFormalArguments(
   if (IsEntryFunc) {
     allocateSpecialEntryInputVGPRs(CCInfo, MF, *TRI, *Info);
     allocateHSAUserSGPRs(CCInfo, MF, *TRI, *Info);
-  } else {
+  } else if (!IsGraphics) {
     // For the fixed ABI, pass workitem IDs in the last argument register.
-    if (AMDGPUTargetMachine::EnableFixedFunctionABI)
-      allocateSpecialInputVGPRsFixed(CCInfo, MF, *TRI, *Info);
+    allocateSpecialInputVGPRsFixed(CCInfo, MF, *TRI, *Info);
   }
 
   if (IsKernel) {
@@ -2549,17 +2546,13 @@ SDValue SITargetLowering::LowerFormalArguments(
     InVals.push_back(Val);
   }
 
-  if (!IsEntryFunc && !AMDGPUTargetMachine::EnableFixedFunctionABI) {
-    // Special inputs come after user arguments.
-    allocateSpecialInputVGPRs(CCInfo, MF, *TRI, *Info);
-  }
-
   // Start adding system SGPRs.
   if (IsEntryFunc) {
     allocateSystemSGPRs(CCInfo, MF, *Info, CallConv, IsGraphics);
   } else {
     CCInfo.AllocateReg(Info->getScratchRSrcReg());
-    allocateSpecialInputSGPRs(CCInfo, MF, *TRI, *Info);
+    if (!IsGraphics)
+      allocateSpecialInputSGPRs(CCInfo, MF, *TRI, *Info);
   }
 
   auto &ArgUsageInfo =
@@ -3123,8 +3116,7 @@ SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI,
   CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
   CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, IsVarArg);
 
-  if (AMDGPUTargetMachine::EnableFixedFunctionABI &&
-      CallConv != CallingConv::AMDGPU_Gfx) {
+  if (CallConv != CallingConv::AMDGPU_Gfx) {
     // With a fixed ABI, allocate fixed registers before user arguments.
     passSpecialInputs(CLI, CCInfo, *Info, RegsToPass, MemOpChains, Chain);
   }
@@ -3263,12 +3255,6 @@ SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI,
     }
   }
 
-  if (!AMDGPUTargetMachine::EnableFixedFunctionABI &&
-      CallConv != CallingConv::AMDGPU_Gfx) {
-    // Copy special input registers after user input arguments.
-    passSpecialInputs(CLI, CCInfo, *Info, RegsToPass, MemOpChains, Chain);
-  }
-
   if (!MemOpChains.empty())
     Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
 
@@ -6282,10 +6268,6 @@ SDValue SITargetLowering::lowerImage(SDValue Op,
     }
   }
 
-  // Push back extra arguments.
-  for (unsigned I = Intr->VAddrStart; I < Intr->GradientStart; I++)
-    VAddrs.push_back(Op.getOperand(ArgOffset + I));
-
   // Check for 16 bit addresses or derivatives and pack if true.
   MVT VAddrVT =
       Op.getOperand(ArgOffset + Intr->GradientStart).getSimpleValueType();
@@ -6298,6 +6280,17 @@ SDValue SITargetLowering::lowerImage(SDValue Op,
   MVT AddrPackVectorVT = VAddrScalarVT == MVT::f16 ? MVT::v2f16 : MVT::v2i16;
   IsA16 = VAddrScalarVT == MVT::f16 || VAddrScalarVT == MVT::i16;
 
+  // Push back extra arguments.
+  for (unsigned I = Intr->VAddrStart; I < Intr->GradientStart; I++) {
+    if (IsA16 && (Op.getOperand(ArgOffset + I).getValueType() == MVT::f16)) {
+      // Special handling of bias when A16 is on. Bias is of type half but
+      // occupies full 32-bit.
+      SDValue bias = DAG.getBuildVector( MVT::v2f16, DL, {Op.getOperand(ArgOffset + I), DAG.getUNDEF(MVT::f16)});
+      VAddrs.push_back(bias);
+    } else
+      VAddrs.push_back(Op.getOperand(ArgOffset + I));
+  }
+
   if (BaseOpcode->Gradients && !ST->hasG16() && (IsA16 != IsG16)) {
     // 16 bit gradients are supported, but are tied to the A16 control
     // so both gradients and addresses must be 16 bit
@@ -7502,8 +7495,8 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
 
     assert(NodePtr.getValueType() == MVT::i32 ||
            NodePtr.getValueType() == MVT::i64);
-    assert(RayDir.getValueType() == MVT::v4f16 ||
-           RayDir.getValueType() == MVT::v4f32);
+    assert(RayDir.getValueType() == MVT::v3f16 ||
+           RayDir.getValueType() == MVT::v3f32);
 
     if (!Subtarget->hasGFX10_AEncoding()) {
       emitRemovedIntrinsicError(DAG, DL, Op.getValueType());
@@ -9837,11 +9830,13 @@ bool SITargetLowering::isCanonicalized(Register Reg, MachineFunction &MF,
   if (Opcode == AMDGPU::G_FCANONICALIZE)
     return true;
 
-  if (Opcode == AMDGPU::G_FCONSTANT) {
-    auto F = MI->getOperand(1).getFPImm()->getValueAPF();
-    if (F.isNaN() && F.isSignaling())
+  Optional<FPValueAndVReg> FCR;
+  // Constant splat (can be padded with undef) or scalar constant.
+  if (mi_match(Reg, MRI, MIPatternMatch::m_GFCstOrSplat(FCR))) {
+    if (FCR->Value.isSignaling())
       return false;
-    return !F.isDenormal() || denormalsEnabledForType(MRI.getType(Reg), MF);
+    return !FCR->Value.isDenormal() ||
+           denormalsEnabledForType(MRI.getType(FCR->VReg), MF);
   }
 
   if (MaxDepth == 0)
@@ -11514,7 +11509,7 @@ void SITargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
     // Prefer VGPRs over AGPRs in mAI instructions where possible.
     // This saves a chain-copy of registers and better ballance register
     // use between vgpr and agpr as agpr tuples tend to be big.
-    if (const MCOperandInfo *OpInfo = MI.getDesc().OpInfo) {
+    if (MI.getDesc().OpInfo) {
       unsigned Opc = MI.getOpcode();
       const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
       for (auto I : { AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0),
@@ -12477,6 +12472,6 @@ SITargetLowering::getTypeLegalizationCost(const DataLayout &DL,
   if (Size <= 256)
     return Cost;
 
-  Cost.first = (Size + 255) / 256;
+  Cost.first += (Size + 255) / 256;
   return Cost;
 }
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index c9d9dd1fb82c..6fbe5d45ce0a 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -30,6 +30,7 @@
 #include "Utils/AMDGPUBaseInfo.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/PostOrderIterator.h"
+#include "llvm/ADT/Sequence.h"
 #include "llvm/CodeGen/MachinePostDominators.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Support/DebugCounter.h"
@@ -51,26 +52,6 @@ static cl::opt<bool> ForceEmitZeroFlag(
   cl::init(false), cl::Hidden);
 
 namespace {
-
-template <typename EnumT>
-class enum_iterator
-    : public iterator_facade_base<enum_iterator<EnumT>,
-                                  std::forward_iterator_tag, const EnumT> {
-  EnumT Value;
-public:
-  enum_iterator() = default;
-  enum_iterator(EnumT Value) : Value(Value) {}
-
-  enum_iterator &operator++() {
-    Value = static_cast<EnumT>(Value + 1);
-    return *this;
-  }
-
-  bool operator==(const enum_iterator &RHS) const { return Value == RHS.Value; }
-
-  EnumT operator*() const { return Value; }
-};
-
 // Class of object that encapsulates latest instruction counter score
 // associated with the operand.  Used for determining whether
 // s_waitcnt instruction needs to be emitted.
@@ -78,27 +59,32 @@ public:
 #define CNT_MASK(t) (1u << (t))
 
 enum InstCounterType { VM_CNT = 0, LGKM_CNT, EXP_CNT, VS_CNT, NUM_INST_CNTS };
+} // namespace
 
-iterator_range<enum_iterator<InstCounterType>> inst_counter_types() {
-  return make_range(enum_iterator<InstCounterType>(VM_CNT),
-                    enum_iterator<InstCounterType>(NUM_INST_CNTS));
-}
+namespace llvm {
+template <> struct enum_iteration_traits<InstCounterType> {
+  static constexpr bool is_iterable = true;
+};
+} // namespace llvm
+
+namespace {
+auto inst_counter_types() { return enum_seq(VM_CNT, NUM_INST_CNTS); }
 
 using RegInterval = std::pair<int, int>;
 
-struct {
+struct HardwareLimits {
   unsigned VmcntMax;
   unsigned ExpcntMax;
   unsigned LgkmcntMax;
   unsigned VscntMax;
-} HardwareLimits;
+};
 
-struct {
+struct RegisterEncoding {
   unsigned VGPR0;
   unsigned VGPRL;
   unsigned SGPR0;
   unsigned SGPRL;
-} RegisterEncoding;
+};
 
 enum WaitEventType {
   VMEM_ACCESS,      // vector-memory read & write
@@ -194,18 +180,20 @@ void addWait(AMDGPU::Waitcnt &Wait, InstCounterType T, unsigned Count) {
 // "s_waitcnt 0" before use.
 class WaitcntBrackets {
 public:
-  WaitcntBrackets(const GCNSubtarget *SubTarget) : ST(SubTarget) {}
+  WaitcntBrackets(const GCNSubtarget *SubTarget, HardwareLimits Limits,
+                  RegisterEncoding Encoding)
+      : ST(SubTarget), Limits(Limits), Encoding(Encoding) {}
 
-  static unsigned getWaitCountMax(InstCounterType T) {
+  unsigned getWaitCountMax(InstCounterType T) const {
     switch (T) {
     case VM_CNT:
-      return HardwareLimits.VmcntMax;
+      return Limits.VmcntMax;
     case LGKM_CNT:
-      return HardwareLimits.LgkmcntMax;
+      return Limits.LgkmcntMax;
     case EXP_CNT:
-      return HardwareLimits.ExpcntMax;
+      return Limits.ExpcntMax;
     case VS_CNT:
-      return HardwareLimits.VscntMax;
+      return Limits.VscntMax;
     default:
       break;
     }
@@ -338,6 +326,8 @@ private:
                    unsigned OpNo, unsigned Val);
 
   const GCNSubtarget *ST = nullptr;
+  HardwareLimits Limits = {};
+  RegisterEncoding Encoding = {};
   unsigned ScoreLBs[NUM_INST_CNTS] = {0};
   unsigned ScoreUBs[NUM_INST_CNTS] = {0};
   unsigned PendingEvents = 0;
@@ -471,14 +461,14 @@ RegInterval WaitcntBrackets::getRegInterval(const MachineInstr *MI,
   unsigned Reg = TRI->getEncodingValue(AMDGPU::getMCReg(Op.getReg(), *ST));
 
   if (TRI->isVectorRegister(*MRI, Op.getReg())) {
-    assert(Reg >= RegisterEncoding.VGPR0 && Reg <= RegisterEncoding.VGPRL);
-    Result.first = Reg - RegisterEncoding.VGPR0;
+    assert(Reg >= Encoding.VGPR0 && Reg <= Encoding.VGPRL);
+    Result.first = Reg - Encoding.VGPR0;
     if (TRI->isAGPR(*MRI, Op.getReg()))
       Result.first += AGPR_OFFSET;
     assert(Result.first >= 0 && Result.first < SQ_MAX_PGM_VGPRS);
   } else if (TRI->isSGPRReg(*MRI, Op.getReg())) {
-    assert(Reg >= RegisterEncoding.SGPR0 && Reg < SQ_MAX_PGM_SGPRS);
-    Result.first = Reg - RegisterEncoding.SGPR0 + NUM_ALL_VGPRS;
+    assert(Reg >= Encoding.SGPR0 && Reg < SQ_MAX_PGM_SGPRS);
+    Result.first = Reg - Encoding.SGPR0 + NUM_ALL_VGPRS;
     assert(Result.first >= NUM_ALL_VGPRS &&
            Result.first < SQ_MAX_PGM_SGPRS + NUM_ALL_VGPRS);
   }
@@ -1589,20 +1579,22 @@ bool SIInsertWaitcnts::runOnMachineFunction(MachineFunction &MF) {
   for (auto T : inst_counter_types())
     ForceEmitWaitcnt[T] = false;
 
-  HardwareLimits.VmcntMax = AMDGPU::getVmcntBitMask(IV);
-  HardwareLimits.ExpcntMax = AMDGPU::getExpcntBitMask(IV);
-  HardwareLimits.LgkmcntMax = AMDGPU::getLgkmcntBitMask(IV);
-  HardwareLimits.VscntMax = ST->hasVscnt() ? 63 : 0;
+  HardwareLimits Limits = {};
+  Limits.VmcntMax = AMDGPU::getVmcntBitMask(IV);
+  Limits.ExpcntMax = AMDGPU::getExpcntBitMask(IV);
+  Limits.LgkmcntMax = AMDGPU::getLgkmcntBitMask(IV);
+  Limits.VscntMax = ST->hasVscnt() ? 63 : 0;
 
   unsigned NumVGPRsMax = ST->getAddressableNumVGPRs();
   unsigned NumSGPRsMax = ST->getAddressableNumSGPRs();
   assert(NumVGPRsMax <= SQ_MAX_PGM_VGPRS);
   assert(NumSGPRsMax <= SQ_MAX_PGM_SGPRS);
 
-  RegisterEncoding.VGPR0 = TRI->getEncodingValue(AMDGPU::VGPR0);
-  RegisterEncoding.VGPRL = RegisterEncoding.VGPR0 + NumVGPRsMax - 1;
-  RegisterEncoding.SGPR0 = TRI->getEncodingValue(AMDGPU::SGPR0);
-  RegisterEncoding.SGPRL = RegisterEncoding.SGPR0 + NumSGPRsMax - 1;
+  RegisterEncoding Encoding = {};
+  Encoding.VGPR0 = TRI->getEncodingValue(AMDGPU::VGPR0);
+  Encoding.VGPRL = Encoding.VGPR0 + NumVGPRsMax - 1;
+  Encoding.SGPR0 = TRI->getEncodingValue(AMDGPU::SGPR0);
+  Encoding.SGPRL = Encoding.SGPR0 + NumSGPRsMax - 1;
 
   TrackedWaitcntSet.clear();
   BlockInfos.clear();
@@ -1652,9 +1644,9 @@ bool SIInsertWaitcnts::runOnMachineFunction(MachineFunction &MF) {
           *Brackets = *BI.Incoming;
       } else {
         if (!Brackets)
-          Brackets = std::make_unique<WaitcntBrackets>(ST);
+          Brackets = std::make_unique<WaitcntBrackets>(ST, Limits, Encoding);
         else
-          *Brackets = WaitcntBrackets(ST);
+          *Brackets = WaitcntBrackets(ST, Limits, Encoding);
       }
 
       Modified |= insertWaitcntInBlock(MF, *BI.MBB, *Brackets);
@@ -1686,45 +1678,47 @@ bool SIInsertWaitcnts::runOnMachineFunction(MachineFunction &MF) {
     }
   } while (Repeat);
 
-  SmallVector<MachineBasicBlock *, 4> EndPgmBlocks;
-
-  bool HaveScalarStores = false;
+  if (ST->hasScalarStores()) {
+    SmallVector<MachineBasicBlock *, 4> EndPgmBlocks;
+    bool HaveScalarStores = false;
 
-  for (MachineBasicBlock &MBB : MF) {
-    for (MachineInstr &MI : MBB) {
-      if (!HaveScalarStores && TII->isScalarStore(MI))
-        HaveScalarStores = true;
+    for (MachineBasicBlock &MBB : MF) {
+      for (MachineInstr &MI : MBB) {
+        if (!HaveScalarStores && TII->isScalarStore(MI))
+          HaveScalarStores = true;
 
-      if (MI.getOpcode() == AMDGPU::S_ENDPGM ||
-          MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG)
-        EndPgmBlocks.push_back(&MBB);
+        if (MI.getOpcode() == AMDGPU::S_ENDPGM ||
+            MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG)
+          EndPgmBlocks.push_back(&MBB);
+      }
     }
-  }
 
-  if (HaveScalarStores) {
-    // If scalar writes are used, the cache must be flushed or else the next
-    // wave to reuse the same scratch memory can be clobbered.
-    //
-    // Insert s_dcache_wb at wave termination points if there were any scalar
-    // stores, and only if the cache hasn't already been flushed. This could be
-    // improved by looking across blocks for flushes in postdominating blocks
-    // from the stores but an explicitly requested flush is probably very rare.
-    for (MachineBasicBlock *MBB : EndPgmBlocks) {
-      bool SeenDCacheWB = false;
-
-      for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end(); I != E;
-           ++I) {
-        if (I->getOpcode() == AMDGPU::S_DCACHE_WB)
-          SeenDCacheWB = true;
-        else if (TII->isScalarStore(*I))
-          SeenDCacheWB = false;
-
-        // FIXME: It would be better to insert this before a waitcnt if any.
-        if ((I->getOpcode() == AMDGPU::S_ENDPGM ||
-             I->getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG) &&
-            !SeenDCacheWB) {
-          Modified = true;
-          BuildMI(*MBB, I, I->getDebugLoc(), TII->get(AMDGPU::S_DCACHE_WB));
+    if (HaveScalarStores) {
+      // If scalar writes are used, the cache must be flushed or else the next
+      // wave to reuse the same scratch memory can be clobbered.
+      //
+      // Insert s_dcache_wb at wave termination points if there were any scalar
+      // stores, and only if the cache hasn't already been flushed. This could
+      // be improved by looking across blocks for flushes in postdominating
+      // blocks from the stores but an explicitly requested flush is probably
+      // very rare.
+      for (MachineBasicBlock *MBB : EndPgmBlocks) {
+        bool SeenDCacheWB = false;
+
+        for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end();
+             I != E; ++I) {
+          if (I->getOpcode() == AMDGPU::S_DCACHE_WB)
+            SeenDCacheWB = true;
+          else if (TII->isScalarStore(*I))
+            SeenDCacheWB = false;
+
+          // FIXME: It would be better to insert this before a waitcnt if any.
+          if ((I->getOpcode() == AMDGPU::S_ENDPGM ||
+               I->getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG) &&
+              !SeenDCacheWB) {
+            Modified = true;
+            BuildMI(*MBB, I, I->getDebugLoc(), TII->get(AMDGPU::S_DCACHE_WB));
+          }
         }
       }
     }
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 92f5322b8ad2..1755b93538ce 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -899,8 +899,12 @@ void SIInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
   unsigned EltSize = 4;
   unsigned Opcode = AMDGPU::V_MOV_B32_e32;
   if (RI.isAGPRClass(RC)) {
-    Opcode = (RI.hasVGPRs(SrcRC)) ?
-      AMDGPU::V_ACCVGPR_WRITE_B32_e64 : AMDGPU::INSTRUCTION_LIST_END;
+    if (ST.hasGFX90AInsts() && RI.isAGPRClass(SrcRC))
+      Opcode = AMDGPU::V_ACCVGPR_MOV_B32;
+    else if (RI.hasVGPRs(SrcRC))
+      Opcode = AMDGPU::V_ACCVGPR_WRITE_B32_e64;
+    else
+      Opcode = AMDGPU::INSTRUCTION_LIST_END;
   } else if (RI.hasVGPRs(RC) && RI.isAGPRClass(SrcRC)) {
     Opcode = AMDGPU::V_ACCVGPR_READ_B32_e64;
   } else if ((Size % 64 == 0) && RI.hasVGPRs(RC) &&
@@ -1417,6 +1421,33 @@ static unsigned getAGPRSpillSaveOpcode(unsigned Size) {
   }
 }
 
+static unsigned getAVSpillSaveOpcode(unsigned Size) {
+  switch (Size) {
+  case 4:
+    return AMDGPU::SI_SPILL_AV32_SAVE;
+  case 8:
+    return AMDGPU::SI_SPILL_AV64_SAVE;
+  case 12:
+    return AMDGPU::SI_SPILL_AV96_SAVE;
+  case 16:
+    return AMDGPU::SI_SPILL_AV128_SAVE;
+  case 20:
+    return AMDGPU::SI_SPILL_AV160_SAVE;
+  case 24:
+    return AMDGPU::SI_SPILL_AV192_SAVE;
+  case 28:
+    return AMDGPU::SI_SPILL_AV224_SAVE;
+  case 32:
+    return AMDGPU::SI_SPILL_AV256_SAVE;
+  case 64:
+    return AMDGPU::SI_SPILL_AV512_SAVE;
+  case 128:
+    return AMDGPU::SI_SPILL_AV1024_SAVE;
+  default:
+    llvm_unreachable("unknown register size");
+  }
+}
+
 void SIInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
                                       MachineBasicBlock::iterator MI,
                                       Register SrcReg, bool isKill,
@@ -1463,21 +1494,11 @@ void SIInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
     return;
   }
 
-  unsigned Opcode = RI.isAGPRClass(RC) ? getAGPRSpillSaveOpcode(SpillSize)
-                                       : getVGPRSpillSaveOpcode(SpillSize);
+  unsigned Opcode = RI.isVectorSuperClass(RC) ? getAVSpillSaveOpcode(SpillSize)
+                    : RI.isAGPRClass(RC) ? getAGPRSpillSaveOpcode(SpillSize)
+                                         : getVGPRSpillSaveOpcode(SpillSize);
   MFI->setHasSpilledVGPRs();
 
-  if (RI.isVectorSuperClass(RC)) {
-    // Convert an AV spill into a VGPR spill. Introduce a copy from AV to an
-    // equivalent VGPR register beforehand. Regalloc might want to introduce
-    // AV spills only to be relevant until rewriter at which they become
-    // either spills of VGPRs or AGPRs.
-    Register TmpVReg = MRI.createVirtualRegister(RI.getEquivalentVGPRClass(RC));
-    BuildMI(MBB, MI, DL, get(TargetOpcode::COPY), TmpVReg)
-        .addReg(SrcReg, RegState::Kill);
-    SrcReg = TmpVReg;
-  }
-
   BuildMI(MBB, MI, DL, get(Opcode))
     .addReg(SrcReg, getKillRegState(isKill)) // data
     .addFrameIndex(FrameIndex)               // addr
@@ -1567,6 +1588,33 @@ static unsigned getAGPRSpillRestoreOpcode(unsigned Size) {
   }
 }
 
+static unsigned getAVSpillRestoreOpcode(unsigned Size) {
+  switch (Size) {
+  case 4:
+    return AMDGPU::SI_SPILL_AV32_RESTORE;
+  case 8:
+    return AMDGPU::SI_SPILL_AV64_RESTORE;
+  case 12:
+    return AMDGPU::SI_SPILL_AV96_RESTORE;
+  case 16:
+    return AMDGPU::SI_SPILL_AV128_RESTORE;
+  case 20:
+    return AMDGPU::SI_SPILL_AV160_RESTORE;
+  case 24:
+    return AMDGPU::SI_SPILL_AV192_RESTORE;
+  case 28:
+    return AMDGPU::SI_SPILL_AV224_RESTORE;
+  case 32:
+    return AMDGPU::SI_SPILL_AV256_RESTORE;
+  case 64:
+    return AMDGPU::SI_SPILL_AV512_RESTORE;
+  case 128:
+    return AMDGPU::SI_SPILL_AV1024_RESTORE;
+  default:
+    llvm_unreachable("unknown register size");
+  }
+}
+
 void SIInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
                                        MachineBasicBlock::iterator MI,
                                        Register DestReg, int FrameIndex,
@@ -1609,26 +1657,15 @@ void SIInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
     return;
   }
 
-  unsigned Opcode = RI.isAGPRClass(RC) ? getAGPRSpillRestoreOpcode(SpillSize)
-                                       : getVGPRSpillRestoreOpcode(SpillSize);
-
-  bool IsVectorSuperClass = RI.isVectorSuperClass(RC);
-  Register TmpReg = DestReg;
-  if (IsVectorSuperClass) {
-    // For AV classes, insert the spill restore to a VGPR followed by a copy
-    // into an equivalent AV register.
-    MachineRegisterInfo &MRI = MF->getRegInfo();
-    DestReg = MRI.createVirtualRegister(RI.getEquivalentVGPRClass(RC));
-  }
+  unsigned Opcode = RI.isVectorSuperClass(RC)
+                        ? getAVSpillRestoreOpcode(SpillSize)
+                    : RI.isAGPRClass(RC) ? getAGPRSpillRestoreOpcode(SpillSize)
+                                         : getVGPRSpillRestoreOpcode(SpillSize);
   BuildMI(MBB, MI, DL, get(Opcode), DestReg)
-    .addFrameIndex(FrameIndex)        // vaddr
-    .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset
-    .addImm(0)                           // offset
-    .addMemOperand(MMO);
-
-  if (IsVectorSuperClass)
-    BuildMI(MBB, MI, DL, get(TargetOpcode::COPY), TmpReg)
-        .addReg(DestReg, RegState::Kill);
+      .addFrameIndex(FrameIndex)           // vaddr
+      .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset
+      .addImm(0)                           // offset
+      .addMemOperand(MMO);
 }
 
 void SIInstrInfo::insertNoop(MachineBasicBlock &MBB,
@@ -2358,8 +2395,6 @@ void SIInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB,
   OffsetLo->setVariableValue(MCBinaryExpr::createAnd(Offset, Mask, MCCtx));
   auto *ShAmt = MCConstantExpr::create(32, MCCtx);
   OffsetHi->setVariableValue(MCBinaryExpr::createAShr(Offset, ShAmt, MCCtx));
-
-  return;
 }
 
 unsigned SIInstrInfo::getBranchOpcode(SIInstrInfo::BranchPredicate Cond) {
@@ -3106,23 +3141,26 @@ bool SIInstrInfo::areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
 }
 
 static bool getFoldableImm(Register Reg, const MachineRegisterInfo &MRI,
-                           int64_t &Imm) {
+                           int64_t &Imm, MachineInstr **DefMI = nullptr) {
   if (Reg.isPhysical())
     return false;
   auto *Def = MRI.getUniqueVRegDef(Reg);
   if (Def && SIInstrInfo::isFoldableCopy(*Def) && Def->getOperand(1).isImm()) {
     Imm = Def->getOperand(1).getImm();
+    if (DefMI)
+      *DefMI = Def;
     return true;
   }
   return false;
 }
 
-static bool getFoldableImm(const MachineOperand *MO, int64_t &Imm) {
+static bool getFoldableImm(const MachineOperand *MO, int64_t &Imm,
+                           MachineInstr **DefMI = nullptr) {
   if (!MO->isReg())
     return false;
   const MachineFunction *MF = MO->getParent()->getParent()->getParent();
   const MachineRegisterInfo &MRI = MF->getRegInfo();
-  return getFoldableImm(MO->getReg(), MRI, Imm);
+  return getFoldableImm(MO->getReg(), MRI, Imm, DefMI);
 }
 
 static void updateLiveVariables(LiveVariables *LV, MachineInstr &MI,
@@ -3195,8 +3233,20 @@ MachineInstr *SIInstrInfo::convertToThreeAddress(MachineInstr &MI,
       // If we have an SGPR input, we will violate the constant bus restriction.
       (ST.getConstantBusLimit(Opc) > 1 || !Src0->isReg() ||
        !RI.isSGPRReg(MBB.getParent()->getRegInfo(), Src0->getReg()))) {
+    MachineInstr *DefMI;
+    const auto killDef = [&DefMI, &MBB, this]() -> void {
+      const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
+      // The only user is the instruction which will be killed.
+      if (!MRI.hasOneNonDBGUse(DefMI->getOperand(0).getReg()))
+        return;
+      // We cannot just remove the DefMI here, calling pass will crash.
+      DefMI->setDesc(get(AMDGPU::IMPLICIT_DEF));
+      for (unsigned I = DefMI->getNumOperands() - 1; I != 0; --I)
+        DefMI->RemoveOperand(I);
+    };
+
     int64_t Imm;
-    if (getFoldableImm(Src2, Imm)) {
+    if (getFoldableImm(Src2, Imm, &DefMI)) {
       unsigned NewOpc =
           IsFMA ? (IsF16 ? AMDGPU::V_FMAAK_F16 : AMDGPU::V_FMAAK_F32)
                 : (IsF16 ? AMDGPU::V_MADAK_F16 : AMDGPU::V_MADAK_F32);
@@ -3209,13 +3259,14 @@ MachineInstr *SIInstrInfo::convertToThreeAddress(MachineInstr &MI,
         updateLiveVariables(LV, MI, *MIB);
         if (LIS)
           LIS->ReplaceMachineInstrInMaps(MI, *MIB);
+        killDef();
         return MIB;
       }
     }
     unsigned NewOpc = IsFMA
                           ? (IsF16 ? AMDGPU::V_FMAMK_F16 : AMDGPU::V_FMAMK_F32)
                           : (IsF16 ? AMDGPU::V_MADMK_F16 : AMDGPU::V_MADMK_F32);
-    if (getFoldableImm(Src1, Imm)) {
+    if (getFoldableImm(Src1, Imm, &DefMI)) {
       if (pseudoToMCOpcode(NewOpc) != -1) {
         MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc))
                   .add(*Dst)
@@ -3225,10 +3276,11 @@ MachineInstr *SIInstrInfo::convertToThreeAddress(MachineInstr &MI,
         updateLiveVariables(LV, MI, *MIB);
         if (LIS)
           LIS->ReplaceMachineInstrInMaps(MI, *MIB);
+        killDef();
         return MIB;
       }
     }
-    if (getFoldableImm(Src0, Imm)) {
+    if (getFoldableImm(Src0, Imm, &DefMI)) {
       if (pseudoToMCOpcode(NewOpc) != -1 &&
           isOperandLegal(
               MI, AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::src0),
@@ -3241,12 +3293,13 @@ MachineInstr *SIInstrInfo::convertToThreeAddress(MachineInstr &MI,
         updateLiveVariables(LV, MI, *MIB);
         if (LIS)
           LIS->ReplaceMachineInstrInMaps(MI, *MIB);
+        killDef();
         return MIB;
       }
     }
   }
 
-  unsigned NewOpc = IsFMA ? (IsF16 ? AMDGPU::V_FMA_F16_e64
+  unsigned NewOpc = IsFMA ? (IsF16 ? AMDGPU::V_FMA_F16_gfx9_e64
                                    : IsF64 ? AMDGPU::V_FMA_F64_e64
                                            : AMDGPU::V_FMA_F32_e64)
                           : (IsF16 ? AMDGPU::V_MAD_F16_e64 : AMDGPU::V_MAD_F32_e64);
@@ -3605,12 +3658,6 @@ bool SIInstrInfo::canShrink(const MachineInstr &MI,
                             const MachineRegisterInfo &MRI) const {
   const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
   // Can't shrink instruction with three operands.
-  // FIXME: v_cndmask_b32 has 3 operands and is shrinkable, but we need to add
-  // a special case for it.  It can only be shrunk if the third operand
-  // is vcc, and src0_modifiers and src1_modifiers are not set.
-  // We should handle this the same way we handle vopc, by addding
-  // a register allocation hint pre-regalloc and then do the shrinking
-  // post-regalloc.
   if (Src2) {
     switch (MI.getOpcode()) {
       default: return false;
@@ -4563,8 +4610,9 @@ static unsigned adjustAllocatableRegClass(const GCNSubtarget &ST,
                                           unsigned RCID,
                                           bool IsAllocatable) {
   if ((IsAllocatable || !ST.hasGFX90AInsts() || !MRI.reservedRegsFrozen()) &&
-      (TID.mayLoad() || TID.mayStore() ||
-      (TID.TSFlags & (SIInstrFlags::DS | SIInstrFlags::MIMG)))) {
+      (((TID.mayLoad() || TID.mayStore()) &&
+        !(TID.TSFlags & SIInstrFlags::VGPRSpill)) ||
+       (TID.TSFlags & (SIInstrFlags::DS | SIInstrFlags::MIMG)))) {
     switch (RCID) {
     case AMDGPU::AV_32RegClassID: return AMDGPU::VGPR_32RegClassID;
     case AMDGPU::AV_64RegClassID: return AMDGPU::VReg_64RegClassID;
@@ -5001,8 +5049,7 @@ void SIInstrInfo::legalizeOperandsVOP3(MachineRegisterInfo &MRI,
     --ConstantBusLimit;
   }
 
-  for (unsigned i = 0; i < 3; ++i) {
-    int Idx = VOP3Idx[i];
+  for (int Idx : VOP3Idx) {
     if (Idx == -1)
       break;
     MachineOperand &MO = MI.getOperand(Idx);
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index 47ee83eb9351..dda92d3d25ff 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -1350,11 +1350,11 @@ def PackedI16InputMods : PackedIntInputMods<PackedI16InputModsMatchClass>;
 // Complex patterns
 //===----------------------------------------------------------------------===//
 
-def DS1Addr1Offset : ComplexPattern<i32, 2, "SelectDS1Addr1Offset">;
-def DS64Bit4ByteAligned : ComplexPattern<i32, 3, "SelectDS64Bit4ByteAligned">;
-def DS128Bit8ByteAligned : ComplexPattern<i64, 3, "SelectDS128Bit8ByteAligned">;
+def DS1Addr1Offset : ComplexPattern<iPTR, 2, "SelectDS1Addr1Offset">;
+def DS64Bit4ByteAligned : ComplexPattern<iPTR, 3, "SelectDS64Bit4ByteAligned">;
+def DS128Bit8ByteAligned : ComplexPattern<iPTR, 3, "SelectDS128Bit8ByteAligned">;
 
-def MOVRELOffset : ComplexPattern<i32, 2, "SelectMOVRELOffset">;
+def MOVRELOffset : ComplexPattern<iPTR, 2, "SelectMOVRELOffset">;
 
 def VOP3Mods0 : ComplexPattern<untyped, 4, "SelectVOP3Mods0">;
 def VOP3Mods  : ComplexPattern<untyped, 2, "SelectVOP3Mods">;
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index d55d8da8699a..636337ede000 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -761,6 +761,17 @@ defm SI_SPILL_A256 : SI_SPILL_VGPR <AReg_256, 1>;
 defm SI_SPILL_A512 : SI_SPILL_VGPR <AReg_512, 1>;
 defm SI_SPILL_A1024 : SI_SPILL_VGPR <AReg_1024, 1>;
 
+defm SI_SPILL_AV32  : SI_SPILL_VGPR <AV_32, 1>;
+defm SI_SPILL_AV64  : SI_SPILL_VGPR <AV_64, 1>;
+defm SI_SPILL_AV96  : SI_SPILL_VGPR <AV_96, 1>;
+defm SI_SPILL_AV128 : SI_SPILL_VGPR <AV_128, 1>;
+defm SI_SPILL_AV160 : SI_SPILL_VGPR <AV_160, 1>;
+defm SI_SPILL_AV192 : SI_SPILL_VGPR <AV_192, 1>;
+defm SI_SPILL_AV224 : SI_SPILL_VGPR <AV_224, 1>;
+defm SI_SPILL_AV256 : SI_SPILL_VGPR <AV_256, 1>;
+defm SI_SPILL_AV512 : SI_SPILL_VGPR <AV_512, 1>;
+defm SI_SPILL_AV1024 : SI_SPILL_VGPR <AV_1024, 1>;
+
 def SI_PC_ADD_REL_OFFSET : SPseudoInstSI <
   (outs SReg_64:$dst),
   (ins si_ga:$ptr_lo, si_ga:$ptr_hi),
@@ -2106,6 +2117,19 @@ def : GCNPat <
 } // end isWave32
 
 def : GCNPat <
+  (i32 (DivergentBinFrag<xor> i32:$src0, (i32 -1))),
+  (V_NOT_B32_e32 $src0)
+>;
+
+def : GCNPat <
+  (i64 (DivergentBinFrag<xor> i64:$src0, (i64 -1))),
+    (REG_SEQUENCE VReg_64,
+      (V_NOT_B32_e32 (i32 (EXTRACT_SUBREG i64:$src0, sub0))), sub0,
+      (V_NOT_B32_e32 (i32 (EXTRACT_SUBREG i64:$src0, sub1))), sub1
+    )
+>;
+
+def : GCNPat <
   (f16 (sint_to_fp i1:$src)),
   (V_CVT_F16_F32_e32 (
       V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0),
@@ -2188,18 +2212,18 @@ def : GCNPat <
 >;
 
 def : GCNPat <
-  (i1 (trunc i32:$a)),
-  (V_CMP_EQ_U32_e64 (S_AND_B32 (i32 1), $a), (i32 1))
+  (i1 (DivergentUnaryFrag<trunc> i32:$a)),
+  (V_CMP_EQ_U32_e64 (V_AND_B32_e64 (i32 1), $a), (i32 1))
 >;
 
 def : GCNPat <
-  (i1 (trunc i16:$a)),
-  (V_CMP_EQ_U32_e64 (S_AND_B32 (i32 1), $a), (i32 1))
+  (i1 (DivergentUnaryFrag<trunc> i16:$a)),
+  (V_CMP_EQ_U32_e64 (V_AND_B32_e64 (i32 1), $a), (i32 1))
 >;
 
 def : GCNPat <
-  (i1 (trunc i64:$a)),
-  (V_CMP_EQ_U32_e64 (S_AND_B32 (i32 1),
+  (i1 (DivergentUnaryFrag<trunc> i64:$a)),
+  (V_CMP_EQ_U32_e64 (V_AND_B32_e64 (i32 1),
                     (i32 (EXTRACT_SUBREG $a, sub0))), (i32 1))
 >;
 
@@ -2405,21 +2429,37 @@ def : GCNPat <
 // COPY is workaround tablegen bug from multiple outputs
 // from S_LSHL_B32's multiple outputs from implicit scc def.
 def : GCNPat <
-  (v2i16 (build_vector (i16 0), (i16 SReg_32:$src1))),
+  (v2i16 (UniformBinFrag<build_vector> (i16 0), (i16 SReg_32:$src1))),
   (S_LSHL_B32 SReg_32:$src1, (i16 16))
 >;
 
 def : GCNPat <
-  (v2i16 (build_vector (i16 SReg_32:$src1), (i16 0))),
+  (v2i16 (DivergentBinFrag<build_vector> (i16 0), (i16 SReg_32:$src1))),
+  (v2i16 (V_LSHLREV_B32_e64 (i16 16), SReg_32:$src1))
+>;
+
+
+def : GCNPat <
+  (v2i16 (UniformBinFrag<build_vector> (i16 SReg_32:$src1), (i16 0))),
   (S_AND_B32 (S_MOV_B32 (i32 0xffff)), SReg_32:$src1)
 >;
 
 def : GCNPat <
-  (v2f16 (build_vector (f16 SReg_32:$src1), (f16 FP_ZERO))),
+  (v2i16 (DivergentBinFrag<build_vector> (i16 SReg_32:$src1), (i16 0))),
+  (v2i16 (V_AND_B32_e64 (i32 (V_MOV_B32_e32 (i32 0xffff))), SReg_32:$src1))
+>;
+
+def : GCNPat <
+  (v2f16 (UniformBinFrag<build_vector> (f16 SReg_32:$src1), (f16 FP_ZERO))),
   (S_AND_B32 (S_MOV_B32 (i32 0xffff)), SReg_32:$src1)
 >;
 
 def : GCNPat <
+  (v2f16 (DivergentBinFrag<build_vector> (f16 SReg_32:$src1), (f16 FP_ZERO))),
+  (v2f16 (V_AND_B32_e64 (i32 (V_MOV_B32_e32 (i32 0xffff))), SReg_32:$src1))
+>;
+
+def : GCNPat <
   (v2i16 (build_vector (i16 SReg_32:$src0), (i16 undef))),
   (COPY_TO_REGCLASS SReg_32:$src0, SReg_32)
 >;
@@ -2435,42 +2475,74 @@ def : GCNPat <
 >;
 
 def : GCNPat <
-  (v2i16 (build_vector (i16 undef), (i16 SReg_32:$src1))),
+  (v2i16 (UniformBinFrag<build_vector> (i16 undef), (i16 SReg_32:$src1))),
   (S_LSHL_B32 SReg_32:$src1, (i32 16))
 >;
 
 def : GCNPat <
-  (v2f16 (build_vector (f16 undef), (f16 SReg_32:$src1))),
+  (v2i16 (DivergentBinFrag<build_vector> (i16 undef), (i16 SReg_32:$src1))),
+  (v2i16 (V_LSHLREV_B32_e64 (i32 16), SReg_32:$src1))
+>;
+
+
+def : GCNPat <
+  (v2f16 (UniformBinFrag<build_vector> (f16 undef), (f16 SReg_32:$src1))),
   (S_LSHL_B32 SReg_32:$src1, (i32 16))
 >;
 
+def : GCNPat <
+  (v2f16 (DivergentBinFrag<build_vector> (f16 undef), (f16 SReg_32:$src1))),
+  (v2f16 (V_LSHLREV_B32_e64 (i32 16), SReg_32:$src1))
+>;
+
 let SubtargetPredicate = HasVOP3PInsts in {
 def : GCNPat <
-  (v2i16 (build_vector (i16 SReg_32:$src0), (i16 SReg_32:$src1))),
+  (v2i16 (UniformBinFrag<build_vector> (i16 SReg_32:$src0), (i16 SReg_32:$src1))),
   (S_PACK_LL_B32_B16 SReg_32:$src0, SReg_32:$src1)
 >;
 
+def : GCNPat <
+  (v2i16 (DivergentBinFrag<build_vector> (i16 SReg_32:$src0), (i16 SReg_32:$src1))),
+  (v2i16 (V_LSHL_OR_B32_e64 $src1, (i32 16), (i32 (V_AND_B32_e64 (i32 (V_MOV_B32_e32 (i32 0xffff))), $src0))))
+>;
+
 // With multiple uses of the shift, this will duplicate the shift and
 // increase register pressure.
 def : GCNPat <
-  (v2i16 (build_vector (i16 SReg_32:$src0), (i16 (trunc (srl_oneuse SReg_32:$src1, (i32 16)))))),
+  (v2i16 (UniformBinFrag<build_vector> (i16 SReg_32:$src0), (i16 (trunc (srl_oneuse SReg_32:$src1, (i32 16)))))),
   (v2i16 (S_PACK_LH_B32_B16 SReg_32:$src0, SReg_32:$src1))
 >;
 
+def : GCNPat <
+  (v2i16 (DivergentBinFrag<build_vector> (i16 SReg_32:$src0), (i16 (trunc (srl_oneuse SReg_32:$src1, (i32 16)))))),
+  (v2i16 (V_BFI_B32_e64 (i32 (V_MOV_B32_e32 (i32 0xffff))), SReg_32:$src0, SReg_32:$src1))
+>;
+
 
 def : GCNPat <
-  (v2i16 (build_vector (i16 (trunc (srl_oneuse SReg_32:$src0, (i32 16)))),
+  (v2i16 (UniformBinFrag<build_vector> (i16 (trunc (srl_oneuse SReg_32:$src0, (i32 16)))),
                        (i16 (trunc (srl_oneuse SReg_32:$src1, (i32 16)))))),
   (S_PACK_HH_B32_B16 SReg_32:$src0, SReg_32:$src1)
 >;
 
-// TODO: Should source modifiers be matched to v_pack_b32_f16?
 def : GCNPat <
-  (v2f16 (build_vector (f16 SReg_32:$src0), (f16 SReg_32:$src1))),
+  (v2i16 (DivergentBinFrag<build_vector> (i16 (trunc (srl_oneuse SReg_32:$src0, (i32 16)))),
+                       (i16 (trunc (srl_oneuse SReg_32:$src1, (i32 16)))))),
+  (v2i16 (V_AND_OR_B32_e64 SReg_32:$src1, (i32 (V_MOV_B32_e32 (i32 0xffff0000))), (i32 (V_LSHRREV_B32_e64 (i32 16), SReg_32:$src0))))
+>;
+
+def : GCNPat <
+  (v2f16 (UniformBinFrag<build_vector> (f16 SReg_32:$src0), (f16 SReg_32:$src1))),
   (S_PACK_LL_B32_B16 SReg_32:$src0, SReg_32:$src1)
 >;
 
 def : GCNPat <
+  (v2f16 (DivergentBinFrag<build_vector> (f16 SReg_32:$src0), (f16 SReg_32:$src1))),
+  (v2f16 (V_LSHL_OR_B32_e64 SReg_32:$src1, (i32 16), (i32 (V_AND_B32_e64 (i32 (V_MOV_B32_e32 (i32 0xffff))), SReg_32:$src0))))
+>;
+
+
+def : GCNPat <
   (v2f16 (is_canonicalized<build_vector> (f16 (VOP3Mods (f16 VGPR_32:$src0), i32:$src0_mods)),
                                          (f16 (VOP3Mods (f16 VGPR_32:$src1), i32:$src1_mods)))),
   (V_PACK_B32_F16_e64 $src0_mods, VGPR_32:$src0, $src1_mods, VGPR_32:$src1)
@@ -2866,6 +2938,18 @@ def G_AMDGPU_UMED3 : AMDGPUGenericInstruction {
   let hasSideEffects = 0;
 }
 
+def G_AMDGPU_FMED3 : AMDGPUGenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src0, type0:$src1, type0:$src2);
+  let hasSideEffects = 0;
+}
+
+def G_AMDGPU_CLAMP : AMDGPUGenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src);
+  let hasSideEffects = 0;
+}
+
 // Atomic cmpxchg. $cmpval ad $newval are packed in a single vector
 // operand Expects a MachineMemOperand in addition to explicit
 // operands.
diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
index c4007f56f350..3ce368ef4db9 100644
--- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
@@ -62,11 +62,6 @@ SIMachineFunctionInfo::SIMachineFunctionInfo(const MachineFunction &MF)
   // calls.
   const bool HasCalls = F.hasFnAttribute("amdgpu-calls");
 
-  // Enable all kernel inputs if we have the fixed ABI. Don't bother if we don't
-  // have any calls.
-  const bool UseFixedABI = AMDGPUTargetMachine::EnableFixedFunctionABI &&
-                           CC != CallingConv::AMDGPU_Gfx &&
-                           (!isEntryFunction() || HasCalls);
   const bool IsKernel = CC == CallingConv::AMDGPU_KERNEL ||
                         CC == CallingConv::SPIR_KERNEL;
 
@@ -80,7 +75,7 @@ SIMachineFunctionInfo::SIMachineFunctionInfo(const MachineFunction &MF)
   }
 
   if (!isEntryFunction()) {
-    if (UseFixedABI)
+    if (CC != CallingConv::AMDGPU_Gfx)
       ArgInfo = AMDGPUArgumentUsageInfo::FixedABIFunctionInfo;
 
     // TODO: Pick a high register, and shift down, similar to a kernel.
@@ -110,20 +105,7 @@ SIMachineFunctionInfo::SIMachineFunctionInfo(const MachineFunction &MF)
   else if (ST.isMesaGfxShader(F))
     ImplicitBufferPtr = true;
 
-  if (UseFixedABI) {
-    DispatchPtr = true;
-    QueuePtr = true;
-    ImplicitArgPtr = true;
-    WorkGroupIDX = true;
-    WorkGroupIDY = true;
-    WorkGroupIDZ = true;
-    WorkItemIDX = true;
-    WorkItemIDY = true;
-    WorkItemIDZ = true;
-
-    // FIXME: We don't need this?
-    DispatchID = true;
-  } else if (!AMDGPU::isGraphics(CC)) {
+  if (!AMDGPU::isGraphics(CC)) {
     if (IsKernel || !F.hasFnAttribute("amdgpu-no-workgroup-id-x"))
       WorkGroupIDX = true;
 
@@ -462,7 +444,7 @@ void SIMachineFunctionInfo::removeDeadFrameIndices(MachineFrameInfo &MFI) {
       MFI.setStackID(i, TargetStackID::Default);
 
   for (auto &R : VGPRToAGPRSpills) {
-    if (R.second.FullyAllocated)
+    if (R.second.IsDead)
       MFI.RemoveStackObject(R.first);
   }
 }
diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
index c305bc20e40d..8accbf611c5f 100644
--- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
@@ -465,6 +465,7 @@ public:
   struct VGPRSpillToAGPR {
     SmallVector<MCPhysReg, 32> Lanes;
     bool FullyAllocated = false;
+    bool IsDead = false;
   };
 
   // Map WWM VGPR to a stack slot that is used to save/restore it in the
@@ -546,6 +547,12 @@ public:
                                          : I->second.Lanes[Lane];
   }
 
+  void setVGPRToAGPRSpillDead(int FrameIndex) {
+    auto I = VGPRToAGPRSpills.find(FrameIndex);
+    if (I != VGPRToAGPRSpills.end())
+      I->second.IsDead = true;
+  }
+
   bool haveFreeLanesForSGPRSpill(const MachineFunction &MF,
                                  unsigned NumLane) const;
   bool allocateSGPRSpillToVGPR(MachineFunction &MF, int FI);
diff --git a/llvm/lib/Target/AMDGPU/SIMachineScheduler.cpp b/llvm/lib/Target/AMDGPU/SIMachineScheduler.cpp
index 5590d84cc3ab..81db66a98ddf 100644
--- a/llvm/lib/Target/AMDGPU/SIMachineScheduler.cpp
+++ b/llvm/lib/Target/AMDGPU/SIMachineScheduler.cpp
@@ -869,29 +869,27 @@ void SIScheduleBlockCreator::colorComputeReservedDependencies() {
 }
 
 void SIScheduleBlockCreator::colorAccordingToReservedDependencies() {
-  unsigned DAGSize = DAG->SUnits.size();
   std::map<std::pair<unsigned, unsigned>, unsigned> ColorCombinations;
 
   // Every combination of colors given by the top down
   // and bottom up Reserved node dependency
 
-  for (unsigned i = 0, e = DAGSize; i != e; ++i) {
-    SUnit *SU = &DAG->SUnits[i];
+  for (const SUnit &SU : DAG->SUnits) {
     std::pair<unsigned, unsigned> SUColors;
 
     // High latency instructions: already given.
-    if (CurrentColoring[SU->NodeNum])
+    if (CurrentColoring[SU.NodeNum])
       continue;
 
-    SUColors.first = CurrentTopDownReservedDependencyColoring[SU->NodeNum];
-    SUColors.second = CurrentBottomUpReservedDependencyColoring[SU->NodeNum];
+    SUColors.first = CurrentTopDownReservedDependencyColoring[SU.NodeNum];
+    SUColors.second = CurrentBottomUpReservedDependencyColoring[SU.NodeNum];
 
     std::map<std::pair<unsigned, unsigned>, unsigned>::iterator Pos =
       ColorCombinations.find(SUColors);
     if (Pos != ColorCombinations.end()) {
-      CurrentColoring[SU->NodeNum] = Pos->second;
+      CurrentColoring[SU.NodeNum] = Pos->second;
     } else {
-      CurrentColoring[SU->NodeNum] = NextNonReservedID;
+      CurrentColoring[SU.NodeNum] = NextNonReservedID;
       ColorCombinations[SUColors] = NextNonReservedID++;
     }
   }
@@ -1232,15 +1230,13 @@ void SIScheduleBlockCreator::createBlocksForVariant(SISchedulerBlockCreatorVaria
   }
 
   // Free root and leafs of all blocks to enable scheduling inside them.
-  for (unsigned i = 0, e = CurrentBlocks.size(); i != e; ++i) {
-    SIScheduleBlock *Block = CurrentBlocks[i];
+  for (SIScheduleBlock *Block : CurrentBlocks)
     Block->finalizeUnits();
-  }
-  LLVM_DEBUG(dbgs() << "Blocks created:\n\n";
-             for (unsigned i = 0, e = CurrentBlocks.size(); i != e; ++i) {
-               SIScheduleBlock *Block = CurrentBlocks[i];
-               Block->printDebug(true);
-             });
+  LLVM_DEBUG({
+    dbgs() << "Blocks created:\n\n";
+    for (SIScheduleBlock *Block : CurrentBlocks)
+      Block->printDebug(true);
+  });
 }
 
 // Two functions taken from Codegen/MachineScheduler.cpp
@@ -1379,9 +1375,9 @@ void SIScheduleBlockCreator::scheduleInsideBlocks() {
     }
   }
 
-  LLVM_DEBUG(for (unsigned i = 0, e = CurrentBlocks.size(); i != e; ++i) {
-    SIScheduleBlock *Block = CurrentBlocks[i];
-    Block->printDebug(true);
+  LLVM_DEBUG({
+    for (SIScheduleBlock *Block : CurrentBlocks)
+      Block->printDebug(true);
   });
 }
 
@@ -1437,8 +1433,7 @@ SIScheduleBlockScheduler::SIScheduleBlockScheduler(SIScheduleDAGMI *DAG,
   // found for several parents, we increment the usage of the one with the
   // highest topological index.
   LiveOutRegsNumUsages.resize(Blocks.size());
-  for (unsigned i = 0, e = Blocks.size(); i != e; ++i) {
-    SIScheduleBlock *Block = Blocks[i];
+  for (SIScheduleBlock *Block : Blocks) {
     for (unsigned Reg : Block->getInRegs()) {
       bool Found = false;
       int topoInd = -1;
@@ -1502,8 +1497,7 @@ SIScheduleBlockScheduler::SIScheduleBlockScheduler(SIScheduleDAGMI *DAG,
 
   // Fill LiveRegsConsumers for regs that were already
   // defined before scheduling.
-  for (unsigned i = 0, e = Blocks.size(); i != e; ++i) {
-    SIScheduleBlock *Block = Blocks[i];
+  for (SIScheduleBlock *Block : Blocks) {
     for (unsigned Reg : Block->getInRegs()) {
       bool Found = false;
       for (SIScheduleBlock* Pred: Block->getPreds()) {
@@ -1700,10 +1694,7 @@ void SIScheduleBlockScheduler::blockScheduled(SIScheduleBlock *Block) {
   decreaseLiveRegs(Block, Block->getInRegs());
   addLiveRegs(Block->getOutRegs());
   releaseBlockSuccs(Block);
-  for (std::map<unsigned, unsigned>::iterator RegI =
-       LiveOutRegsNumUsages[Block->getID()].begin(),
-       E = LiveOutRegsNumUsages[Block->getID()].end(); RegI != E; ++RegI) {
-    std::pair<unsigned, unsigned> RegP = *RegI;
+  for (const auto &RegP : LiveOutRegsNumUsages[Block->getID()]) {
     // We produce this register, thus it must not be previously alive.
     assert(LiveRegsConsumers.find(RegP.first) == LiveRegsConsumers.end() ||
            LiveRegsConsumers[RegP.first] == 0);
@@ -1759,8 +1750,7 @@ SIScheduler::scheduleVariant(SISchedulerBlockCreatorVariant BlockVariant,
 
   ScheduledBlocks = Scheduler.getBlocks();
 
-  for (unsigned b = 0; b < ScheduledBlocks.size(); ++b) {
-    SIScheduleBlock *Block = ScheduledBlocks[b];
+  for (SIScheduleBlock *Block : ScheduledBlocks) {
     std::vector<SUnit*> SUs = Block->getScheduledUnits();
 
     for (SUnit* SU : SUs)
@@ -2000,9 +1990,8 @@ void SIScheduleDAGMI::schedule()
   assert(TopRPTracker.getPos() == RegionBegin && "bad initial Top tracker");
   TopRPTracker.setPos(CurrentTop);
 
-  for (std::vector<unsigned>::iterator I = ScheduledSUnits.begin(),
-       E = ScheduledSUnits.end(); I != E; ++I) {
-    SUnit *SU = &SUnits[*I];
+  for (unsigned I : ScheduledSUnits) {
+    SUnit *SU = &SUnits[I];
 
     scheduleMI(SU, true);
 
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index a1d9a23a5084..21aed4ececb5 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -210,6 +210,7 @@ struct SGPRSpillBuilder {
       auto I = BuildMI(*MBB, MI, DL, TII.get(NotOpc), ExecReg).addReg(ExecReg);
       if (!TmpVGPRLive)
         I.addReg(TmpVGPR, RegState::ImplicitDefine);
+      I->getOperand(2).setIsDead(true); // Mark SCC as dead.
       TRI.buildVGPRSpillLoadStore(*this, TmpVGPRIndex, 0, /*IsLoad*/ false);
     }
   }
@@ -242,9 +243,10 @@ struct SGPRSpillBuilder {
       TRI.buildVGPRSpillLoadStore(*this, TmpVGPRIndex, 0, /*IsLoad*/ true,
                                   /*IsKill*/ false);
       auto I = BuildMI(*MBB, MI, DL, TII.get(NotOpc), ExecReg).addReg(ExecReg);
-      if (!TmpVGPRLive) {
+      if (!TmpVGPRLive)
         I.addReg(TmpVGPR, RegState::ImplicitKill);
-      }
+      I->getOperand(2).setIsDead(true); // Mark SCC as dead.
+
       // Restore active lanes
       if (TmpVGPRLive)
         TRI.buildVGPRSpillLoadStore(*this, TmpVGPRIndex, 0, /*IsLoad*/ true);
@@ -267,9 +269,11 @@ struct SGPRSpillBuilder {
       TRI.buildVGPRSpillLoadStore(*this, Index, Offset, IsLoad,
                                   /*IsKill*/ false);
       // Spill inactive lanes
-      BuildMI(*MBB, MI, DL, TII.get(NotOpc), ExecReg).addReg(ExecReg);
+      auto Not0 = BuildMI(*MBB, MI, DL, TII.get(NotOpc), ExecReg).addReg(ExecReg);
+      Not0->getOperand(2).setIsDead(); // Mark SCC as dead.
       TRI.buildVGPRSpillLoadStore(*this, Index, Offset, IsLoad);
-      BuildMI(*MBB, MI, DL, TII.get(NotOpc), ExecReg).addReg(ExecReg);
+      auto Not1 = BuildMI(*MBB, MI, DL, TII.get(NotOpc), ExecReg).addReg(ExecReg);
+      Not1->getOperand(2).setIsDead(); // Mark SCC as dead.
     }
   }
 
@@ -908,6 +912,8 @@ static unsigned getNumSubRegsForSpillOp(unsigned Op) {
   case AMDGPU::SI_SPILL_V1024_RESTORE:
   case AMDGPU::SI_SPILL_A1024_SAVE:
   case AMDGPU::SI_SPILL_A1024_RESTORE:
+  case AMDGPU::SI_SPILL_AV1024_SAVE:
+  case AMDGPU::SI_SPILL_AV1024_RESTORE:
     return 32;
   case AMDGPU::SI_SPILL_S512_SAVE:
   case AMDGPU::SI_SPILL_S512_RESTORE:
@@ -915,6 +921,8 @@ static unsigned getNumSubRegsForSpillOp(unsigned Op) {
   case AMDGPU::SI_SPILL_V512_RESTORE:
   case AMDGPU::SI_SPILL_A512_SAVE:
   case AMDGPU::SI_SPILL_A512_RESTORE:
+  case AMDGPU::SI_SPILL_AV512_SAVE:
+  case AMDGPU::SI_SPILL_AV512_RESTORE:
     return 16;
   case AMDGPU::SI_SPILL_S256_SAVE:
   case AMDGPU::SI_SPILL_S256_RESTORE:
@@ -922,6 +930,8 @@ static unsigned getNumSubRegsForSpillOp(unsigned Op) {
   case AMDGPU::SI_SPILL_V256_RESTORE:
   case AMDGPU::SI_SPILL_A256_SAVE:
   case AMDGPU::SI_SPILL_A256_RESTORE:
+  case AMDGPU::SI_SPILL_AV256_SAVE:
+  case AMDGPU::SI_SPILL_AV256_RESTORE:
     return 8;
   case AMDGPU::SI_SPILL_S224_SAVE:
   case AMDGPU::SI_SPILL_S224_RESTORE:
@@ -929,6 +939,8 @@ static unsigned getNumSubRegsForSpillOp(unsigned Op) {
   case AMDGPU::SI_SPILL_V224_RESTORE:
   case AMDGPU::SI_SPILL_A224_SAVE:
   case AMDGPU::SI_SPILL_A224_RESTORE:
+  case AMDGPU::SI_SPILL_AV224_SAVE:
+  case AMDGPU::SI_SPILL_AV224_RESTORE:
     return 7;
   case AMDGPU::SI_SPILL_S192_SAVE:
   case AMDGPU::SI_SPILL_S192_RESTORE:
@@ -936,6 +948,8 @@ static unsigned getNumSubRegsForSpillOp(unsigned Op) {
   case AMDGPU::SI_SPILL_V192_RESTORE:
   case AMDGPU::SI_SPILL_A192_SAVE:
   case AMDGPU::SI_SPILL_A192_RESTORE:
+  case AMDGPU::SI_SPILL_AV192_SAVE:
+  case AMDGPU::SI_SPILL_AV192_RESTORE:
     return 6;
   case AMDGPU::SI_SPILL_S160_SAVE:
   case AMDGPU::SI_SPILL_S160_RESTORE:
@@ -943,6 +957,8 @@ static unsigned getNumSubRegsForSpillOp(unsigned Op) {
   case AMDGPU::SI_SPILL_V160_RESTORE:
   case AMDGPU::SI_SPILL_A160_SAVE:
   case AMDGPU::SI_SPILL_A160_RESTORE:
+  case AMDGPU::SI_SPILL_AV160_SAVE:
+  case AMDGPU::SI_SPILL_AV160_RESTORE:
     return 5;
   case AMDGPU::SI_SPILL_S128_SAVE:
   case AMDGPU::SI_SPILL_S128_RESTORE:
@@ -950,6 +966,8 @@ static unsigned getNumSubRegsForSpillOp(unsigned Op) {
   case AMDGPU::SI_SPILL_V128_RESTORE:
   case AMDGPU::SI_SPILL_A128_SAVE:
   case AMDGPU::SI_SPILL_A128_RESTORE:
+  case AMDGPU::SI_SPILL_AV128_SAVE:
+  case AMDGPU::SI_SPILL_AV128_RESTORE:
     return 4;
   case AMDGPU::SI_SPILL_S96_SAVE:
   case AMDGPU::SI_SPILL_S96_RESTORE:
@@ -957,6 +975,8 @@ static unsigned getNumSubRegsForSpillOp(unsigned Op) {
   case AMDGPU::SI_SPILL_V96_RESTORE:
   case AMDGPU::SI_SPILL_A96_SAVE:
   case AMDGPU::SI_SPILL_A96_RESTORE:
+  case AMDGPU::SI_SPILL_AV96_SAVE:
+  case AMDGPU::SI_SPILL_AV96_RESTORE:
     return 3;
   case AMDGPU::SI_SPILL_S64_SAVE:
   case AMDGPU::SI_SPILL_S64_RESTORE:
@@ -964,6 +984,8 @@ static unsigned getNumSubRegsForSpillOp(unsigned Op) {
   case AMDGPU::SI_SPILL_V64_RESTORE:
   case AMDGPU::SI_SPILL_A64_SAVE:
   case AMDGPU::SI_SPILL_A64_RESTORE:
+  case AMDGPU::SI_SPILL_AV64_SAVE:
+  case AMDGPU::SI_SPILL_AV64_RESTORE:
     return 2;
   case AMDGPU::SI_SPILL_S32_SAVE:
   case AMDGPU::SI_SPILL_S32_RESTORE:
@@ -971,6 +993,8 @@ static unsigned getNumSubRegsForSpillOp(unsigned Op) {
   case AMDGPU::SI_SPILL_V32_RESTORE:
   case AMDGPU::SI_SPILL_A32_SAVE:
   case AMDGPU::SI_SPILL_A32_RESTORE:
+  case AMDGPU::SI_SPILL_AV32_SAVE:
+  case AMDGPU::SI_SPILL_AV32_RESTORE:
     return 1;
   default: llvm_unreachable("Invalid spill opcode");
   }
@@ -1240,9 +1264,10 @@ void SIRegisterInfo::buildSpillLoadStore(
     if (ScratchOffsetReg == AMDGPU::NoRegister) {
       BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_MOV_B32), SOffset).addImm(Offset);
     } else {
-      BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_ADD_I32), SOffset)
+      auto Add = BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_ADD_I32), SOffset)
           .addReg(ScratchOffsetReg)
           .addImm(Offset);
+      Add->getOperand(3).setIsDead(); // Mark SCC as dead.
     }
 
     Offset = 0;
@@ -1810,7 +1835,17 @@ void SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
     case AMDGPU::SI_SPILL_A128_SAVE:
     case AMDGPU::SI_SPILL_A96_SAVE:
     case AMDGPU::SI_SPILL_A64_SAVE:
-    case AMDGPU::SI_SPILL_A32_SAVE: {
+    case AMDGPU::SI_SPILL_A32_SAVE:
+    case AMDGPU::SI_SPILL_AV1024_SAVE:
+    case AMDGPU::SI_SPILL_AV512_SAVE:
+    case AMDGPU::SI_SPILL_AV256_SAVE:
+    case AMDGPU::SI_SPILL_AV224_SAVE:
+    case AMDGPU::SI_SPILL_AV192_SAVE:
+    case AMDGPU::SI_SPILL_AV160_SAVE:
+    case AMDGPU::SI_SPILL_AV128_SAVE:
+    case AMDGPU::SI_SPILL_AV96_SAVE:
+    case AMDGPU::SI_SPILL_AV64_SAVE:
+    case AMDGPU::SI_SPILL_AV32_SAVE: {
       const MachineOperand *VData = TII->getNamedOperand(*MI,
                                                          AMDGPU::OpName::vdata);
       assert(TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)->getReg() ==
@@ -1846,7 +1881,17 @@ void SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
     case AMDGPU::SI_SPILL_A224_RESTORE:
     case AMDGPU::SI_SPILL_A256_RESTORE:
     case AMDGPU::SI_SPILL_A512_RESTORE:
-    case AMDGPU::SI_SPILL_A1024_RESTORE: {
+    case AMDGPU::SI_SPILL_A1024_RESTORE:
+    case AMDGPU::SI_SPILL_AV32_RESTORE:
+    case AMDGPU::SI_SPILL_AV64_RESTORE:
+    case AMDGPU::SI_SPILL_AV96_RESTORE:
+    case AMDGPU::SI_SPILL_AV128_RESTORE:
+    case AMDGPU::SI_SPILL_AV160_RESTORE:
+    case AMDGPU::SI_SPILL_AV192_RESTORE:
+    case AMDGPU::SI_SPILL_AV224_RESTORE:
+    case AMDGPU::SI_SPILL_AV256_RESTORE:
+    case AMDGPU::SI_SPILL_AV512_RESTORE:
+    case AMDGPU::SI_SPILL_AV1024_RESTORE: {
       const MachineOperand *VData = TII->getNamedOperand(*MI,
                                                          AMDGPU::OpName::vdata);
       assert(TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)->getReg() ==
diff --git a/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp b/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp
index 3a372d4519fb..c8f1daf26de9 100644
--- a/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp
+++ b/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp
@@ -731,11 +731,6 @@ bool SIShrinkInstructions::runOnMachineFunction(MachineFunction &MF) {
           continue;
       }
 
-      // getVOPe32 could be -1 here if we started with an instruction that had
-      // a 32-bit encoding and then commuted it to an instruction that did not.
-      if (!TII->hasVALU32BitEncoding(MI.getOpcode()))
-        continue;
-
       int Op32 = AMDGPU::getVOPe32(MI.getOpcode());
 
       if (TII->isVOPC(Op32)) {
@@ -776,10 +771,6 @@ bool SIShrinkInstructions::runOnMachineFunction(MachineFunction &MF) {
       const MachineOperand *SDst = TII->getNamedOperand(MI,
                                                         AMDGPU::OpName::sdst);
 
-      // Check the carry-in operand for v_addc_u32_e64.
-      const MachineOperand *Src2 = TII->getNamedOperand(MI,
-                                                        AMDGPU::OpName::src2);
-
       if (SDst) {
         bool Next = false;
 
@@ -791,6 +782,8 @@ bool SIShrinkInstructions::runOnMachineFunction(MachineFunction &MF) {
 
         // All of the instructions with carry outs also have an SGPR input in
         // src2.
+        const MachineOperand *Src2 = TII->getNamedOperand(MI,
+                                                          AMDGPU::OpName::src2);
         if (Src2 && Src2->getReg() != VCCReg) {
           if (Src2->getReg().isVirtual())
             MRI.setRegAllocationHint(Src2->getReg(), 0, VCCReg);
diff --git a/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp b/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp
index 46012e5d7d97..77ee3c0ff0e4 100644
--- a/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp
+++ b/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp
@@ -495,11 +495,10 @@ char SIWholeQuadMode::scanInstructions(MachineFunction &MF,
   // instruction as needing e.g. WQM before visiting it and realizing it needs
   // WQM disabled.
   ReversePostOrderTraversal<MachineFunction *> RPOT(&MF);
-  for (auto BI = RPOT.begin(), BE = RPOT.end(); BI != BE; ++BI) {
-    MachineBasicBlock &MBB = **BI;
-    BlockInfo &BBI = Blocks[&MBB];
+  for (MachineBasicBlock *MBB : RPOT) {
+    BlockInfo &BBI = Blocks[MBB];
 
-    for (MachineInstr &MI : MBB) {
+    for (MachineInstr &MI : *MBB) {
       InstrInfo &III = Instructions[&MI];
       unsigned Opcode = MI.getOpcode();
       char Flags = 0;
@@ -561,7 +560,7 @@ char SIWholeQuadMode::scanInstructions(MachineFunction &MF,
         BBI.Needs |= StateExact;
         if (!(BBI.InNeeds & StateExact)) {
           BBI.InNeeds |= StateExact;
-          Worklist.push_back(&MBB);
+          Worklist.push_back(MBB);
         }
         GlobalFlags |= StateExact;
         III.Disabled = StateWQM | StateStrict;
diff --git a/llvm/lib/Target/AMDGPU/SMInstructions.td b/llvm/lib/Target/AMDGPU/SMInstructions.td
index 8502ed61b366..184c871db775 100644
--- a/llvm/lib/Target/AMDGPU/SMInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SMInstructions.td
@@ -181,15 +181,8 @@ class SM_Time_Pseudo<string opName, SDPatternOperator node = null_frag> : SM_Pse
   " $sdst", [(set i64:$sdst, (node))]> {
   let hasSideEffects = 1;
 
-  // FIXME: This should be definitively mayStore = 0. TableGen
-  // brokenly tries to infer these based on the intrinsic properties
-  // corresponding to the IR attributes. The target intrinsics are
-  // considered as writing to memory for IR dependency purposes, but
-  // those can be modeled with hasSideEffects here. These also end up
-  // inferring differently for llvm.readcyclecounter and the amdgcn
-  // intrinsics.
-  let mayStore = ?;
-  let mayLoad = 1;
+  let mayStore = 0;
+  let mayLoad = 0;
   let has_sbase = 0;
   let has_offset = 0;
 }
@@ -765,11 +758,11 @@ def smrd_load : PatFrag <(ops node:$ptr), (load node:$ptr), [{ return isUniformL
   }];
 }
 
-def SMRDImm         : ComplexPattern<i64, 2, "SelectSMRDImm">;
-def SMRDImm32       : ComplexPattern<i64, 2, "SelectSMRDImm32">;
-def SMRDSgpr        : ComplexPattern<i64, 2, "SelectSMRDSgpr">;
-def SMRDBufferImm   : ComplexPattern<i32, 1, "SelectSMRDBufferImm">;
-def SMRDBufferImm32 : ComplexPattern<i32, 1, "SelectSMRDBufferImm32">;
+def SMRDImm         : ComplexPattern<iPTR, 2, "SelectSMRDImm">;
+def SMRDImm32       : ComplexPattern<iPTR, 2, "SelectSMRDImm32">;
+def SMRDSgpr        : ComplexPattern<iPTR, 2, "SelectSMRDSgpr">;
+def SMRDBufferImm   : ComplexPattern<iPTR, 1, "SelectSMRDBufferImm">;
+def SMRDBufferImm32 : ComplexPattern<iPTR, 1, "SelectSMRDBufferImm32">;
 
 multiclass SMRD_Pattern <string Instr, ValueType vt> {
 
diff --git a/llvm/lib/Target/AMDGPU/SOPInstructions.td b/llvm/lib/Target/AMDGPU/SOPInstructions.td
index 61ecc13620a1..1713586dcf5b 100644
--- a/llvm/lib/Target/AMDGPU/SOPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SOPInstructions.td
@@ -157,6 +157,42 @@ class SOP1_1 <string opName, RegisterClass rc = SReg_64, list<dag> pattern=[]> :
   let has_sdst = 0;
 }
 
+class UniformUnaryFrag<SDPatternOperator Op> : PatFrag <
+  (ops node:$src0),
+  (Op $src0),
+  [{ return !N->isDivergent(); }]> {
+  // This check is unnecessary as it's captured by the result register
+  // bank constraint.
+  //
+  // FIXME: Should add a way for the emitter to recognize this is a
+  // trivially true predicate to eliminate the check.
+  let GISelPredicateCode = [{return true;}];
+}
+
+class UniformBinFrag<SDPatternOperator Op> : PatFrag <
+  (ops node:$src0, node:$src1),
+  (Op $src0, $src1),
+  [{ return !N->isDivergent(); }]> {
+  // This check is unnecessary as it's captured by the result register
+  // bank constraint.
+  //
+  // FIXME: Should add a way for the emitter to recognize this is a
+  // trivially true predicate to eliminate the check.
+  let GISelPredicateCode = [{return true;}];
+}
+
+class DivergentBinFrag<SDPatternOperator Op> : PatFrag <
+  (ops node:$src0, node:$src1),
+  (Op $src0, $src1),
+  [{ return N->isDivergent(); }]> {
+  // This check is unnecessary as it's captured by the result register
+  // bank constraint.
+  //
+  // FIXME: Should add a way for the emitter to recognize this is a
+  // trivially true predicate to eliminate the check.
+  let GISelPredicateCode = [{return true;}];
+}
+
 
 let isMoveImm = 1 in {
   let isReMaterializable = 1, isAsCheapAsAMove = 1 in {
@@ -172,11 +208,11 @@ let isMoveImm = 1 in {
 
 let Defs = [SCC] in {
   def S_NOT_B32 : SOP1_32 <"s_not_b32",
-    [(set i32:$sdst, (not i32:$src0))]
+    [(set i32:$sdst, (UniformUnaryFrag<not> i32:$src0))]
   >;
 
   def S_NOT_B64 : SOP1_64 <"s_not_b64",
-    [(set i64:$sdst, (not i64:$src0))]
+    [(set i64:$sdst, (UniformUnaryFrag<not> i64:$src0))]
   >;
   def S_WQM_B32 : SOP1_32 <"s_wqm_b32">;
   def S_WQM_B64 : SOP1_64 <"s_wqm_b64">;
@@ -221,22 +257,22 @@ let isReMaterializable = 1 in {
 def S_FF0_I32_B32 : SOP1_32 <"s_ff0_i32_b32">;
 def S_FF0_I32_B64 : SOP1_32_64 <"s_ff0_i32_b64">;
 def S_FF1_I32_B64 : SOP1_32_64 <"s_ff1_i32_b64",
-  [(set i32:$sdst, (AMDGPUffbl_b32 i64:$src0))]
+  [(set i32:$sdst, (UniformUnaryFrag<AMDGPUffbl_b32> i64:$src0))]
 >;
 
 def S_FF1_I32_B32 : SOP1_32 <"s_ff1_i32_b32",
-  [(set i32:$sdst, (AMDGPUffbl_b32 i32:$src0))]
+  [(set i32:$sdst, (UniformUnaryFrag<AMDGPUffbl_b32> i32:$src0))]
 >;
 
 def S_FLBIT_I32_B32 : SOP1_32 <"s_flbit_i32_b32",
-  [(set i32:$sdst, (AMDGPUffbh_u32 i32:$src0))]
+  [(set i32:$sdst, (UniformUnaryFrag<AMDGPUffbh_u32> i32:$src0))]
 >;
 
 def S_FLBIT_I32_B64 : SOP1_32_64 <"s_flbit_i32_b64",
-  [(set i32:$sdst, (AMDGPUffbh_u32 i64:$src0))]
+  [(set i32:$sdst, (UniformUnaryFrag<AMDGPUffbh_u32> i64:$src0))]
 >;
 def S_FLBIT_I32 : SOP1_32 <"s_flbit_i32",
-  [(set i32:$sdst, (AMDGPUffbh_i32 i32:$src0))]
+  [(set i32:$sdst, (UniformUnaryFrag<AMDGPUffbh_i32> i32:$src0))]
 >;
 def S_FLBIT_I32_I64 : SOP1_32_64 <"s_flbit_i32_i64">;
 def S_SEXT_I32_I8 : SOP1_32 <"s_sext_i32_i8",
@@ -426,41 +462,6 @@ class SOP2_64_32_32 <string opName, list<dag> pattern=[]> : SOP2_Pseudo <
   "$sdst, $src0, $src1", pattern
 >;
 
-class UniformUnaryFrag<SDPatternOperator Op> : PatFrag <
-  (ops node:$src0),
-  (Op $src0),
-  [{ return !N->isDivergent(); }]> {
-  // This check is unnecessary as it's captured by the result register
-  // bank constraint.
-  //
-  // FIXME: Should add a way for the emitter to recognize this is a
-  // trivially true predicate to eliminate the check.
-  let GISelPredicateCode = [{return true;}];
-}
-
-class UniformBinFrag<SDPatternOperator Op> : PatFrag <
-  (ops node:$src0, node:$src1),
-  (Op $src0, $src1),
-  [{ return !N->isDivergent(); }]> {
-  // This check is unnecessary as it's captured by the result register
-  // bank constraint.
-  //
-  // FIXME: Should add a way for the emitter to recognize this is a
-  // trivially true predicate to eliminate the check.
-  let GISelPredicateCode = [{return true;}];
-}
-
-class DivergentBinFrag<SDPatternOperator Op> : PatFrag <
-  (ops node:$src0, node:$src1),
-  (Op $src0, $src1),
-  [{ return N->isDivergent(); }]> {
-  // This check is unnecessary as it's captured by the result register
-  // bank constraint.
-  //
-  // FIXME: Should add a way for the emitter to recognize this is a
-  // trivially true predicate to eliminate the check.
-  let GISelPredicateCode = [{return true;}];
-}
 
 let Defs = [SCC] in { // Carry out goes to SCC
 let isCommutable = 1 in {
@@ -485,19 +486,18 @@ def S_SUBB_U32 : SOP2_32 <"s_subb_u32",
   [(set i32:$sdst, (UniformBinFrag<sube> (i32 SSrc_b32:$src0), (i32 SSrc_b32:$src1)))]>;
 } // End Uses = [SCC]
 
-
 let isCommutable = 1 in {
 def S_MIN_I32 : SOP2_32 <"s_min_i32",
-  [(set i32:$sdst, (smin i32:$src0, i32:$src1))]
+  [(set i32:$sdst, (UniformBinFrag<smin> i32:$src0, i32:$src1))]
 >;
 def S_MIN_U32 : SOP2_32 <"s_min_u32",
-  [(set i32:$sdst, (umin i32:$src0, i32:$src1))]
+  [(set i32:$sdst, (UniformBinFrag<umin> i32:$src0, i32:$src1))]
 >;
 def S_MAX_I32 : SOP2_32 <"s_max_i32",
-  [(set i32:$sdst, (smax i32:$src0, i32:$src1))]
+  [(set i32:$sdst, (UniformBinFrag<smax> i32:$src0, i32:$src1))]
 >;
 def S_MAX_U32 : SOP2_32 <"s_max_u32",
-  [(set i32:$sdst, (umax i32:$src0, i32:$src1))]
+  [(set i32:$sdst, (UniformBinFrag<umax> i32:$src0, i32:$src1))]
 >;
 } // End isCommutable = 1
 } // End Defs = [SCC]
@@ -870,7 +870,7 @@ def S_GETREG_B32 : SOPK_Pseudo <
 }
 } // End mayLoad = 1
 
-let mayLoad = 0, mayStore = 0, Defs = [MODE], Uses = [MODE] in {
+let Defs = [MODE], Uses = [MODE] in {
 
 // FIXME: Need to truncate immediate to 16-bits.
 class S_SETREG_B32_Pseudo <list<dag> pattern=[]> : SOPK_Pseudo <
@@ -914,7 +914,7 @@ def S_SETREG_IMM32_B32_mode : S_SETREG_IMM32_B32_Pseudo {
   let hasSideEffects = 0;
 }
 
-} // End mayLoad = 0, mayStore = 0, Defs = [MODE], Uses = [MODE]
+} // End Defs = [MODE], Uses = [MODE]
 
 class SOPK_WAITCNT<string opName, list<dag> pat=[]> :
     SOPK_Pseudo<
@@ -1264,7 +1264,7 @@ def S_WAKEUP : SOPP_Pseudo <"s_wakeup", (ins) > {
   let mayStore = 1;
 }
 
-let mayLoad = 0, mayStore = 0, hasSideEffects = 1 in
+let hasSideEffects = 1 in
 def S_WAITCNT : SOPP_Pseudo <"s_waitcnt" , (ins WAIT_FLAG:$simm16), "$simm16",
     [(int_amdgcn_s_waitcnt timm:$simm16)]>;
 def S_SETHALT : SOPP_Pseudo <"s_sethalt" , (ins i32imm:$simm16), "$simm16",
@@ -1278,8 +1278,6 @@ def S_SETKILL : SOPP_Pseudo <"s_setkill" , (ins i16imm:$simm16), "$simm16">;
 def S_SLEEP : SOPP_Pseudo <"s_sleep", (ins i32imm:$simm16),
   "$simm16", [(int_amdgcn_s_sleep timm:$simm16)]> {
   let hasSideEffects = 1;
-  let mayLoad = 0;
-  let mayStore = 0;
 }
 
 def S_SETPRIO : SOPP_Pseudo <"s_setprio" , (ins i16imm:$simm16), "$simm16">;
@@ -1305,14 +1303,10 @@ def S_ICACHE_INV : SOPP_Pseudo <"s_icache_inv", (ins)> {
 def S_INCPERFLEVEL : SOPP_Pseudo <"s_incperflevel", (ins i32imm:$simm16), "$simm16",
   [(int_amdgcn_s_incperflevel timm:$simm16)]> {
   let hasSideEffects = 1;
-  let mayLoad = 0;
-  let mayStore = 0;
 }
 def S_DECPERFLEVEL : SOPP_Pseudo <"s_decperflevel", (ins i32imm:$simm16), "$simm16",
   [(int_amdgcn_s_decperflevel timm:$simm16)]> {
   let hasSideEffects = 1;
-  let mayLoad = 0;
-  let mayStore = 0;
 }
 def S_TTRACEDATA : SOPP_Pseudo <"s_ttracedata", (ins)> {
   let simm16 = 0;
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.cpp
index 2e4d83fbbc39..a83ff6667956 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.cpp
@@ -15,7 +15,6 @@
 #include "Utils/AMDGPUBaseInfo.h"
 #include "llvm/ADT/DepthFirstIterator.h"
 #include "llvm/ADT/SetVector.h"
-#include "llvm/Analysis/CallGraph.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/ReplaceConstant.h"
 
@@ -25,175 +24,6 @@ namespace llvm {
 
 namespace AMDGPU {
 
-// An helper class for collecting all reachable callees for each kernel defined
-// within the module.
-class CollectReachableCallees {
-  Module &M;
-  CallGraph CG;
-  SmallPtrSet<CallGraphNode *, 8> AddressTakenFunctions;
-
-  // Collect all address taken functions within the module.
-  void collectAddressTakenFunctions() {
-    auto *ECNode = CG.getExternalCallingNode();
-
-    for (auto GI = ECNode->begin(), GE = ECNode->end(); GI != GE; ++GI) {
-      auto *CGN = GI->second;
-      auto *F = CGN->getFunction();
-      if (!F || F->isDeclaration() || AMDGPU::isKernelCC(F))
-        continue;
-      AddressTakenFunctions.insert(CGN);
-    }
-  }
-
-  // For given kernel, collect all its reachable non-kernel functions.
-  SmallPtrSet<Function *, 8> collectReachableCallees(Function *K) {
-    SmallPtrSet<Function *, 8> ReachableCallees;
-
-    // Call graph node which represents this kernel.
-    auto *KCGN = CG[K];
-
-    // Go through all call graph nodes reachable from the node representing this
-    // kernel, visit all their call sites, if the call site is direct, add
-    // corresponding callee to reachable callee set, if it is indirect, resolve
-    // the indirect call site to potential reachable callees, add them to
-    // reachable callee set, and repeat the process for the newly added
-    // potential callee nodes.
-    //
-    // FIXME: Need to handle bit-casted function pointers.
-    //
-    SmallVector<CallGraphNode *, 8> CGNStack(df_begin(KCGN), df_end(KCGN));
-    SmallPtrSet<CallGraphNode *, 8> VisitedCGNodes;
-    while (!CGNStack.empty()) {
-      auto *CGN = CGNStack.pop_back_val();
-
-      if (!VisitedCGNodes.insert(CGN).second)
-        continue;
-
-      // Ignore call graph node which does not have associated function or
-      // associated function is not a definition.
-      if (!CGN->getFunction() || CGN->getFunction()->isDeclaration())
-        continue;
-
-      for (auto GI = CGN->begin(), GE = CGN->end(); GI != GE; ++GI) {
-        auto *RCB = cast<CallBase>(GI->first.getValue());
-        auto *RCGN = GI->second;
-
-        if (auto *DCallee = RCGN->getFunction()) {
-          ReachableCallees.insert(DCallee);
-        } else if (RCB->isIndirectCall()) {
-          auto *RCBFTy = RCB->getFunctionType();
-          for (auto *ACGN : AddressTakenFunctions) {
-            auto *ACallee = ACGN->getFunction();
-            if (ACallee->getFunctionType() == RCBFTy) {
-              ReachableCallees.insert(ACallee);
-              CGNStack.append(df_begin(ACGN), df_end(ACGN));
-            }
-          }
-        }
-      }
-    }
-
-    return ReachableCallees;
-  }
-
-public:
-  explicit CollectReachableCallees(Module &M) : M(M), CG(CallGraph(M)) {
-    // Collect address taken functions.
-    collectAddressTakenFunctions();
-  }
-
-  void collectReachableCallees(
-      DenseMap<Function *, SmallPtrSet<Function *, 8>> &KernelToCallees) {
-    // Collect reachable callee set for each kernel defined in the module.
-    for (Function &F : M.functions()) {
-      if (!AMDGPU::isKernelCC(&F))
-        continue;
-      Function *K = &F;
-      KernelToCallees[K] = collectReachableCallees(K);
-    }
-  }
-};
-
-void collectReachableCallees(
-    Module &M,
-    DenseMap<Function *, SmallPtrSet<Function *, 8>> &KernelToCallees) {
-  CollectReachableCallees CRC{M};
-  CRC.collectReachableCallees(KernelToCallees);
-}
-
-SmallPtrSet<Function *, 8> collectNonKernelAccessorsOfLDS(GlobalVariable *GV) {
-  SmallPtrSet<Function *, 8> LDSAccessors;
-  SmallVector<User *, 8> UserStack(GV->users());
-  SmallPtrSet<User *, 8> VisitedUsers;
-
-  while (!UserStack.empty()) {
-    auto *U = UserStack.pop_back_val();
-
-    // `U` is already visited? continue to next one.
-    if (!VisitedUsers.insert(U).second)
-      continue;
-
-    // `U` is a global variable which is initialized with LDS. Ignore LDS.
-    if (isa<GlobalValue>(U))
-      return SmallPtrSet<Function *, 8>();
-
-    // Recursively explore constant users.
-    if (isa<Constant>(U)) {
-      append_range(UserStack, U->users());
-      continue;
-    }
-
-    // `U` should be an instruction, if it belongs to a non-kernel function F,
-    // then collect F.
-    Function *F = cast<Instruction>(U)->getFunction();
-    if (!AMDGPU::isKernelCC(F))
-      LDSAccessors.insert(F);
-  }
-
-  return LDSAccessors;
-}
-
-DenseMap<Function *, SmallPtrSet<Instruction *, 8>>
-getFunctionToInstsMap(User *U, bool CollectKernelInsts) {
-  DenseMap<Function *, SmallPtrSet<Instruction *, 8>> FunctionToInsts;
-  SmallVector<User *, 8> UserStack;
-  SmallPtrSet<User *, 8> VisitedUsers;
-
-  UserStack.push_back(U);
-
-  while (!UserStack.empty()) {
-    auto *UU = UserStack.pop_back_val();
-
-    if (!VisitedUsers.insert(UU).second)
-      continue;
-
-    if (isa<GlobalValue>(UU))
-      continue;
-
-    if (isa<Constant>(UU)) {
-      append_range(UserStack, UU->users());
-      continue;
-    }
-
-    auto *I = cast<Instruction>(UU);
-    Function *F = I->getFunction();
-    if (CollectKernelInsts) {
-      if (!AMDGPU::isKernelCC(F)) {
-        continue;
-      }
-    } else {
-      if (AMDGPU::isKernelCC(F)) {
-        continue;
-      }
-    }
-
-    FunctionToInsts.insert(std::make_pair(F, SmallPtrSet<Instruction *, 8>()));
-    FunctionToInsts[F].insert(I);
-  }
-
-  return FunctionToInsts;
-}
-
 bool isKernelCC(const Function *Func) {
   return AMDGPU::isModuleEntryFunctionCC(Func->getCallingConv());
 }
@@ -232,26 +62,8 @@ void replaceConstantUsesInFunction(ConstantExpr *C, const Function *F) {
   }
 }
 
-bool hasUserInstruction(const GlobalValue *GV) {
-  SmallPtrSet<const User *, 8> Visited;
-  SmallVector<const User *, 16> Stack(GV->users());
-
-  while (!Stack.empty()) {
-    const User *U = Stack.pop_back_val();
-
-    if (!Visited.insert(U).second)
-      continue;
-
-    if (isa<Instruction>(U))
-      return true;
-
-    append_range(Stack, U->users());
-  }
-
-  return false;
-}
-
-bool shouldLowerLDSToStruct(const GlobalVariable &GV, const Function *F) {
+static bool shouldLowerLDSToStruct(const GlobalVariable &GV,
+                                   const Function *F) {
   // We are not interested in kernel LDS lowering for module LDS itself.
   if (F && GV.getName() == "llvm.amdgcn.module.lds")
     return false;
@@ -259,7 +71,6 @@ bool shouldLowerLDSToStruct(const GlobalVariable &GV, const Function *F) {
   bool Ret = false;
   SmallPtrSet<const User *, 8> Visited;
   SmallVector<const User *, 16> Stack(GV.users());
-  SmallPtrSet<const GlobalValue *, 8> GlobalUsers;
 
   assert(!F || isKernelCC(F));
 
@@ -267,15 +78,10 @@ bool shouldLowerLDSToStruct(const GlobalVariable &GV, const Function *F) {
     const User *V = Stack.pop_back_val();
     Visited.insert(V);
 
-    if (auto *G = dyn_cast<GlobalValue>(V)) {
-      StringRef GName = G->getName();
-      if (F && GName != "llvm.used" && GName != "llvm.compiler.used") {
-        // For kernel LDS lowering, if G is not a compiler.used list, then we
-        // cannot lower the lds GV since we cannot replace the use of GV within
-        // G.
-        return false;
-      }
-      GlobalUsers.insert(G);
+    if (isa<GlobalValue>(V)) {
+      // This use of the LDS variable is the initializer of a global variable.
+      // This is ill formed. The address of an LDS variable is kernel dependent
+      // and unknown until runtime. It can't be written to a global variable.
       continue;
     }
 
@@ -297,15 +103,6 @@ bool shouldLowerLDSToStruct(const GlobalVariable &GV, const Function *F) {
     append_range(Stack, V->users());
   }
 
-  if (!F && !Ret) {
-    // For module LDS lowering, we have not yet decided if we should lower GV or
-    // not. Explore all global users of GV, and check if atleast one of these
-    // global users appear as an use within an instruction (possibly nested use
-    // via constant expression), if so, then conservately lower LDS.
-    for (auto *G : GlobalUsers)
-      Ret |= hasUserInstruction(G);
-  }
-
   return Ret;
 }
 
@@ -324,7 +121,7 @@ std::vector<GlobalVariable *> findVariablesToLower(Module &M,
       continue;
     }
     if (!isa<UndefValue>(GV.getInitializer())) {
-      // Initializers are unimplemented for local address space.
+      // Initializers are unimplemented for LDS address space.
       // Leave such variables in place for consistent error reporting.
       continue;
     }
@@ -342,20 +139,6 @@ std::vector<GlobalVariable *> findVariablesToLower(Module &M,
   return LocalVars;
 }
 
-SmallPtrSet<GlobalValue *, 32> getUsedList(Module &M) {
-  SmallPtrSet<GlobalValue *, 32> UsedList;
-
-  SmallVector<GlobalValue *, 32> TmpVec;
-  collectUsedGlobalVariables(M, TmpVec, true);
-  UsedList.insert(TmpVec.begin(), TmpVec.end());
-
-  TmpVec.clear();
-  collectUsedGlobalVariables(M, TmpVec, false);
-  UsedList.insert(TmpVec.begin(), TmpVec.end());
-
-  return UsedList;
-}
-
 } // end namespace AMDGPU
 
 } // end namespace llvm
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.h
index d1c9229bc336..83ef68cc3f60 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.h
@@ -22,44 +22,13 @@ class ConstantExpr;
 
 namespace AMDGPU {
 
-/// Collect reachable callees for each kernel defined in the module \p M and
-/// return collected callees at \p KernelToCallees.
-void collectReachableCallees(
-    Module &M,
-    DenseMap<Function *, SmallPtrSet<Function *, 8>> &KernelToCallees);
-
-/// For the given LDS global \p GV, visit all its users and collect all
-/// non-kernel functions within which \p GV is used and return collected list of
-/// such non-kernel functions.
-SmallPtrSet<Function *, 8> collectNonKernelAccessorsOfLDS(GlobalVariable *GV);
-
-/// Collect all the instructions where user \p U belongs to. \p U could be
-/// instruction itself or it could be a constant expression which is used within
-/// an instruction. If \p CollectKernelInsts is true, collect instructions only
-/// from kernels, otherwise collect instructions only from non-kernel functions.
-DenseMap<Function *, SmallPtrSet<Instruction *, 8>>
-getFunctionToInstsMap(User *U, bool CollectKernelInsts);
-
 bool isKernelCC(const Function *Func);
 
 Align getAlign(DataLayout const &DL, const GlobalVariable *GV);
 
-/// \returns true if a given global variable \p GV (or its global users) appear
-/// as an use within some instruction (either from kernel or from non-kernel).
-bool hasUserInstruction(const GlobalValue *GV);
-
-/// \returns true if an LDS global requires lowering to a module LDS structure
-/// if \p F is not given. If \p F is given it must be a kernel and function
-/// \returns true if an LDS global is directly used from that kernel and it
-/// is safe to replace its uses with a kernel LDS structure member.
-bool shouldLowerLDSToStruct(const GlobalVariable &GV,
-                            const Function *F = nullptr);
-
 std::vector<GlobalVariable *> findVariablesToLower(Module &M,
                                                    const Function *F = nullptr);
 
-SmallPtrSet<GlobalValue *, 32> getUsedList(Module &M);
-
 /// Replace all uses of constant \p C with instructions in \p F.
 void replaceConstantUsesInFunction(ConstantExpr *C, const Function *F);
 } // end namespace AMDGPU
diff --git a/llvm/lib/Target/AMDGPU/VOPInstructions.td b/llvm/lib/Target/AMDGPU/VOPInstructions.td
index a3eccf13cd71..a8368892c565 100644
--- a/llvm/lib/Target/AMDGPU/VOPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOPInstructions.td
@@ -794,6 +794,18 @@ class VOPPatGen<SDPatternOperator Op, VOPProfile P> {
   list<dag> ret =  [!con(Outs, (set Ins))];
 }
 
+class DivergentUnaryFrag<SDPatternOperator Op> : PatFrag <
+  (ops node:$src0),
+  (Op $src0),
+  [{ return N->isDivergent(); }]> {
+  // This check is unnecessary as it's captured by the result register
+  // bank constraint.
+  //
+  // FIXME: Should add a way for the emitter to recognize this is a
+  // trivially true predicate to eliminate the check.
+  let GISelPredicateCode = [{return true;}];
+}
+
 class VOPPatOrNull<SDPatternOperator Op, VOPProfile P> {
   list<dag> ret = !if(!ne(P.NeedPatGen,PatGenMode.NoPattern), VOPPatGen<Op, P>.ret, []);
 }
diff --git a/llvm/lib/Target/ARM/A15SDOptimizer.cpp b/llvm/lib/Target/ARM/A15SDOptimizer.cpp
index f4d0f4a6d6b0..d0efecad63bc 100644
--- a/llvm/lib/Target/ARM/A15SDOptimizer.cpp
+++ b/llvm/lib/Target/ARM/A15SDOptimizer.cpp
@@ -592,16 +592,15 @@ bool A15SDOptimizer::runOnInstruction(MachineInstr *MI) {
   SmallVector<unsigned, 8> Defs = getReadDPRs(MI);
   bool Modified = false;
 
-  for (SmallVectorImpl<unsigned>::iterator I = Defs.begin(), E = Defs.end();
-     I != E; ++I) {
+  for (unsigned I : Defs) {
     // Follow the def-use chain for this DPR through COPYs, and also through
     // PHIs (which are essentially multi-way COPYs). It is because of PHIs that
     // we can end up with multiple defs of this DPR.
 
     SmallVector<MachineInstr *, 8> DefSrcs;
-    if (!Register::isVirtualRegister(*I))
+    if (!Register::isVirtualRegister(I))
       continue;
-    MachineInstr *Def = MRI->getVRegDef(*I);
+    MachineInstr *Def = MRI->getVRegDef(I);
     if (!Def)
       continue;
 
@@ -628,18 +627,17 @@ bool A15SDOptimizer::runOnInstruction(MachineInstr *MI) {
 
       if (NewReg != 0) {
         Modified = true;
-        for (SmallVectorImpl<MachineOperand *>::const_iterator I = Uses.begin(),
-               E = Uses.end(); I != E; ++I) {
+        for (MachineOperand *Use : Uses) {
           // Make sure to constrain the register class of the new register to
           // match what we're replacing. Otherwise we can optimize a DPR_VFP2
           // reference into a plain DPR, and that will end poorly. NewReg is
           // always virtual here, so there will always be a matching subclass
           // to find.
-          MRI->constrainRegClass(NewReg, MRI->getRegClass((*I)->getReg()));
+          MRI->constrainRegClass(NewReg, MRI->getRegClass(Use->getReg()));
 
-          LLVM_DEBUG(dbgs() << "Replacing operand " << **I << " with "
+          LLVM_DEBUG(dbgs() << "Replacing operand " << *Use << " with "
                             << printReg(NewReg) << "\n");
-          (*I)->substVirtReg(NewReg, 0, *TRI);
+          Use->substVirtReg(NewReg, 0, *TRI);
         }
       }
       Replacements[MI] = NewReg;
diff --git a/llvm/lib/Target/ARM/ARM.td b/llvm/lib/Target/ARM/ARM.td
index e03dd597eb65..8173fe4036a8 100644
--- a/llvm/lib/Target/ARM/ARM.td
+++ b/llvm/lib/Target/ARM/ARM.td
@@ -446,6 +446,11 @@ def FeaturePACBTI         : SubtargetFeature<"pacbti", "HasPACBTI", "true",
                                              "Enable Pointer Authentication and Branch "
                                              "Target Identification">;
 
+def FeatureNoBTIAtReturnTwice : SubtargetFeature<"no-bti-at-return-twice",
+                                                 "NoBTIAtReturnTwice", "true",
+                                                 "Don't place a BTI instruction "
+                                                 "after a return-twice">;
+
 //===----------------------------------------------------------------------===//
 // ARM architecture class
 //
diff --git a/llvm/lib/Target/ARM/ARMAsmPrinter.cpp b/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
index 6a88ac485e69..fa09b2567aa9 100644
--- a/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
+++ b/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
@@ -1153,8 +1153,12 @@ void ARMAsmPrinter::EmitUnwindingInstruction(const MachineInstr *MI) {
     unsigned StartOp = 2 + 2;
     // Use all the operands.
     unsigned NumOffset = 0;
-    // Amount of SP adjustment folded into a push.
-    unsigned Pad = 0;
+    // Amount of SP adjustment folded into a push, before the
+    // registers are stored (pad at higher addresses).
+    unsigned PadBefore = 0;
+    // Amount of SP adjustment folded into a push, after the
+    // registers are stored (pad at lower addresses).
+    unsigned PadAfter = 0;
 
     switch (Opc) {
     default:
@@ -1185,7 +1189,7 @@ void ARMAsmPrinter::EmitUnwindingInstruction(const MachineInstr *MI) {
                  "Pad registers must come before restored ones");
           unsigned Width =
             TargetRegInfo->getRegSizeInBits(MO.getReg(), MachineRegInfo) / 8;
-          Pad += Width;
+          PadAfter += Width;
           continue;
         }
         // Check for registers that are remapped (for a Thumb1 prologue that
@@ -1201,14 +1205,32 @@ void ARMAsmPrinter::EmitUnwindingInstruction(const MachineInstr *MI) {
     case ARM::t2STR_PRE:
       assert(MI->getOperand(2).getReg() == ARM::SP &&
              "Only stack pointer as a source reg is supported");
+      if (unsigned RemappedReg = AFI->EHPrologueRemappedRegs.lookup(SrcReg))
+        SrcReg = RemappedReg;
+
+      RegList.push_back(SrcReg);
+      break;
+    case ARM::t2STRD_PRE:
+      assert(MI->getOperand(3).getReg() == ARM::SP &&
+             "Only stack pointer as a source reg is supported");
+      SrcReg = MI->getOperand(1).getReg();
+      if (unsigned RemappedReg = AFI->EHPrologueRemappedRegs.lookup(SrcReg))
+        SrcReg = RemappedReg;
+      RegList.push_back(SrcReg);
+      SrcReg = MI->getOperand(2).getReg();
+      if (unsigned RemappedReg = AFI->EHPrologueRemappedRegs.lookup(SrcReg))
+        SrcReg = RemappedReg;
       RegList.push_back(SrcReg);
+      PadBefore = -MI->getOperand(4).getImm() - 8;
       break;
     }
     if (MAI->getExceptionHandlingType() == ExceptionHandling::ARM) {
+      if (PadBefore)
+        ATS.emitPad(PadBefore);
       ATS.emitRegSave(RegList, Opc == ARM::VSTMDDB_UPD);
       // Account for the SP adjustment, folded into the push.
-      if (Pad)
-        ATS.emitPad(Pad);
+      if (PadAfter)
+        ATS.emitPad(PadAfter);
     }
   } else {
     // Changes of stack / frame pointer.
@@ -1300,6 +1322,10 @@ void ARMAsmPrinter::EmitUnwindingInstruction(const MachineInstr *MI) {
         Offset = MI->getOperand(2).getImm();
         AFI->EHPrologueOffsetInRegs[DstReg] |= (Offset << 16);
         break;
+      case ARM::t2PAC:
+      case ARM::t2PACBTI:
+        AFI->EHPrologueRemappedRegs[ARM::R12] = ARM::RA_AUTH_CODE;
+        break;
       default:
         MI->print(errs());
         llvm_unreachable("Unsupported opcode for unwinding information");
diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
index 2a12947d24a8..884f38ff6c58 100644
--- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
@@ -2629,8 +2629,8 @@ bool llvm::tryFoldSPUpdateIntoPushPop(const ARMSubtarget &Subtarget,
 
   // Add the complete list back in.
   MachineInstrBuilder MIB(MF, &*MI);
-  for (int i = RegList.size() - 1; i >= 0; --i)
-    MIB.add(RegList[i]);
+  for (const MachineOperand &MO : llvm::reverse(RegList))
+    MIB.add(MO);
 
   return true;
 }
@@ -5678,7 +5678,7 @@ bool llvm::HasLowerConstantMaterializationCost(unsigned Val1, unsigned Val2,
 /// |                         | Thumb2 | ARM |
 /// +-------------------------+--------+-----+
 /// | Call overhead in Bytes  |      4 |   4 |
-/// | Frame overhead in Bytes |      4 |   4 |
+/// | Frame overhead in Bytes |      2 |   4 |
 /// | Stack fixup required    |     No |  No |
 /// +-------------------------+--------+-----+
 ///
@@ -5755,7 +5755,7 @@ struct OutlinerCosts {
         CallThunk(target.isThumb() ? 4 : 4),
         FrameThunk(target.isThumb() ? 0 : 0),
         CallNoLRSave(target.isThumb() ? 4 : 4),
-        FrameNoLRSave(target.isThumb() ? 4 : 4),
+        FrameNoLRSave(target.isThumb() ? 2 : 4),
         CallRegSave(target.isThumb() ? 8 : 12),
         FrameRegSave(target.isThumb() ? 2 : 4),
         CallDefault(target.isThumb() ? 8 : 12),
@@ -5868,11 +5868,17 @@ outliner::OutlinedFunction ARMBaseInstrInfo::getOutliningCandidateInfo(
       return outliner::OutlinedFunction();
   }
 
+  // We expect the majority of the outlining candidates to be in consensus with
+  // regard to return address sign and authentication, and branch target
+  // enforcement, in other words, partitioning according to all the four
+  // possible combinations of PAC-RET and BTI is going to yield one big subset
+  // and three small (likely empty) subsets. That allows us to cull incompatible
+  // candidates separately for PAC-RET and BTI.
+
   // Partition the candidates in two sets: one with BTI enabled and one with BTI
-  // disabled. Remove the candidates from the smaller set. We expect the
-  // majority of the candidates to be in consensus with regard to branch target
-  // enforcement with just a few oddballs, but if they are the same number
-  // prefer the non-BTI ones for outlining, since they have less overhead.
+  // disabled. Remove the candidates from the smaller set. If they are the same
+  // number prefer the non-BTI ones for outlining, since they have less
+  // overhead.
   auto NoBTI =
       llvm::partition(RepeatedSequenceLocs, [](const outliner::Candidate &C) {
         const ARMFunctionInfo &AFI = *C.getMF()->getInfo<ARMFunctionInfo>();
@@ -5883,6 +5889,24 @@ outliner::OutlinedFunction ARMBaseInstrInfo::getOutliningCandidateInfo(
     RepeatedSequenceLocs.erase(NoBTI, RepeatedSequenceLocs.end());
   else
     RepeatedSequenceLocs.erase(RepeatedSequenceLocs.begin(), NoBTI);
+
+  if (RepeatedSequenceLocs.size() < 2)
+    return outliner::OutlinedFunction();
+
+  // Likewise, partition the candidates according to PAC-RET enablement.
+  auto NoPAC =
+      llvm::partition(RepeatedSequenceLocs, [](const outliner::Candidate &C) {
+        const ARMFunctionInfo &AFI = *C.getMF()->getInfo<ARMFunctionInfo>();
+        // If the function happens to not spill the LR, do not disqualify it
+        // from the outlining.
+        return AFI.shouldSignReturnAddress(true);
+      });
+  if (std::distance(RepeatedSequenceLocs.begin(), NoPAC) >
+      std::distance(NoPAC, RepeatedSequenceLocs.end()))
+    RepeatedSequenceLocs.erase(NoPAC, RepeatedSequenceLocs.end());
+  else
+    RepeatedSequenceLocs.erase(RepeatedSequenceLocs.begin(), NoPAC);
+
   if (RepeatedSequenceLocs.size() < 2)
     return outliner::OutlinedFunction();
 
@@ -5899,6 +5923,7 @@ outliner::OutlinedFunction ARMBaseInstrInfo::getOutliningCandidateInfo(
       };
 
   OutlinerCosts Costs(Subtarget);
+
   const auto &SomeMFI =
       *RepeatedSequenceLocs.front().getMF()->getInfo<ARMFunctionInfo>();
   // Adjust costs to account for the BTI instructions.
@@ -5909,6 +5934,13 @@ outliner::OutlinedFunction ARMBaseInstrInfo::getOutliningCandidateInfo(
     Costs.FrameTailCall += 4;
     Costs.FrameThunk += 4;
   }
+
+  // Adjust costs to account for sign and authentication instructions.
+  if (SomeMFI.shouldSignReturnAddress(true)) {
+    Costs.CallDefault += 8;          // +PAC instr, +AUT instr
+    Costs.SaveRestoreLROnStack += 8; // +PAC instr, +AUT instr
+  }
+
   unsigned FrameID = MachineOutlinerDefault;
   unsigned NumBytesToCreateFrame = Costs.FrameDefault;
 
@@ -6325,6 +6357,11 @@ ARMBaseInstrInfo::getOutliningType(MachineBasicBlock::iterator &MIT,
     // * LR is available in the range (No save/restore around call)
     // * The range doesn't include calls (No save/restore in outlined frame)
     // are true.
+    // These conditions also ensure correctness of the return address
+    // authentication - we insert sign and authentication instructions only if
+    // we save/restore LR on stack, but then this condition ensures that the
+    // outlined range does not modify the SP, therefore the SP value used for
+    // signing is the same as the one used for authentication.
     // FIXME: This is very restrictive; the flags check the whole block,
     // not just the bit we will try to outline.
     bool MightNeedStackFixUp =
@@ -6369,23 +6406,39 @@ void ARMBaseInstrInfo::fixupPostOutline(MachineBasicBlock &MBB) const {
 }
 
 void ARMBaseInstrInfo::saveLROnStack(MachineBasicBlock &MBB,
-                                     MachineBasicBlock::iterator It) const {
-  unsigned Opc = Subtarget.isThumb() ? ARM::t2STR_PRE : ARM::STR_PRE_IMM;
-  int Align = -Subtarget.getStackAlignment().value();
-  BuildMI(MBB, It, DebugLoc(), get(Opc), ARM::SP)
-    .addReg(ARM::LR, RegState::Kill)
-    .addReg(ARM::SP)
-    .addImm(Align)
-    .add(predOps(ARMCC::AL));
-}
+                                     MachineBasicBlock::iterator It, bool CFI,
+                                     bool Auth) const {
+  int Align = std::max(Subtarget.getStackAlignment().value(), uint64_t(8));
+  assert(Align >= 8 && Align <= 256);
+  if (Auth) {
+    assert(Subtarget.isThumb2());
+    // Compute PAC in R12. Outlining ensures R12 is dead across the outlined
+    // sequence.
+    BuildMI(MBB, It, DebugLoc(), get(ARM::t2PAC))
+        .setMIFlags(MachineInstr::FrameSetup);
+    BuildMI(MBB, It, DebugLoc(), get(ARM::t2STRD_PRE), ARM::SP)
+        .addReg(ARM::R12, RegState::Kill)
+        .addReg(ARM::LR, RegState::Kill)
+        .addReg(ARM::SP)
+        .addImm(-Align)
+        .add(predOps(ARMCC::AL))
+        .setMIFlags(MachineInstr::FrameSetup);
+  } else {
+    unsigned Opc = Subtarget.isThumb() ? ARM::t2STR_PRE : ARM::STR_PRE_IMM;
+    BuildMI(MBB, It, DebugLoc(), get(Opc), ARM::SP)
+        .addReg(ARM::LR, RegState::Kill)
+        .addReg(ARM::SP)
+        .addImm(-Align)
+        .add(predOps(ARMCC::AL))
+        .setMIFlags(MachineInstr::FrameSetup);
+  }
+
+  if (!CFI)
+    return;
 
-void ARMBaseInstrInfo::emitCFIForLRSaveOnStack(
-    MachineBasicBlock &MBB, MachineBasicBlock::iterator It) const {
   MachineFunction &MF = *MBB.getParent();
-  const MCRegisterInfo *MRI = Subtarget.getRegisterInfo();
-  unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true);
-  int Align = Subtarget.getStackAlignment().value();
-  // Add a CFI saying the stack was moved down.
+
+  // Add a CFI, saying CFA is offset by Align bytes from SP.
   int64_t StackPosEntry =
       MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, Align));
   BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION))
@@ -6394,11 +6447,23 @@ void ARMBaseInstrInfo::emitCFIForLRSaveOnStack(
 
   // Add a CFI saying that the LR that we want to find is now higher than
   // before.
-  int64_t LRPosEntry =
-      MF.addFrameInst(MCCFIInstruction::createOffset(nullptr, DwarfLR, -Align));
+  int LROffset = Auth ? Align - 4 : Align;
+  const MCRegisterInfo *MRI = Subtarget.getRegisterInfo();
+  unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true);
+  int64_t LRPosEntry = MF.addFrameInst(
+      MCCFIInstruction::createOffset(nullptr, DwarfLR, -LROffset));
   BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION))
       .addCFIIndex(LRPosEntry)
       .setMIFlags(MachineInstr::FrameSetup);
+  if (Auth) {
+    // Add a CFI for the location of the return adddress PAC.
+    unsigned DwarfRAC = MRI->getDwarfRegNum(ARM::RA_AUTH_CODE, true);
+    int64_t RACPosEntry = MF.addFrameInst(
+        MCCFIInstruction::createOffset(nullptr, DwarfRAC, -Align));
+    BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION))
+        .addCFIIndex(RACPosEntry)
+        .setMIFlags(MachineInstr::FrameSetup);
+  }
 }
 
 void ARMBaseInstrInfo::emitCFIForLRSaveToReg(MachineBasicBlock &MBB,
@@ -6416,35 +6481,64 @@ void ARMBaseInstrInfo::emitCFIForLRSaveToReg(MachineBasicBlock &MBB,
       .setMIFlags(MachineInstr::FrameSetup);
 }
 
-void ARMBaseInstrInfo::restoreLRFromStack(
-    MachineBasicBlock &MBB, MachineBasicBlock::iterator It) const {
-  unsigned Opc = Subtarget.isThumb() ? ARM::t2LDR_POST : ARM::LDR_POST_IMM;
-  MachineInstrBuilder MIB = BuildMI(MBB, It, DebugLoc(), get(Opc), ARM::LR)
-    .addReg(ARM::SP, RegState::Define)
-    .addReg(ARM::SP);
-  if (!Subtarget.isThumb())
-    MIB.addReg(0);
-  MIB.addImm(Subtarget.getStackAlignment().value()).add(predOps(ARMCC::AL));
-}
+void ARMBaseInstrInfo::restoreLRFromStack(MachineBasicBlock &MBB,
+                                          MachineBasicBlock::iterator It,
+                                          bool CFI, bool Auth) const {
+  int Align = Subtarget.getStackAlignment().value();
+  if (Auth) {
+    assert(Subtarget.isThumb2());
+    // Restore return address PAC and LR.
+    BuildMI(MBB, It, DebugLoc(), get(ARM::t2LDRD_POST))
+        .addReg(ARM::R12, RegState::Define)
+        .addReg(ARM::LR, RegState::Define)
+        .addReg(ARM::SP, RegState::Define)
+        .addReg(ARM::SP)
+        .addImm(Align)
+        .add(predOps(ARMCC::AL))
+        .setMIFlags(MachineInstr::FrameDestroy);
+    // LR authentication is after the CFI instructions, below.
+  } else {
+    unsigned Opc = Subtarget.isThumb() ? ARM::t2LDR_POST : ARM::LDR_POST_IMM;
+    MachineInstrBuilder MIB = BuildMI(MBB, It, DebugLoc(), get(Opc), ARM::LR)
+                                  .addReg(ARM::SP, RegState::Define)
+                                  .addReg(ARM::SP);
+    if (!Subtarget.isThumb())
+      MIB.addReg(0);
+    MIB.addImm(Subtarget.getStackAlignment().value())
+        .add(predOps(ARMCC::AL))
+        .setMIFlags(MachineInstr::FrameDestroy);
+  }
 
-void ARMBaseInstrInfo::emitCFIForLRRestoreFromStack(
-    MachineBasicBlock &MBB, MachineBasicBlock::iterator It) const {
-  // Now stack has moved back up...
-  MachineFunction &MF = *MBB.getParent();
-  const MCRegisterInfo *MRI = Subtarget.getRegisterInfo();
-  unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true);
-  int64_t StackPosEntry =
-      MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, 0));
-  BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION))
-      .addCFIIndex(StackPosEntry)
-      .setMIFlags(MachineInstr::FrameDestroy);
+  if (CFI) {
+    // Now stack has moved back up...
+    MachineFunction &MF = *MBB.getParent();
+    const MCRegisterInfo *MRI = Subtarget.getRegisterInfo();
+    unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true);
+    int64_t StackPosEntry =
+        MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, 0));
+    BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION))
+        .addCFIIndex(StackPosEntry)
+        .setMIFlags(MachineInstr::FrameDestroy);
+
+    // ... and we have restored LR.
+    int64_t LRPosEntry =
+        MF.addFrameInst(MCCFIInstruction::createRestore(nullptr, DwarfLR));
+    BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION))
+        .addCFIIndex(LRPosEntry)
+        .setMIFlags(MachineInstr::FrameDestroy);
+
+    if (Auth) {
+      unsigned DwarfRAC = MRI->getDwarfRegNum(ARM::RA_AUTH_CODE, true);
+      int64_t Entry =
+          MF.addFrameInst(MCCFIInstruction::createUndefined(nullptr, DwarfRAC));
+      BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION))
+          .addCFIIndex(Entry)
+          .setMIFlags(MachineInstr::FrameDestroy);
+    }
+  }
 
-  // ... and we have restored LR.
-  int64_t LRPosEntry =
-      MF.addFrameInst(MCCFIInstruction::createRestore(nullptr, DwarfLR));
-  BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION))
-      .addCFIIndex(LRPosEntry)
-      .setMIFlags(MachineInstr::FrameDestroy);
+  if (Auth)
+    BuildMI(MBB, It, DebugLoc(), get(ARM::t2AUT));
 }
 
 void ARMBaseInstrInfo::emitCFIForLRRestoreFromReg(
@@ -6500,8 +6594,11 @@ void ARMBaseInstrInfo::buildOutlinedFrame(
       MBB.addLiveIn(ARM::LR);
 
     // Insert a save before the outlined region
-    saveLROnStack(MBB, It);
-    emitCFIForLRSaveOnStack(MBB, It);
+    bool Auth = OF.Candidates.front()
+                    .getMF()
+                    ->getInfo<ARMFunctionInfo>()
+                    ->shouldSignReturnAddress(true);
+    saveLROnStack(MBB, It, true, Auth);
 
     // Fix up the instructions in the range, since we're going to modify the
     // stack.
@@ -6510,8 +6607,7 @@ void ARMBaseInstrInfo::buildOutlinedFrame(
     fixupPostOutline(MBB);
 
     // Insert a restore before the terminator for the function.  Restore LR.
-    restoreLRFromStack(MBB, Et);
-    emitCFIForLRRestoreFromStack(MBB, Et);
+    restoreLRFromStack(MBB, Et, true, Auth);
   }
 
   // If this is a tail call outlined function, then there's already a return.
@@ -6590,13 +6686,10 @@ MachineBasicBlock::iterator ARMBaseInstrInfo::insertOutlinedCall(
   // We have the default case. Save and restore from SP.
   if (!MBB.isLiveIn(ARM::LR))
     MBB.addLiveIn(ARM::LR);
-  saveLROnStack(MBB, It);
-  if (!AFI.isLRSpilled())
-    emitCFIForLRSaveOnStack(MBB, It);
+  bool Auth = !AFI.isLRSpilled() && AFI.shouldSignReturnAddress(true);
+  saveLROnStack(MBB, It, !AFI.isLRSpilled(), Auth);
   CallPt = MBB.insert(It, CallMIB);
-  restoreLRFromStack(MBB, It);
-  if (!AFI.isLRSpilled())
-    emitCFIForLRRestoreFromStack(MBB, It);
+  restoreLRFromStack(MBB, It, !AFI.isLRSpilled(), Auth);
   It--;
   return CallPt;
 }
diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.h b/llvm/lib/Target/ARM/ARMBaseInstrInfo.h
index 5fa912ae35d7..defce07dd862 100644
--- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.h
+++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.h
@@ -377,20 +377,20 @@ private:
   /// constructing an outlined call if one exists. Returns 0 otherwise.
   unsigned findRegisterToSaveLRTo(const outliner::Candidate &C) const;
 
-  // Adds an instruction which saves the link register on top of the stack into
-  /// the MachineBasicBlock \p MBB at position \p It.
-  void saveLROnStack(MachineBasicBlock &MBB,
-                     MachineBasicBlock::iterator It) const;
+  /// Adds an instruction which saves the link register on top of the stack into
+  /// the MachineBasicBlock \p MBB at position \p It. If \p Auth is true,
+  /// compute and store an authentication code alongiside the link register.
+  /// If \p CFI is true, emit CFI instructions.
+  void saveLROnStack(MachineBasicBlock &MBB, MachineBasicBlock::iterator It,
+                     bool CFI, bool Auth) const;
 
   /// Adds an instruction which restores the link register from the top the
-  /// stack into the MachineBasicBlock \p MBB at position \p It.
+  /// stack into the MachineBasicBlock \p MBB at position \p It. If \p Auth is
+  /// true, restore an authentication code and authenticate LR.
+  /// If \p CFI is true, emit CFI instructions.
   void restoreLRFromStack(MachineBasicBlock &MBB,
-                          MachineBasicBlock::iterator It) const;
-
-  /// Emit CFI instructions into the MachineBasicBlock \p MBB at position \p It,
-  /// for the case when the LR is saved on the stack.
-  void emitCFIForLRSaveOnStack(MachineBasicBlock &MBB,
-                               MachineBasicBlock::iterator It) const;
+                          MachineBasicBlock::iterator It, bool CFI,
+                          bool Auth) const;
 
   /// Emit CFI instructions into the MachineBasicBlock \p MBB at position \p It,
   /// for the case when the LR is saved in the register \p Reg.
@@ -399,11 +399,6 @@ private:
                              Register Reg) const;
 
   /// Emit CFI instructions into the MachineBasicBlock \p MBB at position \p It,
-  /// after the LR is was restored from the stack.
-  void emitCFIForLRRestoreFromStack(MachineBasicBlock &MBB,
-                                    MachineBasicBlock::iterator It) const;
-
-  /// Emit CFI instructions into the MachineBasicBlock \p MBB at position \p It,
   /// after the LR is was restored from a register.
   void emitCFIForLRRestoreFromReg(MachineBasicBlock &MBB,
                                   MachineBasicBlock::iterator It) const;
diff --git a/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp b/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp
index b53efe58e8de..c543d02ff75a 100644
--- a/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp
@@ -530,6 +530,8 @@ getFrameIndexInstrOffset(const MachineInstr *MI, int Idx) const {
   unsigned ImmIdx = 0;
   switch (AddrMode) {
   case ARMII::AddrModeT2_i8:
+  case ARMII::AddrModeT2_i8neg:
+  case ARMII::AddrModeT2_i8pos:
   case ARMII::AddrModeT2_i12:
   case ARMII::AddrMode_i12:
     InstrOffs = MI->getOperand(Idx+1).getImm();
@@ -728,6 +730,8 @@ bool ARMBaseRegisterInfo::isFrameOffsetLegal(const MachineInstr *MI,
   bool isSigned = true;
   switch (AddrMode) {
   case ARMII::AddrModeT2_i8:
+  case ARMII::AddrModeT2_i8pos:
+  case ARMII::AddrModeT2_i8neg:
   case ARMII::AddrModeT2_i12:
     // i8 supports only negative, and i12 supports only positive, so
     // based on Offset sign, consider the appropriate instruction
diff --git a/llvm/lib/Target/ARM/ARMBranchTargets.cpp b/llvm/lib/Target/ARM/ARMBranchTargets.cpp
index 1091c1f970fa..8ba3e627c039 100644
--- a/llvm/lib/Target/ARM/ARMBranchTargets.cpp
+++ b/llvm/lib/Target/ARM/ARMBranchTargets.cpp
@@ -108,6 +108,7 @@ void ARMBranchTargets::addBTI(const ARMInstrInfo &TII, MachineBasicBlock &MBB,
                               bool IsFirstBB) {
   // Which instruction to insert: BTI or PACBTI
   unsigned OpCode = ARM::t2BTI;
+  unsigned MIFlags = 0;
 
   // Skip meta instructions, including EH labels
   auto MBBI = llvm::find_if_not(MBB.instrs(), [](const MachineInstr &MI) {
@@ -121,6 +122,7 @@ void ARMBranchTargets::addBTI(const ARMInstrInfo &TII, MachineBasicBlock &MBB,
       LLVM_DEBUG(dbgs() << "Removing a 'PAC' instr from BB '" << MBB.getName()
                         << "' to replace with PACBTI\n");
       OpCode = ARM::t2PACBTI;
+      MIFlags = MachineInstr::FrameSetup;
       auto NextMBBI = std::next(MBBI);
       MBBI->eraseFromParent();
       MBBI = NextMBBI;
@@ -131,5 +133,6 @@ void ARMBranchTargets::addBTI(const ARMInstrInfo &TII, MachineBasicBlock &MBB,
                     << (OpCode == ARM::t2BTI ? "BTI" : "PACBTI")
                     << "' instr into BB '" << MBB.getName() << "'\n");
   // Finally, insert a new instruction (either PAC or PACBTI)
-  BuildMI(MBB, MBBI, MBB.findDebugLoc(MBBI), TII.get(OpCode));
+  BuildMI(MBB, MBBI, MBB.findDebugLoc(MBBI), TII.get(OpCode))
+      .setMIFlags(MIFlags);
 }
diff --git a/llvm/lib/Target/ARM/ARMCallingConv.cpp b/llvm/lib/Target/ARM/ARMCallingConv.cpp
index d8d9ca3b912f..32f3a4a632f5 100644
--- a/llvm/lib/Target/ARM/ARMCallingConv.cpp
+++ b/llvm/lib/Target/ARM/ARMCallingConv.cpp
@@ -230,10 +230,9 @@ static bool CC_ARM_AAPCS_Custom_Aggregate(unsigned ValNo, MVT ValVT,
 
   unsigned RegResult = State.AllocateRegBlock(RegList, PendingMembers.size());
   if (RegResult) {
-    for (SmallVectorImpl<CCValAssign>::iterator It = PendingMembers.begin();
-         It != PendingMembers.end(); ++It) {
-      It->convertToReg(RegResult);
-      State.addLoc(*It);
+    for (CCValAssign &PendingMember : PendingMembers) {
+      PendingMember.convertToReg(RegResult);
+      State.addLoc(PendingMember);
       ++RegResult;
     }
     PendingMembers.clear();
diff --git a/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp b/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp
index c2ca4708c208..a2a4f1f3bdfd 100644
--- a/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp
+++ b/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp
@@ -310,8 +310,7 @@ void ARMConstantIslands::verify() {
            BBInfo[RHS.getNumber()].postOffset();
   }));
   LLVM_DEBUG(dbgs() << "Verifying " << CPUsers.size() << " CP users.\n");
-  for (unsigned i = 0, e = CPUsers.size(); i != e; ++i) {
-    CPUser &U = CPUsers[i];
+  for (CPUser &U : CPUsers) {
     unsigned UserOffset = getUserOffset(U);
     // Verify offset using the real max displacement without the safety
     // adjustment.
@@ -697,10 +696,9 @@ ARMConstantIslands::findConstPoolEntry(unsigned CPI,
   std::vector<CPEntry> &CPEs = CPEntries[CPI];
   // Number of entries per constpool index should be small, just do a
   // linear search.
-  for (unsigned i = 0, e = CPEs.size(); i != e; ++i) {
-    if (CPEs[i].CPEMI == CPEMI)
-      return &CPEs[i];
-  }
+  for (CPEntry &CPE : CPEs)
+    if (CPE.CPEMI == CPEMI)
+      return &CPE;
   return nullptr;
 }
 
@@ -1234,27 +1232,27 @@ int ARMConstantIslands::findInRangeCPEntry(CPUser& U, unsigned UserOffset) {
   // No.  Look for previously created clones of the CPE that are in range.
   unsigned CPI = getCombinedIndex(CPEMI);
   std::vector<CPEntry> &CPEs = CPEntries[CPI];
-  for (unsigned i = 0, e = CPEs.size(); i != e; ++i) {
+  for (CPEntry &CPE : CPEs) {
     // We already tried this one
-    if (CPEs[i].CPEMI == CPEMI)
+    if (CPE.CPEMI == CPEMI)
       continue;
     // Removing CPEs can leave empty entries, skip
-    if (CPEs[i].CPEMI == nullptr)
+    if (CPE.CPEMI == nullptr)
       continue;
-    if (isCPEntryInRange(UserMI, UserOffset, CPEs[i].CPEMI, U.getMaxDisp(),
-                     U.NegOk)) {
-      LLVM_DEBUG(dbgs() << "Replacing CPE#" << CPI << " with CPE#"
-                        << CPEs[i].CPI << "\n");
+    if (isCPEntryInRange(UserMI, UserOffset, CPE.CPEMI, U.getMaxDisp(),
+                         U.NegOk)) {
+      LLVM_DEBUG(dbgs() << "Replacing CPE#" << CPI << " with CPE#" << CPE.CPI
+                        << "\n");
       // Point the CPUser node to the replacement
-      U.CPEMI = CPEs[i].CPEMI;
+      U.CPEMI = CPE.CPEMI;
       // Change the CPI in the instruction operand to refer to the clone.
       for (MachineOperand &MO : UserMI->operands())
         if (MO.isCPI()) {
-          MO.setIndex(CPEs[i].CPI);
+          MO.setIndex(CPE.CPI);
           break;
         }
       // Adjust the refcount of the clone...
-      CPEs[i].RefCount++;
+      CPE.RefCount++;
       // ...and the original.  If we didn't remove the old entry, none of the
       // addresses changed, so we don't need another pass.
       return decrementCPEReferenceCount(CPI, CPEMI) ? 2 : 1;
@@ -1675,15 +1673,14 @@ void ARMConstantIslands::removeDeadCPEMI(MachineInstr *CPEMI) {
 /// are zero.
 bool ARMConstantIslands::removeUnusedCPEntries() {
   unsigned MadeChange = false;
-  for (unsigned i = 0, e = CPEntries.size(); i != e; ++i) {
-      std::vector<CPEntry> &CPEs = CPEntries[i];
-      for (unsigned j = 0, ee = CPEs.size(); j != ee; ++j) {
-        if (CPEs[j].RefCount == 0 && CPEs[j].CPEMI) {
-          removeDeadCPEMI(CPEs[j].CPEMI);
-          CPEs[j].CPEMI = nullptr;
-          MadeChange = true;
-        }
+  for (std::vector<CPEntry> &CPEs : CPEntries) {
+    for (CPEntry &CPE : CPEs) {
+      if (CPE.RefCount == 0 && CPE.CPEMI) {
+        removeDeadCPEMI(CPE.CPEMI);
+        CPE.CPEMI = nullptr;
+        MadeChange = true;
       }
+    }
   }
   return MadeChange;
 }
@@ -1829,8 +1826,7 @@ bool ARMConstantIslands::optimizeThumb2Instructions() {
   bool MadeChange = false;
 
   // Shrink ADR and LDR from constantpool.
-  for (unsigned i = 0, e = CPUsers.size(); i != e; ++i) {
-    CPUser &U = CPUsers[i];
+  for (CPUser &U : CPUsers) {
     unsigned Opcode = U.MI->getOpcode();
     unsigned NewOpc = 0;
     unsigned Scale = 1;
diff --git a/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp b/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp
index 7a35f252b22a..fa244786a80d 100644
--- a/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp
@@ -2160,6 +2160,11 @@ bool ARMExpandPseudo::ExpandMI(MachineBasicBlock &MBB,
       return true;
     }
     case ARM::tBXNS_RET: {
+      // For v8.0-M.Main we need to authenticate LR before clearing FPRs, which
+      // uses R12 as a scratch register.
+      if (!STI->hasV8_1MMainlineOps() && AFI->shouldSignReturnAddress())
+        BuildMI(MBB, MBBI, DebugLoc(), TII->get(ARM::t2AUT));
+
       MachineBasicBlock &AfterBB = CMSEClearFPRegs(MBB, MBBI);
 
       if (STI->hasV8_1MMainlineOps()) {
@@ -2169,6 +2174,9 @@ bool ARMExpandPseudo::ExpandMI(MachineBasicBlock &MBB,
             .addReg(ARM::SP)
             .addImm(4)
             .add(predOps(ARMCC::AL));
+
+        if (AFI->shouldSignReturnAddress())
+          BuildMI(AfterBB, AfterBB.end(), DebugLoc(), TII->get(ARM::t2AUT));
       }
 
       // Clear all GPR that are not a use of the return instruction.
@@ -3073,6 +3081,22 @@ bool ARMExpandPseudo::ExpandMI(MachineBasicBlock &MBB,
       MI.eraseFromParent();
       return true;
     }
+    case ARM::t2CALL_BTI: {
+      MachineFunction &MF = *MI.getMF();
+      MachineInstrBuilder MIB =
+          BuildMI(MF, MI.getDebugLoc(), TII->get(ARM::tBL));
+      MIB.cloneMemRefs(MI);
+      for (unsigned i = 0; i < MI.getNumOperands(); ++i)
+        MIB.add(MI.getOperand(i));
+      if (MI.isCandidateForCallSiteEntry())
+        MF.moveCallSiteInfo(&MI, MIB.getInstr());
+      MIBundleBuilder Bundler(MBB, MI);
+      Bundler.append(MIB);
+      Bundler.append(BuildMI(MF, MI.getDebugLoc(), TII->get(ARM::t2BTI)));
+      finalizeBundle(MBB, Bundler.begin(), Bundler.end());
+      MI.eraseFromParent();
+      return true;
+    }
     case ARM::LOADDUAL:
     case ARM::STOREDUAL: {
       Register PairReg = MI.getOperand(0).getReg();
diff --git a/llvm/lib/Target/ARM/ARMFrameLowering.cpp b/llvm/lib/Target/ARM/ARMFrameLowering.cpp
index b866cf952ff1..4b59f9cb94ce 100644
--- a/llvm/lib/Target/ARM/ARMFrameLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMFrameLowering.cpp
@@ -503,20 +503,12 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF,
   StackAdjustingInsts DefCFAOffsetCandidates;
   bool HasFP = hasFP(MF);
 
-  // Allocate the vararg register save area.
-  if (ArgRegsSaveSize) {
-    emitSPUpdate(isARM, MBB, MBBI, dl, TII, -ArgRegsSaveSize,
-                 MachineInstr::FrameSetup);
-    DefCFAOffsetCandidates.addInst(std::prev(MBBI), ArgRegsSaveSize, true);
-  }
-
   if (!AFI->hasStackFrame() &&
       (!STI.isTargetWindows() || !WindowsRequiresStackProbe(MF, NumBytes))) {
-    if (NumBytes - ArgRegsSaveSize != 0) {
-      emitSPUpdate(isARM, MBB, MBBI, dl, TII, -(NumBytes - ArgRegsSaveSize),
+    if (NumBytes != 0) {
+      emitSPUpdate(isARM, MBB, MBBI, dl, TII, -NumBytes,
                    MachineInstr::FrameSetup);
-      DefCFAOffsetCandidates.addInst(std::prev(MBBI),
-                                     NumBytes - ArgRegsSaveSize, true);
+      DefCFAOffsetCandidates.addInst(std::prev(MBBI), NumBytes, true);
     }
     DefCFAOffsetCandidates.emitDefCFAOffsets(MBB, dl, TII, HasFP);
     return;
@@ -562,13 +554,26 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF,
     }
   }
 
-  // Move past FPCXT area.
   MachineBasicBlock::iterator LastPush = MBB.end(), GPRCS1Push, GPRCS2Push;
+
+  // Move past the PAC computation.
+  if (AFI->shouldSignReturnAddress())
+    LastPush = MBBI++;
+
+  // Move past FPCXT area.
   if (FPCXTSaveSize > 0) {
     LastPush = MBBI++;
     DefCFAOffsetCandidates.addInst(LastPush, FPCXTSaveSize, true);
   }
 
+  // Allocate the vararg register save area.
+  if (ArgRegsSaveSize) {
+    emitSPUpdate(isARM, MBB, MBBI, dl, TII, -ArgRegsSaveSize,
+                 MachineInstr::FrameSetup);
+    LastPush = std::prev(MBBI);
+    DefCFAOffsetCandidates.addInst(LastPush, ArgRegsSaveSize, true);
+  }
+
   // Move past area 1.
   if (GPRCS1Size > 0) {
     GPRCS1Push = LastPush = MBBI++;
@@ -788,7 +793,8 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF,
       case ARM::R11:
       case ARM::R12:
         if (STI.splitFramePushPop(MF)) {
-          unsigned DwarfReg =  MRI->getDwarfRegNum(Reg, true);
+          unsigned DwarfReg = MRI->getDwarfRegNum(
+              Reg == ARM::R12 ? (unsigned)ARM::RA_AUTH_CODE : Reg, true);
           unsigned Offset = MFI.getObjectOffset(FI);
           unsigned CFIIndex = MF.addFrameInst(
               MCCFIInstruction::createOffset(nullptr, DwarfReg, Offset));
@@ -923,8 +929,9 @@ void ARMFrameLowering::emitEpilogue(MachineFunction &MF,
   DebugLoc dl = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc();
 
   if (!AFI->hasStackFrame()) {
-    if (NumBytes - ReservedArgStack != 0)
-      emitSPUpdate(isARM, MBB, MBBI, dl, TII, NumBytes - ReservedArgStack,
+    if (NumBytes + IncomingArgStackToRestore != 0)
+      emitSPUpdate(isARM, MBB, MBBI, dl, TII,
+                   NumBytes + IncomingArgStackToRestore,
                    MachineInstr::FrameDestroy);
   } else {
     // Unwind MBBI to point to first LDR / VLDRD.
@@ -1007,15 +1014,21 @@ void ARMFrameLowering::emitEpilogue(MachineFunction &MF,
 
     if (AFI->getGPRCalleeSavedArea2Size()) MBBI++;
     if (AFI->getGPRCalleeSavedArea1Size()) MBBI++;
-    if (AFI->getFPCXTSaveAreaSize()) MBBI++;
-  }
 
-  if (ReservedArgStack || IncomingArgStackToRestore) {
-    assert((int)ReservedArgStack + IncomingArgStackToRestore >= 0 &&
-           "attempting to restore negative stack amount");
-    emitSPUpdate(isARM, MBB, MBBI, dl, TII,
-                 ReservedArgStack + IncomingArgStackToRestore,
-                 MachineInstr::FrameDestroy);
+    if (ReservedArgStack || IncomingArgStackToRestore) {
+      assert((int)ReservedArgStack + IncomingArgStackToRestore >= 0 &&
+             "attempting to restore negative stack amount");
+      emitSPUpdate(isARM, MBB, MBBI, dl, TII,
+                   ReservedArgStack + IncomingArgStackToRestore,
+                   MachineInstr::FrameDestroy);
+    }
+
+    // Validate PAC, It should have been already popped into R12. For CMSE entry
+    // function, the validation instruction is emitted during expansion of the
+    // tBXNS_RET, since the validation must use the value of SP at function
+    // entry, before saving, resp. after restoring, FPCXTNS.
+    if (AFI->shouldSignReturnAddress() && !AFI->isCmseNSEntryFunction())
+      BuildMI(MBB, MBBI, DebugLoc(), STI.getInstrInfo()->get(ARM::t2AUT));
   }
 }
 
@@ -1199,6 +1212,7 @@ void ARMFrameLowering::emitPopInst(MachineBasicBlock &MBB,
   const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
   const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
+  bool hasPAC = AFI->shouldSignReturnAddress();
   DebugLoc DL;
   bool isTailCall = false;
   bool isInterrupt = false;
@@ -1231,7 +1245,7 @@ void ARMFrameLowering::emitPopInst(MachineBasicBlock &MBB,
         continue;
       if (Reg == ARM::LR && !isTailCall && !isVarArg && !isInterrupt &&
           !isCmseEntry && !isTrap && AFI->getArgumentStackToRestore() == 0 &&
-          STI.hasV5TOps() && MBB.succ_empty()) {
+          STI.hasV5TOps() && MBB.succ_empty() && !hasPAC) {
         Reg = ARM::PC;
         // Fold the return instruction into the LDM.
         DeleteRet = true;
@@ -1580,6 +1594,11 @@ bool ARMFrameLowering::spillCalleeSavedRegisters(
     ARM::t2STR_PRE : ARM::STR_PRE_IMM;
   unsigned FltOpc = ARM::VSTMDDB_UPD;
   unsigned NumAlignedDPRCS2Regs = AFI->getNumAlignedDPRCS2Regs();
+  // Compute PAC in R12.
+  if (AFI->shouldSignReturnAddress()) {
+    BuildMI(MBB, MI, DebugLoc(), STI.getInstrInfo()->get(ARM::t2PAC))
+        .setMIFlags(MachineInstr::FrameSetup);
+  }
   // Save the non-secure floating point context.
   if (llvm::any_of(CSI, [](const CalleeSavedInfo &C) {
         return C.getReg() == ARM::FPCXTNS;
@@ -1789,6 +1808,13 @@ bool ARMFrameLowering::enableShrinkWrapping(const MachineFunction &MF) const {
       MF.getInfo<ARMFunctionInfo>()->isCmseNSEntryFunction())
     return false;
 
+  // We are disabling shrinkwrapping for now when PAC is enabled, as
+  // shrinkwrapping can cause clobbering of r12 when the PAC code is
+  // generated. A follow-up patch will fix this in a more performant manner.
+  if (MF.getInfo<ARMFunctionInfo>()->shouldSignReturnAddress(
+          false /*SpillsLR */))
+    return false;
+
   return true;
 }
 
@@ -2315,6 +2341,26 @@ bool ARMFrameLowering::assignCalleeSavedSpillSlots(
     CSI.back().setRestored(false);
   }
 
+  // For functions, which sign their return address, upon function entry, the
+  // return address PAC is computed in R12. Treat R12 as a callee-saved register
+  // in this case.
+  const auto &AFI = *MF.getInfo<ARMFunctionInfo>();
+  if (AFI.shouldSignReturnAddress()) {
+    // The order of register must match the order we push them, because the
+    // PEI assigns frame indices in that order. When compiling for return
+    // address sign and authenication, we use split push, therefore the orders
+    // we want are:
+    // LR, R7, R6, R5, R4, <R12>, R11, R10,  R9,  R8, D15-D8
+    CSI.insert(find_if(CSI,
+                       [=](const auto &CS) {
+                         unsigned Reg = CS.getReg();
+                         return Reg == ARM::R10 || Reg == ARM::R11 ||
+                                Reg == ARM::R8 || Reg == ARM::R9 ||
+                                ARM::DPRRegClass.contains(Reg);
+                       }),
+               CalleeSavedInfo(ARM::R12));
+  }
+
   return false;
 }
 
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 33d115945614..3d45db349644 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -391,6 +391,7 @@ void ARMTargetLowering::addMVEVectorTypes(bool HasMVEFP) {
     setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
     setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
     setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
+    setOperationAction(ISD::VSELECT, VT, Legal);
   }
   setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2f64, Legal);
 
@@ -428,7 +429,7 @@ void ARMTargetLowering::addMVEVectorTypes(bool HasMVEFP) {
   }
 
   // Predicate types
-  const MVT pTypes[] = {MVT::v16i1, MVT::v8i1, MVT::v4i1};
+  const MVT pTypes[] = {MVT::v16i1, MVT::v8i1, MVT::v4i1, MVT::v2i1};
   for (auto VT : pTypes) {
     addRegisterClass(VT, &ARM::VCCRRegClass);
     setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
@@ -445,6 +446,16 @@ void ARMTargetLowering::addMVEVectorTypes(bool HasMVEFP) {
     setOperationAction(ISD::VSELECT, VT, Expand);
     setOperationAction(ISD::SELECT, VT, Expand);
   }
+  setOperationAction(ISD::SETCC, MVT::v2i1, Expand);
+  setOperationAction(ISD::TRUNCATE, MVT::v2i1, Expand);
+  setOperationAction(ISD::AND, MVT::v2i1, Expand);
+  setOperationAction(ISD::OR, MVT::v2i1, Expand);
+  setOperationAction(ISD::XOR, MVT::v2i1, Expand);
+  setOperationAction(ISD::SINT_TO_FP, MVT::v2i1, Expand);
+  setOperationAction(ISD::UINT_TO_FP, MVT::v2i1, Expand);
+  setOperationAction(ISD::FP_TO_SINT, MVT::v2i1, Expand);
+  setOperationAction(ISD::FP_TO_UINT, MVT::v2i1, Expand);
+
   setOperationAction(ISD::SIGN_EXTEND, MVT::v8i32, Custom);
   setOperationAction(ISD::SIGN_EXTEND, MVT::v16i16, Custom);
   setOperationAction(ISD::SIGN_EXTEND, MVT::v16i32, Custom);
@@ -1647,6 +1658,7 @@ const char *ARMTargetLowering::getTargetNodeName(unsigned Opcode) const {
     MAKE_CASE(ARMISD::CALL_PRED)
     MAKE_CASE(ARMISD::CALL_NOLINK)
     MAKE_CASE(ARMISD::tSECALL)
+    MAKE_CASE(ARMISD::t2CALL_BTI)
     MAKE_CASE(ARMISD::BRCOND)
     MAKE_CASE(ARMISD::BR_JT)
     MAKE_CASE(ARMISD::BR2_JT)
@@ -1853,8 +1865,10 @@ EVT ARMTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &,
 
   // MVE has a predicate register.
   if ((Subtarget->hasMVEIntegerOps() &&
-       (VT == MVT::v4i32 || VT == MVT::v8i16 || VT == MVT::v16i8)) ||
-      (Subtarget->hasMVEFloatOps() && (VT == MVT::v4f32 || VT == MVT::v8f16)))
+       (VT == MVT::v2i64 || VT == MVT::v4i32 || VT == MVT::v8i16 ||
+        VT == MVT::v16i8)) ||
+      (Subtarget->hasMVEFloatOps() &&
+       (VT == MVT::v2f64 || VT == MVT::v4f32 || VT == MVT::v8f16)))
     return MVT::getVectorVT(MVT::i1, VT.getVectorElementCount());
   return VT.changeVectorElementTypeToInteger();
 }
@@ -2308,6 +2322,12 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   bool isCmseNSCall   = false;
   bool isSibCall = false;
   bool PreferIndirect = false;
+  bool GuardWithBTI = false;
+
+  // Lower 'returns_twice' calls to a pseudo-instruction.
+  if (CLI.CB && CLI.CB->getAttributes().hasFnAttr(Attribute::ReturnsTwice) &&
+      !Subtarget->getNoBTIAtReturnTwice())
+    GuardWithBTI = AFI->branchTargetEnforcement();
 
   // Determine whether this is a non-secure function call.
   if (CLI.CB && CLI.CB->getAttributes().hasFnAttr("cmse_nonsecure_call"))
@@ -2713,7 +2733,9 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   // FIXME: handle tail calls differently.
   unsigned CallOpc;
   if (Subtarget->isThumb()) {
-    if (isCmseNSCall)
+    if (GuardWithBTI)
+      CallOpc = ARMISD::t2CALL_BTI;
+    else if (isCmseNSCall)
       CallOpc = ARMISD::tSECALL;
     else if ((!isDirect || isARMFunc) && !Subtarget->hasV5TOps())
       CallOpc = ARMISD::CALL_NOLINK;
@@ -2930,9 +2952,17 @@ bool ARMTargetLowering::IsEligibleForTailCallOptimization(
   // Indirect tail calls cannot be optimized for Thumb1 if the args
   // to the call take up r0-r3. The reason is that there are no legal registers
   // left to hold the pointer to the function to be called.
-  if (Subtarget->isThumb1Only() && Outs.size() >= 4 &&
-      (!isa<GlobalAddressSDNode>(Callee.getNode()) || isIndirect))
-    return false;
+  // Similarly, if the function uses return address sign and authentication,
+  // r12 is needed to hold the PAC and is not available to hold the callee
+  // address.
+  if (Outs.size() >= 4 &&
+      (!isa<GlobalAddressSDNode>(Callee.getNode()) || isIndirect)) {
+    if (Subtarget->isThumb1Only())
+      return false;
+    // Conservatively assume the function spills LR.
+    if (MF.getInfo<ARMFunctionInfo>()->shouldSignReturnAddress(true))
+      return false;
+  }
 
   // Look for obvious safe cases to perform tail call optimization that do not
   // require ABI changes. This is what gcc calls sibcall.
@@ -7616,7 +7646,10 @@ static SDValue LowerBUILD_VECTOR_i1(SDValue Op, SelectionDAG &DAG,
   unsigned NumElts = VT.getVectorNumElements();
   unsigned BoolMask;
   unsigned BitsPerBool;
-  if (NumElts == 4) {
+  if (NumElts == 2) {
+    BitsPerBool = 8;
+    BoolMask = 0xff;
+  } else if (NumElts == 4) {
     BitsPerBool = 4;
     BoolMask = 0xf;
   } else if (NumElts == 8) {
@@ -7699,6 +7732,46 @@ static SDValue LowerBUILD_VECTORToVIDUP(SDValue Op, SelectionDAG &DAG,
                      DAG.getConstant(N, DL, MVT::i32));
 }
 
+// Returns true if the operation N can be treated as qr instruction variant at
+// operand Op.
+static bool IsQRMVEInstruction(const SDNode *N, const SDNode *Op) {
+  switch (N->getOpcode()) {
+  case ISD::ADD:
+  case ISD::MUL:
+  case ISD::SADDSAT:
+  case ISD::UADDSAT:
+    return true;
+  case ISD::SUB:
+  case ISD::SSUBSAT:
+  case ISD::USUBSAT:
+    return N->getOperand(1).getNode() == Op;
+  case ISD::INTRINSIC_WO_CHAIN:
+    switch (N->getConstantOperandVal(0)) {
+    case Intrinsic::arm_mve_add_predicated:
+    case Intrinsic::arm_mve_mul_predicated:
+    case Intrinsic::arm_mve_qadd_predicated:
+    case Intrinsic::arm_mve_vhadd:
+    case Intrinsic::arm_mve_hadd_predicated:
+    case Intrinsic::arm_mve_vqdmulh:
+    case Intrinsic::arm_mve_qdmulh_predicated:
+    case Intrinsic::arm_mve_vqrdmulh:
+    case Intrinsic::arm_mve_qrdmulh_predicated:
+    case Intrinsic::arm_mve_vqdmull:
+    case Intrinsic::arm_mve_vqdmull_predicated:
+      return true;
+    case Intrinsic::arm_mve_sub_predicated:
+    case Intrinsic::arm_mve_qsub_predicated:
+    case Intrinsic::arm_mve_vhsub:
+    case Intrinsic::arm_mve_hsub_predicated:
+      return N->getOperand(2).getNode() == Op;
+    default:
+      return false;
+    }
+  default:
+    return false;
+  }
+}
+
 // If this is a case we can't handle, return null and let the default
 // expansion code take care of it.
 SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
@@ -7720,6 +7793,20 @@ SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
     if (SplatUndef.isAllOnes())
       return DAG.getUNDEF(VT);
 
+    // If all the users of this constant splat are qr instruction variants,
+    // generate a vdup of the constant.
+    if (ST->hasMVEIntegerOps() && VT.getScalarSizeInBits() == SplatBitSize &&
+        (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32) &&
+        all_of(BVN->uses(),
+               [BVN](const SDNode *U) { return IsQRMVEInstruction(U, BVN); })) {
+      EVT DupVT = SplatBitSize == 32   ? MVT::v4i32
+                  : SplatBitSize == 16 ? MVT::v8i16
+                                       : MVT::v16i8;
+      SDValue Const = DAG.getConstant(SplatBits.getZExtValue(), dl, MVT::i32);
+      SDValue VDup = DAG.getNode(ARMISD::VDUP, dl, DupVT, Const);
+      return DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, VDup);
+    }
+
     if ((ST->hasNEON() && SplatBitSize <= 64) ||
         (ST->hasMVEIntegerOps() && SplatBitSize <= 64)) {
       // Check if an immediate VMOV works.
@@ -8313,9 +8400,8 @@ static SDValue LowerVECTOR_SHUFFLEv8i8(SDValue Op,
   SDLoc DL(Op);
 
   SmallVector<SDValue, 8> VTBLMask;
-  for (ArrayRef<int>::iterator
-         I = ShuffleMask.begin(), E = ShuffleMask.end(); I != E; ++I)
-    VTBLMask.push_back(DAG.getConstant(*I, DL, MVT::i32));
+  for (int I : ShuffleMask)
+    VTBLMask.push_back(DAG.getConstant(I, DL, MVT::i32));
 
   if (V2.getNode()->isUndef())
     return DAG.getNode(ARMISD::VTBL1, DL, MVT::v8i8, V1,
@@ -8346,6 +8432,8 @@ static SDValue LowerReverse_VECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) {
 
 static EVT getVectorTyFromPredicateVector(EVT VT) {
   switch (VT.getSimpleVT().SimpleTy) {
+  case MVT::v2i1:
+    return MVT::v2f64;
   case MVT::v4i1:
     return MVT::v4i32;
   case MVT::v8i1:
@@ -8427,7 +8515,14 @@ static SDValue LowerVECTOR_SHUFFLE_i1(SDValue Op, SelectionDAG &DAG,
                                           DAG.getUNDEF(NewVT), ShuffleMask);
 
   // Now return the result of comparing the shuffled vector with zero,
-  // which will generate a real predicate, i.e. v4i1, v8i1 or v16i1.
+  // which will generate a real predicate, i.e. v4i1, v8i1 or v16i1. For a v2i1
+  // we convert to a v4i1 compare to fill in the two halves of the i64 as i32s.
+  if (VT == MVT::v2i1) {
+    SDValue BC = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, Shuffled);
+    SDValue Cmp = DAG.getNode(ARMISD::VCMPZ, dl, MVT::v4i1, BC,
+                              DAG.getConstant(ARMCC::NE, dl, MVT::i32));
+    return DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::v2i1, Cmp);
+  }
   return DAG.getNode(ARMISD::VCMPZ, dl, VT, Shuffled,
                      DAG.getConstant(ARMCC::NE, dl, MVT::i32));
 }
@@ -8927,8 +9022,15 @@ static SDValue LowerCONCAT_VECTORS_i1(SDValue Op, SelectionDAG &DAG,
     ConVec = ExtractInto(NewV1, ConVec, j);
     ConVec = ExtractInto(NewV2, ConVec, j);
 
-    // Now return the result of comparing the subvector with zero,
-    // which will generate a real predicate, i.e. v4i1, v8i1 or v16i1.
+    // Now return the result of comparing the subvector with zero, which will
+    // generate a real predicate, i.e. v4i1, v8i1 or v16i1. For a v2i1 we
+    // convert to a v4i1 compare to fill in the two halves of the i64 as i32s.
+    if (VT == MVT::v2i1) {
+      SDValue BC = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, ConVec);
+      SDValue Cmp = DAG.getNode(ARMISD::VCMPZ, dl, MVT::v4i1, BC,
+                                DAG.getConstant(ARMCC::NE, dl, MVT::i32));
+      return DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::v2i1, Cmp);
+    }
     return DAG.getNode(ARMISD::VCMPZ, dl, VT, ConVec,
                        DAG.getConstant(ARMCC::NE, dl, MVT::i32));
   };
@@ -8993,6 +9095,22 @@ static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG,
 
   MVT ElType = getVectorTyFromPredicateVector(VT).getScalarType().getSimpleVT();
 
+  if (NumElts == 2) {
+    EVT SubVT = MVT::v4i32;
+    SDValue SubVec = DAG.getNode(ISD::UNDEF, dl, SubVT);
+    for (unsigned i = Index, j = 0; i < (Index + NumElts); i++, j += 2) {
+      SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, NewV1,
+                                DAG.getIntPtrConstant(i, dl));
+      SubVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, SubVT, SubVec, Elt,
+                           DAG.getConstant(j, dl, MVT::i32));
+      SubVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, SubVT, SubVec, Elt,
+                           DAG.getConstant(j + 1, dl, MVT::i32));
+    }
+    SDValue Cmp = DAG.getNode(ARMISD::VCMPZ, dl, MVT::v4i1, SubVec,
+                              DAG.getConstant(ARMCC::NE, dl, MVT::i32));
+    return DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::v2i1, Cmp);
+  }
+
   EVT SubVT = MVT::getVectorVT(ElType, NumElts);
   SDValue SubVec = DAG.getNode(ISD::UNDEF, dl, SubVT);
   for (unsigned i = Index, j = 0; i < (Index + NumElts); i++, j++) {
@@ -9839,16 +9957,17 @@ void ARMTargetLowering::ExpandDIV_Windows(
 static SDValue LowerPredicateLoad(SDValue Op, SelectionDAG &DAG) {
   LoadSDNode *LD = cast<LoadSDNode>(Op.getNode());
   EVT MemVT = LD->getMemoryVT();
-  assert((MemVT == MVT::v4i1 || MemVT == MVT::v8i1 || MemVT == MVT::v16i1) &&
+  assert((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
+          MemVT == MVT::v16i1) &&
          "Expected a predicate type!");
   assert(MemVT == Op.getValueType());
   assert(LD->getExtensionType() == ISD::NON_EXTLOAD &&
          "Expected a non-extending load");
   assert(LD->isUnindexed() && "Expected a unindexed load");
 
-  // The basic MVE VLDR on a v4i1/v8i1 actually loads the entire 16bit
+  // The basic MVE VLDR on a v2i1/v4i1/v8i1 actually loads the entire 16bit
   // predicate, with the "v4i1" bits spread out over the 16 bits loaded. We
-  // need to make sure that 8/4 bits are actually loaded into the correct
+  // need to make sure that 8/4/2 bits are actually loaded into the correct
   // place, which means loading the value and then shuffling the values into
   // the bottom bits of the predicate.
   // Equally, VLDR for an v16i1 will actually load 32bits (so will be incorrect
@@ -9895,14 +10014,15 @@ void ARMTargetLowering::LowerLOAD(SDNode *N, SmallVectorImpl<SDValue> &Results,
 static SDValue LowerPredicateStore(SDValue Op, SelectionDAG &DAG) {
   StoreSDNode *ST = cast<StoreSDNode>(Op.getNode());
   EVT MemVT = ST->getMemoryVT();
-  assert((MemVT == MVT::v4i1 || MemVT == MVT::v8i1 || MemVT == MVT::v16i1) &&
+  assert((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
+          MemVT == MVT::v16i1) &&
          "Expected a predicate type!");
   assert(MemVT == ST->getValue().getValueType());
   assert(!ST->isTruncatingStore() && "Expected a non-extending store");
   assert(ST->isUnindexed() && "Expected a unindexed store");
 
-  // Only store the v4i1 or v8i1 worth of bits, via a buildvector with top bits
-  // unset and a scalar store.
+  // Only store the v2i1 or v4i1 or v8i1 worth of bits, via a buildvector with
+  // top bits unset and a scalar store.
   SDLoc dl(Op);
   SDValue Build = ST->getValue();
   if (MemVT != MVT::v16i1) {
@@ -9953,7 +10073,7 @@ static SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG,
                                    {ST->getChain(), Lo, Hi, ST->getBasePtr()},
                                    MemVT, ST->getMemOperand());
   } else if (Subtarget->hasMVEIntegerOps() &&
-             ((MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
+             ((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
                MemVT == MVT::v16i1))) {
     return LowerPredicateStore(Op, DAG);
   }
@@ -10561,25 +10681,23 @@ void ARMTargetLowering::EmitSjLjDispatchBlock(MachineInstr &MI,
   // associated with.
   DenseMap<unsigned, SmallVector<MachineBasicBlock*, 2>> CallSiteNumToLPad;
   unsigned MaxCSNum = 0;
-  for (MachineFunction::iterator BB = MF->begin(), E = MF->end(); BB != E;
-       ++BB) {
-    if (!BB->isEHPad()) continue;
+  for (MachineBasicBlock &BB : *MF) {
+    if (!BB.isEHPad())
+      continue;
 
     // FIXME: We should assert that the EH_LABEL is the first MI in the landing
     // pad.
-    for (MachineBasicBlock::iterator
-           II = BB->begin(), IE = BB->end(); II != IE; ++II) {
-      if (!II->isEHLabel()) continue;
+    for (MachineInstr &II : BB) {
+      if (!II.isEHLabel())
+        continue;
 
-      MCSymbol *Sym = II->getOperand(0).getMCSymbol();
+      MCSymbol *Sym = II.getOperand(0).getMCSymbol();
       if (!MF->hasCallSiteLandingPad(Sym)) continue;
 
       SmallVectorImpl<unsigned> &CallSiteIdxs = MF->getCallSiteLandingPad(Sym);
-      for (SmallVectorImpl<unsigned>::iterator
-             CSI = CallSiteIdxs.begin(), CSE = CallSiteIdxs.end();
-           CSI != CSE; ++CSI) {
-        CallSiteNumToLPad[*CSI].push_back(&*BB);
-        MaxCSNum = std::max(MaxCSNum, *CSI);
+      for (unsigned Idx : CallSiteIdxs) {
+        CallSiteNumToLPad[Idx].push_back(&BB);
+        MaxCSNum = std::max(MaxCSNum, Idx);
       }
       break;
     }
@@ -14002,8 +14120,8 @@ static SDValue PerformANDCombine(SDNode *N,
   EVT VT = N->getValueType(0);
   SelectionDAG &DAG = DCI.DAG;
 
-  if (!DAG.getTargetLoweringInfo().isTypeLegal(VT) || VT == MVT::v4i1 ||
-      VT == MVT::v8i1 || VT == MVT::v16i1)
+  if (!DAG.getTargetLoweringInfo().isTypeLegal(VT) || VT == MVT::v2i1 ||
+      VT == MVT::v4i1 || VT == MVT::v8i1 || VT == MVT::v16i1)
     return SDValue();
 
   APInt SplatBits, SplatUndef;
@@ -14298,8 +14416,8 @@ static SDValue PerformORCombine(SDNode *N,
   if(!DAG.getTargetLoweringInfo().isTypeLegal(VT))
     return SDValue();
 
-  if (Subtarget->hasMVEIntegerOps() &&
-      (VT == MVT::v4i1 || VT == MVT::v8i1 || VT == MVT::v16i1))
+  if (Subtarget->hasMVEIntegerOps() && (VT == MVT::v2i1 || VT == MVT::v4i1 ||
+                                        VT == MVT::v8i1 || VT == MVT::v16i1))
     return PerformORCombine_i1(N, DAG, Subtarget);
 
   APInt SplatBits, SplatUndef;
@@ -14569,6 +14687,15 @@ static SDValue IsCMPZCSINC(SDNode *Cmp, ARMCC::CondCodes &CC) {
   if (Cmp->getOpcode() != ARMISD::CMPZ || !isNullConstant(Cmp->getOperand(1)))
     return SDValue();
   SDValue CSInc = Cmp->getOperand(0);
+
+  // Ignore any `And 1` nodes that may not yet have been removed. We are
+  // looking for a value that produces 1/0, so these have no effect on the
+  // code.
+  while (CSInc.getOpcode() == ISD::AND &&
+         isa<ConstantSDNode>(CSInc.getOperand(1)) &&
+         CSInc.getConstantOperandVal(1) == 1 && CSInc->hasOneUse())
+    CSInc = CSInc.getOperand(0);
+
   if (CSInc.getOpcode() != ARMISD::CSINC ||
       !isNullConstant(CSInc.getOperand(0)) ||
       !isNullConstant(CSInc.getOperand(1)) || !CSInc->hasOneUse())
@@ -17897,6 +18024,23 @@ ARMTargetLowering::PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const {
   if (!VT.isInteger())
       return SDValue();
 
+  // Fold away an unneccessary CMPZ/CMOV
+  // CMOV A, B, C1, $cpsr, (CMPZ (CMOV 1, 0, C2, D), 0) ->
+  // if C1==EQ -> CMOV A, B, C2, $cpsr, D
+  // if C1==NE -> CMOV A, B, NOT(C2), $cpsr, D
+  if (N->getConstantOperandVal(2) == ARMCC::EQ ||
+      N->getConstantOperandVal(2) == ARMCC::NE) {
+    ARMCC::CondCodes Cond;
+    if (SDValue C = IsCMPZCSINC(N->getOperand(4).getNode(), Cond)) {
+      if (N->getConstantOperandVal(2) == ARMCC::NE)
+        Cond = ARMCC::getOppositeCondition(Cond);
+      return DAG.getNode(N->getOpcode(), SDLoc(N), MVT::i32, N->getOperand(0),
+                         N->getOperand(1),
+                         DAG.getTargetConstant(Cond, SDLoc(N), MVT::i32),
+                         N->getOperand(3), C);
+    }
+  }
+
   // Materialize a boolean comparison for integers so we can avoid branching.
   if (isNullConstant(FalseVal)) {
     if (CC == ARMCC::EQ && isOneConstant(TrueVal)) {
@@ -18564,7 +18708,8 @@ bool ARMTargetLowering::allowsMisalignedMemoryAccesses(EVT VT, unsigned,
     return false;
 
   // These are for predicates
-  if ((Ty == MVT::v16i1 || Ty == MVT::v8i1 || Ty == MVT::v4i1)) {
+  if ((Ty == MVT::v16i1 || Ty == MVT::v8i1 || Ty == MVT::v4i1 ||
+       Ty == MVT::v2i1)) {
     if (Fast)
       *Fast = true;
     return true;
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.h b/llvm/lib/Target/ARM/ARMISelLowering.h
index e3b422358cae..1c5f8389f57c 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.h
+++ b/llvm/lib/Target/ARM/ARMISelLowering.h
@@ -69,6 +69,7 @@ class VectorType;
     CALL_PRED,   // Function call that's predicable.
     CALL_NOLINK, // Function call with branch not branch-and-link.
     tSECALL,     // CMSE non-secure function call.
+    t2CALL_BTI,  // Thumb function call followed by BTI instruction.
     BRCOND,      // Conditional branch.
     BR_JT,       // Jumptable branch.
     BR2_JT,      // Jumptable branch (2 level - jumptable entry is a jump).
diff --git a/llvm/lib/Target/ARM/ARMInstrMVE.td b/llvm/lib/Target/ARM/ARMInstrMVE.td
index f53814a80e01..1ae0354ffc37 100644
--- a/llvm/lib/Target/ARM/ARMInstrMVE.td
+++ b/llvm/lib/Target/ARM/ARMInstrMVE.td
@@ -254,13 +254,6 @@ class MVEVectorVTInfo<ValueType vec, ValueType dblvec,
   // An LLVM ValueType representing a corresponding vector of
   // predicate bits, for use in ISel patterns that handle an IR
   // intrinsic describing the predicated form of the instruction.
-  //
-  // Usually, for a vector of N things, this will be vNi1. But for
-  // vectors of 2 values, we make an exception, and use v4i1 instead
-  // of v2i1. Rationale: MVE codegen doesn't support doing all the
-  // auxiliary operations on v2i1 (vector shuffles etc), and also,
-  // there's no MVE compare instruction that will _generate_ v2i1
-  // directly.
   ValueType Pred = pred;
 
   // Same as Pred but for DblVec rather than Vec.
@@ -294,25 +287,25 @@ class MVEVectorVTInfo<ValueType vec, ValueType dblvec,
 // Integer vector types that don't treat signed and unsigned differently.
 def MVE_v16i8 : MVEVectorVTInfo<v16i8, v8i16, v16i1, v8i1, 0b00, "i", ?>;
 def MVE_v8i16 : MVEVectorVTInfo<v8i16, v4i32, v8i1,  v4i1, 0b01, "i", ?>;
-def MVE_v4i32 : MVEVectorVTInfo<v4i32, v2i64, v4i1,  v4i1, 0b10, "i", ?>;
-def MVE_v2i64 : MVEVectorVTInfo<v2i64, ?,     v4i1,  ?,    0b11, "i", ?>;
+def MVE_v4i32 : MVEVectorVTInfo<v4i32, v2i64, v4i1,  v2i1, 0b10, "i", ?>;
+def MVE_v2i64 : MVEVectorVTInfo<v2i64, ?,     v2i1,  ?,    0b11, "i", ?>;
 
 // Explicitly signed and unsigned integer vectors. They map to the
 // same set of LLVM ValueTypes as above, but are represented
 // differently in assembly and instruction encodings.
 def MVE_v16s8 : MVEVectorVTInfo<v16i8, v8i16, v16i1, v8i1, 0b00, "s", 0b0>;
 def MVE_v8s16 : MVEVectorVTInfo<v8i16, v4i32, v8i1,  v4i1, 0b01, "s", 0b0>;
-def MVE_v4s32 : MVEVectorVTInfo<v4i32, v2i64, v4i1,  v4i1, 0b10, "s", 0b0>;
-def MVE_v2s64 : MVEVectorVTInfo<v2i64, ?,     v4i1,  ?,    0b11, "s", 0b0>;
+def MVE_v4s32 : MVEVectorVTInfo<v4i32, v2i64, v4i1,  v2i1, 0b10, "s", 0b0>;
+def MVE_v2s64 : MVEVectorVTInfo<v2i64, ?,     v2i1,  ?,    0b11, "s", 0b0>;
 def MVE_v16u8 : MVEVectorVTInfo<v16i8, v8i16, v16i1, v8i1, 0b00, "u", 0b1>;
 def MVE_v8u16 : MVEVectorVTInfo<v8i16, v4i32, v8i1,  v4i1, 0b01, "u", 0b1>;
-def MVE_v4u32 : MVEVectorVTInfo<v4i32, v2i64, v4i1,  v4i1, 0b10, "u", 0b1>;
-def MVE_v2u64 : MVEVectorVTInfo<v2i64, ?,     v4i1,  ?,    0b11, "u", 0b1>;
+def MVE_v4u32 : MVEVectorVTInfo<v4i32, v2i64, v4i1,  v2i1, 0b10, "u", 0b1>;
+def MVE_v2u64 : MVEVectorVTInfo<v2i64, ?,     v2i1,  ?,    0b11, "u", 0b1>;
 
 // FP vector types.
 def MVE_v8f16 : MVEVectorVTInfo<v8f16, v4f32, v8i1,  v4i1, 0b01, "f", ?>;
-def MVE_v4f32 : MVEVectorVTInfo<v4f32, v2f64, v4i1,  v4i1, 0b10, "f", ?>;
-def MVE_v2f64 : MVEVectorVTInfo<v2f64, ?,     v4i1,  ?,    0b11, "f", ?>;
+def MVE_v4f32 : MVEVectorVTInfo<v4f32, v2f64, v4i1,  v2i1, 0b10, "f", ?>;
+def MVE_v2f64 : MVEVectorVTInfo<v2f64, ?,     v2i1,  ?,    0b11, "f", ?>;
 
 // Polynomial vector types.
 def MVE_v16p8 : MVEVectorVTInfo<v16i8, v8i16, v16i1, v8i1, 0b11, "p", 0b0>;
@@ -2260,6 +2253,31 @@ let Predicates = [HasMVEInt] in {
                                         (v4i32 (ARMvmovImm (i32 1)))),
                                 (i32 1))),
             (MVE_VRHADDu32 MQPR:$Qm, MQPR:$Qn)>;
+
+  def : Pat<(v16i8 (ARMvshrsImm (addnsw (addnsw (v16i8 MQPR:$Qm), (v16i8 MQPR:$Qn)),
+                                        (v16i8 (ARMvdup (i32 1)))),
+                                (i32 1))),
+            (MVE_VRHADDs8 MQPR:$Qm, MQPR:$Qn)>;
+  def : Pat<(v8i16 (ARMvshrsImm (addnsw (addnsw (v8i16 MQPR:$Qm), (v8i16 MQPR:$Qn)),
+                                        (v8i16 (ARMvdup (i32 1)))),
+                                (i32 1))),
+            (MVE_VRHADDs16 MQPR:$Qm, MQPR:$Qn)>;
+  def : Pat<(v4i32 (ARMvshrsImm (addnsw (addnsw (v4i32 MQPR:$Qm), (v4i32 MQPR:$Qn)),
+                                        (v4i32 (ARMvdup (i32 1)))),
+                                (i32 1))),
+            (MVE_VRHADDs32 MQPR:$Qm, MQPR:$Qn)>;
+  def : Pat<(v16i8 (ARMvshruImm (addnuw (addnuw (v16i8 MQPR:$Qm), (v16i8 MQPR:$Qn)),
+                                        (v16i8 (ARMvdup (i32 1)))),
+                                (i32 1))),
+            (MVE_VRHADDu8 MQPR:$Qm, MQPR:$Qn)>;
+  def : Pat<(v8i16 (ARMvshruImm (addnuw (addnuw (v8i16 MQPR:$Qm), (v8i16 MQPR:$Qn)),
+                                        (v8i16 (ARMvdup (i32 1)))),
+                                (i32 1))),
+            (MVE_VRHADDu16 MQPR:$Qm, MQPR:$Qn)>;
+  def : Pat<(v4i32 (ARMvshruImm (addnuw (addnuw (v4i32 MQPR:$Qm), (v4i32 MQPR:$Qn)),
+                                        (v4i32 (ARMvdup (i32 1)))),
+                                (i32 1))),
+            (MVE_VRHADDu32 MQPR:$Qm, MQPR:$Qn)>;
 }
 
 
@@ -4450,6 +4468,11 @@ multiclass two_predops<SDPatternOperator opnode, Instruction insn> {
                           (insn (i32 (COPY_TO_REGCLASS (v4i1 VCCR:$p1), rGPR)),
                                 (i32 (COPY_TO_REGCLASS (v4i1 VCCR:$p2), rGPR))),
                           VCCR))>;
+  def v2i1  : Pat<(v2i1 (opnode (v2i1 VCCR:$p1), (v2i1 VCCR:$p2))),
+                  (v2i1 (COPY_TO_REGCLASS
+                          (insn (i32 (COPY_TO_REGCLASS (v2i1 VCCR:$p1), rGPR)),
+                                (i32 (COPY_TO_REGCLASS (v2i1 VCCR:$p2), rGPR))),
+                          VCCR))>;
 }
 
 let Predicates = [HasMVEInt] in {
@@ -4469,20 +4492,20 @@ def load_align4 : PatFrag<(ops node:$ptr), (load node:$ptr), [{
 }]>;
 
 let Predicates = [HasMVEInt] in {
-  foreach VT = [ v4i1, v8i1, v16i1 ] in {
+  foreach VT = [ v2i1, v4i1, v8i1, v16i1 ] in {
     def : Pat<(i32 (predicate_cast (VT VCCR:$src))),
               (i32 (COPY_TO_REGCLASS (VT VCCR:$src), VCCR))>;
     def : Pat<(VT  (predicate_cast (i32 VCCR:$src))),
               (VT  (COPY_TO_REGCLASS (i32 VCCR:$src), VCCR))>;
 
-    foreach VT2 = [ v4i1, v8i1, v16i1 ] in
+    foreach VT2 = [ v2i1, v4i1, v8i1, v16i1 ] in
       def : Pat<(VT  (predicate_cast (VT2 VCCR:$src))),
                 (VT  (COPY_TO_REGCLASS (VT2 VCCR:$src), VCCR))>;
   }
 
   // If we happen to be casting from a load we can convert that straight
   // into a predicate load, so long as the load is of the correct type.
-  foreach VT = [ v4i1, v8i1, v16i1 ] in {
+  foreach VT = [ v2i1, v4i1, v8i1, v16i1 ] in {
     def : Pat<(VT (predicate_cast (i32 (load_align4 taddrmode_imm7<2>:$addr)))),
               (VT (VLDR_P0_off taddrmode_imm7<2>:$addr))>;
   }
@@ -5350,33 +5373,40 @@ class MVE_VxADDSUB_qr<string iname, string suffix,
 }
 
 multiclass MVE_VHADDSUB_qr_m<string iname, MVEVectorVTInfo VTI, bit subtract,
-                             Intrinsic unpred_int, Intrinsic pred_int> {
+                             Intrinsic unpred_int, Intrinsic pred_int, PatFrag add_op,
+                             SDNode shift_op> {
   def "" : MVE_VxADDSUB_qr<iname, VTI.Suffix, VTI.Unsigned, VTI.Size, subtract, VTI.Size>;
   defm : MVE_vec_scalar_int_pat_m<!cast<Instruction>(NAME),
                                   VTI, unpred_int, pred_int, 1, 1>;
+  defvar Inst = !cast<Instruction>(NAME);
+
+  let Predicates = [HasMVEInt] in {
+    def : Pat<(VTI.Vec (shift_op (add_op (VTI.Vec MQPR:$Qm), (VTI.Vec (ARMvdup rGPR:$Rn))), (i32 1))),
+              (Inst MQPR:$Qm, rGPR:$Rn)>;
+  }
 }
 
-multiclass MVE_VHADD_qr_m<MVEVectorVTInfo VTI> :
-  MVE_VHADDSUB_qr_m<"vhadd", VTI, 0b0, int_arm_mve_vhadd,
-                                       int_arm_mve_hadd_predicated>;
+multiclass MVE_VHADD_qr_m<MVEVectorVTInfo VTI, PatFrag add_op, SDNode shift_op> :
+  MVE_VHADDSUB_qr_m<"vhadd", VTI, 0b0, int_arm_mve_vhadd, int_arm_mve_hadd_predicated,
+                    add_op, shift_op>;
 
-multiclass MVE_VHSUB_qr_m<MVEVectorVTInfo VTI> :
-  MVE_VHADDSUB_qr_m<"vhsub", VTI, 0b1, int_arm_mve_vhsub,
-                                       int_arm_mve_hsub_predicated>;
+multiclass MVE_VHSUB_qr_m<MVEVectorVTInfo VTI, PatFrag add_op, SDNode shift_op> :
+  MVE_VHADDSUB_qr_m<"vhsub", VTI, 0b1, int_arm_mve_vhsub, int_arm_mve_hsub_predicated,
+                    add_op, shift_op>;
 
-defm MVE_VHADD_qr_s8  : MVE_VHADD_qr_m<MVE_v16s8>;
-defm MVE_VHADD_qr_s16 : MVE_VHADD_qr_m<MVE_v8s16>;
-defm MVE_VHADD_qr_s32 : MVE_VHADD_qr_m<MVE_v4s32>;
-defm MVE_VHADD_qr_u8  : MVE_VHADD_qr_m<MVE_v16u8>;
-defm MVE_VHADD_qr_u16 : MVE_VHADD_qr_m<MVE_v8u16>;
-defm MVE_VHADD_qr_u32 : MVE_VHADD_qr_m<MVE_v4u32>;
+defm MVE_VHADD_qr_s8  : MVE_VHADD_qr_m<MVE_v16s8, addnsw, ARMvshrsImm>;
+defm MVE_VHADD_qr_s16 : MVE_VHADD_qr_m<MVE_v8s16, addnsw, ARMvshrsImm>;
+defm MVE_VHADD_qr_s32 : MVE_VHADD_qr_m<MVE_v4s32, addnsw, ARMvshrsImm>;
+defm MVE_VHADD_qr_u8  : MVE_VHADD_qr_m<MVE_v16u8, addnuw, ARMvshruImm>;
+defm MVE_VHADD_qr_u16 : MVE_VHADD_qr_m<MVE_v8u16, addnuw, ARMvshruImm>;
+defm MVE_VHADD_qr_u32 : MVE_VHADD_qr_m<MVE_v4u32, addnuw, ARMvshruImm>;
 
-defm MVE_VHSUB_qr_s8  : MVE_VHSUB_qr_m<MVE_v16s8>;
-defm MVE_VHSUB_qr_s16 : MVE_VHSUB_qr_m<MVE_v8s16>;
-defm MVE_VHSUB_qr_s32 : MVE_VHSUB_qr_m<MVE_v4s32>;
-defm MVE_VHSUB_qr_u8  : MVE_VHSUB_qr_m<MVE_v16u8>;
-defm MVE_VHSUB_qr_u16 : MVE_VHSUB_qr_m<MVE_v8u16>;
-defm MVE_VHSUB_qr_u32 : MVE_VHSUB_qr_m<MVE_v4u32>;
+defm MVE_VHSUB_qr_s8  : MVE_VHSUB_qr_m<MVE_v16s8, subnsw, ARMvshrsImm>;
+defm MVE_VHSUB_qr_s16 : MVE_VHSUB_qr_m<MVE_v8s16, subnsw, ARMvshrsImm>;
+defm MVE_VHSUB_qr_s32 : MVE_VHSUB_qr_m<MVE_v4s32, subnsw, ARMvshrsImm>;
+defm MVE_VHSUB_qr_u8  : MVE_VHSUB_qr_m<MVE_v16u8, subnuw, ARMvshruImm>;
+defm MVE_VHSUB_qr_u16 : MVE_VHSUB_qr_m<MVE_v8u16, subnuw, ARMvshruImm>;
+defm MVE_VHSUB_qr_u32 : MVE_VHSUB_qr_m<MVE_v4u32, subnuw, ARMvshruImm>;
 
 multiclass MVE_VADDSUB_qr_f<string iname, MVEVectorVTInfo VTI, bit subtract,
                             SDNode Op, Intrinsic PredInt, SDPatternOperator IdentityVec> {
@@ -6778,11 +6808,15 @@ let Predicates = [HasMVEInt] in {
             (v8i16 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, VCCR:$pred, zero_reg))>;
   def : Pat<(v4i32 (vselect (v4i1 VCCR:$pred), (v4i32 MQPR:$v1), (v4i32 MQPR:$v2))),
             (v4i32 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, VCCR:$pred, zero_reg))>;
+  def : Pat<(v2i64 (vselect (v2i1 VCCR:$pred), (v2i64 MQPR:$v1), (v2i64 MQPR:$v2))),
+            (v2i64 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, VCCR:$pred, zero_reg))>;
 
   def : Pat<(v8f16 (vselect (v8i1 VCCR:$pred), (v8f16 MQPR:$v1), (v8f16 MQPR:$v2))),
             (v8f16 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, VCCR:$pred, zero_reg))>;
   def : Pat<(v4f32 (vselect (v4i1 VCCR:$pred), (v4f32 MQPR:$v1), (v4f32 MQPR:$v2))),
             (v4f32 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, VCCR:$pred, zero_reg))>;
+  def : Pat<(v2f64 (vselect (v2i1 VCCR:$pred), (v2f64 MQPR:$v1), (v2f64 MQPR:$v2))),
+            (v2f64 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, VCCR:$pred, zero_reg))>;
 
   def : Pat<(v16i8 (vselect (v16i8 MQPR:$pred), (v16i8 MQPR:$v1), (v16i8 MQPR:$v2))),
             (v16i8 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone,
@@ -6808,6 +6842,8 @@ let Predicates = [HasMVEInt] in {
             (v8i16 (MVE_VPSEL (MVE_VMOVimmi16 1), (MVE_VMOVimmi16 0), ARMVCCNone, VCCR:$pred, zero_reg))>;
   def : Pat<(v4i32 (zext  (v4i1  VCCR:$pred))),
             (v4i32 (MVE_VPSEL (MVE_VMOVimmi32 1), (MVE_VMOVimmi32 0), ARMVCCNone, VCCR:$pred, zero_reg))>;
+  def : Pat<(v2i64 (zext  (v2i1  VCCR:$pred))),
+            (v2i64 (MVE_VPSEL (MVE_VMOVimmi64 1), (MVE_VMOVimmi32 0), ARMVCCNone, VCCR:$pred, zero_reg))>;
 
   def : Pat<(v16i8 (sext  (v16i1 VCCR:$pred))),
             (v16i8 (MVE_VPSEL (MVE_VMOVimmi8 255), (MVE_VMOVimmi8 0), ARMVCCNone, VCCR:$pred, zero_reg))>;
@@ -6815,6 +6851,8 @@ let Predicates = [HasMVEInt] in {
             (v8i16 (MVE_VPSEL (MVE_VMOVimmi8 255), (MVE_VMOVimmi16 0), ARMVCCNone, VCCR:$pred, zero_reg))>;
   def : Pat<(v4i32 (sext  (v4i1  VCCR:$pred))),
             (v4i32 (MVE_VPSEL (MVE_VMOVimmi8 255), (MVE_VMOVimmi32 0), ARMVCCNone, VCCR:$pred, zero_reg))>;
+  def : Pat<(v2i64 (sext  (v2i1  VCCR:$pred))),
+            (v2i64 (MVE_VPSEL (MVE_VMOVimmi8 255), (MVE_VMOVimmi32 0), ARMVCCNone, VCCR:$pred, zero_reg))>;
 
   def : Pat<(v16i8 (anyext  (v16i1 VCCR:$pred))),
             (v16i8 (MVE_VPSEL (MVE_VMOVimmi8 1), (MVE_VMOVimmi8 0), ARMVCCNone, VCCR:$pred, zero_reg))>;
@@ -6822,6 +6860,8 @@ let Predicates = [HasMVEInt] in {
             (v8i16 (MVE_VPSEL (MVE_VMOVimmi16 1), (MVE_VMOVimmi16 0), ARMVCCNone, VCCR:$pred, zero_reg))>;
   def : Pat<(v4i32 (anyext  (v4i1  VCCR:$pred))),
             (v4i32 (MVE_VPSEL (MVE_VMOVimmi32 1), (MVE_VMOVimmi32 0), ARMVCCNone, VCCR:$pred, zero_reg))>;
+  def : Pat<(v2i64 (anyext  (v2i1  VCCR:$pred))),
+            (v2i64 (MVE_VPSEL (MVE_VMOVimmi64 1), (MVE_VMOVimmi32 0), ARMVCCNone, VCCR:$pred, zero_reg))>;
 }
 
 let Predicates = [HasMVEFloat] in {
@@ -6862,6 +6902,8 @@ def MVE_VPNOT : MVE_p<(outs VCCR:$P0), (ins VCCR:$P0_in), NoItinerary,
 }
 
 let Predicates = [HasMVEInt] in {
+  def : Pat<(v2i1 (xor (v2i1 VCCR:$pred), (v2i1 (predicate_cast (i32 65535))))),
+            (v2i1 (MVE_VPNOT (v2i1 VCCR:$pred)))>;
   def : Pat<(v4i1 (xor (v4i1 VCCR:$pred), (v4i1 (predicate_cast (i32 65535))))),
             (v4i1 (MVE_VPNOT (v4i1 VCCR:$pred)))>;
   def : Pat<(v8i1 (xor (v8i1 VCCR:$pred), (v8i1 (predicate_cast (i32 65535))))),
diff --git a/llvm/lib/Target/ARM/ARMInstrThumb2.td b/llvm/lib/Target/ARM/ARMInstrThumb2.td
index 4471317f4ea4..6e8e61ca2b8e 100644
--- a/llvm/lib/Target/ARM/ARMInstrThumb2.td
+++ b/llvm/lib/Target/ARM/ARMInstrThumb2.td
@@ -5736,3 +5736,10 @@ def t2BTI    : PACBTIHintSpaceNoOpsInst<"bti", 0b00001111>;
 def t2AUT    : PACBTIHintSpaceUseInst<"aut", 0b00101101> {
   let hasSideEffects = 1;
 }
+
+def ARMt2CallBTI : SDNode<"ARMISD::t2CALL_BTI", SDT_ARMcall,
+                   [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue, SDNPVariadic]>;
+
+def t2CALL_BTI : PseudoInst<(outs), (ins pred:$p, thumb_bl_target:$func),
+                 IIC_Br, [(ARMt2CallBTI tglobaladdr:$func)]>,
+                 Requires<[IsThumb2]>, Sched<[WriteBrL]>;
diff --git a/llvm/lib/Target/ARM/ARMInstrVFP.td b/llvm/lib/Target/ARM/ARMInstrVFP.td
index 9d1bfa414dff..dc5f1b92a6c2 100644
--- a/llvm/lib/Target/ARM/ARMInstrVFP.td
+++ b/llvm/lib/Target/ARM/ARMInstrVFP.td
@@ -1076,6 +1076,9 @@ multiclass vrint_inst_anpm<string opc, bits<2> rm,
     }
   }
 
+  def : InstAlias<!strconcat("vrint", opc, ".f16.f16\t$Sd, $Sm"),
+                  (!cast<Instruction>(NAME#"H") HPR:$Sd, HPR:$Sm), 0>,
+        Requires<[HasFullFP16]>;
   def : InstAlias<!strconcat("vrint", opc, ".f32.f32\t$Sd, $Sm"),
                   (!cast<Instruction>(NAME#"S") SPR:$Sd, SPR:$Sm), 0>,
         Requires<[HasFPARMv8]>;
diff --git a/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp b/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
index 3b10c60a0654..ef5fc12feb54 100644
--- a/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
@@ -2121,7 +2121,7 @@ bool ARMLoadStoreOpt::runOnMachineFunction(MachineFunction &Fn) {
   bool Modified = false;
   for (MachineBasicBlock &MBB : Fn) {
     Modified |= LoadStoreMultipleOpti(MBB);
-    if (STI->hasV5TOps())
+    if (STI->hasV5TOps() && !AFI->shouldSignReturnAddress())
       Modified |= MergeReturnIntoLDM(MBB);
     if (isThumb1)
       Modified |= CombineMovBx(MBB);
@@ -2349,9 +2349,8 @@ bool ARMPreAllocLoadStoreOpt::RescheduleOps(MachineBasicBlock *MBB,
     unsigned LastOpcode = 0;
     unsigned LastBytes = 0;
     unsigned NumMove = 0;
-    for (int i = Ops.size() - 1; i >= 0; --i) {
+    for (MachineInstr *Op : llvm::reverse(Ops)) {
       // Make sure each operation has the same kind.
-      MachineInstr *Op = Ops[i];
       unsigned LSMOpcode
         = getLoadStoreMultipleOpcode(Op->getOpcode(), ARM_AM::ia);
       if (LastOpcode && LSMOpcode != LastOpcode)
diff --git a/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp b/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp
index 3874db5792d6..f822672c4477 100644
--- a/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp
+++ b/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp
@@ -251,10 +251,7 @@ namespace {
       SetVector<MachineInstr *> &Predicates = PredicatedInsts[MI]->Predicates;
       if (Exclusive && Predicates.size() != 1)
         return false;
-      for (auto *PredMI : Predicates)
-        if (isVCTP(PredMI))
-          return true;
-      return false;
+      return llvm::any_of(Predicates, isVCTP);
     }
 
     // Is the VPST, controlling the block entry, predicated upon a VCTP.
@@ -351,10 +348,7 @@ namespace {
     }
 
     bool containsVCTP() const {
-      for (auto *MI : Insts)
-        if (isVCTP(MI))
-          return true;
-      return false;
+      return llvm::any_of(Insts, isVCTP);
     }
 
     unsigned size() const { return Insts.size(); }
@@ -1334,8 +1328,8 @@ bool ARMLowOverheadLoops::ProcessLoop(MachineLoop *ML) {
   bool Changed = false;
 
   // Process inner loops first.
-  for (auto I = ML->begin(), E = ML->end(); I != E; ++I)
-    Changed |= ProcessLoop(*I);
+  for (MachineLoop *L : *ML)
+    Changed |= ProcessLoop(L);
 
   LLVM_DEBUG({
     dbgs() << "ARM Loops: Processing loop containing:\n";
@@ -1699,7 +1693,7 @@ void ARMLowOverheadLoops::ConvertVPTBlocks(LowOverheadLoop &LoLoop) {
         // If any of the instructions between the VCMP and VPST are predicated
         // then a different code path is expected to have merged the VCMP and
         // VPST already.
-        if (!std::any_of(++MachineBasicBlock::iterator(VCMP),
+        if (std::none_of(++MachineBasicBlock::iterator(VCMP),
                          MachineBasicBlock::iterator(VPST), hasVPRUse) &&
             RDA->hasSameReachingDef(VCMP, VPST, VCMP->getOperand(1).getReg()) &&
             RDA->hasSameReachingDef(VCMP, VPST, VCMP->getOperand(2).getReg())) {
diff --git a/llvm/lib/Target/ARM/ARMMachineFunctionInfo.h b/llvm/lib/Target/ARM/ARMMachineFunctionInfo.h
index 4077fc058217..d8d937055d23 100644
--- a/llvm/lib/Target/ARM/ARMMachineFunctionInfo.h
+++ b/llvm/lib/Target/ARM/ARMMachineFunctionInfo.h
@@ -289,7 +289,7 @@ public:
       return false;
     if (SignReturnAddressAll)
       return true;
-    return LRSpilled;
+    return SpillsLR;
   }
 
   bool branchTargetEnforcement() const { return BranchTargetEnforcement; }
diff --git a/llvm/lib/Target/ARM/ARMRegisterInfo.td b/llvm/lib/Target/ARM/ARMRegisterInfo.td
index 760a5a5a20cf..194d65cad8d1 100644
--- a/llvm/lib/Target/ARM/ARMRegisterInfo.td
+++ b/llvm/lib/Target/ARM/ARMRegisterInfo.td
@@ -211,6 +211,8 @@ def FPCXTS  : ARMReg<15, "fpcxts">;
 
 def ZR  : ARMReg<15, "zr">,  DwarfRegNum<[15]>;
 
+def RA_AUTH_CODE : ARMReg<12, "ra_auth_code">, DwarfRegNum<[143]>;
+
 // Register classes.
 //
 // pc  == Program Counter
@@ -395,7 +397,7 @@ def CCR : RegisterClass<"ARM", [i32], 32, (add CPSR)> {
 }
 
 // MVE Condition code register.
-def VCCR : RegisterClass<"ARM", [i32, v16i1, v8i1, v4i1], 32, (add VPR)> {
+def VCCR : RegisterClass<"ARM", [i32, v16i1, v8i1, v4i1, v2i1], 32, (add VPR)> {
 //  let CopyCost = -1;  // Don't allow copying of status registers.
 }
 
diff --git a/llvm/lib/Target/ARM/ARMSubtarget.h b/llvm/lib/Target/ARM/ARMSubtarget.h
index d51a888c951f..e61b90af31b0 100644
--- a/llvm/lib/Target/ARM/ARMSubtarget.h
+++ b/llvm/lib/Target/ARM/ARMSubtarget.h
@@ -18,6 +18,7 @@
 #include "ARMConstantPoolValue.h"
 #include "ARMFrameLowering.h"
 #include "ARMISelLowering.h"
+#include "ARMMachineFunctionInfo.h"
 #include "ARMSelectionDAGInfo.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
@@ -534,6 +535,10 @@ protected:
   /// Selected instruction itineraries (one entry per itinerary class.)
   InstrItineraryData InstrItins;
 
+  /// NoBTIAtReturnTwice - Don't place a BTI instruction after
+  /// return-twice constructs (setjmp)
+  bool NoBTIAtReturnTwice = false;
+
   /// Options passed via command line that could influence the target
   const TargetOptions &Options;
 
@@ -840,6 +845,8 @@ public:
   /// to lr. This is always required on Thumb1-only targets, as the push and
   /// pop instructions can't access the high registers.
   bool splitFramePushPop(const MachineFunction &MF) const {
+    if (MF.getInfo<ARMFunctionInfo>()->shouldSignReturnAddress())
+      return true;
     return (getFramePointerReg() == ARM::R7 &&
             MF.getTarget().Options.DisableFramePointerElim(MF)) ||
            isThumb1Only();
@@ -948,6 +955,8 @@ public:
   bool hardenSlsRetBr() const { return HardenSlsRetBr; }
   bool hardenSlsBlr() const { return HardenSlsBlr; }
   bool hardenSlsNoComdat() const { return HardenSlsNoComdat; }
+
+  bool getNoBTIAtReturnTwice() const { return NoBTIAtReturnTwice; }
 };
 
 } // end namespace llvm
diff --git a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
index 39f407ba7149..bfe078b06861 100644
--- a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
+++ b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
@@ -137,21 +137,18 @@ public:
   int getFPReg() const { return FPReg; }
 
   void emitFnStartLocNotes() const {
-    for (Locs::const_iterator FI = FnStartLocs.begin(), FE = FnStartLocs.end();
-         FI != FE; ++FI)
-      Parser.Note(*FI, ".fnstart was specified here");
+    for (const SMLoc &Loc : FnStartLocs)
+      Parser.Note(Loc, ".fnstart was specified here");
   }
 
   void emitCantUnwindLocNotes() const {
-    for (Locs::const_iterator UI = CantUnwindLocs.begin(),
-                              UE = CantUnwindLocs.end(); UI != UE; ++UI)
-      Parser.Note(*UI, ".cantunwind was specified here");
+    for (const SMLoc &Loc : CantUnwindLocs)
+      Parser.Note(Loc, ".cantunwind was specified here");
   }
 
   void emitHandlerDataLocNotes() const {
-    for (Locs::const_iterator HI = HandlerDataLocs.begin(),
-                              HE = HandlerDataLocs.end(); HI != HE; ++HI)
-      Parser.Note(*HI, ".handlerdata was specified here");
+    for (const SMLoc &Loc : HandlerDataLocs)
+      Parser.Note(Loc, ".handlerdata was specified here");
   }
 
   void emitPersonalityLocNotes() const {
@@ -452,7 +449,8 @@ class ARMAsmParser : public MCTargetAsmParser {
   int tryParseRegister();
   bool tryParseRegisterWithWriteBack(OperandVector &);
   int tryParseShiftRegister(OperandVector &);
-  bool parseRegisterList(OperandVector &, bool EnforceOrder = true);
+  bool parseRegisterList(OperandVector &, bool EnforceOrder = true,
+                         bool AllowRAAC = false);
   bool parseMemory(OperandVector &);
   bool parseOperand(OperandVector &, StringRef Mnemonic);
   bool parsePrefix(ARMMCExpr::VariantKind &RefKind);
@@ -2572,17 +2570,15 @@ public:
   void addRegListOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     const SmallVectorImpl<unsigned> &RegList = getRegList();
-    for (SmallVectorImpl<unsigned>::const_iterator
-           I = RegList.begin(), E = RegList.end(); I != E; ++I)
-      Inst.addOperand(MCOperand::createReg(*I));
+    for (unsigned Reg : RegList)
+      Inst.addOperand(MCOperand::createReg(Reg));
   }
 
   void addRegListWithAPSROperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     const SmallVectorImpl<unsigned> &RegList = getRegList();
-    for (SmallVectorImpl<unsigned>::const_iterator
-           I = RegList.begin(), E = RegList.end(); I != E; ++I)
-      Inst.addOperand(MCOperand::createReg(*I));
+    for (unsigned Reg : RegList)
+      Inst.addOperand(MCOperand::createReg(Reg));
   }
 
   void addDPRRegListOperands(MCInst &Inst, unsigned N) const {
@@ -4464,8 +4460,8 @@ insertNoDuplicates(SmallVectorImpl<std::pair<unsigned, unsigned>> &Regs,
 }
 
 /// Parse a register list.
-bool ARMAsmParser::parseRegisterList(OperandVector &Operands,
-                                     bool EnforceOrder) {
+bool ARMAsmParser::parseRegisterList(OperandVector &Operands, bool EnforceOrder,
+                                     bool AllowRAAC) {
   MCAsmParser &Parser = getParser();
   if (Parser.getTok().isNot(AsmToken::LCurly))
     return TokError("Token is not a Left Curly Brace");
@@ -4478,7 +4474,8 @@ bool ARMAsmParser::parseRegisterList(OperandVector &Operands,
   int Reg = tryParseRegister();
   if (Reg == -1)
     return Error(RegLoc, "register expected");
-
+  if (!AllowRAAC && Reg == ARM::RA_AUTH_CODE)
+    return Error(RegLoc, "pseudo-register not allowed");
   // The reglist instructions have at most 16 registers, so reserve
   // space for that many.
   int EReg = 0;
@@ -4492,7 +4489,8 @@ bool ARMAsmParser::parseRegisterList(OperandVector &Operands,
     ++Reg;
   }
   const MCRegisterClass *RC;
-  if (ARMMCRegisterClasses[ARM::GPRRegClassID].contains(Reg))
+  if (Reg == ARM::RA_AUTH_CODE ||
+      ARMMCRegisterClasses[ARM::GPRRegClassID].contains(Reg))
     RC = &ARMMCRegisterClasses[ARM::GPRRegClassID];
   else if (ARMMCRegisterClasses[ARM::DPRRegClassID].contains(Reg))
     RC = &ARMMCRegisterClasses[ARM::DPRRegClassID];
@@ -4513,11 +4511,15 @@ bool ARMAsmParser::parseRegisterList(OperandVector &Operands,
   while (Parser.getTok().is(AsmToken::Comma) ||
          Parser.getTok().is(AsmToken::Minus)) {
     if (Parser.getTok().is(AsmToken::Minus)) {
+      if (Reg == ARM::RA_AUTH_CODE)
+        return Error(RegLoc, "pseudo-register not allowed");
       Parser.Lex(); // Eat the minus.
       SMLoc AfterMinusLoc = Parser.getTok().getLoc();
       int EndReg = tryParseRegister();
       if (EndReg == -1)
         return Error(AfterMinusLoc, "register expected");
+      if (EndReg == ARM::RA_AUTH_CODE)
+        return Error(AfterMinusLoc, "pseudo-register not allowed");
       // Allow Q regs and just interpret them as the two D sub-registers.
       if (ARMMCRegisterClasses[ARM::QPRRegClassID].contains(EndReg))
         EndReg = getDRegFromQReg(EndReg) + 1;
@@ -4526,7 +4528,9 @@ bool ARMAsmParser::parseRegisterList(OperandVector &Operands,
       if (Reg == EndReg)
         continue;
       // The register must be in the same register class as the first.
-      if (!RC->contains(EndReg))
+      if ((Reg == ARM::RA_AUTH_CODE &&
+           RC != &ARMMCRegisterClasses[ARM::GPRRegClassID]) ||
+          (Reg != ARM::RA_AUTH_CODE && !RC->contains(Reg)))
         return Error(AfterMinusLoc, "invalid register in register list");
       // Ranges must go from low to high.
       if (MRI->getEncodingValue(Reg) > MRI->getEncodingValue(EndReg))
@@ -4551,13 +4555,15 @@ bool ARMAsmParser::parseRegisterList(OperandVector &Operands,
     Reg = tryParseRegister();
     if (Reg == -1)
       return Error(RegLoc, "register expected");
+    if (!AllowRAAC && Reg == ARM::RA_AUTH_CODE)
+      return Error(RegLoc, "pseudo-register not allowed");
     // Allow Q regs and just interpret them as the two D sub-registers.
     bool isQReg = false;
     if (ARMMCRegisterClasses[ARM::QPRRegClassID].contains(Reg)) {
       Reg = getDRegFromQReg(Reg);
       isQReg = true;
     }
-    if (!RC->contains(Reg) &&
+    if (Reg != ARM::RA_AUTH_CODE && !RC->contains(Reg) &&
         RC->getID() == ARMMCRegisterClasses[ARM::GPRRegClassID].getID() &&
         ARMMCRegisterClasses[ARM::GPRwithAPSRnospRegClassID].contains(Reg)) {
       // switch the register classes, as GPRwithAPSRnospRegClassID is a partial
@@ -4577,7 +4583,9 @@ bool ARMAsmParser::parseRegisterList(OperandVector &Operands,
       continue;
     }
     // The register must be in the same register class as the first.
-    if (!RC->contains(Reg))
+    if ((Reg == ARM::RA_AUTH_CODE &&
+         RC != &ARMMCRegisterClasses[ARM::GPRRegClassID]) ||
+        (Reg != ARM::RA_AUTH_CODE && !RC->contains(Reg)))
       return Error(RegLoc, "invalid register in register list");
     // In most cases, the list must be monotonically increasing. An
     // exception is CLRM, which is order-independent anyway, so
@@ -7106,13 +7114,12 @@ bool ARMAsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
       return Error(Loc, "too many conditions on VPT instruction");
     }
     unsigned Mask = 8;
-    for (unsigned i = ITMask.size(); i != 0; --i) {
-      char pos = ITMask[i - 1];
-      if (pos != 't' && pos != 'e') {
+    for (char Pos : llvm::reverse(ITMask)) {
+      if (Pos != 't' && Pos != 'e') {
         return Error(Loc, "illegal IT block condition mask '" + ITMask + "'");
       }
       Mask >>= 1;
-      if (ITMask[i - 1] == 'e')
+      if (Pos == 'e')
         Mask |= 8;
     }
     Operands.push_back(ARMOperand::CreateITMask(Mask, Loc));
@@ -11685,7 +11692,7 @@ bool ARMAsmParser::parseDirectiveRegSave(SMLoc L, bool IsVector) {
   SmallVector<std::unique_ptr<MCParsedAsmOperand>, 1> Operands;
 
   // Parse the register list
-  if (parseRegisterList(Operands) ||
+  if (parseRegisterList(Operands, true, true) ||
       parseToken(AsmToken::EndOfStatement, "unexpected token in directive"))
     return true;
   ARMOperand &Op = (ARMOperand &)*Operands[0];
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp
index 896b104e8d97..e060e59e3759 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp
@@ -1289,34 +1289,65 @@ void ARMELFStreamer::emitPad(int64_t Offset) {
   PendingOffset -= Offset;
 }
 
-void ARMELFStreamer::emitRegSave(const SmallVectorImpl<unsigned> &RegList,
-                                 bool IsVector) {
-  // Collect the registers in the register list
-  unsigned Count = 0;
+static std::pair<unsigned, unsigned>
+collectHWRegs(const MCRegisterInfo &MRI, unsigned Idx,
+              const SmallVectorImpl<unsigned> &RegList, bool IsVector,
+              uint32_t &Mask_) {
   uint32_t Mask = 0;
-  const MCRegisterInfo *MRI = getContext().getRegisterInfo();
-  for (size_t i = 0; i < RegList.size(); ++i) {
-    unsigned Reg = MRI->getEncodingValue(RegList[i]);
+  unsigned Count = 0;
+  while (Idx > 0) {
+    unsigned Reg = RegList[Idx - 1];
+    if (Reg == ARM::RA_AUTH_CODE)
+      break;
+    Reg = MRI.getEncodingValue(Reg);
     assert(Reg < (IsVector ? 32U : 16U) && "Register out of range");
     unsigned Bit = (1u << Reg);
     if ((Mask & Bit) == 0) {
       Mask |= Bit;
       ++Count;
     }
+    --Idx;
   }
 
-  // Track the change the $sp offset: For the .save directive, the
-  // corresponding push instruction will decrease the $sp by (4 * Count).
-  // For the .vsave directive, the corresponding vpush instruction will
-  // decrease $sp by (8 * Count).
-  SPOffset -= Count * (IsVector ? 8 : 4);
+  Mask_ = Mask;
+  return {Idx, Count};
+}
 
-  // Emit the opcode
-  FlushPendingOffset();
-  if (IsVector)
-    UnwindOpAsm.EmitVFPRegSave(Mask);
-  else
-    UnwindOpAsm.EmitRegSave(Mask);
+void ARMELFStreamer::emitRegSave(const SmallVectorImpl<unsigned> &RegList,
+                                 bool IsVector) {
+  uint32_t Mask;
+  unsigned Idx, Count;
+  const MCRegisterInfo &MRI = *getContext().getRegisterInfo();
+
+  // Collect the registers in the register list. Issue unwinding instructions in
+  // three parts: ordinary hardware registers, return address authentication
+  // code pseudo register, the rest of the registers. The RA PAC is kept in an
+  // architectural register (usually r12), but we treat it as a special case in
+  // order to distinguish between that register containing RA PAC or a general
+  // value.
+  Idx = RegList.size();
+  while (Idx > 0) {
+    std::tie(Idx, Count) = collectHWRegs(MRI, Idx, RegList, IsVector, Mask);
+    if (Count) {
+      // Track the change the $sp offset: For the .save directive, the
+      // corresponding push instruction will decrease the $sp by (4 * Count).
+      // For the .vsave directive, the corresponding vpush instruction will
+      // decrease $sp by (8 * Count).
+      SPOffset -= Count * (IsVector ? 8 : 4);
+
+      // Emit the opcode
+      FlushPendingOffset();
+      if (IsVector)
+        UnwindOpAsm.EmitVFPRegSave(Mask);
+      else
+        UnwindOpAsm.EmitRegSave(Mask);
+    } else if (Idx > 0 && RegList[Idx - 1] == ARM::RA_AUTH_CODE) {
+      --Idx;
+      SPOffset -= 4;
+      FlushPendingOffset();
+      UnwindOpAsm.EmitRegSave(0);
+    }
+  }
 }
 
 void ARMELFStreamer::emitUnwindRaw(int64_t Offset,
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMUnwindOpAsm.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMUnwindOpAsm.cpp
index 781627c3c425..50f416b23db2 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMUnwindOpAsm.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMUnwindOpAsm.cpp
@@ -64,8 +64,11 @@ namespace {
 } // end anonymous namespace
 
 void UnwindOpcodeAssembler::EmitRegSave(uint32_t RegSave) {
-  if (RegSave == 0u)
+  if (RegSave == 0u) {
+    // That's the special case for RA PAC.
+    EmitInt8(ARM::EHABI::UNWIND_OPCODE_POP_RA_AUTH_CODE);
     return;
+  }
 
   // One byte opcode to save register r14 and r11-r4
   if (RegSave & (1u << 4)) {
diff --git a/llvm/lib/Target/ARM/MVETPAndVPTOptimisationsPass.cpp b/llvm/lib/Target/ARM/MVETPAndVPTOptimisationsPass.cpp
index dc58b5427425..7e31ea77f4f5 100644
--- a/llvm/lib/Target/ARM/MVETPAndVPTOptimisationsPass.cpp
+++ b/llvm/lib/Target/ARM/MVETPAndVPTOptimisationsPass.cpp
@@ -366,7 +366,7 @@ bool MVETPAndVPTOptimisations::MergeLoopEnd(MachineLoop *ML) {
     while (!Worklist.empty()) {
       Register Reg = Worklist.pop_back_val();
       for (MachineInstr &MI : MRI->use_nodbg_instructions(Reg)) {
-        if (count(ExpectedUsers, &MI))
+        if (llvm::is_contained(ExpectedUsers, &MI))
           continue;
         if (MI.getOpcode() != TargetOpcode::COPY ||
             !MI.getOperand(0).getReg().isVirtual()) {
diff --git a/llvm/lib/Target/ARM/MVETailPredication.cpp b/llvm/lib/Target/ARM/MVETailPredication.cpp
index 6a5bc9284266..0e6960bce32b 100644
--- a/llvm/lib/Target/ARM/MVETailPredication.cpp
+++ b/llvm/lib/Target/ARM/MVETailPredication.cpp
@@ -213,7 +213,8 @@ bool MVETailPredication::IsSafeActiveMask(IntrinsicInst *ActiveLaneMask,
   auto *TC = SE->getSCEV(TripCount);
   int VectorWidth =
       cast<FixedVectorType>(ActiveLaneMask->getType())->getNumElements();
-  if (VectorWidth != 4 && VectorWidth != 8 && VectorWidth != 16)
+  if (VectorWidth != 2 && VectorWidth != 4 && VectorWidth != 8 &&
+      VectorWidth != 16)
     return false;
   ConstantInt *ConstElemCount = nullptr;
 
@@ -371,15 +372,10 @@ void MVETailPredication::InsertVCTPIntrinsic(IntrinsicInst *ActiveLaneMask,
   switch (VectorWidth) {
   default:
     llvm_unreachable("unexpected number of lanes");
+  case 2:  VCTPID = Intrinsic::arm_mve_vctp64; break;
   case 4:  VCTPID = Intrinsic::arm_mve_vctp32; break;
   case 8:  VCTPID = Intrinsic::arm_mve_vctp16; break;
   case 16: VCTPID = Intrinsic::arm_mve_vctp8; break;
-
-    // FIXME: vctp64 currently not supported because the predicate
-    // vector wants to be <2 x i1>, but v2i1 is not a legal MVE
-    // type, so problems happen at isel time.
-    // Intrinsic::arm_mve_vctp64 exists for ACLE intrinsics
-    // purposes, but takes a v4i1 instead of a v2i1.
   }
   Function *VCTP = Intrinsic::getDeclaration(M, VCTPID);
   Value *VCTPCall = Builder.CreateCall(VCTP, Processed);
diff --git a/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp b/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp
index 224c61b9f065..54e80a095dd4 100644
--- a/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp
+++ b/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp
@@ -824,8 +824,8 @@ bool Thumb1FrameLowering::spillCalleeSavedRegisters(
   ARMRegSet CopyRegs;     // Registers which can be used after pushing
                           // LoRegs for saving HiRegs.
 
-  for (unsigned i = CSI.size(); i != 0; --i) {
-    unsigned Reg = CSI[i-1].getReg();
+  for (const CalleeSavedInfo &I : llvm::reverse(CSI)) {
+    unsigned Reg = I.getReg();
 
     if (ARM::tGPRRegClass.contains(Reg) || Reg == ARM::LR) {
       LoRegsToSave[Reg] = true;
@@ -1021,8 +1021,7 @@ bool Thumb1FrameLowering::restoreCalleeSavedRegisters(
       BuildMI(MF, DL, TII.get(ARM::tPOP)).add(predOps(ARMCC::AL));
 
   bool NeedsPop = false;
-  for (unsigned i = CSI.size(); i != 0; --i) {
-    CalleeSavedInfo &Info = CSI[i-1];
+  for (CalleeSavedInfo &Info : llvm::reverse(CSI)) {
     unsigned Reg = Info.getReg();
 
     // High registers (excluding lr) have already been dealt with
@@ -1067,7 +1066,7 @@ bool Thumb1FrameLowering::restoreCalleeSavedRegisters(
   if (NeedsPop)
     MBB.insert(MI, &*MIB);
   else
-    MF.DeleteMachineInstr(MIB);
+    MF.deleteMachineInstr(MIB);
 
   return true;
 }
diff --git a/llvm/lib/Target/AVR/AVRFrameLowering.cpp b/llvm/lib/Target/AVR/AVRFrameLowering.cpp
index 672611ea2234..543d94875037 100644
--- a/llvm/lib/Target/AVR/AVRFrameLowering.cpp
+++ b/llvm/lib/Target/AVR/AVRFrameLowering.cpp
@@ -247,8 +247,8 @@ bool AVRFrameLowering::spillCalleeSavedRegisters(
   const TargetInstrInfo &TII = *STI.getInstrInfo();
   AVRMachineFunctionInfo *AVRFI = MF.getInfo<AVRMachineFunctionInfo>();
 
-  for (unsigned i = CSI.size(); i != 0; --i) {
-    unsigned Reg = CSI[i - 1].getReg();
+  for (const CalleeSavedInfo &I : llvm::reverse(CSI)) {
+    unsigned Reg = I.getReg();
     bool IsNotLiveIn = !MBB.isLiveIn(Reg);
 
     assert(TRI->getRegSizeInBits(*TRI->getMinimalPhysRegClass(Reg)) == 8 &&
diff --git a/llvm/lib/Target/AVR/AVRInstrInfo.cpp b/llvm/lib/Target/AVR/AVRInstrInfo.cpp
index 798d08393eae..51060018a5ca 100644
--- a/llvm/lib/Target/AVR/AVRInstrInfo.cpp
+++ b/llvm/lib/Target/AVR/AVRInstrInfo.cpp
@@ -571,8 +571,6 @@ void AVRInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB,
   // See lib/CodeGen/RegisterRelaxation.cpp for details.
   // We end up here when a jump is too long for a RJMP instruction.
   BuildMI(&MBB, DL, get(AVR::JMPk)).addMBB(&NewDestBB);
-
-  return;
 }
 
 } // end of namespace llvm
diff --git a/llvm/lib/Target/BPF/BPFPreserveDIType.cpp b/llvm/lib/Target/BPF/BPFPreserveDIType.cpp
index 0348e2200acb..36237b2fc4fd 100644
--- a/llvm/lib/Target/BPF/BPFPreserveDIType.cpp
+++ b/llvm/lib/Target/BPF/BPFPreserveDIType.cpp
@@ -93,8 +93,13 @@ static bool BPFPreserveDITypeImpl(Function &F) {
         Ty = DTy->getBaseType();
       }
 
-      if (Ty->getName().empty())
-        report_fatal_error("Empty type name for BTF_TYPE_ID_REMOTE reloc");
+      if (Ty->getName().empty()) {
+        if (isa<DISubroutineType>(Ty))
+          report_fatal_error(
+              "SubroutineType not supported for BTF_TYPE_ID_REMOTE reloc");
+        else
+          report_fatal_error("Empty type name for BTF_TYPE_ID_REMOTE reloc");
+      }
       MD = Ty;
     }
 
diff --git a/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp b/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp
index ebc04b40d428..29b99a84a6cd 100644
--- a/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp
+++ b/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp
@@ -11,6 +11,7 @@
 #include "MCTargetDesc/CSKYMCTargetDesc.h"
 #include "TargetInfo/CSKYTargetInfo.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/CodeGen/Register.h"
 #include "llvm/MC/MCContext.h"
@@ -25,11 +26,24 @@
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/TargetRegistry.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 
+using namespace llvm;
+
 #define DEBUG_TYPE "csky-asm-parser"
 
-using namespace llvm;
+// Include the auto-generated portion of the compress emitter.
+#define GEN_COMPRESS_INSTR
+#include "CSKYGenCompressInstEmitter.inc"
+
+STATISTIC(CSKYNumInstrsCompressed,
+          "Number of C-SKY Compressed instructions emitted");
+
+static cl::opt<bool>
+    EnableCompressedInst("enable-csky-asm-compressed-inst", cl::Hidden,
+                         cl::init(false),
+                         cl::desc("Enable C-SKY asm compressed instruction"));
 
 namespace {
 struct CSKYOperand;
@@ -55,6 +69,10 @@ class CSKYAsmParser : public MCTargetAsmParser {
 
   bool ParseDirective(AsmToken DirectiveID) override;
 
+  // Helper to actually emit an instruction to the MCStreamer. Also, when
+  // possible, compression of the instruction is performed.
+  void emitToStreamer(MCStreamer &S, const MCInst &Inst);
+
   OperandMatchResultTy tryParseRegister(unsigned &RegNo, SMLoc &StartLoc,
                                         SMLoc &EndLoc) override;
 
@@ -264,12 +282,6 @@ public:
   bool isConstpool() const { return isConstPoolOp(); }
   bool isDataSymbol() const { return isConstPoolOp(); }
 
-  bool isSPOperand() const {
-    if (!isReg())
-      return false;
-    return getReg() == CSKY::R14;
-  }
-
   bool isPSRFlag() const {
     int64_t Imm;
     // Must be of 'immediate' type and a constant.
@@ -755,10 +767,6 @@ bool CSKYAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
     SMLoc ErrorLoc = ((CSKYOperand &)*Operands[ErrorInfo]).getStartLoc();
     return Error(ErrorLoc, "register is out of range");
   }
-  case Match_InvalidSPOperand: {
-    SMLoc ErrorLoc = ((CSKYOperand &)*Operands[ErrorInfo]).getStartLoc();
-    return Error(ErrorLoc, "operand must be sp register");
-  }
   case Match_RequiresSameSrcAndDst: {
     SMLoc ErrorLoc = ((CSKYOperand &)*Operands[ErrorInfo]).getStartLoc();
     return Error(ErrorLoc, "src and dst operand must be same");
@@ -776,27 +784,62 @@ bool CSKYAsmParser::processInstruction(MCInst &Inst, SMLoc IDLoc,
                                        OperandVector &Operands,
                                        MCStreamer &Out) {
 
-  if (Inst.getOpcode() == CSKY::LDQ32 || Inst.getOpcode() == CSKY::STQ32) {
+  switch (Inst.getOpcode()) {
+  default:
+    break;
+  case CSKY::LDQ32:
+  case CSKY::STQ32:
     if (Inst.getOperand(1).getReg() != CSKY::R4 ||
         Inst.getOperand(2).getReg() != CSKY::R7) {
       return Error(IDLoc, "Register sequence is not valid. 'r4-r7' expected");
     }
     Inst.setOpcode(Inst.getOpcode() == CSKY::LDQ32 ? CSKY::LDM32 : CSKY::STM32);
-    Out.emitInstruction(Inst, getSTI());
-    return false;
-  } else if (Inst.getOpcode() == CSKY::SEXT32 ||
-             Inst.getOpcode() == CSKY::ZEXT32) {
+    break;
+  case CSKY::SEXT32:
+  case CSKY::ZEXT32:
     if (Inst.getOperand(2).getImm() < Inst.getOperand(3).getImm())
       return Error(IDLoc, "msb must be greater or equal to lsb");
-  } else if (Inst.getOpcode() == CSKY::INS32) {
+    break;
+  case CSKY::INS32:
     if (Inst.getOperand(3).getImm() < Inst.getOperand(4).getImm())
       return Error(IDLoc, "msb must be greater or equal to lsb");
-  } else if (Inst.getOpcode() == CSKY::IDLY32) {
+    break;
+  case CSKY::IDLY32:
     if (Inst.getOperand(0).getImm() > 32 || Inst.getOperand(0).getImm() < 0)
       return Error(IDLoc, "n must be in range [0,32]");
+    break;
+  case CSKY::ADDC32:
+  case CSKY::SUBC32:
+  case CSKY::ADDC16:
+  case CSKY::SUBC16:
+    Inst.erase(std::next(Inst.begin()));
+    Inst.erase(std::prev(Inst.end()));
+    Inst.insert(std::next(Inst.begin()), MCOperand::createReg(CSKY::C));
+    Inst.insert(Inst.end(), MCOperand::createReg(CSKY::C));
+    break;
+  case CSKY::CMPNEI32:
+  case CSKY::CMPNEI16:
+  case CSKY::CMPNE32:
+  case CSKY::CMPNE16:
+  case CSKY::CMPHSI32:
+  case CSKY::CMPHSI16:
+  case CSKY::CMPHS32:
+  case CSKY::CMPHS16:
+  case CSKY::CMPLTI32:
+  case CSKY::CMPLTI16:
+  case CSKY::CMPLT32:
+  case CSKY::CMPLT16:
+  case CSKY::BTSTI32:
+    Inst.erase(Inst.begin());
+    Inst.insert(Inst.begin(), MCOperand::createReg(CSKY::C));
+    break;
+  case CSKY::MVCV32:
+    Inst.erase(std::next(Inst.begin()));
+    Inst.insert(Inst.end(), MCOperand::createReg(CSKY::C));
+    break;
   }
 
-  Out.emitInstruction(Inst, getSTI());
+  emitToStreamer(Out, Inst);
   return false;
 }
 
@@ -1422,6 +1465,16 @@ OperandMatchResultTy CSKYAsmParser::tryParseRegister(unsigned &RegNo,
 
 bool CSKYAsmParser::ParseDirective(AsmToken DirectiveID) { return true; }
 
+void CSKYAsmParser::emitToStreamer(MCStreamer &S, const MCInst &Inst) {
+  MCInst CInst;
+  bool Res = false;
+  if (EnableCompressedInst)
+    Res = compressInst(CInst, Inst, getSTI(), S.getContext());
+  if (Res)
+    ++CSKYNumInstrsCompressed;
+  S.emitInstruction((Res ? CInst : Inst), getSTI());
+}
+
 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeCSKYAsmParser() {
   RegisterMCAsmParser<CSKYAsmParser> X(getTheCSKYTarget());
 }
diff --git a/llvm/lib/Target/CSKY/CSKYAsmPrinter.cpp b/llvm/lib/Target/CSKY/CSKYAsmPrinter.cpp
index 1c38c5d1fde6..85129f78e726 100644
--- a/llvm/lib/Target/CSKY/CSKYAsmPrinter.cpp
+++ b/llvm/lib/Target/CSKY/CSKYAsmPrinter.cpp
@@ -30,6 +30,9 @@ using namespace llvm;
 
 #define DEBUG_TYPE "csky-asm-printer"
 
+STATISTIC(CSKYNumInstrsCompressed,
+          "Number of C-SKY Compressed instructions emitted");
+
 CSKYAsmPrinter::CSKYAsmPrinter(llvm::TargetMachine &TM,
                                std::unique_ptr<llvm::MCStreamer> Streamer)
     : AsmPrinter(TM, std::move(Streamer)), MCInstLowering(OutContext, *this) {}
@@ -39,6 +42,16 @@ bool CSKYAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
   return AsmPrinter::runOnMachineFunction(MF);
 }
 
+#define GEN_COMPRESS_INSTR
+#include "CSKYGenCompressInstEmitter.inc"
+void CSKYAsmPrinter::EmitToStreamer(MCStreamer &S, const MCInst &Inst) {
+  MCInst CInst;
+  bool Res = compressInst(CInst, Inst, *Subtarget, OutStreamer->getContext());
+  if (Res)
+    ++CSKYNumInstrsCompressed;
+  AsmPrinter::EmitToStreamer(*OutStreamer, Res ? CInst : Inst);
+}
+
 // Simple pseudo-instructions have their lowering (with expansion to real
 // instructions) auto-generated.
 #include "CSKYGenMCPseudoLowering.inc"
diff --git a/llvm/lib/Target/CSKY/CSKYAsmPrinter.h b/llvm/lib/Target/CSKY/CSKYAsmPrinter.h
index f0f5d8657c04..b30311e0ca64 100644
--- a/llvm/lib/Target/CSKY/CSKYAsmPrinter.h
+++ b/llvm/lib/Target/CSKY/CSKYAsmPrinter.h
@@ -26,6 +26,8 @@ public:
 
   StringRef getPassName() const override { return "CSKY Assembly Printer"; }
 
+  void EmitToStreamer(MCStreamer &S, const MCInst &Inst);
+
   /// tblgen'erated driver function for lowering simple MI->MC
   /// pseudo instructions.
   bool emitPseudoExpansionLowering(MCStreamer &OutStreamer,
diff --git a/llvm/lib/Target/CSKY/CSKYCallingConv.td b/llvm/lib/Target/CSKY/CSKYCallingConv.td
index 87e2e6b9dc31..91102e3714df 100644
--- a/llvm/lib/Target/CSKY/CSKYCallingConv.td
+++ b/llvm/lib/Target/CSKY/CSKYCallingConv.td
@@ -79,4 +79,4 @@ def RetCC_CSKY_ABIV2_FP : CallingConv<[
   CCIfType<[i32], CCAssignToReg<[R0, R1]>>,
   CCIfType<[f32], CCAssignToReg<[F0_32]>>,
   CCIfType<[f64], CCAssignToReg<[F0_64]>>
-]>;
\ No newline at end of file
+]>;
diff --git a/llvm/lib/Target/CSKY/CSKYFrameLowering.cpp b/llvm/lib/Target/CSKY/CSKYFrameLowering.cpp
index 9b22c95cfe21..3a8ee5713584 100644
--- a/llvm/lib/Target/CSKY/CSKYFrameLowering.cpp
+++ b/llvm/lib/Target/CSKY/CSKYFrameLowering.cpp
@@ -54,4 +54,4 @@ void CSKYFrameLowering::emitPrologue(MachineFunction &MF,
 void CSKYFrameLowering::emitEpilogue(MachineFunction &MF,
                                      MachineBasicBlock &MBB) const {
   // FIXME: Implement this when we have function calls
-}
\ No newline at end of file
+}
diff --git a/llvm/lib/Target/CSKY/CSKYISelDAGToDAG.cpp b/llvm/lib/Target/CSKY/CSKYISelDAGToDAG.cpp
index fc9ef8bfd9d9..8dc91904b8cc 100644
--- a/llvm/lib/Target/CSKY/CSKYISelDAGToDAG.cpp
+++ b/llvm/lib/Target/CSKY/CSKYISelDAGToDAG.cpp
@@ -40,6 +40,8 @@ public:
   }
 
   void Select(SDNode *N) override;
+  bool selectAddCarry(SDNode *N);
+  bool selectSubCarry(SDNode *N);
 
 #include "CSKYGenDAGISel.inc"
 };
@@ -60,7 +62,12 @@ void CSKYDAGToDAGISel::Select(SDNode *N) {
   switch (Opcode) {
   default:
     break;
-    // FIXME: Add selection nodes needed later.
+  case ISD::ADDCARRY:
+    IsSelected = selectAddCarry(N);
+    break;
+  case ISD::SUBCARRY:
+    IsSelected = selectSubCarry(N);
+    break;
   }
 
   if (IsSelected)
@@ -70,6 +77,86 @@ void CSKYDAGToDAGISel::Select(SDNode *N) {
   SelectCode(N);
 }
 
+bool CSKYDAGToDAGISel::selectAddCarry(SDNode *N) {
+  MachineSDNode *NewNode = nullptr;
+  auto Type0 = N->getValueType(0);
+  auto Type1 = N->getValueType(1);
+  auto Op0 = N->getOperand(0);
+  auto Op1 = N->getOperand(1);
+  auto Op2 = N->getOperand(2);
+
+  SDLoc Dl(N);
+
+  if (isNullConstant(Op2)) {
+    auto *CA = CurDAG->getMachineNode(
+        Subtarget->has2E3() ? CSKY::CLRC32 : CSKY::CLRC16, Dl, Type1);
+    NewNode = CurDAG->getMachineNode(
+        Subtarget->has2E3() ? CSKY::ADDC32 : CSKY::ADDC16, Dl, {Type0, Type1},
+        {Op0, Op1, SDValue(CA, 0)});
+  } else if (isOneConstant(Op2)) {
+    auto *CA = CurDAG->getMachineNode(
+        Subtarget->has2E3() ? CSKY::SETC32 : CSKY::SETC16, Dl, Type1);
+    NewNode = CurDAG->getMachineNode(
+        Subtarget->has2E3() ? CSKY::ADDC32 : CSKY::ADDC16, Dl, {Type0, Type1},
+        {Op0, Op1, SDValue(CA, 0)});
+  } else {
+    NewNode = CurDAG->getMachineNode(Subtarget->has2E3() ? CSKY::ADDC32
+                                                         : CSKY::ADDC16,
+                                     Dl, {Type0, Type1}, {Op0, Op1, Op2});
+  }
+  ReplaceNode(N, NewNode);
+  return true;
+}
+
+static SDValue InvertCarryFlag(const CSKYSubtarget *Subtarget,
+                               SelectionDAG *DAG, SDLoc Dl, SDValue OldCarry) {
+  auto NewCarryReg =
+      DAG->getMachineNode(Subtarget->has2E3() ? CSKY::MVCV32 : CSKY::MVCV16, Dl,
+                          MVT::i32, OldCarry);
+  auto NewCarry =
+      DAG->getMachineNode(Subtarget->hasE2() ? CSKY::BTSTI32 : CSKY::BTSTI16,
+                          Dl, OldCarry.getValueType(), SDValue(NewCarryReg, 0),
+                          DAG->getTargetConstant(0, Dl, MVT::i32));
+  return SDValue(NewCarry, 0);
+}
+
+bool CSKYDAGToDAGISel::selectSubCarry(SDNode *N) {
+  MachineSDNode *NewNode = nullptr;
+  auto Type0 = N->getValueType(0);
+  auto Type1 = N->getValueType(1);
+  auto Op0 = N->getOperand(0);
+  auto Op1 = N->getOperand(1);
+  auto Op2 = N->getOperand(2);
+
+  SDLoc Dl(N);
+
+  if (isNullConstant(Op2)) {
+    auto *CA = CurDAG->getMachineNode(
+        Subtarget->has2E3() ? CSKY::SETC32 : CSKY::SETC16, Dl, Type1);
+    NewNode = CurDAG->getMachineNode(
+        Subtarget->has2E3() ? CSKY::SUBC32 : CSKY::SUBC16, Dl, {Type0, Type1},
+        {Op0, Op1, SDValue(CA, 0)});
+  } else if (isOneConstant(Op2)) {
+    auto *CA = CurDAG->getMachineNode(
+        Subtarget->has2E3() ? CSKY::CLRC32 : CSKY::CLRC16, Dl, Type1);
+    NewNode = CurDAG->getMachineNode(
+        Subtarget->has2E3() ? CSKY::SUBC32 : CSKY::SUBC16, Dl, {Type0, Type1},
+        {Op0, Op1, SDValue(CA, 0)});
+  } else {
+    auto CarryIn = InvertCarryFlag(Subtarget, CurDAG, Dl, Op2);
+    NewNode = CurDAG->getMachineNode(Subtarget->has2E3() ? CSKY::SUBC32
+                                                         : CSKY::SUBC16,
+                                     Dl, {Type0, Type1}, {Op0, Op1, CarryIn});
+  }
+  auto CarryOut = InvertCarryFlag(Subtarget, CurDAG, Dl, SDValue(NewNode, 1));
+
+  ReplaceUses(SDValue(N, 0), SDValue(NewNode, 0));
+  ReplaceUses(SDValue(N, 1), CarryOut);
+  CurDAG->RemoveDeadNode(N);
+
+  return true;
+}
+
 FunctionPass *llvm::createCSKYISelDag(CSKYTargetMachine &TM) {
   return new CSKYDAGToDAGISel(TM);
 }
diff --git a/llvm/lib/Target/CSKY/CSKYISelLowering.cpp b/llvm/lib/Target/CSKY/CSKYISelLowering.cpp
index ac6d069e592c..a1f7cc685d4c 100644
--- a/llvm/lib/Target/CSKY/CSKYISelLowering.cpp
+++ b/llvm/lib/Target/CSKY/CSKYISelLowering.cpp
@@ -37,6 +37,46 @@ CSKYTargetLowering::CSKYTargetLowering(const TargetMachine &TM,
   // Register Class
   addRegisterClass(MVT::i32, &CSKY::GPRRegClass);
 
+  setOperationAction(ISD::ADDCARRY, MVT::i32, Legal);
+  setOperationAction(ISD::SUBCARRY, MVT::i32, Legal);
+  setOperationAction(ISD::BITREVERSE, MVT::i32, Legal);
+
+  setOperationAction(ISD::SREM, MVT::i32, Expand);
+  setOperationAction(ISD::UREM, MVT::i32, Expand);
+  setOperationAction(ISD::UDIVREM, MVT::i32, Expand);
+  setOperationAction(ISD::SDIVREM, MVT::i32, Expand);
+  setOperationAction(ISD::CTTZ, MVT::i32, Expand);
+  setOperationAction(ISD::CTPOP, MVT::i32, Expand);
+  setOperationAction(ISD::ROTR, MVT::i32, Expand);
+  setOperationAction(ISD::SHL_PARTS, MVT::i32, Expand);
+  setOperationAction(ISD::SRL_PARTS, MVT::i32, Expand);
+  setOperationAction(ISD::SRA_PARTS, MVT::i32, Expand);
+  setOperationAction(ISD::UMUL_LOHI, MVT::i32, Expand);
+  setOperationAction(ISD::SMUL_LOHI, MVT::i32, Expand);
+  setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Expand);
+  setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
+  setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
+  setOperationAction(ISD::MULHS, MVT::i32, Expand);
+  setOperationAction(ISD::MULHU, MVT::i32, Expand);
+
+  setLoadExtAction(ISD::EXTLOAD, MVT::i32, MVT::i1, Promote);
+  setLoadExtAction(ISD::SEXTLOAD, MVT::i32, MVT::i1, Promote);
+  setLoadExtAction(ISD::ZEXTLOAD, MVT::i32, MVT::i1, Promote);
+
+  if (!Subtarget.hasE2()) {
+    setLoadExtAction(ISD::SEXTLOAD, MVT::i32, MVT::i8, Expand);
+    setLoadExtAction(ISD::SEXTLOAD, MVT::i32, MVT::i16, Expand);
+    setOperationAction(ISD::CTLZ, MVT::i32, Expand);
+    setOperationAction(ISD::BSWAP, MVT::i32, Expand);
+  }
+
+  if (!Subtarget.has2E3()) {
+    setOperationAction(ISD::ABS, MVT::i32, Expand);
+    setOperationAction(ISD::BITREVERSE, MVT::i32, Expand);
+    setOperationAction(ISD::SDIV, MVT::i32, Expand);
+    setOperationAction(ISD::UDIV, MVT::i32, Expand);
+  }
+
   // Compute derived properties from the register classes.
   computeRegisterProperties(STI.getRegisterInfo());
 
diff --git a/llvm/lib/Target/CSKY/CSKYInstrFormats16Instr.td b/llvm/lib/Target/CSKY/CSKYInstrFormats16Instr.td
index 6d42bddcdd78..ea0761d97545 100644
--- a/llvm/lib/Target/CSKY/CSKYInstrFormats16Instr.td
+++ b/llvm/lib/Target/CSKY/CSKYInstrFormats16Instr.td
@@ -88,6 +88,19 @@ class R16_XZ_UNOP<bits<4> op, bits<2> sop, string opstr> : CSKY16Inst<
   let Inst{1, 0} = sop;
 }
 
+class R16_Z_UNOP<bits<4> op, bits<2> sop, string opstr> : CSKY16Inst<
+  AddrModeNone, (outs sGPR:$rz), (ins sGPR:$rx), !strconcat(opstr, "\t$rz"),
+  []> {
+  bits<4> rz;
+  bits<4> rx;
+  let Inst{15, 14} = 0b01;
+  let Inst{13 - 10} = op;
+  let Inst{9 - 6} = rz;
+  let Inst{5 - 2} = rx;
+  let Inst{1, 0} = sop;
+  let Constraints = "$rz = $rx";
+}
+
 class R16_XY_CMP<bits<2> sop, string opstr> : CSKY16Inst<
   AddrModeNone, (outs CARRY:$ca), (ins sGPR:$rx, sGPR:$ry), !strconcat(opstr, "\t$rx, $ry"),
   []> {
@@ -146,7 +159,7 @@ class I16_X_CMP<bits<3> sop, string opstr, Operand Immoperand> : CSKY16Inst<
 }
 
 class I16_SP_IMM7<bits<3> sop, string opstr> : CSKY16Inst<
-  AddrModeNone, (outs SPOp:$sp2), (ins SPOp:$sp1, uimm7_2:$imm7),
+  AddrModeNone, (outs GPRSP:$sp2), (ins GPRSP:$sp1, uimm7_2:$imm7),
   !strconcat(opstr, "\t$sp2, $sp1, $imm7"), []> {
   bits<7> imm7;
   let Inst{15, 14} = 0b00;
diff --git a/llvm/lib/Target/CSKY/CSKYInstrInfo.cpp b/llvm/lib/Target/CSKY/CSKYInstrInfo.cpp
index e12235cf9478..6fcb136cd99b 100644
--- a/llvm/lib/Target/CSKY/CSKYInstrInfo.cpp
+++ b/llvm/lib/Target/CSKY/CSKYInstrInfo.cpp
@@ -11,6 +11,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "CSKYInstrInfo.h"
+#include "CSKYMachineFunctionInfo.h"
+#include "CSKYTargetMachine.h"
 #include "llvm/MC/MCContext.h"
 
 #define DEBUG_TYPE "csky-instr-info"
@@ -23,3 +25,289 @@ using namespace llvm;
 CSKYInstrInfo::CSKYInstrInfo(CSKYSubtarget &STI)
     : CSKYGenInstrInfo(CSKY::ADJCALLSTACKDOWN, CSKY::ADJCALLSTACKUP), STI(STI) {
 }
+
+Register CSKYInstrInfo::movImm(MachineBasicBlock &MBB,
+                               MachineBasicBlock::iterator MBBI,
+                               const DebugLoc &DL, int64_t Val,
+                               MachineInstr::MIFlag Flag) const {
+  assert(isUInt<32>(Val) && "should be uint32");
+
+  MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
+
+  Register DstReg;
+  if (STI.hasE2()) {
+    DstReg = MRI.createVirtualRegister(&CSKY::GPRRegClass);
+
+    if (isUInt<16>(Val)) {
+      BuildMI(MBB, MBBI, DL, get(CSKY::MOVI32), DstReg)
+          .addImm(Val & 0xFFFF)
+          .setMIFlags(Flag);
+    } else if (isShiftedUInt<16, 16>(Val)) {
+      BuildMI(MBB, MBBI, DL, get(CSKY::MOVIH32), DstReg)
+          .addImm((Val >> 16) & 0xFFFF)
+          .setMIFlags(Flag);
+    } else {
+      BuildMI(MBB, MBBI, DL, get(CSKY::MOVIH32), DstReg)
+          .addImm((Val >> 16) & 0xFFFF)
+          .setMIFlags(Flag);
+      BuildMI(MBB, MBBI, DL, get(CSKY::ORI32), DstReg)
+          .addReg(DstReg)
+          .addImm(Val & 0xFFFF)
+          .setMIFlags(Flag);
+    }
+
+  } else {
+    DstReg = MRI.createVirtualRegister(&CSKY::mGPRRegClass);
+    if (isUInt<8>(Val)) {
+      BuildMI(MBB, MBBI, DL, get(CSKY::MOVI16), DstReg)
+          .addImm(Val & 0xFF)
+          .setMIFlags(Flag);
+    } else if (isUInt<16>(Val)) {
+      BuildMI(MBB, MBBI, DL, get(CSKY::MOVI16), DstReg)
+          .addImm((Val >> 8) & 0xFF)
+          .setMIFlags(Flag);
+      BuildMI(MBB, MBBI, DL, get(CSKY::LSLI16), DstReg)
+          .addReg(DstReg)
+          .addImm(8)
+          .setMIFlags(Flag);
+      if ((Val & 0xFF) != 0)
+        BuildMI(MBB, MBBI, DL, get(CSKY::ADDI16), DstReg)
+            .addReg(DstReg)
+            .addImm(Val & 0xFF)
+            .setMIFlags(Flag);
+    } else if (isUInt<24>(Val)) {
+      BuildMI(MBB, MBBI, DL, get(CSKY::MOVI16), DstReg)
+          .addImm((Val >> 16) & 0xFF)
+          .setMIFlags(Flag);
+      BuildMI(MBB, MBBI, DL, get(CSKY::LSLI16), DstReg)
+          .addReg(DstReg)
+          .addImm(8)
+          .setMIFlags(Flag);
+      if (((Val >> 8) & 0xFF) != 0)
+        BuildMI(MBB, MBBI, DL, get(CSKY::ADDI16), DstReg)
+            .addReg(DstReg)
+            .addImm((Val >> 8) & 0xFF)
+            .setMIFlags(Flag);
+      BuildMI(MBB, MBBI, DL, get(CSKY::LSLI16), DstReg)
+          .addReg(DstReg)
+          .addImm(8)
+          .setMIFlags(Flag);
+      if ((Val & 0xFF) != 0)
+        BuildMI(MBB, MBBI, DL, get(CSKY::ADDI16), DstReg)
+            .addReg(DstReg)
+            .addImm(Val & 0xFF)
+            .setMIFlags(Flag);
+    } else {
+      BuildMI(MBB, MBBI, DL, get(CSKY::MOVI16), DstReg)
+          .addImm((Val >> 24) & 0xFF)
+          .setMIFlags(Flag);
+      BuildMI(MBB, MBBI, DL, get(CSKY::LSLI16), DstReg)
+          .addReg(DstReg)
+          .addImm(8)
+          .setMIFlags(Flag);
+      if (((Val >> 16) & 0xFF) != 0)
+        BuildMI(MBB, MBBI, DL, get(CSKY::ADDI16), DstReg)
+            .addReg(DstReg)
+            .addImm((Val >> 16) & 0xFF)
+            .setMIFlags(Flag);
+      BuildMI(MBB, MBBI, DL, get(CSKY::LSLI16), DstReg)
+          .addReg(DstReg)
+          .addImm(8)
+          .setMIFlags(Flag);
+      if (((Val >> 8) & 0xFF) != 0)
+        BuildMI(MBB, MBBI, DL, get(CSKY::ADDI16), DstReg)
+            .addReg(DstReg)
+            .addImm((Val >> 8) & 0xFF)
+            .setMIFlags(Flag);
+      BuildMI(MBB, MBBI, DL, get(CSKY::LSLI16), DstReg)
+          .addReg(DstReg)
+          .addImm(8)
+          .setMIFlags(Flag);
+      if ((Val & 0xFF) != 0)
+        BuildMI(MBB, MBBI, DL, get(CSKY::ADDI16), DstReg)
+            .addReg(DstReg)
+            .addImm(Val & 0xFF)
+            .setMIFlags(Flag);
+    }
+  }
+
+  return DstReg;
+}
+
+unsigned CSKYInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
+                                            int &FrameIndex) const {
+  switch (MI.getOpcode()) {
+  default:
+    return 0;
+  case CSKY::LD16B:
+  case CSKY::LD16H:
+  case CSKY::LD16W:
+  case CSKY::LD32B:
+  case CSKY::LD32BS:
+  case CSKY::LD32H:
+  case CSKY::LD32HS:
+  case CSKY::LD32W:
+  case CSKY::RESTORE_CARRY:
+    break;
+  }
+
+  if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() &&
+      MI.getOperand(2).getImm() == 0) {
+    FrameIndex = MI.getOperand(1).getIndex();
+    return MI.getOperand(0).getReg();
+  }
+
+  return 0;
+}
+
+unsigned CSKYInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
+                                           int &FrameIndex) const {
+  switch (MI.getOpcode()) {
+  default:
+    return 0;
+  case CSKY::ST16B:
+  case CSKY::ST16H:
+  case CSKY::ST16W:
+  case CSKY::ST32B:
+  case CSKY::ST32H:
+  case CSKY::ST32W:
+  case CSKY::SPILL_CARRY:
+    break;
+  }
+
+  if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() &&
+      MI.getOperand(2).getImm() == 0) {
+    FrameIndex = MI.getOperand(1).getIndex();
+    return MI.getOperand(0).getReg();
+  }
+
+  return 0;
+}
+
+void CSKYInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
+                                        MachineBasicBlock::iterator I,
+                                        Register SrcReg, bool IsKill, int FI,
+                                        const TargetRegisterClass *RC,
+                                        const TargetRegisterInfo *TRI) const {
+  DebugLoc DL;
+  if (I != MBB.end())
+    DL = I->getDebugLoc();
+
+  MachineFunction &MF = *MBB.getParent();
+  CSKYMachineFunctionInfo *CFI = MF.getInfo<CSKYMachineFunctionInfo>();
+  MachineFrameInfo &MFI = MF.getFrameInfo();
+
+  unsigned Opcode = 0;
+
+  if (CSKY::GPRRegClass.hasSubClassEq(RC)) {
+    Opcode = CSKY::ST32W; // Optimize for 16bit
+  } else if (CSKY::CARRYRegClass.hasSubClassEq(RC)) {
+    Opcode = CSKY::SPILL_CARRY;
+    CFI->setSpillsCR();
+  } else {
+    llvm_unreachable("Unknown RegisterClass");
+  }
+
+  MachineMemOperand *MMO = MF.getMachineMemOperand(
+      MachinePointerInfo::getFixedStack(MF, FI), MachineMemOperand::MOStore,
+      MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
+
+  BuildMI(MBB, I, DL, get(Opcode))
+      .addReg(SrcReg, getKillRegState(IsKill))
+      .addFrameIndex(FI)
+      .addImm(0)
+      .addMemOperand(MMO);
+}
+
+void CSKYInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
+                                         MachineBasicBlock::iterator I,
+                                         Register DestReg, int FI,
+                                         const TargetRegisterClass *RC,
+                                         const TargetRegisterInfo *TRI) const {
+  DebugLoc DL;
+  if (I != MBB.end())
+    DL = I->getDebugLoc();
+
+  MachineFunction &MF = *MBB.getParent();
+  CSKYMachineFunctionInfo *CFI = MF.getInfo<CSKYMachineFunctionInfo>();
+  MachineFrameInfo &MFI = MF.getFrameInfo();
+
+  unsigned Opcode = 0;
+
+  if (CSKY::GPRRegClass.hasSubClassEq(RC)) {
+    Opcode = CSKY::LD32W;
+  } else if (CSKY::CARRYRegClass.hasSubClassEq(RC)) {
+    Opcode = CSKY::RESTORE_CARRY;
+    CFI->setSpillsCR();
+  } else {
+    llvm_unreachable("Unknown RegisterClass");
+  }
+
+  MachineMemOperand *MMO = MF.getMachineMemOperand(
+      MachinePointerInfo::getFixedStack(MF, FI), MachineMemOperand::MOLoad,
+      MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
+
+  BuildMI(MBB, I, DL, get(Opcode), DestReg)
+      .addFrameIndex(FI)
+      .addImm(0)
+      .addMemOperand(MMO);
+}
+
+void CSKYInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
+                                MachineBasicBlock::iterator I,
+                                const DebugLoc &DL, MCRegister DestReg,
+                                MCRegister SrcReg, bool KillSrc) const {
+
+  MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
+
+  if (CSKY::GPRRegClass.contains(SrcReg) &&
+      CSKY::CARRYRegClass.contains(DestReg)) {
+    if (STI.hasE2()) {
+      BuildMI(MBB, I, DL, get(CSKY::BTSTI32), DestReg)
+          .addReg(SrcReg, getKillRegState(KillSrc))
+          .addImm(0);
+    } else {
+      assert(SrcReg < CSKY::R8);
+      BuildMI(MBB, I, DL, get(CSKY::BTSTI16), DestReg)
+          .addReg(SrcReg, getKillRegState(KillSrc))
+          .addImm(0);
+    }
+    return;
+  }
+
+  if (CSKY::CARRYRegClass.contains(SrcReg) &&
+      CSKY::GPRRegClass.contains(DestReg)) {
+
+    if (STI.hasE2()) {
+      BuildMI(MBB, I, DL, get(CSKY::MVC32), DestReg)
+          .addReg(SrcReg, getKillRegState(KillSrc));
+    } else {
+      assert(DestReg < CSKY::R16);
+      assert(DestReg < CSKY::R8);
+      BuildMI(MBB, I, DL, get(CSKY::MOVI16), DestReg).addImm(0);
+      BuildMI(MBB, I, DL, get(CSKY::ADDC16))
+          .addReg(DestReg, RegState::Define)
+          .addReg(SrcReg, RegState::Define)
+          .addReg(DestReg, getKillRegState(true))
+          .addReg(DestReg, getKillRegState(true))
+          .addReg(SrcReg, getKillRegState(true));
+      BuildMI(MBB, I, DL, get(CSKY::BTSTI16))
+          .addReg(SrcReg, RegState::Define | getDeadRegState(KillSrc))
+          .addReg(DestReg)
+          .addImm(0);
+    }
+    return;
+  }
+
+  unsigned Opcode = 0;
+  if (CSKY::GPRRegClass.contains(DestReg, SrcReg))
+    Opcode = CSKY::MOV32;
+  else {
+    LLVM_DEBUG(dbgs() << "src = " << SrcReg << ", dst = " << DestReg);
+    LLVM_DEBUG(I->dump());
+    llvm_unreachable("Unknown RegisterClass");
+  }
+
+  BuildMI(MBB, I, DL, get(Opcode), DestReg)
+      .addReg(SrcReg, getKillRegState(KillSrc));
+}
diff --git a/llvm/lib/Target/CSKY/CSKYInstrInfo.h b/llvm/lib/Target/CSKY/CSKYInstrInfo.h
index 04be9da27b57..450641d96b74 100644
--- a/llvm/lib/Target/CSKY/CSKYInstrInfo.h
+++ b/llvm/lib/Target/CSKY/CSKYInstrInfo.h
@@ -29,6 +29,31 @@ protected:
 
 public:
   explicit CSKYInstrInfo(CSKYSubtarget &STI);
+
+  unsigned isLoadFromStackSlot(const MachineInstr &MI,
+                               int &FrameIndex) const override;
+  unsigned isStoreToStackSlot(const MachineInstr &MI,
+                              int &FrameIndex) const override;
+
+  void storeRegToStackSlot(MachineBasicBlock &MBB,
+                           MachineBasicBlock::iterator MI, Register SrcReg,
+                           bool IsKill, int FrameIndex,
+                           const TargetRegisterClass *RC,
+                           const TargetRegisterInfo *TRI) const override;
+
+  void loadRegFromStackSlot(MachineBasicBlock &MBB,
+                            MachineBasicBlock::iterator MI, Register DestReg,
+                            int FrameIndex, const TargetRegisterClass *RC,
+                            const TargetRegisterInfo *TRI) const override;
+
+  void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
+                   const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg,
+                   bool KillSrc) const override;
+
+  // Materializes the given integer Val into DstReg.
+  Register movImm(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
+                  const DebugLoc &DL, int64_t Val,
+                  MachineInstr::MIFlag Flag = MachineInstr::NoFlags) const;
 };
 
 } // namespace llvm
diff --git a/llvm/lib/Target/CSKY/CSKYInstrInfo.td b/llvm/lib/Target/CSKY/CSKYInstrInfo.td
index 9dda3159e446..30d9206eec68 100644
--- a/llvm/lib/Target/CSKY/CSKYInstrInfo.td
+++ b/llvm/lib/Target/CSKY/CSKYInstrInfo.td
@@ -52,6 +52,11 @@ class OImmAsmOperand<int width, string suffix = "">
     : ImmAsmOperand<"O", width, suffix> {
 }
 
+def to_tframeindex : SDNodeXForm<frameindex, [{
+  auto FI = cast<FrameIndexSDNode>(N);
+  return CurDAG->getTargetFrameIndex(FI->getIndex(), TLI->getPointerTy(CurDAG->getDataLayout()));
+}]>;
+
 class oimm<int num> : Operand<i32>,
   ImmLeaf<i32, "return isUInt<"#num#">(Imm - 1);"> {
   let EncoderMethod = "getOImmOpValue";
@@ -166,9 +171,23 @@ def bare_symbol : Operand<iPTR> {
   let OperandType = "OPERAND_PCREL";
 }
 
-def oimm3 : oimm<3>;
+def oimm3 : oimm<3> {
+  let MCOperandPredicate = [{
+    int64_t Imm;
+    if (MCOp.evaluateAsConstantImm(Imm))
+      return isUInt<3>(Imm - 1);
+    return MCOp.isBareSymbolRef();
+  }];
+}
 def oimm4 : oimm<4>;
-def oimm5 : oimm<5>;
+def oimm5 : oimm<5> {
+  let MCOperandPredicate = [{
+    int64_t Imm;
+    if (MCOp.evaluateAsConstantImm(Imm))
+      return isUInt<5>(Imm - 1);
+    return MCOp.isBareSymbolRef();
+  }];
+}
 def oimm6 : oimm<6>;
 
 def imm5_idly : Operand<i32>, ImmLeaf<i32,
@@ -177,9 +196,30 @@ def imm5_idly : Operand<i32>, ImmLeaf<i32,
   let DecoderMethod = "decodeOImmOperand<5>";
 }
 
-def oimm8 : oimm<8>;
-def oimm12 : oimm<12>;
-def oimm16 : oimm<16>;
+def oimm8 : oimm<8> {
+  let MCOperandPredicate = [{
+    int64_t Imm;
+    if (MCOp.evaluateAsConstantImm(Imm))
+      return isUInt<8>(Imm - 1);
+    return MCOp.isBareSymbolRef();
+  }];
+}
+def oimm12 : oimm<12> {
+  let MCOperandPredicate = [{
+    int64_t Imm;
+    if (MCOp.evaluateAsConstantImm(Imm))
+      return isUInt<12>(Imm - 1);
+    return MCOp.isBareSymbolRef();
+  }];
+}
+def oimm16 : oimm<16> {
+  let MCOperandPredicate = [{
+    int64_t Imm;
+    if (MCOp.evaluateAsConstantImm(Imm))
+      return isUInt<16>(Imm - 1);
+    return MCOp.isBareSymbolRef();
+  }];
+}
 
 def nimm12 : nimm<12>;
 
@@ -195,28 +235,98 @@ def uimm2_jmpix : Operand<i32>,
 
 def uimm3 : uimm<3>;
 def uimm4 : uimm<4>;
-def uimm5 : uimm<5>;
+def uimm5 : uimm<5> {
+  let MCOperandPredicate = [{
+    int64_t Imm;
+    if (MCOp.evaluateAsConstantImm(Imm))
+      return isShiftedUInt<5, 0>(Imm);
+    return MCOp.isBareSymbolRef();
+  }];
+}
 def uimm5_msb_size : uimm<5> {
   let EncoderMethod = "getImmOpValueMSBSize";
 }
 
-def uimm5_1 : uimm<5, 1>;
-def uimm5_2 : uimm<5, 2>;
+def uimm5_1 : uimm<5, 1> {
+  let MCOperandPredicate = [{
+    int64_t Imm;
+    if (MCOp.evaluateAsConstantImm(Imm))
+      return isShiftedUInt<5, 1>(Imm);
+    return MCOp.isBareSymbolRef();
+  }];
+}
+def uimm5_2 : uimm<5, 2> {
+  let MCOperandPredicate = [{
+    int64_t Imm;
+    if (MCOp.evaluateAsConstantImm(Imm))
+      return isShiftedUInt<5, 2>(Imm);
+    return MCOp.isBareSymbolRef();
+  }];
+}
 def uimm6 : uimm<6>;
 def uimm7 : uimm<7>;
 def uimm7_1 : uimm<7, 1>;
-def uimm7_2 : uimm<7, 2>;
+def uimm7_2 : uimm<7, 2>{
+  let MCOperandPredicate = [{
+    int64_t Imm;
+    if (MCOp.evaluateAsConstantImm(Imm))
+      return isShiftedUInt<7, 2>(Imm);
+    return MCOp.isBareSymbolRef();
+  }];
+}
 def uimm7_3 : uimm<7, 3>;
-def uimm8 : uimm<8>;
-def uimm8_2 : uimm<8, 2>;
+def uimm8 : uimm<8> {
+  let MCOperandPredicate = [{
+    int64_t Imm;
+    if (MCOp.evaluateAsConstantImm(Imm))
+      return isShiftedUInt<8, 0>(Imm);
+    return MCOp.isBareSymbolRef();
+  }];
+}
+def uimm8_2 : uimm<8, 2> {
+  let MCOperandPredicate = [{
+    int64_t Imm;
+    if (MCOp.evaluateAsConstantImm(Imm))
+      return isShiftedUInt<8, 2>(Imm);
+    return MCOp.isBareSymbolRef();
+  }];
+}
 def uimm8_3 : uimm<8, 3>;
 def uimm8_8 : uimm<8, 8>;
 def uimm8_16 : uimm<8, 16>;
 def uimm8_24 : uimm<8, 24>;
-def uimm12 : uimm<12>;
-def uimm12_1 : uimm<12, 1>;
-def uimm12_2 : uimm<12, 2>;
-def uimm16 : uimm<16>;
+def uimm12 : uimm<12>  {
+  let MCOperandPredicate = [{
+    int64_t Imm;
+    if (MCOp.evaluateAsConstantImm(Imm))
+      return isShiftedUInt<12, 0>(Imm);
+    return MCOp.isBareSymbolRef();
+  }];
+}
+def uimm12_1 : uimm<12, 1> {
+  let MCOperandPredicate = [{
+    int64_t Imm;
+    if (MCOp.evaluateAsConstantImm(Imm))
+      return isShiftedUInt<12, 1>(Imm);
+    return MCOp.isBareSymbolRef();
+  }];
+}
+def uimm12_2 : uimm<12, 2> {
+  let MCOperandPredicate = [{
+    int64_t Imm;
+    if (MCOp.evaluateAsConstantImm(Imm))
+      return isShiftedUInt<12, 2>(Imm);
+    return MCOp.isBareSymbolRef();
+  }];
+}
+def uimm16 : uimm<16> {
+  let MCOperandPredicate = [{
+    int64_t Imm;
+    if (MCOp.evaluateAsConstantImm(Imm))
+      return isShiftedUInt<16, 0>(Imm);
+    return MCOp.isBareSymbolRef();
+  }];
+}
 def uimm16_8 : uimm<16, 8>;
 def uimm16_16 : uimm<16, 16>;
 def uimm20 : uimm<20>;
@@ -642,11 +752,6 @@ def BSR32_BR : J<0x38, (outs), (ins call_symbol:$offset), "bsr32", []>{
   let Defs = [ R15 ];
 }
 
-let Predicates = [iHasE2], isCodeGenOnly = 1 in {
-  def RTS32 : I_16_RET<0x6, 0xF, "rts32", [(CSKY_RET)]>;
-}
-
-
 //===----------------------------------------------------------------------===//
 // Symbol address instructions.
 //===----------------------------------------------------------------------===//
@@ -872,6 +977,102 @@ def TRAP32 : CSKY32Inst<AddrModeNone, 0x30, (outs), (ins uimm2:$imm2), "trap32 $
 
 }
 
+//===----------------------------------------------------------------------===//
+// Instruction Patterns.
+//===----------------------------------------------------------------------===//
+
+// Load & Store Patterns
+multiclass LdPat<PatFrag LoadOp, ImmLeaf imm_type, Instruction Inst, ValueType Type> {
+  def : Pat<(Type (LoadOp GPR:$rs1)), (Inst GPR:$rs1, 0)>;
+  def : Pat<(Type (LoadOp (i32 frameindex:$rs1))), (Inst (i32 (to_tframeindex tframeindex:$rs1)), 0)>;
+  def : Pat<(Type (LoadOp (add GPR:$rs1, imm_type:$uimm))),
+            (Inst GPR:$rs1, imm_type:$uimm)>;
+  def : Pat<(Type (LoadOp (add frameindex:$rs1, imm_type:$uimm))),
+            (Inst (i32 (to_tframeindex tframeindex:$rs1)), imm_type:$uimm)>;
+  def : Pat<(Type (LoadOp (eqToAdd frameindex:$rs1, imm_type:$uimm))),
+            (Inst (i32 (to_tframeindex tframeindex:$rs1)), imm_type:$uimm)>;
+  def : Pat<(Type (LoadOp (add GPR:$rs1, tglobaladdr:$gd))),
+            (Inst GPR:$rs1, tglobaladdr:$gd)>;
+}
+
+defm : LdPat<extloadi8, uimm12, LD32B, i32>;
+defm : LdPat<zextloadi8, uimm12, LD32B, i32>;
+let Predicates = [iHasE2] in {
+  defm : LdPat<sextloadi8, uimm12, LD32BS, i32>;
+}
+defm : LdPat<extloadi16, uimm12_1, LD32H, i32>;
+defm : LdPat<zextloadi16, uimm12_1, LD32H, i32>;
+let Predicates = [iHasE2] in {
+defm : LdPat<sextloadi16, uimm12_1, LD32HS, i32>;
+}
+defm : LdPat<load, uimm12_2, LD32W, i32>;
+
+multiclass LdrPat<PatFrag LoadOp, Instruction Inst, ValueType Type> {
+  def : Pat<(Type (LoadOp (add GPR:$rs1, GPR:$rs2))), (Inst GPR:$rs1, GPR:$rs2, 0)>;
+  def : Pat<(Type (LoadOp (add GPR:$rs1, (shl GPR:$rs2, (i32 1))))), (Inst GPR:$rs1, GPR:$rs2, 1)>;
+  def : Pat<(Type (LoadOp (add GPR:$rs1, (shl GPR:$rs2, (i32 2))))), (Inst GPR:$rs1, GPR:$rs2, 2)>;
+  def : Pat<(Type (LoadOp (add GPR:$rs1, (shl GPR:$rs2, (i32 3))))), (Inst GPR:$rs1, GPR:$rs2, 3)>;
+}
+
+let Predicates = [iHas2E3] in {
+  defm : LdrPat<zextloadi8, LDR32B, i32>;
+  defm : LdrPat<sextloadi8, LDR32BS, i32>;
+  defm : LdrPat<extloadi8, LDR32BS, i32>;
+  defm : LdrPat<zextloadi16, LDR32H, i32>;
+  defm : LdrPat<sextloadi16, LDR32HS, i32>;
+  defm : LdrPat<extloadi16, LDR32HS, i32>;
+  defm : LdrPat<load, LDR32W, i32>;
+}
+
+multiclass StPat<PatFrag StoreOp, ValueType Type, ImmLeaf imm_type, Instruction Inst> {
+  def : Pat<(StoreOp Type:$rs2, GPR:$rs1), (Inst Type:$rs2, GPR:$rs1, 0)>;
+  def : Pat<(StoreOp Type:$rs2, frameindex:$rs1), (Inst Type:$rs2, (i32 (to_tframeindex tframeindex:$rs1)), 0)>;
+  def : Pat<(StoreOp Type:$rs2, (add GPR:$rs1, imm_type:$uimm12)),
+            (Inst Type:$rs2, GPR:$rs1, imm_type:$uimm12)>;
+  def : Pat<(StoreOp Type:$rs2, (add frameindex:$rs1, imm_type:$uimm12)),
+            (Inst Type:$rs2, (i32 (to_tframeindex tframeindex:$rs1)), imm_type:$uimm12)>;
+  def : Pat<(StoreOp Type:$rs2, (eqToAdd frameindex:$rs1, imm_type:$uimm12)),
+            (Inst Type:$rs2, (i32 (to_tframeindex tframeindex:$rs1)), imm_type:$uimm12)>;
+}
+
+defm : StPat<truncstorei8, i32, uimm12, ST32B>;
+defm : StPat<truncstorei16, i32, uimm12_1, ST32H>;
+defm : StPat<store, i32, uimm12_2, ST32W>;
+
+multiclass StrPat<PatFrag StoreOp, ValueType Type, Instruction Inst> {
+  def : Pat<(StoreOp Type:$rz, (add GPR:$rs1, GPR:$rs2)), (Inst Type:$rz, GPR:$rs1, GPR:$rs2, 0)>;
+  def : Pat<(StoreOp Type:$rz, (add GPR:$rs1, (shl GPR:$rs2, (i32 1)))), (Inst Type:$rz, GPR:$rs1, GPR:$rs2, 1)>;
+  def : Pat<(StoreOp Type:$rz, (add GPR:$rs1, (shl GPR:$rs2, (i32 2)))), (Inst Type:$rz, GPR:$rs1, GPR:$rs2, 2)>;
+  def : Pat<(StoreOp Type:$rz, (add GPR:$rs1, (shl GPR:$rs2, (i32 3)))), (Inst Type:$rz, GPR:$rs1, GPR:$rs2, 3)>;
+}
+
+let Predicates = [iHas2E3] in {
+  defm : StrPat<truncstorei8, i32, STR32B>;
+  defm : StrPat<truncstorei16, i32, STR32H>;
+  defm : StrPat<store, i32, STR32W>;
+
+  // Sext & Zext Patterns
+  def : Pat<(sext_inreg GPR:$src, i1), (SEXT32 GPR:$src, 0, 0)>;
+  def : Pat<(and GPR:$src, 255), (ZEXT32 GPR:$src, 7, 0)>;
+  def : Pat<(and GPR:$src, 65535), (ZEXT32 GPR:$src, 15, 0)>;
+}
+
+// Constant materialize patterns.
+let Predicates = [iHasE2] in
+  def : Pat<(i32 imm:$imm),
+            (ORI32 (MOVIH32 (uimm32_hi16 imm:$imm)), (uimm32_lo16 imm:$imm))>;
+
+
+// Other operations.
+let Predicates = [iHasE2] in {
+  def : Pat<(rotl GPR:$rs1, GPR:$rs2),
+            (ROTL32 GPR:$rs1, (ANDI32 GPR:$rs2, 0x1f))>;
+  let Predicates = [iHas2E3] in {
+    def : Pat<(bitreverse GPR:$rx), (BREV32 GPR:$rx)>;
+    def : Pat<(bswap GPR:$rx), (REVB32 GPR:$rx)>;
+  }
+  def : Pat<(i32 (ctlz GPR:$rx)), (FF1 GPR:$rx)>;
+}
 
 //===----------------------------------------------------------------------===//
 // Pseudo for assembly
diff --git a/llvm/lib/Target/CSKY/CSKYInstrInfo16Instr.td b/llvm/lib/Target/CSKY/CSKYInstrInfo16Instr.td
index c98f43622155..6a9dd03dfa1d 100644
--- a/llvm/lib/Target/CSKY/CSKYInstrInfo16Instr.td
+++ b/llvm/lib/Target/CSKY/CSKYInstrInfo16Instr.td
@@ -33,16 +33,6 @@ def br_symbol_16bit : Operand<iPTR> {
   let OperandType = "OPERAND_PCREL";
 }
 
-def SPOperand : AsmOperandClass {
-  let Name = "SPOperand";
-  let RenderMethod = "addRegOperands";
-  let DiagnosticType = !strconcat("Invalid", Name);
-}
-
-def SPOp : RegisterOperand<GPR> {
- let ParserMatchClass = SPOperand;
-}
-
 def constpool_symbol_16bit : Operand<iPTR> {
   let ParserMatchClass = Constpool;
   let EncoderMethod =
@@ -83,7 +73,7 @@ let isCommutable = 1 in {
   def XOR16 : R16_XZ_BINOP<0b1011, 0b01, "xor16", BinOpFrag<(xor node:$LHS, node:$RHS)>>;
   def NOR16 : R16_XZ_BINOP<0b1011, 0b10, "nor16", BinOpFrag<(not (or node:$LHS, node:$RHS))>>;
   let isCodeGenOnly = 1 in
-  def NOT16 : R16_XZ_UNOP<0b1011, 0b10, "not16">;
+  def NOT16 : R16_Z_UNOP<0b1011, 0b10, "not16">;
   def MULT16 :  R16_XZ_BINOP<0b1111, 0b00, "mult16", BinOpFrag<(mul node:$LHS, node:$RHS)>>;
 }
 def SUBU16XZ : R16_XZ_BINOP<0b1000, 0b10, "subu16", BinOpFrag<(sub node:$LHS, node:$RHS)>>;
@@ -108,7 +98,7 @@ let Constraints = "$rZ = $rz", isReMaterializable = 1, isAsCheapAsAMove = 1 in {
 }
 
 let isAdd = 1 in
-def ADDI16ZSP : I16_Z_8<0b011, (ins SPOp:$sp, uimm8_2:$imm8),
+def ADDI16ZSP : I16_Z_8<0b011, (ins GPRSP:$sp, uimm8_2:$imm8),
                         "addi16\t$rz, $sp, $imm8">;
 
 let isAdd = 1 in
@@ -150,9 +140,9 @@ def ST16W : I16_XZ_LDST<AddrMode16W, 0b110, "st16.w",
   (outs), (ins mGPR:$rz, mGPR:$rx, uimm5_2:$imm)>;
 
 def LD16WSP : I16_ZSP_LDST<AddrMode16W, 0b011, "ld16.w",
-  (outs mGPR:$rz), (ins SPOp:$sp, uimm8_2:$addr)>;
+  (outs mGPR:$rz), (ins GPRSP:$sp, uimm8_2:$addr)>;
 def ST16WSP : I16_ZSP_LDST<AddrMode16W, 0b111, "st16.w",
-  (outs), (ins mGPR:$rz, SPOp:$sp, uimm8_2:$addr)>;
+  (outs), (ins mGPR:$rz, GPRSP:$sp, uimm8_2:$addr)>;
 
 //===----------------------------------------------------------------------===//
 // Compare instructions.
@@ -450,3 +440,150 @@ def JBF16 : JBranchPseudo<(outs),
 let mayLoad = 1, Size = 2, isCodeGenOnly = 0 in
 def PseudoLRW16 : CSKYPseudo<(outs mGPR:$rz),
   (ins bare_symbol:$src), "lrw16 $rz, $src", []>;
+
+
+//===----------------------------------------------------------------------===//
+// Compress Instruction tablegen backend.
+//===----------------------------------------------------------------------===//
+
+def : CompressPat<(ADDU32 sGPR:$rd, sGPR:$rd, sGPR:$rs2),
+                  (ADDU16XZ sGPR:$rd, sGPR:$rs2)>;
+def : CompressPat<(ADDU32 sGPR:$rd, sGPR:$rs1, sGPR:$rd),
+                  (ADDU16XZ sGPR:$rd, sGPR:$rs1)>;
+def : CompressPat<(ADDU32 mGPR:$rd, mGPR:$rs1, mGPR:$rs2),
+                  (ADDU16 mGPR:$rd, mGPR:$rs1, mGPR:$rs2)>;
+def : CompressPat<(SUBU32 sGPR:$rd, sGPR:$rd, sGPR:$rs2),
+                  (SUBU16XZ sGPR:$rd, sGPR:$rs2)>;
+def : CompressPat<(SUBU32 mGPR:$rd, mGPR:$rs1, mGPR:$rs2),
+                  (SUBU16 mGPR:$rd, mGPR:$rs1, mGPR:$rs2)>;
+
+def : CompressPat<
+  (ADDC32 sGPR:$rd, CARRY:$cout, sGPR:$rd, sGPR:$rs2, CARRY:$cout),
+  (ADDC16 sGPR:$rd, CARRY:$cout, sGPR:$rs2, CARRY:$cout)
+  >;
+def : CompressPat<
+  (SUBC32 sGPR:$rd, CARRY:$cout, sGPR:$rd, sGPR:$rs2, CARRY:$cout),
+  (SUBC16 sGPR:$rd, CARRY:$cout, sGPR:$rs2, CARRY:$cout)
+  >;
+
+def : CompressPat<(ADDI32 mGPR:$rd, mGPR:$rs, oimm3:$imm),
+                  (ADDI16XZ mGPR:$rd, mGPR:$rs, oimm3:$imm)>;
+def : CompressPat<(SUBI32 mGPR:$rd, mGPR:$rs, oimm3:$imm),
+                  (SUBI16XZ mGPR:$rd, mGPR:$rs, oimm3:$imm)>;
+
+def : CompressPat<(ADDI32 mGPR:$rd, mGPR:$rd, oimm8:$imm),
+                  (ADDI16 mGPR:$rd, oimm8:$imm)>;
+def : CompressPat<(SUBI32 mGPR:$rd, mGPR:$rd, oimm8:$imm),
+                  (SUBI16 mGPR:$rd, oimm8:$imm)>;
+
+def : CompressPat<(ADDI32 GPRSP:$sp, GPRSP:$sp, uimm7_2:$imm),
+                  (ADDI16SPSP GPRSP:$sp, GPRSP:$sp, uimm7_2:$imm)>;
+def : CompressPat<(SUBI32 GPRSP:$sp, GPRSP:$sp, uimm7_2:$imm),
+                  (SUBI16SPSP GPRSP:$sp, GPRSP:$sp, uimm7_2:$imm)>;
+
+def : CompressPat<(ADDI32 mGPR:$rd, GPRSP:$sp, uimm8_2:$imm),
+                  (ADDI16ZSP mGPR:$rd, GPRSP:$sp, uimm8_2:$imm)>;
+
+def : CompressPat<(MULT32 sGPR:$rd, sGPR:$rd, sGPR:$rs2),
+                  (MULT16 sGPR:$rd, sGPR:$rs2)>;
+def : CompressPat<(MULT32 sGPR:$rd, sGPR:$rs1, sGPR:$rd),
+                  (MULT16 sGPR:$rd, sGPR:$rs1)>;
+def : CompressPat<(AND32 sGPR:$rd, sGPR:$rd, sGPR:$rs2),
+                  (AND16 sGPR:$rd, sGPR:$rs2)>;
+def : CompressPat<(AND32 sGPR:$rd, sGPR:$rs1, sGPR:$rd),
+                  (AND16 sGPR:$rd, sGPR:$rs1)>;
+def : CompressPat<(OR32 sGPR:$rd, sGPR:$rd, sGPR:$rs2),
+                  (OR16 sGPR:$rd, sGPR:$rs2)>;
+def : CompressPat<(OR32 sGPR:$rd, sGPR:$rs1, sGPR:$rd),
+                  (OR16 sGPR:$rd, sGPR:$rs1)>;
+def : CompressPat<(XOR32 sGPR:$rd, sGPR:$rd, sGPR:$rs2),
+                  (XOR16 sGPR:$rd, sGPR:$rs2)>;
+def : CompressPat<(XOR32 sGPR:$rd, sGPR:$rs1, sGPR:$rd),
+                  (XOR16 sGPR:$rd, sGPR:$rs1)>;
+
+def : CompressPat<(ANDN32 sGPR:$rd, sGPR:$rd, sGPR:$rs2),
+                  (ANDN16 sGPR:$rd, sGPR:$rs2)>;
+def : CompressPat<(NOR32 sGPR:$rd, sGPR:$rd, sGPR:$rs2),
+                  (NOR16 sGPR:$rd, sGPR:$rs2)>;
+def : CompressPat<(LSL32 sGPR:$rd, sGPR:$rd, sGPR:$rs2),
+                  (LSL16 sGPR:$rd, sGPR:$rs2)>;
+def : CompressPat<(LSR32 sGPR:$rd, sGPR:$rd, sGPR:$rs2),
+                  (LSR16 sGPR:$rd, sGPR:$rs2)>;
+def : CompressPat<(ASR32 sGPR:$rd, sGPR:$rd, sGPR:$rs2),
+                  (ASR16 sGPR:$rd, sGPR:$rs2)>;
+def : CompressPat<(ROTL32 sGPR:$rd, sGPR:$rd, sGPR:$rs2),
+                  (ROTL16 sGPR:$rd, sGPR:$rs2)>;
+
+def : CompressPat<(NOT32 sGPR:$rd, sGPR:$rd),
+                  (NOT16 sGPR:$rd)>;
+
+let Predicates = [iHas2E3] in
+def : CompressPat<(REVB32 sGPR:$rd, sGPR:$rs),
+                  (REVB16 sGPR:$rd, sGPR:$rs)>;
+
+def : CompressPat<(LSLI32 mGPR:$rd, mGPR:$rs, uimm5:$imm),
+                  (LSLI16 mGPR:$rd, mGPR:$rs, uimm5:$imm)>;
+def : CompressPat<(LSRI32 mGPR:$rd, mGPR:$rs, uimm5:$imm),
+                  (LSRI16 mGPR:$rd, mGPR:$rs, uimm5:$imm)>;
+def : CompressPat<(ASRI32 mGPR:$rd, mGPR:$rs, uimm5:$imm),
+                  (ASRI16 mGPR:$rd, mGPR:$rs, uimm5:$imm)>;
+
+def : CompressPat<(CMPHS32 CARRY:$ca, sGPR:$rs1, sGPR:$rs2),
+                  (CMPHS16 CARRY:$ca, sGPR:$rs1, sGPR:$rs2)>;
+def : CompressPat<(CMPLT32 CARRY:$ca, sGPR:$rs1, sGPR:$rs2),
+                  (CMPLT16 CARRY:$ca, sGPR:$rs1, sGPR:$rs2)>;
+def : CompressPat<(CMPNE32 CARRY:$ca, sGPR:$rs1, sGPR:$rs2),
+                  (CMPNE16 CARRY:$ca, sGPR:$rs1, sGPR:$rs2)>;
+
+def : CompressPat<(CMPHSI32 CARRY:$ca, mGPR:$rs, oimm5:$imm),
+                  (CMPHSI16 CARRY:$ca, mGPR:$rs, oimm5:$imm)>;
+def : CompressPat<(CMPLTI32 CARRY:$ca, mGPR:$rs, oimm5:$imm),
+                  (CMPLTI16 CARRY:$ca, mGPR:$rs, oimm5:$imm)>;
+def : CompressPat<(CMPNEI32 CARRY:$ca, mGPR:$rs, uimm5:$imm),
+                  (CMPNEI16 CARRY:$ca, mGPR:$rs, uimm5:$imm)>;
+
+def : CompressPat<(JSR32 sGPR:$rd),
+                  (JSR16 sGPR:$rd)>;
+
+
+def : CompressPat<(MVCV32 sGPR:$rd, CARRY:$ca),
+                  (MVCV16 sGPR:$rd, CARRY:$ca)>;
+def : CompressPat<(MOV32 sGPR:$rd, sGPR:$ca),
+                  (MOV16 sGPR:$rd, sGPR:$ca)>;
+def : CompressPat<(MOVI32 mGPR:$rd, uimm8:$imm),
+                  (MOVI16 mGPR:$rd, uimm8:$imm)>;
+
+def : CompressPat<(LD32B mGPR:$rd, mGPR:$rs, uimm5:$imm),
+                  (LD16B mGPR:$rd, mGPR:$rs, uimm5:$imm)>;
+def : CompressPat<(LD32H mGPR:$rd, mGPR:$rs, uimm5_1:$imm),
+                  (LD16H mGPR:$rd, mGPR:$rs, uimm5_1:$imm)>;
+def : CompressPat<(LD32W mGPR:$rd, mGPR:$rs, uimm5_2:$imm),
+                  (LD16W mGPR:$rd, mGPR:$rs, uimm5_2:$imm)>;
+def : CompressPat<(LD32W mGPR:$rd, GPRSP:$sp, uimm8_2:$imm),
+                  (LD16WSP mGPR:$rd, GPRSP:$sp, uimm8_2:$imm)>;
+
+def : CompressPat<(ST32B mGPR:$rd, mGPR:$rs, uimm5:$imm),
+                  (ST16B mGPR:$rd, mGPR:$rs, uimm5:$imm)>;
+def : CompressPat<(ST32H mGPR:$rd, mGPR:$rs, uimm5_1:$imm),
+                  (ST16H mGPR:$rd, mGPR:$rs, uimm5_1:$imm)>;
+def : CompressPat<(ST32W mGPR:$rd, mGPR:$rs, uimm5_2:$imm),
+                  (ST16W mGPR:$rd, mGPR:$rs, uimm5_2:$imm)>;
+def : CompressPat<(ST32W mGPR:$rd, GPRSP:$sp, uimm8_2:$imm),
+                  (ST16WSP mGPR:$rd, GPRSP:$sp, uimm8_2:$imm)>;
+
+let Predicates = [HasBTST16] in
+def : CompressPat<(BTSTI32 CARRY:$ca, mGPR:$rs, uimm5:$imm),
+                  (BTSTI16 CARRY:$ca, mGPR:$rs, uimm5:$imm)>;
+def : CompressPat<(BCLRI32 mGPR:$rd, mGPR:$rd, uimm5:$imm),
+                  (BCLRI16 mGPR:$rd, uimm5:$imm)>;
+def : CompressPat<(BSETI32 mGPR:$rd, mGPR:$rd, uimm5:$imm),
+                  (BSETI16 mGPR:$rd, uimm5:$imm)>;
+
+def : CompressPat<(ZEXTB32 sGPR:$rd, sGPR:$rs),
+                  (ZEXTB16 sGPR:$rd, sGPR:$rs)>;
+def : CompressPat<(ZEXTH32 sGPR:$rd, sGPR:$rs),
+                  (ZEXTH16 sGPR:$rd, sGPR:$rs)>;
+def : CompressPat<(SEXTB32 sGPR:$rd, sGPR:$rs),
+                  (SEXTB16 sGPR:$rd, sGPR:$rs)>;
+def : CompressPat<(SEXTH32 sGPR:$rd, sGPR:$rs),
+                  (SEXTH16 sGPR:$rd, sGPR:$rs)>;
diff --git a/llvm/lib/Target/CSKY/CSKYMCInstLower.cpp b/llvm/lib/Target/CSKY/CSKYMCInstLower.cpp
index c42a56bfb04e..7e0b9bcd7549 100644
--- a/llvm/lib/Target/CSKY/CSKYMCInstLower.cpp
+++ b/llvm/lib/Target/CSKY/CSKYMCInstLower.cpp
@@ -114,4 +114,4 @@ bool CSKYMCInstLower::lowerOperand(const MachineOperand &MO,
     break;
   }
   return true;
-}
\ No newline at end of file
+}
diff --git a/llvm/lib/Target/CSKY/CSKYRegisterInfo.cpp b/llvm/lib/Target/CSKY/CSKYRegisterInfo.cpp
index a1d45fea534b..57b6ae3c27b5 100644
--- a/llvm/lib/Target/CSKY/CSKYRegisterInfo.cpp
+++ b/llvm/lib/Target/CSKY/CSKYRegisterInfo.cpp
@@ -88,8 +88,187 @@ CSKYRegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
   return CSR_I32_SaveList;
 }
 
+static bool IsLegalOffset(const CSKYInstrInfo *TII, MachineInstr *MI,
+                          int &Offset) {
+  const MCInstrDesc &Desc = MI->getDesc();
+  unsigned AddrMode = (Desc.TSFlags & CSKYII::AddrModeMask);
+  unsigned i = 0;
+  for (; !MI->getOperand(i).isFI(); ++i) {
+    assert(i + 1 < MI->getNumOperands() &&
+           "Instr doesn't have FrameIndex operand!");
+  }
+
+  if (MI->getOpcode() == CSKY::ADDI32) {
+    if (!isUInt<12>(std::abs(Offset) - 1))
+      return false;
+    if (Offset < 0) {
+      MI->setDesc(TII->get(CSKY::SUBI32));
+      Offset = -Offset;
+    }
+
+    return true;
+  }
+
+  if (MI->getOpcode() == CSKY::ADDI16XZ)
+    return false;
+
+  if (Offset < 0)
+    return false;
+
+  unsigned NumBits = 0;
+  unsigned Scale = 1;
+  switch (AddrMode) {
+  case CSKYII::AddrMode32B:
+    Scale = 1;
+    NumBits = 12;
+    break;
+  case CSKYII::AddrMode32H:
+    Scale = 2;
+    NumBits = 12;
+    break;
+  case CSKYII::AddrMode32WD:
+    Scale = 4;
+    NumBits = 12;
+    break;
+  case CSKYII::AddrMode16B:
+    Scale = 1;
+    NumBits = 5;
+    break;
+  case CSKYII::AddrMode16H:
+    Scale = 2;
+    NumBits = 5;
+    break;
+  case CSKYII::AddrMode16W:
+    Scale = 4;
+    NumBits = 5;
+    break;
+  case CSKYII::AddrMode32SDF:
+    Scale = 4;
+    NumBits = 8;
+    break;
+  default:
+    llvm_unreachable("Unsupported addressing mode!");
+  }
+
+  // Cannot encode offset.
+  if ((Offset & (Scale - 1)) != 0)
+    return false;
+
+  unsigned Mask = (1 << NumBits) - 1;
+  if ((unsigned)Offset <= Mask * Scale)
+    return true;
+
+  // Offset out of range.
+  return false;
+}
+
 void CSKYRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
                                            int SPAdj, unsigned FIOperandNum,
                                            RegScavenger *RS) const {
   assert(SPAdj == 0 && "Unexpected non-zero SPAdj value");
-}
\ No newline at end of file
+
+  MachineInstr *MI = &*II;
+  MachineBasicBlock &MBB = *MI->getParent();
+  MachineFunction &MF = *MI->getParent()->getParent();
+  MachineRegisterInfo &MRI = MF.getRegInfo();
+  const CSKYInstrInfo *TII = MF.getSubtarget<CSKYSubtarget>().getInstrInfo();
+  DebugLoc DL = MI->getDebugLoc();
+  const CSKYSubtarget &STI = MF.getSubtarget<CSKYSubtarget>();
+
+  switch (MI->getOpcode()) {
+  default:
+    break;
+  case CSKY::RESTORE_CARRY: {
+    Register NewReg = STI.hasE2()
+                          ? MRI.createVirtualRegister(&CSKY::GPRRegClass)
+                          : MRI.createVirtualRegister(&CSKY::mGPRRegClass);
+
+    auto *Temp = BuildMI(MBB, II, DL, TII->get(CSKY::LD32W), NewReg)
+                     .add(MI->getOperand(1))
+                     .add(MI->getOperand(2))
+                     .getInstr();
+
+    BuildMI(MBB, II, DL, TII->get(STI.hasE2() ? CSKY::BTSTI32 : CSKY::BTSTI16),
+            MI->getOperand(0).getReg())
+        .addReg(NewReg, getKillRegState(true))
+        .addImm(0);
+
+    MI = Temp;
+
+    MBB.erase(II);
+    break;
+  }
+  case CSKY::SPILL_CARRY: {
+    Register NewReg;
+    if (STI.hasE2()) {
+      NewReg = MRI.createVirtualRegister(&CSKY::GPRRegClass);
+      BuildMI(MBB, II, DL, TII->get(CSKY::MVC32), NewReg)
+          .add(MI->getOperand(0));
+    } else {
+      NewReg = MRI.createVirtualRegister(&CSKY::mGPRRegClass);
+      BuildMI(MBB, II, DL, TII->get(CSKY::MOVI16), NewReg).addImm(0);
+      BuildMI(MBB, II, DL, TII->get(CSKY::ADDC16))
+          .addReg(NewReg, RegState::Define)
+          .addReg(MI->getOperand(0).getReg(), RegState::Define)
+          .addReg(NewReg, getKillRegState(true))
+          .addReg(NewReg, getKillRegState(true))
+          .addReg(MI->getOperand(0).getReg());
+
+      BuildMI(MBB, II, DL, TII->get(CSKY::BTSTI16), MI->getOperand(0).getReg())
+          .addReg(NewReg)
+          .addImm(0);
+    }
+
+    MI = BuildMI(MBB, II, DL, TII->get(CSKY::ST32W))
+             .addReg(NewReg, getKillRegState(true))
+             .add(MI->getOperand(1))
+             .add(MI->getOperand(2))
+             .getInstr();
+
+    MBB.erase(II);
+
+    break;
+  }
+  }
+
+  int FrameIndex = MI->getOperand(FIOperandNum).getIndex();
+  Register FrameReg;
+  int Offset = getFrameLowering(MF)
+                   ->getFrameIndexReference(MF, FrameIndex, FrameReg)
+                   .getFixed() +
+               MI->getOperand(FIOperandNum + 1).getImm();
+
+  if (!isInt<32>(Offset))
+    report_fatal_error(
+        "Frame offsets outside of the signed 32-bit range not supported");
+
+  bool FrameRegIsKill = false;
+  MachineBasicBlock::iterator NewII(MI);
+  if (!IsLegalOffset(TII, MI, Offset)) {
+    assert(isInt<32>(Offset) && "Int32 expected");
+    // The offset won't fit in an immediate, so use a scratch register instead
+    // Modify Offset and FrameReg appropriately
+    assert(Offset >= 0);
+    Register ScratchReg = TII->movImm(MBB, NewII, DL, Offset);
+    BuildMI(MBB, NewII, DL,
+            TII->get(STI.hasE2() ? CSKY::ADDU32 : CSKY::ADDU16XZ), ScratchReg)
+        .addReg(ScratchReg, RegState::Kill)
+        .addReg(FrameReg);
+
+    Offset = 0;
+    FrameReg = ScratchReg;
+    FrameRegIsKill = true;
+  }
+
+  if (Offset == 0 &&
+      (MI->getOpcode() == CSKY::ADDI32 || MI->getOpcode() == CSKY::ADDI16XZ)) {
+    MI->setDesc(TII->get(TargetOpcode::COPY));
+    MI->getOperand(FIOperandNum)
+        .ChangeToRegister(FrameReg, false, false, FrameRegIsKill);
+    MI->RemoveOperand(FIOperandNum + 1);
+  } else {
+    MI->getOperand(FIOperandNum)
+        .ChangeToRegister(FrameReg, false, false, FrameRegIsKill);
+    MI->getOperand(FIOperandNum + 1).ChangeToImmediate(Offset);
+  }
+}
diff --git a/llvm/lib/Target/CSKY/CSKYRegisterInfo.h b/llvm/lib/Target/CSKY/CSKYRegisterInfo.h
index 779ea6493c7e..5b3b62ec0db2 100644
--- a/llvm/lib/Target/CSKY/CSKYRegisterInfo.h
+++ b/llvm/lib/Target/CSKY/CSKYRegisterInfo.h
@@ -38,6 +38,18 @@ public:
   void eliminateFrameIndex(MachineBasicBlock::iterator MI, int SPAdj,
                            unsigned FIOperandNum,
                            RegScavenger *RS) const override;
+
+  bool requiresFrameIndexScavenging(const MachineFunction &MF) const override {
+    return true;
+  }
+
+  bool requiresRegisterScavenging(const MachineFunction &MF) const override {
+    return true;
+  }
+
+  bool useFPForScavengingIndex(const MachineFunction &MF) const override {
+    return false;
+  }
 };
 
 } // namespace llvm
diff --git a/llvm/lib/Target/CSKY/CSKYRegisterInfo.td b/llvm/lib/Target/CSKY/CSKYRegisterInfo.td
index 7548c22bb2c5..ade5c7f795af 100644
--- a/llvm/lib/Target/CSKY/CSKYRegisterInfo.td
+++ b/llvm/lib/Target/CSKY/CSKYRegisterInfo.td
@@ -168,6 +168,11 @@ def mGPR : RegisterClass<"CSKY", [i32], 32,
   let Size = 32;
 }
 
+// Register class for SP only.
+def GPRSP : RegisterClass<"CSKY", [i32], 32, (add R14)> {
+  let Size = 32;
+}
+
 def GPRPair : RegisterClass<"CSKY", [untyped], 32, (add GPRTuple)> {
   let Size = 64;
 }
diff --git a/llvm/lib/Target/Hexagon/Hexagon.td b/llvm/lib/Target/Hexagon/Hexagon.td
index 7518fd774a48..ae811b30434d 100644
--- a/llvm/lib/Target/Hexagon/Hexagon.td
+++ b/llvm/lib/Target/Hexagon/Hexagon.td
@@ -29,6 +29,8 @@ def ProcTinyCore: SubtargetFeature<"tinycore", "HexagonProcFamily",
 // Hexagon ISA Extensions
 def ExtensionZReg: SubtargetFeature<"zreg", "UseZRegOps", "true",
       "Hexagon ZReg extension instructions">;
+def ExtensionHVXQFloat: SubtargetFeature<"hvx-qfloat", "UseHVXQFloatOps",
+      "true", "Hexagon HVX QFloating point instructions">;
 
 def ExtensionHVX: SubtargetFeature<"hvx", "HexagonHVXVersion",
       "Hexagon::ArchEnum::V60", "Hexagon HVX instructions">;
@@ -52,6 +54,10 @@ def ExtensionHVXV68: SubtargetFeature<"hvxv68", "HexagonHVXVersion",
       "Hexagon::ArchEnum::V68", "Hexagon HVX instructions",
       [ExtensionHVXV60, ExtensionHVXV62, ExtensionHVXV65, ExtensionHVXV66,
        ExtensionHVXV67]>;
+def ExtensionHVXV69: SubtargetFeature<"hvxv69", "HexagonHVXVersion",
+      "Hexagon::ArchEnum::V69", "Hexagon HVX instructions",
+      [ExtensionHVXV60, ExtensionHVXV62, ExtensionHVXV65, ExtensionHVXV66,
+       ExtensionHVXV67, ExtensionHVXV68]>;
 
 def ExtensionHVX64B: SubtargetFeature<"hvx-length64b", "UseHVX64BOps",
       "true", "Hexagon HVX 64B instructions", [ExtensionHVX]>;
@@ -61,6 +67,9 @@ def ExtensionHVX128B: SubtargetFeature<"hvx-length128b", "UseHVX128BOps",
 def ExtensionAudio: SubtargetFeature<"audio", "UseAudioOps", "true",
       "Hexagon Audio extension instructions">;
 
+def ExtensionHVXIEEEFP: SubtargetFeature<"hvx-ieee-fp", "UseHVXIEEEFPOps",
+      "true", "Hexagon HVX IEEE floating point instructions">;
+
 def FeatureCompound: SubtargetFeature<"compound", "UseCompound", "true",
       "Use compound instructions">;
 def FeaturePackets: SubtargetFeature<"packets", "UsePackets", "true",
@@ -88,6 +97,8 @@ def FeatureReservedR19: SubtargetFeature<"reserved-r19", "ReservedR19",
 def FeatureNoreturnStackElim: SubtargetFeature<"noreturn-stack-elim",
       "NoreturnStackElim", "true",
       "Eliminate stack allocation in a noreturn function when possible">;
+def FeatureCabac: SubtargetFeature<"cabac", "UseCabac", "false",
+      "Emit the CABAC instruction">;
 
 //===----------------------------------------------------------------------===//
 // Hexagon Instruction Predicate Definitions.
@@ -112,6 +123,8 @@ def UseHVXV67          : Predicate<"HST->useHVXV67Ops()">,
                          AssemblerPredicate<(all_of ExtensionHVXV67)>;
 def UseHVXV68          : Predicate<"HST->useHVXV68Ops()">,
                          AssemblerPredicate<(all_of ExtensionHVXV68)>;
+def UseHVXV69          : Predicate<"HST->useHVXV69Ops()">,
+                         AssemblerPredicate<(all_of ExtensionHVXV69)>;
 def UseAudio           : Predicate<"HST->useAudioOps()">,
                          AssemblerPredicate<(all_of ExtensionAudio)>;
 def UseZReg            : Predicate<"HST->useZRegOps()">,
@@ -119,6 +132,11 @@ def UseZReg            : Predicate<"HST->useZRegOps()">,
 def UseCompound        : Predicate<"HST->useCompound()">;
 def HasPreV65          : Predicate<"HST->hasPreV65()">,
                          AssemblerPredicate<(all_of FeaturePreV65)>;
+def UseHVXIEEEFP       : Predicate<"HST->useHVXIEEEFPOps()">,
+                         AssemblerPredicate<(all_of ExtensionHVXIEEEFP)>;
+def UseHVXQFloat       : Predicate<"HST->useHVXQFloatOps()">,
+                         AssemblerPredicate<(all_of ExtensionHVXQFloat)>;
+def UseHVXFloatingPoint: Predicate<"HST->useHVXFloatingPoint()">;
 def HasMemNoShuf       : Predicate<"HST->hasMemNoShuf()">,
                          AssemblerPredicate<(all_of FeatureMemNoShuf)>;
 def UseUnsafeMath      : Predicate<"HST->useUnsafeMath()">;
@@ -127,6 +145,8 @@ def NotOptTinyCore     : Predicate<"!HST->isTinyCore() ||"
   let RecomputePerFunction = 1;
 }
 def UseSmallData       : Predicate<"HST->useSmallData()">;
+def UseCabac           : Predicate<"HST->useCabac()">,
+                         AssemblerPredicate<(any_of FeatureCabac)>;
 
 def Hvx64:  HwMode<"+hvx-length64b">;
 def Hvx128: HwMode<"+hvx-length128b">;
@@ -299,7 +319,7 @@ def changeAddrMode_rr_ur: InstrMapping {
   let ValueCols = [["BaseLongOffset"]];
 }
 
-def changeAddrMode_ur_rr : InstrMapping {
+def changeAddrMode_ur_rr: InstrMapping {
   let FilterClass = "ImmRegShl";
   let RowFields = ["CextOpcode", "PredSense", "PNewValue", "isNVStore"];
   let ColFields = ["addrMode"];
@@ -370,40 +390,55 @@ class Proc<string Name, SchedMachineModel Model,
 def : Proc<"generic", HexagonModelV60,
            [ArchV5, ArchV55, ArchV60,
             FeatureCompound, FeatureDuplex, FeaturePreV65, FeatureMemops,
-            FeatureNVJ, FeatureNVS, FeaturePackets, FeatureSmallData]>;
+            FeatureNVJ, FeatureNVS, FeaturePackets, FeatureSmallData,
+            FeatureCabac]>;
 def : Proc<"hexagonv5",  HexagonModelV5,
            [ArchV5,
             FeatureCompound, FeatureDuplex, FeaturePreV65, FeatureMemops,
-            FeatureNVJ, FeatureNVS, FeaturePackets, FeatureSmallData]>;
+            FeatureNVJ, FeatureNVS, FeaturePackets, FeatureSmallData,
+            FeatureCabac]>;
 def : Proc<"hexagonv55", HexagonModelV55,
            [ArchV5, ArchV55,
             FeatureCompound, FeatureDuplex, FeaturePreV65, FeatureMemops,
-            FeatureNVJ, FeatureNVS, FeaturePackets, FeatureSmallData]>;
+            FeatureNVJ, FeatureNVS, FeaturePackets, FeatureSmallData,
+            FeatureCabac]>;
 def : Proc<"hexagonv60", HexagonModelV60,
            [ArchV5, ArchV55, ArchV60,
             FeatureCompound, FeatureDuplex, FeaturePreV65, FeatureMemops,
-            FeatureNVJ, FeatureNVS, FeaturePackets, FeatureSmallData]>;
+            FeatureNVJ, FeatureNVS, FeaturePackets, FeatureSmallData,
+            FeatureCabac]>;
 def : Proc<"hexagonv62", HexagonModelV62,
            [ArchV5, ArchV55, ArchV60, ArchV62,
             FeatureCompound, FeatureDuplex, FeaturePreV65, FeatureMemops,
-            FeatureNVJ, FeatureNVS, FeaturePackets, FeatureSmallData]>;
+            FeatureNVJ, FeatureNVS, FeaturePackets, FeatureSmallData,
+            FeatureCabac]>;
 def : Proc<"hexagonv65", HexagonModelV65,
            [ArchV5, ArchV55, ArchV60, ArchV62, ArchV65,
             FeatureCompound, FeatureDuplex, FeatureMemNoShuf, FeatureMemops,
-            FeatureNVJ, FeatureNVS, FeaturePackets, FeatureSmallData]>;
+            FeatureNVJ, FeatureNVS, FeaturePackets, FeatureSmallData,
+            FeatureCabac]>;
 def : Proc<"hexagonv66", HexagonModelV66,
            [ArchV5, ArchV55, ArchV60, ArchV62, ArchV65, ArchV66,
             FeatureCompound, FeatureDuplex, FeatureMemNoShuf, FeatureMemops,
-            FeatureNVJ, FeatureNVS, FeaturePackets, FeatureSmallData]>;
+            FeatureNVJ, FeatureNVS, FeaturePackets, FeatureSmallData,
+            FeatureCabac]>;
 def : Proc<"hexagonv67", HexagonModelV67,
            [ArchV5, ArchV55, ArchV60, ArchV62, ArchV65, ArchV66, ArchV67,
             FeatureCompound, FeatureDuplex, FeatureMemNoShuf, FeatureMemops,
-            FeatureNVJ, FeatureNVS, FeaturePackets, FeatureSmallData]>;
+            FeatureNVJ, FeatureNVS, FeaturePackets, FeatureSmallData,
+            FeatureCabac]>;
 def : Proc<"hexagonv68", HexagonModelV68,
            [ArchV5, ArchV55, ArchV60, ArchV62, ArchV65, ArchV66, ArchV67,
             ArchV68,
             FeatureCompound, FeatureDuplex, FeatureMemNoShuf, FeatureMemops,
-            FeatureNVJ, FeatureNVS, FeaturePackets, FeatureSmallData]>;
+            FeatureNVJ, FeatureNVS, FeaturePackets, FeatureSmallData,
+            FeatureCabac]>;
+def : Proc<"hexagonv69", HexagonModelV69,
+           [ArchV5, ArchV55, ArchV60, ArchV62, ArchV65, ArchV66, ArchV67,
+            ArchV68, ArchV69,
+            FeatureCompound, FeatureDuplex, FeatureMemNoShuf, FeatureMemops,
+            FeatureNVJ, FeatureNVS, FeaturePackets, FeatureSmallData,
+            FeatureCabac]>;
 // Need to update the correct features for tiny core.
 // Disable NewValueJumps since the packetizer is unable to handle a packet with
 // a new value jump and another SLOT0 instruction.
diff --git a/llvm/lib/Target/Hexagon/HexagonAsmPrinter.cpp b/llvm/lib/Target/Hexagon/HexagonAsmPrinter.cpp
index 8e6a01e3a186..411078052e0f 100644
--- a/llvm/lib/Target/Hexagon/HexagonAsmPrinter.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonAsmPrinter.cpp
@@ -773,6 +773,67 @@ void HexagonAsmPrinter::emitInstruction(const MachineInstr *MI) {
   OutStreamer->emitInstruction(MCB, getSubtargetInfo());
 }
 
+void HexagonAsmPrinter::EmitSled(const MachineInstr &MI, SledKind Kind) {
+  static const int8_t NoopsInSledCount = 4;
+  // We want to emit the following pattern:
+  //
+  // .L_xray_sled_N:
+  // <xray_sled_base>:
+  // { 	jump .Ltmp0 }
+  // {  nop
+  //    nop
+  //    nop
+  //    nop }
+  // .Ltmp0:
+  //
+  // We need the 4 nop words because at runtime, we'd be patching over the
+  // full 5 words with the following pattern:
+  //
+  // <xray_sled_n>:
+  // { 	immext(#...) // upper 26-bits of trampoline
+  //    r6 = ##...   // lower  6-bits of trampoline
+  //    immext(#...) // upper 26-bits of func id
+  //    r7 = ##... }  // lower 6 bits of func id
+  // { 	callr r6 }
+  //
+  //
+  auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
+  OutStreamer->emitLabel(CurSled);
+
+  MCInst *SledJump = new (OutContext) MCInst();
+  SledJump->setOpcode(Hexagon::J2_jump);
+  auto PostSled = OutContext.createTempSymbol();
+  SledJump->addOperand(MCOperand::createExpr(HexagonMCExpr::create(
+      MCSymbolRefExpr::create(PostSled, OutContext), OutContext)));
+
+  // Emit "jump PostSled" instruction, which jumps over the nop series.
+  MCInst SledJumpPacket;
+  SledJumpPacket.setOpcode(Hexagon::BUNDLE);
+  SledJumpPacket.addOperand(MCOperand::createImm(0));
+  SledJumpPacket.addOperand(MCOperand::createInst(SledJump));
+
+  EmitToStreamer(*OutStreamer, SledJumpPacket);
+
+  // FIXME: this will emit individual packets, we should
+  // special-case this and combine them into a single packet.
+  emitNops(NoopsInSledCount);
+
+  OutStreamer->emitLabel(PostSled);
+  recordSled(CurSled, MI, Kind, 0);
+}
+
+void HexagonAsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI) {
+  EmitSled(MI, SledKind::FUNCTION_ENTER);
+}
+
+void HexagonAsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI) {
+  EmitSled(MI, SledKind::FUNCTION_EXIT);
+}
+
+void HexagonAsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI) {
+  EmitSled(MI, SledKind::TAIL_CALL);
+}
+
 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeHexagonAsmPrinter() {
   RegisterAsmPrinter<HexagonAsmPrinter> X(getTheHexagonTarget());
 }
diff --git a/llvm/lib/Target/Hexagon/HexagonAsmPrinter.h b/llvm/lib/Target/Hexagon/HexagonAsmPrinter.h
index 3932def87854..93d5f1dce7af 100644
--- a/llvm/lib/Target/Hexagon/HexagonAsmPrinter.h
+++ b/llvm/lib/Target/Hexagon/HexagonAsmPrinter.h
@@ -36,7 +36,11 @@ class TargetMachine;
 
     bool runOnMachineFunction(MachineFunction &Fn) override {
       Subtarget = &Fn.getSubtarget<HexagonSubtarget>();
-      return AsmPrinter::runOnMachineFunction(Fn);
+      const bool Modified = AsmPrinter::runOnMachineFunction(Fn);
+      // Emit the XRay table for this function.
+      emitXRayTable();
+
+      return Modified;
     }
 
     StringRef getPassName() const override {
@@ -47,6 +51,16 @@ class TargetMachine;
           const override;
 
     void emitInstruction(const MachineInstr *MI) override;
+
+    //===------------------------------------------------------------------===//
+    // XRay implementation
+    //===------------------------------------------------------------------===//
+    // XRay-specific lowering for Hexagon.
+    void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI);
+    void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI);
+    void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI);
+    void EmitSled(const MachineInstr &MI, SledKind Kind);
+
     void HexagonProcessInstruction(MCInst &Inst, const MachineInstr &MBB);
 
     void printOperand(const MachineInstr *MI, unsigned OpNo, raw_ostream &O);
diff --git a/llvm/lib/Target/Hexagon/HexagonBitSimplify.cpp b/llvm/lib/Target/Hexagon/HexagonBitSimplify.cpp
index 2c5ad3b589d2..428d25da6dbc 100644
--- a/llvm/lib/Target/Hexagon/HexagonBitSimplify.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonBitSimplify.cpp
@@ -995,8 +995,8 @@ bool DeadCodeElimination::runOnNode(MachineDomTreeNode *N) {
 
   MachineBasicBlock *B = N->getBlock();
   std::vector<MachineInstr*> Instrs;
-  for (auto I = B->rbegin(), E = B->rend(); I != E; ++I)
-    Instrs.push_back(&*I);
+  for (MachineInstr &MI : llvm::reverse(*B))
+    Instrs.push_back(&MI);
 
   for (auto MI : Instrs) {
     unsigned Opc = MI->getOpcode();
@@ -3084,8 +3084,7 @@ void HexagonLoopRescheduling::moveGroup(InstrGroup &G, MachineBasicBlock &LB,
     .addMBB(&LB);
   RegMap.insert(std::make_pair(G.Inp.Reg, PhiR));
 
-  for (unsigned i = G.Ins.size(); i > 0; --i) {
-    const MachineInstr *SI = G.Ins[i-1];
+  for (const MachineInstr *SI : llvm::reverse(G.Ins)) {
     unsigned DR = getDefReg(SI);
     const TargetRegisterClass *RC = MRI->getRegClass(DR);
     Register NewDR = MRI->createVirtualRegister(RC);
@@ -3156,20 +3155,20 @@ bool HexagonLoopRescheduling::processLoop(LoopCand &C) {
   // if that instruction could potentially be moved to the front of the loop:
   // the output of the loop cannot be used in a non-shuffling instruction
   // in this loop.
-  for (auto I = C.LB->rbegin(), E = C.LB->rend(); I != E; ++I) {
-    if (I->isTerminator())
+  for (MachineInstr &MI : llvm::reverse(*C.LB)) {
+    if (MI.isTerminator())
       continue;
-    if (I->isPHI())
+    if (MI.isPHI())
       break;
 
     RegisterSet Defs;
-    HBS::getInstrDefs(*I, Defs);
+    HBS::getInstrDefs(MI, Defs);
     if (Defs.count() != 1)
       continue;
     Register DefR = Defs.find_first();
     if (!DefR.isVirtual())
       continue;
-    if (!isBitShuffle(&*I, DefR))
+    if (!isBitShuffle(&MI, DefR))
       continue;
 
     bool BadUse = false;
@@ -3183,8 +3182,7 @@ bool HexagonLoopRescheduling::processLoop(LoopCand &C) {
           if (UseI->getOperand(Idx+1).getMBB() != C.LB)
             BadUse = true;
         } else {
-          auto F = find(ShufIns, UseI);
-          if (F == ShufIns.end())
+          if (!llvm::is_contained(ShufIns, UseI))
             BadUse = true;
         }
       } else {
@@ -3199,7 +3197,7 @@ bool HexagonLoopRescheduling::processLoop(LoopCand &C) {
 
     if (BadUse)
       continue;
-    ShufIns.push_back(&*I);
+    ShufIns.push_back(&MI);
   }
 
   // Partition the list of shuffling instructions into instruction groups,
diff --git a/llvm/lib/Target/Hexagon/HexagonCommonGEP.cpp b/llvm/lib/Target/Hexagon/HexagonCommonGEP.cpp
index 8c3b9572201e..a53efeb96961 100644
--- a/llvm/lib/Target/Hexagon/HexagonCommonGEP.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonCommonGEP.cpp
@@ -1256,15 +1256,11 @@ void HexagonCommonGEP::removeDeadCode() {
       BO.push_back(DTN->getBlock());
   }
 
-  for (unsigned i = BO.size(); i > 0; --i) {
-    BasicBlock *B = cast<BasicBlock>(BO[i-1]);
-    BasicBlock::InstListType &IL = B->getInstList();
-
-    using reverse_iterator = BasicBlock::InstListType::reverse_iterator;
-
+  for (Value *V : llvm::reverse(BO)) {
+    BasicBlock *B = cast<BasicBlock>(V);
     ValueVect Ins;
-    for (reverse_iterator I = IL.rbegin(), E = IL.rend(); I != E; ++I)
-      Ins.push_back(&*I);
+    for (Instruction &I : llvm::reverse(*B))
+      Ins.push_back(&I);
     for (ValueVect::iterator I = Ins.begin(), E = Ins.end(); I != E; ++I) {
       Instruction *In = cast<Instruction>(*I);
       if (isInstructionTriviallyDead(In))
diff --git a/llvm/lib/Target/Hexagon/HexagonDepArch.h b/llvm/lib/Target/Hexagon/HexagonDepArch.h
index 7a43a4440b2d..56174dc7e136 100644
--- a/llvm/lib/Target/Hexagon/HexagonDepArch.h
+++ b/llvm/lib/Target/Hexagon/HexagonDepArch.h
@@ -21,31 +21,32 @@
 
 namespace llvm {
 namespace Hexagon {
-enum class ArchEnum { NoArch, Generic, V5, V55, V60, V62, V65, V66, V67, V68 };
+enum class ArchEnum { NoArch, Generic, V5, V55, V60, V62, V65, V66, V67, V68, V69 };
 
-static constexpr unsigned ArchValsNumArray[] = {5, 55, 60, 62, 65, 66, 67, 68};
+static constexpr unsigned ArchValsNumArray[] = {5, 55, 60, 62, 65, 66, 67, 68, 69};
 static constexpr ArrayRef<unsigned> ArchValsNum(ArchValsNumArray);
 
-static constexpr StringLiteral ArchValsTextArray[] = { "v5", "v55", "v60", "v62", "v65", "v66", "v67", "v68" };
+static constexpr StringLiteral ArchValsTextArray[] = { "v5", "v55", "v60", "v62", "v65", "v66", "v67", "v68", "v69" };
 static constexpr ArrayRef<StringLiteral> ArchValsText(ArchValsTextArray);
 
-static constexpr StringLiteral CpuValsTextArray[] = { "hexagonv5", "hexagonv55", "hexagonv60", "hexagonv62", "hexagonv65", "hexagonv66", "hexagonv67", "hexagonv67t", "hexagonv68" };
+static constexpr StringLiteral CpuValsTextArray[] = { "hexagonv5", "hexagonv55", "hexagonv60", "hexagonv62", "hexagonv65", "hexagonv66", "hexagonv67", "hexagonv67t", "hexagonv68", "hexagonv69" };
 static constexpr ArrayRef<StringLiteral> CpuValsText(CpuValsTextArray);
 
-static constexpr StringLiteral CpuNickTextArray[] = { "v5", "v55", "v60", "v62", "v65", "v66", "v67", "v67t", "v68" };
+static constexpr StringLiteral CpuNickTextArray[] = { "v5", "v55", "v60", "v62", "v65", "v66", "v67", "v67t", "v68", "v69" };
 static constexpr ArrayRef<StringLiteral> CpuNickText(CpuNickTextArray);
 
 static const std::map<std::string, ArchEnum> CpuTable{
-  {"generic", Hexagon::ArchEnum::V5},
-  {"hexagonv5", Hexagon::ArchEnum::V5},
-  {"hexagonv55", Hexagon::ArchEnum::V55},
-  {"hexagonv60", Hexagon::ArchEnum::V60},
-  {"hexagonv62", Hexagon::ArchEnum::V62},
-  {"hexagonv65", Hexagon::ArchEnum::V65},
-  {"hexagonv66", Hexagon::ArchEnum::V66},
-  {"hexagonv67", Hexagon::ArchEnum::V67},
-  {"hexagonv67t", Hexagon::ArchEnum::V67},
-  {"hexagonv68", Hexagon::ArchEnum::V68},
+    {"generic", Hexagon::ArchEnum::V5},
+    {"hexagonv5", Hexagon::ArchEnum::V5},
+    {"hexagonv55", Hexagon::ArchEnum::V55},
+    {"hexagonv60", Hexagon::ArchEnum::V60},
+    {"hexagonv62", Hexagon::ArchEnum::V62},
+    {"hexagonv65", Hexagon::ArchEnum::V65},
+    {"hexagonv66", Hexagon::ArchEnum::V66},
+    {"hexagonv67", Hexagon::ArchEnum::V67},
+    {"hexagonv67t", Hexagon::ArchEnum::V67},
+    {"hexagonv68", Hexagon::ArchEnum::V68},
+    {"hexagonv69", Hexagon::ArchEnum::V69},
 };
 
 static const std::map<std::string, unsigned> ElfFlagsByCpuStr = {
@@ -59,6 +60,7 @@ static const std::map<std::string, unsigned> ElfFlagsByCpuStr = {
   {"hexagonv67", llvm::ELF::EF_HEXAGON_MACH_V67},
   {"hexagonv67t", llvm::ELF::EF_HEXAGON_MACH_V67T},
   {"hexagonv68", llvm::ELF::EF_HEXAGON_MACH_V68},
+  {"hexagonv69", llvm::ELF::EF_HEXAGON_MACH_V69},
 };
 static const std::map<unsigned, std::string> ElfArchByMachFlags = {
   {llvm::ELF::EF_HEXAGON_MACH_V5, "V5"},
@@ -70,6 +72,7 @@ static const std::map<unsigned, std::string> ElfArchByMachFlags = {
   {llvm::ELF::EF_HEXAGON_MACH_V67, "V67"},
   {llvm::ELF::EF_HEXAGON_MACH_V67T, "V67T"},
   {llvm::ELF::EF_HEXAGON_MACH_V68, "V68"},
+  {llvm::ELF::EF_HEXAGON_MACH_V69, "V69"},
 };
 static const std::map<unsigned, std::string> ElfCpuByMachFlags = {
   {llvm::ELF::EF_HEXAGON_MACH_V5, "hexagonv5"},
@@ -81,6 +84,7 @@ static const std::map<unsigned, std::string> ElfCpuByMachFlags = {
   {llvm::ELF::EF_HEXAGON_MACH_V67, "hexagonv67"},
   {llvm::ELF::EF_HEXAGON_MACH_V67T, "hexagonv67t"},
   {llvm::ELF::EF_HEXAGON_MACH_V68, "hexagonv68"},
+  {llvm::ELF::EF_HEXAGON_MACH_V69, "hexagonv69"},
 };
 
 } // namespace Hexagon
diff --git a/llvm/lib/Target/Hexagon/HexagonDepArch.td b/llvm/lib/Target/Hexagon/HexagonDepArch.td
index e743a291f1e5..e4f24e3c2e66 100644
--- a/llvm/lib/Target/Hexagon/HexagonDepArch.td
+++ b/llvm/lib/Target/Hexagon/HexagonDepArch.td
@@ -24,3 +24,5 @@ def ArchV67: SubtargetFeature<"v67", "HexagonArchVersion", "Hexagon::ArchEnum::V
 def HasV67 : Predicate<"HST->hasV67Ops()">, AssemblerPredicate<(all_of ArchV67)>;
 def ArchV68: SubtargetFeature<"v68", "HexagonArchVersion", "Hexagon::ArchEnum::V68", "Enable Hexagon V68 architecture">;
 def HasV68 : Predicate<"HST->hasV68Ops()">, AssemblerPredicate<(all_of ArchV68)>;
+def ArchV69: SubtargetFeature<"v69", "HexagonArchVersion", "Hexagon::ArchEnum::V69", "Enable Hexagon V69 architecture">;
+def HasV69 : Predicate<"HST->hasV69Ops()">, AssemblerPredicate<(all_of ArchV69)>;
diff --git a/llvm/lib/Target/Hexagon/HexagonDepDecoders.inc b/llvm/lib/Target/Hexagon/HexagonDepDecoders.inc
index 40f6e14aed13..7164af3ad5c6 100644
--- a/llvm/lib/Target/Hexagon/HexagonDepDecoders.inc
+++ b/llvm/lib/Target/Hexagon/HexagonDepDecoders.inc
@@ -8,6 +8,7 @@
 // Automatically generated file, do not edit!
 //===----------------------------------------------------------------------===//
 
+
 #if defined(__clang__)
 #pragma clang diagnostic push
 #pragma clang diagnostic ignored "-Wunused-function"
diff --git a/llvm/lib/Target/Hexagon/HexagonDepIICHVX.td b/llvm/lib/Target/Hexagon/HexagonDepIICHVX.td
index a1db3ae7239d..d195df918293 100644
--- a/llvm/lib/Target/Hexagon/HexagonDepIICHVX.td
+++ b/llvm/lib/Target/Hexagon/HexagonDepIICHVX.td
@@ -11,6 +11,7 @@
 def tc_04da405a : InstrItinClass;
 def tc_05ca8cfd : InstrItinClass;
 def tc_08a4f1b6 : InstrItinClass;
+def tc_0afc8be9 : InstrItinClass;
 def tc_0b04c6c7 : InstrItinClass;
 def tc_0ec46cf9 : InstrItinClass;
 def tc_131f1c81 : InstrItinClass;
@@ -21,6 +22,7 @@ def tc_191381c1 : InstrItinClass;
 def tc_1ad8a370 : InstrItinClass;
 def tc_1ba8a0cd : InstrItinClass;
 def tc_20a4bbec : InstrItinClass;
+def tc_2120355e : InstrItinClass;
 def tc_257f6f7c : InstrItinClass;
 def tc_26a377fe : InstrItinClass;
 def tc_2b4c548e : InstrItinClass;
@@ -28,15 +30,18 @@ def tc_2c745bb8 : InstrItinClass;
 def tc_2d4051cd : InstrItinClass;
 def tc_2e8f5f6e : InstrItinClass;
 def tc_309dbb4f : InstrItinClass;
+def tc_37820f4c : InstrItinClass;
 def tc_3904b926 : InstrItinClass;
 def tc_3aacf4a8 : InstrItinClass;
 def tc_3ad719fb : InstrItinClass;
 def tc_3c56e5ce : InstrItinClass;
+def tc_3c8c15d0 : InstrItinClass;
 def tc_3ce09744 : InstrItinClass;
 def tc_3e2aaafc : InstrItinClass;
 def tc_447d9895 : InstrItinClass;
 def tc_453fe68d : InstrItinClass;
 def tc_46d6c3e0 : InstrItinClass;
+def tc_4942646a : InstrItinClass;
 def tc_51d0ecc3 : InstrItinClass;
 def tc_52447ecc : InstrItinClass;
 def tc_540c3da3 : InstrItinClass;
@@ -46,6 +51,7 @@ def tc_56c4f9fe : InstrItinClass;
 def tc_56e64202 : InstrItinClass;
 def tc_58d21193 : InstrItinClass;
 def tc_5bf8afbb : InstrItinClass;
+def tc_5cdf8c84 : InstrItinClass;
 def tc_61bf7c03 : InstrItinClass;
 def tc_649072c2 : InstrItinClass;
 def tc_660769f1 : InstrItinClass;
@@ -57,6 +63,8 @@ def tc_71646d06 : InstrItinClass;
 def tc_7177e272 : InstrItinClass;
 def tc_718b5c53 : InstrItinClass;
 def tc_7273323b : InstrItinClass;
+def tc_72e2b393 : InstrItinClass;
+def tc_73efe966 : InstrItinClass;
 def tc_7417e785 : InstrItinClass;
 def tc_767c4e9d : InstrItinClass;
 def tc_7d68d5c2 : InstrItinClass;
@@ -71,9 +79,11 @@ def tc_9d1dc972 : InstrItinClass;
 def tc_9f363d21 : InstrItinClass;
 def tc_a02a10a8 : InstrItinClass;
 def tc_a0dbea28 : InstrItinClass;
+def tc_a19b9305 : InstrItinClass;
 def tc_a28f32b5 : InstrItinClass;
 def tc_a69eeee1 : InstrItinClass;
 def tc_a7e6707d : InstrItinClass;
+def tc_aa047364 : InstrItinClass;
 def tc_ab23f776 : InstrItinClass;
 def tc_abe8c3b2 : InstrItinClass;
 def tc_ac4046bc : InstrItinClass;
@@ -89,8 +99,10 @@ def tc_c4edf264 : InstrItinClass;
 def tc_c5dba46e : InstrItinClass;
 def tc_c7039829 : InstrItinClass;
 def tc_cd94bfe0 : InstrItinClass;
+def tc_cda936da : InstrItinClass;
 def tc_d8287c14 : InstrItinClass;
 def tc_db5555f3 : InstrItinClass;
+def tc_dcca380f : InstrItinClass;
 def tc_dd5b0695 : InstrItinClass;
 def tc_df80eeb0 : InstrItinClass;
 def tc_e2d2e9e5 : InstrItinClass;
@@ -99,6 +111,7 @@ def tc_e3f68a46 : InstrItinClass;
 def tc_e675c45a : InstrItinClass;
 def tc_e699ae41 : InstrItinClass;
 def tc_e99d4c2e : InstrItinClass;
+def tc_f175e046 : InstrItinClass;
 def tc_f1de44ef : InstrItinClass;
 def tc_f21e8abb : InstrItinClass;
 
@@ -119,6 +132,11 @@ class DepHVXItinV55 {
        InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 5],
       [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>,
 
+    InstrItinData <tc_0afc8be9, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 5],
+      [HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_0b04c6c7, /*SLOT23,VX_DV*/
       [InstrStage<1, [SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_MPY01]>], [9, 5, 2],
@@ -174,6 +192,10 @@ class DepHVXItinV55 {
        InstrStage<1, [CVI_ST]>], [3, 1, 2],
       [Hex_FWD, Hex_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_2120355e, /*SLOT0123*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [9, 7],
+      [HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_257f6f7c, /*SLOT0123,VA*/
       [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [9, 7, 7, 7],
@@ -209,6 +231,11 @@ class DepHVXItinV55 {
        InstrStage<1, [CVI_SHIFT]>], [9, 7, 5, 2],
       [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_37820f4c, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 5],
+      [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_3904b926, /*SLOT01,LOAD*/
       [InstrStage<1, [SLOT0, SLOT1], 0>,
        InstrStage<1, [CVI_LD]>], [9, 2, 1, 2],
@@ -231,6 +258,11 @@ class DepHVXItinV55 {
        InstrStage<1, [CVI_XLANE]>], [9, 3, 1, 2],
       [HVX_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_3c8c15d0, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5],
+      [HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_3ce09744, /*SLOT0,STORE*/
       [InstrStage<1, [SLOT0], 0>,
        InstrStage<1, [CVI_ST]>], [1, 2],
@@ -259,6 +291,11 @@ class DepHVXItinV55 {
        InstrStage<1, [CVI_XLANE]>], [9, 5, 5],
       [HVX_FWD, HVX_FWD, HVX_FWD]>,
 
+    InstrItinData <tc_4942646a, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 5, 5, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
     InstrItinData <tc_51d0ecc3, /*SLOT0123,VS*/
       [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_SHIFT]>], [9, 5],
@@ -306,6 +343,11 @@ class DepHVXItinV55 {
        InstrStage<1, [CVI_XLANE]>], [9, 2],
       [HVX_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_5cdf8c84, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7],
+      [HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_61bf7c03, /*SLOT23,4SLOT_MPY*/
       [InstrStage<1, [SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_ALL_NOMEM]>], [9, 5, 2],
@@ -363,6 +405,16 @@ class DepHVXItinV55 {
        InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [1, 2, 7, 7],
       [Hex_FWD, Hex_FWD, HVX_FWD, HVX_FWD]>,
 
+    InstrItinData <tc_72e2b393, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_73efe966, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 5, 5],
+      [HVX_FWD, HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_7417e785, /*SLOT0123,VS*/
       [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_SHIFT]>], [9, 5, 2],
@@ -437,6 +489,11 @@ class DepHVXItinV55 {
        InstrStage<1, [CVI_ZW]>], [3, 1, 2],
       [Hex_FWD, Hex_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_a19b9305, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 5, 5],
+      [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_a28f32b5, /*SLOT1,LOAD,VA*/
       [InstrStage<1, [SLOT1], 0>,
        InstrStage<1, [CVI_LD], 0>,
@@ -456,6 +513,10 @@ class DepHVXItinV55 {
        InstrStage<1, [CVI_XLANE]>], [9, 1, 2],
       [HVX_FWD, Hex_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_aa047364, /*SLOT0123*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [9, 7, 7],
+      [HVX_FWD, HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_ab23f776, /*SLOT0,STORE*/
       [InstrStage<1, [SLOT0], 0>,
        InstrStage<1, [CVI_ST]>], [1, 2, 5],
@@ -537,6 +598,11 @@ class DepHVXItinV55 {
        InstrStage<1, [CVI_SHIFT, CVI_XLANE]>], [9, 5, 2],
       [HVX_FWD, HVX_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_cda936da, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 7],
+      [HVX_FWD, HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_d8287c14, /*SLOT23,VX_DV*/
       [InstrStage<1, [SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_MPY01]>], [9, 5, 5],
@@ -547,6 +613,11 @@ class DepHVXItinV55 {
        InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [9, 7, 7],
       [HVX_FWD, HVX_FWD, HVX_FWD]>,
 
+    InstrItinData <tc_dcca380f, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 5, 2],
+      [HVX_FWD, HVX_FWD, Hex_FWD]>,
+
     InstrItinData <tc_dd5b0695, /*SLOT01,ZW*/
       [InstrStage<1, [SLOT0, SLOT1], 0>,
        InstrStage<1, [CVI_ZW]>], [2, 1, 2],
@@ -589,6 +660,11 @@ class DepHVXItinV55 {
        InstrStage<1, [CVI_ST]>], [3, 2, 1, 2, 5],
       [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>,
 
+    InstrItinData <tc_f175e046, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5, 5, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
     InstrItinData <tc_f1de44ef, /*SLOT2,VX_DV*/
       [InstrStage<1, [SLOT2], 0>,
        InstrStage<1, [CVI_MPY01]>], [9, 5, 2],
@@ -620,6 +696,11 @@ class DepHVXItinV60 {
        InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 5],
       [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>,
 
+    InstrItinData <tc_0afc8be9, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 5],
+      [HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_0b04c6c7, /*SLOT23,VX_DV*/
       [InstrStage<1, [SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_MPY01]>], [9, 5, 2],
@@ -675,6 +756,10 @@ class DepHVXItinV60 {
        InstrStage<1, [CVI_ST]>], [3, 1, 2],
       [Hex_FWD, Hex_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_2120355e, /*SLOT0123*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [9, 7],
+      [HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_257f6f7c, /*SLOT0123,VA*/
       [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [9, 7, 7, 7],
@@ -710,6 +795,11 @@ class DepHVXItinV60 {
        InstrStage<1, [CVI_SHIFT]>], [9, 7, 5, 2],
       [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_37820f4c, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 5],
+      [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_3904b926, /*SLOT01,LOAD*/
       [InstrStage<1, [SLOT0, SLOT1], 0>,
        InstrStage<1, [CVI_LD]>], [9, 2, 1, 2],
@@ -732,6 +822,11 @@ class DepHVXItinV60 {
        InstrStage<1, [CVI_XLANE]>], [9, 3, 1, 2],
       [HVX_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_3c8c15d0, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5],
+      [HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_3ce09744, /*SLOT0,STORE*/
       [InstrStage<1, [SLOT0], 0>,
        InstrStage<1, [CVI_ST]>], [1, 2],
@@ -760,6 +855,11 @@ class DepHVXItinV60 {
        InstrStage<1, [CVI_XLANE]>], [9, 5, 5],
       [HVX_FWD, HVX_FWD, HVX_FWD]>,
 
+    InstrItinData <tc_4942646a, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 5, 5, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
     InstrItinData <tc_51d0ecc3, /*SLOT0123,VS*/
       [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_SHIFT]>], [9, 5],
@@ -807,6 +907,11 @@ class DepHVXItinV60 {
        InstrStage<1, [CVI_XLANE]>], [9, 2],
       [HVX_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_5cdf8c84, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7],
+      [HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_61bf7c03, /*SLOT23,4SLOT_MPY*/
       [InstrStage<1, [SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_ALL_NOMEM]>], [9, 5, 2],
@@ -864,6 +969,16 @@ class DepHVXItinV60 {
        InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [1, 2, 7, 7],
       [Hex_FWD, Hex_FWD, HVX_FWD, HVX_FWD]>,
 
+    InstrItinData <tc_72e2b393, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_73efe966, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 5, 5],
+      [HVX_FWD, HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_7417e785, /*SLOT0123,VS*/
       [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_SHIFT]>], [9, 5, 2],
@@ -938,6 +1053,11 @@ class DepHVXItinV60 {
        InstrStage<1, [CVI_ZW]>], [3, 1, 2],
       [Hex_FWD, Hex_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_a19b9305, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 5, 5],
+      [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_a28f32b5, /*SLOT1,LOAD,VA*/
       [InstrStage<1, [SLOT1], 0>,
        InstrStage<1, [CVI_LD], 0>,
@@ -957,6 +1077,10 @@ class DepHVXItinV60 {
        InstrStage<1, [CVI_XLANE]>], [9, 1, 2],
       [HVX_FWD, Hex_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_aa047364, /*SLOT0123*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [9, 7, 7],
+      [HVX_FWD, HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_ab23f776, /*SLOT0,STORE*/
       [InstrStage<1, [SLOT0], 0>,
        InstrStage<1, [CVI_ST]>], [1, 2, 5],
@@ -1038,6 +1162,11 @@ class DepHVXItinV60 {
        InstrStage<1, [CVI_SHIFT, CVI_XLANE]>], [9, 5, 2],
       [HVX_FWD, HVX_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_cda936da, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 7],
+      [HVX_FWD, HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_d8287c14, /*SLOT23,VX_DV*/
       [InstrStage<1, [SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_MPY01]>], [9, 5, 5],
@@ -1048,6 +1177,11 @@ class DepHVXItinV60 {
        InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [9, 7, 7],
       [HVX_FWD, HVX_FWD, HVX_FWD]>,
 
+    InstrItinData <tc_dcca380f, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 5, 2],
+      [HVX_FWD, HVX_FWD, Hex_FWD]>,
+
     InstrItinData <tc_dd5b0695, /*SLOT01,ZW*/
       [InstrStage<1, [SLOT0, SLOT1], 0>,
        InstrStage<1, [CVI_ZW]>], [2, 1, 2],
@@ -1090,6 +1224,11 @@ class DepHVXItinV60 {
        InstrStage<1, [CVI_ST]>], [3, 2, 1, 2, 5],
       [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>,
 
+    InstrItinData <tc_f175e046, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5, 5, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
     InstrItinData <tc_f1de44ef, /*SLOT2,VX_DV*/
       [InstrStage<1, [SLOT2], 0>,
        InstrStage<1, [CVI_MPY01]>], [9, 5, 2],
@@ -1121,6 +1260,11 @@ class DepHVXItinV62 {
        InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 5],
       [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>,
 
+    InstrItinData <tc_0afc8be9, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 5],
+      [HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_0b04c6c7, /*SLOT23,VX_DV*/
       [InstrStage<1, [SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_MPY01]>], [9, 5, 2],
@@ -1176,6 +1320,10 @@ class DepHVXItinV62 {
        InstrStage<1, [CVI_ST]>], [3, 1, 2],
       [Hex_FWD, Hex_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_2120355e, /*SLOT0123*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [9, 7],
+      [HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_257f6f7c, /*SLOT0123,VA*/
       [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [9, 7, 7, 7],
@@ -1211,6 +1359,11 @@ class DepHVXItinV62 {
        InstrStage<1, [CVI_SHIFT]>], [9, 7, 5, 2],
       [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_37820f4c, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 5],
+      [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_3904b926, /*SLOT01,LOAD*/
       [InstrStage<1, [SLOT0, SLOT1], 0>,
        InstrStage<1, [CVI_LD]>], [9, 2, 1, 2],
@@ -1233,6 +1386,11 @@ class DepHVXItinV62 {
        InstrStage<1, [CVI_XLANE]>], [9, 3, 1, 2],
       [HVX_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_3c8c15d0, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5],
+      [HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_3ce09744, /*SLOT0,STORE*/
       [InstrStage<1, [SLOT0], 0>,
        InstrStage<1, [CVI_ST]>], [1, 2],
@@ -1261,6 +1419,11 @@ class DepHVXItinV62 {
        InstrStage<1, [CVI_XLANE]>], [9, 5, 5],
       [HVX_FWD, HVX_FWD, HVX_FWD]>,
 
+    InstrItinData <tc_4942646a, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 5, 5, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
     InstrItinData <tc_51d0ecc3, /*SLOT0123,VS*/
       [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_SHIFT]>], [9, 5],
@@ -1308,6 +1471,11 @@ class DepHVXItinV62 {
        InstrStage<1, [CVI_XLANE]>], [9, 2],
       [HVX_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_5cdf8c84, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7],
+      [HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_61bf7c03, /*SLOT23,4SLOT_MPY*/
       [InstrStage<1, [SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_ALL_NOMEM]>], [9, 5, 2],
@@ -1365,6 +1533,16 @@ class DepHVXItinV62 {
        InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [1, 2, 7, 7],
       [Hex_FWD, Hex_FWD, HVX_FWD, HVX_FWD]>,
 
+    InstrItinData <tc_72e2b393, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_73efe966, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 5, 5],
+      [HVX_FWD, HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_7417e785, /*SLOT0123,VS*/
       [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_SHIFT]>], [9, 5, 2],
@@ -1439,6 +1617,11 @@ class DepHVXItinV62 {
        InstrStage<1, [CVI_ZW]>], [3, 1, 2],
       [Hex_FWD, Hex_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_a19b9305, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 5, 5],
+      [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_a28f32b5, /*SLOT1,LOAD,VA*/
       [InstrStage<1, [SLOT1], 0>,
        InstrStage<1, [CVI_LD], 0>,
@@ -1458,6 +1641,10 @@ class DepHVXItinV62 {
        InstrStage<1, [CVI_XLANE]>], [9, 1, 2],
       [HVX_FWD, Hex_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_aa047364, /*SLOT0123*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [9, 7, 7],
+      [HVX_FWD, HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_ab23f776, /*SLOT0,STORE*/
       [InstrStage<1, [SLOT0], 0>,
        InstrStage<1, [CVI_ST]>], [1, 2, 5],
@@ -1539,6 +1726,11 @@ class DepHVXItinV62 {
        InstrStage<1, [CVI_SHIFT, CVI_XLANE]>], [9, 5, 2],
       [HVX_FWD, HVX_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_cda936da, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 7],
+      [HVX_FWD, HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_d8287c14, /*SLOT23,VX_DV*/
       [InstrStage<1, [SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_MPY01]>], [9, 5, 5],
@@ -1549,6 +1741,11 @@ class DepHVXItinV62 {
        InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [9, 7, 7],
       [HVX_FWD, HVX_FWD, HVX_FWD]>,
 
+    InstrItinData <tc_dcca380f, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 5, 2],
+      [HVX_FWD, HVX_FWD, Hex_FWD]>,
+
     InstrItinData <tc_dd5b0695, /*SLOT01,ZW*/
       [InstrStage<1, [SLOT0, SLOT1], 0>,
        InstrStage<1, [CVI_ZW]>], [2, 1, 2],
@@ -1591,6 +1788,11 @@ class DepHVXItinV62 {
        InstrStage<1, [CVI_ST]>], [3, 2, 1, 2, 5],
       [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>,
 
+    InstrItinData <tc_f175e046, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5, 5, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
     InstrItinData <tc_f1de44ef, /*SLOT2,VX_DV*/
       [InstrStage<1, [SLOT2], 0>,
        InstrStage<1, [CVI_MPY01]>], [9, 5, 2],
@@ -1622,6 +1824,11 @@ class DepHVXItinV65 {
        InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 5],
       [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>,
 
+    InstrItinData <tc_0afc8be9, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 5],
+      [HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_0b04c6c7, /*SLOT23,VX_DV*/
       [InstrStage<1, [SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_MPY01]>], [9, 5, 2],
@@ -1677,6 +1884,10 @@ class DepHVXItinV65 {
        InstrStage<1, [CVI_ST]>], [3, 1, 2],
       [Hex_FWD, Hex_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_2120355e, /*SLOT0123*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [9, 7],
+      [HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_257f6f7c, /*SLOT0123,VA*/
       [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [9, 7, 7, 7],
@@ -1712,6 +1923,11 @@ class DepHVXItinV65 {
        InstrStage<1, [CVI_SHIFT]>], [9, 7, 5, 2],
       [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_37820f4c, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 5],
+      [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_3904b926, /*SLOT01,LOAD*/
       [InstrStage<1, [SLOT0, SLOT1], 0>,
        InstrStage<1, [CVI_LD]>], [9, 2, 1, 2],
@@ -1734,6 +1950,11 @@ class DepHVXItinV65 {
        InstrStage<1, [CVI_XLANE]>], [9, 3, 1, 2],
       [HVX_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_3c8c15d0, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5],
+      [HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_3ce09744, /*SLOT0,STORE*/
       [InstrStage<1, [SLOT0], 0>,
        InstrStage<1, [CVI_ST]>], [1, 2],
@@ -1762,6 +1983,11 @@ class DepHVXItinV65 {
        InstrStage<1, [CVI_XLANE]>], [9, 5, 5],
       [HVX_FWD, HVX_FWD, HVX_FWD]>,
 
+    InstrItinData <tc_4942646a, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 5, 5, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
     InstrItinData <tc_51d0ecc3, /*SLOT0123,VS*/
       [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_SHIFT]>], [9, 5],
@@ -1809,6 +2035,11 @@ class DepHVXItinV65 {
        InstrStage<1, [CVI_XLANE]>], [9, 2],
       [HVX_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_5cdf8c84, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7],
+      [HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_61bf7c03, /*SLOT23,4SLOT_MPY*/
       [InstrStage<1, [SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_ALL_NOMEM]>], [9, 5, 2],
@@ -1866,6 +2097,16 @@ class DepHVXItinV65 {
        InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [1, 2, 7, 7],
       [Hex_FWD, Hex_FWD, HVX_FWD, HVX_FWD]>,
 
+    InstrItinData <tc_72e2b393, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_73efe966, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 5, 5],
+      [HVX_FWD, HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_7417e785, /*SLOT0123,VS*/
       [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_SHIFT]>], [9, 5, 2],
@@ -1940,6 +2181,11 @@ class DepHVXItinV65 {
        InstrStage<1, [CVI_ZW]>], [3, 1, 2],
       [Hex_FWD, Hex_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_a19b9305, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 5, 5],
+      [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_a28f32b5, /*SLOT1,LOAD,VA*/
       [InstrStage<1, [SLOT1], 0>,
        InstrStage<1, [CVI_LD], 0>,
@@ -1959,6 +2205,10 @@ class DepHVXItinV65 {
        InstrStage<1, [CVI_XLANE]>], [9, 1, 2],
       [HVX_FWD, Hex_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_aa047364, /*SLOT0123*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [9, 7, 7],
+      [HVX_FWD, HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_ab23f776, /*SLOT0,STORE*/
       [InstrStage<1, [SLOT0], 0>,
        InstrStage<1, [CVI_ST]>], [1, 2, 5],
@@ -2040,6 +2290,11 @@ class DepHVXItinV65 {
        InstrStage<1, [CVI_SHIFT, CVI_XLANE]>], [9, 5, 2],
       [HVX_FWD, HVX_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_cda936da, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 7],
+      [HVX_FWD, HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_d8287c14, /*SLOT23,VX_DV*/
       [InstrStage<1, [SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_MPY01]>], [9, 5, 5],
@@ -2050,6 +2305,11 @@ class DepHVXItinV65 {
        InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [9, 7, 7],
       [HVX_FWD, HVX_FWD, HVX_FWD]>,
 
+    InstrItinData <tc_dcca380f, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 5, 2],
+      [HVX_FWD, HVX_FWD, Hex_FWD]>,
+
     InstrItinData <tc_dd5b0695, /*SLOT01,ZW*/
       [InstrStage<1, [SLOT0, SLOT1], 0>,
        InstrStage<1, [CVI_ZW]>], [2, 1, 2],
@@ -2092,6 +2352,11 @@ class DepHVXItinV65 {
        InstrStage<1, [CVI_ST]>], [3, 2, 1, 2, 5],
       [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>,
 
+    InstrItinData <tc_f175e046, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5, 5, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
     InstrItinData <tc_f1de44ef, /*SLOT2,VX_DV*/
       [InstrStage<1, [SLOT2], 0>,
        InstrStage<1, [CVI_MPY01]>], [9, 5, 2],
@@ -2123,6 +2388,11 @@ class DepHVXItinV66 {
        InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 5],
       [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>,
 
+    InstrItinData <tc_0afc8be9, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 5],
+      [HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_0b04c6c7, /*SLOT23,VX_DV*/
       [InstrStage<1, [SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_MPY01]>], [9, 5, 2],
@@ -2178,6 +2448,10 @@ class DepHVXItinV66 {
        InstrStage<1, [CVI_ST]>], [3, 1, 2],
       [Hex_FWD, Hex_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_2120355e, /*SLOT0123*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [9, 7],
+      [HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_257f6f7c, /*SLOT0123,VA*/
       [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [9, 7, 7, 7],
@@ -2213,6 +2487,11 @@ class DepHVXItinV66 {
        InstrStage<1, [CVI_SHIFT]>], [9, 7, 5, 2],
       [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_37820f4c, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 5],
+      [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_3904b926, /*SLOT01,LOAD*/
       [InstrStage<1, [SLOT0, SLOT1], 0>,
        InstrStage<1, [CVI_LD]>], [9, 2, 1, 2],
@@ -2235,6 +2514,11 @@ class DepHVXItinV66 {
        InstrStage<1, [CVI_XLANE]>], [9, 3, 1, 2],
       [HVX_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_3c8c15d0, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5],
+      [HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_3ce09744, /*SLOT0,STORE*/
       [InstrStage<1, [SLOT0], 0>,
        InstrStage<1, [CVI_ST]>], [1, 2],
@@ -2263,6 +2547,11 @@ class DepHVXItinV66 {
        InstrStage<1, [CVI_XLANE]>], [9, 5, 5],
       [HVX_FWD, HVX_FWD, HVX_FWD]>,
 
+    InstrItinData <tc_4942646a, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 5, 5, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
     InstrItinData <tc_51d0ecc3, /*SLOT0123,VS*/
       [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_SHIFT]>], [9, 5],
@@ -2310,6 +2599,11 @@ class DepHVXItinV66 {
        InstrStage<1, [CVI_XLANE]>], [9, 2],
       [HVX_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_5cdf8c84, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7],
+      [HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_61bf7c03, /*SLOT23,4SLOT_MPY*/
       [InstrStage<1, [SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_ALL_NOMEM]>], [9, 5, 2],
@@ -2367,6 +2661,16 @@ class DepHVXItinV66 {
        InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [1, 2, 7, 7],
       [Hex_FWD, Hex_FWD, HVX_FWD, HVX_FWD]>,
 
+    InstrItinData <tc_72e2b393, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_73efe966, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 5, 5],
+      [HVX_FWD, HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_7417e785, /*SLOT0123,VS*/
       [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_SHIFT]>], [9, 5, 2],
@@ -2441,6 +2745,11 @@ class DepHVXItinV66 {
        InstrStage<1, [CVI_ZW]>], [3, 1, 2],
       [Hex_FWD, Hex_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_a19b9305, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 5, 5],
+      [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_a28f32b5, /*SLOT1,LOAD,VA*/
       [InstrStage<1, [SLOT1], 0>,
        InstrStage<1, [CVI_LD], 0>,
@@ -2460,6 +2769,10 @@ class DepHVXItinV66 {
        InstrStage<1, [CVI_XLANE]>], [9, 1, 2],
       [HVX_FWD, Hex_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_aa047364, /*SLOT0123*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [9, 7, 7],
+      [HVX_FWD, HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_ab23f776, /*SLOT0,STORE*/
       [InstrStage<1, [SLOT0], 0>,
        InstrStage<1, [CVI_ST]>], [1, 2, 5],
@@ -2541,6 +2854,11 @@ class DepHVXItinV66 {
        InstrStage<1, [CVI_SHIFT, CVI_XLANE]>], [9, 5, 2],
       [HVX_FWD, HVX_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_cda936da, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 7],
+      [HVX_FWD, HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_d8287c14, /*SLOT23,VX_DV*/
       [InstrStage<1, [SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_MPY01]>], [9, 5, 5],
@@ -2551,6 +2869,11 @@ class DepHVXItinV66 {
        InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [9, 7, 7],
       [HVX_FWD, HVX_FWD, HVX_FWD]>,
 
+    InstrItinData <tc_dcca380f, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 5, 2],
+      [HVX_FWD, HVX_FWD, Hex_FWD]>,
+
     InstrItinData <tc_dd5b0695, /*SLOT01,ZW*/
       [InstrStage<1, [SLOT0, SLOT1], 0>,
        InstrStage<1, [CVI_ZW]>], [2, 1, 2],
@@ -2593,6 +2916,11 @@ class DepHVXItinV66 {
        InstrStage<1, [CVI_ST]>], [3, 2, 1, 2, 5],
       [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>,
 
+    InstrItinData <tc_f175e046, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5, 5, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
     InstrItinData <tc_f1de44ef, /*SLOT2,VX_DV*/
       [InstrStage<1, [SLOT2], 0>,
        InstrStage<1, [CVI_MPY01]>], [9, 5, 2],
@@ -2624,6 +2952,11 @@ class DepHVXItinV67 {
        InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 5],
       [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>,
 
+    InstrItinData <tc_0afc8be9, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 5],
+      [HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_0b04c6c7, /*SLOT23,VX_DV*/
       [InstrStage<1, [SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_MPY01]>], [9, 5, 2],
@@ -2679,6 +3012,10 @@ class DepHVXItinV67 {
        InstrStage<1, [CVI_ST]>], [3, 1, 2],
       [Hex_FWD, Hex_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_2120355e, /*SLOT0123*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [9, 7],
+      [HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_257f6f7c, /*SLOT0123,VA*/
       [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [9, 7, 7, 7],
@@ -2714,6 +3051,11 @@ class DepHVXItinV67 {
        InstrStage<1, [CVI_SHIFT]>], [9, 7, 5, 2],
       [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_37820f4c, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 5],
+      [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_3904b926, /*SLOT01,LOAD*/
       [InstrStage<1, [SLOT0, SLOT1], 0>,
        InstrStage<1, [CVI_LD]>], [9, 2, 1, 2],
@@ -2736,6 +3078,11 @@ class DepHVXItinV67 {
        InstrStage<1, [CVI_XLANE]>], [9, 3, 1, 2],
       [HVX_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_3c8c15d0, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5],
+      [HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_3ce09744, /*SLOT0,STORE*/
       [InstrStage<1, [SLOT0], 0>,
        InstrStage<1, [CVI_ST]>], [1, 2],
@@ -2764,6 +3111,11 @@ class DepHVXItinV67 {
        InstrStage<1, [CVI_XLANE]>], [9, 5, 5],
       [HVX_FWD, HVX_FWD, HVX_FWD]>,
 
+    InstrItinData <tc_4942646a, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 5, 5, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
     InstrItinData <tc_51d0ecc3, /*SLOT0123,VS*/
       [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_SHIFT]>], [9, 5],
@@ -2811,6 +3163,11 @@ class DepHVXItinV67 {
        InstrStage<1, [CVI_XLANE]>], [9, 2],
       [HVX_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_5cdf8c84, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7],
+      [HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_61bf7c03, /*SLOT23,4SLOT_MPY*/
       [InstrStage<1, [SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_ALL_NOMEM]>], [9, 5, 2],
@@ -2868,6 +3225,16 @@ class DepHVXItinV67 {
        InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [1, 2, 7, 7],
       [Hex_FWD, Hex_FWD, HVX_FWD, HVX_FWD]>,
 
+    InstrItinData <tc_72e2b393, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_73efe966, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 5, 5],
+      [HVX_FWD, HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_7417e785, /*SLOT0123,VS*/
       [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_SHIFT]>], [9, 5, 2],
@@ -2942,6 +3309,11 @@ class DepHVXItinV67 {
        InstrStage<1, [CVI_ZW]>], [3, 1, 2],
       [Hex_FWD, Hex_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_a19b9305, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 5, 5],
+      [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_a28f32b5, /*SLOT1,LOAD,VA*/
       [InstrStage<1, [SLOT1], 0>,
        InstrStage<1, [CVI_LD], 0>,
@@ -2961,6 +3333,10 @@ class DepHVXItinV67 {
        InstrStage<1, [CVI_XLANE]>], [9, 1, 2],
       [HVX_FWD, Hex_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_aa047364, /*SLOT0123*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [9, 7, 7],
+      [HVX_FWD, HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_ab23f776, /*SLOT0,STORE*/
       [InstrStage<1, [SLOT0], 0>,
        InstrStage<1, [CVI_ST]>], [1, 2, 5],
@@ -3042,6 +3418,11 @@ class DepHVXItinV67 {
        InstrStage<1, [CVI_SHIFT, CVI_XLANE]>], [9, 5, 2],
       [HVX_FWD, HVX_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_cda936da, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 7],
+      [HVX_FWD, HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_d8287c14, /*SLOT23,VX_DV*/
       [InstrStage<1, [SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_MPY01]>], [9, 5, 5],
@@ -3052,6 +3433,11 @@ class DepHVXItinV67 {
        InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [9, 7, 7],
       [HVX_FWD, HVX_FWD, HVX_FWD]>,
 
+    InstrItinData <tc_dcca380f, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 5, 2],
+      [HVX_FWD, HVX_FWD, Hex_FWD]>,
+
     InstrItinData <tc_dd5b0695, /*SLOT01,ZW*/
       [InstrStage<1, [SLOT0, SLOT1], 0>,
        InstrStage<1, [CVI_ZW]>], [2, 1, 2],
@@ -3094,6 +3480,11 @@ class DepHVXItinV67 {
        InstrStage<1, [CVI_ST]>], [3, 2, 1, 2, 5],
       [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>,
 
+    InstrItinData <tc_f175e046, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5, 5, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
     InstrItinData <tc_f1de44ef, /*SLOT2,VX_DV*/
       [InstrStage<1, [SLOT2], 0>,
        InstrStage<1, [CVI_MPY01]>], [9, 5, 2],
@@ -3125,6 +3516,575 @@ class DepHVXItinV68 {
        InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 5],
       [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>,
 
+    InstrItinData <tc_0afc8be9, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 5],
+      [HVX_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_0b04c6c7, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 5, 2],
+      [HVX_FWD, HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_0ec46cf9, /*SLOT0123,VA*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [9, 7],
+      [HVX_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_131f1c81, /*SLOT0,NOSLOT1,STORE,VP*/
+      [InstrStage<1, [SLOT0], 0>,
+       InstrStage<1, [SLOT1], 0>,
+       InstrStage<1, [CVI_ST], 0>,
+       InstrStage<1, [CVI_XLANE]>], [2, 1, 2, 5],
+      [Hex_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_1381a97c, /*SLOT0123,4SLOT*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_ALL]>], [],
+      []>,
+
+    InstrItinData <tc_15fdf750, /*SLOT23,VS_VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1], 0>,
+       InstrStage<1, [CVI_SHIFT, CVI_XLANE]>], [9, 7, 5, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_16ff9ef8, /*SLOT0123,VS*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_SHIFT]>], [9, 5, 5, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_191381c1, /*SLOT0,STORE,VA*/
+      [InstrStage<1, [SLOT0], 0>,
+       InstrStage<1, [CVI_ST], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [3, 7, 1, 2, 7],
+      [Hex_FWD, HVX_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_1ad8a370, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 5, 2, 2],
+      [HVX_FWD, HVX_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_1ba8a0cd, /*SLOT01,LOAD,VA*/
+      [InstrStage<1, [SLOT0, SLOT1], 0>,
+       InstrStage<1, [CVI_LD], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [9, 3, 1, 2],
+      [HVX_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_20a4bbec, /*SLOT0,STORE*/
+      [InstrStage<1, [SLOT0], 0>,
+       InstrStage<1, [CVI_ST]>], [3, 1, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_2120355e, /*SLOT0123*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [9, 7],
+      [HVX_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_257f6f7c, /*SLOT0123,VA*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [9, 7, 7, 7],
+      [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_26a377fe, /*SLOT23,4SLOT_MPY*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_ALL_NOMEM]>], [9, 3, 5, 2],
+      [HVX_FWD, Hex_FWD, HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_2b4c548e, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 5, 5, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_2c745bb8, /*SLOT0123,VP_VS*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_XLSHF]>], [9, 7, 5],
+      [HVX_FWD, HVX_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_2d4051cd, /*SLOT23,4SLOT_MPY*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_ALL_NOMEM]>], [9, 3, 7, 5, 2],
+      [HVX_FWD, Hex_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_2e8f5f6e, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 7, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_309dbb4f, /*SLOT0123,VS*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_SHIFT]>], [9, 7, 5, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_37820f4c, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 5],
+      [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_3904b926, /*SLOT01,LOAD*/
+      [InstrStage<1, [SLOT0, SLOT1], 0>,
+       InstrStage<1, [CVI_LD]>], [9, 2, 1, 2],
+      [HVX_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_3aacf4a8, /*SLOT0123,VA*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [9, 2, 7],
+      [HVX_FWD, Hex_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_3ad719fb, /*SLOT01,ZW*/
+      [InstrStage<1, [SLOT0, SLOT1], 0>,
+       InstrStage<1, [CVI_ZW]>], [3, 2, 1, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_3c56e5ce, /*SLOT0,NOSLOT1,LOAD,VP*/
+      [InstrStage<1, [SLOT0], 0>,
+       InstrStage<1, [SLOT1], 0>,
+       InstrStage<1, [CVI_LD], 0>,
+       InstrStage<1, [CVI_XLANE]>], [9, 3, 1, 2],
+      [HVX_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_3c8c15d0, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5],
+      [HVX_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_3ce09744, /*SLOT0,STORE*/
+      [InstrStage<1, [SLOT0], 0>,
+       InstrStage<1, [CVI_ST]>], [1, 2],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_3e2aaafc, /*SLOT0,STORE,VA*/
+      [InstrStage<1, [SLOT0], 0>,
+       InstrStage<1, [CVI_ST], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [3, 1, 2, 7],
+      [Hex_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_447d9895, /*SLOT0,STORE,VA*/
+      [InstrStage<1, [SLOT0], 0>,
+       InstrStage<1, [CVI_ST], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [7, 1, 2, 7],
+      [HVX_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_453fe68d, /*SLOT01,LOAD,VA*/
+      [InstrStage<1, [SLOT0, SLOT1], 0>,
+       InstrStage<1, [CVI_LD], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [9, 3, 2, 1, 2],
+      [HVX_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_46d6c3e0, /*SLOT0123,VP*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_XLANE]>], [9, 5, 5],
+      [HVX_FWD, HVX_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_4942646a, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 5, 5, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_51d0ecc3, /*SLOT0123,VS*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_SHIFT]>], [9, 5],
+      [HVX_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_52447ecc, /*SLOT01,LOAD*/
+      [InstrStage<1, [SLOT0, SLOT1], 0>,
+       InstrStage<1, [CVI_LD]>], [9, 1, 2],
+      [HVX_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_540c3da3, /*SLOT0,VA*/
+      [InstrStage<1, [SLOT0], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [4, 7, 1],
+      [Hex_FWD, HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_54a0dc47, /*SLOT0,STORE,VA*/
+      [InstrStage<1, [SLOT0], 0>,
+       InstrStage<1, [CVI_ST], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [3, 2, 1, 2, 7],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_561aaa58, /*SLOT0123,VP_VS*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_XLSHF]>], [9, 9, 5, 5, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_56c4f9fe, /*SLOT0123,VA*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [9, 7, 7],
+      [HVX_FWD, HVX_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_56e64202, /*SLOT0123,VP*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_XLANE]>], [9, 5, 5, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_58d21193, /*SLOT0,STORE,VA_DV*/
+      [InstrStage<1, [SLOT0], 0>,
+       InstrStage<1, [CVI_ST], 0>,
+       InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [7, 1, 2, 7, 7],
+      [HVX_FWD, Hex_FWD, Hex_FWD, HVX_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_5bf8afbb, /*SLOT0123,VP*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_XLANE]>], [9, 2],
+      [HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_5cdf8c84, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7],
+      [HVX_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_61bf7c03, /*SLOT23,4SLOT_MPY*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_ALL_NOMEM]>], [9, 5, 2],
+      [HVX_FWD, HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_649072c2, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5, 2],
+      [HVX_FWD, HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_660769f1, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_663c80a7, /*SLOT01,LOAD*/
+      [InstrStage<1, [SLOT0, SLOT1], 0>,
+       InstrStage<1, [CVI_LD]>], [9, 3, 1, 2],
+      [HVX_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_6942b6e0, /*SLOT0,STORE*/
+      [InstrStage<1, [SLOT0], 0>,
+       InstrStage<1, [CVI_ST]>], [3, 1, 2, 5],
+      [Hex_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_6e7fa133, /*SLOT0123,VP*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_XLANE]>], [9, 5, 2],
+      [HVX_FWD, HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_7095ecba, /*SLOT01,LOAD,VA_DV*/
+      [InstrStage<1, [SLOT0, SLOT1], 0>,
+       InstrStage<1, [CVI_LD], 0>,
+       InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [1, 2, 7],
+      [Hex_FWD, Hex_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_71646d06, /*SLOT0123,VA_DV*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [9, 7, 7, 7],
+      [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_7177e272, /*SLOT0,STORE*/
+      [InstrStage<1, [SLOT0], 0>,
+       InstrStage<1, [CVI_ST]>], [2, 1, 2, 5],
+      [Hex_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_718b5c53, /*SLOT0123,VA_DV*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [9],
+      [HVX_FWD]>,
+
+    InstrItinData <tc_7273323b, /*SLOT0,STORE,VA_DV*/
+      [InstrStage<1, [SLOT0], 0>,
+       InstrStage<1, [CVI_ST], 0>,
+       InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [1, 2, 7, 7],
+      [Hex_FWD, Hex_FWD, HVX_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_72e2b393, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_73efe966, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 5, 5],
+      [HVX_FWD, HVX_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_7417e785, /*SLOT0123,VS*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_SHIFT]>], [9, 5, 2],
+      [HVX_FWD, HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_767c4e9d, /*SLOT0123,4SLOT*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_ALL]>], [3, 2],
+      [HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_7d68d5c2, /*SLOT01,LOAD,VA*/
+      [InstrStage<1, [SLOT0, SLOT1], 0>,
+       InstrStage<1, [CVI_LD], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [7, 1, 2, 7],
+      [HVX_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_7e6a3e89, /*SLOT0123,VA*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [9, 9, 7, 7, 7],
+      [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_8772086c, /*SLOT0123,VA*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [9, 7, 7],
+      [HVX_FWD, HVX_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_87adc037, /*SLOT0123,VP_VS*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_XLSHF]>], [9, 5, 5, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_8e420e4d, /*SLOT0,STORE,VA*/
+      [InstrStage<1, [SLOT0], 0>,
+       InstrStage<1, [CVI_ST], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [7, 1, 2, 7, 7],
+      [HVX_FWD, Hex_FWD, Hex_FWD, HVX_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_90bcc1db, /*SLOT2,VX_DV*/
+      [InstrStage<1, [SLOT2], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 5, 5, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_933f2b39, /*SLOT23,4SLOT_MPY*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_ALL_NOMEM]>], [9, 7, 5, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_946013d8, /*SLOT0123,VP*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_XLANE]>], [9, 5],
+      [HVX_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_9d1dc972, /*SLOT0123,VP_VS*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_XLSHF]>], [9, 7, 5, 5, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_9f363d21, /*SLOT0,STORE,VA*/
+      [InstrStage<1, [SLOT0], 0>,
+       InstrStage<1, [CVI_ST], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [1, 2, 7, 7],
+      [Hex_FWD, Hex_FWD, HVX_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_a02a10a8, /*SLOT0,STORE,VA*/
+      [InstrStage<1, [SLOT0], 0>,
+       InstrStage<1, [CVI_ST], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [2, 1, 2, 7],
+      [Hex_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_a0dbea28, /*SLOT01,ZW*/
+      [InstrStage<1, [SLOT0, SLOT1], 0>,
+       InstrStage<1, [CVI_ZW]>], [3, 1, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_a19b9305, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 5, 5],
+      [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_a28f32b5, /*SLOT01,LOAD,VA*/
+      [InstrStage<1, [SLOT0, SLOT1], 0>,
+       InstrStage<1, [CVI_LD], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [1, 2, 7],
+      [Hex_FWD, Hex_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_a69eeee1, /*SLOT01,LOAD,VA_DV*/
+      [InstrStage<1, [SLOT0, SLOT1], 0>,
+       InstrStage<1, [CVI_LD], 0>,
+       InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [7, 1, 2, 7],
+      [HVX_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_a7e6707d, /*SLOT0,NOSLOT1,LOAD,VP*/
+      [InstrStage<1, [SLOT0], 0>,
+       InstrStage<1, [SLOT1], 0>,
+       InstrStage<1, [CVI_LD], 0>,
+       InstrStage<1, [CVI_XLANE]>], [9, 1, 2],
+      [HVX_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_aa047364, /*SLOT0123*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [9, 7, 7],
+      [HVX_FWD, HVX_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_ab23f776, /*SLOT0,STORE*/
+      [InstrStage<1, [SLOT0], 0>,
+       InstrStage<1, [CVI_ST]>], [1, 2, 5],
+      [Hex_FWD, Hex_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_abe8c3b2, /*SLOT01,LOAD,VA*/
+      [InstrStage<1, [SLOT0, SLOT1], 0>,
+       InstrStage<1, [CVI_LD], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [9, 2, 1, 2],
+      [HVX_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_ac4046bc, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 2],
+      [HVX_FWD, HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_af25efd9, /*SLOT0123,VA_DV*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [9, 2, 7, 7],
+      [HVX_FWD, Hex_FWD, HVX_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_b091f1c6, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 5, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_b28e51aa, /*SLOT0123,4SLOT*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_ALL]>], [2],
+      [Hex_FWD]>,
+
+    InstrItinData <tc_b4416217, /*SLOT0123,VA_DV*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [9, 7],
+      [HVX_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_b9db8205, /*SLOT01,LOAD*/
+      [InstrStage<1, [SLOT0, SLOT1], 0>,
+       InstrStage<1, [CVI_LD]>], [9, 3, 2, 1, 2],
+      [HVX_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_bb599486, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 5, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_c0749f3c, /*SLOT01,LOAD,VA*/
+      [InstrStage<1, [SLOT0, SLOT1], 0>,
+       InstrStage<1, [CVI_LD], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [9, 1, 2],
+      [HVX_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_c127de3a, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5, 5],
+      [HVX_FWD, HVX_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_c4edf264, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 2],
+      [HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_c5dba46e, /*SLOT0,STORE,VA*/
+      [InstrStage<1, [SLOT0], 0>,
+       InstrStage<1, [CVI_ST], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [1, 2, 7],
+      [Hex_FWD, Hex_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_c7039829, /*SLOT0,NOSLOT1,STORE,VP*/
+      [InstrStage<1, [SLOT0], 0>,
+       InstrStage<1, [SLOT1], 0>,
+       InstrStage<1, [CVI_ST], 0>,
+       InstrStage<1, [CVI_XLANE]>], [3, 2, 1, 2, 5],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_cd94bfe0, /*SLOT23,VS_VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1], 0>,
+       InstrStage<1, [CVI_SHIFT, CVI_XLANE]>], [9, 5, 2],
+      [HVX_FWD, HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_cda936da, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 7],
+      [HVX_FWD, HVX_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_d8287c14, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 5, 5],
+      [HVX_FWD, HVX_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_db5555f3, /*SLOT0123,VA_DV*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [9, 7, 7],
+      [HVX_FWD, HVX_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_dcca380f, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 5, 2],
+      [HVX_FWD, HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_dd5b0695, /*SLOT01,ZW*/
+      [InstrStage<1, [SLOT0, SLOT1], 0>,
+       InstrStage<1, [CVI_ZW]>], [2, 1, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_df80eeb0, /*SLOT0123,VP_VS*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_XLSHF]>], [9, 7, 5, 5],
+      [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_e2d2e9e5, /*SLOT0,NOSLOT1,STORE,VP*/
+      [InstrStage<1, [SLOT0], 0>,
+       InstrStage<1, [SLOT1], 0>,
+       InstrStage<1, [CVI_ST], 0>,
+       InstrStage<1, [CVI_XLANE]>], [3, 1, 2, 5],
+      [Hex_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_e35c1e93, /*SLOT0123,VA*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [9, 9, 7, 7],
+      [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_e3f68a46, /*SLOT0123,4SLOT*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_ALL]>], [3],
+      [HVX_FWD]>,
+
+    InstrItinData <tc_e675c45a, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 2, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_e699ae41, /*SLOT01,ZW*/
+      [InstrStage<1, [SLOT0, SLOT1], 0>,
+       InstrStage<1, [CVI_ZW]>], [1, 2],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_e99d4c2e, /*SLOT0,STORE*/
+      [InstrStage<1, [SLOT0], 0>,
+       InstrStage<1, [CVI_ST]>], [3, 2, 1, 2, 5],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_f175e046, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5, 5, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_f1de44ef, /*SLOT2,VX_DV*/
+      [InstrStage<1, [SLOT2], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 5, 2],
+      [HVX_FWD, HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_f21e8abb, /*SLOT0,NOSLOT1,STORE,VP*/
+      [InstrStage<1, [SLOT0], 0>,
+       InstrStage<1, [SLOT1], 0>,
+       InstrStage<1, [CVI_ST], 0>,
+       InstrStage<1, [CVI_XLANE]>], [1, 2, 5],
+      [Hex_FWD, Hex_FWD, HVX_FWD]>
+  ];
+}
+
+class DepHVXItinV69 {
+  list<InstrItinData> DepHVXItinV69_list = [
+    InstrItinData <tc_04da405a, /*SLOT0123,VP_VS*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_XLSHF]>], [9, 5],
+      [HVX_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_05ca8cfd, /*SLOT0123,VS*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_SHIFT]>], [9, 5, 5],
+      [HVX_FWD, HVX_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_08a4f1b6, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 5],
+      [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>,
+
+    InstrItinData <tc_0afc8be9, /*SLOT23,VX_DV*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY01]>], [9, 5],
+      [HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_0b04c6c7, /*SLOT23,VX_DV*/
       [InstrStage<1, [SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_MPY01]>], [9, 5, 2],
@@ -3180,6 +4140,10 @@ class DepHVXItinV68 {
        InstrStage<1, [CVI_ST]>], [3, 1, 2],
       [Hex_FWD, Hex_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_2120355e, /*SLOT0123*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [9, 7],
+      [HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_257f6f7c, /*SLOT0123,VA*/
       [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [9, 7, 7, 7],
@@ -3215,6 +4179,11 @@ class DepHVXItinV68 {
        InstrStage<1, [CVI_SHIFT]>], [9, 7, 5, 2],
       [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_37820f4c, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 5, 5],
+      [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_3904b926, /*SLOT01,LOAD*/
       [InstrStage<1, [SLOT0, SLOT1], 0>,
        InstrStage<1, [CVI_LD]>], [9, 2, 1, 2],
@@ -3237,6 +4206,11 @@ class DepHVXItinV68 {
        InstrStage<1, [CVI_XLANE]>], [9, 3, 1, 2],
       [HVX_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_3c8c15d0, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5],
+      [HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_3ce09744, /*SLOT0,STORE*/
       [InstrStage<1, [SLOT0], 0>,
        InstrStage<1, [CVI_ST]>], [1, 2],
@@ -3265,6 +4239,11 @@ class DepHVXItinV68 {
        InstrStage<1, [CVI_XLANE]>], [9, 5, 5],
       [HVX_FWD, HVX_FWD, HVX_FWD]>,
 
+    InstrItinData <tc_4942646a, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 5, 5, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
     InstrItinData <tc_51d0ecc3, /*SLOT0123,VS*/
       [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_SHIFT]>], [9, 5],
@@ -3312,6 +4291,11 @@ class DepHVXItinV68 {
        InstrStage<1, [CVI_XLANE]>], [9, 2],
       [HVX_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_5cdf8c84, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7],
+      [HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_61bf7c03, /*SLOT23,4SLOT_MPY*/
       [InstrStage<1, [SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_ALL_NOMEM]>], [9, 5, 2],
@@ -3369,6 +4353,16 @@ class DepHVXItinV68 {
        InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [1, 2, 7, 7],
       [Hex_FWD, Hex_FWD, HVX_FWD, HVX_FWD]>,
 
+    InstrItinData <tc_72e2b393, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 5, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_73efe966, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5, 5],
+      [HVX_FWD, HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_7417e785, /*SLOT0123,VS*/
       [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_SHIFT]>], [9, 5, 2],
@@ -3443,6 +4437,11 @@ class DepHVXItinV68 {
        InstrStage<1, [CVI_ZW]>], [3, 1, 2],
       [Hex_FWD, Hex_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_a19b9305, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 5, 5],
+      [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_a28f32b5, /*SLOT01,LOAD,VA*/
       [InstrStage<1, [SLOT0, SLOT1], 0>,
        InstrStage<1, [CVI_LD], 0>,
@@ -3462,6 +4461,10 @@ class DepHVXItinV68 {
        InstrStage<1, [CVI_XLANE]>], [9, 1, 2],
       [HVX_FWD, Hex_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_aa047364, /*SLOT0123*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [9, 7, 7],
+      [HVX_FWD, HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_ab23f776, /*SLOT0,STORE*/
       [InstrStage<1, [SLOT0], 0>,
        InstrStage<1, [CVI_ST]>], [1, 2, 5],
@@ -3543,6 +4546,11 @@ class DepHVXItinV68 {
        InstrStage<1, [CVI_SHIFT, CVI_XLANE]>], [9, 5, 2],
       [HVX_FWD, HVX_FWD, Hex_FWD]>,
 
+    InstrItinData <tc_cda936da, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 7],
+      [HVX_FWD, HVX_FWD, HVX_FWD]>,
+
     InstrItinData <tc_d8287c14, /*SLOT23,VX_DV*/
       [InstrStage<1, [SLOT2, SLOT3], 0>,
        InstrStage<1, [CVI_MPY01]>], [9, 5, 5],
@@ -3553,6 +4561,11 @@ class DepHVXItinV68 {
        InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [9, 7, 7],
       [HVX_FWD, HVX_FWD, HVX_FWD]>,
 
+    InstrItinData <tc_dcca380f, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5, 2],
+      [HVX_FWD, HVX_FWD, Hex_FWD]>,
+
     InstrItinData <tc_dd5b0695, /*SLOT01,ZW*/
       [InstrStage<1, [SLOT0, SLOT1], 0>,
        InstrStage<1, [CVI_ZW]>], [2, 1, 2],
@@ -3595,6 +4608,11 @@ class DepHVXItinV68 {
        InstrStage<1, [CVI_ST]>], [3, 2, 1, 2, 5],
       [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>,
 
+    InstrItinData <tc_f175e046, /*SLOT23,VX*/
+      [InstrStage<1, [SLOT2, SLOT3], 0>,
+       InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5, 5, 2],
+      [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>,
+
     InstrItinData <tc_f1de44ef, /*SLOT2,VX_DV*/
       [InstrStage<1, [SLOT2], 0>,
        InstrStage<1, [CVI_MPY01]>], [9, 5, 2],
diff --git a/llvm/lib/Target/Hexagon/HexagonDepIICScalar.td b/llvm/lib/Target/Hexagon/HexagonDepIICScalar.td
index a3766652794b..a979bafe8e33 100644
--- a/llvm/lib/Target/Hexagon/HexagonDepIICScalar.td
+++ b/llvm/lib/Target/Hexagon/HexagonDepIICScalar.td
@@ -7338,3 +7338,771 @@ class DepScalarItinV68 {
       [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>
   ];
 }
+
+class DepScalarItinV69 {
+  list<InstrItinData> DepScalarItinV69_list = [
+    InstrItinData <tc_011e0e9d, /*tc_st*/
+      [InstrStage<1, [SLOT0]>], [2, 1, 2, 3],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_01d44cb2, /*tc_2*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [4, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_01e1be3b, /*tc_3x*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [4, 2, 1, 1],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_02fe1c65, /*tc_4x*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [5, 1, 1],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_0655b949, /*tc_st*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [2, 3],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_075c8dd8, /*tc_ld*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [4, 3, 1, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_0a195f2c, /*tc_4x*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [5, 2, 1, 1],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_0a6c20ae, /*tc_st*/
+      [InstrStage<1, [SLOT0]>], [2, 1, 1, 2, 3],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_0ba0d5da, /*tc_3stall*/
+      [InstrStage<1, [SLOT2]>], [1],
+      [Hex_FWD]>,
+
+    InstrItinData <tc_0dfac0a7, /*tc_2*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [4, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_0fac1eb8, /*tc_st*/
+      [InstrStage<1, [SLOT0]>], [3, 2, 3],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_112d30d6, /*tc_1*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [2],
+      [Hex_FWD]>,
+
+    InstrItinData <tc_1242dc2a, /*tc_ld*/
+      [InstrStage<1, [SLOT0]>], [2],
+      [Hex_FWD]>,
+
+    InstrItinData <tc_1248597c, /*tc_3x*/
+      [InstrStage<1, [SLOT3]>], [2, 2],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_14ab4f41, /*tc_newvjump*/
+      [InstrStage<1, [SLOT0]>], [3, 3, 1],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_151bf368, /*tc_1*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [3, 2],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_158aa3f7, /*tc_st*/
+      [InstrStage<1, [SLOT0]>], [1, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_197dce51, /*tc_3x*/
+      [InstrStage<1, [SLOT3]>], [4, 2, 1, 1],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_1981450d, /*tc_newvjump*/
+      [InstrStage<1, [SLOT0]>], [3],
+      [Hex_FWD]>,
+
+    InstrItinData <tc_1c2c7a4a, /*tc_1*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [3, 2, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_1c7522a8, /*tc_ld*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [4, 3, 2, 1, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_1d41f8b7, /*tc_1*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [3, 4, 2, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_1fcb8495, /*tc_2*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [4, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_1fe4ab69, /*tc_st*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [2, 1, 1, 2, 3],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_20131976, /*tc_2*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [4, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_2237d952, /*tc_ld*/
+      [InstrStage<1, [SLOT0]>], [1, 2],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_23708a21, /*tc_1*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [],
+      []>,
+
+    InstrItinData <tc_2471c1c8, /*tc_ld*/
+      [InstrStage<1, [SLOT0]>], [4, 1],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_24e109c7, /*tc_newvjump*/
+      [InstrStage<1, [SLOT0]>], [3, 3, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_24f426ab, /*tc_1*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [2, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_280f7fe1, /*tc_st*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [1, 1, 2, 3],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_28e55c6f, /*tc_3x*/
+      [InstrStage<1, [SLOT3]>], [1, 1],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_2c13e7f5, /*tc_2*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [4, 2, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_2c3e17fc, /*tc_3x*/
+      [InstrStage<1, [SLOT3]>], [1],
+      [Hex_FWD]>,
+
+    InstrItinData <tc_2f573607, /*tc_1*/
+      [InstrStage<1, [SLOT2]>], [2, 2],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_362b0be2, /*tc_3*/
+      [InstrStage<1, [SLOT2]>], [1],
+      [Hex_FWD]>,
+
+    InstrItinData <tc_38382228, /*tc_3x*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [4, 1, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_388f9897, /*tc_1*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [3, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_38e0bae9, /*tc_3x*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [4, 4, 2, 1, 1],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_3d14a17b, /*tc_1*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [3, 2],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_3edca78f, /*tc_2*/
+      [InstrStage<1, [SLOT3]>], [4, 2],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_3fbf1042, /*tc_1*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [3],
+      [Hex_FWD]>,
+
+    InstrItinData <tc_407e96f9, /*tc_1*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [3, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_40d64c94, /*tc_newvjump*/
+      [InstrStage<1, [SLOT0]>], [3, 1],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_4222e6bf, /*tc_ld*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [4, 1, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_42ff66ba, /*tc_1*/
+      [InstrStage<1, [SLOT2]>], [2, 2],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_442395f3, /*tc_2latepred*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [4, 3, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_449acf79, /*tc_latepredstaia*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [4, 3, 1, 2, 1],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_44d5a428, /*tc_st*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [1, 2],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_44fffc58, /*tc_3*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [2],
+      [Hex_FWD]>,
+
+    InstrItinData <tc_45791fb8, /*tc_ld*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [4, 2, 1, 1, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_45f9d1be, /*tc_2early*/
+      [InstrStage<1, [SLOT2]>], [2],
+      [Hex_FWD]>,
+
+    InstrItinData <tc_49fdfd4b, /*tc_3stall*/
+      [InstrStage<1, [SLOT3]>], [4, 1],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_4a55d03c, /*tc_1*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [3, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_4abdbdc6, /*tc_3x*/
+      [InstrStage<1, [SLOT3]>], [2, 2],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_4ac61d92, /*tc_2latepred*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [4, 3, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_4bf903b0, /*tc_st*/
+      [InstrStage<1, [SLOT0]>], [3],
+      [Hex_FWD]>,
+
+    InstrItinData <tc_503ce0f3, /*tc_3x*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [4, 2, 2, 1],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_53c851ab, /*tc_3stall*/
+      [InstrStage<1, [SLOT2]>], [4, 1, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_5502c366, /*tc_1*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [3, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_55255f2b, /*tc_3stall*/
+      [InstrStage<1, [SLOT3]>], [],
+      []>,
+
+    InstrItinData <tc_556f6577, /*tc_3x*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [4, 1, 1],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_55a9a350, /*tc_st*/
+      [InstrStage<1, [SLOT0]>], [1, 2, 2, 3],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_55b33fda, /*tc_1*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [3, 2],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_56a124a7, /*tc_1*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [2, 2],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_57a55b54, /*tc_1*/
+      [InstrStage<1, [SLOT3]>], [2, 2],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_5944960d, /*tc_ld*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [1, 1, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_59a7822c, /*tc_1*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [2, 2],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_5a4b5e58, /*tc_3x*/
+      [InstrStage<1, [SLOT3]>], [4, 1, 1],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_5b347363, /*tc_1*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [3, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_5ceb2f9e, /*tc_ld*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [4, 3, 1, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_5da50c4b, /*tc_1*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [3, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_5deb5e47, /*tc_st*/
+      [InstrStage<1, [SLOT0]>], [1, 2, 3],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_5e4cf0e8, /*tc_2*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [4, 2, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_5f2afaf7, /*tc_latepredldaia*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [4, 4, 3, 1, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_60e324ff, /*tc_1*/
+      [InstrStage<1, [SLOT2]>], [2],
+      [Hex_FWD]>,
+
+    InstrItinData <tc_63567288, /*tc_2latepred*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [4],
+      [Hex_FWD]>,
+
+    InstrItinData <tc_64b00d8a, /*tc_ld*/
+      [InstrStage<1, [SLOT0]>], [4, 1],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_651cbe02, /*tc_1*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [3, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_65279839, /*tc_2*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [4, 2],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_65cbd974, /*tc_st*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [3, 1, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_69bfb303, /*tc_3*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [2, 2],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_6ae3426b, /*tc_3x*/
+      [InstrStage<1, [SLOT3]>], [4, 1],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_6d861a95, /*tc_3x*/
+      [InstrStage<1, [SLOT3]>], [2, 1],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_6e20402a, /*tc_st*/
+      [InstrStage<1, [SLOT0]>], [2, 3],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_6f42bc60, /*tc_3stall*/
+      [InstrStage<1, [SLOT0]>], [4, 1, 1],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_6fc5dbea, /*tc_1*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [3, 2, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_711c805f, /*tc_1*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [2, 2],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_713b66bf, /*tc_1*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [3, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_7401744f, /*tc_2*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [4, 4, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_7476d766, /*tc_3stall*/
+      [InstrStage<1, [SLOT3]>], [4, 2],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_74a42bda, /*tc_ld*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [3, 1, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_76bb5435, /*tc_ld*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [4, 3, 2, 1, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_77f94a5e, /*tc_st*/
+      [InstrStage<1, [SLOT0]>], [],
+      []>,
+
+    InstrItinData <tc_788b1d09, /*tc_3x*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [4, 1, 1, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_7af3a37e, /*tc_st*/
+      [InstrStage<1, [SLOT0]>], [1, 3],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_7b9187d3, /*tc_newvjump*/
+      [InstrStage<1, [SLOT0]>], [3, 2],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_7c31e19a, /*tc_st*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [1, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_7c6d32e4, /*tc_ld*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [4, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_7f7f45f5, /*tc_4x*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [5, 5, 1],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_7f8ae742, /*tc_3x*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [4, 2, 1, 1],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_8035e91f, /*tc_st*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [2, 1, 2, 3],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_822c3c68, /*tc_ld*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [4, 3, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_829d8a86, /*tc_st*/
+      [InstrStage<1, [SLOT0]>], [3, 1, 1, 2, 3],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_838c4d7a, /*tc_st*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [1, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_84a7500d, /*tc_2*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [4, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_86173609, /*tc_2latepred*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [4, 3, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_887d1bb7, /*tc_st*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [1, 2, 2, 3],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_8a6d0d94, /*tc_ld*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [4, 2],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_8a825db2, /*tc_2*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [4, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_8b5bd4f5, /*tc_2*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [4, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_8e82e8ca, /*tc_st*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [3, 1, 1, 2, 3],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_9124c04f, /*tc_1*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [3, 2],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_92240447, /*tc_st*/
+      [InstrStage<1, [SLOT0]>], [3, 1, 2, 3],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_934753bb, /*tc_ld*/
+      [InstrStage<1, [SLOT0]>], [3, 1, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_937dd41c, /*tc_ld*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [],
+      []>,
+
+    InstrItinData <tc_9406230a, /*tc_3x*/
+      [InstrStage<1, [SLOT3]>], [2, 1],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_95a33176, /*tc_2*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [4, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_96ef76ef, /*tc_st*/
+      [InstrStage<1, [SLOT0]>], [1, 1, 2, 3],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_975a4e54, /*tc_newvjump*/
+      [InstrStage<1, [SLOT0]>], [3, 3, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_9783714b, /*tc_4x*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [5, 1],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_9b34f5e0, /*tc_3stall*/
+      [InstrStage<1, [SLOT2]>], [],
+      []>,
+
+    InstrItinData <tc_9b3c0462, /*tc_2*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [4, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_9bcfb2ee, /*tc_st*/
+      [InstrStage<1, [SLOT0]>], [1, 2, 3],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_9c52f549, /*tc_1*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [3, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_9e27f2f9, /*tc_1*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [2, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_9e72dc89, /*tc_4x*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [5, 2, 1, 1],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_9edb7c77, /*tc_4x*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [5, 2, 1, 1, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_9edefe01, /*tc_st*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [3, 2, 1, 2, 3],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_9f6cd987, /*tc_1*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [3, 2],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_a08b630b, /*tc_2*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [4, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_a1297125, /*tc_1*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [3, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_a154b476, /*tc_3x*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [4, 2, 1, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_a2b365d2, /*tc_st*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [3, 1, 2, 3],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_a3070909, /*tc_3stall*/
+      [InstrStage<1, [SLOT0]>], [1, 1],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_a32e03e7, /*tc_ld*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [4, 2, 1, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_a38c45dc, /*tc_3x*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [4, 2, 1, 1, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_a4e22bbd, /*tc_2*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [4, 2, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_a4ee89db, /*tc_2early*/
+      [InstrStage<1, [SLOT0]>], [],
+      []>,
+
+    InstrItinData <tc_a7a13fac, /*tc_1*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [3, 2, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_a7bdb22c, /*tc_2*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [4, 2],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_a9edeffa, /*tc_st*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [1, 2, 3],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_abfd9a6d, /*tc_ld*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [4, 1, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_ac65613f, /*tc_ld*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [4, 3, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_addc37a8, /*tc_st*/
+      [InstrStage<1, [SLOT0]>], [3, 1, 2, 2, 3],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_ae5babd7, /*tc_st*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [1, 2, 3],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_aee6250c, /*tc_ld*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [4, 1],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_b1ae5f67, /*tc_st*/
+      [InstrStage<1, [SLOT0]>], [1],
+      [Hex_FWD]>,
+
+    InstrItinData <tc_b4dc7630, /*tc_st*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [3, 1, 2, 2, 3],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_b7c4062a, /*tc_ld*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [4, 3, 1, 1, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_b837298f, /*tc_1*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [],
+      []>,
+
+    InstrItinData <tc_ba9255a6, /*tc_st*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [2, 2, 3],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_bb07f2c5, /*tc_st*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [3, 2, 3],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_bb831a7c, /*tc_2*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [4, 2, 2, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_bf2ffc0f, /*tc_ld*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [4, 1, 1, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_c20701f0, /*tc_2*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [4, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_c21d7447, /*tc_3x*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [4, 1, 1],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_c57d9f39, /*tc_1*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [3, 2],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_c818ff7f, /*tc_newvjump*/
+      [InstrStage<1, [SLOT0]>], [],
+      []>,
+
+    InstrItinData <tc_ce59038e, /*tc_st*/
+      [InstrStage<1, [SLOT0]>], [3, 2, 1, 2, 3],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_cfa0e29b, /*tc_st*/
+      [InstrStage<1, [SLOT0]>], [2, 2, 3],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_d03278fd, /*tc_st*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [2, 1, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_d33e5eee, /*tc_1*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [3, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_d3632d88, /*tc_2*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [4, 2],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_d45ba9cd, /*tc_ld*/
+      [InstrStage<1, [SLOT0]>], [1],
+      [Hex_FWD]>,
+
+    InstrItinData <tc_d57d649c, /*tc_3stall*/
+      [InstrStage<1, [SLOT2]>], [2],
+      [Hex_FWD]>,
+
+    InstrItinData <tc_d61dfdc3, /*tc_2*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [4, 2],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_d68dca5c, /*tc_3stall*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [4, 1, 1],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_d7718fbe, /*tc_3x*/
+      [InstrStage<1, [SLOT3]>], [1],
+      [Hex_FWD]>,
+
+    InstrItinData <tc_db596beb, /*tc_3x*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [4, 1, 1],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_db96aa6b, /*tc_st*/
+      [InstrStage<1, [SLOT0]>], [1],
+      [Hex_FWD]>,
+
+    InstrItinData <tc_dc51281d, /*tc_3*/
+      [InstrStage<1, [SLOT2]>], [2, 1],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_decdde8a, /*tc_1*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [2],
+      [Hex_FWD]>,
+
+    InstrItinData <tc_df5d53f9, /*tc_newvjump*/
+      [InstrStage<1, [SLOT0]>], [3, 2, 1],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_e3d699e3, /*tc_2*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [4, 2],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_e9170fb7, /*tc_ld*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [4, 1],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_ed03645c, /*tc_1*/
+      [InstrStage<1, [SLOT2]>], [3, 2],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_eed07714, /*tc_ld*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [4, 1, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_eeda4109, /*tc_1*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [3, 2],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_ef921005, /*tc_1*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [3, 2],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_f098b237, /*tc_2*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [4, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_f0cdeccf, /*tc_3x*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [4, 1, 1, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_f0e8e832, /*tc_4x*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [5, 1, 1],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_f34c1c21, /*tc_2*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [4, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_f38f92e1, /*tc_newvjump*/
+      [InstrStage<1, [SLOT0]>], [2],
+      [Hex_FWD]>,
+
+    InstrItinData <tc_f529831b, /*tc_latepredstaia*/
+      [InstrStage<1, [SLOT0]>], [4, 3, 1, 2, 3],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_f6e2aff9, /*tc_newvjump*/
+      [InstrStage<1, [SLOT0]>], [3, 2, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_f7569068, /*tc_4x*/
+      [InstrStage<1, [SLOT2, SLOT3]>], [5, 5, 1, 1],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_f999c66e, /*tc_1*/
+      [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [2, 2],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_fae9dfa5, /*tc_3x*/
+      [InstrStage<1, [SLOT3]>], [4, 2],
+      [Hex_FWD, Hex_FWD]>,
+
+    InstrItinData <tc_fedb7e19, /*tc_ld*/
+      [InstrStage<1, [SLOT0, SLOT1]>], [4, 2, 1, 2],
+      [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>
+  ];
+}
diff --git a/llvm/lib/Target/Hexagon/HexagonDepInstrFormats.td b/llvm/lib/Target/Hexagon/HexagonDepInstrFormats.td
index b3f1b6638193..65d36924ba48 100644
--- a/llvm/lib/Target/Hexagon/HexagonDepInstrFormats.td
+++ b/llvm/lib/Target/Hexagon/HexagonDepInstrFormats.td
@@ -2288,6 +2288,12 @@ class Enc_a30110 : OpcodeHexagon {
   bits <5> Vd32;
   let Inst{4-0} = Vd32{4-0};
 }
+class Enc_a33d04 : OpcodeHexagon {
+  bits <5> Vuu32;
+  let Inst{12-8} = Vuu32{4-0};
+  bits <5> Vd32;
+  let Inst{4-0} = Vd32{4-0};
+}
 class Enc_a42857 : OpcodeHexagon {
   bits <11> Ii;
   let Inst{21-20} = Ii{10-9};
@@ -3109,6 +3115,14 @@ class Enc_de0214 : OpcodeHexagon {
   bits <5> Rd32;
   let Inst{4-0} = Rd32{4-0};
 }
+class Enc_de5ea0 : OpcodeHexagon {
+  bits <5> Vuu32;
+  let Inst{12-8} = Vuu32{4-0};
+  bits <5> Vv32;
+  let Inst{20-16} = Vv32{4-0};
+  bits <5> Vd32;
+  let Inst{4-0} = Vd32{4-0};
+}
 class Enc_e07374 : OpcodeHexagon {
   bits <5> Rs32;
   let Inst{20-16} = Rs32{4-0};
diff --git a/llvm/lib/Target/Hexagon/HexagonDepInstrInfo.td b/llvm/lib/Target/Hexagon/HexagonDepInstrInfo.td
index 4f00409c336c..c02988266584 100644
--- a/llvm/lib/Target/Hexagon/HexagonDepInstrInfo.td
+++ b/llvm/lib/Target/Hexagon/HexagonDepInstrInfo.td
@@ -5824,8 +5824,8 @@ let Inst{31-21} = 0b01010100100;
 let hasNewValue = 1;
 let opNewValue = 0;
 let isSolo = 1;
-let Uses = [GOSP];
-let Defs = [GOSP, PC];
+let Uses = [CCR, GOSP];
+let Defs = [CCR, GOSP, PC];
 let hasSideEffects = 1;
 let Constraints = "$Rx32 = $Rx32in";
 }
@@ -8500,6 +8500,8 @@ let Inst{31-21} = 0b01010010101;
 let isTerminator = 1;
 let isIndirectBranch = 1;
 let isBranch = 1;
+let cofRelax1 = 1;
+let cofRelax2 = 1;
 let cofMax1 = 1;
 }
 def J4_jumpseti : HInst<
@@ -18210,16 +18212,6 @@ let opExtentBits = 18;
 let opExtentAlign = 2;
 let opNewValue = 1;
 }
-def PS_trap1 : HInst<
-(outs),
-(ins u8_0Imm:$Ii),
-"trap1(#$Ii)",
-tc_53c851ab, TypeJ>, Enc_a51a9a, Requires<[HasPreV65]> {
-let Inst{1-0} = 0b00;
-let Inst{7-5} = 0b000;
-let Inst{13-13} = 0b0;
-let Inst{31-16} = 0b0101010010000000;
-}
 def R6_release_at_vi : HInst<
 (outs),
 (ins IntRegs:$Rs32),
@@ -18964,7 +18956,7 @@ def S2_cabacdecbin : HInst<
 (outs DoubleRegs:$Rdd32),
 (ins DoubleRegs:$Rss32, DoubleRegs:$Rtt32),
 "$Rdd32 = decbin($Rss32,$Rtt32)",
-tc_db596beb, TypeS_3op>, Enc_a56825 {
+tc_db596beb, TypeS_3op>, Enc_a56825, Requires<[UseCabac]> {
 let Inst{7-5} = 0b110;
 let Inst{13-13} = 0b0;
 let Inst{31-21} = 0b11000001110;
@@ -26883,17 +26875,6 @@ let isPseudo = 1;
 let isCodeGenOnly = 1;
 let DecoderNamespace = "EXT_mmvec";
 }
-def V6_ldntnt0 : HInst<
-(outs HvxVR:$Vd32),
-(ins IntRegs:$Rt32),
-"$Vd32 = vmem($Rt32):nt",
-PSEUDO, TypeMAPPING>, Requires<[HasV62]> {
-let hasNewValue = 1;
-let opNewValue = 0;
-let isPseudo = 1;
-let isCodeGenOnly = 1;
-let DecoderNamespace = "EXT_mmvec";
-}
 def V6_ldp0 : HInst<
 (outs HvxVR:$Vd32),
 (ins PredRegs:$Pv4, IntRegs:$Rt32),
@@ -27312,6 +27293,30 @@ let isPseudo = 1;
 let isCodeGenOnly = 1;
 let DecoderNamespace = "EXT_mmvec";
 }
+def V6_v10mpyubs10 : HInst<
+(outs HvxWR:$Vdd32),
+(ins HvxWR:$Vuu32, HvxWR:$Vvv32, u1_0Imm:$Ii),
+"$Vdd32.w = v10mpy($Vuu32.ub,$Vvv32.b,#$Ii)",
+tc_f175e046, TypeCVI_VX>, Requires<[UseHVXV69]> {
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let isPseudo = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_v10mpyubs10_vxx : HInst<
+(outs HvxWR:$Vxx32),
+(ins HvxWR:$Vxx32in, HvxWR:$Vuu32, HvxWR:$Vvv32, u1_0Imm:$Ii),
+"$Vxx32.w += v10mpy($Vuu32.ub,$Vvv32.b,#$Ii)",
+tc_4942646a, TypeCVI_VX>, Requires<[UseHVXV69]> {
+let hasNewValue = 1;
+let opNewValue = 0;
+let isAccumulator = 1;
+let isCVI = 1;
+let isPseudo = 1;
+let DecoderNamespace = "EXT_mmvec";
+let Constraints = "$Vxx32 = $Vxx32in";
+}
 def V6_v6mpyhubs10 : HInst<
 (outs HvxWR:$Vdd32),
 (ins HvxWR:$Vuu32, HvxWR:$Vvv32, u2_0Imm:$Ii),
@@ -27396,7 +27401,7 @@ def V6_vL32Ub_ai : HInst<
 (outs HvxVR:$Vd32),
 (ins IntRegs:$Rt32, s4_0Imm:$Ii),
 "$Vd32 = vmemu($Rt32+#$Ii)",
-tc_a7e6707d, TypeCVI_VM_VP_LDU>, Enc_f3f408, Requires<[UseHVXV60]> {
+tc_a7e6707d, TypeCVI_VM_VP_LDU>, Enc_f3f408, Requires<[UseHVXV60]>, PostInc_BaseImm {
 let Inst{7-5} = 0b111;
 let Inst{12-11} = 0b00;
 let Inst{31-21} = 0b00101000000;
@@ -27408,13 +27413,15 @@ let isCVLoad = 1;
 let isCVI = 1;
 let mayLoad = 1;
 let isRestrictNoSlot1Store = 1;
+let BaseOpcode = "V6_vL32Ub_ai";
+let CextOpcode = "V6_vL32Ub";
 let DecoderNamespace = "EXT_mmvec";
 }
 def V6_vL32Ub_pi : HInst<
 (outs HvxVR:$Vd32, IntRegs:$Rx32),
 (ins IntRegs:$Rx32in, s3_0Imm:$Ii),
 "$Vd32 = vmemu($Rx32++#$Ii)",
-tc_3c56e5ce, TypeCVI_VM_VP_LDU>, Enc_a255dc, Requires<[UseHVXV60]> {
+tc_3c56e5ce, TypeCVI_VM_VP_LDU>, Enc_a255dc, Requires<[UseHVXV60]>, PostInc_BaseImm {
 let Inst{7-5} = 0b111;
 let Inst{13-11} = 0b000;
 let Inst{31-21} = 0b00101001000;
@@ -27427,6 +27434,7 @@ let isCVI = 1;
 let mayLoad = 1;
 let isRestrictNoSlot1Store = 1;
 let BaseOpcode = "V6_vL32b_pi";
+let CextOpcode = "V6_vL32Ub";
 let DecoderNamespace = "EXT_mmvec";
 let Constraints = "$Rx32 = $Rx32in";
 }
@@ -27452,7 +27460,7 @@ def V6_vL32b_ai : HInst<
 (outs HvxVR:$Vd32),
 (ins IntRegs:$Rt32, s4_0Imm:$Ii),
 "$Vd32 = vmem($Rt32+#$Ii)",
-tc_c0749f3c, TypeCVI_VM_LD>, Enc_f3f408, Requires<[UseHVXV60]>, PredRel {
+tc_c0749f3c, TypeCVI_VM_LD>, Enc_f3f408, Requires<[UseHVXV60]>, PredRel, PostInc_BaseImm {
 let Inst{7-5} = 0b000;
 let Inst{12-11} = 0b00;
 let Inst{31-21} = 0b00101000000;
@@ -27465,6 +27473,7 @@ let isCVI = 1;
 let mayLoad = 1;
 let isRestrictNoSlot1Store = 1;
 let BaseOpcode = "V6_vL32b_ai";
+let CextOpcode = "V6_vL32b";
 let isCVLoadable = 1;
 let isPredicable = 1;
 let DecoderNamespace = "EXT_mmvec";
@@ -27473,7 +27482,7 @@ def V6_vL32b_cur_ai : HInst<
 (outs HvxVR:$Vd32),
 (ins IntRegs:$Rt32, s4_0Imm:$Ii),
 "$Vd32.cur = vmem($Rt32+#$Ii)",
-tc_c0749f3c, TypeCVI_VM_LD>, Enc_f3f408, Requires<[UseHVXV60]>, PredRel {
+tc_c0749f3c, TypeCVI_VM_LD>, Enc_f3f408, Requires<[UseHVXV60]>, PredRel, PostInc_BaseImm {
 let Inst{7-5} = 0b001;
 let Inst{12-11} = 0b00;
 let Inst{31-21} = 0b00101000000;
@@ -27487,6 +27496,7 @@ let CVINew = 1;
 let mayLoad = 1;
 let isRestrictNoSlot1Store = 1;
 let BaseOpcode = "V6_vL32b_cur_ai";
+let CextOpcode = "V6_vL32b_cur";
 let isPredicable = 1;
 let DecoderNamespace = "EXT_mmvec";
 }
@@ -27560,7 +27570,7 @@ def V6_vL32b_cur_pi : HInst<
 (outs HvxVR:$Vd32, IntRegs:$Rx32),
 (ins IntRegs:$Rx32in, s3_0Imm:$Ii),
 "$Vd32.cur = vmem($Rx32++#$Ii)",
-tc_1ba8a0cd, TypeCVI_VM_LD>, Enc_a255dc, Requires<[UseHVXV60]>, PredRel {
+tc_1ba8a0cd, TypeCVI_VM_LD>, Enc_a255dc, Requires<[UseHVXV60]>, PredRel, PostInc_BaseImm {
 let Inst{7-5} = 0b001;
 let Inst{13-11} = 0b000;
 let Inst{31-21} = 0b00101001000;
@@ -27574,6 +27584,7 @@ let CVINew = 1;
 let mayLoad = 1;
 let isRestrictNoSlot1Store = 1;
 let BaseOpcode = "V6_vL32b_cur_pi";
+let CextOpcode = "V6_vL32b_cur";
 let isPredicable = 1;
 let DecoderNamespace = "EXT_mmvec";
 let Constraints = "$Rx32 = $Rx32in";
@@ -27729,7 +27740,7 @@ def V6_vL32b_nt_ai : HInst<
 (outs HvxVR:$Vd32),
 (ins IntRegs:$Rt32, s4_0Imm:$Ii),
 "$Vd32 = vmem($Rt32+#$Ii):nt",
-tc_c0749f3c, TypeCVI_VM_LD>, Enc_f3f408, Requires<[UseHVXV60]>, PredRel {
+tc_c0749f3c, TypeCVI_VM_LD>, Enc_f3f408, Requires<[UseHVXV60]>, PredRel, PostInc_BaseImm {
 let Inst{7-5} = 0b000;
 let Inst{12-11} = 0b00;
 let Inst{31-21} = 0b00101000010;
@@ -27743,6 +27754,7 @@ let mayLoad = 1;
 let isNonTemporal = 1;
 let isRestrictNoSlot1Store = 1;
 let BaseOpcode = "V6_vL32b_nt_ai";
+let CextOpcode = "V6_vL32b_nt";
 let isCVLoadable = 1;
 let isPredicable = 1;
 let DecoderNamespace = "EXT_mmvec";
@@ -27751,7 +27763,7 @@ def V6_vL32b_nt_cur_ai : HInst<
 (outs HvxVR:$Vd32),
 (ins IntRegs:$Rt32, s4_0Imm:$Ii),
 "$Vd32.cur = vmem($Rt32+#$Ii):nt",
-tc_c0749f3c, TypeCVI_VM_LD>, Enc_f3f408, Requires<[UseHVXV60]>, PredRel {
+tc_c0749f3c, TypeCVI_VM_LD>, Enc_f3f408, Requires<[UseHVXV60]>, PredRel, PostInc_BaseImm {
 let Inst{7-5} = 0b001;
 let Inst{12-11} = 0b00;
 let Inst{31-21} = 0b00101000010;
@@ -27766,6 +27778,7 @@ let mayLoad = 1;
 let isNonTemporal = 1;
 let isRestrictNoSlot1Store = 1;
 let BaseOpcode = "V6_vL32b_nt_cur_ai";
+let CextOpcode = "V6_vL32b_nt_cur";
 let isPredicable = 1;
 let DecoderNamespace = "EXT_mmvec";
 }
@@ -27842,7 +27855,7 @@ def V6_vL32b_nt_cur_pi : HInst<
 (outs HvxVR:$Vd32, IntRegs:$Rx32),
 (ins IntRegs:$Rx32in, s3_0Imm:$Ii),
 "$Vd32.cur = vmem($Rx32++#$Ii):nt",
-tc_1ba8a0cd, TypeCVI_VM_LD>, Enc_a255dc, Requires<[UseHVXV60]>, PredRel {
+tc_1ba8a0cd, TypeCVI_VM_LD>, Enc_a255dc, Requires<[UseHVXV60]>, PredRel, PostInc_BaseImm {
 let Inst{7-5} = 0b001;
 let Inst{13-11} = 0b000;
 let Inst{31-21} = 0b00101001010;
@@ -27857,6 +27870,7 @@ let mayLoad = 1;
 let isNonTemporal = 1;
 let isRestrictNoSlot1Store = 1;
 let BaseOpcode = "V6_vL32b_nt_cur_pi";
+let CextOpcode = "V6_vL32b_nt_cur";
 let isPredicable = 1;
 let DecoderNamespace = "EXT_mmvec";
 let Constraints = "$Rx32 = $Rx32in";
@@ -28019,7 +28033,7 @@ def V6_vL32b_nt_pi : HInst<
 (outs HvxVR:$Vd32, IntRegs:$Rx32),
 (ins IntRegs:$Rx32in, s3_0Imm:$Ii),
 "$Vd32 = vmem($Rx32++#$Ii):nt",
-tc_1ba8a0cd, TypeCVI_VM_LD>, Enc_a255dc, Requires<[UseHVXV60]>, PredRel {
+tc_1ba8a0cd, TypeCVI_VM_LD>, Enc_a255dc, Requires<[UseHVXV60]>, PredRel, PostInc_BaseImm {
 let Inst{7-5} = 0b000;
 let Inst{13-11} = 0b000;
 let Inst{31-21} = 0b00101001010;
@@ -28033,6 +28047,7 @@ let mayLoad = 1;
 let isNonTemporal = 1;
 let isRestrictNoSlot1Store = 1;
 let BaseOpcode = "V6_vL32b_nt_pi";
+let CextOpcode = "V6_vL32b_nt";
 let isCVLoadable = 1;
 let isPredicable = 1;
 let DecoderNamespace = "EXT_mmvec";
@@ -28127,7 +28142,7 @@ def V6_vL32b_nt_tmp_ai : HInst<
 (outs HvxVR:$Vd32),
 (ins IntRegs:$Rt32, s4_0Imm:$Ii),
 "$Vd32.tmp = vmem($Rt32+#$Ii):nt",
-tc_52447ecc, TypeCVI_VM_TMP_LD>, Enc_f3f408, Requires<[UseHVXV60]>, PredRel {
+tc_52447ecc, TypeCVI_VM_TMP_LD>, Enc_f3f408, Requires<[UseHVXV60]>, PredRel, PostInc_BaseImm {
 let Inst{7-5} = 0b010;
 let Inst{12-11} = 0b00;
 let Inst{31-21} = 0b00101000010;
@@ -28137,11 +28152,12 @@ let addrMode = BaseImmOffset;
 let accessSize = HVXVectorAccess;
 let isCVLoad = 1;
 let isCVI = 1;
-let hasTmpDst = 1;
+let hasHvxTmp = 1;
 let mayLoad = 1;
 let isNonTemporal = 1;
 let isRestrictNoSlot1Store = 1;
 let BaseOpcode = "V6_vL32b_nt_tmp_ai";
+let CextOpcode = "V6_vL32b_nt_tmp";
 let isPredicable = 1;
 let DecoderNamespace = "EXT_mmvec";
 }
@@ -28160,7 +28176,7 @@ let addrMode = BaseImmOffset;
 let accessSize = HVXVectorAccess;
 let isCVLoad = 1;
 let isCVI = 1;
-let hasTmpDst = 1;
+let hasHvxTmp = 1;
 let mayLoad = 1;
 let isNonTemporal = 1;
 let isRestrictNoSlot1Store = 1;
@@ -28183,7 +28199,7 @@ let addrMode = PostInc;
 let accessSize = HVXVectorAccess;
 let isCVLoad = 1;
 let isCVI = 1;
-let hasTmpDst = 1;
+let hasHvxTmp = 1;
 let mayLoad = 1;
 let isNonTemporal = 1;
 let isRestrictNoSlot1Store = 1;
@@ -28206,7 +28222,7 @@ let addrMode = PostInc;
 let accessSize = HVXVectorAccess;
 let isCVLoad = 1;
 let isCVI = 1;
-let hasTmpDst = 1;
+let hasHvxTmp = 1;
 let mayLoad = 1;
 let isNonTemporal = 1;
 let isRestrictNoSlot1Store = 1;
@@ -28218,7 +28234,7 @@ def V6_vL32b_nt_tmp_pi : HInst<
 (outs HvxVR:$Vd32, IntRegs:$Rx32),
 (ins IntRegs:$Rx32in, s3_0Imm:$Ii),
 "$Vd32.tmp = vmem($Rx32++#$Ii):nt",
-tc_663c80a7, TypeCVI_VM_TMP_LD>, Enc_a255dc, Requires<[UseHVXV60]>, PredRel {
+tc_663c80a7, TypeCVI_VM_TMP_LD>, Enc_a255dc, Requires<[UseHVXV60]>, PredRel, PostInc_BaseImm {
 let Inst{7-5} = 0b010;
 let Inst{13-11} = 0b000;
 let Inst{31-21} = 0b00101001010;
@@ -28228,11 +28244,12 @@ let addrMode = PostInc;
 let accessSize = HVXVectorAccess;
 let isCVLoad = 1;
 let isCVI = 1;
-let hasTmpDst = 1;
+let hasHvxTmp = 1;
 let mayLoad = 1;
 let isNonTemporal = 1;
 let isRestrictNoSlot1Store = 1;
 let BaseOpcode = "V6_vL32b_nt_tmp_pi";
+let CextOpcode = "V6_vL32b_nt_tmp";
 let isPredicable = 1;
 let DecoderNamespace = "EXT_mmvec";
 let Constraints = "$Rx32 = $Rx32in";
@@ -28250,7 +28267,7 @@ let addrMode = PostInc;
 let accessSize = HVXVectorAccess;
 let isCVLoad = 1;
 let isCVI = 1;
-let hasTmpDst = 1;
+let hasHvxTmp = 1;
 let mayLoad = 1;
 let isNonTemporal = 1;
 let isRestrictNoSlot1Store = 1;
@@ -28273,7 +28290,7 @@ let addrMode = BaseImmOffset;
 let accessSize = HVXVectorAccess;
 let isCVLoad = 1;
 let isCVI = 1;
-let hasTmpDst = 1;
+let hasHvxTmp = 1;
 let mayLoad = 1;
 let isNonTemporal = 1;
 let isRestrictNoSlot1Store = 1;
@@ -28295,7 +28312,7 @@ let addrMode = PostInc;
 let accessSize = HVXVectorAccess;
 let isCVLoad = 1;
 let isCVI = 1;
-let hasTmpDst = 1;
+let hasHvxTmp = 1;
 let mayLoad = 1;
 let isNonTemporal = 1;
 let isRestrictNoSlot1Store = 1;
@@ -28317,7 +28334,7 @@ let addrMode = PostInc;
 let accessSize = HVXVectorAccess;
 let isCVLoad = 1;
 let isCVI = 1;
-let hasTmpDst = 1;
+let hasHvxTmp = 1;
 let mayLoad = 1;
 let isNonTemporal = 1;
 let isRestrictNoSlot1Store = 1;
@@ -28329,7 +28346,7 @@ def V6_vL32b_pi : HInst<
 (outs HvxVR:$Vd32, IntRegs:$Rx32),
 (ins IntRegs:$Rx32in, s3_0Imm:$Ii),
 "$Vd32 = vmem($Rx32++#$Ii)",
-tc_1ba8a0cd, TypeCVI_VM_LD>, Enc_a255dc, Requires<[UseHVXV60]>, PredRel {
+tc_1ba8a0cd, TypeCVI_VM_LD>, Enc_a255dc, Requires<[UseHVXV60]>, PredRel, PostInc_BaseImm {
 let Inst{7-5} = 0b000;
 let Inst{13-11} = 0b000;
 let Inst{31-21} = 0b00101001000;
@@ -28342,6 +28359,7 @@ let isCVI = 1;
 let mayLoad = 1;
 let isRestrictNoSlot1Store = 1;
 let BaseOpcode = "V6_vL32b_pi";
+let CextOpcode = "V6_vL32b";
 let isCVLoadable = 1;
 let isPredicable = 1;
 let DecoderNamespace = "EXT_mmvec";
@@ -28432,7 +28450,7 @@ def V6_vL32b_tmp_ai : HInst<
 (outs HvxVR:$Vd32),
 (ins IntRegs:$Rt32, s4_0Imm:$Ii),
 "$Vd32.tmp = vmem($Rt32+#$Ii)",
-tc_52447ecc, TypeCVI_VM_TMP_LD>, Enc_f3f408, Requires<[UseHVXV60]>, PredRel {
+tc_52447ecc, TypeCVI_VM_TMP_LD>, Enc_f3f408, Requires<[UseHVXV60]>, PredRel, PostInc_BaseImm {
 let Inst{7-5} = 0b010;
 let Inst{12-11} = 0b00;
 let Inst{31-21} = 0b00101000000;
@@ -28442,10 +28460,11 @@ let addrMode = BaseImmOffset;
 let accessSize = HVXVectorAccess;
 let isCVLoad = 1;
 let isCVI = 1;
-let hasTmpDst = 1;
+let hasHvxTmp = 1;
 let mayLoad = 1;
 let isRestrictNoSlot1Store = 1;
 let BaseOpcode = "V6_vL32b_tmp_ai";
+let CextOpcode = "V6_vL32b_tmp";
 let isPredicable = 1;
 let DecoderNamespace = "EXT_mmvec";
 }
@@ -28464,7 +28483,7 @@ let addrMode = BaseImmOffset;
 let accessSize = HVXVectorAccess;
 let isCVLoad = 1;
 let isCVI = 1;
-let hasTmpDst = 1;
+let hasHvxTmp = 1;
 let mayLoad = 1;
 let isRestrictNoSlot1Store = 1;
 let BaseOpcode = "V6_vL32b_tmp_ai";
@@ -28486,7 +28505,7 @@ let addrMode = PostInc;
 let accessSize = HVXVectorAccess;
 let isCVLoad = 1;
 let isCVI = 1;
-let hasTmpDst = 1;
+let hasHvxTmp = 1;
 let mayLoad = 1;
 let isRestrictNoSlot1Store = 1;
 let BaseOpcode = "V6_vL32b_tmp_pi";
@@ -28508,7 +28527,7 @@ let addrMode = PostInc;
 let accessSize = HVXVectorAccess;
 let isCVLoad = 1;
 let isCVI = 1;
-let hasTmpDst = 1;
+let hasHvxTmp = 1;
 let mayLoad = 1;
 let isRestrictNoSlot1Store = 1;
 let BaseOpcode = "V6_vL32b_tmp_ppu";
@@ -28519,7 +28538,7 @@ def V6_vL32b_tmp_pi : HInst<
 (outs HvxVR:$Vd32, IntRegs:$Rx32),
 (ins IntRegs:$Rx32in, s3_0Imm:$Ii),
 "$Vd32.tmp = vmem($Rx32++#$Ii)",
-tc_663c80a7, TypeCVI_VM_TMP_LD>, Enc_a255dc, Requires<[UseHVXV60]>, PredRel {
+tc_663c80a7, TypeCVI_VM_TMP_LD>, Enc_a255dc, Requires<[UseHVXV60]>, PredRel, PostInc_BaseImm {
 let Inst{7-5} = 0b010;
 let Inst{13-11} = 0b000;
 let Inst{31-21} = 0b00101001000;
@@ -28529,10 +28548,11 @@ let addrMode = PostInc;
 let accessSize = HVXVectorAccess;
 let isCVLoad = 1;
 let isCVI = 1;
-let hasTmpDst = 1;
+let hasHvxTmp = 1;
 let mayLoad = 1;
 let isRestrictNoSlot1Store = 1;
 let BaseOpcode = "V6_vL32b_tmp_pi";
+let CextOpcode = "V6_vL32b_tmp";
 let isPredicable = 1;
 let DecoderNamespace = "EXT_mmvec";
 let Constraints = "$Rx32 = $Rx32in";
@@ -28550,7 +28570,7 @@ let addrMode = PostInc;
 let accessSize = HVXVectorAccess;
 let isCVLoad = 1;
 let isCVI = 1;
-let hasTmpDst = 1;
+let hasHvxTmp = 1;
 let mayLoad = 1;
 let isRestrictNoSlot1Store = 1;
 let BaseOpcode = "V6_vL32b_tmp_ppu";
@@ -28572,7 +28592,7 @@ let addrMode = BaseImmOffset;
 let accessSize = HVXVectorAccess;
 let isCVLoad = 1;
 let isCVI = 1;
-let hasTmpDst = 1;
+let hasHvxTmp = 1;
 let mayLoad = 1;
 let isRestrictNoSlot1Store = 1;
 let BaseOpcode = "V6_vL32b_tmp_ai";
@@ -28593,7 +28613,7 @@ let addrMode = PostInc;
 let accessSize = HVXVectorAccess;
 let isCVLoad = 1;
 let isCVI = 1;
-let hasTmpDst = 1;
+let hasHvxTmp = 1;
 let mayLoad = 1;
 let isRestrictNoSlot1Store = 1;
 let BaseOpcode = "V6_vL32b_tmp_pi";
@@ -28614,7 +28634,7 @@ let addrMode = PostInc;
 let accessSize = HVXVectorAccess;
 let isCVLoad = 1;
 let isCVI = 1;
-let hasTmpDst = 1;
+let hasHvxTmp = 1;
 let mayLoad = 1;
 let isRestrictNoSlot1Store = 1;
 let BaseOpcode = "V6_vL32b_tmp_ppu";
@@ -28625,7 +28645,7 @@ def V6_vS32Ub_ai : HInst<
 (outs),
 (ins IntRegs:$Rt32, s4_0Imm:$Ii, HvxVR:$Vs32),
 "vmemu($Rt32+#$Ii) = $Vs32",
-tc_f21e8abb, TypeCVI_VM_STU>, Enc_c9e3bc, Requires<[UseHVXV60]>, NewValueRel {
+tc_f21e8abb, TypeCVI_VM_STU>, Enc_c9e3bc, Requires<[UseHVXV60]>, NewValueRel, PostInc_BaseImm {
 let Inst{7-5} = 0b111;
 let Inst{12-11} = 0b00;
 let Inst{31-21} = 0b00101000001;
@@ -28634,6 +28654,7 @@ let accessSize = HVXVectorAccess;
 let isCVI = 1;
 let mayStore = 1;
 let BaseOpcode = "V6_vS32Ub_ai";
+let CextOpcode = "V6_vS32Ub";
 let isPredicable = 1;
 let DecoderNamespace = "EXT_mmvec";
 }
@@ -28692,7 +28713,7 @@ def V6_vS32Ub_pi : HInst<
 (outs IntRegs:$Rx32),
 (ins IntRegs:$Rx32in, s3_0Imm:$Ii, HvxVR:$Vs32),
 "vmemu($Rx32++#$Ii) = $Vs32",
-tc_e2d2e9e5, TypeCVI_VM_STU>, Enc_b62ef7, Requires<[UseHVXV60]>, NewValueRel {
+tc_e2d2e9e5, TypeCVI_VM_STU>, Enc_b62ef7, Requires<[UseHVXV60]>, NewValueRel, PostInc_BaseImm {
 let Inst{7-5} = 0b111;
 let Inst{13-11} = 0b000;
 let Inst{31-21} = 0b00101001001;
@@ -28701,6 +28722,7 @@ let accessSize = HVXVectorAccess;
 let isCVI = 1;
 let mayStore = 1;
 let BaseOpcode = "V6_vS32Ub_pi";
+let CextOpcode = "V6_vS32Ub";
 let isPredicable = 1;
 let DecoderNamespace = "EXT_mmvec";
 let Constraints = "$Rx32 = $Rx32in";
@@ -28773,7 +28795,7 @@ def V6_vS32b_ai : HInst<
 (outs),
 (ins IntRegs:$Rt32, s4_0Imm:$Ii, HvxVR:$Vs32),
 "vmem($Rt32+#$Ii) = $Vs32",
-tc_c5dba46e, TypeCVI_VM_ST>, Enc_c9e3bc, Requires<[UseHVXV60]>, NewValueRel {
+tc_c5dba46e, TypeCVI_VM_ST>, Enc_c9e3bc, Requires<[UseHVXV60]>, NewValueRel, PostInc_BaseImm {
 let Inst{7-5} = 0b000;
 let Inst{12-11} = 0b00;
 let Inst{31-21} = 0b00101000001;
@@ -28782,6 +28804,7 @@ let accessSize = HVXVectorAccess;
 let isCVI = 1;
 let mayStore = 1;
 let BaseOpcode = "V6_vS32b_ai";
+let CextOpcode = "V6_vS32b";
 let isNVStorable = 1;
 let isPredicable = 1;
 let DecoderNamespace = "EXT_mmvec";
@@ -28790,7 +28813,7 @@ def V6_vS32b_new_ai : HInst<
 (outs),
 (ins IntRegs:$Rt32, s4_0Imm:$Ii, HvxVR:$Os8),
 "vmem($Rt32+#$Ii) = $Os8.new",
-tc_ab23f776, TypeCVI_VM_NEW_ST>, Enc_f77fbc, Requires<[UseHVXV60]>, NewValueRel {
+tc_ab23f776, TypeCVI_VM_NEW_ST>, Enc_f77fbc, Requires<[UseHVXV60]>, NewValueRel, PostInc_BaseImm {
 let Inst{7-3} = 0b00100;
 let Inst{12-11} = 0b00;
 let Inst{31-21} = 0b00101000001;
@@ -28802,6 +28825,7 @@ let CVINew = 1;
 let isNewValue = 1;
 let mayStore = 1;
 let BaseOpcode = "V6_vS32b_ai";
+let CextOpcode = "V6_vS32b_new";
 let isPredicable = 1;
 let DecoderNamespace = "EXT_mmvec";
 let opNewValue = 2;
@@ -28873,7 +28897,7 @@ def V6_vS32b_new_pi : HInst<
 (outs IntRegs:$Rx32),
 (ins IntRegs:$Rx32in, s3_0Imm:$Ii, HvxVR:$Os8),
 "vmem($Rx32++#$Ii) = $Os8.new",
-tc_6942b6e0, TypeCVI_VM_NEW_ST>, Enc_1aaec1, Requires<[UseHVXV60]>, NewValueRel {
+tc_6942b6e0, TypeCVI_VM_NEW_ST>, Enc_1aaec1, Requires<[UseHVXV60]>, NewValueRel, PostInc_BaseImm {
 let Inst{7-3} = 0b00100;
 let Inst{13-11} = 0b000;
 let Inst{31-21} = 0b00101001001;
@@ -28885,6 +28909,7 @@ let CVINew = 1;
 let isNewValue = 1;
 let mayStore = 1;
 let BaseOpcode = "V6_vS32b_pi";
+let CextOpcode = "V6_vS32b_new";
 let isPredicable = 1;
 let DecoderNamespace = "EXT_mmvec";
 let opNewValue = 3;
@@ -29070,7 +29095,7 @@ def V6_vS32b_nt_ai : HInst<
 (outs),
 (ins IntRegs:$Rt32, s4_0Imm:$Ii, HvxVR:$Vs32),
 "vmem($Rt32+#$Ii):nt = $Vs32",
-tc_c5dba46e, TypeCVI_VM_ST>, Enc_c9e3bc, Requires<[UseHVXV60]>, NewValueRel {
+tc_c5dba46e, TypeCVI_VM_ST>, Enc_c9e3bc, Requires<[UseHVXV60]>, NewValueRel, PostInc_BaseImm {
 let Inst{7-5} = 0b000;
 let Inst{12-11} = 0b00;
 let Inst{31-21} = 0b00101000011;
@@ -29080,6 +29105,7 @@ let isCVI = 1;
 let isNonTemporal = 1;
 let mayStore = 1;
 let BaseOpcode = "V6_vS32b_ai";
+let CextOpcode = "V6_vS32b_nt";
 let isNVStorable = 1;
 let isPredicable = 1;
 let DecoderNamespace = "EXT_mmvec";
@@ -29088,7 +29114,7 @@ def V6_vS32b_nt_new_ai : HInst<
 (outs),
 (ins IntRegs:$Rt32, s4_0Imm:$Ii, HvxVR:$Os8),
 "vmem($Rt32+#$Ii):nt = $Os8.new",
-tc_ab23f776, TypeCVI_VM_NEW_ST>, Enc_f77fbc, Requires<[UseHVXV60]>, NewValueRel {
+tc_ab23f776, TypeCVI_VM_NEW_ST>, Enc_f77fbc, Requires<[UseHVXV60]>, NewValueRel, PostInc_BaseImm {
 let Inst{7-3} = 0b00100;
 let Inst{12-11} = 0b00;
 let Inst{31-21} = 0b00101000011;
@@ -29101,6 +29127,7 @@ let isNewValue = 1;
 let isNonTemporal = 1;
 let mayStore = 1;
 let BaseOpcode = "V6_vS32b_ai";
+let CextOpcode = "V6_vS32b_nt_new";
 let isPredicable = 1;
 let DecoderNamespace = "EXT_mmvec";
 let opNewValue = 2;
@@ -29175,7 +29202,7 @@ def V6_vS32b_nt_new_pi : HInst<
 (outs IntRegs:$Rx32),
 (ins IntRegs:$Rx32in, s3_0Imm:$Ii, HvxVR:$Os8),
 "vmem($Rx32++#$Ii):nt = $Os8.new",
-tc_6942b6e0, TypeCVI_VM_NEW_ST>, Enc_1aaec1, Requires<[UseHVXV60]>, NewValueRel {
+tc_6942b6e0, TypeCVI_VM_NEW_ST>, Enc_1aaec1, Requires<[UseHVXV60]>, NewValueRel, PostInc_BaseImm {
 let Inst{7-3} = 0b00100;
 let Inst{13-11} = 0b000;
 let Inst{31-21} = 0b00101001011;
@@ -29188,6 +29215,7 @@ let isNewValue = 1;
 let isNonTemporal = 1;
 let mayStore = 1;
 let BaseOpcode = "V6_vS32b_pi";
+let CextOpcode = "V6_vS32b_nt_new";
 let isPredicable = 1;
 let DecoderNamespace = "EXT_mmvec";
 let opNewValue = 3;
@@ -29383,7 +29411,7 @@ def V6_vS32b_nt_pi : HInst<
 (outs IntRegs:$Rx32),
 (ins IntRegs:$Rx32in, s3_0Imm:$Ii, HvxVR:$Vs32),
 "vmem($Rx32++#$Ii):nt = $Vs32",
-tc_3e2aaafc, TypeCVI_VM_ST>, Enc_b62ef7, Requires<[UseHVXV60]>, NewValueRel {
+tc_3e2aaafc, TypeCVI_VM_ST>, Enc_b62ef7, Requires<[UseHVXV60]>, NewValueRel, PostInc_BaseImm {
 let Inst{7-5} = 0b000;
 let Inst{13-11} = 0b000;
 let Inst{31-21} = 0b00101001011;
@@ -29393,6 +29421,7 @@ let isCVI = 1;
 let isNonTemporal = 1;
 let mayStore = 1;
 let BaseOpcode = "V6_vS32b_pi";
+let CextOpcode = "V6_vS32b_nt";
 let isNVStorable = 1;
 let isPredicable = 1;
 let DecoderNamespace = "EXT_mmvec";
@@ -29519,7 +29548,7 @@ def V6_vS32b_pi : HInst<
 (outs IntRegs:$Rx32),
 (ins IntRegs:$Rx32in, s3_0Imm:$Ii, HvxVR:$Vs32),
 "vmem($Rx32++#$Ii) = $Vs32",
-tc_3e2aaafc, TypeCVI_VM_ST>, Enc_b62ef7, Requires<[UseHVXV60]>, NewValueRel {
+tc_3e2aaafc, TypeCVI_VM_ST>, Enc_b62ef7, Requires<[UseHVXV60]>, NewValueRel, PostInc_BaseImm {
 let Inst{7-5} = 0b000;
 let Inst{13-11} = 0b000;
 let Inst{31-21} = 0b00101001001;
@@ -29528,6 +29557,7 @@ let accessSize = HVXVectorAccess;
 let isCVI = 1;
 let mayStore = 1;
 let BaseOpcode = "V6_vS32b_pi";
+let CextOpcode = "V6_vS32b";
 let isNVStorable = 1;
 let isPredicable = 1;
 let DecoderNamespace = "EXT_mmvec";
@@ -29689,6 +29719,32 @@ let mayStore = 1;
 let DecoderNamespace = "EXT_mmvec";
 let Constraints = "$Rx32 = $Rx32in";
 }
+def V6_vabs_hf : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32),
+"$Vd32.hf = vabs($Vu32.hf)",
+tc_5cdf8c84, TypeCVI_VX_LATE>, Enc_e7581c, Requires<[UseHVXV68,UseHVXIEEEFP]> {
+let Inst{7-5} = 0b100;
+let Inst{13-13} = 0b1;
+let Inst{31-16} = 0b0001111000000110;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vabs_sf : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32),
+"$Vd32.sf = vabs($Vu32.sf)",
+tc_5cdf8c84, TypeCVI_VX_LATE>, Enc_e7581c, Requires<[UseHVXV68,UseHVXIEEEFP]> {
+let Inst{7-5} = 0b101;
+let Inst{13-13} = 0b1;
+let Inst{31-16} = 0b0001111000000110;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
 def V6_vabsb : HInst<
 (outs HvxVR:$Vd32),
 (ins HvxVR:$Vu32),
@@ -29975,6 +30031,123 @@ let isPseudo = 1;
 let isCodeGenOnly = 1;
 let DecoderNamespace = "EXT_mmvec";
 }
+def V6_vadd_hf : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vd32.qf16 = vadd($Vu32.hf,$Vv32.hf)",
+tc_05ca8cfd, TypeCVI_VS>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> {
+let Inst{7-5} = 0b011;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011111011;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vadd_hf_hf : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vd32.hf = vadd($Vu32.hf,$Vv32.hf)",
+tc_c127de3a, TypeCVI_VX>, Enc_45364e, Requires<[UseHVXV68,UseHVXIEEEFP]> {
+let Inst{7-5} = 0b111;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011111101;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vadd_qf16 : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vd32.qf16 = vadd($Vu32.qf16,$Vv32.qf16)",
+tc_05ca8cfd, TypeCVI_VS>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> {
+let Inst{7-5} = 0b010;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011111011;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vadd_qf16_mix : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vd32.qf16 = vadd($Vu32.qf16,$Vv32.hf)",
+tc_05ca8cfd, TypeCVI_VS>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> {
+let Inst{7-5} = 0b100;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011111011;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vadd_qf32 : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vd32.qf32 = vadd($Vu32.qf32,$Vv32.qf32)",
+tc_05ca8cfd, TypeCVI_VS>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> {
+let Inst{7-5} = 0b000;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011111101;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vadd_qf32_mix : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vd32.qf32 = vadd($Vu32.qf32,$Vv32.sf)",
+tc_05ca8cfd, TypeCVI_VS>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> {
+let Inst{7-5} = 0b010;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011111101;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vadd_sf : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vd32.qf32 = vadd($Vu32.sf,$Vv32.sf)",
+tc_05ca8cfd, TypeCVI_VS>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> {
+let Inst{7-5} = 0b001;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011111101;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vadd_sf_hf : HInst<
+(outs HvxWR:$Vdd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vdd32.sf = vadd($Vu32.hf,$Vv32.hf)",
+tc_d8287c14, TypeCVI_VX_DV>, Enc_71bb9b, Requires<[UseHVXV68,UseHVXIEEEFP]> {
+let Inst{7-5} = 0b100;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011111100;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vadd_sf_sf : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vd32.sf = vadd($Vu32.sf,$Vv32.sf)",
+tc_c127de3a, TypeCVI_VX>, Enc_45364e, Requires<[UseHVXV68,UseHVXIEEEFP]> {
+let Inst{7-5} = 0b110;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011111100;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
 def V6_vaddb : HInst<
 (outs HvxVR:$Vd32),
 (ins HvxVR:$Vu32, HvxVR:$Vv32),
@@ -31440,6 +31613,58 @@ let opNewValue = 0;
 let isCVI = 1;
 let DecoderNamespace = "EXT_mmvec";
 }
+def V6_vasrvuhubrndsat : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxWR:$Vuu32, HvxVR:$Vv32),
+"$Vd32.ub = vasr($Vuu32.uh,$Vv32.ub):rnd:sat",
+tc_05ca8cfd, TypeCVI_VS>, Enc_de5ea0, Requires<[UseHVXV69]> {
+let Inst{7-5} = 0b011;
+let Inst{13-13} = 0b0;
+let Inst{31-21} = 0b00011101000;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vasrvuhubsat : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxWR:$Vuu32, HvxVR:$Vv32),
+"$Vd32.ub = vasr($Vuu32.uh,$Vv32.ub):sat",
+tc_05ca8cfd, TypeCVI_VS>, Enc_de5ea0, Requires<[UseHVXV69]> {
+let Inst{7-5} = 0b010;
+let Inst{13-13} = 0b0;
+let Inst{31-21} = 0b00011101000;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vasrvwuhrndsat : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxWR:$Vuu32, HvxVR:$Vv32),
+"$Vd32.uh = vasr($Vuu32.w,$Vv32.uh):rnd:sat",
+tc_05ca8cfd, TypeCVI_VS>, Enc_de5ea0, Requires<[UseHVXV69]> {
+let Inst{7-5} = 0b001;
+let Inst{13-13} = 0b0;
+let Inst{31-21} = 0b00011101000;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vasrvwuhsat : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxWR:$Vuu32, HvxVR:$Vv32),
+"$Vd32.uh = vasr($Vuu32.w,$Vv32.uh):sat",
+tc_05ca8cfd, TypeCVI_VS>, Enc_de5ea0, Requires<[UseHVXV69]> {
+let Inst{7-5} = 0b000;
+let Inst{13-13} = 0b0;
+let Inst{31-21} = 0b00011101000;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
 def V6_vasrw : HInst<
 (outs HvxVR:$Vd32),
 (ins HvxVR:$Vu32, IntRegs:$Rt32),
@@ -31597,6 +31822,33 @@ let opNewValue = 0;
 let isCVI = 1;
 let DecoderNamespace = "EXT_mmvec";
 }
+def V6_vassign_fp : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32),
+"$Vd32.w = vfmv($Vu32.w)",
+tc_5cdf8c84, TypeCVI_VX_LATE>, Enc_e7581c, Requires<[UseHVXV68,UseHVXIEEEFP]> {
+let Inst{7-5} = 0b001;
+let Inst{13-13} = 0b1;
+let Inst{31-16} = 0b0001111000000110;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vassign_tmp : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32),
+"$Vd32.tmp = $Vu32",
+tc_2120355e, TypeCVI_VX>, Enc_e7581c, Requires<[UseHVXV69]> {
+let Inst{7-5} = 0b110;
+let Inst{13-13} = 0b0;
+let Inst{31-16} = 0b0001111000000001;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let hasHvxTmp = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
 def V6_vassignp : HInst<
 (outs HvxWR:$Vdd32),
 (ins HvxWR:$Vuu32),
@@ -32000,6 +32252,189 @@ let isCVI = 1;
 let isRegSequence = 1;
 let DecoderNamespace = "EXT_mmvec";
 }
+def V6_vcombine_tmp : HInst<
+(outs HvxWR:$Vdd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vdd32.tmp = vcombine($Vu32,$Vv32)",
+tc_aa047364, TypeCVI_VX>, Enc_71bb9b, Requires<[UseHVXV69]> {
+let Inst{7-5} = 0b111;
+let Inst{13-13} = 0b0;
+let Inst{31-21} = 0b00011110101;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let hasHvxTmp = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vconv_hf_qf16 : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32),
+"$Vd32.hf = $Vu32.qf16",
+tc_51d0ecc3, TypeCVI_VS>, Enc_e7581c, Requires<[UseHVXV68,UseHVXQFloat]> {
+let Inst{7-5} = 0b011;
+let Inst{13-13} = 0b1;
+let Inst{31-16} = 0b0001111000000100;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vconv_hf_qf32 : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxWR:$Vuu32),
+"$Vd32.hf = $Vuu32.qf32",
+tc_51d0ecc3, TypeCVI_VS>, Enc_a33d04, Requires<[UseHVXV68,UseHVXQFloat]> {
+let Inst{7-5} = 0b110;
+let Inst{13-13} = 0b1;
+let Inst{31-16} = 0b0001111000000100;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vconv_sf_qf32 : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32),
+"$Vd32.sf = $Vu32.qf32",
+tc_51d0ecc3, TypeCVI_VS>, Enc_e7581c, Requires<[UseHVXV68,UseHVXQFloat]> {
+let Inst{7-5} = 0b000;
+let Inst{13-13} = 0b1;
+let Inst{31-16} = 0b0001111000000100;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vcvt_b_hf : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vd32.b = vcvt($Vu32.hf,$Vv32.hf)",
+tc_c127de3a, TypeCVI_VX>, Enc_45364e, Requires<[UseHVXV68,UseHVXIEEEFP]> {
+let Inst{7-5} = 0b110;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011111110;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vcvt_h_hf : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32),
+"$Vd32.h = vcvt($Vu32.hf)",
+tc_3c8c15d0, TypeCVI_VX>, Enc_e7581c, Requires<[UseHVXV68,UseHVXIEEEFP]> {
+let Inst{7-5} = 0b000;
+let Inst{13-13} = 0b1;
+let Inst{31-16} = 0b0001111000000110;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vcvt_hf_b : HInst<
+(outs HvxWR:$Vdd32),
+(ins HvxVR:$Vu32),
+"$Vdd32.hf = vcvt($Vu32.b)",
+tc_0afc8be9, TypeCVI_VX_DV>, Enc_dd766a, Requires<[UseHVXV68,UseHVXIEEEFP]> {
+let Inst{7-5} = 0b010;
+let Inst{13-13} = 0b1;
+let Inst{31-16} = 0b0001111000000100;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vcvt_hf_h : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32),
+"$Vd32.hf = vcvt($Vu32.h)",
+tc_3c8c15d0, TypeCVI_VX>, Enc_e7581c, Requires<[UseHVXV68,UseHVXIEEEFP]> {
+let Inst{7-5} = 0b111;
+let Inst{13-13} = 0b1;
+let Inst{31-16} = 0b0001111000000100;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vcvt_hf_sf : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vd32.hf = vcvt($Vu32.sf,$Vv32.sf)",
+tc_c127de3a, TypeCVI_VX>, Enc_45364e, Requires<[UseHVXV68,UseHVXIEEEFP]> {
+let Inst{7-5} = 0b001;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011111011;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vcvt_hf_ub : HInst<
+(outs HvxWR:$Vdd32),
+(ins HvxVR:$Vu32),
+"$Vdd32.hf = vcvt($Vu32.ub)",
+tc_0afc8be9, TypeCVI_VX_DV>, Enc_dd766a, Requires<[UseHVXV68,UseHVXIEEEFP]> {
+let Inst{7-5} = 0b001;
+let Inst{13-13} = 0b1;
+let Inst{31-16} = 0b0001111000000100;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vcvt_hf_uh : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32),
+"$Vd32.hf = vcvt($Vu32.uh)",
+tc_3c8c15d0, TypeCVI_VX>, Enc_e7581c, Requires<[UseHVXV68,UseHVXIEEEFP]> {
+let Inst{7-5} = 0b101;
+let Inst{13-13} = 0b1;
+let Inst{31-16} = 0b0001111000000100;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vcvt_sf_hf : HInst<
+(outs HvxWR:$Vdd32),
+(ins HvxVR:$Vu32),
+"$Vdd32.sf = vcvt($Vu32.hf)",
+tc_0afc8be9, TypeCVI_VX_DV>, Enc_dd766a, Requires<[UseHVXV68,UseHVXIEEEFP]> {
+let Inst{7-5} = 0b100;
+let Inst{13-13} = 0b1;
+let Inst{31-16} = 0b0001111000000100;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vcvt_ub_hf : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vd32.ub = vcvt($Vu32.hf,$Vv32.hf)",
+tc_c127de3a, TypeCVI_VX>, Enc_45364e, Requires<[UseHVXV68,UseHVXIEEEFP]> {
+let Inst{7-5} = 0b101;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011111110;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vcvt_uh_hf : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32),
+"$Vd32.uh = vcvt($Vu32.hf)",
+tc_3c8c15d0, TypeCVI_VX>, Enc_e7581c, Requires<[UseHVXV68,UseHVXIEEEFP]> {
+let Inst{7-5} = 0b000;
+let Inst{13-13} = 0b1;
+let Inst{31-16} = 0b0001111000000101;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
 def V6_vd0 : HInst<
 (outs HvxVR:$Vd32),
 (ins),
@@ -32141,6 +32576,34 @@ let opNewValue = 0;
 let isCVI = 1;
 let DecoderNamespace = "EXT_mmvec";
 }
+def V6_vdmpy_sf_hf : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vd32.sf = vdmpy($Vu32.hf,$Vv32.hf)",
+tc_c127de3a, TypeCVI_VX>, Enc_45364e, Requires<[UseHVXV68,UseHVXIEEEFP]> {
+let Inst{7-5} = 0b110;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011111101;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vdmpy_sf_hf_acc : HInst<
+(outs HvxVR:$Vx32),
+(ins HvxVR:$Vx32in, HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vx32.sf += vdmpy($Vu32.hf,$Vv32.hf)",
+tc_a19b9305, TypeCVI_VX>, Enc_a7341a, Requires<[UseHVXV68,UseHVXIEEEFP]> {
+let Inst{7-5} = 0b011;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011100010;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isAccumulator = 1;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+let Constraints = "$Vx32 = $Vx32in";
+}
 def V6_vdmpybus : HInst<
 (outs HvxVR:$Vd32),
 (ins HvxVR:$Vu32, IntRegs:$Rt32),
@@ -32415,7 +32878,7 @@ def V6_vdmpyhsat : HInst<
 (outs HvxVR:$Vd32),
 (ins HvxVR:$Vu32, IntRegs:$Rt32),
 "$Vd32.w = vdmpy($Vu32.h,$Rt32.h):sat",
-tc_0b04c6c7, TypeCVI_VX_DV>, Enc_b087ac, Requires<[UseHVXV60]> {
+tc_dcca380f, TypeCVI_VX>, Enc_b087ac, Requires<[UseHVXV60]> {
 let Inst{7-5} = 0b010;
 let Inst{13-13} = 0b0;
 let Inst{31-21} = 0b00011001001;
@@ -32428,7 +32891,7 @@ def V6_vdmpyhsat_acc : HInst<
 (outs HvxVR:$Vx32),
 (ins HvxVR:$Vx32in, HvxVR:$Vu32, IntRegs:$Rt32),
 "$Vx32.w += vdmpy($Vu32.h,$Rt32.h):sat",
-tc_660769f1, TypeCVI_VX_DV>, Enc_5138b3, Requires<[UseHVXV60]> {
+tc_72e2b393, TypeCVI_VX>, Enc_5138b3, Requires<[UseHVXV60]> {
 let Inst{7-5} = 0b011;
 let Inst{13-13} = 0b1;
 let Inst{31-21} = 0b00011001001;
@@ -32523,7 +32986,7 @@ def V6_vdmpyhsusat : HInst<
 (outs HvxVR:$Vd32),
 (ins HvxVR:$Vu32, IntRegs:$Rt32),
 "$Vd32.w = vdmpy($Vu32.h,$Rt32.uh):sat",
-tc_0b04c6c7, TypeCVI_VX_DV>, Enc_b087ac, Requires<[UseHVXV60]> {
+tc_dcca380f, TypeCVI_VX>, Enc_b087ac, Requires<[UseHVXV60]> {
 let Inst{7-5} = 0b000;
 let Inst{13-13} = 0b0;
 let Inst{31-21} = 0b00011001001;
@@ -32536,7 +32999,7 @@ def V6_vdmpyhsusat_acc : HInst<
 (outs HvxVR:$Vx32),
 (ins HvxVR:$Vx32in, HvxVR:$Vu32, IntRegs:$Rt32),
 "$Vx32.w += vdmpy($Vu32.h,$Rt32.uh):sat",
-tc_660769f1, TypeCVI_VX_DV>, Enc_5138b3, Requires<[UseHVXV60]> {
+tc_72e2b393, TypeCVI_VX>, Enc_5138b3, Requires<[UseHVXV60]> {
 let Inst{7-5} = 0b000;
 let Inst{13-13} = 0b1;
 let Inst{31-21} = 0b00011001001;
@@ -32577,7 +33040,7 @@ def V6_vdmpyhvsat : HInst<
 (outs HvxVR:$Vd32),
 (ins HvxVR:$Vu32, HvxVR:$Vv32),
 "$Vd32.w = vdmpy($Vu32.h,$Vv32.h):sat",
-tc_d8287c14, TypeCVI_VX_DV>, Enc_45364e, Requires<[UseHVXV60]> {
+tc_73efe966, TypeCVI_VX>, Enc_45364e, Requires<[UseHVXV60]> {
 let Inst{7-5} = 0b011;
 let Inst{13-13} = 0b0;
 let Inst{31-21} = 0b00011100000;
@@ -32831,6 +33294,84 @@ let isCVI = 1;
 let DecoderNamespace = "EXT_mmvec";
 let Constraints = "$Qx4 = $Qx4in";
 }
+def V6_vfmax_hf : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vd32.hf = vfmax($Vu32.hf,$Vv32.hf)",
+tc_cda936da, TypeCVI_VX_LATE>, Enc_45364e, Requires<[UseHVXV68,UseHVXIEEEFP]> {
+let Inst{7-5} = 0b010;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011100011;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vfmax_sf : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vd32.sf = vfmax($Vu32.sf,$Vv32.sf)",
+tc_cda936da, TypeCVI_VX_LATE>, Enc_45364e, Requires<[UseHVXV68,UseHVXIEEEFP]> {
+let Inst{7-5} = 0b011;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011100011;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vfmin_hf : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vd32.hf = vfmin($Vu32.hf,$Vv32.hf)",
+tc_cda936da, TypeCVI_VX_LATE>, Enc_45364e, Requires<[UseHVXV68,UseHVXIEEEFP]> {
+let Inst{7-5} = 0b000;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011100011;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vfmin_sf : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vd32.sf = vfmin($Vu32.sf,$Vv32.sf)",
+tc_cda936da, TypeCVI_VX_LATE>, Enc_45364e, Requires<[UseHVXV68,UseHVXIEEEFP]> {
+let Inst{7-5} = 0b001;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011100011;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vfneg_hf : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32),
+"$Vd32.hf = vfneg($Vu32.hf)",
+tc_5cdf8c84, TypeCVI_VX_LATE>, Enc_e7581c, Requires<[UseHVXV68,UseHVXIEEEFP]> {
+let Inst{7-5} = 0b010;
+let Inst{13-13} = 0b1;
+let Inst{31-16} = 0b0001111000000110;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vfneg_sf : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32),
+"$Vd32.sf = vfneg($Vu32.sf)",
+tc_5cdf8c84, TypeCVI_VX_LATE>, Enc_e7581c, Requires<[UseHVXV68,UseHVXIEEEFP]> {
+let Inst{7-5} = 0b011;
+let Inst{13-13} = 0b1;
+let Inst{31-16} = 0b0001111000000110;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
 def V6_vgathermh : HInst<
 (outs),
 (ins IntRegs:$Rt32, ModRegs:$Mu2, HvxVR:$Vv32),
@@ -32843,7 +33384,6 @@ let opNewValue = 0;
 let accessSize = HalfWordAccess;
 let isCVLoad = 1;
 let isCVI = 1;
-let hasTmpDst = 1;
 let mayLoad = 1;
 let Defs = [VTMP];
 let DecoderNamespace = "EXT_mmvec";
@@ -32860,7 +33400,6 @@ let opNewValue = 0;
 let accessSize = HalfWordAccess;
 let isCVLoad = 1;
 let isCVI = 1;
-let hasTmpDst = 1;
 let mayLoad = 1;
 let Defs = [VTMP];
 let DecoderNamespace = "EXT_mmvec";
@@ -32877,7 +33416,6 @@ let opNewValue = 0;
 let accessSize = HalfWordAccess;
 let isCVLoad = 1;
 let isCVI = 1;
-let hasTmpDst = 1;
 let mayLoad = 1;
 let Defs = [VTMP];
 let DecoderNamespace = "EXT_mmvec";
@@ -32894,7 +33432,6 @@ let opNewValue = 0;
 let accessSize = HalfWordAccess;
 let isCVLoad = 1;
 let isCVI = 1;
-let hasTmpDst = 1;
 let mayLoad = 1;
 let Defs = [VTMP];
 let DecoderNamespace = "EXT_mmvec";
@@ -32911,7 +33448,6 @@ let opNewValue = 0;
 let accessSize = WordAccess;
 let isCVLoad = 1;
 let isCVI = 1;
-let hasTmpDst = 1;
 let mayLoad = 1;
 let Defs = [VTMP];
 let DecoderNamespace = "EXT_mmvec";
@@ -32928,7 +33464,6 @@ let opNewValue = 0;
 let accessSize = WordAccess;
 let isCVLoad = 1;
 let isCVI = 1;
-let hasTmpDst = 1;
 let mayLoad = 1;
 let Defs = [VTMP];
 let DecoderNamespace = "EXT_mmvec";
@@ -33033,6 +33568,106 @@ let isCVI = 1;
 let DecoderNamespace = "EXT_mmvec";
 let Constraints = "$Qx4 = $Qx4in";
 }
+def V6_vgthf : HInst<
+(outs HvxQR:$Qd4),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Qd4 = vcmp.gt($Vu32.hf,$Vv32.hf)",
+tc_56c4f9fe, TypeCVI_VA>, Enc_95441f, Requires<[UseHVXV68,UseHVXFloatingPoint]> {
+let Inst{7-2} = 0b011101;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011100100;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vgthf_and : HInst<
+(outs HvxQR:$Qx4),
+(ins HvxQR:$Qx4in, HvxVR:$Vu32, HvxVR:$Vv32),
+"$Qx4 &= vcmp.gt($Vu32.hf,$Vv32.hf)",
+tc_257f6f7c, TypeCVI_VA>, Enc_eaa9f8, Requires<[UseHVXV68,UseHVXFloatingPoint]> {
+let Inst{7-2} = 0b110011;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011100100;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+let Constraints = "$Qx4 = $Qx4in";
+}
+def V6_vgthf_or : HInst<
+(outs HvxQR:$Qx4),
+(ins HvxQR:$Qx4in, HvxVR:$Vu32, HvxVR:$Vv32),
+"$Qx4 |= vcmp.gt($Vu32.hf,$Vv32.hf)",
+tc_257f6f7c, TypeCVI_VA>, Enc_eaa9f8, Requires<[UseHVXV68,UseHVXFloatingPoint]> {
+let Inst{7-2} = 0b001101;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011100100;
+let isAccumulator = 1;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+let Constraints = "$Qx4 = $Qx4in";
+}
+def V6_vgthf_xor : HInst<
+(outs HvxQR:$Qx4),
+(ins HvxQR:$Qx4in, HvxVR:$Vu32, HvxVR:$Vv32),
+"$Qx4 ^= vcmp.gt($Vu32.hf,$Vv32.hf)",
+tc_257f6f7c, TypeCVI_VA>, Enc_eaa9f8, Requires<[UseHVXV68,UseHVXFloatingPoint]> {
+let Inst{7-2} = 0b111011;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011100100;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+let Constraints = "$Qx4 = $Qx4in";
+}
+def V6_vgtsf : HInst<
+(outs HvxQR:$Qd4),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Qd4 = vcmp.gt($Vu32.sf,$Vv32.sf)",
+tc_56c4f9fe, TypeCVI_VA>, Enc_95441f, Requires<[UseHVXV68,UseHVXFloatingPoint]> {
+let Inst{7-2} = 0b011100;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011100100;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vgtsf_and : HInst<
+(outs HvxQR:$Qx4),
+(ins HvxQR:$Qx4in, HvxVR:$Vu32, HvxVR:$Vv32),
+"$Qx4 &= vcmp.gt($Vu32.sf,$Vv32.sf)",
+tc_257f6f7c, TypeCVI_VA>, Enc_eaa9f8, Requires<[UseHVXV68,UseHVXFloatingPoint]> {
+let Inst{7-2} = 0b110010;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011100100;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+let Constraints = "$Qx4 = $Qx4in";
+}
+def V6_vgtsf_or : HInst<
+(outs HvxQR:$Qx4),
+(ins HvxQR:$Qx4in, HvxVR:$Vu32, HvxVR:$Vv32),
+"$Qx4 |= vcmp.gt($Vu32.sf,$Vv32.sf)",
+tc_257f6f7c, TypeCVI_VA>, Enc_eaa9f8, Requires<[UseHVXV68,UseHVXFloatingPoint]> {
+let Inst{7-2} = 0b001100;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011100100;
+let isAccumulator = 1;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+let Constraints = "$Qx4 = $Qx4in";
+}
+def V6_vgtsf_xor : HInst<
+(outs HvxQR:$Qx4),
+(ins HvxQR:$Qx4in, HvxVR:$Vu32, HvxVR:$Vv32),
+"$Qx4 ^= vcmp.gt($Vu32.sf,$Vv32.sf)",
+tc_257f6f7c, TypeCVI_VA>, Enc_eaa9f8, Requires<[UseHVXV68,UseHVXFloatingPoint]> {
+let Inst{7-2} = 0b111010;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011100100;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+let Constraints = "$Qx4 = $Qx4in";
+}
 def V6_vgtub : HInst<
 (outs HvxQR:$Qd4),
 (ins HvxVR:$Vu32, HvxVR:$Vv32),
@@ -33552,6 +34187,32 @@ let opNewValue = 0;
 let isCVI = 1;
 let DecoderNamespace = "EXT_mmvec";
 }
+def V6_vmax_hf : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vd32.hf = vmax($Vu32.hf,$Vv32.hf)",
+tc_56c4f9fe, TypeCVI_VA>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> {
+let Inst{7-5} = 0b011;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011111110;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vmax_sf : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vd32.sf = vmax($Vu32.sf,$Vv32.sf)",
+tc_56c4f9fe, TypeCVI_VA>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> {
+let Inst{7-5} = 0b001;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011111110;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
 def V6_vmaxb : HInst<
 (outs HvxVR:$Vd32),
 (ins HvxVR:$Vu32, HvxVR:$Vv32),
@@ -33677,6 +34338,32 @@ let isPseudo = 1;
 let isCodeGenOnly = 1;
 let DecoderNamespace = "EXT_mmvec";
 }
+def V6_vmin_hf : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vd32.hf = vmin($Vu32.hf,$Vv32.hf)",
+tc_56c4f9fe, TypeCVI_VA>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> {
+let Inst{7-5} = 0b100;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011111110;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vmin_sf : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vd32.sf = vmin($Vu32.sf,$Vv32.sf)",
+tc_56c4f9fe, TypeCVI_VA>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> {
+let Inst{7-5} = 0b010;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011111110;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
 def V6_vminb : HInst<
 (outs HvxVR:$Vd32),
 (ins HvxVR:$Vu32, HvxVR:$Vv32),
@@ -34110,6 +34797,179 @@ let isCVI = 1;
 let DecoderNamespace = "EXT_mmvec";
 let Constraints = "$Vx32 = $Vx32in";
 }
+def V6_vmpy_hf_hf : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vd32.hf = vmpy($Vu32.hf,$Vv32.hf)",
+tc_c127de3a, TypeCVI_VX>, Enc_45364e, Requires<[UseHVXV68,UseHVXIEEEFP]> {
+let Inst{7-5} = 0b011;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011111100;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vmpy_hf_hf_acc : HInst<
+(outs HvxVR:$Vx32),
+(ins HvxVR:$Vx32in, HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vx32.hf += vmpy($Vu32.hf,$Vv32.hf)",
+tc_a19b9305, TypeCVI_VX>, Enc_a7341a, Requires<[UseHVXV68,UseHVXIEEEFP]> {
+let Inst{7-5} = 0b010;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011100010;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isAccumulator = 1;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+let Constraints = "$Vx32 = $Vx32in";
+}
+def V6_vmpy_qf16 : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vd32.qf16 = vmpy($Vu32.qf16,$Vv32.qf16)",
+tc_d8287c14, TypeCVI_VX_DV>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> {
+let Inst{7-5} = 0b011;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011111111;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vmpy_qf16_hf : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vd32.qf16 = vmpy($Vu32.hf,$Vv32.hf)",
+tc_d8287c14, TypeCVI_VX_DV>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> {
+let Inst{7-5} = 0b100;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011111111;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vmpy_qf16_mix_hf : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vd32.qf16 = vmpy($Vu32.qf16,$Vv32.hf)",
+tc_d8287c14, TypeCVI_VX_DV>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> {
+let Inst{7-5} = 0b101;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011111111;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vmpy_qf32 : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vd32.qf32 = vmpy($Vu32.qf32,$Vv32.qf32)",
+tc_d8287c14, TypeCVI_VX_DV>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> {
+let Inst{7-5} = 0b000;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011111111;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vmpy_qf32_hf : HInst<
+(outs HvxWR:$Vdd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vdd32.qf32 = vmpy($Vu32.hf,$Vv32.hf)",
+tc_d8287c14, TypeCVI_VX_DV>, Enc_71bb9b, Requires<[UseHVXV68,UseHVXQFloat]> {
+let Inst{7-5} = 0b111;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011111111;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vmpy_qf32_mix_hf : HInst<
+(outs HvxWR:$Vdd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vdd32.qf32 = vmpy($Vu32.qf16,$Vv32.hf)",
+tc_d8287c14, TypeCVI_VX_DV>, Enc_71bb9b, Requires<[UseHVXV68,UseHVXQFloat]> {
+let Inst{7-5} = 0b000;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011111100;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vmpy_qf32_qf16 : HInst<
+(outs HvxWR:$Vdd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vdd32.qf32 = vmpy($Vu32.qf16,$Vv32.qf16)",
+tc_d8287c14, TypeCVI_VX_DV>, Enc_71bb9b, Requires<[UseHVXV68,UseHVXQFloat]> {
+let Inst{7-5} = 0b110;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011111111;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vmpy_qf32_sf : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vd32.qf32 = vmpy($Vu32.sf,$Vv32.sf)",
+tc_d8287c14, TypeCVI_VX_DV>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> {
+let Inst{7-5} = 0b001;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011111111;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vmpy_sf_hf : HInst<
+(outs HvxWR:$Vdd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vdd32.sf = vmpy($Vu32.hf,$Vv32.hf)",
+tc_d8287c14, TypeCVI_VX_DV>, Enc_71bb9b, Requires<[UseHVXV68,UseHVXIEEEFP]> {
+let Inst{7-5} = 0b010;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011111100;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vmpy_sf_hf_acc : HInst<
+(outs HvxWR:$Vxx32),
+(ins HvxWR:$Vxx32in, HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vxx32.sf += vmpy($Vu32.hf,$Vv32.hf)",
+tc_08a4f1b6, TypeCVI_VX_DV>, Enc_3fc427, Requires<[UseHVXV68,UseHVXIEEEFP]> {
+let Inst{7-5} = 0b001;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011100010;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isAccumulator = 1;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+let Constraints = "$Vxx32 = $Vxx32in";
+}
+def V6_vmpy_sf_sf : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vd32.sf = vmpy($Vu32.sf,$Vv32.sf)",
+tc_d8287c14, TypeCVI_VX_DV>, Enc_45364e, Requires<[UseHVXV68,UseHVXIEEEFP]> {
+let Inst{7-5} = 0b001;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011111100;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
 def V6_vmpybus : HInst<
 (outs HvxWR:$Vdd32),
 (ins HvxVR:$Vu32, IntRegs:$Rt32),
@@ -34397,7 +35257,7 @@ def V6_vmpyhsrs : HInst<
 (outs HvxVR:$Vd32),
 (ins HvxVR:$Vu32, IntRegs:$Rt32),
 "$Vd32.h = vmpy($Vu32.h,$Rt32.h):<<1:rnd:sat",
-tc_0b04c6c7, TypeCVI_VX_DV>, Enc_b087ac, Requires<[UseHVXV60]> {
+tc_dcca380f, TypeCVI_VX>, Enc_b087ac, Requires<[UseHVXV60]> {
 let Inst{7-5} = 0b010;
 let Inst{13-13} = 0b0;
 let Inst{31-21} = 0b00011001010;
@@ -34422,7 +35282,7 @@ def V6_vmpyhss : HInst<
 (outs HvxVR:$Vd32),
 (ins HvxVR:$Vu32, IntRegs:$Rt32),
 "$Vd32.h = vmpy($Vu32.h,$Rt32.h):<<1:sat",
-tc_0b04c6c7, TypeCVI_VX_DV>, Enc_b087ac, Requires<[UseHVXV60]> {
+tc_dcca380f, TypeCVI_VX>, Enc_b087ac, Requires<[UseHVXV60]> {
 let Inst{7-5} = 0b001;
 let Inst{13-13} = 0b0;
 let Inst{31-21} = 0b00011001010;
@@ -34555,7 +35415,7 @@ def V6_vmpyhvsrs : HInst<
 (outs HvxVR:$Vd32),
 (ins HvxVR:$Vu32, HvxVR:$Vv32),
 "$Vd32.h = vmpy($Vu32.h,$Vv32.h):<<1:rnd:sat",
-tc_d8287c14, TypeCVI_VX_DV>, Enc_45364e, Requires<[UseHVXV60]> {
+tc_73efe966, TypeCVI_VX>, Enc_45364e, Requires<[UseHVXV60]> {
 let Inst{7-5} = 0b001;
 let Inst{13-13} = 0b0;
 let Inst{31-21} = 0b00011100001;
@@ -35332,6 +36192,19 @@ let isPseudo = 1;
 let isCodeGenOnly = 1;
 let DecoderNamespace = "EXT_mmvec";
 }
+def V6_vmpyuhvs : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vd32.uh = vmpy($Vu32.uh,$Vv32.uh):>>16",
+tc_c127de3a, TypeCVI_VX>, Enc_45364e, Requires<[UseHVXV69]> {
+let Inst{7-5} = 0b111;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011111110;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
 def V6_vmux : HInst<
 (outs HvxVR:$Vd32),
 (ins HvxQR:$Qt4, HvxVR:$Vu32, HvxVR:$Vv32),
@@ -36007,7 +36880,7 @@ def V6_vrmpybusv_acc : HInst<
 (outs HvxVR:$Vx32),
 (ins HvxVR:$Vx32in, HvxVR:$Vu32, HvxVR:$Vv32),
 "$Vx32.w += vrmpy($Vu32.ub,$Vv32.b)",
-tc_08a4f1b6, TypeCVI_VX_DV>, Enc_a7341a, Requires<[UseHVXV60]> {
+tc_37820f4c, TypeCVI_VX>, Enc_a7341a, Requires<[UseHVXV60]> {
 let Inst{7-5} = 0b010;
 let Inst{13-13} = 0b1;
 let Inst{31-21} = 0b00011100000;
@@ -36061,7 +36934,7 @@ def V6_vrmpybv_acc : HInst<
 (outs HvxVR:$Vx32),
 (ins HvxVR:$Vx32in, HvxVR:$Vu32, HvxVR:$Vv32),
 "$Vx32.w += vrmpy($Vu32.b,$Vv32.b)",
-tc_08a4f1b6, TypeCVI_VX_DV>, Enc_a7341a, Requires<[UseHVXV60]> {
+tc_37820f4c, TypeCVI_VX>, Enc_a7341a, Requires<[UseHVXV60]> {
 let Inst{7-5} = 0b001;
 let Inst{13-13} = 0b1;
 let Inst{31-21} = 0b00011100000;
@@ -36277,7 +37150,7 @@ def V6_vrmpyubv_acc : HInst<
 (outs HvxVR:$Vx32),
 (ins HvxVR:$Vx32in, HvxVR:$Vu32, HvxVR:$Vv32),
 "$Vx32.uw += vrmpy($Vu32.ub,$Vv32.ub)",
-tc_08a4f1b6, TypeCVI_VX_DV>, Enc_a7341a, Requires<[UseHVXV60]> {
+tc_37820f4c, TypeCVI_VX>, Enc_a7341a, Requires<[UseHVXV60]> {
 let Inst{7-5} = 0b000;
 let Inst{13-13} = 0b1;
 let Inst{31-21} = 0b00011100000;
@@ -37412,6 +38285,123 @@ let isPseudo = 1;
 let isCodeGenOnly = 1;
 let DecoderNamespace = "EXT_mmvec";
 }
+def V6_vsub_hf : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vd32.qf16 = vsub($Vu32.hf,$Vv32.hf)",
+tc_05ca8cfd, TypeCVI_VS>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> {
+let Inst{7-5} = 0b110;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011111011;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vsub_hf_hf : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vd32.hf = vsub($Vu32.hf,$Vv32.hf)",
+tc_c127de3a, TypeCVI_VX>, Enc_45364e, Requires<[UseHVXV68,UseHVXIEEEFP]> {
+let Inst{7-5} = 0b000;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011111011;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vsub_qf16 : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vd32.qf16 = vsub($Vu32.qf16,$Vv32.qf16)",
+tc_05ca8cfd, TypeCVI_VS>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> {
+let Inst{7-5} = 0b101;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011111011;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vsub_qf16_mix : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vd32.qf16 = vsub($Vu32.qf16,$Vv32.hf)",
+tc_05ca8cfd, TypeCVI_VS>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> {
+let Inst{7-5} = 0b111;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011111011;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vsub_qf32 : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vd32.qf32 = vsub($Vu32.qf32,$Vv32.qf32)",
+tc_05ca8cfd, TypeCVI_VS>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> {
+let Inst{7-5} = 0b011;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011111101;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vsub_qf32_mix : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vd32.qf32 = vsub($Vu32.qf32,$Vv32.sf)",
+tc_05ca8cfd, TypeCVI_VS>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> {
+let Inst{7-5} = 0b101;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011111101;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vsub_sf : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vd32.qf32 = vsub($Vu32.sf,$Vv32.sf)",
+tc_05ca8cfd, TypeCVI_VS>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> {
+let Inst{7-5} = 0b100;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011111101;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vsub_sf_hf : HInst<
+(outs HvxWR:$Vdd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vdd32.sf = vsub($Vu32.hf,$Vv32.hf)",
+tc_d8287c14, TypeCVI_VX_DV>, Enc_71bb9b, Requires<[UseHVXV68,UseHVXIEEEFP]> {
+let Inst{7-5} = 0b101;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011111100;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
+def V6_vsub_sf_sf : HInst<
+(outs HvxVR:$Vd32),
+(ins HvxVR:$Vu32, HvxVR:$Vv32),
+"$Vd32.sf = vsub($Vu32.sf,$Vv32.sf)",
+tc_c127de3a, TypeCVI_VX>, Enc_45364e, Requires<[UseHVXV68,UseHVXIEEEFP]> {
+let Inst{7-5} = 0b111;
+let Inst{13-13} = 0b1;
+let Inst{31-21} = 0b00011111100;
+let hasNewValue = 1;
+let opNewValue = 0;
+let isCVI = 1;
+let DecoderNamespace = "EXT_mmvec";
+}
 def V6_vsubb : HInst<
 (outs HvxVR:$Vd32),
 (ins HvxVR:$Vu32, HvxVR:$Vv32),
@@ -38647,7 +39637,7 @@ def V6_zLd_ai : HInst<
 (outs),
 (ins IntRegs:$Rt32, s4_0Imm:$Ii),
 "z = vmem($Rt32+#$Ii)",
-tc_e699ae41, TypeCVI_ZW>, Enc_ff3442, Requires<[UseHVXV66,UseZReg]> {
+tc_e699ae41, TypeCVI_ZW>, Enc_ff3442, Requires<[UseHVXV66,UseZReg]>, PostInc_BaseImm {
 let Inst{7-0} = 0b00000000;
 let Inst{12-11} = 0b00;
 let Inst{31-21} = 0b00101100000;
@@ -38655,13 +39645,14 @@ let addrMode = BaseImmOffset;
 let isCVI = 1;
 let mayLoad = 1;
 let isRestrictNoSlot1Store = 1;
+let CextOpcode = "V6_zLd";
 let DecoderNamespace = "EXT_mmvec";
 }
 def V6_zLd_pi : HInst<
 (outs IntRegs:$Rx32),
 (ins IntRegs:$Rx32in, s3_0Imm:$Ii),
 "z = vmem($Rx32++#$Ii)",
-tc_a0dbea28, TypeCVI_ZW>, Enc_6c9ee0, Requires<[UseHVXV66,UseZReg]> {
+tc_a0dbea28, TypeCVI_ZW>, Enc_6c9ee0, Requires<[UseHVXV66,UseZReg]>, PostInc_BaseImm {
 let Inst{7-0} = 0b00000000;
 let Inst{13-11} = 0b000;
 let Inst{31-21} = 0b00101101000;
@@ -38669,6 +39660,7 @@ let addrMode = PostInc;
 let isCVI = 1;
 let mayLoad = 1;
 let isRestrictNoSlot1Store = 1;
+let CextOpcode = "V6_zLd";
 let DecoderNamespace = "EXT_mmvec";
 let Constraints = "$Rx32 = $Rx32in";
 }
@@ -38782,6 +39774,17 @@ let Inst{13-0} = 0b00000000000000;
 let Inst{31-16} = 0b0110110000100000;
 let isSolo = 1;
 }
+def Y2_crswap_old : HInst<
+(outs IntRegs:$Rx32),
+(ins IntRegs:$Rx32in),
+"crswap($Rx32,sgp)",
+PSEUDO, TypeMAPPING> {
+let hasNewValue = 1;
+let opNewValue = 0;
+let isPseudo = 1;
+let isCodeGenOnly = 1;
+let Constraints = "$Rx32 = $Rx32in";
+}
 def Y2_dccleana : HInst<
 (outs),
 (ins IntRegs:$Rs32),
@@ -38861,6 +39864,22 @@ let Inst{13-0} = 0b00000000000010;
 let Inst{31-16} = 0b0101011111000000;
 let isSolo = 1;
 }
+def Y2_k1lock_map : HInst<
+(outs),
+(ins),
+"k1lock",
+PSEUDO, TypeMAPPING>, Requires<[HasV65]> {
+let isPseudo = 1;
+let isCodeGenOnly = 1;
+}
+def Y2_k1unlock_map : HInst<
+(outs),
+(ins),
+"k1unlock",
+PSEUDO, TypeMAPPING>, Requires<[HasV65]> {
+let isPseudo = 1;
+let isCodeGenOnly = 1;
+}
 def Y2_syncht : HInst<
 (outs),
 (ins),
@@ -39083,7 +40102,7 @@ def dup_A2_add : HInst<
 (outs IntRegs:$Rd32),
 (ins IntRegs:$Rs32, IntRegs:$Rt32),
 "$Rd32 = add($Rs32,$Rt32)",
-tc_388f9897, TypeALU32_3op>, Requires<[HasV68]> {
+tc_388f9897, TypeALU32_3op>, Requires<[HasV69]> {
 let hasNewValue = 1;
 let opNewValue = 0;
 let AsmVariantName = "NonParsable";
@@ -39093,7 +40112,7 @@ def dup_A2_addi : HInst<
 (outs IntRegs:$Rd32),
 (ins IntRegs:$Rs32, s32_0Imm:$Ii),
 "$Rd32 = add($Rs32,#$Ii)",
-tc_388f9897, TypeALU32_ADDI>, Requires<[HasV68]> {
+tc_388f9897, TypeALU32_ADDI>, Requires<[HasV69]> {
 let hasNewValue = 1;
 let opNewValue = 0;
 let AsmVariantName = "NonParsable";
@@ -39108,7 +40127,7 @@ def dup_A2_andir : HInst<
 (outs IntRegs:$Rd32),
 (ins IntRegs:$Rs32, s32_0Imm:$Ii),
 "$Rd32 = and($Rs32,#$Ii)",
-tc_388f9897, TypeALU32_2op>, Requires<[HasV68]> {
+tc_388f9897, TypeALU32_2op>, Requires<[HasV69]> {
 let hasNewValue = 1;
 let opNewValue = 0;
 let AsmVariantName = "NonParsable";
@@ -39123,7 +40142,7 @@ def dup_A2_combineii : HInst<
 (outs DoubleRegs:$Rdd32),
 (ins s32_0Imm:$Ii, s8_0Imm:$II),
 "$Rdd32 = combine(#$Ii,#$II)",
-tc_388f9897, TypeALU32_2op>, Requires<[HasV68]> {
+tc_388f9897, TypeALU32_2op>, Requires<[HasV69]> {
 let AsmVariantName = "NonParsable";
 let isPseudo = 1;
 let isExtendable = 1;
@@ -39136,7 +40155,7 @@ def dup_A2_sxtb : HInst<
 (outs IntRegs:$Rd32),
 (ins IntRegs:$Rs32),
 "$Rd32 = sxtb($Rs32)",
-tc_9124c04f, TypeALU32_2op>, Requires<[HasV68]> {
+tc_9124c04f, TypeALU32_2op>, Requires<[HasV69]> {
 let hasNewValue = 1;
 let opNewValue = 0;
 let AsmVariantName = "NonParsable";
@@ -39146,7 +40165,7 @@ def dup_A2_sxth : HInst<
 (outs IntRegs:$Rd32),
 (ins IntRegs:$Rs32),
 "$Rd32 = sxth($Rs32)",
-tc_9124c04f, TypeALU32_2op>, Requires<[HasV68]> {
+tc_9124c04f, TypeALU32_2op>, Requires<[HasV69]> {
 let hasNewValue = 1;
 let opNewValue = 0;
 let AsmVariantName = "NonParsable";
@@ -39156,7 +40175,7 @@ def dup_A2_tfr : HInst<
 (outs IntRegs:$Rd32),
 (ins IntRegs:$Rs32),
 "$Rd32 = $Rs32",
-tc_9124c04f, TypeALU32_2op>, Requires<[HasV68]> {
+tc_9124c04f, TypeALU32_2op>, Requires<[HasV69]> {
 let hasNewValue = 1;
 let opNewValue = 0;
 let AsmVariantName = "NonParsable";
@@ -39166,7 +40185,7 @@ def dup_A2_tfrsi : HInst<
 (outs IntRegs:$Rd32),
 (ins s32_0Imm:$Ii),
 "$Rd32 = #$Ii",
-tc_9124c04f, TypeALU32_2op>, Requires<[HasV68]> {
+tc_9124c04f, TypeALU32_2op>, Requires<[HasV69]> {
 let hasNewValue = 1;
 let opNewValue = 0;
 let AsmVariantName = "NonParsable";
@@ -39181,7 +40200,7 @@ def dup_A2_zxtb : HInst<
 (outs IntRegs:$Rd32),
 (ins IntRegs:$Rs32),
 "$Rd32 = zxtb($Rs32)",
-PSEUDO, TypeMAPPING>, Requires<[HasV68]> {
+PSEUDO, TypeMAPPING>, Requires<[HasV69]> {
 let hasNewValue = 1;
 let opNewValue = 0;
 let AsmVariantName = "NonParsable";
@@ -39191,7 +40210,7 @@ def dup_A2_zxth : HInst<
 (outs IntRegs:$Rd32),
 (ins IntRegs:$Rs32),
 "$Rd32 = zxth($Rs32)",
-tc_9124c04f, TypeALU32_2op>, Requires<[HasV68]> {
+tc_9124c04f, TypeALU32_2op>, Requires<[HasV69]> {
 let hasNewValue = 1;
 let opNewValue = 0;
 let AsmVariantName = "NonParsable";
@@ -39201,7 +40220,7 @@ def dup_A4_combineii : HInst<
 (outs DoubleRegs:$Rdd32),
 (ins s8_0Imm:$Ii, u32_0Imm:$II),
 "$Rdd32 = combine(#$Ii,#$II)",
-tc_388f9897, TypeALU32_2op>, Requires<[HasV68]> {
+tc_388f9897, TypeALU32_2op>, Requires<[HasV69]> {
 let AsmVariantName = "NonParsable";
 let isPseudo = 1;
 let isExtendable = 1;
@@ -39214,7 +40233,7 @@ def dup_A4_combineir : HInst<
 (outs DoubleRegs:$Rdd32),
 (ins s32_0Imm:$Ii, IntRegs:$Rs32),
 "$Rdd32 = combine(#$Ii,$Rs32)",
-tc_388f9897, TypeALU32_2op>, Requires<[HasV68]> {
+tc_388f9897, TypeALU32_2op>, Requires<[HasV69]> {
 let AsmVariantName = "NonParsable";
 let isPseudo = 1;
 let isExtendable = 1;
@@ -39227,7 +40246,7 @@ def dup_A4_combineri : HInst<
 (outs DoubleRegs:$Rdd32),
 (ins IntRegs:$Rs32, s32_0Imm:$Ii),
 "$Rdd32 = combine($Rs32,#$Ii)",
-tc_388f9897, TypeALU32_2op>, Requires<[HasV68]> {
+tc_388f9897, TypeALU32_2op>, Requires<[HasV69]> {
 let AsmVariantName = "NonParsable";
 let isPseudo = 1;
 let isExtendable = 1;
@@ -39240,7 +40259,7 @@ def dup_C2_cmoveif : HInst<
 (outs IntRegs:$Rd32),
 (ins PredRegs:$Pu4, s32_0Imm:$Ii),
 "if (!$Pu4) $Rd32 = #$Ii",
-tc_388f9897, TypeALU32_2op>, Requires<[HasV68]> {
+tc_388f9897, TypeALU32_2op>, Requires<[HasV69]> {
 let isPredicated = 1;
 let isPredicatedFalse = 1;
 let hasNewValue = 1;
@@ -39257,7 +40276,7 @@ def dup_C2_cmoveit : HInst<
 (outs IntRegs:$Rd32),
 (ins PredRegs:$Pu4, s32_0Imm:$Ii),
 "if ($Pu4) $Rd32 = #$Ii",
-tc_388f9897, TypeALU32_2op>, Requires<[HasV68]> {
+tc_388f9897, TypeALU32_2op>, Requires<[HasV69]> {
 let isPredicated = 1;
 let hasNewValue = 1;
 let opNewValue = 0;
@@ -39273,7 +40292,7 @@ def dup_C2_cmovenewif : HInst<
 (outs IntRegs:$Rd32),
 (ins PredRegs:$Pu4, s32_0Imm:$Ii),
 "if (!$Pu4.new) $Rd32 = #$Ii",
-tc_4ac61d92, TypeALU32_2op>, Requires<[HasV68]> {
+tc_4ac61d92, TypeALU32_2op>, Requires<[HasV69]> {
 let isPredicated = 1;
 let isPredicatedFalse = 1;
 let hasNewValue = 1;
@@ -39291,7 +40310,7 @@ def dup_C2_cmovenewit : HInst<
 (outs IntRegs:$Rd32),
 (ins PredRegs:$Pu4, s32_0Imm:$Ii),
 "if ($Pu4.new) $Rd32 = #$Ii",
-tc_4ac61d92, TypeALU32_2op>, Requires<[HasV68]> {
+tc_4ac61d92, TypeALU32_2op>, Requires<[HasV69]> {
 let isPredicated = 1;
 let hasNewValue = 1;
 let opNewValue = 0;
@@ -39308,7 +40327,7 @@ def dup_C2_cmpeqi : HInst<
 (outs PredRegs:$Pd4),
 (ins IntRegs:$Rs32, s32_0Imm:$Ii),
 "$Pd4 = cmp.eq($Rs32,#$Ii)",
-tc_388f9897, TypeALU32_2op>, Requires<[HasV68]> {
+tc_388f9897, TypeALU32_2op>, Requires<[HasV69]> {
 let AsmVariantName = "NonParsable";
 let isPseudo = 1;
 let isExtendable = 1;
@@ -39321,7 +40340,7 @@ def dup_L2_deallocframe : HInst<
 (outs DoubleRegs:$Rdd32),
 (ins IntRegs:$Rs32),
 "$Rdd32 = deallocframe($Rs32):raw",
-tc_aee6250c, TypeLD>, Requires<[HasV68]> {
+tc_aee6250c, TypeLD>, Requires<[HasV69]> {
 let accessSize = DoubleWordAccess;
 let AsmVariantName = "NonParsable";
 let mayLoad = 1;
@@ -39333,7 +40352,7 @@ def dup_L2_loadrb_io : HInst<
 (outs IntRegs:$Rd32),
 (ins IntRegs:$Rs32, s32_0Imm:$Ii),
 "$Rd32 = memb($Rs32+#$Ii)",
-tc_eed07714, TypeLD>, Requires<[HasV68]> {
+tc_eed07714, TypeLD>, Requires<[HasV69]> {
 let hasNewValue = 1;
 let opNewValue = 0;
 let addrMode = BaseImmOffset;
@@ -39351,7 +40370,7 @@ def dup_L2_loadrd_io : HInst<
 (outs DoubleRegs:$Rdd32),
 (ins IntRegs:$Rs32, s29_3Imm:$Ii),
 "$Rdd32 = memd($Rs32+#$Ii)",
-tc_eed07714, TypeLD>, Requires<[HasV68]> {
+tc_eed07714, TypeLD>, Requires<[HasV69]> {
 let addrMode = BaseImmOffset;
 let accessSize = DoubleWordAccess;
 let AsmVariantName = "NonParsable";
@@ -39367,7 +40386,7 @@ def dup_L2_loadrh_io : HInst<
 (outs IntRegs:$Rd32),
 (ins IntRegs:$Rs32, s31_1Imm:$Ii),
 "$Rd32 = memh($Rs32+#$Ii)",
-tc_eed07714, TypeLD>, Requires<[HasV68]> {
+tc_eed07714, TypeLD>, Requires<[HasV69]> {
 let hasNewValue = 1;
 let opNewValue = 0;
 let addrMode = BaseImmOffset;
@@ -39385,7 +40404,7 @@ def dup_L2_loadri_io : HInst<
 (outs IntRegs:$Rd32),
 (ins IntRegs:$Rs32, s30_2Imm:$Ii),
 "$Rd32 = memw($Rs32+#$Ii)",
-tc_eed07714, TypeLD>, Requires<[HasV68]> {
+tc_eed07714, TypeLD>, Requires<[HasV69]> {
 let hasNewValue = 1;
 let opNewValue = 0;
 let addrMode = BaseImmOffset;
@@ -39403,7 +40422,7 @@ def dup_L2_loadrub_io : HInst<
 (outs IntRegs:$Rd32),
 (ins IntRegs:$Rs32, s32_0Imm:$Ii),
 "$Rd32 = memub($Rs32+#$Ii)",
-tc_eed07714, TypeLD>, Requires<[HasV68]> {
+tc_eed07714, TypeLD>, Requires<[HasV69]> {
 let hasNewValue = 1;
 let opNewValue = 0;
 let addrMode = BaseImmOffset;
@@ -39421,7 +40440,7 @@ def dup_L2_loadruh_io : HInst<
 (outs IntRegs:$Rd32),
 (ins IntRegs:$Rs32, s31_1Imm:$Ii),
 "$Rd32 = memuh($Rs32+#$Ii)",
-tc_eed07714, TypeLD>, Requires<[HasV68]> {
+tc_eed07714, TypeLD>, Requires<[HasV69]> {
 let hasNewValue = 1;
 let opNewValue = 0;
 let addrMode = BaseImmOffset;
@@ -39439,7 +40458,7 @@ def dup_S2_allocframe : HInst<
 (outs IntRegs:$Rx32),
 (ins IntRegs:$Rx32in, u11_3Imm:$Ii),
 "allocframe($Rx32,#$Ii):raw",
-tc_74a42bda, TypeST>, Requires<[HasV68]> {
+tc_74a42bda, TypeST>, Requires<[HasV69]> {
 let hasNewValue = 1;
 let opNewValue = 0;
 let addrMode = BaseImmOffset;
@@ -39455,7 +40474,7 @@ def dup_S2_storerb_io : HInst<
 (outs),
 (ins IntRegs:$Rs32, s32_0Imm:$Ii, IntRegs:$Rt32),
 "memb($Rs32+#$Ii) = $Rt32",
-tc_a9edeffa, TypeST>, Requires<[HasV68]> {
+tc_a9edeffa, TypeST>, Requires<[HasV69]> {
 let addrMode = BaseImmOffset;
 let accessSize = ByteAccess;
 let AsmVariantName = "NonParsable";
@@ -39471,7 +40490,7 @@ def dup_S2_storerd_io : HInst<
 (outs),
 (ins IntRegs:$Rs32, s29_3Imm:$Ii, DoubleRegs:$Rtt32),
 "memd($Rs32+#$Ii) = $Rtt32",
-tc_a9edeffa, TypeST>, Requires<[HasV68]> {
+tc_a9edeffa, TypeST>, Requires<[HasV69]> {
 let addrMode = BaseImmOffset;
 let accessSize = DoubleWordAccess;
 let AsmVariantName = "NonParsable";
@@ -39487,7 +40506,7 @@ def dup_S2_storerh_io : HInst<
 (outs),
 (ins IntRegs:$Rs32, s31_1Imm:$Ii, IntRegs:$Rt32),
 "memh($Rs32+#$Ii) = $Rt32",
-tc_a9edeffa, TypeST>, Requires<[HasV68]> {
+tc_a9edeffa, TypeST>, Requires<[HasV69]> {
 let addrMode = BaseImmOffset;
 let accessSize = HalfWordAccess;
 let AsmVariantName = "NonParsable";
@@ -39503,7 +40522,7 @@ def dup_S2_storeri_io : HInst<
 (outs),
 (ins IntRegs:$Rs32, s30_2Imm:$Ii, IntRegs:$Rt32),
 "memw($Rs32+#$Ii) = $Rt32",
-tc_a9edeffa, TypeST>, Requires<[HasV68]> {
+tc_a9edeffa, TypeST>, Requires<[HasV69]> {
 let addrMode = BaseImmOffset;
 let accessSize = WordAccess;
 let AsmVariantName = "NonParsable";
@@ -39519,7 +40538,7 @@ def dup_S4_storeirb_io : HInst<
 (outs),
 (ins IntRegs:$Rs32, u6_0Imm:$Ii, s32_0Imm:$II),
 "memb($Rs32+#$Ii) = #$II",
-tc_838c4d7a, TypeV4LDST>, Requires<[HasV68]> {
+tc_838c4d7a, TypeV4LDST>, Requires<[HasV69]> {
 let addrMode = BaseImmOffset;
 let accessSize = ByteAccess;
 let AsmVariantName = "NonParsable";
@@ -39535,7 +40554,7 @@ def dup_S4_storeiri_io : HInst<
 (outs),
 (ins IntRegs:$Rs32, u6_2Imm:$Ii, s32_0Imm:$II),
 "memw($Rs32+#$Ii) = #$II",
-tc_838c4d7a, TypeV4LDST>, Requires<[HasV68]> {
+tc_838c4d7a, TypeV4LDST>, Requires<[HasV69]> {
 let addrMode = BaseImmOffset;
 let accessSize = WordAccess;
 let AsmVariantName = "NonParsable";
diff --git a/llvm/lib/Target/Hexagon/HexagonDepMapAsm2Intrin.td b/llvm/lib/Target/Hexagon/HexagonDepMapAsm2Intrin.td
index e5c78d122c9e..64bc5091d1d1 100644
--- a/llvm/lib/Target/Hexagon/HexagonDepMapAsm2Intrin.td
+++ b/llvm/lib/Target/Hexagon/HexagonDepMapAsm2Intrin.td
@@ -1661,8 +1661,6 @@ def: Pat<(int_hexagon_Y2_dccleana IntRegs:$src1),
          (Y2_dccleana IntRegs:$src1)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_Y2_dccleaninva IntRegs:$src1),
          (Y2_dccleaninva IntRegs:$src1)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_Y2_dcfetch IntRegs:$src1),
-         (Y2_dcfetch IntRegs:$src1)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_Y2_dcinva IntRegs:$src1),
          (Y2_dcinva IntRegs:$src1)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_Y2_dczeroa IntRegs:$src1),
@@ -3380,3 +3378,294 @@ def: Pat<(int_hexagon_V6_v6mpyvubs10_vxx HvxWR:$src1, HvxWR:$src2, HvxWR:$src3,
          (V6_v6mpyvubs10_vxx HvxWR:$src1, HvxWR:$src2, HvxWR:$src3, u2_0ImmPred_timm:$src4)>, Requires<[HasV68, UseHVX64B]>;
 def: Pat<(int_hexagon_V6_v6mpyvubs10_vxx_128B HvxWR:$src1, HvxWR:$src2, HvxWR:$src3, u2_0ImmPred_timm:$src4),
          (V6_v6mpyvubs10_vxx HvxWR:$src1, HvxWR:$src2, HvxWR:$src3, u2_0ImmPred_timm:$src4)>, Requires<[HasV68, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vabs_hf HvxVR:$src1),
+         (V6_vabs_hf HvxVR:$src1)>, Requires<[HasV68, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vabs_hf_128B HvxVR:$src1),
+         (V6_vabs_hf HvxVR:$src1)>, Requires<[HasV68, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vabs_sf HvxVR:$src1),
+         (V6_vabs_sf HvxVR:$src1)>, Requires<[HasV68, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vabs_sf_128B HvxVR:$src1),
+         (V6_vabs_sf HvxVR:$src1)>, Requires<[HasV68, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vadd_hf HvxVR:$src1, HvxVR:$src2),
+         (V6_vadd_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vadd_hf_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vadd_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vadd_hf_hf HvxVR:$src1, HvxVR:$src2),
+         (V6_vadd_hf_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vadd_hf_hf_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vadd_hf_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vadd_qf16 HvxVR:$src1, HvxVR:$src2),
+         (V6_vadd_qf16 HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vadd_qf16_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vadd_qf16 HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vadd_qf16_mix HvxVR:$src1, HvxVR:$src2),
+         (V6_vadd_qf16_mix HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vadd_qf16_mix_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vadd_qf16_mix HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vadd_qf32 HvxVR:$src1, HvxVR:$src2),
+         (V6_vadd_qf32 HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vadd_qf32_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vadd_qf32 HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vadd_qf32_mix HvxVR:$src1, HvxVR:$src2),
+         (V6_vadd_qf32_mix HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vadd_qf32_mix_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vadd_qf32_mix HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vadd_sf HvxVR:$src1, HvxVR:$src2),
+         (V6_vadd_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vadd_sf_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vadd_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vadd_sf_hf HvxVR:$src1, HvxVR:$src2),
+         (V6_vadd_sf_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vadd_sf_hf_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vadd_sf_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vadd_sf_sf HvxVR:$src1, HvxVR:$src2),
+         (V6_vadd_sf_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vadd_sf_sf_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vadd_sf_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vassign_fp HvxVR:$src1),
+         (V6_vassign_fp HvxVR:$src1)>, Requires<[HasV68, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vassign_fp_128B HvxVR:$src1),
+         (V6_vassign_fp HvxVR:$src1)>, Requires<[HasV68, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vconv_hf_qf16 HvxVR:$src1),
+         (V6_vconv_hf_qf16 HvxVR:$src1)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vconv_hf_qf16_128B HvxVR:$src1),
+         (V6_vconv_hf_qf16 HvxVR:$src1)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vconv_hf_qf32 HvxWR:$src1),
+         (V6_vconv_hf_qf32 HvxWR:$src1)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vconv_hf_qf32_128B HvxWR:$src1),
+         (V6_vconv_hf_qf32 HvxWR:$src1)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vconv_sf_qf32 HvxVR:$src1),
+         (V6_vconv_sf_qf32 HvxVR:$src1)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vconv_sf_qf32_128B HvxVR:$src1),
+         (V6_vconv_sf_qf32 HvxVR:$src1)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vcvt_b_hf HvxVR:$src1, HvxVR:$src2),
+         (V6_vcvt_b_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vcvt_b_hf_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vcvt_b_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vcvt_h_hf HvxVR:$src1),
+         (V6_vcvt_h_hf HvxVR:$src1)>, Requires<[HasV68, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vcvt_h_hf_128B HvxVR:$src1),
+         (V6_vcvt_h_hf HvxVR:$src1)>, Requires<[HasV68, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vcvt_hf_b HvxVR:$src1),
+         (V6_vcvt_hf_b HvxVR:$src1)>, Requires<[HasV68, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vcvt_hf_b_128B HvxVR:$src1),
+         (V6_vcvt_hf_b HvxVR:$src1)>, Requires<[HasV68, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vcvt_hf_h HvxVR:$src1),
+         (V6_vcvt_hf_h HvxVR:$src1)>, Requires<[HasV68, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vcvt_hf_h_128B HvxVR:$src1),
+         (V6_vcvt_hf_h HvxVR:$src1)>, Requires<[HasV68, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vcvt_hf_sf HvxVR:$src1, HvxVR:$src2),
+         (V6_vcvt_hf_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vcvt_hf_sf_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vcvt_hf_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vcvt_hf_ub HvxVR:$src1),
+         (V6_vcvt_hf_ub HvxVR:$src1)>, Requires<[HasV68, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vcvt_hf_ub_128B HvxVR:$src1),
+         (V6_vcvt_hf_ub HvxVR:$src1)>, Requires<[HasV68, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vcvt_hf_uh HvxVR:$src1),
+         (V6_vcvt_hf_uh HvxVR:$src1)>, Requires<[HasV68, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vcvt_hf_uh_128B HvxVR:$src1),
+         (V6_vcvt_hf_uh HvxVR:$src1)>, Requires<[HasV68, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vcvt_sf_hf HvxVR:$src1),
+         (V6_vcvt_sf_hf HvxVR:$src1)>, Requires<[HasV68, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vcvt_sf_hf_128B HvxVR:$src1),
+         (V6_vcvt_sf_hf HvxVR:$src1)>, Requires<[HasV68, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vcvt_ub_hf HvxVR:$src1, HvxVR:$src2),
+         (V6_vcvt_ub_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vcvt_ub_hf_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vcvt_ub_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vcvt_uh_hf HvxVR:$src1),
+         (V6_vcvt_uh_hf HvxVR:$src1)>, Requires<[HasV68, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vcvt_uh_hf_128B HvxVR:$src1),
+         (V6_vcvt_uh_hf HvxVR:$src1)>, Requires<[HasV68, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vdmpy_sf_hf HvxVR:$src1, HvxVR:$src2),
+         (V6_vdmpy_sf_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vdmpy_sf_hf_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vdmpy_sf_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vdmpy_sf_hf_acc HvxVR:$src1, HvxVR:$src2, HvxVR:$src3),
+         (V6_vdmpy_sf_hf_acc HvxVR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV68, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vdmpy_sf_hf_acc_128B HvxVR:$src1, HvxVR:$src2, HvxVR:$src3),
+         (V6_vdmpy_sf_hf_acc HvxVR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV68, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vfmax_hf HvxVR:$src1, HvxVR:$src2),
+         (V6_vfmax_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vfmax_hf_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vfmax_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vfmax_sf HvxVR:$src1, HvxVR:$src2),
+         (V6_vfmax_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vfmax_sf_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vfmax_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vfmin_hf HvxVR:$src1, HvxVR:$src2),
+         (V6_vfmin_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vfmin_hf_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vfmin_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vfmin_sf HvxVR:$src1, HvxVR:$src2),
+         (V6_vfmin_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vfmin_sf_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vfmin_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vfneg_hf HvxVR:$src1),
+         (V6_vfneg_hf HvxVR:$src1)>, Requires<[HasV68, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vfneg_hf_128B HvxVR:$src1),
+         (V6_vfneg_hf HvxVR:$src1)>, Requires<[HasV68, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vfneg_sf HvxVR:$src1),
+         (V6_vfneg_sf HvxVR:$src1)>, Requires<[HasV68, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vfneg_sf_128B HvxVR:$src1),
+         (V6_vfneg_sf HvxVR:$src1)>, Requires<[HasV68, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vgthf HvxVR:$src1, HvxVR:$src2),
+         (V6_vgthf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vgthf_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vgthf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vgthf_and HvxQR:$src1, HvxVR:$src2, HvxVR:$src3),
+         (V6_vgthf_and HvxQR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vgthf_and_128B HvxQR:$src1, HvxVR:$src2, HvxVR:$src3),
+         (V6_vgthf_and HvxQR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vgthf_or HvxQR:$src1, HvxVR:$src2, HvxVR:$src3),
+         (V6_vgthf_or HvxQR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vgthf_or_128B HvxQR:$src1, HvxVR:$src2, HvxVR:$src3),
+         (V6_vgthf_or HvxQR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vgthf_xor HvxQR:$src1, HvxVR:$src2, HvxVR:$src3),
+         (V6_vgthf_xor HvxQR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vgthf_xor_128B HvxQR:$src1, HvxVR:$src2, HvxVR:$src3),
+         (V6_vgthf_xor HvxQR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vgtsf HvxVR:$src1, HvxVR:$src2),
+         (V6_vgtsf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vgtsf_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vgtsf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vgtsf_and HvxQR:$src1, HvxVR:$src2, HvxVR:$src3),
+         (V6_vgtsf_and HvxQR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vgtsf_and_128B HvxQR:$src1, HvxVR:$src2, HvxVR:$src3),
+         (V6_vgtsf_and HvxQR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vgtsf_or HvxQR:$src1, HvxVR:$src2, HvxVR:$src3),
+         (V6_vgtsf_or HvxQR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vgtsf_or_128B HvxQR:$src1, HvxVR:$src2, HvxVR:$src3),
+         (V6_vgtsf_or HvxQR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vgtsf_xor HvxQR:$src1, HvxVR:$src2, HvxVR:$src3),
+         (V6_vgtsf_xor HvxQR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vgtsf_xor_128B HvxQR:$src1, HvxVR:$src2, HvxVR:$src3),
+         (V6_vgtsf_xor HvxQR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vmax_hf HvxVR:$src1, HvxVR:$src2),
+         (V6_vmax_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vmax_hf_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vmax_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vmax_sf HvxVR:$src1, HvxVR:$src2),
+         (V6_vmax_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vmax_sf_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vmax_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vmin_hf HvxVR:$src1, HvxVR:$src2),
+         (V6_vmin_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vmin_hf_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vmin_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vmin_sf HvxVR:$src1, HvxVR:$src2),
+         (V6_vmin_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vmin_sf_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vmin_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vmpy_hf_hf HvxVR:$src1, HvxVR:$src2),
+         (V6_vmpy_hf_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vmpy_hf_hf_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vmpy_hf_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vmpy_hf_hf_acc HvxVR:$src1, HvxVR:$src2, HvxVR:$src3),
+         (V6_vmpy_hf_hf_acc HvxVR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV68, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vmpy_hf_hf_acc_128B HvxVR:$src1, HvxVR:$src2, HvxVR:$src3),
+         (V6_vmpy_hf_hf_acc HvxVR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV68, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vmpy_qf16 HvxVR:$src1, HvxVR:$src2),
+         (V6_vmpy_qf16 HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vmpy_qf16_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vmpy_qf16 HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vmpy_qf16_hf HvxVR:$src1, HvxVR:$src2),
+         (V6_vmpy_qf16_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vmpy_qf16_hf_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vmpy_qf16_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vmpy_qf16_mix_hf HvxVR:$src1, HvxVR:$src2),
+         (V6_vmpy_qf16_mix_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vmpy_qf16_mix_hf_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vmpy_qf16_mix_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vmpy_qf32 HvxVR:$src1, HvxVR:$src2),
+         (V6_vmpy_qf32 HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vmpy_qf32_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vmpy_qf32 HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vmpy_qf32_hf HvxVR:$src1, HvxVR:$src2),
+         (V6_vmpy_qf32_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vmpy_qf32_hf_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vmpy_qf32_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vmpy_qf32_mix_hf HvxVR:$src1, HvxVR:$src2),
+         (V6_vmpy_qf32_mix_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vmpy_qf32_mix_hf_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vmpy_qf32_mix_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vmpy_qf32_qf16 HvxVR:$src1, HvxVR:$src2),
+         (V6_vmpy_qf32_qf16 HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vmpy_qf32_qf16_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vmpy_qf32_qf16 HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vmpy_qf32_sf HvxVR:$src1, HvxVR:$src2),
+         (V6_vmpy_qf32_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vmpy_qf32_sf_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vmpy_qf32_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vmpy_sf_hf HvxVR:$src1, HvxVR:$src2),
+         (V6_vmpy_sf_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vmpy_sf_hf_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vmpy_sf_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vmpy_sf_hf_acc HvxWR:$src1, HvxVR:$src2, HvxVR:$src3),
+         (V6_vmpy_sf_hf_acc HvxWR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV68, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vmpy_sf_hf_acc_128B HvxWR:$src1, HvxVR:$src2, HvxVR:$src3),
+         (V6_vmpy_sf_hf_acc HvxWR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV68, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vmpy_sf_sf HvxVR:$src1, HvxVR:$src2),
+         (V6_vmpy_sf_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vmpy_sf_sf_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vmpy_sf_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vsub_hf HvxVR:$src1, HvxVR:$src2),
+         (V6_vsub_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vsub_hf_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vsub_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vsub_hf_hf HvxVR:$src1, HvxVR:$src2),
+         (V6_vsub_hf_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vsub_hf_hf_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vsub_hf_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vsub_qf16 HvxVR:$src1, HvxVR:$src2),
+         (V6_vsub_qf16 HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vsub_qf16_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vsub_qf16 HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vsub_qf16_mix HvxVR:$src1, HvxVR:$src2),
+         (V6_vsub_qf16_mix HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vsub_qf16_mix_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vsub_qf16_mix HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vsub_qf32 HvxVR:$src1, HvxVR:$src2),
+         (V6_vsub_qf32 HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vsub_qf32_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vsub_qf32 HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vsub_qf32_mix HvxVR:$src1, HvxVR:$src2),
+         (V6_vsub_qf32_mix HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vsub_qf32_mix_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vsub_qf32_mix HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vsub_sf HvxVR:$src1, HvxVR:$src2),
+         (V6_vsub_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vsub_sf_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vsub_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>;
+def: Pat<(int_hexagon_V6_vsub_sf_hf HvxVR:$src1, HvxVR:$src2),
+         (V6_vsub_sf_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vsub_sf_hf_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vsub_sf_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vsub_sf_sf HvxVR:$src1, HvxVR:$src2),
+         (V6_vsub_sf_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vsub_sf_sf_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vsub_sf_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B]>;
+
+// V69 HVX Instructions.
+
+def: Pat<(int_hexagon_V6_vasrvuhubrndsat HvxWR:$src1, HvxVR:$src2),
+         (V6_vasrvuhubrndsat HvxWR:$src1, HvxVR:$src2)>, Requires<[HasV69, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vasrvuhubrndsat_128B HvxWR:$src1, HvxVR:$src2),
+         (V6_vasrvuhubrndsat HvxWR:$src1, HvxVR:$src2)>, Requires<[HasV69, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vasrvuhubsat HvxWR:$src1, HvxVR:$src2),
+         (V6_vasrvuhubsat HvxWR:$src1, HvxVR:$src2)>, Requires<[HasV69, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vasrvuhubsat_128B HvxWR:$src1, HvxVR:$src2),
+         (V6_vasrvuhubsat HvxWR:$src1, HvxVR:$src2)>, Requires<[HasV69, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vasrvwuhrndsat HvxWR:$src1, HvxVR:$src2),
+         (V6_vasrvwuhrndsat HvxWR:$src1, HvxVR:$src2)>, Requires<[HasV69, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vasrvwuhrndsat_128B HvxWR:$src1, HvxVR:$src2),
+         (V6_vasrvwuhrndsat HvxWR:$src1, HvxVR:$src2)>, Requires<[HasV69, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vasrvwuhsat HvxWR:$src1, HvxVR:$src2),
+         (V6_vasrvwuhsat HvxWR:$src1, HvxVR:$src2)>, Requires<[HasV69, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vasrvwuhsat_128B HvxWR:$src1, HvxVR:$src2),
+         (V6_vasrvwuhsat HvxWR:$src1, HvxVR:$src2)>, Requires<[HasV69, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vmpyuhvs HvxVR:$src1, HvxVR:$src2),
+         (V6_vmpyuhvs HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV69, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vmpyuhvs_128B HvxVR:$src1, HvxVR:$src2),
+         (V6_vmpyuhvs HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV69, UseHVX128B]>;
diff --git a/llvm/lib/Target/Hexagon/HexagonDepMappings.td b/llvm/lib/Target/Hexagon/HexagonDepMappings.td
index 919cb996ad15..2f7b76b893a9 100644
--- a/llvm/lib/Target/Hexagon/HexagonDepMappings.td
+++ b/llvm/lib/Target/Hexagon/HexagonDepMappings.td
@@ -174,7 +174,6 @@ def V6_ldcpnt0Alias : InstAlias<"if ($Pv4) $Vd32.cur = vmem($Rt32):nt", (V6_vL32
 def V6_ldnp0Alias : InstAlias<"if (!$Pv4) $Vd32 = vmem($Rt32)", (V6_vL32b_npred_pi HvxVR:$Vd32, IntRegs:$Rt32, PredRegs:$Pv4, 0)>, Requires<[UseHVX]>;
 def V6_ldnpnt0Alias : InstAlias<"if (!$Pv4) $Vd32 = vmem($Rt32):nt", (V6_vL32b_nt_npred_pi HvxVR:$Vd32, IntRegs:$Rt32, PredRegs:$Pv4, 0)>, Requires<[UseHVX]>;
 def V6_ldnt0Alias : InstAlias<"$Vd32 = vmem($Rt32):nt", (V6_vL32b_nt_ai HvxVR:$Vd32, IntRegs:$Rt32, 0)>, Requires<[UseHVX]>;
-def V6_ldntnt0Alias : InstAlias<"$Vd32 = vmem($Rt32):nt", (V6_vL32b_nt_ai HvxVR:$Vd32, IntRegs:$Rt32, 0)>;
 def V6_ldp0Alias : InstAlias<"if ($Pv4) $Vd32 = vmem($Rt32)", (V6_vL32b_pred_ai HvxVR:$Vd32, PredRegs:$Pv4, IntRegs:$Rt32, 0)>, Requires<[UseHVX]>;
 def V6_ldpnt0Alias : InstAlias<"if ($Pv4) $Vd32 = vmem($Rt32):nt", (V6_vL32b_nt_pred_ai HvxVR:$Vd32, PredRegs:$Pv4, IntRegs:$Rt32, 0)>, Requires<[UseHVX]>;
 def V6_ldtnp0Alias : InstAlias<"if (!$Pv4) $Vd32.tmp = vmem($Rt32)", (V6_vL32b_npred_ai HvxVR:$Vd32, PredRegs:$Pv4, IntRegs:$Rt32, 0)>, Requires<[UseHVX]>;
diff --git a/llvm/lib/Target/Hexagon/HexagonGenInsert.cpp b/llvm/lib/Target/Hexagon/HexagonGenInsert.cpp
index 46c1fbc6eeb2..85230cac9d7c 100644
--- a/llvm/lib/Target/Hexagon/HexagonGenInsert.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonGenInsert.cpp
@@ -1445,8 +1445,8 @@ bool HexagonGenInsert::removeDeadCode(MachineDomTreeNode *N) {
 
   MachineBasicBlock *B = N->getBlock();
   std::vector<MachineInstr*> Instrs;
-  for (auto I = B->rbegin(), E = B->rend(); I != E; ++I)
-    Instrs.push_back(&*I);
+  for (MachineInstr &MI : llvm::reverse(*B))
+    Instrs.push_back(&MI);
 
   for (MachineInstr *MI : Instrs) {
     unsigned Opc = MI->getOpcode();
diff --git a/llvm/lib/Target/Hexagon/HexagonHazardRecognizer.cpp b/llvm/lib/Target/Hexagon/HexagonHazardRecognizer.cpp
index e45126bec6ef..44679d429de5 100644
--- a/llvm/lib/Target/Hexagon/HexagonHazardRecognizer.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonHazardRecognizer.cpp
@@ -60,7 +60,7 @@ HexagonHazardRecognizer::getHazardType(SUnit *SU, int stalls) {
         RetVal = NoHazard;
       LLVM_DEBUG(dbgs() << "*** Try .new version? " << (RetVal == NoHazard)
                         << "\n");
-      MF->DeleteMachineInstr(NewMI);
+      MF->deleteMachineInstr(NewMI);
     }
     return RetVal;
   }
@@ -129,7 +129,7 @@ void HexagonHazardRecognizer::EmitInstruction(SUnit *SU) {
                                MI->getDebugLoc());
     assert(Resources->canReserveResources(*NewMI));
     Resources->reserveResources(*NewMI);
-    MF->DeleteMachineInstr(NewMI);
+    MF->deleteMachineInstr(NewMI);
   }
   else
     Resources->reserveResources(*MI);
diff --git a/llvm/lib/Target/Hexagon/HexagonInstrFormats.td b/llvm/lib/Target/Hexagon/HexagonInstrFormats.td
index 45adaf50774f..898ef51bd48f 100644
--- a/llvm/lib/Target/Hexagon/HexagonInstrFormats.td
+++ b/llvm/lib/Target/Hexagon/HexagonInstrFormats.td
@@ -146,9 +146,6 @@ class InstHexagon<dag outs, dag ins, string asmstr, list<dag> pattern,
   bits<1> isFP = 0;
   let TSFlags {50} = isFP; // Floating-point.
 
-  bits<1> isSomeOK = 0;
-  let TSFlags {51} = isSomeOK; // Relax some grouping constraints.
-
   bits<1> hasNewValue2 = 0;
   let TSFlags{52} = hasNewValue2; // Second New-value producer insn.
   bits<3> opNewValue2 = 0;
@@ -160,8 +157,8 @@ class InstHexagon<dag outs, dag ins, string asmstr, list<dag> pattern,
   bits<1> prefersSlot3 = 0;
   let TSFlags{57} = prefersSlot3; // Complex XU
 
-  bits<1> hasTmpDst = 0;
-  let TSFlags{60} = hasTmpDst;  // v65 : 'fake" register VTMP is set
+  bits<1> hasHvxTmp = 0;
+  let TSFlags{60} = hasHvxTmp;  // vector register vX.tmp false-write
 
   bit CVINew = 0;
   let TSFlags{62} = CVINew;
diff --git a/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp b/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
index b6984d40f78e..931b0c0e0090 100644
--- a/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
@@ -40,6 +40,7 @@
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/IR/DebugLoc.h"
 #include "llvm/MC/MCAsmInfo.h"
+#include "llvm/MC/MCInstBuilder.h"
 #include "llvm/MC/MCInstrDesc.h"
 #include "llvm/MC/MCInstrItineraries.h"
 #include "llvm/MC/MCRegisterInfo.h"
@@ -4655,3 +4656,11 @@ short HexagonInstrInfo::changeAddrMode_rr_ur(short Opc) const {
 short HexagonInstrInfo::changeAddrMode_ur_rr(short Opc) const {
   return Opc >= 0 ? Hexagon::changeAddrMode_ur_rr(Opc) : Opc;
 }
+
+MCInst HexagonInstrInfo::getNop() const {
+  static const MCInst Nop = MCInstBuilder(Hexagon::A2_nop);
+
+  return MCInstBuilder(Hexagon::BUNDLE)
+    .addImm(0)
+    .addInst(&Nop);
+}
diff --git a/llvm/lib/Target/Hexagon/HexagonInstrInfo.h b/llvm/lib/Target/Hexagon/HexagonInstrInfo.h
index eaaf9f7046c7..830f04d9eac3 100644
--- a/llvm/lib/Target/Hexagon/HexagonInstrInfo.h
+++ b/llvm/lib/Target/Hexagon/HexagonInstrInfo.h
@@ -524,6 +524,8 @@ public:
   short changeAddrMode_ur_rr(const MachineInstr &MI) const {
     return changeAddrMode_ur_rr(MI.getOpcode());
   }
+
+  MCInst getNop() const override;
 };
 
 } // end namespace llvm
diff --git a/llvm/lib/Target/Hexagon/HexagonMCInstLower.cpp b/llvm/lib/Target/Hexagon/HexagonMCInstLower.cpp
index 987c4a5fa6c4..d5c34ac467c3 100644
--- a/llvm/lib/Target/Hexagon/HexagonMCInstLower.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonMCInstLower.cpp
@@ -104,6 +104,19 @@ void llvm::HexagonLowerToMC(const MCInstrInfo &MCII, const MachineInstr *MI,
     HexagonMCInstrInfo::setOuterLoop(MCB);
     return;
   }
+  if (MI->getOpcode() == Hexagon::PATCHABLE_FUNCTION_ENTER) {
+    AP.EmitSled(*MI, HexagonAsmPrinter::SledKind::FUNCTION_ENTER);
+    return;
+  }
+  if (MI->getOpcode() == Hexagon::PATCHABLE_FUNCTION_EXIT) {
+    AP.EmitSled(*MI, HexagonAsmPrinter::SledKind::FUNCTION_EXIT);
+    return;
+  }
+  if (MI->getOpcode() == Hexagon::PATCHABLE_TAIL_CALL) {
+    AP.EmitSled(*MI, HexagonAsmPrinter::SledKind::TAIL_CALL);
+    return;
+  }
+
   MCInst *MCI = AP.OutContext.createMCInst();
   MCI->setOpcode(MI->getOpcode());
   assert(MCI->getOpcode() == static_cast<unsigned>(MI->getOpcode()) &&
diff --git a/llvm/lib/Target/Hexagon/HexagonMachineScheduler.cpp b/llvm/lib/Target/Hexagon/HexagonMachineScheduler.cpp
index 60d58f421bbb..53e82ac66b85 100644
--- a/llvm/lib/Target/Hexagon/HexagonMachineScheduler.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonMachineScheduler.cpp
@@ -14,676 +14,44 @@
 #include "HexagonMachineScheduler.h"
 #include "HexagonInstrInfo.h"
 #include "HexagonSubtarget.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/CodeGen/DFAPacketizer.h"
-#include "llvm/CodeGen/MachineBasicBlock.h"
-#include "llvm/CodeGen/MachineFunction.h"
-#include "llvm/CodeGen/MachineInstr.h"
-#include "llvm/CodeGen/MachineLoopInfo.h"
-#include "llvm/CodeGen/RegisterClassInfo.h"
-#include "llvm/CodeGen/RegisterPressure.h"
+#include "llvm/CodeGen/MachineScheduler.h"
 #include "llvm/CodeGen/ScheduleDAG.h"
-#include "llvm/CodeGen/ScheduleHazardRecognizer.h"
-#include "llvm/CodeGen/TargetInstrInfo.h"
-#include "llvm/CodeGen/TargetOpcodes.h"
-#include "llvm/CodeGen/TargetRegisterInfo.h"
-#include "llvm/CodeGen/TargetSchedule.h"
-#include "llvm/CodeGen/TargetSubtargetInfo.h"
-#include "llvm/IR/Function.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/raw_ostream.h"
-#include <algorithm>
-#include <cassert>
-#include <iomanip>
-#include <limits>
-#include <memory>
-#include <sstream>
+#include "llvm/CodeGen/VLIWMachineScheduler.h"
 
 using namespace llvm;
 
 #define DEBUG_TYPE "machine-scheduler"
 
-static cl::opt<bool> IgnoreBBRegPressure("ignore-bb-reg-pressure",
-    cl::Hidden, cl::ZeroOrMore, cl::init(false));
-
-static cl::opt<bool> UseNewerCandidate("use-newer-candidate",
-    cl::Hidden, cl::ZeroOrMore, cl::init(true));
-
-static cl::opt<unsigned> SchedDebugVerboseLevel("misched-verbose-level",
-    cl::Hidden, cl::ZeroOrMore, cl::init(1));
-
-// Check if the scheduler should penalize instructions that are available to
-// early due to a zero-latency dependence.
-static cl::opt<bool> CheckEarlyAvail("check-early-avail", cl::Hidden,
-    cl::ZeroOrMore, cl::init(true));
-
-// This value is used to determine if a register class is a high pressure set.
-// We compute the maximum number of registers needed and divided by the total
-// available. Then, we compare the result to this value.
-static cl::opt<float> RPThreshold("hexagon-reg-pressure", cl::Hidden,
-    cl::init(0.75f), cl::desc("High register pressure threhold."));
-
 /// Return true if there is a dependence between SUd and SUu.
-static bool hasDependence(const SUnit *SUd, const SUnit *SUu,
-                          const HexagonInstrInfo &QII) {
-  if (SUd->Succs.size() == 0)
-    return false;
+bool HexagonVLIWResourceModel::hasDependence(const SUnit *SUd,
+                                             const SUnit *SUu) {
+  const auto *QII = static_cast<const HexagonInstrInfo *>(TII);
 
   // Enable .cur formation.
-  if (QII.mayBeCurLoad(*SUd->getInstr()))
+  if (QII->mayBeCurLoad(*SUd->getInstr()))
     return false;
 
-  if (QII.canExecuteInBundle(*SUd->getInstr(), *SUu->getInstr()))
-    return false;
-
-  for (const auto &S : SUd->Succs) {
-    // Since we do not add pseudos to packets, might as well
-    // ignore order dependencies.
-    if (S.isCtrl())
-      continue;
-
-    if (S.getSUnit() == SUu && S.getLatency() > 0)
-      return true;
-  }
-  return false;
-}
-
-/// Check if scheduling of this SU is possible
-/// in the current packet.
-/// It is _not_ precise (statefull), it is more like
-/// another heuristic. Many corner cases are figured
-/// empirically.
-bool VLIWResourceModel::isResourceAvailable(SUnit *SU, bool IsTop) {
-  if (!SU || !SU->getInstr())
+  if (QII->canExecuteInBundle(*SUd->getInstr(), *SUu->getInstr()))
     return false;
 
-  // First see if the pipeline could receive this instruction
-  // in the current cycle.
-  switch (SU->getInstr()->getOpcode()) {
-  default:
-    if (!ResourcesModel->canReserveResources(*SU->getInstr()))
-      return false;
-    break;
-  case TargetOpcode::EXTRACT_SUBREG:
-  case TargetOpcode::INSERT_SUBREG:
-  case TargetOpcode::SUBREG_TO_REG:
-  case TargetOpcode::REG_SEQUENCE:
-  case TargetOpcode::IMPLICIT_DEF:
-  case TargetOpcode::COPY:
-  case TargetOpcode::INLINEASM:
-  case TargetOpcode::INLINEASM_BR:
-    break;
-  }
-
-  MachineBasicBlock *MBB = SU->getInstr()->getParent();
-  auto &QST = MBB->getParent()->getSubtarget<HexagonSubtarget>();
-  const auto &QII = *QST.getInstrInfo();
-
-  // Now see if there are no other dependencies to instructions already
-  // in the packet. 
-  if (IsTop) {
-    for (unsigned i = 0, e = Packet.size(); i != e; ++i)
-      if (hasDependence(Packet[i], SU, QII))
-        return false;
-  } else {
-    for (unsigned i = 0, e = Packet.size(); i != e; ++i)
-      if (hasDependence(SU, Packet[i], QII))
-        return false;
-  }
-  return true;
-}
-
-/// Keep track of available resources.
-bool VLIWResourceModel::reserveResources(SUnit *SU, bool IsTop) {
-  bool startNewCycle = false;
-  // Artificially reset state.
-  if (!SU) {
-    ResourcesModel->clearResources();
-    Packet.clear();
-    TotalPackets++;
-    return false;
-  }
-  // If this SU does not fit in the packet or the packet is now full
-  // start a new one.
-  if (!isResourceAvailable(SU, IsTop) ||
-      Packet.size() >= SchedModel->getIssueWidth()) {
-    ResourcesModel->clearResources();
-    Packet.clear();
-    TotalPackets++;
-    startNewCycle = true;
-  }
-
-  switch (SU->getInstr()->getOpcode()) {
-  default:
-    ResourcesModel->reserveResources(*SU->getInstr());
-    break;
-  case TargetOpcode::EXTRACT_SUBREG:
-  case TargetOpcode::INSERT_SUBREG:
-  case TargetOpcode::SUBREG_TO_REG:
-  case TargetOpcode::REG_SEQUENCE:
-  case TargetOpcode::IMPLICIT_DEF:
-  case TargetOpcode::KILL:
-  case TargetOpcode::CFI_INSTRUCTION:
-  case TargetOpcode::EH_LABEL:
-  case TargetOpcode::COPY:
-  case TargetOpcode::INLINEASM:
-  case TargetOpcode::INLINEASM_BR:
-    break;
-  }
-  Packet.push_back(SU);
-
-#ifndef NDEBUG
-  LLVM_DEBUG(dbgs() << "Packet[" << TotalPackets << "]:\n");
-  for (unsigned i = 0, e = Packet.size(); i != e; ++i) {
-    LLVM_DEBUG(dbgs() << "\t[" << i << "] SU(");
-    LLVM_DEBUG(dbgs() << Packet[i]->NodeNum << ")\t");
-    LLVM_DEBUG(Packet[i]->getInstr()->dump());
-  }
-#endif
-
-  return startNewCycle;
+  return VLIWResourceModel::hasDependence(SUd, SUu);
 }
 
-/// schedule - Called back from MachineScheduler::runOnMachineFunction
-/// after setting up the current scheduling region. [RegionBegin, RegionEnd)
-/// only includes instructions that have DAG nodes, not scheduling boundaries.
-void VLIWMachineScheduler::schedule() {
-  LLVM_DEBUG(dbgs() << "********** MI Converging Scheduling VLIW "
-                    << printMBBReference(*BB) << " " << BB->getName()
-                    << " in_func " << BB->getParent()->getName()
-                    << " at loop depth " << MLI->getLoopDepth(BB) << " \n");
-
-  buildDAGWithRegPressure();
-
-  Topo.InitDAGTopologicalSorting();
-
-  // Postprocess the DAG to add platform-specific artificial dependencies.
-  postprocessDAG();
-
-  SmallVector<SUnit*, 8> TopRoots, BotRoots;
-  findRootsAndBiasEdges(TopRoots, BotRoots);
-
-  // Initialize the strategy before modifying the DAG.
-  SchedImpl->initialize(this);
-
-  LLVM_DEBUG(unsigned maxH = 0;
-             for (unsigned su = 0, e = SUnits.size(); su != e;
-                  ++su) if (SUnits[su].getHeight() > maxH) maxH =
-                 SUnits[su].getHeight();
-             dbgs() << "Max Height " << maxH << "\n";);
-  LLVM_DEBUG(unsigned maxD = 0;
-             for (unsigned su = 0, e = SUnits.size(); su != e;
-                  ++su) if (SUnits[su].getDepth() > maxD) maxD =
-                 SUnits[su].getDepth();
-             dbgs() << "Max Depth " << maxD << "\n";);
-  LLVM_DEBUG(dump());
-
-  initQueues(TopRoots, BotRoots);
-
-  bool IsTopNode = false;
-  while (true) {
-    LLVM_DEBUG(
-        dbgs() << "** VLIWMachineScheduler::schedule picking next node\n");
-    SUnit *SU = SchedImpl->pickNode(IsTopNode);
-    if (!SU) break;
-
-    if (!checkSchedLimit())
-      break;
-
-    scheduleMI(SU, IsTopNode);
-
-    // Notify the scheduling strategy after updating the DAG.
-    SchedImpl->schedNode(SU, IsTopNode);
-
-    updateQueues(SU, IsTopNode);
-  }
-  assert(CurrentTop == CurrentBottom && "Nonempty unscheduled zone.");
-
-  placeDebugValues();
-
-  LLVM_DEBUG({
-    dbgs() << "*** Final schedule for "
-           << printMBBReference(*begin()->getParent()) << " ***\n";
-    dumpSchedule();
-    dbgs() << '\n';
-  });
+VLIWResourceModel *HexagonConvergingVLIWScheduler::createVLIWResourceModel(
+    const TargetSubtargetInfo &STI, const TargetSchedModel *SchedModel) const {
+  return new HexagonVLIWResourceModel(STI, SchedModel);
 }
 
-void ConvergingVLIWScheduler::initialize(ScheduleDAGMI *dag) {
-  DAG = static_cast<VLIWMachineScheduler*>(dag);
-  SchedModel = DAG->getSchedModel();
-
-  Top.init(DAG, SchedModel);
-  Bot.init(DAG, SchedModel);
-
-  // Initialize the HazardRecognizers. If itineraries don't exist, are empty, or
-  // are disabled, then these HazardRecs will be disabled.
-  const InstrItineraryData *Itin = DAG->getSchedModel()->getInstrItineraries();
-  const TargetSubtargetInfo &STI = DAG->MF.getSubtarget();
-  const TargetInstrInfo *TII = STI.getInstrInfo();
-  delete Top.HazardRec;
-  delete Bot.HazardRec;
-  Top.HazardRec = TII->CreateTargetMIHazardRecognizer(Itin, DAG);
-  Bot.HazardRec = TII->CreateTargetMIHazardRecognizer(Itin, DAG);
-
-  delete Top.ResourceModel;
-  delete Bot.ResourceModel;
-  Top.ResourceModel = new VLIWResourceModel(STI, DAG->getSchedModel());
-  Bot.ResourceModel = new VLIWResourceModel(STI, DAG->getSchedModel());
-
-  const std::vector<unsigned> &MaxPressure =
-    DAG->getRegPressure().MaxSetPressure;
-  HighPressureSets.assign(MaxPressure.size(), 0);
-  for (unsigned i = 0, e = MaxPressure.size(); i < e; ++i) {
-    unsigned Limit = DAG->getRegClassInfo()->getRegPressureSetLimit(i);
-    HighPressureSets[i] =
-      ((float) MaxPressure[i] > ((float) Limit * RPThreshold));
-  }
-
-  assert((!ForceTopDown || !ForceBottomUp) &&
-         "-misched-topdown incompatible with -misched-bottomup");
-}
-
-void ConvergingVLIWScheduler::releaseTopNode(SUnit *SU) {
-  for (const SDep &PI : SU->Preds) {
-    unsigned PredReadyCycle = PI.getSUnit()->TopReadyCycle;
-    unsigned MinLatency = PI.getLatency();
-#ifndef NDEBUG
-    Top.MaxMinLatency = std::max(MinLatency, Top.MaxMinLatency);
-#endif
-    if (SU->TopReadyCycle < PredReadyCycle + MinLatency)
-      SU->TopReadyCycle = PredReadyCycle + MinLatency;
-  }
-
-  if (!SU->isScheduled)
-    Top.releaseNode(SU, SU->TopReadyCycle);
-}
-
-void ConvergingVLIWScheduler::releaseBottomNode(SUnit *SU) {
-  assert(SU->getInstr() && "Scheduled SUnit must have instr");
-
-  for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
-       I != E; ++I) {
-    unsigned SuccReadyCycle = I->getSUnit()->BotReadyCycle;
-    unsigned MinLatency = I->getLatency();
-#ifndef NDEBUG
-    Bot.MaxMinLatency = std::max(MinLatency, Bot.MaxMinLatency);
-#endif
-    if (SU->BotReadyCycle < SuccReadyCycle + MinLatency)
-      SU->BotReadyCycle = SuccReadyCycle + MinLatency;
-  }
-
-  if (!SU->isScheduled)
-    Bot.releaseNode(SU, SU->BotReadyCycle);
-}
-
-/// Does this SU have a hazard within the current instruction group.
-///
-/// The scheduler supports two modes of hazard recognition. The first is the
-/// ScheduleHazardRecognizer API. It is a fully general hazard recognizer that
-/// supports highly complicated in-order reservation tables
-/// (ScoreboardHazardRecognizer) and arbitrary target-specific logic.
-///
-/// The second is a streamlined mechanism that checks for hazards based on
-/// simple counters that the scheduler itself maintains. It explicitly checks
-/// for instruction dispatch limitations, including the number of micro-ops that
-/// can dispatch per cycle.
-///
-/// TODO: Also check whether the SU must start a new group.
-bool ConvergingVLIWScheduler::VLIWSchedBoundary::checkHazard(SUnit *SU) {
-  if (HazardRec->isEnabled())
-    return HazardRec->getHazardType(SU) != ScheduleHazardRecognizer::NoHazard;
-
-  unsigned uops = SchedModel->getNumMicroOps(SU->getInstr());
-  if (IssueCount + uops > SchedModel->getIssueWidth())
-    return true;
-
-  return false;
-}
-
-void ConvergingVLIWScheduler::VLIWSchedBoundary::releaseNode(SUnit *SU,
-                                                     unsigned ReadyCycle) {
-  if (ReadyCycle < MinReadyCycle)
-    MinReadyCycle = ReadyCycle;
-
-  // Check for interlocks first. For the purpose of other heuristics, an
-  // instruction that cannot issue appears as if it's not in the ReadyQueue.
-  if (ReadyCycle > CurrCycle || checkHazard(SU))
-
-    Pending.push(SU);
-  else
-    Available.push(SU);
-}
-
-/// Move the boundary of scheduled code by one cycle.
-void ConvergingVLIWScheduler::VLIWSchedBoundary::bumpCycle() {
-  unsigned Width = SchedModel->getIssueWidth();
-  IssueCount = (IssueCount <= Width) ? 0 : IssueCount - Width;
-
-  assert(MinReadyCycle < std::numeric_limits<unsigned>::max() &&
-         "MinReadyCycle uninitialized");
-  unsigned NextCycle = std::max(CurrCycle + 1, MinReadyCycle);
-
-  if (!HazardRec->isEnabled()) {
-    // Bypass HazardRec virtual calls.
-    CurrCycle = NextCycle;
-  } else {
-    // Bypass getHazardType calls in case of long latency.
-    for (; CurrCycle != NextCycle; ++CurrCycle) {
-      if (isTop())
-        HazardRec->AdvanceCycle();
-      else
-        HazardRec->RecedeCycle();
-    }
-  }
-  CheckPending = true;
-
-  LLVM_DEBUG(dbgs() << "*** Next cycle " << Available.getName() << " cycle "
-                    << CurrCycle << '\n');
-}
-
-/// Move the boundary of scheduled code by one SUnit.
-void ConvergingVLIWScheduler::VLIWSchedBoundary::bumpNode(SUnit *SU) {
-  bool startNewCycle = false;
-
-  // Update the reservation table.
-  if (HazardRec->isEnabled()) {
-    if (!isTop() && SU->isCall) {
-      // Calls are scheduled with their preceding instructions. For bottom-up
-      // scheduling, clear the pipeline state before emitting.
-      HazardRec->Reset();
-    }
-    HazardRec->EmitInstruction(SU);
-  }
-
-  // Update DFA model.
-  startNewCycle = ResourceModel->reserveResources(SU, isTop());
-
-  // Check the instruction group dispatch limit.
-  // TODO: Check if this SU must end a dispatch group.
-  IssueCount += SchedModel->getNumMicroOps(SU->getInstr());
-  if (startNewCycle) {
-    LLVM_DEBUG(dbgs() << "*** Max instrs at cycle " << CurrCycle << '\n');
-    bumpCycle();
-  }
-  else
-    LLVM_DEBUG(dbgs() << "*** IssueCount " << IssueCount << " at cycle "
-                      << CurrCycle << '\n');
-}
-
-/// Release pending ready nodes in to the available queue. This makes them
-/// visible to heuristics.
-void ConvergingVLIWScheduler::VLIWSchedBoundary::releasePending() {
-  // If the available queue is empty, it is safe to reset MinReadyCycle.
-  if (Available.empty())
-    MinReadyCycle = std::numeric_limits<unsigned>::max();
-
-  // Check to see if any of the pending instructions are ready to issue.  If
-  // so, add them to the available queue.
-  for (unsigned i = 0, e = Pending.size(); i != e; ++i) {
-    SUnit *SU = *(Pending.begin()+i);
-    unsigned ReadyCycle = isTop() ? SU->TopReadyCycle : SU->BotReadyCycle;
-
-    if (ReadyCycle < MinReadyCycle)
-      MinReadyCycle = ReadyCycle;
-
-    if (ReadyCycle > CurrCycle)
-      continue;
-
-    if (checkHazard(SU))
-      continue;
-
-    Available.push(SU);
-    Pending.remove(Pending.begin()+i);
-    --i; --e;
-  }
-  CheckPending = false;
-}
-
-/// Remove SU from the ready set for this boundary.
-void ConvergingVLIWScheduler::VLIWSchedBoundary::removeReady(SUnit *SU) {
-  if (Available.isInQueue(SU))
-    Available.remove(Available.find(SU));
-  else {
-    assert(Pending.isInQueue(SU) && "bad ready count");
-    Pending.remove(Pending.find(SU));
-  }
-}
+int HexagonConvergingVLIWScheduler::SchedulingCost(ReadyQueue &Q, SUnit *SU,
+                                                   SchedCandidate &Candidate,
+                                                   RegPressureDelta &Delta,
+                                                   bool verbose) {
+  int ResCount =
+      ConvergingVLIWScheduler::SchedulingCost(Q, SU, Candidate, Delta, verbose);
 
-/// If this queue only has one ready candidate, return it. As a side effect,
-/// advance the cycle until at least one node is ready. If multiple instructions
-/// are ready, return NULL.
-SUnit *ConvergingVLIWScheduler::VLIWSchedBoundary::pickOnlyChoice() {
-  if (CheckPending)
-    releasePending();
-
-  auto AdvanceCycle = [this]() {
-    if (Available.empty())
-      return true;
-    if (Available.size() == 1 && Pending.size() > 0)
-      return !ResourceModel->isResourceAvailable(*Available.begin(), isTop()) ||
-        getWeakLeft(*Available.begin(), isTop()) != 0;
-    return false;
-  };
-  for (unsigned i = 0; AdvanceCycle(); ++i) {
-    assert(i <= (HazardRec->getMaxLookAhead() + MaxMinLatency) &&
-           "permanent hazard"); (void)i;
-    ResourceModel->reserveResources(nullptr, isTop());
-    bumpCycle();
-    releasePending();
-  }
-  if (Available.size() == 1)
-    return *Available.begin();
-  return nullptr;
-}
-
-#ifndef NDEBUG
-void ConvergingVLIWScheduler::traceCandidate(const char *Label,
-      const ReadyQueue &Q, SUnit *SU, int Cost, PressureChange P) {
-  dbgs() << Label << " " << Q.getName() << " ";
-  if (P.isValid())
-    dbgs() << DAG->TRI->getRegPressureSetName(P.getPSet()) << ":"
-           << P.getUnitInc() << " ";
-  else
-    dbgs() << "     ";
-  dbgs() << "cost(" << Cost << ")\t";
-  DAG->dumpNode(*SU);
-}
-
-// Very detailed queue dump, to be used with higher verbosity levels.
-void ConvergingVLIWScheduler::readyQueueVerboseDump(
-      const RegPressureTracker &RPTracker, SchedCandidate &Candidate,
-      ReadyQueue &Q) {
-  RegPressureTracker &TempTracker = const_cast<RegPressureTracker &>(RPTracker);
-
-  dbgs() << ">>> " << Q.getName() << "\n";
-  for (ReadyQueue::iterator I = Q.begin(), E = Q.end(); I != E; ++I) {
-    RegPressureDelta RPDelta;
-    TempTracker.getMaxPressureDelta((*I)->getInstr(), RPDelta,
-                                    DAG->getRegionCriticalPSets(),
-                                    DAG->getRegPressure().MaxSetPressure);
-    std::stringstream dbgstr;
-    dbgstr << "SU(" << std::setw(3) << (*I)->NodeNum << ")";
-    dbgs() << dbgstr.str();
-    SchedulingCost(Q, *I, Candidate, RPDelta, true);
-    dbgs() << "\t";
-    (*I)->getInstr()->dump();
-  }
-  dbgs() << "\n";
-}
-#endif
-
-/// isSingleUnscheduledPred - If SU2 is the only unscheduled predecessor
-/// of SU, return true (we may have duplicates)
-static inline bool isSingleUnscheduledPred(SUnit *SU, SUnit *SU2) {
-  if (SU->NumPredsLeft == 0)
-    return false;
-
-  for (auto &Pred : SU->Preds) {
-    // We found an available, but not scheduled, predecessor.
-    if (!Pred.getSUnit()->isScheduled && (Pred.getSUnit() != SU2))
-      return false;
-  }
-
-  return true;
-}
-
-/// isSingleUnscheduledSucc - If SU2 is the only unscheduled successor
-/// of SU, return true (we may have duplicates)
-static inline bool isSingleUnscheduledSucc(SUnit *SU, SUnit *SU2) {
-  if (SU->NumSuccsLeft == 0)
-    return false;
-
-  for (auto &Succ : SU->Succs) {
-    // We found an available, but not scheduled, successor.
-    if (!Succ.getSUnit()->isScheduled && (Succ.getSUnit() != SU2))
-      return false;
-  }
-  return true;
-}
-
-/// Check if the instruction changes the register pressure of a register in the
-/// high pressure set. The function returns a negative value if the pressure
-/// decreases and a positive value is the pressure increases. If the instruction
-/// doesn't use a high pressure register or doesn't change the register
-/// pressure, then return 0.
-int ConvergingVLIWScheduler::pressureChange(const SUnit *SU, bool isBotUp) {
-  PressureDiff &PD = DAG->getPressureDiff(SU);
-  for (auto &P : PD) {
-    if (!P.isValid())
-      continue;
-    // The pressure differences are computed bottom-up, so the comparision for
-    // an increase is positive in the bottom direction, but negative in the
-    //  top-down direction.
-    if (HighPressureSets[P.getPSet()])
-      return (isBotUp ? P.getUnitInc() : -P.getUnitInc());
-  }
-  return 0;
-}
-
-// Constants used to denote relative importance of
-// heuristic components for cost computation.
-static const unsigned PriorityOne = 200;
-static const unsigned PriorityTwo = 50;
-static const unsigned PriorityThree = 75;
-static const unsigned ScaleTwo = 10;
-
-/// Single point to compute overall scheduling cost.
-/// TODO: More heuristics will be used soon.
-int ConvergingVLIWScheduler::SchedulingCost(ReadyQueue &Q, SUnit *SU,
-                                            SchedCandidate &Candidate,
-                                            RegPressureDelta &Delta,
-                                            bool verbose) {
-  // Initial trivial priority.
-  int ResCount = 1;
-
-  // Do not waste time on a node that is already scheduled.
   if (!SU || SU->isScheduled)
     return ResCount;
 
-  LLVM_DEBUG(if (verbose) dbgs()
-             << ((Q.getID() == TopQID) ? "(top|" : "(bot|"));
-  // Forced priority is high.
-  if (SU->isScheduleHigh) {
-    ResCount += PriorityOne;
-    LLVM_DEBUG(dbgs() << "H|");
-  }
-
-  unsigned IsAvailableAmt = 0;
-  // Critical path first.
-  if (Q.getID() == TopQID) {
-    if (Top.isLatencyBound(SU)) {
-      LLVM_DEBUG(if (verbose) dbgs() << "LB|");
-      ResCount += (SU->getHeight() * ScaleTwo);
-    }
-
-    LLVM_DEBUG(if (verbose) {
-      std::stringstream dbgstr;
-      dbgstr << "h" << std::setw(3) << SU->getHeight() << "|";
-      dbgs() << dbgstr.str();
-    });
-
-    // If resources are available for it, multiply the
-    // chance of scheduling.
-    if (Top.ResourceModel->isResourceAvailable(SU, true)) {
-      IsAvailableAmt = (PriorityTwo + PriorityThree);
-      ResCount += IsAvailableAmt;
-      LLVM_DEBUG(if (verbose) dbgs() << "A|");
-    } else
-      LLVM_DEBUG(if (verbose) dbgs() << " |");
-  } else {
-    if (Bot.isLatencyBound(SU)) {
-      LLVM_DEBUG(if (verbose) dbgs() << "LB|");
-      ResCount += (SU->getDepth() * ScaleTwo);
-    }
-
-    LLVM_DEBUG(if (verbose) {
-      std::stringstream dbgstr;
-      dbgstr << "d" << std::setw(3) << SU->getDepth() << "|";
-      dbgs() << dbgstr.str();
-    });
-
-    // If resources are available for it, multiply the
-    // chance of scheduling.
-    if (Bot.ResourceModel->isResourceAvailable(SU, false)) {
-      IsAvailableAmt = (PriorityTwo + PriorityThree);
-      ResCount += IsAvailableAmt;
-      LLVM_DEBUG(if (verbose) dbgs() << "A|");
-    } else
-      LLVM_DEBUG(if (verbose) dbgs() << " |");
-  }
-
-  unsigned NumNodesBlocking = 0;
-  if (Q.getID() == TopQID) {
-    // How many SUs does it block from scheduling?
-    // Look at all of the successors of this node.
-    // Count the number of nodes that
-    // this node is the sole unscheduled node for.
-    if (Top.isLatencyBound(SU))
-      for (const SDep &SI : SU->Succs)
-        if (isSingleUnscheduledPred(SI.getSUnit(), SU))
-          ++NumNodesBlocking;
-  } else {
-    // How many unscheduled predecessors block this node?
-    if (Bot.isLatencyBound(SU))
-      for (const SDep &PI : SU->Preds)
-        if (isSingleUnscheduledSucc(PI.getSUnit(), SU))
-          ++NumNodesBlocking;
-  }
-  ResCount += (NumNodesBlocking * ScaleTwo);
-
-  LLVM_DEBUG(if (verbose) {
-    std::stringstream dbgstr;
-    dbgstr << "blk " << std::setw(2) << NumNodesBlocking << ")|";
-    dbgs() << dbgstr.str();
-  });
-
-  // Factor in reg pressure as a heuristic.
-  if (!IgnoreBBRegPressure) {
-    // Decrease priority by the amount that register pressure exceeds the limit.
-    ResCount -= (Delta.Excess.getUnitInc()*PriorityOne);
-    // Decrease priority if register pressure exceeds the limit.
-    ResCount -= (Delta.CriticalMax.getUnitInc()*PriorityOne);
-    // Decrease priority slightly if register pressure would increase over the
-    // current maximum.
-    ResCount -= (Delta.CurrentMax.getUnitInc()*PriorityTwo);
-    // If there are register pressure issues, then we remove the value added for
-    // the instruction being available. The rationale is that we really don't
-    // want to schedule an instruction that causes a spill.
-    if (IsAvailableAmt && pressureChange(SU, Q.getID() != TopQID) > 0 &&
-        (Delta.Excess.getUnitInc() || Delta.CriticalMax.getUnitInc() ||
-         Delta.CurrentMax.getUnitInc()))
-      ResCount -= IsAvailableAmt;
-    LLVM_DEBUG(if (verbose) {
-      dbgs() << "RP " << Delta.Excess.getUnitInc() << "/"
-             << Delta.CriticalMax.getUnitInc() << "/"
-             << Delta.CurrentMax.getUnitInc() << ")|";
-    });
-  }
-
-  // Give a little extra priority to a .cur instruction if there is a resource
-  // available for it.
   auto &QST = DAG->MF.getSubtarget<HexagonSubtarget>();
   auto &QII = *QST.getInstrInfo();
   if (SU->isInstr() && QII.mayBeCurLoad(*SU->getInstr())) {
@@ -698,303 +66,5 @@ int ConvergingVLIWScheduler::SchedulingCost(ReadyQueue &Q, SUnit *SU,
     }
   }
 
-  // Give preference to a zero latency instruction if the dependent
-  // instruction is in the current packet.
-  if (Q.getID() == TopQID && getWeakLeft(SU, true) == 0) {
-    for (const SDep &PI : SU->Preds) {
-      if (!PI.getSUnit()->getInstr()->isPseudo() && PI.isAssignedRegDep() &&
-          PI.getLatency() == 0 &&
-          Top.ResourceModel->isInPacket(PI.getSUnit())) {
-        ResCount += PriorityThree;
-        LLVM_DEBUG(if (verbose) dbgs() << "Z|");
-      }
-    }
-  } else if (Q.getID() == BotQID && getWeakLeft(SU, false) == 0) {
-    for (const SDep &SI : SU->Succs) {
-      if (!SI.getSUnit()->getInstr()->isPseudo() && SI.isAssignedRegDep() &&
-          SI.getLatency() == 0 &&
-          Bot.ResourceModel->isInPacket(SI.getSUnit())) {
-        ResCount += PriorityThree;
-        LLVM_DEBUG(if (verbose) dbgs() << "Z|");
-      }
-    }
-  }
-
-  // If the instruction has a non-zero latency dependence with an instruction in
-  // the current packet, then it should not be scheduled yet. The case occurs
-  // when the dependent instruction is scheduled in a new packet, so the
-  // scheduler updates the current cycle and pending instructions become
-  // available.
-  if (CheckEarlyAvail) {
-    if (Q.getID() == TopQID) {
-      for (const auto &PI : SU->Preds) {
-        if (PI.getLatency() > 0 &&
-            Top.ResourceModel->isInPacket(PI.getSUnit())) {
-          ResCount -= PriorityOne;
-          LLVM_DEBUG(if (verbose) dbgs() << "D|");
-        }
-      }
-    } else {
-      for (const auto &SI : SU->Succs) {
-        if (SI.getLatency() > 0 &&
-            Bot.ResourceModel->isInPacket(SI.getSUnit())) {
-          ResCount -= PriorityOne;
-          LLVM_DEBUG(if (verbose) dbgs() << "D|");
-        }
-      }
-    }
-  }
-
-  LLVM_DEBUG(if (verbose) {
-    std::stringstream dbgstr;
-    dbgstr << "Total " << std::setw(4) << ResCount << ")";
-    dbgs() << dbgstr.str();
-  });
-
   return ResCount;
 }
-
-/// Pick the best candidate from the top queue.
-///
-/// TODO: getMaxPressureDelta results can be mostly cached for each SUnit during
-/// DAG building. To adjust for the current scheduling location we need to
-/// maintain the number of vreg uses remaining to be top-scheduled.
-ConvergingVLIWScheduler::CandResult ConvergingVLIWScheduler::
-pickNodeFromQueue(VLIWSchedBoundary &Zone, const RegPressureTracker &RPTracker,
-                  SchedCandidate &Candidate) {
-  ReadyQueue &Q = Zone.Available;
-  LLVM_DEBUG(if (SchedDebugVerboseLevel > 1)
-                 readyQueueVerboseDump(RPTracker, Candidate, Q);
-             else Q.dump(););
-
-  // getMaxPressureDelta temporarily modifies the tracker.
-  RegPressureTracker &TempTracker = const_cast<RegPressureTracker&>(RPTracker);
-
-  // BestSU remains NULL if no top candidates beat the best existing candidate.
-  CandResult FoundCandidate = NoCand;
-  for (ReadyQueue::iterator I = Q.begin(), E = Q.end(); I != E; ++I) {
-    RegPressureDelta RPDelta;
-    TempTracker.getMaxPressureDelta((*I)->getInstr(), RPDelta,
-                                    DAG->getRegionCriticalPSets(),
-                                    DAG->getRegPressure().MaxSetPressure);
-
-    int CurrentCost = SchedulingCost(Q, *I, Candidate, RPDelta, false);
-
-    // Initialize the candidate if needed.
-    if (!Candidate.SU) {
-      LLVM_DEBUG(traceCandidate("DCAND", Q, *I, CurrentCost));
-      Candidate.SU = *I;
-      Candidate.RPDelta = RPDelta;
-      Candidate.SCost = CurrentCost;
-      FoundCandidate = NodeOrder;
-      continue;
-    }
-
-    // Choose node order for negative cost candidates. There is no good
-    // candidate in this case.
-    if (CurrentCost < 0 && Candidate.SCost < 0) {
-      if ((Q.getID() == TopQID && (*I)->NodeNum < Candidate.SU->NodeNum)
-          || (Q.getID() == BotQID && (*I)->NodeNum > Candidate.SU->NodeNum)) {
-        LLVM_DEBUG(traceCandidate("NCAND", Q, *I, CurrentCost));
-        Candidate.SU = *I;
-        Candidate.RPDelta = RPDelta;
-        Candidate.SCost = CurrentCost;
-        FoundCandidate = NodeOrder;
-      }
-      continue;
-    }
-
-    // Best cost.
-    if (CurrentCost > Candidate.SCost) {
-      LLVM_DEBUG(traceCandidate("CCAND", Q, *I, CurrentCost));
-      Candidate.SU = *I;
-      Candidate.RPDelta = RPDelta;
-      Candidate.SCost = CurrentCost;
-      FoundCandidate = BestCost;
-      continue;
-    }
-
-    // Choose an instruction that does not depend on an artificial edge.
-    unsigned CurrWeak = getWeakLeft(*I, (Q.getID() == TopQID));
-    unsigned CandWeak = getWeakLeft(Candidate.SU, (Q.getID() == TopQID));
-    if (CurrWeak != CandWeak) {
-      if (CurrWeak < CandWeak) {
-        LLVM_DEBUG(traceCandidate("WCAND", Q, *I, CurrentCost));
-        Candidate.SU = *I;
-        Candidate.RPDelta = RPDelta;
-        Candidate.SCost = CurrentCost;
-        FoundCandidate = Weak;
-      }
-      continue;
-    }
-
-    if (CurrentCost == Candidate.SCost && Zone.isLatencyBound(*I)) {
-      unsigned CurrSize, CandSize;
-      if (Q.getID() == TopQID) {
-        CurrSize = (*I)->Succs.size();
-        CandSize = Candidate.SU->Succs.size();
-      } else {
-        CurrSize = (*I)->Preds.size();
-        CandSize = Candidate.SU->Preds.size();
-      }
-      if (CurrSize > CandSize) {
-        LLVM_DEBUG(traceCandidate("SPCAND", Q, *I, CurrentCost));
-        Candidate.SU = *I;
-        Candidate.RPDelta = RPDelta;
-        Candidate.SCost = CurrentCost;
-        FoundCandidate = BestCost;
-      }
-      // Keep the old candidate if it's a better candidate. That is, don't use
-      // the subsequent tie breaker.
-      if (CurrSize != CandSize)
-        continue;
-    }
-
-    // Tie breaker.
-    // To avoid scheduling indeterminism, we need a tie breaker
-    // for the case when cost is identical for two nodes.
-    if (UseNewerCandidate && CurrentCost == Candidate.SCost) {
-      if ((Q.getID() == TopQID && (*I)->NodeNum < Candidate.SU->NodeNum)
-          || (Q.getID() == BotQID && (*I)->NodeNum > Candidate.SU->NodeNum)) {
-        LLVM_DEBUG(traceCandidate("TCAND", Q, *I, CurrentCost));
-        Candidate.SU = *I;
-        Candidate.RPDelta = RPDelta;
-        Candidate.SCost = CurrentCost;
-        FoundCandidate = NodeOrder;
-        continue;
-      }
-    }
-
-    // Fall through to original instruction order.
-    // Only consider node order if Candidate was chosen from this Q.
-    if (FoundCandidate == NoCand)
-      continue;
-  }
-  return FoundCandidate;
-}
-
-/// Pick the best candidate node from either the top or bottom queue.
-SUnit *ConvergingVLIWScheduler::pickNodeBidrectional(bool &IsTopNode) {
-  // Schedule as far as possible in the direction of no choice. This is most
-  // efficient, but also provides the best heuristics for CriticalPSets.
-  if (SUnit *SU = Bot.pickOnlyChoice()) {
-    LLVM_DEBUG(dbgs() << "Picked only Bottom\n");
-    IsTopNode = false;
-    return SU;
-  }
-  if (SUnit *SU = Top.pickOnlyChoice()) {
-    LLVM_DEBUG(dbgs() << "Picked only Top\n");
-    IsTopNode = true;
-    return SU;
-  }
-  SchedCandidate BotCand;
-  // Prefer bottom scheduling when heuristics are silent.
-  CandResult BotResult = pickNodeFromQueue(Bot,
-                                           DAG->getBotRPTracker(), BotCand);
-  assert(BotResult != NoCand && "failed to find the first candidate");
-
-  // If either Q has a single candidate that provides the least increase in
-  // Excess pressure, we can immediately schedule from that Q.
-  //
-  // RegionCriticalPSets summarizes the pressure within the scheduled region and
-  // affects picking from either Q. If scheduling in one direction must
-  // increase pressure for one of the excess PSets, then schedule in that
-  // direction first to provide more freedom in the other direction.
-  if (BotResult == SingleExcess || BotResult == SingleCritical) {
-    LLVM_DEBUG(dbgs() << "Prefered Bottom Node\n");
-    IsTopNode = false;
-    return BotCand.SU;
-  }
-  // Check if the top Q has a better candidate.
-  SchedCandidate TopCand;
-  CandResult TopResult = pickNodeFromQueue(Top,
-                                           DAG->getTopRPTracker(), TopCand);
-  assert(TopResult != NoCand && "failed to find the first candidate");
-
-  if (TopResult == SingleExcess || TopResult == SingleCritical) {
-    LLVM_DEBUG(dbgs() << "Prefered Top Node\n");
-    IsTopNode = true;
-    return TopCand.SU;
-  }
-  // If either Q has a single candidate that minimizes pressure above the
-  // original region's pressure pick it.
-  if (BotResult == SingleMax) {
-    LLVM_DEBUG(dbgs() << "Prefered Bottom Node SingleMax\n");
-    IsTopNode = false;
-    return BotCand.SU;
-  }
-  if (TopResult == SingleMax) {
-    LLVM_DEBUG(dbgs() << "Prefered Top Node SingleMax\n");
-    IsTopNode = true;
-    return TopCand.SU;
-  }
-  if (TopCand.SCost > BotCand.SCost) {
-    LLVM_DEBUG(dbgs() << "Prefered Top Node Cost\n");
-    IsTopNode = true;
-    return TopCand.SU;
-  }
-  // Otherwise prefer the bottom candidate in node order.
-  LLVM_DEBUG(dbgs() << "Prefered Bottom in Node order\n");
-  IsTopNode = false;
-  return BotCand.SU;
-}
-
-/// Pick the best node to balance the schedule. Implements MachineSchedStrategy.
-SUnit *ConvergingVLIWScheduler::pickNode(bool &IsTopNode) {
-  if (DAG->top() == DAG->bottom()) {
-    assert(Top.Available.empty() && Top.Pending.empty() &&
-           Bot.Available.empty() && Bot.Pending.empty() && "ReadyQ garbage");
-    return nullptr;
-  }
-  SUnit *SU;
-  if (ForceTopDown) {
-    SU = Top.pickOnlyChoice();
-    if (!SU) {
-      SchedCandidate TopCand;
-      CandResult TopResult =
-        pickNodeFromQueue(Top, DAG->getTopRPTracker(), TopCand);
-      assert(TopResult != NoCand && "failed to find the first candidate");
-      (void)TopResult;
-      SU = TopCand.SU;
-    }
-    IsTopNode = true;
-  } else if (ForceBottomUp) {
-    SU = Bot.pickOnlyChoice();
-    if (!SU) {
-      SchedCandidate BotCand;
-      CandResult BotResult =
-        pickNodeFromQueue(Bot, DAG->getBotRPTracker(), BotCand);
-      assert(BotResult != NoCand && "failed to find the first candidate");
-      (void)BotResult;
-      SU = BotCand.SU;
-    }
-    IsTopNode = false;
-  } else {
-    SU = pickNodeBidrectional(IsTopNode);
-  }
-  if (SU->isTopReady())
-    Top.removeReady(SU);
-  if (SU->isBottomReady())
-    Bot.removeReady(SU);
-
-  LLVM_DEBUG(dbgs() << "*** " << (IsTopNode ? "Top" : "Bottom")
-                    << " Scheduling instruction in cycle "
-                    << (IsTopNode ? Top.CurrCycle : Bot.CurrCycle) << " ("
-                    << reportPackets() << ")\n";
-             DAG->dumpNode(*SU));
-  return SU;
-}
-
-/// Update the scheduler's state after scheduling a node. This is the same node
-/// that was just returned by pickNode(). However, VLIWMachineScheduler needs
-/// to update it's state based on the current cycle before MachineSchedStrategy
-/// does.
-void ConvergingVLIWScheduler::schedNode(SUnit *SU, bool IsTopNode) {
-  if (IsTopNode) {
-    Top.bumpNode(SU);
-    SU->TopReadyCycle = Top.CurrCycle;
-  } else {
-    Bot.bumpNode(SU);
-    SU->BotReadyCycle = Bot.CurrCycle;
-  }
-}
diff --git a/llvm/lib/Target/Hexagon/HexagonMachineScheduler.h b/llvm/lib/Target/Hexagon/HexagonMachineScheduler.h
index fb0a7abd339b..3d8f557dc787 100644
--- a/llvm/lib/Target/Hexagon/HexagonMachineScheduler.h
+++ b/llvm/lib/Target/Hexagon/HexagonMachineScheduler.h
@@ -13,261 +13,28 @@
 #ifndef LLVM_LIB_TARGET_HEXAGON_HEXAGONMACHINESCHEDULER_H
 #define LLVM_LIB_TARGET_HEXAGON_HEXAGONMACHINESCHEDULER_H
 
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/Twine.h"
-#include "llvm/CodeGen/DFAPacketizer.h"
 #include "llvm/CodeGen/MachineScheduler.h"
 #include "llvm/CodeGen/RegisterPressure.h"
-#include "llvm/CodeGen/ScheduleHazardRecognizer.h"
-#include "llvm/CodeGen/TargetInstrInfo.h"
-#include "llvm/CodeGen/TargetSchedule.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
-#include <algorithm>
-#include <cassert>
-#include <limits>
-#include <memory>
-#include <vector>
+#include "llvm/CodeGen/VLIWMachineScheduler.h"
 
 namespace llvm {
 
 class SUnit;
 
-class VLIWResourceModel {
-  /// ResourcesModel - Represents VLIW state.
-  /// Not limited to VLIW targets per se, but assumes
-  /// definition of DFA by a target.
-  DFAPacketizer *ResourcesModel;
-
-  const TargetSchedModel *SchedModel;
-
-  /// Local packet/bundle model. Purely
-  /// internal to the MI schedulre at the time.
-  std::vector<SUnit *> Packet;
-
-  /// Total packets created.
-  unsigned TotalPackets = 0;
-
+class HexagonVLIWResourceModel : public VLIWResourceModel {
 public:
-  VLIWResourceModel(const TargetSubtargetInfo &STI, const TargetSchedModel *SM)
-      : SchedModel(SM) {
-    ResourcesModel = STI.getInstrInfo()->CreateTargetScheduleState(STI);
-
-    // This hard requirement could be relaxed,
-    // but for now do not let it proceed.
-    assert(ResourcesModel && "Unimplemented CreateTargetScheduleState.");
-
-    Packet.resize(SchedModel->getIssueWidth());
-    Packet.clear();
-    ResourcesModel->clearResources();
-  }
-
-  ~VLIWResourceModel() {
-    delete ResourcesModel;
-  }
-
-  void resetPacketState() {
-    Packet.clear();
-  }
-
-  void resetDFA() {
-    ResourcesModel->clearResources();
-  }
-
-  void reset() {
-    Packet.clear();
-    ResourcesModel->clearResources();
-  }
-
-  bool isResourceAvailable(SUnit *SU, bool IsTop);
-  bool reserveResources(SUnit *SU, bool IsTop);
-  unsigned getTotalPackets() const { return TotalPackets; }
-  bool isInPacket(SUnit *SU) const { return is_contained(Packet, SU); }
+  using VLIWResourceModel::VLIWResourceModel;
+  bool hasDependence(const SUnit *SUd, const SUnit *SUu) override;
 };
 
-/// Extend the standard ScheduleDAGMI to provide more context and override the
-/// top-level schedule() driver.
-class VLIWMachineScheduler : public ScheduleDAGMILive {
-public:
-  VLIWMachineScheduler(MachineSchedContext *C,
-                       std::unique_ptr<MachineSchedStrategy> S)
-      : ScheduleDAGMILive(C, std::move(S)) {}
-
-  /// Schedule - This is called back from ScheduleDAGInstrs::Run() when it's
-  /// time to do some work.
-  void schedule() override;
-
-  RegisterClassInfo *getRegClassInfo() { return RegClassInfo; }
-  int getBBSize() { return BB->size(); }
-};
-
-//===----------------------------------------------------------------------===//
-// ConvergingVLIWScheduler - Implementation of the standard
-// MachineSchedStrategy.
-//===----------------------------------------------------------------------===//
-
-/// ConvergingVLIWScheduler shrinks the unscheduled zone using heuristics
-/// to balance the schedule.
-class ConvergingVLIWScheduler : public MachineSchedStrategy {
-  /// Store the state used by ConvergingVLIWScheduler heuristics, required
-  ///  for the lifetime of one invocation of pickNode().
-  struct SchedCandidate {
-    // The best SUnit candidate.
-    SUnit *SU = nullptr;
-
-    // Register pressure values for the best candidate.
-    RegPressureDelta RPDelta;
-
-    // Best scheduling cost.
-    int SCost = 0;
-
-    SchedCandidate() = default;
-  };
-  /// Represent the type of SchedCandidate found within a single queue.
-  enum CandResult {
-    NoCand, NodeOrder, SingleExcess, SingleCritical, SingleMax, MultiPressure,
-    BestCost, Weak};
-
-  /// Each Scheduling boundary is associated with ready queues. It tracks the
-  /// current cycle in whichever direction at has moved, and maintains the state
-  /// of "hazards" and other interlocks at the current cycle.
-  struct VLIWSchedBoundary {
-    VLIWMachineScheduler *DAG = nullptr;
-    const TargetSchedModel *SchedModel = nullptr;
-
-    ReadyQueue Available;
-    ReadyQueue Pending;
-    bool CheckPending = false;
-
-    ScheduleHazardRecognizer *HazardRec = nullptr;
-    VLIWResourceModel *ResourceModel = nullptr;
-
-    unsigned CurrCycle = 0;
-    unsigned IssueCount = 0;
-    unsigned CriticalPathLength = 0;
-
-    /// MinReadyCycle - Cycle of the soonest available instruction.
-    unsigned MinReadyCycle = std::numeric_limits<unsigned>::max();
-
-    // Remember the greatest min operand latency.
-    unsigned MaxMinLatency = 0;
-
-    /// Pending queues extend the ready queues with the same ID and the
-    /// PendingFlag set.
-    VLIWSchedBoundary(unsigned ID, const Twine &Name)
-        : Available(ID, Name+".A"),
-          Pending(ID << ConvergingVLIWScheduler::LogMaxQID, Name+".P") {}
-
-    ~VLIWSchedBoundary() {
-      delete ResourceModel;
-      delete HazardRec;
-    }
-
-    void init(VLIWMachineScheduler *dag, const TargetSchedModel *smodel) {
-      DAG = dag;
-      SchedModel = smodel;
-      CurrCycle = 0;
-      IssueCount = 0;
-      // Initialize the critical path length limit, which used by the scheduling
-      // cost model to determine the value for scheduling an instruction. We use
-      // a slightly different heuristic for small and large functions. For small
-      // functions, it's important to use the height/depth of the instruction.
-      // For large functions, prioritizing by height or depth increases spills.
-      CriticalPathLength = DAG->getBBSize() / SchedModel->getIssueWidth();
-      if (DAG->getBBSize() < 50)
-        // We divide by two as a cheap and simple heuristic to reduce the
-        // critcal path length, which increases the priority of using the graph
-        // height/depth in the scheduler's cost computation.
-        CriticalPathLength >>= 1;
-      else {
-        // For large basic blocks, we prefer a larger critical path length to
-        // decrease the priority of using the graph height/depth.
-        unsigned MaxPath = 0;
-        for (auto &SU : DAG->SUnits)
-          MaxPath = std::max(MaxPath, isTop() ? SU.getHeight() : SU.getDepth());
-        CriticalPathLength = std::max(CriticalPathLength, MaxPath) + 1;
-      }
-    }
-
-    bool isTop() const {
-      return Available.getID() == ConvergingVLIWScheduler::TopQID;
-    }
-
-    bool checkHazard(SUnit *SU);
-
-    void releaseNode(SUnit *SU, unsigned ReadyCycle);
-
-    void bumpCycle();
-
-    void bumpNode(SUnit *SU);
-
-    void releasePending();
-
-    void removeReady(SUnit *SU);
-
-    SUnit *pickOnlyChoice();
-
-    bool isLatencyBound(SUnit *SU) {
-      if (CurrCycle >= CriticalPathLength)
-        return true;
-      unsigned PathLength = isTop() ? SU->getHeight() : SU->getDepth();
-      return CriticalPathLength - CurrCycle <= PathLength;
-    }
-  };
-
-  VLIWMachineScheduler *DAG = nullptr;
-  const TargetSchedModel *SchedModel = nullptr;
-
-  // State of the top and bottom scheduled instruction boundaries.
-  VLIWSchedBoundary Top;
-  VLIWSchedBoundary Bot;
-
-  /// List of pressure sets that have a high pressure level in the region.
-  std::vector<bool> HighPressureSets;
-
-public:
-  /// SUnit::NodeQueueId: 0 (none), 1 (top), 2 (bot), 3 (both)
-  enum {
-    TopQID = 1,
-    BotQID = 2,
-    LogMaxQID = 2
-  };
-
-  ConvergingVLIWScheduler() : Top(TopQID, "TopQ"), Bot(BotQID, "BotQ") {}
-
-  void initialize(ScheduleDAGMI *dag) override;
-
-  SUnit *pickNode(bool &IsTopNode) override;
-
-  void schedNode(SUnit *SU, bool IsTopNode) override;
-
-  void releaseTopNode(SUnit *SU) override;
-
-  void releaseBottomNode(SUnit *SU) override;
-
-  unsigned reportPackets() {
-    return Top.ResourceModel->getTotalPackets() +
-           Bot.ResourceModel->getTotalPackets();
-  }
-
+class HexagonConvergingVLIWScheduler : public ConvergingVLIWScheduler {
 protected:
-  SUnit *pickNodeBidrectional(bool &IsTopNode);
-
-  int pressureChange(const SUnit *SU, bool isBotUp);
-
-  int SchedulingCost(ReadyQueue &Q,
-                     SUnit *SU, SchedCandidate &Candidate,
-                     RegPressureDelta &Delta, bool verbose);
-
-  CandResult pickNodeFromQueue(VLIWSchedBoundary &Zone,
-                               const RegPressureTracker &RPTracker,
-                               SchedCandidate &Candidate);
-#ifndef NDEBUG
-  void traceCandidate(const char *Label, const ReadyQueue &Q, SUnit *SU,
-                      int Cost, PressureChange P = PressureChange());
-
-  void readyQueueVerboseDump(const RegPressureTracker &RPTracker,
-                             SchedCandidate &Candidate, ReadyQueue &Q);
-#endif
+  VLIWResourceModel *
+  createVLIWResourceModel(const TargetSubtargetInfo &STI,
+                          const TargetSchedModel *SchedModel) const override;
+  int SchedulingCost(ReadyQueue &Q, SUnit *SU, SchedCandidate &Candidate,
+                     RegPressureDelta &Delta, bool verbose) override;
 };
 
 } // end namespace llvm
diff --git a/llvm/lib/Target/Hexagon/HexagonPseudo.td b/llvm/lib/Target/Hexagon/HexagonPseudo.td
index 11f8af7c41a0..afd63d6d4aa7 100644
--- a/llvm/lib/Target/Hexagon/HexagonPseudo.td
+++ b/llvm/lib/Target/Hexagon/HexagonPseudo.td
@@ -572,3 +572,14 @@ defm PS_storerd : NewCircularStore<DoubleRegs, WordAccess>;
 // __builtin_trap.
 let hasSideEffects = 1, isPseudo = 1, isCodeGenOnly = 1, isSolo = 1 in
 def PS_crash: InstHexagon<(outs), (ins), "", [], "", PSEUDO, TypePSEUDO>;
+
+// This is actual trap1 instruction from before v65. It's here since it is
+// no longer included in DepInstrInfo.td.
+def PS_trap1 : HInst<(outs), (ins u8_0Imm:$Ii), "trap1(#$Ii)", tc_53c851ab,
+                     TypeJ>, Enc_a51a9a, Requires<[HasPreV65]> {
+  let Inst{1-0} = 0b00;
+  let Inst{7-5} = 0b000;
+  let Inst{13-13} = 0b0;
+  let Inst{31-16} = 0b0101010010000000;
+}
+
diff --git a/llvm/lib/Target/Hexagon/HexagonSchedule.td b/llvm/lib/Target/Hexagon/HexagonSchedule.td
index 88d775f16a7f..931578c9e78d 100644
--- a/llvm/lib/Target/Hexagon/HexagonSchedule.td
+++ b/llvm/lib/Target/Hexagon/HexagonSchedule.td
@@ -69,3 +69,4 @@ include "HexagonScheduleV66.td"
 include "HexagonScheduleV67.td"
 include "HexagonScheduleV67T.td"
 include "HexagonScheduleV68.td"
+include "HexagonScheduleV69.td"
diff --git a/llvm/lib/Target/Hexagon/HexagonScheduleV69.td b/llvm/lib/Target/Hexagon/HexagonScheduleV69.td
new file mode 100644
index 000000000000..ddd246866e20
--- /dev/null
+++ b/llvm/lib/Target/Hexagon/HexagonScheduleV69.td
@@ -0,0 +1,40 @@
+//=-HexagonScheduleV69.td - HexagonV69 Scheduling Definitions *- tablegen -*-=//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+//
+// ScalarItin and HVXItin contain some old itineraries
+// still used by a handful of instructions. Hopefully, we will be able
+// to get rid of them soon.
+def HexagonV69ItinList : DepScalarItinV69, ScalarItin,
+                         DepHVXItinV69, HVXItin, PseudoItin {
+  list<InstrItinData> ItinList =
+    !listconcat(DepScalarItinV69_list, ScalarItin_list,
+                DepHVXItinV69_list, HVXItin_list, PseudoItin_list);
+}
+
+def HexagonItinerariesV69 :
+      ProcessorItineraries<[SLOT0, SLOT1, SLOT2, SLOT3, SLOT_ENDLOOP,
+                            CVI_ST, CVI_XLANE, CVI_SHIFT, CVI_MPY0, CVI_MPY1,
+                            CVI_LD, CVI_XLSHF, CVI_MPY01, CVI_ALL,
+                            CVI_ALL_NOMEM, CVI_ZW],
+                            [Hex_FWD, HVX_FWD],
+                            HexagonV69ItinList.ItinList>;
+
+def HexagonModelV69 : SchedMachineModel {
+  // Max issue per cycle == bundle width.
+  let IssueWidth = 4;
+  let Itineraries = HexagonItinerariesV69;
+  let LoadLatency = 1;
+  let CompleteModel = 0;
+}
+
+//===----------------------------------------------------------------------===//
+// Hexagon V69 Resource Definitions -
+//===----------------------------------------------------------------------===//
+
diff --git a/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp b/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp
index ecb2f88d8096..08bb4580b585 100644
--- a/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp
@@ -75,6 +75,10 @@ static cl::opt<bool> EnableCheckBankConflict("hexagon-check-bank-conflict",
   cl::Hidden, cl::ZeroOrMore, cl::init(true),
   cl::desc("Enable checking for cache bank conflicts"));
 
+static cl::opt<bool> EnableV68FloatCodeGen(
+    "force-hvx-float", cl::Hidden, cl::ZeroOrMore, cl::init(false),
+    cl::desc("Enable the code-generation for vector float instructions on v68."));
+
 HexagonSubtarget::HexagonSubtarget(const Triple &TT, StringRef CPU,
                                    StringRef FS, const TargetMachine &TM)
     : HexagonGenSubtargetInfo(TT, CPU, /*TuneCPU*/ CPU, FS),
@@ -103,13 +107,71 @@ HexagonSubtarget::initializeSubtargetDependencies(StringRef CPU, StringRef FS) {
   UseAudioOps = false;
   UseLongCalls = false;
 
-  UseBSBScheduling = hasV60Ops() && EnableBSBSched;
+  SubtargetFeatures Features(FS);
 
-  ParseSubtargetFeatures(CPUString, /*TuneCPU*/ CPUString, FS);
+  // Turn on QFloat if the HVX version is v68+.
+  // The function ParseSubtargetFeatures will set feature bits and initialize
+  // subtarget's variables all in one, so there isn't a good way to preprocess
+  // the feature string, other than by tinkering with it directly.
+  auto IsQFloatFS = [](StringRef F) {
+    return F == "+hvx-qfloat" || F == "-hvx-qfloat";
+  };
+  if (!llvm::count_if(Features.getFeatures(), IsQFloatFS)) {
+    auto getHvxVersion = [&Features](StringRef FS) -> StringRef {
+      for (StringRef F : llvm::reverse(Features.getFeatures())) {
+        if (F.startswith("+hvxv"))
+          return F;
+      }
+      for (StringRef F : llvm::reverse(Features.getFeatures())) {
+        if (F == "-hvx")
+          return StringRef();
+        if (F.startswith("+hvx") || F == "-hvx")
+          return F.take_front(4);  // Return "+hvx" or "-hvx".
+      }
+      return StringRef();
+    };
+
+    bool AddQFloat = false;
+    StringRef HvxVer = getHvxVersion(FS);
+    if (HvxVer.startswith("+hvxv")) {
+      int Ver = 0;
+      if (!HvxVer.drop_front(5).consumeInteger(10, Ver) && Ver >= 68)
+        AddQFloat = true;
+    } else if (HvxVer == "+hvx") {
+      if (hasV68Ops())
+        AddQFloat = true;
+    }
+
+    if (AddQFloat)
+      Features.AddFeature("+hvx-qfloat");
+  }
+
+  std::string FeatureString = Features.getString();
+  ParseSubtargetFeatures(CPUString, /*TuneCPU*/ CPUString, FeatureString);
+
+  // Enable float code generation only if the flag(s) are set and
+  // the feature is enabled. v68 is guarded by additional flags.
+  bool GreaterThanV68 = false;
+  if (useHVXV69Ops())
+    GreaterThanV68 = true;
+
+  // Support for deprecated qfloat/ieee codegen flags
+  if (!GreaterThanV68) {
+    if (EnableV68FloatCodeGen)
+      UseHVXFloatingPoint = true;
+  } else {
+    UseHVXFloatingPoint = true;
+  }
+
+  if (UseHVXQFloatOps && UseHVXIEEEFPOps && UseHVXFloatingPoint)
+    LLVM_DEBUG(
+        dbgs() << "Behavior is undefined for simultaneous qfloat and ieee hvx codegen...");
 
   if (OverrideLongCalls.getPosition())
     UseLongCalls = OverrideLongCalls;
 
+  UseBSBScheduling = hasV60Ops() && EnableBSBSched;
+
   if (isTinyCore()) {
     // Tiny core has a single thread, so back-to-back scheduling is enabled by
     // default.
@@ -117,10 +179,10 @@ HexagonSubtarget::initializeSubtargetDependencies(StringRef CPU, StringRef FS) {
       UseBSBScheduling = false;
   }
 
-  FeatureBitset Features = getFeatureBits();
+  FeatureBitset FeatureBits = getFeatureBits();
   if (HexagonDisableDuplex)
-    setFeatureBits(Features.reset(Hexagon::FeatureDuplex));
-  setFeatureBits(Hexagon_MC::completeHVXFeatures(Features));
+    setFeatureBits(FeatureBits.reset(Hexagon::FeatureDuplex));
+  setFeatureBits(Hexagon_MC::completeHVXFeatures(FeatureBits));
 
   return *this;
 }
diff --git a/llvm/lib/Target/Hexagon/HexagonSubtarget.h b/llvm/lib/Target/Hexagon/HexagonSubtarget.h
index a4f2e159bf4b..e4f375440be1 100644
--- a/llvm/lib/Target/Hexagon/HexagonSubtarget.h
+++ b/llvm/lib/Target/Hexagon/HexagonSubtarget.h
@@ -56,6 +56,10 @@ class HexagonSubtarget : public HexagonGenSubtargetInfo {
   bool UseSmallData = false;
   bool UseUnsafeMath = false;
   bool UseZRegOps = false;
+  bool UseHVXIEEEFPOps = false;
+  bool UseHVXQFloatOps = false;
+  bool UseHVXFloatingPoint = false;
+  bool UseCabac = false;
 
   bool HasPreV65 = false;
   bool HasMemNoShuf = false;
@@ -138,6 +142,8 @@ public:
   /// subtarget options.  Definition of function is auto generated by tblgen.
   void ParseSubtargetFeatures(StringRef CPU, StringRef TuneCPU, StringRef FS);
 
+  bool isXRaySupported() const override { return true; }
+
   bool hasV5Ops() const {
     return getHexagonArchVersion() >= Hexagon::ArchEnum::V5;
   }
@@ -186,6 +192,12 @@ public:
   bool hasV68OpsOnly() const {
     return getHexagonArchVersion() == Hexagon::ArchEnum::V68;
   }
+  bool hasV69Ops() const {
+    return getHexagonArchVersion() >= Hexagon::ArchEnum::V69;
+  }
+  bool hasV69OpsOnly() const {
+    return getHexagonArchVersion() == Hexagon::ArchEnum::V69;
+  }
 
   bool useAudioOps() const { return UseAudioOps; }
   bool useCompound() const { return UseCompound; }
@@ -197,10 +209,16 @@ public:
   bool useSmallData() const { return UseSmallData; }
   bool useUnsafeMath() const { return UseUnsafeMath; }
   bool useZRegOps() const { return UseZRegOps; }
+  bool useCabac() const { return UseCabac; }
 
   bool isTinyCore() const { return HexagonProcFamily == TinyCore; }
   bool isTinyCoreWithDuplex() const { return isTinyCore() && EnableDuplex; }
 
+  bool useHVXIEEEFPOps() const { return UseHVXIEEEFPOps && useHVXOps(); }
+  bool useHVXQFloatOps() const {
+    return UseHVXQFloatOps && HexagonHVXVersion >= Hexagon::ArchEnum::V68;
+  }
+  bool useHVXFloatingPoint() const { return UseHVXFloatingPoint; }
   bool useHVXOps() const {
     return HexagonHVXVersion > Hexagon::ArchEnum::NoArch;
   }
@@ -222,6 +240,9 @@ public:
   bool useHVXV68Ops() const {
     return HexagonHVXVersion >= Hexagon::ArchEnum::V68;
   }
+  bool useHVXV69Ops() const {
+    return HexagonHVXVersion >= Hexagon::ArchEnum::V69;
+  }
   bool useHVX128BOps() const { return useHVXOps() && UseHVX128BOps; }
   bool useHVX64BOps() const { return useHVXOps() && UseHVX64BOps; }
 
@@ -281,7 +302,11 @@ public:
   }
 
   ArrayRef<MVT> getHVXElementTypes() const {
-    static MVT Types[] = { MVT::i8, MVT::i16, MVT::i32 };
+    static MVT Types[] = {MVT::i8, MVT::i16, MVT::i32};
+    static MVT TypesV68[] = {MVT::i8, MVT::i16, MVT::i32, MVT::f16, MVT::f32};
+
+    if (useHVXV68Ops() && useHVXFloatingPoint())
+      return makeArrayRef(TypesV68);
     return makeArrayRef(Types);
   }
 
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp b/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp
index 66de698182d7..fcf829b522cc 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp
@@ -21,6 +21,7 @@
 #include "TargetInfo/HexagonTargetInfo.h"
 #include "llvm/CodeGen/Passes.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/CodeGen/VLIWMachineScheduler.h"
 #include "llvm/IR/LegacyPassManager.h"
 #include "llvm/IR/Module.h"
 #include "llvm/MC/TargetRegistry.h"
@@ -120,8 +121,8 @@ extern "C" int HexagonTargetMachineModule;
 int HexagonTargetMachineModule = 0;
 
 static ScheduleDAGInstrs *createVLIWMachineSched(MachineSchedContext *C) {
-  ScheduleDAGMILive *DAG =
-    new VLIWMachineScheduler(C, std::make_unique<ConvergingVLIWScheduler>());
+  ScheduleDAGMILive *DAG = new VLIWMachineScheduler(
+      C, std::make_unique<HexagonConvergingVLIWScheduler>());
   DAG->addMutation(std::make_unique<HexagonSubtarget::UsrOverflowMutation>());
   DAG->addMutation(std::make_unique<HexagonSubtarget::HVXMemLatencyMutation>());
   DAG->addMutation(std::make_unique<HexagonSubtarget::CallMutation>());
diff --git a/llvm/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp b/llvm/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp
index 1d325553f45a..85ec0cdcd8f0 100644
--- a/llvm/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp
@@ -294,7 +294,7 @@ bool HexagonPacketizerList::tryAllocateResourcesForConstExt(bool Reserve) {
   bool Avail = ResourceTracker->canReserveResources(*ExtMI);
   if (Reserve && Avail)
     ResourceTracker->reserveResources(*ExtMI);
-  MF.DeleteMachineInstr(ExtMI);
+  MF.deleteMachineInstr(ExtMI);
   return Avail;
 }
 
@@ -890,7 +890,7 @@ bool HexagonPacketizerList::canPromoteToDotNew(const MachineInstr &MI,
   const MCInstrDesc &D = HII->get(NewOpcode);
   MachineInstr *NewMI = MF.CreateMachineInstr(D, DebugLoc());
   bool ResourcesAvailable = ResourceTracker->canReserveResources(*NewMI);
-  MF.DeleteMachineInstr(NewMI);
+  MF.deleteMachineInstr(NewMI);
   if (!ResourcesAvailable)
     return false;
 
@@ -1082,6 +1082,11 @@ bool HexagonPacketizerList::isSoloInstruction(const MachineInstr &MI) {
   if (HII->isSolo(MI))
     return true;
 
+  if (MI.getOpcode() == Hexagon::PATCHABLE_FUNCTION_ENTER ||
+      MI.getOpcode() == Hexagon::PATCHABLE_FUNCTION_EXIT ||
+      MI.getOpcode() == Hexagon::PATCHABLE_TAIL_CALL)
+    return true;
+
   if (MI.getOpcode() == Hexagon::A2_nop)
     return true;
 
diff --git a/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp b/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
index ea2798a3b44e..21386a91c7b3 100644
--- a/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
@@ -536,7 +536,7 @@ auto AlignVectors::createAddressGroups() -> bool {
   erase_if(AddrGroups, [](auto &G) { return G.second.size() == 1; });
   // Remove groups that don't use HVX types.
   erase_if(AddrGroups, [&](auto &G) {
-    return !llvm::any_of(
+    return llvm::none_of(
         G.second, [&](auto &I) { return HVC.HST.isTypeForHVX(I.ValTy); });
   });
 
diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonBaseInfo.h b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonBaseInfo.h
index 4125566bc58a..c9a1781a4543 100644
--- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonBaseInfo.h
+++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonBaseInfo.h
@@ -154,9 +154,8 @@ namespace HexagonII {
     PrefersSlot3Pos = 57,
     PrefersSlot3Mask = 0x1,
 
-    // v65
-    HasTmpDstPos = 60,
-    HasTmpDstMask = 0x1,
+    HasHvxTmpPos = 60,
+    HasHvxTmpMask = 0x1,
 
     CVINewPos = 62,
     CVINewMask = 0x1,
diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCChecker.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCChecker.cpp
index fee1acdbbe8a..96c2965296ca 100644
--- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCChecker.cpp
+++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCChecker.cpp
@@ -98,6 +98,10 @@ void HexagonMCChecker::init(MCInst const &MCI) {
   for (unsigned i = 0; i < MCID.getNumImplicitUses(); ++i)
     initReg(MCI, MCID.getImplicitUses()[i], PredReg, isTrue);
 
+  const bool IgnoreTmpDst = (HexagonMCInstrInfo::hasTmpDst(MCII, MCI) ||
+                             HexagonMCInstrInfo::hasHvxTmp(MCII, MCI)) &&
+                            STI.getFeatureBits()[Hexagon::ArchV69];
+
   // Get implicit register definitions.
   if (const MCPhysReg *ImpDef = MCID.getImplicitDefs())
     for (; *ImpDef; ++ImpDef) {
@@ -123,7 +127,7 @@ void HexagonMCChecker::init(MCInst const &MCI) {
                HexagonMCInstrInfo::isPredicateLate(MCII, MCI))
         // Include implicit late predicates.
         LatePreds.insert(R);
-      else
+      else if (!IgnoreTmpDst)
         Defs[R].insert(PredSense(PredReg, isTrue));
     }
 
@@ -178,7 +182,7 @@ void HexagonMCChecker::init(MCInst const &MCI) {
         // vshuff(Vx, Vy, Rx) <- Vx(0) and Vy(1) are both source and
         // destination registers with this instruction. same for vdeal(Vx,Vy,Rx)
         Uses.insert(*SRI);
-      else
+      else if (!IgnoreTmpDst)
         Defs[*SRI].insert(PredSense(PredReg, isTrue));
     }
   }
@@ -227,9 +231,11 @@ bool HexagonMCChecker::check(bool FullCheck) {
   bool chkAXOK = checkAXOK();
   bool chkCofMax1 = checkCOFMax1();
   bool chkHWLoop = checkHWLoop();
+  bool chkValidTmpDst = FullCheck ? checkValidTmpDst() : true;
   bool chkLegalVecRegPair = checkLegalVecRegPair();
   bool chk = chkP && chkNV && chkR && chkRRO && chkS && chkSh && chkSl &&
-             chkAXOK && chkCofMax1 && chkHWLoop && chkLegalVecRegPair;
+             chkAXOK && chkCofMax1 && chkHWLoop && chkValidTmpDst &&
+             chkLegalVecRegPair;
 
   return chk;
 }
@@ -676,6 +682,32 @@ bool HexagonMCChecker::checkShuffle() {
   return MCSDX.check();
 }
 
+bool HexagonMCChecker::checkValidTmpDst() {
+  if (!STI.getFeatureBits()[Hexagon::ArchV69]) {
+    return true;
+  }
+  auto HasTmp = [&](MCInst const &I) {
+    return HexagonMCInstrInfo::hasTmpDst(MCII, I) ||
+           HexagonMCInstrInfo::hasHvxTmp(MCII, I);
+  };
+  unsigned HasTmpCount =
+      llvm::count_if(HexagonMCInstrInfo::bundleInstructions(MCII, MCB), HasTmp);
+
+  if (HasTmpCount > 1) {
+    reportError(
+        MCB.getLoc(),
+        "this packet has more than one HVX vtmp/.tmp destination instruction");
+
+    for (auto const &I : HexagonMCInstrInfo::bundleInstructions(MCII, MCB))
+      if (HasTmp(I))
+        reportNote(I.getLoc(),
+                   "this is an HVX vtmp/.tmp destination instruction");
+
+    return false;
+  }
+  return true;
+}
+
 void HexagonMCChecker::compoundRegisterMap(unsigned &Register) {
   switch (Register) {
   default:
diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCChecker.h b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCChecker.h
index 00afdb664ba5..dbd3d8ae45e6 100644
--- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCChecker.h
+++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCChecker.h
@@ -99,6 +99,7 @@ class HexagonMCChecker {
   bool checkHWLoop();
   bool checkCOFMax1();
   bool checkLegalVecRegPair();
+  bool checkValidTmpDst();
 
   static void compoundRegisterMap(unsigned &);
 
diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.cpp
index fa12fe1da448..68ccb20f4f15 100644
--- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.cpp
+++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.cpp
@@ -939,10 +939,24 @@ bool HexagonMCInstrInfo::prefersSlot3(MCInstrInfo const &MCII,
   return (F >> HexagonII::PrefersSlot3Pos) & HexagonII::PrefersSlot3Mask;
 }
 
-/// return true if instruction has hasTmpDst attribute.
 bool HexagonMCInstrInfo::hasTmpDst(MCInstrInfo const &MCII, MCInst const &MCI) {
+  switch (MCI.getOpcode()) {
+  default:
+    return false;
+  case Hexagon::V6_vgathermh:
+  case Hexagon::V6_vgathermhq:
+  case Hexagon::V6_vgathermhw:
+  case Hexagon::V6_vgathermhwq:
+  case Hexagon::V6_vgathermw:
+  case Hexagon::V6_vgathermwq:
+    return true;
+  }
+  return false;
+}
+
+bool HexagonMCInstrInfo::hasHvxTmp(MCInstrInfo const &MCII, MCInst const &MCI) {
   const uint64_t F = HexagonMCInstrInfo::getDesc(MCII, MCI).TSFlags;
-  return (F >> HexagonII::HasTmpDstPos) & HexagonII::HasTmpDstMask;
+  return (F >> HexagonII::HasHvxTmpPos) & HexagonII::HasHvxTmpMask;
 }
 
 bool HexagonMCInstrInfo::requiresSlot(MCSubtargetInfo const &STI,
diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.h b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.h
index 7b3c079880f8..5c56db14798f 100644
--- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.h
+++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.h
@@ -41,7 +41,8 @@ public:
 
 namespace Hexagon {
 
-class PacketIterator {
+class PacketIterator : public std::iterator<std::forward_iterator_tag,
+    PacketIterator> {
   MCInstrInfo const &MCII;
   MCInst::const_iterator BundleCurrent;
   MCInst::const_iterator BundleEnd;
@@ -188,6 +189,7 @@ bool hasImmExt(MCInst const &MCI);
 bool hasNewValue(MCInstrInfo const &MCII, MCInst const &MCI);
 bool hasNewValue2(MCInstrInfo const &MCII, MCInst const &MCI);
 bool hasTmpDst(MCInstrInfo const &MCII, MCInst const &MCI);
+bool hasHvxTmp(MCInstrInfo const &MCII, MCInst const &MCI);
 unsigned iClassOfDuplexPair(unsigned Ga, unsigned Gb);
 
 int64_t minConstant(MCInst const &MCI, size_t Index);
diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp
index d832a756cb92..dfdddb50657c 100644
--- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp
+++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp
@@ -80,6 +80,8 @@ cl::opt<bool> MV67T("mv67t", cl::Hidden, cl::desc("Build for Hexagon V67T"),
                     cl::init(false));
 cl::opt<bool> MV68("mv68", cl::Hidden, cl::desc("Build for Hexagon V68"),
                    cl::init(false));
+cl::opt<bool> MV69("mv69", cl::Hidden, cl::desc("Build for Hexagon V69"),
+                   cl::init(false));
 
 cl::opt<Hexagon::ArchEnum>
     EnableHVX("mhvx",
@@ -91,6 +93,7 @@ cl::opt<Hexagon::ArchEnum>
         clEnumValN(Hexagon::ArchEnum::V66, "v66", "Build for HVX v66"),
         clEnumValN(Hexagon::ArchEnum::V67, "v67", "Build for HVX v67"),
         clEnumValN(Hexagon::ArchEnum::V68, "v68", "Build for HVX v68"),
+        clEnumValN(Hexagon::ArchEnum::V69, "v69", "Build for HVX v69"),
         // Sentinel for no value specified.
         clEnumValN(Hexagon::ArchEnum::Generic, "", "")),
       // Sentinel for flag not present.
@@ -101,6 +104,11 @@ static cl::opt<bool>
   DisableHVX("mno-hvx", cl::Hidden,
              cl::desc("Disable Hexagon Vector eXtensions"));
 
+static cl::opt<bool>
+    EnableHvxIeeeFp("mhvx-ieee-fp", cl::Hidden,
+                    cl::desc("Enable HVX IEEE floating point extensions"));
+static cl::opt<bool> EnableHexagonCabac
+  ("mcabac", cl::desc("tbd"), cl::init(false));
 
 static StringRef DefaultArch = "hexagonv60";
 
@@ -123,6 +131,8 @@ static StringRef HexagonGetArchVariant() {
     return "hexagonv67t";
   if (MV68)
     return "hexagonv68";
+  if (MV69)
+    return "hexagonv69";
   return "";
 }
 
@@ -371,6 +381,9 @@ std::string selectHexagonFS(StringRef CPU, StringRef FS) {
   case Hexagon::ArchEnum::V68:
     Result.push_back("+hvxv68");
     break;
+  case Hexagon::ArchEnum::V69:
+    Result.push_back("+hvxv69");
+    break;
   case Hexagon::ArchEnum::Generic:{
     Result.push_back(StringSwitch<StringRef>(CPU)
              .Case("hexagonv60", "+hvxv60")
@@ -379,13 +392,19 @@ std::string selectHexagonFS(StringRef CPU, StringRef FS) {
              .Case("hexagonv66", "+hvxv66")
              .Case("hexagonv67", "+hvxv67")
              .Case("hexagonv67t", "+hvxv67")
-             .Case("hexagonv68", "+hvxv68"));
+             .Case("hexagonv68", "+hvxv68")
+             .Case("hexagonv69", "+hvxv69"));
     break;
   }
   case Hexagon::ArchEnum::NoArch:
     // Sentinel if -mhvx isn't specified
     break;
   }
+  if (EnableHvxIeeeFp)
+    Result.push_back("+hvx-ieee-fp");
+  if (EnableHexagonCabac)
+    Result.push_back("+cabac");
+
   return join(Result.begin(), Result.end(), ",");
 }
 }
@@ -422,8 +441,8 @@ FeatureBitset Hexagon_MC::completeHVXFeatures(const FeatureBitset &S) {
   // turns on hvxvNN, corresponding to the existing ArchVNN.
   FeatureBitset FB = S;
   unsigned CpuArch = ArchV5;
-  for (unsigned F : {ArchV68, ArchV67, ArchV66, ArchV65, ArchV62, ArchV60,
-                     ArchV55, ArchV5}) {
+  for (unsigned F : {ArchV69, ArchV68, ArchV67, ArchV66, ArchV65, ArchV62,
+                     ArchV60, ArchV55, ArchV5}) {
     if (!FB.test(F))
       continue;
     CpuArch = F;
@@ -438,7 +457,8 @@ FeatureBitset Hexagon_MC::completeHVXFeatures(const FeatureBitset &S) {
   }
   bool HasHvxVer = false;
   for (unsigned F : {ExtensionHVXV60, ExtensionHVXV62, ExtensionHVXV65,
-                     ExtensionHVXV66, ExtensionHVXV67, ExtensionHVXV68}) {
+                     ExtensionHVXV66, ExtensionHVXV67, ExtensionHVXV68,
+                     ExtensionHVXV69}) {
     if (!FB.test(F))
       continue;
     HasHvxVer = true;
@@ -451,6 +471,9 @@ FeatureBitset Hexagon_MC::completeHVXFeatures(const FeatureBitset &S) {
 
   // HasHvxVer is false, and UseHvx is true.
   switch (CpuArch) {
+  case ArchV69:
+    FB.set(ExtensionHVXV69);
+    LLVM_FALLTHROUGH;
     case ArchV68:
       FB.set(ExtensionHVXV68);
       LLVM_FALLTHROUGH;
@@ -538,6 +561,7 @@ unsigned Hexagon_MC::GetELFFlags(const MCSubtargetInfo &STI) {
     {"hexagonv67", ELF::EF_HEXAGON_MACH_V67},
     {"hexagonv67t", ELF::EF_HEXAGON_MACH_V67T},
     {"hexagonv68", ELF::EF_HEXAGON_MACH_V68},
+    {"hexagonv69", ELF::EF_HEXAGON_MACH_V69},
   };
 
   auto F = ElfFlags.find(STI.getCPU());
diff --git a/llvm/lib/Target/M68k/M68kInstrControl.td b/llvm/lib/Target/M68k/M68kInstrControl.td
index 708474726861..9f87833ab0e2 100644
--- a/llvm/lib/Target/M68k/M68kInstrControl.td
+++ b/llvm/lib/Target/M68k/M68kInstrControl.td
@@ -118,13 +118,13 @@ def SET#"p8"#cc : MxSccM<cc, MxType8.POp, MxType8.PPat, MxEncEAp_0, MxExtI16_0>;
 /// 0  1  0  0  1  1  1  0  1  1 |  MODE   |   REG
 ///------------------------------+---------+---------
 let isBranch = 1, isTerminator = 1, isBarrier = 1, isIndirectBranch = 1 in
-class MxJMP<MxOperand LOCOp, ComplexPattern LOCPat, MxEncEA EA, MxEncExt EXT>
+class MxJMP<MxOperand LOCOp, MxEncEA EA, MxEncExt EXT>
     : MxInst<(outs), (ins LOCOp:$dst), "jmp\t$dst", [(brind iPTR:$dst)],
              MxEncoding<EA.Reg, EA.DA, EA.Mode, MxBead2Bits<0b11>,
                         MxBead4Bits<0b1110>, MxBead4Bits<0b0100>,
                         EXT.Imm, EXT.B8, EXT.Scale, EXT.WL, EXT.DAReg>>;
 
-def JMP32j : MxJMP<MxARI32, MxCP_ARI, MxEncEAj_0, MxExtEmpty>;
+def JMP32j : MxJMP<MxARI32, MxEncEAj_0, MxExtEmpty>;
 
 
 // FIXME Support 16 bit indirect jump.
@@ -147,17 +147,17 @@ def JMP32j : MxJMP<MxARI32, MxCP_ARI, MxEncEAj_0, MxExtEmpty>;
 ///  32-BIT DISPLACEMENT IF 8-BIT DISPLACEMENT = $FF
 /// --------------------------------------------------
 let isBranch = 1, isTerminator = 1, Uses = [CCR] in
-class MxBcc<string cc, Operand TARGET, MxType TYPE, MxEncoding ENC = MxEncEmpty>
+class MxBcc<string cc, Operand TARGET, MxEncoding ENC = MxEncEmpty>
     : MxInst<(outs), (ins TARGET:$dst), "b"#cc#"\t$dst", [], ENC>;
 
 foreach cc = [ "cc", "ls", "lt", "eq", "mi", "ne", "ge",
                "cs", "pl", "gt", "hi", "vc", "le", "vs"] in {
   def B#cc#"8"
-    : MxBcc<cc, MxBrTarget8, MxType8,
+    : MxBcc<cc, MxBrTarget8,
             MxEncoding<MxBead8Disp<0>,
                        !cast<MxBead4Bits>("MxCC"#cc), MxBead4Bits<0x6>>>;
   def B#cc#"16"
-    : MxBcc<cc, MxBrTarget16, MxType16,
+    : MxBcc<cc, MxBrTarget16,
             MxEncoding<MxBead4Bits<0x0>,
                        MxBead4Bits<0x0>, !cast<MxBead4Bits>("MxCC"#cc),
                        MxBead4Bits<0x6>, MxBead16Imm<0>>>;
@@ -179,13 +179,13 @@ def : Pat<(MxBrCond bb:$target, !cast<PatLeaf>("MxCOND"#cc), CCR),
 ///  32-BIT DISPLACEMENT IF 8-BIT DISPLACEMENT = $FF
 /// -------------------------------------------------
 let isBranch = 1, isTerminator = 1, isBarrier=1 in
-class MxBra<Operand TARGET, MxType TYPE, MxEncoding ENC = MxEncEmpty>
+class MxBra<Operand TARGET, MxEncoding ENC = MxEncEmpty>
     : MxInst<(outs), (ins TARGET:$dst), "bra\t$dst", [], ENC>;
 
-def BRA8  : MxBra<MxBrTarget8, MxType8,
+def BRA8  : MxBra<MxBrTarget8,
                   MxEncoding<MxBead8Disp<0>, MxBead4Bits<0x0>,
                              MxBead4Bits<0x6>>>;
-def BRA16 : MxBra<MxBrTarget16, MxType16,
+def BRA16 : MxBra<MxBrTarget16,
                   MxEncoding<MxBead4Bits<0x0>, MxBead4Bits<0x0>,
                              MxBead4Bits<0x0>, MxBead4Bits<0x6>,
                              MxBead16Imm<0>>>;
diff --git a/llvm/lib/Target/MSP430/MSP430FrameLowering.cpp b/llvm/lib/Target/MSP430/MSP430FrameLowering.cpp
index 2a77a150f9aa..4ef9a567d453 100644
--- a/llvm/lib/Target/MSP430/MSP430FrameLowering.cpp
+++ b/llvm/lib/Target/MSP430/MSP430FrameLowering.cpp
@@ -189,8 +189,8 @@ bool MSP430FrameLowering::spillCalleeSavedRegisters(
   MSP430MachineFunctionInfo *MFI = MF.getInfo<MSP430MachineFunctionInfo>();
   MFI->setCalleeSavedFrameSize(CSI.size() * 2);
 
-  for (unsigned i = CSI.size(); i != 0; --i) {
-    unsigned Reg = CSI[i-1].getReg();
+  for (const CalleeSavedInfo &I : llvm::reverse(CSI)) {
+    unsigned Reg = I.getReg();
     // Add the callee-saved register as live-in. It's killed at the spill.
     MBB.addLiveIn(Reg);
     BuildMI(MBB, MI, DL, TII.get(MSP430::PUSH16r))
diff --git a/llvm/lib/Target/Mips/Mips16HardFloat.cpp b/llvm/lib/Target/Mips/Mips16HardFloat.cpp
index 203e05dde7ad..419f0ac1a8a7 100644
--- a/llvm/lib/Target/Mips/Mips16HardFloat.cpp
+++ b/llvm/lib/Target/Mips/Mips16HardFloat.cpp
@@ -479,14 +479,12 @@ static void createFPFnStub(Function *F, Module *M, FPParamVariant PV,
 
 // remove the use-soft-float attribute
 static void removeUseSoftFloat(Function &F) {
-  AttrBuilder B;
   LLVM_DEBUG(errs() << "removing -use-soft-float\n");
-  B.addAttribute("use-soft-float", "false");
-  F.removeFnAttrs(B);
+  F.removeFnAttr("use-soft-float");
   if (F.hasFnAttribute("use-soft-float")) {
     LLVM_DEBUG(errs() << "still has -use-soft-float\n");
   }
-  F.addFnAttrs(B);
+  F.addFnAttr("use-soft-float", "false");
 }
 
 // This pass only makes sense when the underlying chip has floating point but
diff --git a/llvm/lib/Target/Mips/MipsBranchExpansion.cpp b/llvm/lib/Target/Mips/MipsBranchExpansion.cpp
index aa8e298fa759..4e9a23d077da 100644
--- a/llvm/lib/Target/Mips/MipsBranchExpansion.cpp
+++ b/llvm/lib/Target/Mips/MipsBranchExpansion.cpp
@@ -36,7 +36,7 @@
 ///
 /// Regarding compact branch hazard prevention:
 ///
-/// Hazards handled: forbidden slots for MIPSR6.
+/// Hazards handled: forbidden slots for MIPSR6, FPU slots for MIPS3 and below.
 ///
 /// A forbidden slot hazard occurs when a compact branch instruction is executed
 /// and the adjacent instruction in memory is a control transfer instruction
@@ -160,7 +160,10 @@ private:
   bool buildProperJumpMI(MachineBasicBlock *MBB,
                          MachineBasicBlock::iterator Pos, DebugLoc DL);
   void expandToLongBranch(MBBInfo &Info);
+  template <typename Pred, typename Safe>
+  bool handleSlot(Pred Predicate, Safe SafeInSlot);
   bool handleForbiddenSlot();
+  bool handleFPUDelaySlot();
   bool handlePossibleLongBranch();
 
   const MipsSubtarget *STI;
@@ -738,30 +741,27 @@ static void emitGPDisp(MachineFunction &F, const MipsInstrInfo *TII) {
   MBB.removeLiveIn(Mips::V0);
 }
 
-bool MipsBranchExpansion::handleForbiddenSlot() {
-  // Forbidden slot hazards are only defined for MIPSR6 but not microMIPSR6.
-  if (!STI->hasMips32r6() || STI->inMicroMipsMode())
-    return false;
-
+template <typename Pred, typename Safe>
+bool MipsBranchExpansion::handleSlot(Pred Predicate, Safe SafeInSlot) {
   bool Changed = false;
 
   for (MachineFunction::iterator FI = MFp->begin(); FI != MFp->end(); ++FI) {
     for (Iter I = FI->begin(); I != FI->end(); ++I) {
 
-      // Forbidden slot hazard handling. Use lookahead over state.
-      if (!TII->HasForbiddenSlot(*I))
+      // Delay slot hazard handling. Use lookahead over state.
+      if (!Predicate(*I))
         continue;
 
-      Iter Inst;
+      Iter IInSlot;
       bool LastInstInFunction =
           std::next(I) == FI->end() && std::next(FI) == MFp->end();
       if (!LastInstInFunction) {
         std::pair<Iter, bool> Res = getNextMachineInstr(std::next(I), &*FI);
         LastInstInFunction |= Res.second;
-        Inst = Res.first;
+        IInSlot = Res.first;
       }
 
-      if (LastInstInFunction || !TII->SafeInForbiddenSlot(*Inst)) {
+      if (LastInstInFunction || !SafeInSlot(*IInSlot, *I)) {
 
         MachineBasicBlock::instr_iterator Iit = I->getIterator();
         if (std::next(Iit) == FI->end() ||
@@ -778,6 +778,29 @@ bool MipsBranchExpansion::handleForbiddenSlot() {
   return Changed;
 }
 
+bool MipsBranchExpansion::handleForbiddenSlot() {
+  // Forbidden slot hazards are only defined for MIPSR6 but not microMIPSR6.
+  if (!STI->hasMips32r6() || STI->inMicroMipsMode())
+    return false;
+
+  return handleSlot(
+      [this](auto &I) -> bool { return TII->HasForbiddenSlot(I); },
+      [this](auto &IInSlot, auto &I) -> bool {
+        return TII->SafeInForbiddenSlot(IInSlot);
+      });
+}
+
+bool MipsBranchExpansion::handleFPUDelaySlot() {
+  // FPU delay slots are only defined for MIPS3 and below.
+  if (STI->hasMips32() || STI->hasMips4())
+    return false;
+
+  return handleSlot([this](auto &I) -> bool { return TII->HasFPUDelaySlot(I); },
+                    [this](auto &IInSlot, auto &I) -> bool {
+                      return TII->SafeInFPUDelaySlot(IInSlot, I);
+                    });
+}
+
 bool MipsBranchExpansion::handlePossibleLongBranch() {
   if (STI->inMips16Mode() || !STI->enableLongBranchPass())
     return false;
@@ -857,13 +880,16 @@ bool MipsBranchExpansion::runOnMachineFunction(MachineFunction &MF) {
   // Run these two at least once
   bool longBranchChanged = handlePossibleLongBranch();
   bool forbiddenSlotChanged = handleForbiddenSlot();
+  bool fpuDelaySlotChanged = handleFPUDelaySlot();
 
-  bool Changed = longBranchChanged || forbiddenSlotChanged;
+  bool Changed =
+      longBranchChanged || forbiddenSlotChanged || fpuDelaySlotChanged;
 
   // Then run them alternatively while there are changes
   while (forbiddenSlotChanged) {
     longBranchChanged = handlePossibleLongBranch();
-    if (!longBranchChanged)
+    fpuDelaySlotChanged = handleFPUDelaySlot();
+    if (!longBranchChanged && !fpuDelaySlotChanged)
       break;
     forbiddenSlotChanged = handleForbiddenSlot();
   }
diff --git a/llvm/lib/Target/Mips/MipsISelLowering.cpp b/llvm/lib/Target/Mips/MipsISelLowering.cpp
index 4f364ef6afc7..9377e83524e1 100644
--- a/llvm/lib/Target/Mips/MipsISelLowering.cpp
+++ b/llvm/lib/Target/Mips/MipsISelLowering.cpp
@@ -4121,7 +4121,7 @@ MipsTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
     case 'd': // Address register. Same as 'r' unless generating MIPS16 code.
     case 'y': // Same as 'r'. Exists for compatibility.
     case 'r':
-      if (VT == MVT::i32 || VT == MVT::i16 || VT == MVT::i8) {
+      if (VT == MVT::i32 || VT == MVT::i16 || VT == MVT::i8 || VT == MVT::i1) {
         if (Subtarget.inMips16Mode())
           return std::make_pair(0U, &Mips::CPU16RegsRegClass);
         return std::make_pair(0U, &Mips::GPR32RegClass);
diff --git a/llvm/lib/Target/Mips/MipsInstrInfo.cpp b/llvm/lib/Target/Mips/MipsInstrInfo.cpp
index 94828a976695..2bf8562895d7 100644
--- a/llvm/lib/Target/Mips/MipsInstrInfo.cpp
+++ b/llvm/lib/Target/Mips/MipsInstrInfo.cpp
@@ -568,11 +568,60 @@ bool MipsInstrInfo::SafeInForbiddenSlot(const MachineInstr &MI) const {
   return (MI.getDesc().TSFlags & MipsII::IsCTI) == 0;
 }
 
+bool MipsInstrInfo::SafeInFPUDelaySlot(const MachineInstr &MIInSlot,
+                                       const MachineInstr &FPUMI) const {
+  if (MIInSlot.isInlineAsm())
+    return false;
+
+  if (HasFPUDelaySlot(MIInSlot))
+    return false;
+
+  switch (MIInSlot.getOpcode()) {
+  case Mips::BC1F:
+  case Mips::BC1FL:
+  case Mips::BC1T:
+  case Mips::BC1TL:
+    return false;
+  }
+
+  for (const MachineOperand &Op : FPUMI.defs()) {
+    if (!Op.isReg())
+      continue;
+
+    bool Reads, Writes;
+    std::tie(Reads, Writes) = MIInSlot.readsWritesVirtualRegister(Op.getReg());
+
+    if (Reads || Writes)
+      return false;
+  }
+
+  return true;
+}
+
 /// Predicate for distingushing instructions that have forbidden slots.
 bool MipsInstrInfo::HasForbiddenSlot(const MachineInstr &MI) const {
   return (MI.getDesc().TSFlags & MipsII::HasForbiddenSlot) != 0;
 }
 
+/// Predicate for distingushing instructions that have FPU delay slots.
+bool MipsInstrInfo::HasFPUDelaySlot(const MachineInstr &MI) const {
+  switch (MI.getOpcode()) {
+  case Mips::MTC1:
+  case Mips::MFC1:
+  case Mips::MTC1_D64:
+  case Mips::MFC1_D64:
+  case Mips::DMTC1:
+  case Mips::DMFC1:
+  case Mips::FCMP_S32:
+  case Mips::FCMP_D32:
+  case Mips::FCMP_D64:
+    return true;
+
+  default:
+    return false;
+  }
+}
+
 /// Return the number of bytes of code the specified instruction may be.
 unsigned MipsInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
   switch (MI.getOpcode()) {
diff --git a/llvm/lib/Target/Mips/MipsInstrInfo.h b/llvm/lib/Target/Mips/MipsInstrInfo.h
index c96ed202df30..46c1b73d512f 100644
--- a/llvm/lib/Target/Mips/MipsInstrInfo.h
+++ b/llvm/lib/Target/Mips/MipsInstrInfo.h
@@ -92,9 +92,16 @@ public:
   /// Predicate to determine if an instruction can go in a forbidden slot.
   bool SafeInForbiddenSlot(const MachineInstr &MI) const;
 
+  /// Predicate to determine if an instruction can go in an FPU delay slot.
+  bool SafeInFPUDelaySlot(const MachineInstr &MIInSlot,
+                          const MachineInstr &FPUMI) const;
+
   /// Predicate to determine if an instruction has a forbidden slot.
   bool HasForbiddenSlot(const MachineInstr &MI) const;
 
+  /// Predicate to determine if an instruction has an FPU delay slot.
+  bool HasFPUDelaySlot(const MachineInstr &MI) const;
+
   /// Insert nop instruction when hazard condition is found
   void insertNoop(MachineBasicBlock &MBB,
                   MachineBasicBlock::iterator MI) const override;
diff --git a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
index c35e67d6726f..16add48d4602 100644
--- a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
@@ -1098,10 +1098,10 @@ void NVPTXAsmPrinter::printModuleLevelGV(const GlobalVariable *GVar,
     O << " .attribute(.managed)";
   }
 
-  if (GVar->getAlignment() == 0)
-    O << " .align " << (int)DL.getPrefTypeAlignment(ETy);
+  if (MaybeAlign A = GVar->getAlign())
+    O << " .align " << A->value();
   else
-    O << " .align " << GVar->getAlignment();
+    O << " .align " << (int)DL.getPrefTypeAlignment(ETy);
 
   if (ETy->isFloatingPointTy() || ETy->isPointerTy() ||
       (ETy->isIntegerTy() && ETy->getScalarSizeInBits() <= 64)) {
@@ -1290,10 +1290,10 @@ void NVPTXAsmPrinter::emitPTXGlobalVariable(const GlobalVariable *GVar,
 
   O << ".";
   emitPTXAddressSpace(GVar->getType()->getAddressSpace(), O);
-  if (GVar->getAlignment() == 0)
-    O << " .align " << (int)DL.getPrefTypeAlignment(ETy);
+  if (MaybeAlign A = GVar->getAlign())
+    O << " .align " << A->value();
   else
-    O << " .align " << GVar->getAlignment();
+    O << " .align " << (int)DL.getPrefTypeAlignment(ETy);
 
   // Special case for i128
   if (ETy->isIntegerTy(128)) {
diff --git a/llvm/lib/Target/NVPTX/NVPTXPeephole.cpp b/llvm/lib/Target/NVPTX/NVPTXPeephole.cpp
index 1f3b4c9440d8..bf3c87df2e08 100644
--- a/llvm/lib/Target/NVPTX/NVPTXPeephole.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXPeephole.cpp
@@ -126,9 +126,9 @@ static void CombineCVTAToLocal(MachineInstr &Root) {
 
   // Check if MRI has only one non dbg use, which is Root
   if (MRI.hasOneNonDBGUse(Prev.getOperand(0).getReg())) {
-    Prev.eraseFromParentAndMarkDBGValuesForRemoval();
+    Prev.eraseFromParent();
   }
-  Root.eraseFromParentAndMarkDBGValuesForRemoval();
+  Root.eraseFromParent();
 }
 
 bool NVPTXPeephole::runOnMachineFunction(MachineFunction &MF) {
@@ -157,7 +157,7 @@ bool NVPTXPeephole::runOnMachineFunction(MachineFunction &MF) {
   const auto &MRI = MF.getRegInfo();
   if (MRI.use_empty(NRI->getFrameRegister(MF))) {
     if (auto MI = MRI.getUniqueVRegDef(NRI->getFrameRegister(MF))) {
-      MI->eraseFromParentAndMarkDBGValuesForRemoval();
+      MI->eraseFromParent();
     }
   }
 
diff --git a/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp b/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
index 9e181d4052d6..ded922329ebf 100644
--- a/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
+++ b/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
@@ -1576,6 +1576,16 @@ bool PPCAsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
     std::swap(Operands[2], Operands[1]);
   }
 
+  // Handle base mnemonic for atomic loads where the EH bit is zero.
+  if (Name == "lqarx" || Name == "ldarx" || Name == "lwarx" ||
+      Name == "lharx" || Name == "lbarx") {
+    if (Operands.size() != 5)
+      return false;
+    PPCOperand &EHOp = (PPCOperand &)*Operands[4];
+    if (EHOp.isU1Imm() && EHOp.getImm() == 0)
+      Operands.pop_back();
+  }
+
   return false;
 }
 
@@ -1745,7 +1755,7 @@ unsigned PPCAsmParser::validateTargetOperandClass(MCParsedAsmOperand &AsmOp,
   }
 
   PPCOperand &Op = static_cast<PPCOperand &>(AsmOp);
-  if (Op.isImm() && Op.getImm() == ImmVal)
+  if (Op.isU3Imm() && Op.getImm() == ImmVal)
     return Match_Success;
 
   return Match_InvalidOperand;
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp
index 22b948a83c34..d6e02d0d0862 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp
@@ -28,6 +28,7 @@
 #include "llvm/MC/MCDwarf.h"
 #include "llvm/MC/MCELFStreamer.h"
 #include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCInstrAnalysis.h"
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCRegisterInfo.h"
@@ -368,6 +369,31 @@ static MCInstPrinter *createPPCMCInstPrinter(const Triple &T,
   return new PPCInstPrinter(MAI, MII, MRI, T);
 }
 
+namespace {
+
+class PPCMCInstrAnalysis : public MCInstrAnalysis {
+public:
+  explicit PPCMCInstrAnalysis(const MCInstrInfo *Info)
+      : MCInstrAnalysis(Info) {}
+
+  bool evaluateBranch(const MCInst &Inst, uint64_t Addr, uint64_t Size,
+                      uint64_t &Target) const override {
+    unsigned NumOps = Inst.getNumOperands();
+    if (NumOps == 0 ||
+        Info->get(Inst.getOpcode()).OpInfo[NumOps - 1].OperandType !=
+            MCOI::OPERAND_PCREL)
+      return false;
+    Target = Addr + Inst.getOperand(NumOps - 1).getImm() * Size;
+    return true;
+  }
+};
+
+} // end anonymous namespace
+
+static MCInstrAnalysis *createPPCMCInstrAnalysis(const MCInstrInfo *Info) {
+  return new PPCMCInstrAnalysis(Info);
+}
+
 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializePowerPCTargetMC() {
   for (Target *T : {&getThePPC32Target(), &getThePPC32LETarget(),
                     &getThePPC64Target(), &getThePPC64LETarget()}) {
@@ -383,6 +409,9 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializePowerPCTargetMC() {
     // Register the MC subtarget info.
     TargetRegistry::RegisterMCSubtargetInfo(*T, createPPCMCSubtargetInfo);
 
+    // Register the MC instruction analyzer.
+    TargetRegistry::RegisterMCInstrAnalysis(*T, createPPCMCInstrAnalysis);
+
     // Register the MC Code Emitter
     TargetRegistry::RegisterMCCodeEmitter(*T, createPPCMCCodeEmitter);
 
diff --git a/llvm/lib/Target/PowerPC/PPC.td b/llvm/lib/Target/PowerPC/PPC.td
index 422bd11dca52..bbd5f5fd1941 100644
--- a/llvm/lib/Target/PowerPC/PPC.td
+++ b/llvm/lib/Target/PowerPC/PPC.td
@@ -219,6 +219,10 @@ def FeatureZeroMoveFusion:
   SubtargetFeature<"fuse-zeromove", "HasZeroMoveFusion", "true",
                    "Target supports move to SPR with branch fusion",
                    [FeatureFusion]>;
+def FeatureBack2BackFusion:
+  SubtargetFeature<"fuse-back2back", "HasBack2BackFusion", "true",
+                   "Target supports general back to back fusion",
+                   [FeatureFusion]>;
 def FeatureUnalignedFloats :
   SubtargetFeature<"allow-unaligned-fp-access", "AllowsUnalignedFPAccess",
                    "true", "CPU does not trap on unaligned FP access">;
diff --git a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
index 16e3b2b85c2e..f26c15667a0b 100644
--- a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
+++ b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
@@ -347,7 +347,6 @@ bool PPCAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
 // At the moment, all inline asm memory operands are a single register.
 // In any case, the output of this routine should always be just one
 // assembler operand.
-
 bool PPCAsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNo,
                                           const char *ExtraCode,
                                           raw_ostream &O) {
diff --git a/llvm/lib/Target/PowerPC/PPCBack2BackFusion.def b/llvm/lib/Target/PowerPC/PPCBack2BackFusion.def
new file mode 100644
index 000000000000..38ed5f2e78e3
--- /dev/null
+++ b/llvm/lib/Target/PowerPC/PPCBack2BackFusion.def
@@ -0,0 +1,1042 @@
+// Automatically generated file, do not edit!
+//
+// This file defines instruction list for general back2back fusion.
+//===----------------------------------------------------------------------===//
+FUSION_FEATURE(GeneralBack2Back, hasBack2BackFusion, -1,
+  FUSION_OP_SET(ADD4,
+    ADD4O,
+    ADD4TLS,
+    ADD4_rec,
+    ADD8,
+    ADD8O,
+    ADD8TLS,
+    ADD8TLS_,
+    ADD8_rec,
+    ADDE,
+    ADDE8,
+    ADDE8O,
+    ADDEO,
+    ADDEX,
+    ADDEX8,
+    ADDI,
+    ADDI8,
+    ADDIC,
+    ADDIC8,
+    ADDIS,
+    ADDIS8,
+    ADDISdtprelHA32,
+    ADDIStocHA,
+    ADDIStocHA8,
+    ADDIdtprelL32,
+    ADDItlsldLADDR32,
+    ADDItocL,
+    ADDME,
+    ADDME8,
+    ADDME8O,
+    ADDMEO,
+    ADDZE,
+    ADDZE8,
+    ADDZE8O,
+    ADDZEO,
+    AND,
+    AND8,
+    AND8_rec,
+    ANDC,
+    ANDC8,
+    ANDC8_rec,
+    ANDC_rec,
+    ANDI8_rec,
+    ANDIS8_rec,
+    ANDIS_rec,
+    ANDI_rec,
+    AND_rec,
+    CMPB,
+    CMPB8,
+    CNTLZD,
+    CNTLZD_rec,
+    CNTLZW,
+    CNTLZW8,
+    CNTLZW8_rec,
+    CNTLZW_rec,
+    CNTTZD,
+    CNTTZD_rec,
+    CNTTZW,
+    CNTTZW8,
+    CNTTZW8_rec,
+    CNTTZW_rec,
+    EQV,
+    EQV8,
+    EQV8_rec,
+    EQV_rec,
+    EXTSB,
+    EXTSB8,
+    EXTSB8_32_64,
+    EXTSB8_rec,
+    EXTSB_rec,
+    EXTSH,
+    EXTSH8,
+    EXTSH8_32_64,
+    EXTSH8_rec,
+    EXTSH_rec,
+    EXTSW,
+    EXTSWSLI,
+    EXTSWSLI_32_64,
+    EXTSWSLI_32_64_rec,
+    EXTSWSLI_rec,
+    EXTSW_32,
+    EXTSW_32_64,
+    EXTSW_32_64_rec,
+    EXTSW_rec,
+    FABSD,
+    FABSS,
+    FCPSGND,
+    FCPSGNS,
+    FMR,
+    FNABSD,
+    FNABSS,
+    FNEGD,
+    FNEGS,
+    ISEL,
+    ISEL8,
+    LI,
+    LI8,
+    LIS,
+    LIS8,
+    MFCTR,
+    MFCTR8,
+    MFLR,
+    MFLR8,
+    MFOCRF,
+    MFOCRF8,
+    MFVRD,
+    MFVRWZ,
+    MFVSRD,
+    MFVSRWZ,
+    MTVRD,
+    MTVRWA,
+    MTVRWZ,
+    MTVSRBM,
+    MTVSRBMI,
+    MTVSRD,
+    MTVSRDM,
+    MTVSRHM,
+    MTVSRQM,
+    MTVSRWA,
+    MTVSRWM,
+    MTVSRWZ,
+    NAND,
+    NAND8,
+    NAND8_rec,
+    NAND_rec,
+    NEG,
+    NEG8,
+    NEG8O,
+    NEG8_rec,
+    NEGO,
+    NEG_rec,
+    NOP,
+    NOP_GT_PWR6,
+    NOP_GT_PWR7,
+    NOR,
+    NOR8,
+    NOR8_rec,
+    NOR_rec,
+    OR,
+    OR8,
+    OR8_rec,
+    ORC,
+    ORC8,
+    ORC8_rec,
+    ORC_rec,
+    ORI,
+    ORI8,
+    ORIS,
+    ORIS8,
+    OR_rec,
+    POPCNTB,
+    POPCNTB8,
+    POPCNTD,
+    POPCNTW,
+    RLDCL,
+    RLDCL_rec,
+    RLDCR,
+    RLDCR_rec,
+    RLDIC,
+    RLDICL,
+    RLDICL_32,
+    RLDICL_32_64,
+    RLDICL_32_rec,
+    RLDICL_rec,
+    RLDICR,
+    RLDICR_32,
+    RLDICR_rec,
+    RLDIC_rec,
+    RLDIMI,
+    RLDIMI_rec,
+    RLWIMI,
+    RLWIMI8,
+    RLWIMI8_rec,
+    RLWIMI_rec,
+    RLWINM,
+    RLWINM8,
+    RLWINM8_rec,
+    RLWINM_rec,
+    RLWNM,
+    RLWNM8,
+    RLWNM8_rec,
+    RLWNM_rec,
+    SETB,
+    SETB8,
+    SETBC,
+    SETBC8,
+    SETBCR,
+    SETBCR8,
+    SETNBC,
+    SETNBC8,
+    SETNBCR,
+    SETNBCR8,
+    SLD,
+    SLD_rec,
+    SLW,
+    SLW8,
+    SLW8_rec,
+    SLW_rec,
+    SRAD,
+    SRADI,
+    SRADI_32,
+    SRAW,
+    SRAWI,
+    SRD,
+    SRD_rec,
+    SRW,
+    SRW8,
+    SRW8_rec,
+    SRW_rec,
+    SUBF,
+    SUBF8,
+    SUBF8O,
+    SUBF8_rec,
+    SUBFE,
+    SUBFE8,
+    SUBFE8O,
+    SUBFEO,
+    SUBFIC,
+    SUBFIC8,
+    SUBFME,
+    SUBFME8,
+    SUBFME8O,
+    SUBFMEO,
+    SUBFO,
+    SUBFZE,
+    SUBFZE8,
+    SUBFZE8O,
+    SUBFZEO,
+    SUBF_rec,
+    VABSDUB,
+    VABSDUH,
+    VABSDUW,
+    VADDCUW,
+    VADDSBS,
+    VADDSHS,
+    VADDSWS,
+    VADDUBM,
+    VADDUBS,
+    VADDUDM,
+    VADDUHM,
+    VADDUHS,
+    VADDUWM,
+    VADDUWS,
+    VAND,
+    VANDC,
+    VAVGSB,
+    VAVGSH,
+    VAVGSW,
+    VAVGUB,
+    VAVGUH,
+    VAVGUW,
+    VCLZB,
+    VCLZD,
+    VCLZH,
+    VCLZW,
+    VCMPBFP,
+    VCMPBFP_rec,
+    VCMPEQFP,
+    VCMPEQFP_rec,
+    VCMPEQUB,
+    VCMPEQUB_rec,
+    VCMPEQUD,
+    VCMPEQUD_rec,
+    VCMPEQUH,
+    VCMPEQUH_rec,
+    VCMPEQUQ,
+    VCMPEQUQ_rec,
+    VCMPEQUW,
+    VCMPEQUW_rec,
+    VCMPGEFP,
+    VCMPGEFP_rec,
+    VCMPGTFP,
+    VCMPGTFP_rec,
+    VCMPGTSB,
+    VCMPGTSB_rec,
+    VCMPGTSD,
+    VCMPGTSD_rec,
+    VCMPGTSH,
+    VCMPGTSH_rec,
+    VCMPGTSQ,
+    VCMPGTSQ_rec,
+    VCMPGTSW,
+    VCMPGTSW_rec,
+    VCMPGTUB,
+    VCMPGTUB_rec,
+    VCMPGTUD,
+    VCMPGTUD_rec,
+    VCMPGTUH,
+    VCMPGTUH_rec,
+    VCMPGTUQ,
+    VCMPGTUQ_rec,
+    VCMPGTUW,
+    VCMPGTUW_rec,
+    VCMPNEB,
+    VCMPNEB_rec,
+    VCMPNEH,
+    VCMPNEH_rec,
+    VCMPNEW,
+    VCMPNEW_rec,
+    VCMPNEZB,
+    VCMPNEZB_rec,
+    VCMPNEZH,
+    VCMPNEZH_rec,
+    VCMPNEZW,
+    VCMPNEZW_rec,
+    VCNTMBB,
+    VCNTMBD,
+    VCNTMBH,
+    VCNTMBW,
+    VCTZB,
+    VCTZD,
+    VCTZH,
+    VCTZW,
+    VEQV,
+    VEXPANDBM,
+    VEXPANDDM,
+    VEXPANDHM,
+    VEXPANDQM,
+    VEXPANDWM,
+    VEXTRACTBM,
+    VEXTRACTDM,
+    VEXTRACTHM,
+    VEXTRACTQM,
+    VEXTRACTWM,
+    VEXTSB2D,
+    VEXTSB2Ds,
+    VEXTSB2W,
+    VEXTSB2Ws,
+    VEXTSD2Q,
+    VEXTSH2D,
+    VEXTSH2Ds,
+    VEXTSH2W,
+    VEXTSH2Ws,
+    VEXTSW2D,
+    VEXTSW2Ds,
+    VMAXFP,
+    VMAXSB,
+    VMAXSD,
+    VMAXSH,
+    VMAXSW,
+    VMAXUB,
+    VMAXUD,
+    VMAXUH,
+    VMAXUW,
+    VMINFP,
+    VMINSB,
+    VMINSD,
+    VMINSH,
+    VMINSW,
+    VMINUB,
+    VMINUD,
+    VMINUH,
+    VMINUW,
+    VMRGEW,
+    VMRGOW,
+    VNAND,
+    VNEGD,
+    VNEGW,
+    VNOR,
+    VOR,
+    VORC,
+    VPOPCNTB,
+    VPOPCNTD,
+    VPOPCNTH,
+    VPOPCNTW,
+    VPRTYBD,
+    VPRTYBW,
+    VRLB,
+    VRLD,
+    VRLDMI,
+    VRLDNM,
+    VRLH,
+    VRLW,
+    VRLWMI,
+    VRLWNM,
+    VSEL,
+    VSHASIGMAD,
+    VSHASIGMAW,
+    VSLB,
+    VSLD,
+    VSLH,
+    VSLW,
+    VSRAB,
+    VSRAD,
+    VSRAH,
+    VSRAW,
+    VSRB,
+    VSRD,
+    VSRH,
+    VSRW,
+    VSUBCUW,
+    VSUBSBS,
+    VSUBSHS,
+    VSUBSWS,
+    VSUBUBM,
+    VSUBUBS,
+    VSUBUDM,
+    VSUBUHM,
+    VSUBUHS,
+    VSUBUWM,
+    VSUBUWS,
+    VXOR,
+    V_SET0,
+    V_SET0B,
+    V_SET0H,
+    XOR,
+    XOR8,
+    XOR8_rec,
+    XORI,
+    XORI8,
+    XORIS,
+    XORIS8,
+    XOR_rec,
+    XSABSDP,
+    XSABSQP,
+    XSCMPEQDP,
+    XSCMPGEDP,
+    XSCMPGTDP,
+    XSCPSGNDP,
+    XSCPSGNQP,
+    XSCVHPDP,
+    XSCVSPDPN,
+    XSIEXPDP,
+    XSIEXPQP,
+    XSMAXCDP,
+    XSMAXDP,
+    XSMAXJDP,
+    XSMINCDP,
+    XSMINDP,
+    XSMINJDP,
+    XSNABSDP,
+    XSNABSQP,
+    XSNEGDP,
+    XSNEGQP,
+    XSXEXPDP,
+    XSXEXPQP,
+    XSXSIGDP,
+    XVABSDP,
+    XVABSSP,
+    XVCMPEQDP,
+    XVCMPEQDP_rec,
+    XVCMPEQSP,
+    XVCMPEQSP_rec,
+    XVCMPGEDP,
+    XVCMPGEDP_rec,
+    XVCMPGESP,
+    XVCMPGESP_rec,
+    XVCMPGTDP,
+    XVCMPGTDP_rec,
+    XVCMPGTSP,
+    XVCMPGTSP_rec,
+    XVCPSGNDP,
+    XVCPSGNSP,
+    XVCVHPSP,
+    XVIEXPDP,
+    XVIEXPSP,
+    XVMAXDP,
+    XVMAXSP,
+    XVMINDP,
+    XVMINSP,
+    XVNABSDP,
+    XVNABSSP,
+    XVNEGDP,
+    XVNEGSP,
+    XVTSTDCDP,
+    XVTSTDCSP,
+    XVXEXPDP,
+    XVXEXPSP,
+    XVXSIGDP,
+    XVXSIGSP,
+    XXLAND,
+    XXLANDC,
+    XXLEQV,
+    XXLEQVOnes,
+    XXLNAND,
+    XXLNOR,
+    XXLOR,
+    XXLORC,
+    XXLORf,
+    XXLXOR,
+    XXLXORdpz,
+    XXLXORspz,
+    XXLXORz,
+    XXSEL),
+  FUSION_OP_SET(ADD4,
+    ADD4O,
+    ADD4TLS,
+    ADD4_rec,
+    ADD8,
+    ADD8O,
+    ADD8TLS,
+    ADD8TLS_,
+    ADD8_rec,
+    ADDE,
+    ADDE8,
+    ADDE8O,
+    ADDEO,
+    ADDEX,
+    ADDEX8,
+    ADDI,
+    ADDI8,
+    ADDIC,
+    ADDIC8,
+    ADDIS,
+    ADDIS8,
+    ADDISdtprelHA32,
+    ADDIStocHA,
+    ADDIStocHA8,
+    ADDIdtprelL32,
+    ADDItlsldLADDR32,
+    ADDItocL,
+    ADDME,
+    ADDME8,
+    ADDME8O,
+    ADDMEO,
+    ADDZE,
+    ADDZE8,
+    ADDZE8O,
+    ADDZEO,
+    AND,
+    AND8,
+    AND8_rec,
+    ANDC,
+    ANDC8,
+    ANDC8_rec,
+    ANDC_rec,
+    ANDI8_rec,
+    ANDIS8_rec,
+    ANDIS_rec,
+    ANDI_rec,
+    AND_rec,
+    CMPB,
+    CMPB8,
+    CMPD,
+    CMPDI,
+    CMPEQB,
+    CMPLD,
+    CMPLDI,
+    CMPLW,
+    CMPLWI,
+    CMPRB,
+    CMPRB8,
+    CMPW,
+    CMPWI,
+    CNTLZD,
+    CNTLZD_rec,
+    CNTLZW,
+    CNTLZW8,
+    CNTLZW8_rec,
+    CNTLZW_rec,
+    CNTTZD,
+    CNTTZD_rec,
+    CNTTZW,
+    CNTTZW8,
+    CNTTZW8_rec,
+    CNTTZW_rec,
+    CR6SET,
+    CR6UNSET,
+    CRAND,
+    CRANDC,
+    CREQV,
+    CRNAND,
+    CRNOR,
+    CROR,
+    CRORC,
+    CRSET,
+    CRUNSET,
+    CRXOR,
+    DSS,
+    DSSALL,
+    DST,
+    DST64,
+    DSTST,
+    DSTST64,
+    DSTSTT,
+    DSTSTT64,
+    DSTT,
+    DSTT64,
+    EQV,
+    EQV8,
+    EQV8_rec,
+    EQV_rec,
+    EXTSB,
+    EXTSB8,
+    EXTSB8_32_64,
+    EXTSB8_rec,
+    EXTSB_rec,
+    EXTSH,
+    EXTSH8,
+    EXTSH8_32_64,
+    EXTSH8_rec,
+    EXTSH_rec,
+    EXTSW,
+    EXTSWSLI,
+    EXTSWSLI_32_64,
+    EXTSWSLI_32_64_rec,
+    EXTSWSLI_rec,
+    EXTSW_32,
+    EXTSW_32_64,
+    EXTSW_32_64_rec,
+    EXTSW_rec,
+    FABSD,
+    FABSS,
+    FCMPOD,
+    FCMPOS,
+    FCMPUD,
+    FCMPUS,
+    FCPSGND,
+    FCPSGNS,
+    FMR,
+    FNABSD,
+    FNABSS,
+    FNEGD,
+    FNEGS,
+    FTDIV,
+    FTSQRT,
+    ISEL,
+    ISEL8,
+    LI,
+    LI8,
+    LIS,
+    LIS8,
+    MCRF,
+    MCRXRX,
+    MFCTR,
+    MFCTR8,
+    MFLR,
+    MFLR8,
+    MFOCRF,
+    MFOCRF8,
+    MFVRD,
+    MFVRWZ,
+    MFVSRD,
+    MFVSRWZ,
+    MTCTR,
+    MTCTR8,
+    MTCTR8loop,
+    MTCTRloop,
+    MTLR,
+    MTLR8,
+    MTOCRF,
+    MTOCRF8,
+    MTVRD,
+    MTVRWA,
+    MTVRWZ,
+    MTVSRBM,
+    MTVSRBMI,
+    MTVSRD,
+    MTVSRDM,
+    MTVSRHM,
+    MTVSRQM,
+    MTVSRWA,
+    MTVSRWM,
+    MTVSRWZ,
+    NAND,
+    NAND8,
+    NAND8_rec,
+    NAND_rec,
+    NEG,
+    NEG8,
+    NEG8O,
+    NEG8_rec,
+    NEGO,
+    NEG_rec,
+    NOP,
+    NOP_GT_PWR6,
+    NOP_GT_PWR7,
+    NOR,
+    NOR8,
+    NOR8_rec,
+    NOR_rec,
+    OR,
+    OR8,
+    OR8_rec,
+    ORC,
+    ORC8,
+    ORC8_rec,
+    ORC_rec,
+    ORI,
+    ORI8,
+    ORIS,
+    ORIS8,
+    OR_rec,
+    POPCNTB,
+    POPCNTB8,
+    POPCNTD,
+    POPCNTW,
+    RLDCL,
+    RLDCL_rec,
+    RLDCR,
+    RLDCR_rec,
+    RLDIC,
+    RLDICL,
+    RLDICL_32,
+    RLDICL_32_64,
+    RLDICL_32_rec,
+    RLDICL_rec,
+    RLDICR,
+    RLDICR_32,
+    RLDICR_rec,
+    RLDIC_rec,
+    RLDIMI,
+    RLDIMI_rec,
+    RLWIMI,
+    RLWIMI8,
+    RLWIMI8_rec,
+    RLWIMI_rec,
+    RLWINM,
+    RLWINM8,
+    RLWINM8_rec,
+    RLWINM_rec,
+    RLWNM,
+    RLWNM8,
+    RLWNM8_rec,
+    RLWNM_rec,
+    SETB,
+    SETB8,
+    SETBC,
+    SETBC8,
+    SETBCR,
+    SETBCR8,
+    SETNBC,
+    SETNBC8,
+    SETNBCR,
+    SETNBCR8,
+    SLD,
+    SLD_rec,
+    SLW,
+    SLW8,
+    SLW8_rec,
+    SLW_rec,
+    SRAD,
+    SRADI,
+    SRADI_32,
+    SRAW,
+    SRAWI,
+    SRD,
+    SRD_rec,
+    SRW,
+    SRW8,
+    SRW8_rec,
+    SRW_rec,
+    SUBF,
+    SUBF8,
+    SUBF8O,
+    SUBF8_rec,
+    SUBFE,
+    SUBFE8,
+    SUBFE8O,
+    SUBFEO,
+    SUBFIC,
+    SUBFIC8,
+    SUBFME,
+    SUBFME8,
+    SUBFME8O,
+    SUBFMEO,
+    SUBFO,
+    SUBFZE,
+    SUBFZE8,
+    SUBFZE8O,
+    SUBFZEO,
+    SUBF_rec,
+    TD,
+    TDI,
+    TRAP,
+    TW,
+    TWI,
+    VABSDUB,
+    VABSDUH,
+    VABSDUW,
+    VADDCUW,
+    VADDSBS,
+    VADDSHS,
+    VADDSWS,
+    VADDUBM,
+    VADDUBS,
+    VADDUDM,
+    VADDUHM,
+    VADDUHS,
+    VADDUWM,
+    VADDUWS,
+    VAND,
+    VANDC,
+    VAVGSB,
+    VAVGSH,
+    VAVGSW,
+    VAVGUB,
+    VAVGUH,
+    VAVGUW,
+    VCLZB,
+    VCLZD,
+    VCLZH,
+    VCLZW,
+    VCMPBFP,
+    VCMPBFP_rec,
+    VCMPEQFP,
+    VCMPEQFP_rec,
+    VCMPEQUB,
+    VCMPEQUB_rec,
+    VCMPEQUD,
+    VCMPEQUD_rec,
+    VCMPEQUH,
+    VCMPEQUH_rec,
+    VCMPEQUQ,
+    VCMPEQUQ_rec,
+    VCMPEQUW,
+    VCMPEQUW_rec,
+    VCMPGEFP,
+    VCMPGEFP_rec,
+    VCMPGTFP,
+    VCMPGTFP_rec,
+    VCMPGTSB,
+    VCMPGTSB_rec,
+    VCMPGTSD,
+    VCMPGTSD_rec,
+    VCMPGTSH,
+    VCMPGTSH_rec,
+    VCMPGTSQ,
+    VCMPGTSQ_rec,
+    VCMPGTSW,
+    VCMPGTSW_rec,
+    VCMPGTUB,
+    VCMPGTUB_rec,
+    VCMPGTUD,
+    VCMPGTUD_rec,
+    VCMPGTUH,
+    VCMPGTUH_rec,
+    VCMPGTUQ,
+    VCMPGTUQ_rec,
+    VCMPGTUW,
+    VCMPGTUW_rec,
+    VCMPNEB,
+    VCMPNEB_rec,
+    VCMPNEH,
+    VCMPNEH_rec,
+    VCMPNEW,
+    VCMPNEW_rec,
+    VCMPNEZB,
+    VCMPNEZB_rec,
+    VCMPNEZH,
+    VCMPNEZH_rec,
+    VCMPNEZW,
+    VCMPNEZW_rec,
+    VCMPSQ,
+    VCMPUQ,
+    VCNTMBB,
+    VCNTMBD,
+    VCNTMBH,
+    VCNTMBW,
+    VCTZB,
+    VCTZD,
+    VCTZH,
+    VCTZW,
+    VEQV,
+    VEXPANDBM,
+    VEXPANDDM,
+    VEXPANDHM,
+    VEXPANDQM,
+    VEXPANDWM,
+    VEXTRACTBM,
+    VEXTRACTDM,
+    VEXTRACTHM,
+    VEXTRACTQM,
+    VEXTRACTWM,
+    VEXTSB2D,
+    VEXTSB2Ds,
+    VEXTSB2W,
+    VEXTSB2Ws,
+    VEXTSD2Q,
+    VEXTSH2D,
+    VEXTSH2Ds,
+    VEXTSH2W,
+    VEXTSH2Ws,
+    VEXTSW2D,
+    VEXTSW2Ds,
+    VMAXFP,
+    VMAXSB,
+    VMAXSD,
+    VMAXSH,
+    VMAXSW,
+    VMAXUB,
+    VMAXUD,
+    VMAXUH,
+    VMAXUW,
+    VMINFP,
+    VMINSB,
+    VMINSD,
+    VMINSH,
+    VMINSW,
+    VMINUB,
+    VMINUD,
+    VMINUH,
+    VMINUW,
+    VMRGEW,
+    VMRGOW,
+    VNAND,
+    VNEGD,
+    VNEGW,
+    VNOR,
+    VOR,
+    VORC,
+    VPOPCNTB,
+    VPOPCNTD,
+    VPOPCNTH,
+    VPOPCNTW,
+    VPRTYBD,
+    VPRTYBW,
+    VRLB,
+    VRLD,
+    VRLDMI,
+    VRLDNM,
+    VRLH,
+    VRLW,
+    VRLWMI,
+    VRLWNM,
+    VSEL,
+    VSHASIGMAD,
+    VSHASIGMAW,
+    VSLB,
+    VSLD,
+    VSLH,
+    VSLW,
+    VSRAB,
+    VSRAD,
+    VSRAH,
+    VSRAW,
+    VSRB,
+    VSRD,
+    VSRH,
+    VSRW,
+    VSUBCUW,
+    VSUBSBS,
+    VSUBSHS,
+    VSUBSWS,
+    VSUBUBM,
+    VSUBUBS,
+    VSUBUDM,
+    VSUBUHM,
+    VSUBUHS,
+    VSUBUWM,
+    VSUBUWS,
+    VXOR,
+    V_SET0,
+    V_SET0B,
+    V_SET0H,
+    WAIT,
+    XOR,
+    XOR8,
+    XOR8_rec,
+    XORI,
+    XORI8,
+    XORIS,
+    XORIS8,
+    XOR_rec,
+    XSABSDP,
+    XSABSQP,
+    XSCMPEQDP,
+    XSCMPEXPDP,
+    XSCMPGEDP,
+    XSCMPGTDP,
+    XSCMPODP,
+    XSCMPUDP,
+    XSCPSGNDP,
+    XSCPSGNQP,
+    XSCVHPDP,
+    XSCVSPDPN,
+    XSIEXPDP,
+    XSIEXPQP,
+    XSMAXCDP,
+    XSMAXDP,
+    XSMAXJDP,
+    XSMINCDP,
+    XSMINDP,
+    XSMINJDP,
+    XSNABSDP,
+    XSNABSQP,
+    XSNEGDP,
+    XSNEGQP,
+    XSTDIVDP,
+    XSTSQRTDP,
+    XSTSTDCDP,
+    XSTSTDCSP,
+    XSXEXPDP,
+    XSXEXPQP,
+    XSXSIGDP,
+    XVABSDP,
+    XVABSSP,
+    XVCMPEQDP,
+    XVCMPEQDP_rec,
+    XVCMPEQSP,
+    XVCMPEQSP_rec,
+    XVCMPGEDP,
+    XVCMPGEDP_rec,
+    XVCMPGESP,
+    XVCMPGESP_rec,
+    XVCMPGTDP,
+    XVCMPGTDP_rec,
+    XVCMPGTSP,
+    XVCMPGTSP_rec,
+    XVCPSGNDP,
+    XVCPSGNSP,
+    XVCVHPSP,
+    XVIEXPDP,
+    XVIEXPSP,
+    XVMAXDP,
+    XVMAXSP,
+    XVMINDP,
+    XVMINSP,
+    XVNABSDP,
+    XVNABSSP,
+    XVNEGDP,
+    XVNEGSP,
+    XVTDIVDP,
+    XVTDIVSP,
+    XVTLSBB,
+    XVTSQRTDP,
+    XVTSQRTSP,
+    XVTSTDCDP,
+    XVTSTDCSP,
+    XVXEXPDP,
+    XVXEXPSP,
+    XVXSIGDP,
+    XVXSIGSP,
+    XXLAND,
+    XXLANDC,
+    XXLEQV,
+    XXLEQVOnes,
+    XXLNAND,
+    XXLNOR,
+    XXLOR,
+    XXLORC,
+    XXLORf,
+    XXLXOR,
+    XXLXORdpz,
+    XXLXORspz,
+    XXLXORz,
+    XXSEL))
\ No newline at end of file
diff --git a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
index a2664bcff4ab..ba74af5ef5f7 100644
--- a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
@@ -4464,9 +4464,10 @@ bool PPCDAGToDAGISel::trySETCC(SDNode *N) {
 bool PPCDAGToDAGISel::isOffsetMultipleOf(SDNode *N, unsigned Val) const {
   LoadSDNode *LDN = dyn_cast<LoadSDNode>(N);
   StoreSDNode *STN = dyn_cast<StoreSDNode>(N);
+  MemIntrinsicSDNode *MIN = dyn_cast<MemIntrinsicSDNode>(N);
   SDValue AddrOp;
-  if (LDN)
-    AddrOp = LDN->getOperand(1);
+  if (LDN || (MIN && MIN->getOpcode() == PPCISD::LD_SPLAT))
+    AddrOp = N->getOperand(1);
   else if (STN)
     AddrOp = STN->getOperand(2);
 
@@ -5973,6 +5974,15 @@ void PPCDAGToDAGISel::Select(SDNode *N) {
     if (Type != MVT::v16i8 && Type != MVT::v8i16)
       break;
 
+    // If the alignment for the load is 16 or bigger, we don't need the
+    // permutated mask to get the required value. The value must be the 0
+    // element in big endian target or 7/15 in little endian target in the
+    // result vsx register of lvx instruction.
+    // Select the instruction in the .td file.
+    if (cast<MemIntrinsicSDNode>(N)->getAlign() >= Align(16) &&
+        isOffsetMultipleOf(N, 16))
+      break;
+
     SDValue ZeroReg =
         CurDAG->getRegister(Subtarget->isPPC64() ? PPC::ZERO8 : PPC::ZERO,
                             Subtarget->isPPC64() ? MVT::i64 : MVT::i32);
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index ec7e30d7e362..8d6edf07bc53 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -3500,15 +3500,16 @@ SDValue PPCTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
     if (LHS.getValueType() == MVT::v2i64) {
       // Equality can be handled by casting to the legal type for Altivec
       // comparisons, everything else needs to be expanded.
-      if (CC == ISD::SETEQ || CC == ISD::SETNE) {
-        return DAG.getNode(
-            ISD::BITCAST, dl, MVT::v2i64,
-            DAG.getSetCC(dl, MVT::v4i32,
-                         DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, LHS),
-                         DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, RHS), CC));
-      }
-
-      return SDValue();
+      if (CC != ISD::SETEQ && CC != ISD::SETNE)
+        return SDValue();
+      SDValue SetCC32 = DAG.getSetCC(
+          dl, MVT::v4i32, DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, LHS),
+          DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, RHS), CC);
+      int ShuffV[] = {1, 0, 3, 2};
+      SDValue Shuff =
+          DAG.getVectorShuffle(MVT::v4i32, dl, SetCC32, SetCC32, ShuffV);
+      return DAG.getBitcast(
+          MVT::v2i64, DAG.getNode(ISD::AND, dl, MVT::v4i32, Shuff, SetCC32));
     }
 
     // We handle most of these in the usual way.
@@ -6206,20 +6207,13 @@ SDValue PPCTargetLowering::LowerCall_64SVR4(
         ArgOffset += PtrByteSize;
         continue;
       }
-      // Copy entire object into memory.  There are cases where gcc-generated
-      // code assumes it is there, even if it could be put entirely into
-      // registers.  (This is not what the doc says.)
-
-      // FIXME: The above statement is likely due to a misunderstanding of the
-      // documents.  All arguments must be copied into the parameter area BY
-      // THE CALLEE in the event that the callee takes the address of any
-      // formal argument.  That has not yet been implemented.  However, it is
-      // reasonable to use the stack area as a staging area for the register
-      // load.
-
-      // Skip this for small aggregates, as we will use the same slot for a
-      // right-justified copy, below.
-      if (Size >= 8)
+      // Copy the object to parameter save area if it can not be entirely passed 
+      // by registers.
+      // FIXME: we only need to copy the parts which need to be passed in
+      // parameter save area. For the parts passed by registers, we don't need
+      // to copy them to the stack although we need to allocate space for them
+      // in parameter save area.
+      if ((NumGPRs - GPR_idx) * PtrByteSize < Size)
         Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, PtrOff,
                                                           CallSeqStart,
                                                           Flags, DAG, dl);
@@ -17548,14 +17542,14 @@ unsigned PPCTargetLowering::computeMOFlags(const SDNode *Parent, SDValue N,
   if (Subtarget.isISA3_1() && ((ParentOp == ISD::INTRINSIC_W_CHAIN) ||
                                (ParentOp == ISD::INTRINSIC_VOID))) {
     unsigned ID = cast<ConstantSDNode>(Parent->getOperand(1))->getZExtValue();
-    assert(
-        ((ID == Intrinsic::ppc_vsx_lxvp) || (ID == Intrinsic::ppc_vsx_stxvp)) &&
-        "Only the paired load and store (lxvp/stxvp) intrinsics are valid.");
-    SDValue IntrinOp = (ID == Intrinsic::ppc_vsx_lxvp) ? Parent->getOperand(2)
-                                                       : Parent->getOperand(3);
-    computeFlagsForAddressComputation(IntrinOp, FlagSet, DAG);
-    FlagSet |= PPC::MOF_Vector;
-    return FlagSet;
+    if ((ID == Intrinsic::ppc_vsx_lxvp) || (ID == Intrinsic::ppc_vsx_stxvp)) {
+      SDValue IntrinOp = (ID == Intrinsic::ppc_vsx_lxvp)
+                             ? Parent->getOperand(2)
+                             : Parent->getOperand(3);
+      computeFlagsForAddressComputation(IntrinOp, FlagSet, DAG);
+      FlagSet |= PPC::MOF_Vector;
+      return FlagSet;
+    }
   }
 
   // Mark this as something we don't want to handle here if it is atomic
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.h b/llvm/lib/Target/PowerPC/PPCInstrInfo.h
index 2cfd53de3290..c16e146da247 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.h
@@ -393,7 +393,9 @@ public:
                              MachineInstr &NewMI1,
                              MachineInstr &NewMI2) const override;
 
-  void setSpecialOperandAttr(MachineInstr &MI, uint16_t Flags) const override;
+  // PowerPC specific version of setSpecialOperandAttr that copies Flags to MI
+  // and clears nuw, nsw, and exact flags.
+  void setSpecialOperandAttr(MachineInstr &MI, uint16_t Flags) const;
 
   bool isCoalescableExtInstr(const MachineInstr &MI,
                              Register &SrcReg, Register &DstReg,
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.td b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
index d83ecc699b19..2340be5b5915 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
@@ -4780,6 +4780,7 @@ class PPCAsmPseudo<string asm, dag iops>
 def : InstAlias<"sc", (SC 0)>;
 
 def : InstAlias<"sync", (SYNC 0)>, Requires<[HasSYNC]>;
+def : InstAlias<"hwsync", (SYNC 0), 0>, Requires<[HasSYNC]>;
 def : InstAlias<"msync", (SYNC 0), 0>, Requires<[HasSYNC]>;
 def : InstAlias<"lwsync", (SYNC 1)>, Requires<[HasSYNC]>;
 def : InstAlias<"ptesync", (SYNC 2)>, Requires<[HasSYNC]>;
diff --git a/llvm/lib/Target/PowerPC/PPCInstrVSX.td b/llvm/lib/Target/PowerPC/PPCInstrVSX.td
index d92a10c5b208..110f7d79fbc5 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrVSX.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrVSX.td
@@ -158,6 +158,11 @@ def HasP9Vector : Predicate<"Subtarget->hasP9Vector()">;
 def NoP9Altivec : Predicate<"!Subtarget->hasP9Altivec()">;
 def NoP10Vector: Predicate<"!Subtarget->hasP10Vector()">;
 
+def PPCldsplatAlign16 : PatFrag<(ops node:$ptr), (PPCldsplat node:$ptr), [{
+  return cast<MemIntrinsicSDNode>(N)->getAlign() >= Align(16) &&
+         isOffsetMultipleOf(N, 16);
+}]>;
+
 //--------------------- VSX-specific instruction formats ---------------------//
 // By default, all VSX instructions are to be selected over their Altivec
 // counter parts and they do not have unmodeled sideeffects.
@@ -3180,6 +3185,12 @@ defm : ScalToVecWPermute<
   v2f64, (f64 (load ForceXForm:$src)),
   (XXPERMDIs (XFLOADf64 ForceXForm:$src), 2),
   (SUBREG_TO_REG (i64 1), (XFLOADf64 ForceXForm:$src), sub_64)>;
+
+// Splat loads.
+def : Pat<(v8i16 (PPCldsplatAlign16 ForceXForm:$A)),
+          (v8i16 (VSPLTH 7, (LVX ForceXForm:$A)))>;
+def : Pat<(v16i8 (PPCldsplatAlign16 ForceXForm:$A)),
+          (v16i8 (VSPLTB 15, (LVX ForceXForm:$A)))>;
 } // HasVSX, NoP9Vector, IsLittleEndian
 
 let Predicates = [HasVSX, NoP9Vector, IsBigEndian] in {
@@ -3187,6 +3198,12 @@ let Predicates = [HasVSX, NoP9Vector, IsBigEndian] in {
             (LXVD2X ForceXForm:$src)>;
   def : Pat<(int_ppc_vsx_stxvd2x v2f64:$rS, ForceXForm:$dst),
             (STXVD2X $rS, ForceXForm:$dst)>;
+
+  // Splat loads.
+  def : Pat<(v8i16 (PPCldsplatAlign16 ForceXForm:$A)),
+            (v8i16 (VSPLTH 0, (LVX ForceXForm:$A)))>;
+  def : Pat<(v16i8 (PPCldsplatAlign16 ForceXForm:$A)),
+            (v16i8 (VSPLTB 0, (LVX ForceXForm:$A)))>;
 } // HasVSX, NoP9Vector, IsBigEndian
 
 // Any VSX subtarget that only has loads and stores that load in big endian
diff --git a/llvm/lib/Target/PowerPC/PPCLoopInstrFormPrep.cpp b/llvm/lib/Target/PowerPC/PPCLoopInstrFormPrep.cpp
index 7f63827afbd6..0c7be96a0595 100644
--- a/llvm/lib/Target/PowerPC/PPCLoopInstrFormPrep.cpp
+++ b/llvm/lib/Target/PowerPC/PPCLoopInstrFormPrep.cpp
@@ -413,9 +413,9 @@ bool PPCLoopInstrFormPrep::runOnFunction(Function &F) {
 
   bool MadeChange = false;
 
-  for (auto I = LI->begin(), IE = LI->end(); I != IE; ++I)
-    for (auto L = df_begin(*I), LE = df_end(*I); L != LE; ++L)
-      MadeChange |= runOnLoop(*L);
+  for (Loop *I : *LI)
+    for (Loop *L : depth_first(I))
+      MadeChange |= runOnLoop(L);
 
   return MadeChange;
 }
diff --git a/llvm/lib/Target/PowerPC/PPCMacroFusion.def b/llvm/lib/Target/PowerPC/PPCMacroFusion.def
index e4954b722fd0..6b8ad22639c8 100644
--- a/llvm/lib/Target/PowerPC/PPCMacroFusion.def
+++ b/llvm/lib/Target/PowerPC/PPCMacroFusion.def
@@ -153,5 +153,7 @@ FUSION_FEATURE(ZeroMoveLR, hasZeroMoveFusion, -1,
                FUSION_OP_SET(MTLR8, MTLR, MTSPR8, MTSPR),
                FUSION_OP_SET(BCLR, BCLRn, gBCLR, BCLRL, BCLRLn, gBCLRL))
 
+#include "PPCBack2BackFusion.def"
+
 #undef FUSION_FEATURE
 #undef FUSION_OP_SET
diff --git a/llvm/lib/Target/PowerPC/PPCSubtarget.cpp b/llvm/lib/Target/PowerPC/PPCSubtarget.cpp
index 1258a1281597..f11b4e14073e 100644
--- a/llvm/lib/Target/PowerPC/PPCSubtarget.cpp
+++ b/llvm/lib/Target/PowerPC/PPCSubtarget.cpp
@@ -135,6 +135,7 @@ void PPCSubtarget::initializeEnvironment() {
   HasCompareFusion = false;
   HasWideImmFusion = false;
   HasZeroMoveFusion = false;
+  HasBack2BackFusion = false;
   IsISA2_06 = false;
   IsISA2_07 = false;
   IsISA3_0 = false;
diff --git a/llvm/lib/Target/PowerPC/PPCSubtarget.h b/llvm/lib/Target/PowerPC/PPCSubtarget.h
index d52833cb1465..1300b62b623a 100644
--- a/llvm/lib/Target/PowerPC/PPCSubtarget.h
+++ b/llvm/lib/Target/PowerPC/PPCSubtarget.h
@@ -155,6 +155,7 @@ protected:
   bool HasCompareFusion;
   bool HasWideImmFusion;
   bool HasZeroMoveFusion;
+  bool HasBack2BackFusion;
   bool IsISA2_06;
   bool IsISA2_07;
   bool IsISA3_0;
@@ -348,6 +349,7 @@ public:
   bool hasWideImmFusion() const { return HasWideImmFusion; }
   bool hasSha3Fusion() const { return HasSha3Fusion; }
   bool hasZeroMoveFusion() const { return HasZeroMoveFusion; }
+  bool hasBack2BackFusion() const { return HasBack2BackFusion; }
   bool needsSwapsForVSXMemOps() const {
     return hasVSX() && isLittleEndian() && !hasP9Vector();
   }
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
index 5d6f58a77a39..ed28731b8ef2 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
@@ -328,10 +328,6 @@ static bool isMMAType(Type *Ty) {
 InstructionCost PPCTTIImpl::getUserCost(const User *U,
                                         ArrayRef<const Value *> Operands,
                                         TTI::TargetCostKind CostKind) {
-  // Set the max cost if an MMA type is present (v256i1, v512i1).
-  if (isMMAType(U->getType()))
-    return InstructionCost::getMax();
-
   // We already implement getCastInstrCost and getMemoryOpCost where we perform
   // the vector adjustment there.
   if (isa<CastInst>(U) || isa<LoadInst>(U) || isa<StoreInst>(U))
@@ -1276,23 +1272,21 @@ PPCTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
   return BaseT::getIntrinsicInstrCost(ICA, CostKind);
 }
 
-bool PPCTTIImpl::areFunctionArgsABICompatible(
-    const Function *Caller, const Function *Callee,
-    SmallPtrSetImpl<Argument *> &Args) const {
+bool PPCTTIImpl::areTypesABICompatible(const Function *Caller,
+                                       const Function *Callee,
+                                       const ArrayRef<Type *> &Types) const {
 
   // We need to ensure that argument promotion does not
   // attempt to promote pointers to MMA types (__vector_pair
   // and __vector_quad) since these types explicitly cannot be
   // passed as arguments. Both of these types are larger than
   // the 128-bit Altivec vectors and have a scalar size of 1 bit.
-  if (!BaseT::areFunctionArgsABICompatible(Caller, Callee, Args))
+  if (!BaseT::areTypesABICompatible(Caller, Callee, Types))
     return false;
 
-  return llvm::none_of(Args, [](Argument *A) {
-    auto *EltTy = cast<PointerType>(A->getType())->getElementType();
-    if (EltTy->isSized())
-      return (EltTy->isIntOrIntVectorTy(1) &&
-              EltTy->getPrimitiveSizeInBits() > 128);
+  return llvm::none_of(Types, [](Type *Ty) {
+    if (Ty->isSized())
+      return Ty->isIntOrIntVectorTy(1) && Ty->getPrimitiveSizeInBits() > 128;
     return false;
   });
 }
@@ -1388,3 +1382,86 @@ bool PPCTTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
 
   return false;
 }
+
+bool PPCTTIImpl::hasActiveVectorLength(unsigned Opcode, Type *DataType,
+                                       Align Alignment) const {
+  // Only load and stores instructions can have variable vector length on Power.
+  if (Opcode != Instruction::Load && Opcode != Instruction::Store)
+    return false;
+  // Loads/stores with length instructions use bits 0-7 of the GPR operand and
+  // therefore cannot be used in 32-bit mode.
+  if ((!ST->hasP9Vector() && !ST->hasP10Vector()) || !ST->isPPC64())
+    return false;
+  if (isa<FixedVectorType>(DataType)) {
+    unsigned VecWidth = DataType->getPrimitiveSizeInBits();
+    return VecWidth == 128;
+  }
+  Type *ScalarTy = DataType->getScalarType();
+
+  if (ScalarTy->isPointerTy())
+    return true;
+
+  if (ScalarTy->isFloatTy() || ScalarTy->isDoubleTy())
+    return true;
+
+  if (!ScalarTy->isIntegerTy())
+    return false;
+
+  unsigned IntWidth = ScalarTy->getIntegerBitWidth();
+  return IntWidth == 8 || IntWidth == 16 || IntWidth == 32 || IntWidth == 64;
+}
+
+InstructionCost PPCTTIImpl::getVPMemoryOpCost(unsigned Opcode, Type *Src,
+                                              Align Alignment,
+                                              unsigned AddressSpace,
+                                              TTI::TargetCostKind CostKind,
+                                              const Instruction *I) {
+  InstructionCost Cost = BaseT::getVPMemoryOpCost(Opcode, Src, Alignment,
+                                                  AddressSpace, CostKind, I);
+  if (TLI->getValueType(DL, Src, true) == MVT::Other)
+    return Cost;
+  // TODO: Handle other cost kinds.
+  if (CostKind != TTI::TCK_RecipThroughput)
+    return Cost;
+
+  assert((Opcode == Instruction::Load || Opcode == Instruction::Store) &&
+         "Invalid Opcode");
+
+  auto *SrcVTy = dyn_cast<FixedVectorType>(Src);
+  assert(SrcVTy && "Expected a vector type for VP memory operations");
+
+  if (hasActiveVectorLength(Opcode, Src, Alignment)) {
+    std::pair<InstructionCost, MVT> LT =
+        TLI->getTypeLegalizationCost(DL, SrcVTy);
+
+    InstructionCost CostFactor =
+        vectorCostAdjustmentFactor(Opcode, Src, nullptr);
+    if (!CostFactor.isValid())
+      return InstructionCost::getMax();
+
+    InstructionCost Cost = LT.first * CostFactor;
+    assert(Cost.isValid() && "Expected valid cost");
+
+    // On P9 but not on P10, if the op is misaligned then it will cause a
+    // pipeline flush. Otherwise the VSX masked memops cost the same as unmasked
+    // ones.
+    const Align DesiredAlignment(16);
+    if (Alignment >= DesiredAlignment || ST->getCPUDirective() != PPC::DIR_PWR9)
+      return Cost;
+
+    // Since alignment may be under estimated, we try to compute the probability
+    // that the actual address is aligned to the desired boundary. For example
+    // an 8-byte aligned load is assumed to be actually 16-byte aligned half the
+    // time, while a 4-byte aligned load has a 25% chance of being 16-byte
+    // aligned.
+    float AlignmentProb = ((float)Alignment.value()) / DesiredAlignment.value();
+    float MisalignmentProb = 1.0 - AlignmentProb;
+    return (MisalignmentProb * P9PipelineFlushEstimate) +
+           (AlignmentProb * *Cost.getValue());
+  }
+
+  // Usually we should not get to this point, but the following is an attempt to
+  // model the cost of legalization. Currently we can only lower intrinsics with
+  // evl but no mask, on Power 9/10. Otherwise, we must scalarize.
+  return getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind);
+}
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
index 7aeb0c59d503..0af6f2a308d9 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
@@ -134,9 +134,19 @@ public:
       bool UseMaskForCond = false, bool UseMaskForGaps = false);
   InstructionCost getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
                                         TTI::TargetCostKind CostKind);
-  bool areFunctionArgsABICompatible(const Function *Caller,
-                                    const Function *Callee,
-                                    SmallPtrSetImpl<Argument *> &Args) const;
+  bool areTypesABICompatible(const Function *Caller, const Function *Callee,
+                             const ArrayRef<Type *> &Types) const;
+  bool hasActiveVectorLength(unsigned Opcode, Type *DataType,
+                             Align Alignment) const;
+  InstructionCost getVPMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
+                                    unsigned AddressSpace,
+                                    TTI::TargetCostKind CostKind,
+                                    const Instruction *I = nullptr);
+
+private:
+  // The following constant is used for estimating costs on power9.
+  static const InstructionCost::CostType P9PipelineFlushEstimate = 80;
+
   /// @}
 };
 
diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
index f00813f1301a..75592dd4c6f5 100644
--- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
+++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
@@ -169,6 +169,7 @@ class RISCVAsmParser : public MCTargetAsmParser {
   OperandMatchResultTy parseJALOffset(OperandVector &Operands);
   OperandMatchResultTy parseVTypeI(OperandVector &Operands);
   OperandMatchResultTy parseMaskReg(OperandVector &Operands);
+  OperandMatchResultTy parseInsnDirectiveOpcode(OperandVector &Operands);
 
   bool parseOperand(OperandVector &Operands, StringRef Mnemonic);
 
@@ -827,6 +828,7 @@ public:
     Op->SysReg.Length = Str.size();
     Op->SysReg.Encoding = Encoding;
     Op->StartLoc = S;
+    Op->EndLoc = S;
     Op->IsRV64 = IsRV64;
     return Op;
   }
@@ -836,6 +838,7 @@ public:
     auto Op = std::make_unique<RISCVOperand>(KindTy::VType);
     Op->VType.Val = VTypeI;
     Op->StartLoc = S;
+    Op->EndLoc = S;
     Op->IsRV64 = IsRV64;
     return Op;
   }
@@ -1291,7 +1294,7 @@ OperandMatchResultTy RISCVAsmParser::parseRegister(OperandVector &Operands,
     if (HadParens)
       Operands.push_back(RISCVOperand::createToken("(", FirstS, isRV64()));
     SMLoc S = getLoc();
-    SMLoc E = SMLoc::getFromPointer(S.getPointer() - 1);
+    SMLoc E = SMLoc::getFromPointer(S.getPointer() + Name.size());
     getLexer().Lex();
     Operands.push_back(RISCVOperand::createReg(RegNo, S, E, isRV64()));
   }
@@ -1305,6 +1308,67 @@ OperandMatchResultTy RISCVAsmParser::parseRegister(OperandVector &Operands,
 }
 
 OperandMatchResultTy
+RISCVAsmParser::parseInsnDirectiveOpcode(OperandVector &Operands) {
+  SMLoc S = getLoc();
+  SMLoc E;
+  const MCExpr *Res;
+
+  switch (getLexer().getKind()) {
+  default:
+    return MatchOperand_NoMatch;
+  case AsmToken::LParen:
+  case AsmToken::Minus:
+  case AsmToken::Plus:
+  case AsmToken::Exclaim:
+  case AsmToken::Tilde:
+  case AsmToken::Integer:
+  case AsmToken::String: {
+    if (getParser().parseExpression(Res, E))
+      return MatchOperand_ParseFail;
+
+    auto *CE = dyn_cast<MCConstantExpr>(Res);
+    if (CE) {
+      int64_t Imm = CE->getValue();
+      if (isUInt<7>(Imm)) {
+        Operands.push_back(RISCVOperand::createImm(Res, S, E, isRV64()));
+        return MatchOperand_Success;
+      }
+    }
+
+    Twine Msg = "immediate must be an integer in the range";
+    Error(S, Msg + " [" + Twine(0) + ", " + Twine((1 << 7) - 1) + "]");
+    return MatchOperand_ParseFail;
+  }
+  case AsmToken::Identifier: {
+    StringRef Identifier;
+    if (getParser().parseIdentifier(Identifier))
+      return MatchOperand_ParseFail;
+
+    auto Opcode = RISCVInsnOpcode::lookupRISCVOpcodeByName(Identifier);
+    if (Opcode) {
+      Res = MCConstantExpr::create(Opcode->Value, getContext());
+      E = SMLoc::getFromPointer(S.getPointer() + Identifier.size());
+      Operands.push_back(RISCVOperand::createImm(Res, S, E, isRV64()));
+      return MatchOperand_Success;
+    }
+
+    Twine Msg = "operand must be a valid opcode name or an "
+                "integer in the range";
+    Error(S, Msg + " [" + Twine(0) + ", " + Twine((1 << 7) - 1) + "]");
+    return MatchOperand_ParseFail;
+  }
+  case AsmToken::Percent: {
+    // Discard operand with modifier.
+    Twine Msg = "immediate must be an integer in the range";
+    Error(S, Msg + " [" + Twine(0) + ", " + Twine((1 << 7) - 1) + "]");
+    return MatchOperand_ParseFail;
+  }
+  }
+
+  return MatchOperand_NoMatch;
+}
+
+OperandMatchResultTy
 RISCVAsmParser::parseCSRSystemRegister(OperandVector &Operands) {
   SMLoc S = getLoc();
   const MCExpr *Res;
@@ -1381,7 +1445,7 @@ RISCVAsmParser::parseCSRSystemRegister(OperandVector &Operands) {
 
 OperandMatchResultTy RISCVAsmParser::parseImmediate(OperandVector &Operands) {
   SMLoc S = getLoc();
-  SMLoc E = SMLoc::getFromPointer(S.getPointer() - 1);
+  SMLoc E;
   const MCExpr *Res;
 
   switch (getLexer().getKind()) {
@@ -1396,7 +1460,7 @@ OperandMatchResultTy RISCVAsmParser::parseImmediate(OperandVector &Operands) {
   case AsmToken::Integer:
   case AsmToken::String:
   case AsmToken::Identifier:
-    if (getParser().parseExpression(Res))
+    if (getParser().parseExpression(Res, E))
       return MatchOperand_ParseFail;
     break;
   case AsmToken::Percent:
@@ -1410,7 +1474,7 @@ OperandMatchResultTy RISCVAsmParser::parseImmediate(OperandVector &Operands) {
 OperandMatchResultTy
 RISCVAsmParser::parseOperandWithModifier(OperandVector &Operands) {
   SMLoc S = getLoc();
-  SMLoc E = SMLoc::getFromPointer(S.getPointer() - 1);
+  SMLoc E;
 
   if (getLexer().getKind() != AsmToken::Percent) {
     Error(getLoc(), "expected '%' for operand modifier");
@@ -1449,7 +1513,6 @@ RISCVAsmParser::parseOperandWithModifier(OperandVector &Operands) {
 
 OperandMatchResultTy RISCVAsmParser::parseBareSymbol(OperandVector &Operands) {
   SMLoc S = getLoc();
-  SMLoc E = SMLoc::getFromPointer(S.getPointer() - 1);
   const MCExpr *Res;
 
   if (getLexer().getKind() != AsmToken::Identifier)
@@ -1461,6 +1524,8 @@ OperandMatchResultTy RISCVAsmParser::parseBareSymbol(OperandVector &Operands) {
   if (getParser().parseIdentifier(Identifier))
     return MatchOperand_ParseFail;
 
+  SMLoc E = SMLoc::getFromPointer(S.getPointer() + Identifier.size());
+
   if (Identifier.consume_back("@plt")) {
     Error(getLoc(), "'@plt' operand not valid for instruction");
     return MatchOperand_ParseFail;
@@ -1492,7 +1557,7 @@ OperandMatchResultTy RISCVAsmParser::parseBareSymbol(OperandVector &Operands) {
   }
 
   const MCExpr *Expr;
-  if (getParser().parseExpression(Expr))
+  if (getParser().parseExpression(Expr, E))
     return MatchOperand_ParseFail;
   Res = MCBinaryExpr::create(Opcode, Res, Expr, getContext());
   Operands.push_back(RISCVOperand::createImm(Res, S, E, isRV64()));
@@ -1501,7 +1566,6 @@ OperandMatchResultTy RISCVAsmParser::parseBareSymbol(OperandVector &Operands) {
 
 OperandMatchResultTy RISCVAsmParser::parseCallSymbol(OperandVector &Operands) {
   SMLoc S = getLoc();
-  SMLoc E = SMLoc::getFromPointer(S.getPointer() - 1);
   const MCExpr *Res;
 
   if (getLexer().getKind() != AsmToken::Identifier)
@@ -1515,6 +1579,8 @@ OperandMatchResultTy RISCVAsmParser::parseCallSymbol(OperandVector &Operands) {
   if (getParser().parseIdentifier(Identifier))
     return MatchOperand_ParseFail;
 
+  SMLoc E = SMLoc::getFromPointer(S.getPointer() + Identifier.size());
+
   RISCVMCExpr::VariantKind Kind = RISCVMCExpr::VK_RISCV_CALL;
   if (Identifier.consume_back("@plt"))
     Kind = RISCVMCExpr::VK_RISCV_CALL_PLT;
@@ -1529,10 +1595,10 @@ OperandMatchResultTy RISCVAsmParser::parseCallSymbol(OperandVector &Operands) {
 OperandMatchResultTy
 RISCVAsmParser::parsePseudoJumpSymbol(OperandVector &Operands) {
   SMLoc S = getLoc();
-  SMLoc E = SMLoc::getFromPointer(S.getPointer() - 1);
+  SMLoc E;
   const MCExpr *Res;
 
-  if (getParser().parseExpression(Res))
+  if (getParser().parseExpression(Res, E))
     return MatchOperand_ParseFail;
 
   if (Res->getKind() != MCExpr::ExprKind::SymbolRef ||
@@ -1662,7 +1728,7 @@ OperandMatchResultTy RISCVAsmParser::parseMaskReg(OperandVector &Operands) {
     if (RegNo != RISCV::V0)
       return MatchOperand_NoMatch;
     SMLoc S = getLoc();
-    SMLoc E = SMLoc::getFromPointer(S.getPointer() - 1);
+    SMLoc E = SMLoc::getFromPointer(S.getPointer() + Name.size());
     getLexer().Lex();
     Operands.push_back(RISCVOperand::createReg(RegNo, S, E, isRV64()));
   }
@@ -2062,7 +2128,11 @@ bool RISCVAsmParser::parseDirectiveAttribute() {
                         "unexpected token in '.attribute' directive"))
     return true;
 
-  if (Tag == RISCVAttrs::ARCH) {
+  if (IsIntegerValue)
+    getTargetStreamer().emitAttribute(Tag, IntegerValue);
+  else if (Tag != RISCVAttrs::ARCH)
+    getTargetStreamer().emitTextAttribute(Tag, StringValue);
+  else {
     StringRef Arch = StringValue;
     for (auto Feature : RISCVFeatureKV)
       if (llvm::RISCVISAInfo::isSupportedExtensionFeature(Feature.Key))
@@ -2070,7 +2140,7 @@ bool RISCVAsmParser::parseDirectiveAttribute() {
 
     auto ParseResult = llvm::RISCVISAInfo::parseArchString(
         StringValue, /*EnableExperimentalExtension=*/true,
-        /*ExperimentalExtensionVersionCheck=*/false);
+        /*ExperimentalExtensionVersionCheck=*/true);
     if (!ParseResult) {
       std::string Buffer;
       raw_string_ostream OutputErrMsg(Buffer);
@@ -2093,35 +2163,9 @@ bool RISCVAsmParser::parseDirectiveAttribute() {
       setFeatureBits(RISCV::Feature64Bit, "64bit");
     else
       return Error(ValueExprLoc, "bad arch string " + Arch);
-  }
 
-  if (IsIntegerValue)
-    getTargetStreamer().emitAttribute(Tag, IntegerValue);
-  else {
-    if (Tag != RISCVAttrs::ARCH) {
-      getTargetStreamer().emitTextAttribute(Tag, StringValue);
-    } else {
-      std::vector<std::string> FeatureVector;
-      RISCVFeatures::toFeatureVector(FeatureVector, getSTI().getFeatureBits());
-
-      // Parse that by RISCVISAInfo->
-      unsigned XLen = getFeatureBits(RISCV::Feature64Bit) ? 64 : 32;
-      auto ParseResult = llvm::RISCVISAInfo::parseFeatures(XLen, FeatureVector);
-      if (!ParseResult) {
-        std::string Buffer;
-        raw_string_ostream OutputErrMsg(Buffer);
-        handleAllErrors(ParseResult.takeError(),
-                        [&](llvm::StringError &ErrMsg) {
-                          OutputErrMsg << ErrMsg.getMessage();
-                        });
-
-        return Error(ValueExprLoc, OutputErrMsg.str());
-      }
-      auto &ISAInfo = *ParseResult;
-
-      // Then emit the arch string.
-      getTargetStreamer().emitTextAttribute(Tag, ISAInfo->toString());
-    }
+    // Then emit the arch string.
+    getTargetStreamer().emitTextAttribute(Tag, ISAInfo->toString());
   }
 
   return false;
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp
index 0aba18b20f0d..144e761f002d 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp
@@ -27,6 +27,11 @@ namespace RISCVSysReg {
 #include "RISCVGenSearchableTables.inc"
 } // namespace RISCVSysReg
 
+namespace RISCVInsnOpcode {
+#define GET_RISCVOpcodesList_IMPL
+#include "RISCVGenSearchableTables.inc"
+} // namespace RISCVInsnOpcode
+
 namespace RISCVABI {
 ABI computeTargetABI(const Triple &TT, FeatureBitset FeatureBits,
                      StringRef ABIName) {
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
index d8f4403c824f..9cfd36745f46 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
@@ -299,6 +299,16 @@ struct SysReg {
 #include "RISCVGenSearchableTables.inc"
 } // end namespace RISCVSysReg
 
+namespace RISCVInsnOpcode {
+struct RISCVOpcode {
+  const char *Name;
+  unsigned Value;
+};
+
+#define GET_RISCVOpcodesList_DECL
+#include "RISCVGenSearchableTables.inc"
+} // end namespace RISCVInsnOpcode
+
 namespace RISCVABI {
 
 enum ABI {
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp
index f1c3810f4ee5..89a7d54f60f8 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp
@@ -171,9 +171,9 @@ void RISCVInstPrinter::printVTypeI(const MCInst *MI, unsigned OpNo,
                                    const MCSubtargetInfo &STI, raw_ostream &O) {
   unsigned Imm = MI->getOperand(OpNo).getImm();
   // Print the raw immediate for reserved values: vlmul[2:0]=4, vsew[2:0]=0b1xx,
-  // or non-zero bits 8/9/10.
+  // or non-zero in bits 8 and above.
   if (RISCVVType::getVLMUL(Imm) == RISCVII::VLMUL::LMUL_RESERVED ||
-      RISCVVType::getSEW(Imm) > 64 || (Imm & 0x700) != 0) {
+      RISCVVType::getSEW(Imm) > 64 || (Imm >> 8) != 0) {
     O << Imm;
     return;
   }
diff --git a/llvm/lib/Target/RISCV/RISCV.td b/llvm/lib/Target/RISCV/RISCV.td
index 772a4f8ecd53..6aa915c01929 100644
--- a/llvm/lib/Target/RISCV/RISCV.td
+++ b/llvm/lib/Target/RISCV/RISCV.td
@@ -168,14 +168,6 @@ def HasStdExtZvlsseg : Predicate<"Subtarget->hasStdExtZvlsseg()">,
                                  AssemblerPredicate<(all_of FeatureStdExtZvlsseg),
                                  "'Zvlsseg' (Vector segment load/store instructions)">;
 
-def FeatureStdExtZvamo
-    : SubtargetFeature<"experimental-zvamo", "HasStdExtZvamo", "true",
-                       "'Zvamo' (Vector AMO Operations)",
-                       [FeatureStdExtV]>;
-def HasStdExtZvamo : Predicate<"Subtarget->hasStdExtZvamo()">,
-                               AssemblerPredicate<(all_of FeatureStdExtZvamo),
-                               "'Zvamo' (Vector AMO Operations)">;
-
 def Feature64Bit
     : SubtargetFeature<"64bit", "HasRV64", "true", "Implements RV64">;
 def IsRV64 : Predicate<"Subtarget->is64Bit()">,
diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
index 66a34d73dd37..b24eb5f7bbf4 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
@@ -718,6 +718,71 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
 
     break;
   }
+  case ISD::MUL: {
+    // Special case for calculating (mul (and X, C2), C1) where the full product
+    // fits in XLen bits. We can shift X left by the number of leading zeros in
+    // C2 and shift C1 left by XLen-lzcnt(C2). This will ensure the final
+    // product has XLen trailing zeros, putting it in the output of MULHU. This
+    // can avoid materializing a constant in a register for C2.
+
+    // RHS should be a constant.
+    auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
+    if (!N1C || !N1C->hasOneUse())
+      break;
+
+    // LHS should be an AND with constant.
+    SDValue N0 = Node->getOperand(0);
+    if (N0.getOpcode() != ISD::AND || !isa<ConstantSDNode>(N0.getOperand(1)))
+      break;
+
+    uint64_t C2 = cast<ConstantSDNode>(N0.getOperand(1))->getZExtValue();
+
+    // Constant should be a mask.
+    if (!isMask_64(C2))
+      break;
+
+    // This should be the only use of the AND unless we will use
+    // (SRLI (SLLI X, 32), 32). We don't use a shift pair for other AND
+    // constants.
+    if (!N0.hasOneUse() && C2 != UINT64_C(0xFFFFFFFF))
+      break;
+
+    // If this can be an ANDI, ZEXT.H or ZEXT.W we don't need to do this
+    // optimization.
+    if (isInt<12>(C2) ||
+        (C2 == UINT64_C(0xFFFF) &&
+         (Subtarget->hasStdExtZbb() || Subtarget->hasStdExtZbp())) ||
+        (C2 == UINT64_C(0xFFFFFFFF) && Subtarget->hasStdExtZba()))
+      break;
+
+    // We need to shift left the AND input and C1 by a total of XLen bits.
+
+    // How far left do we need to shift the AND input?
+    unsigned XLen = Subtarget->getXLen();
+    unsigned LeadingZeros = XLen - (64 - countLeadingZeros(C2));
+
+    // The constant gets shifted by the remaining amount unless that would
+    // shift bits out.
+    uint64_t C1 = N1C->getZExtValue();
+    unsigned ConstantShift = XLen - LeadingZeros;
+    if (ConstantShift > (XLen - (64 - countLeadingZeros(C1))))
+      break;
+
+    uint64_t ShiftedC1 = C1 << ConstantShift;
+    // If this RV32, we need to sign extend the constant.
+    if (XLen == 32)
+      ShiftedC1 = SignExtend64(ShiftedC1, 32);
+
+    // Create (mulhu (slli X, lzcnt(C2)), C1 << (XLen - lzcnt(C2))).
+    SDNode *Imm = selectImm(CurDAG, DL, ShiftedC1, *Subtarget);
+    SDNode *SLLI =
+        CurDAG->getMachineNode(RISCV::SLLI, DL, VT, N0.getOperand(0),
+                               CurDAG->getTargetConstant(LeadingZeros, DL, VT));
+    SDNode *MULHU = CurDAG->getMachineNode(RISCV::MULHU, DL, VT,
+                                           SDValue(SLLI, 0), SDValue(Imm, 0));
+    ReplaceNode(Node, MULHU);
+    return;
+  }
   case ISD::INTRINSIC_WO_CHAIN: {
     unsigned IntNo = Node->getConstantOperandVal(0);
     switch (IntNo) {
@@ -1450,6 +1515,7 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
     ReplaceNode(Node, Extract.getNode());
     return;
   }
+  case ISD::SPLAT_VECTOR:
   case RISCVISD::VMV_V_X_VL:
   case RISCVISD::VFMV_V_F_VL: {
     // Try to match splat of a scalar load to a strided load with stride of x0.
@@ -1466,7 +1532,10 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
       break;
 
     SDValue VL;
-    selectVLOp(Node->getOperand(1), VL);
+    if (Node->getOpcode() == ISD::SPLAT_VECTOR)
+      VL = CurDAG->getTargetConstant(RISCV::VLMaxSentinel, DL, XLenVT);
+    else
+      selectVLOp(Node->getOperand(1), VL);
 
     unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
     SDValue SEW = CurDAG->getTargetConstant(Log2SEW, DL, XLenVT);
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index f3331571fc55..4f5512e6fb37 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -330,6 +330,14 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::LLRINT, MVT::f16, Legal);
     setOperationAction(ISD::LROUND, MVT::f16, Legal);
     setOperationAction(ISD::LLROUND, MVT::f16, Legal);
+    setOperationAction(ISD::STRICT_FADD, MVT::f16, Legal);
+    setOperationAction(ISD::STRICT_FMA, MVT::f16, Legal);
+    setOperationAction(ISD::STRICT_FSUB, MVT::f16, Legal);
+    setOperationAction(ISD::STRICT_FMUL, MVT::f16, Legal);
+    setOperationAction(ISD::STRICT_FDIV, MVT::f16, Legal);
+    setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Legal);
+    setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Legal);
+    setOperationAction(ISD::STRICT_FSQRT, MVT::f16, Legal);
     for (auto CC : FPCCToExpand)
       setCondCodeAction(CC, MVT::f16, Expand);
     setOperationAction(ISD::SELECT_CC, MVT::f16, Expand);
@@ -367,6 +375,12 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::LLRINT, MVT::f32, Legal);
     setOperationAction(ISD::LROUND, MVT::f32, Legal);
     setOperationAction(ISD::LLROUND, MVT::f32, Legal);
+    setOperationAction(ISD::STRICT_FADD, MVT::f32, Legal);
+    setOperationAction(ISD::STRICT_FMA, MVT::f32, Legal);
+    setOperationAction(ISD::STRICT_FSUB, MVT::f32, Legal);
+    setOperationAction(ISD::STRICT_FMUL, MVT::f32, Legal);
+    setOperationAction(ISD::STRICT_FDIV, MVT::f32, Legal);
+    setOperationAction(ISD::STRICT_FSQRT, MVT::f32, Legal);
     for (auto CC : FPCCToExpand)
       setCondCodeAction(CC, MVT::f32, Expand);
     setOperationAction(ISD::SELECT_CC, MVT::f32, Expand);
@@ -388,6 +402,14 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::LLRINT, MVT::f64, Legal);
     setOperationAction(ISD::LROUND, MVT::f64, Legal);
     setOperationAction(ISD::LLROUND, MVT::f64, Legal);
+    setOperationAction(ISD::STRICT_FMA, MVT::f64, Legal);
+    setOperationAction(ISD::STRICT_FADD, MVT::f64, Legal);
+    setOperationAction(ISD::STRICT_FSUB, MVT::f64, Legal);
+    setOperationAction(ISD::STRICT_FMUL, MVT::f64, Legal);
+    setOperationAction(ISD::STRICT_FDIV, MVT::f64, Legal);
+    setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Legal);
+    setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Legal);
+    setOperationAction(ISD::STRICT_FSQRT, MVT::f64, Legal);
     for (auto CC : FPCCToExpand)
       setCondCodeAction(CC, MVT::f64, Expand);
     setOperationAction(ISD::SELECT_CC, MVT::f64, Expand);
@@ -412,6 +434,11 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::FP_TO_UINT_SAT, XLenVT, Custom);
     setOperationAction(ISD::FP_TO_SINT_SAT, XLenVT, Custom);
 
+    setOperationAction(ISD::STRICT_FP_TO_UINT, XLenVT, Legal);
+    setOperationAction(ISD::STRICT_FP_TO_SINT, XLenVT, Legal);
+    setOperationAction(ISD::STRICT_UINT_TO_FP, XLenVT, Legal);
+    setOperationAction(ISD::STRICT_SINT_TO_FP, XLenVT, Legal);
+
     setOperationAction(ISD::FLT_ROUNDS_, XLenVT, Custom);
     setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom);
   }
@@ -471,12 +498,13 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
         ISD::VP_XOR,         ISD::VP_ASHR,        ISD::VP_LSHR,
         ISD::VP_SHL,         ISD::VP_REDUCE_ADD,  ISD::VP_REDUCE_AND,
         ISD::VP_REDUCE_OR,   ISD::VP_REDUCE_XOR,  ISD::VP_REDUCE_SMAX,
-        ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN};
+        ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN,
+        ISD::VP_SELECT};
 
     static const unsigned FloatingPointVPOps[] = {
         ISD::VP_FADD,        ISD::VP_FSUB,        ISD::VP_FMUL,
         ISD::VP_FDIV,        ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
-        ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX};
+        ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_SELECT};
 
     if (!Subtarget.is64Bit()) {
       // We must custom-lower certain vXi64 operations on RV32 due to the vector
@@ -519,6 +547,10 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
       setOperationAction(ISD::SELECT_CC, VT, Expand);
       setOperationAction(ISD::VSELECT, VT, Expand);
 
+      setOperationAction(ISD::VP_AND, VT, Custom);
+      setOperationAction(ISD::VP_OR, VT, Custom);
+      setOperationAction(ISD::VP_XOR, VT, Custom);
+
       setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
       setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
       setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
@@ -803,6 +835,9 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
 
         // Operations below are different for between masks and other vectors.
         if (VT.getVectorElementType() == MVT::i1) {
+          setOperationAction(ISD::VP_AND, VT, Custom);
+          setOperationAction(ISD::VP_OR, VT, Custom);
+          setOperationAction(ISD::VP_XOR, VT, Custom);
           setOperationAction(ISD::AND, VT, Custom);
           setOperationAction(ISD::OR, VT, Custom);
           setOperationAction(ISD::XOR, VT, Custom);
@@ -1147,7 +1182,7 @@ bool RISCVTargetLowering::isCheapToSpeculateCtlz() const {
   return Subtarget.hasStdExtZbb();
 }
 
-bool RISCVTargetLowering::hasAndNot(SDValue Y) const {
+bool RISCVTargetLowering::hasAndNotCompare(SDValue Y) const {
   EVT VT = Y.getValueType();
 
   // FIXME: Support vectors once we have tests.
@@ -1235,7 +1270,8 @@ bool RISCVTargetLowering::shouldSinkOperands(
 
 bool RISCVTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
                                        bool ForCodeSize) const {
-  if (VT == MVT::f16 && !Subtarget.hasStdExtZfhmin())
+  // FIXME: Change to Zfhmin once f16 becomes a legal type with Zfhmin.
+  if (VT == MVT::f16 && !Subtarget.hasStdExtZfh())
     return false;
   if (VT == MVT::f32 && !Subtarget.hasStdExtF())
     return false;
@@ -1255,9 +1291,10 @@ bool RISCVTargetLowering::hasBitPreservingFPLogic(EVT VT) const {
 MVT RISCVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
                                                       CallingConv::ID CC,
                                                       EVT VT) const {
-  // Use f32 to pass f16 if it is legal and Zfhmin/Zfh is not enabled.
+  // Use f32 to pass f16 if it is legal and Zfh is not enabled.
   // We might still end up using a GPR but that will be decided based on ABI.
-  if (VT == MVT::f16 && Subtarget.hasStdExtF() && !Subtarget.hasStdExtZfhmin())
+  // FIXME: Change to Zfhmin once f16 becomes a legal type with Zfhmin.
+  if (VT == MVT::f16 && Subtarget.hasStdExtF() && !Subtarget.hasStdExtZfh())
     return MVT::f32;
 
   return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
@@ -1266,9 +1303,10 @@ MVT RISCVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
 unsigned RISCVTargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
                                                            CallingConv::ID CC,
                                                            EVT VT) const {
-  // Use f32 to pass f16 if it is legal and Zfhmin/Zfh is not enabled.
+  // Use f32 to pass f16 if it is legal and Zfh is not enabled.
   // We might still end up using a GPR but that will be decided based on ABI.
-  if (VT == MVT::f16 && Subtarget.hasStdExtF() && !Subtarget.hasStdExtZfhmin())
+  // FIXME: Change to Zfhmin once f16 becomes a legal type with Zfhmin.
+  if (VT == MVT::f16 && Subtarget.hasStdExtF() && !Subtarget.hasStdExtZfh())
     return 1;
 
   return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
@@ -1959,29 +1997,37 @@ static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
     int64_t StepNumerator = SimpleVID->StepNumerator;
     unsigned StepDenominator = SimpleVID->StepDenominator;
     int64_t Addend = SimpleVID->Addend;
+
+    assert(StepNumerator != 0 && "Invalid step");
+    bool Negate = false;
+    int64_t SplatStepVal = StepNumerator;
+    unsigned StepOpcode = ISD::MUL;
+    if (StepNumerator != 1) {
+      if (isPowerOf2_64(std::abs(StepNumerator))) {
+        Negate = StepNumerator < 0;
+        StepOpcode = ISD::SHL;
+        SplatStepVal = Log2_64(std::abs(StepNumerator));
+      }
+    }
+
     // Only emit VIDs with suitably-small steps/addends. We use imm5 is a
     // threshold since it's the immediate value many RVV instructions accept.
-    if (isInt<5>(StepNumerator) && isPowerOf2_32(StepDenominator) &&
-        isInt<5>(Addend)) {
+    // There is no vmul.vi instruction so ensure multiply constant can fit in
+    // a single addi instruction.
+    if (((StepOpcode == ISD::MUL && isInt<12>(SplatStepVal)) ||
+         (StepOpcode == ISD::SHL && isUInt<5>(SplatStepVal))) &&
+        isPowerOf2_32(StepDenominator) && isInt<5>(Addend)) {
       SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, ContainerVT, Mask, VL);
       // Convert right out of the scalable type so we can use standard ISD
       // nodes for the rest of the computation. If we used scalable types with
       // these, we'd lose the fixed-length vector info and generate worse
       // vsetvli code.
       VID = convertFromScalableVector(VT, VID, DAG, Subtarget);
-      assert(StepNumerator != 0 && "Invalid step");
-      bool Negate = false;
-      if (StepNumerator != 1) {
-        int64_t SplatStepVal = StepNumerator;
-        unsigned Opcode = ISD::MUL;
-        if (isPowerOf2_64(std::abs(StepNumerator))) {
-          Negate = StepNumerator < 0;
-          Opcode = ISD::SHL;
-          SplatStepVal = Log2_64(std::abs(StepNumerator));
-        }
+      if ((StepOpcode == ISD::MUL && SplatStepVal != 1) ||
+          (StepOpcode == ISD::SHL && SplatStepVal != 0)) {
         SDValue SplatStep = DAG.getSplatVector(
             VT, DL, DAG.getConstant(SplatStepVal, DL, XLenVT));
-        VID = DAG.getNode(Opcode, DL, VT, VID, SplatStep);
+        VID = DAG.getNode(StepOpcode, DL, VT, VID, SplatStep);
       }
       if (StepDenominator != 1) {
         SDValue SplatStep = DAG.getSplatVector(
@@ -3133,6 +3179,8 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
     return lowerGET_ROUNDING(Op, DAG);
   case ISD::SET_ROUNDING:
     return lowerSET_ROUNDING(Op, DAG);
+  case ISD::VP_SELECT:
+    return lowerVPOp(Op, DAG, RISCVISD::VSELECT_VL);
   case ISD::VP_ADD:
     return lowerVPOp(Op, DAG, RISCVISD::ADD_VL);
   case ISD::VP_SUB:
@@ -3148,11 +3196,11 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
   case ISD::VP_UREM:
     return lowerVPOp(Op, DAG, RISCVISD::UREM_VL);
   case ISD::VP_AND:
-    return lowerVPOp(Op, DAG, RISCVISD::AND_VL);
+    return lowerLogicVPOp(Op, DAG, RISCVISD::VMAND_VL, RISCVISD::AND_VL);
   case ISD::VP_OR:
-    return lowerVPOp(Op, DAG, RISCVISD::OR_VL);
+    return lowerLogicVPOp(Op, DAG, RISCVISD::VMOR_VL, RISCVISD::OR_VL);
   case ISD::VP_XOR:
-    return lowerVPOp(Op, DAG, RISCVISD::XOR_VL);
+    return lowerLogicVPOp(Op, DAG, RISCVISD::VMXOR_VL, RISCVISD::XOR_VL);
   case ISD::VP_ASHR:
     return lowerVPOp(Op, DAG, RISCVISD::SRA_VL);
   case ISD::VP_LSHR:
@@ -4469,19 +4517,19 @@ SDValue RISCVTargetLowering::lowerVECREDUCE(SDValue Op,
   }
 
   MVT M1VT = getLMUL1VT(ContainerVT);
+  MVT XLenVT = Subtarget.getXLenVT();
 
   SDValue Mask, VL;
   std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
 
-  // FIXME: This is a VLMAX splat which might be too large and can prevent
-  // vsetvli removal.
   SDValue NeutralElem =
       DAG.getNeutralElement(BaseOpc, DL, VecEltVT, SDNodeFlags());
-  SDValue IdentitySplat = DAG.getSplatVector(M1VT, DL, NeutralElem);
+  SDValue IdentitySplat = lowerScalarSplat(
+      NeutralElem, DAG.getConstant(1, DL, XLenVT), M1VT, DL, DAG, Subtarget);
   SDValue Reduction = DAG.getNode(RVVOpcode, DL, M1VT, DAG.getUNDEF(M1VT), Vec,
                                   IdentitySplat, Mask, VL);
   SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VecEltVT, Reduction,
-                             DAG.getConstant(0, DL, Subtarget.getXLenVT()));
+                             DAG.getConstant(0, DL, XLenVT));
   return DAG.getSExtOrTrunc(Elt0, DL, Op.getValueType());
 }
 
@@ -4497,9 +4545,12 @@ getRVVFPReductionOpAndOperands(SDValue Op, SelectionDAG &DAG, EVT EltVT) {
   switch (Opcode) {
   default:
     llvm_unreachable("Unhandled reduction");
-  case ISD::VECREDUCE_FADD:
-    return std::make_tuple(RISCVISD::VECREDUCE_FADD_VL, Op.getOperand(0),
-                           DAG.getNeutralElement(BaseOpcode, DL, EltVT, Flags));
+  case ISD::VECREDUCE_FADD: {
+    // Use positive zero if we can. It is cheaper to materialize.
+    SDValue Zero =
+        DAG.getConstantFP(Flags.hasNoSignedZeros() ? 0.0 : -0.0, DL, EltVT);
+    return std::make_tuple(RISCVISD::VECREDUCE_FADD_VL, Op.getOperand(0), Zero);
+  }
   case ISD::VECREDUCE_SEQ_FADD:
     return std::make_tuple(RISCVISD::VECREDUCE_SEQ_FADD_VL, Op.getOperand(1),
                            Op.getOperand(0));
@@ -4530,17 +4581,17 @@ SDValue RISCVTargetLowering::lowerFPVECREDUCE(SDValue Op,
   }
 
   MVT M1VT = getLMUL1VT(VectorVal.getSimpleValueType());
+  MVT XLenVT = Subtarget.getXLenVT();
 
   SDValue Mask, VL;
   std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
 
-  // FIXME: This is a VLMAX splat which might be too large and can prevent
-  // vsetvli removal.
-  SDValue ScalarSplat = DAG.getSplatVector(M1VT, DL, ScalarVal);
+  SDValue ScalarSplat = lowerScalarSplat(
+      ScalarVal, DAG.getConstant(1, DL, XLenVT), M1VT, DL, DAG, Subtarget);
   SDValue Reduction = DAG.getNode(RVVOpcode, DL, M1VT, DAG.getUNDEF(M1VT),
                                   VectorVal, ScalarSplat, Mask, VL);
   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VecEltVT, Reduction,
-                     DAG.getConstant(0, DL, Subtarget.getXLenVT()));
+                     DAG.getConstant(0, DL, XLenVT));
 }
 
 static unsigned getRVVVPReductionOp(unsigned ISDOpcode) {
@@ -4602,13 +4653,13 @@ SDValue RISCVTargetLowering::lowerVPREDUCE(SDValue Op,
   MVT XLenVT = Subtarget.getXLenVT();
   MVT ResVT = !VecVT.isInteger() || VecEltVT.bitsGE(XLenVT) ? VecEltVT : XLenVT;
 
-  // FIXME: This is a VLMAX splat which might be too large and can prevent
-  // vsetvli removal.
-  SDValue StartSplat = DAG.getSplatVector(M1VT, DL, Op.getOperand(0));
+  SDValue StartSplat =
+      lowerScalarSplat(Op.getOperand(0), DAG.getConstant(1, DL, XLenVT), M1VT,
+                       DL, DAG, Subtarget);
   SDValue Reduction =
       DAG.getNode(RVVOpcode, DL, M1VT, StartSplat, Vec, StartSplat, Mask, VL);
   SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResVT, Reduction,
-                             DAG.getConstant(0, DL, Subtarget.getXLenVT()));
+                             DAG.getConstant(0, DL, XLenVT));
   if (!VecVT.isInteger())
     return Elt0;
   return DAG.getSExtOrTrunc(Elt0, DL, Op.getValueType());
@@ -5365,6 +5416,33 @@ SDValue RISCVTargetLowering::lowerVPOp(SDValue Op, SelectionDAG &DAG,
   return convertFromScalableVector(VT, VPOp, DAG, Subtarget);
 }
 
+SDValue RISCVTargetLowering::lowerLogicVPOp(SDValue Op, SelectionDAG &DAG,
+                                            unsigned MaskOpc,
+                                            unsigned VecOpc) const {
+  MVT VT = Op.getSimpleValueType();
+  if (VT.getVectorElementType() != MVT::i1)
+    return lowerVPOp(Op, DAG, VecOpc);
+
+  // It is safe to drop mask parameter as masked-off elements are undef.
+  SDValue Op1 = Op->getOperand(0);
+  SDValue Op2 = Op->getOperand(1);
+  SDValue VL = Op->getOperand(3);
+
+  MVT ContainerVT = VT;
+  const bool IsFixed = VT.isFixedLengthVector();
+  if (IsFixed) {
+    ContainerVT = getContainerForFixedLengthVector(VT);
+    Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget);
+    Op2 = convertToScalableVector(ContainerVT, Op2, DAG, Subtarget);
+  }
+
+  SDLoc DL(Op);
+  SDValue Val = DAG.getNode(MaskOpc, DL, ContainerVT, Op1, Op2, VL);
+  if (!IsFixed)
+    return Val;
+  return convertFromScalableVector(VT, Val, DAG, Subtarget);
+}
+
 // Custom lower MGATHER/VP_GATHER to a legalized form for RVV. It will then be
 // matched to a RVV indexed load. The RVV indexed load instructions only
 // support the "unsigned unscaled" addressing mode; indices are implicitly
@@ -5695,11 +5773,17 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
     SDValue Op0 = IsStrict ? N->getOperand(1) : N->getOperand(0);
     if (getTypeAction(*DAG.getContext(), Op0.getValueType()) !=
         TargetLowering::TypeSoftenFloat) {
-      // FIXME: Support strict FP.
-      if (IsStrict)
-        return;
       if (!isTypeLegal(Op0.getValueType()))
         return;
+      if (IsStrict) {
+        unsigned Opc = IsSigned ? RISCVISD::STRICT_FCVT_W_RTZ_RV64
+                                : RISCVISD::STRICT_FCVT_WU_RTZ_RV64;
+        SDVTList VTs = DAG.getVTList(MVT::i64, MVT::Other);
+        SDValue Res = DAG.getNode(Opc, DL, VTs, N->getOperand(0), Op0);
+        Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
+        Results.push_back(Res.getValue(1));
+        return;
+      }
       unsigned Opc =
           IsSigned ? RISCVISD::FCVT_W_RTZ_RV64 : RISCVISD::FCVT_WU_RTZ_RV64;
       SDValue Res = DAG.getNode(Opc, DL, MVT::i64, Op0);
@@ -7026,7 +7110,7 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
     if (SimplifyDemandedLowBitsHelper(1, Log2_32(BitWidth)))
       return SDValue(N, 0);
 
-    return combineGREVI_GORCI(N, DCI.DAG);
+    return combineGREVI_GORCI(N, DAG);
   }
   case RISCVISD::GREVW:
   case RISCVISD::GORCW: {
@@ -7035,7 +7119,7 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
         SimplifyDemandedLowBitsHelper(1, 5))
       return SDValue(N, 0);
 
-    return combineGREVI_GORCI(N, DCI.DAG);
+    return combineGREVI_GORCI(N, DAG);
   }
   case RISCVISD::SHFL:
   case RISCVISD::UNSHFL: {
@@ -7120,11 +7204,23 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
     // Fold (zero_extend (fp_to_uint X)) to prevent forming fcvt+zexti32 during
     // type legalization. This is safe because fp_to_uint produces poison if
     // it overflows.
-    if (N->getValueType(0) == MVT::i64 && Subtarget.is64Bit() &&
-        N->getOperand(0).getOpcode() == ISD::FP_TO_UINT &&
-        isTypeLegal(N->getOperand(0).getOperand(0).getValueType()))
-      return DAG.getNode(ISD::FP_TO_UINT, SDLoc(N), MVT::i64,
-                         N->getOperand(0).getOperand(0));
+    if (N->getValueType(0) == MVT::i64 && Subtarget.is64Bit()) {
+      SDValue Src = N->getOperand(0);
+      if (Src.getOpcode() == ISD::FP_TO_UINT &&
+          isTypeLegal(Src.getOperand(0).getValueType()))
+        return DAG.getNode(ISD::FP_TO_UINT, SDLoc(N), MVT::i64,
+                           Src.getOperand(0));
+      if (Src.getOpcode() == ISD::STRICT_FP_TO_UINT && Src.hasOneUse() &&
+          isTypeLegal(Src.getOperand(1).getValueType())) {
+        SDVTList VTs = DAG.getVTList(MVT::i64, MVT::Other);
+        SDValue Res = DAG.getNode(ISD::STRICT_FP_TO_UINT, SDLoc(N), VTs,
+                                  Src.getOperand(0), Src.getOperand(1));
+        DCI.CombineTo(N, Res);
+        DAG.ReplaceAllUsesOfValueWith(Src.getValue(1), Res.getValue(1));
+        DCI.recursivelyDeleteUnusedNodes(Src.getNode());
+        return SDValue(N, 0); // Return N so it doesn't get rechecked.
+      }
+    }
     return SDValue();
   case RISCVISD::SELECT_CC: {
     // Transform
@@ -7685,6 +7781,8 @@ unsigned RISCVTargetLowering::ComputeNumSignBitsForTargetNode(
   case RISCVISD::BDECOMPRESSW:
   case RISCVISD::FCVT_W_RTZ_RV64:
   case RISCVISD::FCVT_WU_RTZ_RV64:
+  case RISCVISD::STRICT_FCVT_W_RTZ_RV64:
+  case RISCVISD::STRICT_FCVT_WU_RTZ_RV64:
     // TODO: As the result is sign-extended, this is conservatively correct. A
     // more precise answer could be calculated for SRAW depending on known
     // bits in the shift amount.
@@ -8004,6 +8102,22 @@ RISCVTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
   }
 }
 
+void RISCVTargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
+                                                        SDNode *Node) const {
+  // Add FRM dependency to any instructions with dynamic rounding mode.
+  unsigned Opc = MI.getOpcode();
+  auto Idx = RISCV::getNamedOperandIdx(Opc, RISCV::OpName::frm);
+  if (Idx < 0)
+    return;
+  if (MI.getOperand(Idx).getImm() != RISCVFPRndMode::DYN)
+    return;
+  // If the instruction already reads FRM, don't add another read.
+  if (MI.readsRegister(RISCV::FRM))
+    return;
+  MI.addOperand(
+      MachineOperand::CreateReg(RISCV::FRM, /*isDef*/ false, /*isImp*/ true));
+}
+
 // Calling Convention Implementation.
 // The expectations for frontend ABI lowering vary from target to target.
 // Ideally, an LLVM frontend would be able to avoid worrying about many ABI
@@ -9400,6 +9514,8 @@ const char *RISCVTargetLowering::getTargetNodeName(unsigned Opcode) const {
   NODE_NAME_CASE(FCVT_XU_RTZ)
   NODE_NAME_CASE(FCVT_W_RTZ_RV64)
   NODE_NAME_CASE(FCVT_WU_RTZ_RV64)
+  NODE_NAME_CASE(STRICT_FCVT_W_RTZ_RV64)
+  NODE_NAME_CASE(STRICT_FCVT_WU_RTZ_RV64)
   NODE_NAME_CASE(READ_CYCLE_WIDE)
   NODE_NAME_CASE(GREV)
   NODE_NAME_CASE(GREVW)
@@ -9541,6 +9657,9 @@ RISCVTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
   if (Constraint.size() == 1) {
     switch (Constraint[0]) {
     case 'r':
+      // TODO: Support fixed vectors up to XLen for P extension?
+      if (VT.isVector())
+        break;
       return std::make_pair(0U, &RISCV::GPRRegClass);
     case 'f':
       if (Subtarget.hasStdExtZfh() && VT == MVT::f16)
@@ -9553,17 +9672,15 @@ RISCVTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
     default:
       break;
     }
-  } else {
-    if (Constraint == "vr") {
-      for (const auto *RC : {&RISCV::VRRegClass, &RISCV::VRM2RegClass,
-                             &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
-        if (TRI->isTypeLegalForClass(*RC, VT.SimpleTy))
-          return std::make_pair(0U, RC);
-      }
-    } else if (Constraint == "vm") {
-      if (TRI->isTypeLegalForClass(RISCV::VMRegClass, VT.SimpleTy))
-        return std::make_pair(0U, &RISCV::VMRegClass);
+  } else if (Constraint == "vr") {
+    for (const auto *RC : {&RISCV::VRRegClass, &RISCV::VRM2RegClass,
+                           &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
+      if (TRI->isTypeLegalForClass(*RC, VT.SimpleTy))
+        return std::make_pair(0U, RC);
     }
+  } else if (Constraint == "vm") {
+    if (TRI->isTypeLegalForClass(RISCV::VMV0RegClass, VT.SimpleTy))
+      return std::make_pair(0U, &RISCV::VMV0RegClass);
   }
 
   // Clang will correctly decode the usage of register name aliases into their
@@ -10101,17 +10218,29 @@ bool RISCVTargetLowering::splitValueIntoRegisterParts(
     unsigned ValueVTBitSize = ValueVT.getSizeInBits().getKnownMinSize();
     unsigned PartVTBitSize = PartVT.getSizeInBits().getKnownMinSize();
     if (PartVTBitSize % ValueVTBitSize == 0) {
+      assert(PartVTBitSize >= ValueVTBitSize);
       // If the element types are different, bitcast to the same element type of
       // PartVT first.
+      // Give an example here, we want copy a <vscale x 1 x i8> value to
+      // <vscale x 4 x i16>.
+      // We need to convert <vscale x 1 x i8> to <vscale x 8 x i8> by insert
+      // subvector, then we can bitcast to <vscale x 4 x i16>.
       if (ValueEltVT != PartEltVT) {
-        unsigned Count = ValueVTBitSize / PartEltVT.getSizeInBits();
-        assert(Count != 0 && "The number of element should not be zero.");
-        EVT SameEltTypeVT =
-            EVT::getVectorVT(Context, PartEltVT, Count, /*IsScalable=*/true);
-        Val = DAG.getNode(ISD::BITCAST, DL, SameEltTypeVT, Val);
+        if (PartVTBitSize > ValueVTBitSize) {
+          unsigned Count = PartVTBitSize / ValueEltVT.getFixedSizeInBits();
+          assert(Count != 0 && "The number of element should not be zero.");
+          EVT SameEltTypeVT =
+              EVT::getVectorVT(Context, ValueEltVT, Count, /*IsScalable=*/true);
+          Val = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SameEltTypeVT,
+                            DAG.getUNDEF(SameEltTypeVT), Val,
+                            DAG.getVectorIdxConstant(0, DL));
+        }
+        Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
+      } else {
+        Val =
+            DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT),
+                        Val, DAG.getVectorIdxConstant(0, DL));
       }
-      Val = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT),
-                        Val, DAG.getConstant(0, DL, Subtarget.getXLenVT()));
       Parts[0] = Val;
       return true;
     }
@@ -10141,19 +10270,23 @@ SDValue RISCVTargetLowering::joinRegisterPartsIntoValue(
     unsigned ValueVTBitSize = ValueVT.getSizeInBits().getKnownMinSize();
     unsigned PartVTBitSize = PartVT.getSizeInBits().getKnownMinSize();
     if (PartVTBitSize % ValueVTBitSize == 0) {
+      assert(PartVTBitSize >= ValueVTBitSize);
       EVT SameEltTypeVT = ValueVT;
       // If the element types are different, convert it to the same element type
       // of PartVT.
+      // Give an example here, we want copy a <vscale x 1 x i8> value from
+      // <vscale x 4 x i16>.
+      // We need to convert <vscale x 4 x i16> to <vscale x 8 x i8> first,
+      // then we can extract <vscale x 1 x i8>.
       if (ValueEltVT != PartEltVT) {
-        unsigned Count = ValueVTBitSize / PartEltVT.getSizeInBits();
+        unsigned Count = PartVTBitSize / ValueEltVT.getFixedSizeInBits();
         assert(Count != 0 && "The number of element should not be zero.");
         SameEltTypeVT =
-            EVT::getVectorVT(Context, PartEltVT, Count, /*IsScalable=*/true);
+            EVT::getVectorVT(Context, ValueEltVT, Count, /*IsScalable=*/true);
+        Val = DAG.getNode(ISD::BITCAST, DL, SameEltTypeVT, Val);
       }
-      Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SameEltTypeVT, Val,
-                        DAG.getConstant(0, DL, Subtarget.getXLenVT()));
-      if (ValueEltVT != PartEltVT)
-        Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
+      Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
+                        DAG.getVectorIdxConstant(0, DL));
       return Val;
     }
   }
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index 849928eb46ae..48c5ce730933 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -282,6 +282,11 @@ enum NodeType : unsigned {
   // the value read before the modification and the new chain pointer.
   SWAP_CSR,
 
+  // FP to 32 bit int conversions for RV64. These are used to keep track of the
+  // result being sign extended to 64 bit. These saturate out of range inputs.
+  STRICT_FCVT_W_RTZ_RV64 = ISD::FIRST_TARGET_STRICTFP_OPCODE,
+  STRICT_FCVT_WU_RTZ_RV64,
+
   // Memory opcodes start here.
   VLE_VL = ISD::FIRST_TARGET_MEMORY_OPCODE,
   VSE_VL,
@@ -315,7 +320,7 @@ public:
   bool isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const override;
   bool isCheapToSpeculateCttz() const override;
   bool isCheapToSpeculateCtlz() const override;
-  bool hasAndNot(SDValue Y) const override;
+  bool hasAndNotCompare(SDValue Y) const override;
   bool shouldSinkOperands(Instruction *I,
                           SmallVectorImpl<Use *> &Ops) const override;
   bool isFPImmLegal(const APFloat &Imm, EVT VT,
@@ -383,6 +388,9 @@ public:
   EmitInstrWithCustomInserter(MachineInstr &MI,
                               MachineBasicBlock *BB) const override;
 
+  void AdjustInstrPostInstrSelection(MachineInstr &MI,
+                                     SDNode *Node) const override;
+
   EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,
                          EVT VT) const override;
 
@@ -593,6 +601,8 @@ private:
   SDValue lowerToScalableOp(SDValue Op, SelectionDAG &DAG, unsigned NewOpc,
                             bool HasMask = true) const;
   SDValue lowerVPOp(SDValue Op, SelectionDAG &DAG, unsigned RISCVISDOpc) const;
+  SDValue lowerLogicVPOp(SDValue Op, SelectionDAG &DAG, unsigned MaskOpc,
+                         unsigned VecOpc) const;
   SDValue lowerFixedLengthVectorExtendToRVV(SDValue Op, SelectionDAG &DAG,
                                             unsigned ExtendOpc) const;
   SDValue lowerGET_ROUNDING(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrFormats.td b/llvm/lib/Target/RISCV/RISCVInstrFormats.td
index cfad4cdb9364..6a16b6354f95 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrFormats.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrFormats.td
@@ -107,31 +107,44 @@ def Vcompress    : RISCVVConstraint<!or(VS2Constraint.Value,
 
 // The following opcode names match those given in Table 19.1 in the
 // RISC-V User-level ISA specification ("RISC-V base opcode map").
-class RISCVOpcode<bits<7> val> {
+class RISCVOpcode<string name, bits<7> val> {
+  string Name = name;
   bits<7> Value = val;
 }
-def OPC_LOAD      : RISCVOpcode<0b0000011>;
-def OPC_LOAD_FP   : RISCVOpcode<0b0000111>;
-def OPC_MISC_MEM  : RISCVOpcode<0b0001111>;
-def OPC_OP_IMM    : RISCVOpcode<0b0010011>;
-def OPC_AUIPC     : RISCVOpcode<0b0010111>;
-def OPC_OP_IMM_32 : RISCVOpcode<0b0011011>;
-def OPC_STORE     : RISCVOpcode<0b0100011>;
-def OPC_STORE_FP  : RISCVOpcode<0b0100111>;
-def OPC_AMO       : RISCVOpcode<0b0101111>;
-def OPC_OP        : RISCVOpcode<0b0110011>;
-def OPC_LUI       : RISCVOpcode<0b0110111>;
-def OPC_OP_32     : RISCVOpcode<0b0111011>;
-def OPC_MADD      : RISCVOpcode<0b1000011>;
-def OPC_MSUB      : RISCVOpcode<0b1000111>;
-def OPC_NMSUB     : RISCVOpcode<0b1001011>;
-def OPC_NMADD     : RISCVOpcode<0b1001111>;
-def OPC_OP_FP     : RISCVOpcode<0b1010011>;
-def OPC_OP_V      : RISCVOpcode<0b1010111>;
-def OPC_BRANCH    : RISCVOpcode<0b1100011>;
-def OPC_JALR      : RISCVOpcode<0b1100111>;
-def OPC_JAL       : RISCVOpcode<0b1101111>;
-def OPC_SYSTEM    : RISCVOpcode<0b1110011>;
+def RISCVOpcodesList : GenericTable {
+  let FilterClass = "RISCVOpcode";
+  let Fields = [
+    "Name", "Value"
+  ];
+  let PrimaryKey = [ "Value" ];
+  let PrimaryKeyName = "lookupRISCVOpcodeByValue";
+}
+def lookupRISCVOpcodeByName : SearchIndex {
+  let Table = RISCVOpcodesList;
+  let Key = [ "Name" ];
+}
+def OPC_LOAD      : RISCVOpcode<"LOAD",      0b0000011>;
+def OPC_LOAD_FP   : RISCVOpcode<"LOAD_FP",   0b0000111>;
+def OPC_MISC_MEM  : RISCVOpcode<"MISC_MEM",  0b0001111>;
+def OPC_OP_IMM    : RISCVOpcode<"OP_IMM",    0b0010011>;
+def OPC_AUIPC     : RISCVOpcode<"AUIPC",     0b0010111>;
+def OPC_OP_IMM_32 : RISCVOpcode<"OP_IMM_32", 0b0011011>;
+def OPC_STORE     : RISCVOpcode<"STORE",     0b0100011>;
+def OPC_STORE_FP  : RISCVOpcode<"STORE_FP",  0b0100111>;
+def OPC_AMO       : RISCVOpcode<"AMO",       0b0101111>;
+def OPC_OP        : RISCVOpcode<"OP",        0b0110011>;
+def OPC_LUI       : RISCVOpcode<"LUI",       0b0110111>;
+def OPC_OP_32     : RISCVOpcode<"OP_32",     0b0111011>;
+def OPC_MADD      : RISCVOpcode<"MADD",      0b1000011>;
+def OPC_MSUB      : RISCVOpcode<"MSUB",      0b1000111>;
+def OPC_NMSUB     : RISCVOpcode<"NMSUB",     0b1001011>;
+def OPC_NMADD     : RISCVOpcode<"NMADD",     0b1001111>;
+def OPC_OP_FP     : RISCVOpcode<"OP_FP",     0b1010011>;
+def OPC_OP_V      : RISCVOpcode<"OP_V",      0b1010111>;
+def OPC_BRANCH    : RISCVOpcode<"BRANCH",    0b1100011>;
+def OPC_JALR      : RISCVOpcode<"JALR",      0b1100111>;
+def OPC_JAL       : RISCVOpcode<"JAL",       0b1101111>;
+def OPC_SYSTEM    : RISCVOpcode<"SYSTEM",    0b1110011>;
 
 class RVInst<dag outs, dag ins, string opcodestr, string argstr,
              list<dag> pattern, InstFormat format>
@@ -188,8 +201,7 @@ class RVInst<dag outs, dag ins, string opcodestr, string argstr,
 
 // Pseudo instructions
 class Pseudo<dag outs, dag ins, list<dag> pattern, string opcodestr = "", string argstr = "">
-    : RVInst<outs, ins, opcodestr, argstr, pattern, InstFormatPseudo>,
-      Sched<[]> {
+    : RVInst<outs, ins, opcodestr, argstr, pattern, InstFormatPseudo> {
   let isPseudo = 1;
   let isCodeGenOnly = 1;
 }
@@ -265,14 +277,14 @@ class RVInstR4Frm<bits<2> funct2, RISCVOpcode opcode, dag outs, dag ins,
   bits<5> rs3;
   bits<5> rs2;
   bits<5> rs1;
-  bits<3> funct3;
+  bits<3> frm;
   bits<5> rd;
 
   let Inst{31-27} = rs3;
   let Inst{26-25} = funct2;
   let Inst{24-20} = rs2;
   let Inst{19-15} = rs1;
-  let Inst{14-12} = funct3;
+  let Inst{14-12} = frm;
   let Inst{11-7} = rd;
   let Opcode = opcode.Value;
 }
@@ -300,13 +312,13 @@ class RVInstRFrm<bits<7> funct7, RISCVOpcode opcode, dag outs, dag ins,
     : RVInst<outs, ins, opcodestr, argstr, [], InstFormatR> {
   bits<5> rs2;
   bits<5> rs1;
-  bits<3> funct3;
+  bits<3> frm;
   bits<5> rd;
 
   let Inst{31-25} = funct7;
   let Inst{24-20} = rs2;
   let Inst{19-15} = rs1;
-  let Inst{14-12} = funct3;
+  let Inst{14-12} = frm;
   let Inst{11-7} = rd;
   let Opcode = opcode.Value;
 }
diff --git a/llvm/lib/Target/RISCV/RISCVInstrFormatsV.td b/llvm/lib/Target/RISCV/RISCVInstrFormatsV.td
index 80f46b73bfd7..69e9d3553b30 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrFormatsV.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrFormatsV.td
@@ -45,19 +45,6 @@ def SUMOPUnitStride  : RISCVLSUMOP<0b00000>;
 def SUMOPUnitStrideMask : RISCVLSUMOP<0b01011>;
 def SUMOPUnitStrideWholeReg : RISCVLSUMOP<0b01000>;
 
-class RISCVAMOOP<bits<5> val> {
-  bits<5> Value = val;
-}
-def AMOOPVamoSwap : RISCVAMOOP<0b00001>;
-def AMOOPVamoAdd : RISCVAMOOP<0b00000>;
-def AMOOPVamoXor : RISCVAMOOP<0b00100>;
-def AMOOPVamoAnd : RISCVAMOOP<0b01100>;
-def AMOOPVamoOr : RISCVAMOOP<0b01000>;
-def AMOOPVamoMin : RISCVAMOOP<0b10000>;
-def AMOOPVamoMax : RISCVAMOOP<0b10100>;
-def AMOOPVamoMinu : RISCVAMOOP<0b11000>;
-def AMOOPVamoMaxu : RISCVAMOOP<0b11100>;
-
 class RISCVWidth<bits<4> val> {
   bits<4> Value = val;
 }
@@ -342,22 +329,3 @@ class RVInstVSX<bits<3> nf, bit mew, RISCVMOP mop, bits<3> width,
 
   let Uses = [VTYPE, VL];
 }
-
-class RVInstVAMO<RISCVAMOOP amoop, bits<3> width, dag outs, 
-                 dag ins, string opcodestr, string argstr>
-    : RVInst<outs, ins, opcodestr, argstr, [], InstFormatR> {
-  bits<5> vs2;
-  bits<5> rs1;
-  bit wd;
-  bit vm;
-
-  let Inst{31-27} = amoop.Value;
-  let Inst{26} = wd;
-  let Inst{25} = vm;
-  let Inst{24-20} = vs2;
-  let Inst{19-15} = rs1;
-  let Inst{14-12} = width;
-  let Opcode = OPC_AMO.Value;
-
-  let Uses = [VTYPE, VL];
-}
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index 547d82550cac..2e2e00886d57 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -35,6 +35,7 @@ using namespace llvm;
 #include "RISCVGenCompressInstEmitter.inc"
 
 #define GET_INSTRINFO_CTOR_DTOR
+#define GET_INSTRINFO_NAMED_OPS
 #include "RISCVGenInstrInfo.inc"
 
 static cl::opt<bool> PreferWholeRegisterMove(
@@ -1059,6 +1060,7 @@ bool RISCVInstrInfo::isAsCheapAsAMove(const MachineInstr &MI) const {
     break;
   case RISCV::FSGNJ_D:
   case RISCV::FSGNJ_S:
+  case RISCV::FSGNJ_H:
     // The canonical floating-point move is fsgnj rd, rs, rs.
     return MI.getOperand(1).isReg() && MI.getOperand(2).isReg() &&
            MI.getOperand(1).getReg() == MI.getOperand(2).getReg();
@@ -1087,6 +1089,7 @@ RISCVInstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {
     break;
   case RISCV::FSGNJ_D:
   case RISCV::FSGNJ_S:
+  case RISCV::FSGNJ_H:
     // The canonical floating-point move is fsgnj rd, rs, rs.
     if (MI.getOperand(1).isReg() && MI.getOperand(2).isReg() &&
         MI.getOperand(1).getReg() == MI.getOperand(2).getReg())
@@ -1254,7 +1257,7 @@ bool RISCVInstrInfo::isFunctionSafeToOutlineFrom(
 bool RISCVInstrInfo::isMBBSafeToOutlineFrom(MachineBasicBlock &MBB,
                                             unsigned &Flags) const {
   // More accurate safety checking is done in getOutliningCandidateInfo.
-  return true;
+  return TargetInstrInfo::isMBBSafeToOutlineFrom(MBB, Flags);
 }
 
 // Enum values indicating how an outlined call should be constructed.
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.h b/llvm/lib/Target/RISCV/RISCVInstrInfo.h
index 2bfad7844c43..da0877c4299a 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.h
@@ -18,6 +18,7 @@
 #include "llvm/IR/DiagnosticInfo.h"
 
 #define GET_INSTRINFO_HEADER
+#define GET_INSTRINFO_OPERAND_ENUM
 #include "RISCVGenInstrInfo.inc"
 
 namespace llvm {
@@ -181,6 +182,10 @@ protected:
 };
 
 namespace RISCV {
+
+// Implemented in RISCVGenInstrInfo.inc
+int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIndex);
+
 // Special immediate for AVL operand of V pseudo instructions to indicate VLMax.
 static constexpr int64_t VLMaxSentinel = -1LL;
 } // namespace RISCV
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.td b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
index 6f9cde966132..71eb6f01a4f4 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
@@ -174,6 +174,20 @@ def uimm5 : Operand<XLenVT>, ImmLeaf<XLenVT, [{return isUInt<5>(Imm);}]> {
   let OperandNamespace = "RISCVOp";
 }
 
+def InsnDirectiveOpcode : AsmOperandClass {
+  let Name = "InsnDirectiveOpcode";
+  let ParserMethod = "parseInsnDirectiveOpcode";
+  let RenderMethod = "addImmOperands";
+  let PredicateMethod = "isImm";
+}
+
+def uimm7_opcode : Operand<XLenVT> {
+  let ParserMatchClass = InsnDirectiveOpcode;
+  let DecoderMethod = "decodeUImmOperand<7>";
+  let OperandType = "OPERAND_UIMM7";
+  let OperandNamespace = "RISCVOp";
+}
+
 def uimm7 : Operand<XLenVT> {
   let ParserMatchClass = UImmAsmOperand<7>;
   let DecoderMethod = "decodeUImmOperand<7>";
@@ -878,35 +892,35 @@ def : InstAlias<"zext.b $rd, $rs", (ANDI GPR:$rd, GPR:$rs, 0xFF), 0>;
 // isCodeGenOnly = 1 to hide them from the tablegened assembly parser.
 let isCodeGenOnly = 1, hasSideEffects = 1, mayLoad = 1, mayStore = 1,
     hasNoSchedulingInfo = 1 in {
-def InsnR : DirectiveInsnR<(outs AnyReg:$rd), (ins uimm7:$opcode, uimm3:$funct3,
+def InsnR : DirectiveInsnR<(outs AnyReg:$rd), (ins uimm7_opcode:$opcode, uimm3:$funct3,
                                                    uimm7:$funct7, AnyReg:$rs1,
                                                    AnyReg:$rs2),
                            "$opcode, $funct3, $funct7, $rd, $rs1, $rs2">;
-def InsnR4 : DirectiveInsnR4<(outs AnyReg:$rd), (ins uimm7:$opcode,
+def InsnR4 : DirectiveInsnR4<(outs AnyReg:$rd), (ins uimm7_opcode:$opcode,
                                                      uimm3:$funct3,
                                                      uimm2:$funct2,
                                                      AnyReg:$rs1, AnyReg:$rs2,
                                                      AnyReg:$rs3),
                             "$opcode, $funct3, $funct2, $rd, $rs1, $rs2, $rs3">;
-def InsnI : DirectiveInsnI<(outs AnyReg:$rd), (ins uimm7:$opcode, uimm3:$funct3,
+def InsnI : DirectiveInsnI<(outs AnyReg:$rd), (ins uimm7_opcode:$opcode, uimm3:$funct3,
                                                    AnyReg:$rs1, simm12:$imm12),
                            "$opcode, $funct3, $rd, $rs1, $imm12">;
-def InsnI_Mem : DirectiveInsnI<(outs AnyReg:$rd), (ins uimm7:$opcode,
+def InsnI_Mem : DirectiveInsnI<(outs AnyReg:$rd), (ins uimm7_opcode:$opcode,
                                                        uimm3:$funct3,
                                                        AnyReg:$rs1,
                                                        simm12:$imm12),
                                "$opcode, $funct3, $rd, ${imm12}(${rs1})">;
-def InsnB : DirectiveInsnB<(outs), (ins uimm7:$opcode, uimm3:$funct3,
+def InsnB : DirectiveInsnB<(outs), (ins uimm7_opcode:$opcode, uimm3:$funct3,
                                         AnyReg:$rs1, AnyReg:$rs2,
                                         simm13_lsb0:$imm12),
                            "$opcode, $funct3, $rs1, $rs2, $imm12">;
-def InsnU : DirectiveInsnU<(outs AnyReg:$rd), (ins uimm7:$opcode,
+def InsnU : DirectiveInsnU<(outs AnyReg:$rd), (ins uimm7_opcode:$opcode,
                                                    uimm20_lui:$imm20),
                            "$opcode, $rd, $imm20">;
-def InsnJ : DirectiveInsnJ<(outs AnyReg:$rd), (ins uimm7:$opcode,
+def InsnJ : DirectiveInsnJ<(outs AnyReg:$rd), (ins uimm7_opcode:$opcode,
                                                    simm21_lsb0_jal:$imm20),
                            "$opcode, $rd, $imm20">;
-def InsnS : DirectiveInsnS<(outs), (ins uimm7:$opcode, uimm3:$funct3,
+def InsnS : DirectiveInsnS<(outs), (ins uimm7_opcode:$opcode, uimm3:$funct3,
                                         AnyReg:$rs2, AnyReg:$rs1,
                                         simm12:$imm12),
                            "$opcode, $funct3, $rs2, ${imm12}(${rs1})">;
@@ -918,37 +932,37 @@ def InsnS : DirectiveInsnS<(outs), (ins uimm7:$opcode, uimm3:$funct3,
 // for known formats.
 let EmitPriority = 0 in {
 def : InstAlias<".insn_r $opcode, $funct3, $funct7, $rd, $rs1, $rs2",
-                (InsnR AnyReg:$rd, uimm7:$opcode, uimm3:$funct3, uimm7:$funct7,
+                (InsnR AnyReg:$rd, uimm7_opcode:$opcode, uimm3:$funct3, uimm7:$funct7,
                        AnyReg:$rs1, AnyReg:$rs2)>;
 // Accept 4 register form of ".insn r" as alias for ".insn r4".
 def : InstAlias<".insn_r $opcode, $funct3, $funct2, $rd, $rs1, $rs2, $rs3",
-                (InsnR4 AnyReg:$rd, uimm7:$opcode, uimm3:$funct3, uimm2:$funct2,
+                (InsnR4 AnyReg:$rd, uimm7_opcode:$opcode, uimm3:$funct3, uimm2:$funct2,
                         AnyReg:$rs1, AnyReg:$rs2, AnyReg:$rs3)>;
 def : InstAlias<".insn_r4 $opcode, $funct3, $funct2, $rd, $rs1, $rs2, $rs3",
-                (InsnR4 AnyReg:$rd, uimm7:$opcode, uimm3:$funct3, uimm2:$funct2,
+                (InsnR4 AnyReg:$rd, uimm7_opcode:$opcode, uimm3:$funct3, uimm2:$funct2,
                         AnyReg:$rs1, AnyReg:$rs2, AnyReg:$rs3)>;
 def : InstAlias<".insn_i $opcode, $funct3, $rd, $rs1, $imm12",
-                (InsnI AnyReg:$rd, uimm7:$opcode, uimm3:$funct3, AnyReg:$rs1,
+                (InsnI AnyReg:$rd, uimm7_opcode:$opcode, uimm3:$funct3, AnyReg:$rs1,
                        simm12:$imm12)>;
 def : InstAlias<".insn_i $opcode, $funct3, $rd, ${imm12}(${rs1})",
-                (InsnI_Mem AnyReg:$rd, uimm7:$opcode, uimm3:$funct3,
+                (InsnI_Mem AnyReg:$rd, uimm7_opcode:$opcode, uimm3:$funct3,
                            AnyReg:$rs1, simm12:$imm12)>;
 def : InstAlias<".insn_b $opcode, $funct3, $rs1, $rs2, $imm12",
-                (InsnB uimm7:$opcode, uimm3:$funct3, AnyReg:$rs1,
+                (InsnB uimm7_opcode:$opcode, uimm3:$funct3, AnyReg:$rs1,
                        AnyReg:$rs2, simm13_lsb0:$imm12)>;
 // Accept sb as an alias for b.
 def : InstAlias<".insn_sb $opcode, $funct3, $rs1, $rs2, $imm12",
-                (InsnB uimm7:$opcode, uimm3:$funct3, AnyReg:$rs1,
+                (InsnB uimm7_opcode:$opcode, uimm3:$funct3, AnyReg:$rs1,
                        AnyReg:$rs2, simm13_lsb0:$imm12)>;
 def : InstAlias<".insn_u $opcode, $rd, $imm20",
-                (InsnU AnyReg:$rd, uimm7:$opcode, uimm20_lui:$imm20)>;
+                (InsnU AnyReg:$rd, uimm7_opcode:$opcode, uimm20_lui:$imm20)>;
 def : InstAlias<".insn_j $opcode, $rd, $imm20",
-                (InsnJ AnyReg:$rd, uimm7:$opcode, simm21_lsb0_jal:$imm20)>;
+                (InsnJ AnyReg:$rd, uimm7_opcode:$opcode, simm21_lsb0_jal:$imm20)>;
 // Accept uj as an alias for j.
 def : InstAlias<".insn_uj $opcode, $rd, $imm20",
-                (InsnJ AnyReg:$rd, uimm7:$opcode, simm21_lsb0_jal:$imm20)>;
+                (InsnJ AnyReg:$rd, uimm7_opcode:$opcode, simm21_lsb0_jal:$imm20)>;
 def : InstAlias<".insn_s $opcode, $funct3, $rs2, ${imm12}(${rs1})",
-                (InsnS uimm7:$opcode, uimm3:$funct3, AnyReg:$rs2,
+                (InsnS uimm7_opcode:$opcode, uimm3:$funct3, AnyReg:$rs2,
                        AnyReg:$rs1, simm12:$imm12)>;
 }
 
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoD.td b/llvm/lib/Target/RISCV/RISCVInstrInfoD.td
index 2cd011a02345..d6c31c4804db 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoD.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoD.td
@@ -26,41 +26,6 @@ def RISCVBuildPairF64 : SDNode<"RISCVISD::BuildPairF64", SDT_RISCVBuildPairF64>;
 def RISCVSplitF64     : SDNode<"RISCVISD::SplitF64", SDT_RISCVSplitF64>;
 
 //===----------------------------------------------------------------------===//
-// Instruction Class Templates
-//===----------------------------------------------------------------------===//
-
-let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
-class FPFMAD_rrr_frm<RISCVOpcode opcode, string opcodestr>
-    : RVInstR4Frm<0b01, opcode, (outs FPR64:$rd),
-                  (ins FPR64:$rs1, FPR64:$rs2, FPR64:$rs3, frmarg:$funct3),
-                  opcodestr, "$rd, $rs1, $rs2, $rs3, $funct3">;
-
-class FPFMADDynFrmAlias<FPFMAD_rrr_frm Inst, string OpcodeStr>
-    : InstAlias<OpcodeStr#" $rd, $rs1, $rs2, $rs3",
-                (Inst FPR64:$rd, FPR64:$rs1, FPR64:$rs2, FPR64:$rs3, 0b111)>;
-
-let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
-class FPALUD_rr<bits<7> funct7, bits<3> funct3, string opcodestr>
-    : RVInstR<funct7, funct3, OPC_OP_FP, (outs FPR64:$rd),
-              (ins FPR64:$rs1, FPR64:$rs2), opcodestr, "$rd, $rs1, $rs2">;
-
-let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
-class FPALUD_rr_frm<bits<7> funct7, string opcodestr>
-    : RVInstRFrm<funct7, OPC_OP_FP, (outs FPR64:$rd),
-                (ins FPR64:$rs1, FPR64:$rs2, frmarg:$funct3), opcodestr,
-                 "$rd, $rs1, $rs2, $funct3">;
-
-class FPALUDDynFrmAlias<FPALUD_rr_frm Inst, string OpcodeStr>
-    : InstAlias<OpcodeStr#" $rd, $rs1, $rs2",
-                (Inst FPR64:$rd, FPR64:$rs1, FPR64:$rs2, 0b111)>;
-
-let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
-class FPCmpD_rr<bits<3> funct3, string opcodestr>
-    : RVInstR<0b1010001, funct3, OPC_OP_FP, (outs GPR:$rd),
-              (ins FPR64:$rs1, FPR64:$rs2), opcodestr, "$rd, $rs1, $rs2">,
-      Sched<[WriteFCmp64, ReadFCmp64, ReadFCmp64]>;
-
-//===----------------------------------------------------------------------===//
 // Instructions
 //===----------------------------------------------------------------------===//
 
@@ -81,126 +46,104 @@ def FSD : RVInstS<0b011, OPC_STORE_FP, (outs),
                    "fsd", "$rs2, ${imm12}(${rs1})">,
           Sched<[WriteFST64, ReadStoreData, ReadFMemBase]>;
 
-def FMADD_D  : FPFMAD_rrr_frm<OPC_MADD, "fmadd.d">,
-               Sched<[WriteFMA64, ReadFMA64, ReadFMA64, ReadFMA64]>;
-def          : FPFMADDynFrmAlias<FMADD_D, "fmadd.d">;
-def FMSUB_D  : FPFMAD_rrr_frm<OPC_MSUB, "fmsub.d">,
-               Sched<[WriteFMA64, ReadFMA64, ReadFMA64, ReadFMA64]>;
-def          : FPFMADDynFrmAlias<FMSUB_D, "fmsub.d">;
-def FNMSUB_D : FPFMAD_rrr_frm<OPC_NMSUB, "fnmsub.d">,
-               Sched<[WriteFMA64, ReadFMA64, ReadFMA64, ReadFMA64]>;
-def          : FPFMADDynFrmAlias<FNMSUB_D, "fnmsub.d">;
-def FNMADD_D : FPFMAD_rrr_frm<OPC_NMADD, "fnmadd.d">,
-               Sched<[WriteFMA64, ReadFMA64, ReadFMA64, ReadFMA64]>;
-def          : FPFMADDynFrmAlias<FNMADD_D, "fnmadd.d">;
-
-def FADD_D : FPALUD_rr_frm<0b0000001, "fadd.d">,
+let SchedRW = [WriteFMA64, ReadFMA64, ReadFMA64, ReadFMA64] in {
+def FMADD_D  : FPFMA_rrr_frm<OPC_MADD,  0b01, "fmadd.d",  FPR64>;
+def FMSUB_D  : FPFMA_rrr_frm<OPC_MSUB,  0b01, "fmsub.d",  FPR64>;
+def FNMSUB_D : FPFMA_rrr_frm<OPC_NMSUB, 0b01, "fnmsub.d", FPR64>;
+def FNMADD_D : FPFMA_rrr_frm<OPC_NMADD, 0b01, "fnmadd.d", FPR64>;
+}
+
+def : FPFMADynFrmAlias<FMADD_D,  "fmadd.d",  FPR64>;
+def : FPFMADynFrmAlias<FMSUB_D,  "fmsub.d",  FPR64>;
+def : FPFMADynFrmAlias<FNMSUB_D, "fnmsub.d", FPR64>;
+def : FPFMADynFrmAlias<FNMADD_D, "fnmadd.d", FPR64>;
+
+def FADD_D : FPALU_rr_frm<0b0000001, "fadd.d", FPR64>,
              Sched<[WriteFALU64, ReadFALU64, ReadFALU64]>;
-def        : FPALUDDynFrmAlias<FADD_D, "fadd.d">;
-def FSUB_D : FPALUD_rr_frm<0b0000101, "fsub.d">,
+def FSUB_D : FPALU_rr_frm<0b0000101, "fsub.d", FPR64>,
              Sched<[WriteFALU64, ReadFALU64, ReadFALU64]>;
-def        : FPALUDDynFrmAlias<FSUB_D, "fsub.d">;
-def FMUL_D : FPALUD_rr_frm<0b0001001, "fmul.d">,
+def FMUL_D : FPALU_rr_frm<0b0001001, "fmul.d", FPR64>,
              Sched<[WriteFMul64, ReadFMul64, ReadFMul64]>;
-def        : FPALUDDynFrmAlias<FMUL_D, "fmul.d">;
-def FDIV_D : FPALUD_rr_frm<0b0001101, "fdiv.d">,
+def FDIV_D : FPALU_rr_frm<0b0001101, "fdiv.d", FPR64>,
              Sched<[WriteFDiv64, ReadFDiv64, ReadFDiv64]>;
-def        : FPALUDDynFrmAlias<FDIV_D, "fdiv.d">;
 
-def FSQRT_D : FPUnaryOp_r_frm<0b0101101, FPR64, FPR64, "fsqrt.d">,
-              Sched<[WriteFSqrt64, ReadFSqrt64]> {
-  let rs2 = 0b00000;
-}
+def        : FPALUDynFrmAlias<FADD_D, "fadd.d", FPR64>;
+def        : FPALUDynFrmAlias<FSUB_D, "fsub.d", FPR64>;
+def        : FPALUDynFrmAlias<FMUL_D, "fmul.d", FPR64>;
+def        : FPALUDynFrmAlias<FDIV_D, "fdiv.d", FPR64>;
+
+def FSQRT_D : FPUnaryOp_r_frm<0b0101101, 0b00000, FPR64, FPR64, "fsqrt.d">,
+              Sched<[WriteFSqrt64, ReadFSqrt64]>;
 def         : FPUnaryOpDynFrmAlias<FSQRT_D, "fsqrt.d", FPR64, FPR64>;
 
-def FSGNJ_D  : FPALUD_rr<0b0010001, 0b000, "fsgnj.d">,
-               Sched<[WriteFSGNJ64, ReadFSGNJ64, ReadFSGNJ64]>;
-def FSGNJN_D : FPALUD_rr<0b0010001, 0b001, "fsgnjn.d">,
-               Sched<[WriteFSGNJ64, ReadFSGNJ64, ReadFSGNJ64]>;
-def FSGNJX_D : FPALUD_rr<0b0010001, 0b010, "fsgnjx.d">,
-               Sched<[WriteFSGNJ64, ReadFSGNJ64, ReadFSGNJ64]>;
-def FMIN_D   : FPALUD_rr<0b0010101, 0b000, "fmin.d">,
-               Sched<[WriteFMinMax64, ReadFMinMax64, ReadFMinMax64]>;
-def FMAX_D   : FPALUD_rr<0b0010101, 0b001, "fmax.d">,
-               Sched<[WriteFMinMax64, ReadFMinMax64, ReadFMinMax64]>;
-
-def FCVT_S_D : FPUnaryOp_r_frm<0b0100000, FPR32, FPR64, "fcvt.s.d">,
-               Sched<[WriteFCvtF64ToF32, ReadFCvtF64ToF32]> {
-  let rs2 = 0b00001;
+let SchedRW = [WriteFSGNJ64, ReadFSGNJ64, ReadFSGNJ64],
+    mayRaiseFPException = 0 in {
+def FSGNJ_D  : FPALU_rr<0b0010001, 0b000, "fsgnj.d", FPR64>;
+def FSGNJN_D : FPALU_rr<0b0010001, 0b001, "fsgnjn.d", FPR64>;
+def FSGNJX_D : FPALU_rr<0b0010001, 0b010, "fsgnjx.d", FPR64>;
 }
-def          : FPUnaryOpDynFrmAlias<FCVT_S_D, "fcvt.s.d", FPR32, FPR64>;
 
-def FCVT_D_S : FPUnaryOp_r<0b0100001, 0b000, FPR64, FPR32, "fcvt.d.s">,
-               Sched<[WriteFCvtF32ToF64, ReadFCvtF32ToF64]> {
-  let rs2 = 0b00000;
+let SchedRW = [WriteFMinMax64, ReadFMinMax64, ReadFMinMax64] in {
+def FMIN_D   : FPALU_rr<0b0010101, 0b000, "fmin.d", FPR64>;
+def FMAX_D   : FPALU_rr<0b0010101, 0b001, "fmax.d", FPR64>;
 }
 
-def FEQ_D : FPCmpD_rr<0b010, "feq.d">;
-def FLT_D : FPCmpD_rr<0b001, "flt.d">;
-def FLE_D : FPCmpD_rr<0b000, "fle.d">;
+def FCVT_S_D : FPUnaryOp_r_frm<0b0100000, 0b00001, FPR32, FPR64, "fcvt.s.d">,
+               Sched<[WriteFCvtF64ToF32, ReadFCvtF64ToF32]>;
+def          : FPUnaryOpDynFrmAlias<FCVT_S_D, "fcvt.s.d", FPR32, FPR64>;
 
-def FCLASS_D : FPUnaryOp_r<0b1110001, 0b001, GPR, FPR64, "fclass.d">,
-               Sched<[WriteFClass64, ReadFClass64]> {
-  let rs2 = 0b00000;
-}
+def FCVT_D_S : FPUnaryOp_r<0b0100001, 0b00000, 0b000, FPR64, FPR32, "fcvt.d.s">,
+               Sched<[WriteFCvtF32ToF64, ReadFCvtF32ToF64]>;
 
-def FCVT_W_D : FPUnaryOp_r_frm<0b1100001, GPR, FPR64, "fcvt.w.d">,
-               Sched<[WriteFCvtF64ToI32, ReadFCvtF64ToI32]> {
-  let rs2 = 0b00000;
+let SchedRW = [WriteFCmp64, ReadFCmp64, ReadFCmp64] in {
+def FEQ_D : FPCmp_rr<0b1010001, 0b010, "feq.d", FPR64>;
+def FLT_D : FPCmp_rr<0b1010001, 0b001, "flt.d", FPR64>;
+def FLE_D : FPCmp_rr<0b1010001, 0b000, "fle.d", FPR64>;
 }
+
+let mayRaiseFPException = 0 in
+def FCLASS_D : FPUnaryOp_r<0b1110001, 0b00000, 0b001, GPR, FPR64, "fclass.d">,
+               Sched<[WriteFClass64, ReadFClass64]>;
+
+def FCVT_W_D : FPUnaryOp_r_frm<0b1100001, 0b00000, GPR, FPR64, "fcvt.w.d">,
+               Sched<[WriteFCvtF64ToI32, ReadFCvtF64ToI32]>;
 def          : FPUnaryOpDynFrmAlias<FCVT_W_D, "fcvt.w.d", GPR, FPR64>;
 
-def FCVT_WU_D : FPUnaryOp_r_frm<0b1100001, GPR, FPR64, "fcvt.wu.d">,
-                Sched<[WriteFCvtF64ToI32, ReadFCvtF64ToI32]> {
-  let rs2 = 0b00001;
-}
+def FCVT_WU_D : FPUnaryOp_r_frm<0b1100001, 0b00001, GPR, FPR64, "fcvt.wu.d">,
+                Sched<[WriteFCvtF64ToI32, ReadFCvtF64ToI32]>;
 def           : FPUnaryOpDynFrmAlias<FCVT_WU_D, "fcvt.wu.d", GPR, FPR64>;
 
-def FCVT_D_W : FPUnaryOp_r<0b1101001, 0b000, FPR64, GPR, "fcvt.d.w">,
-               Sched<[WriteFCvtI32ToF64, ReadFCvtI32ToF64]> {
-  let rs2 = 0b00000;
-}
+def FCVT_D_W : FPUnaryOp_r<0b1101001, 0b00000, 0b000, FPR64, GPR, "fcvt.d.w">,
+               Sched<[WriteFCvtI32ToF64, ReadFCvtI32ToF64]>;
 
-def FCVT_D_WU : FPUnaryOp_r<0b1101001, 0b000, FPR64, GPR, "fcvt.d.wu">,
-                Sched<[WriteFCvtI32ToF64, ReadFCvtI32ToF64]> {
-  let rs2 = 0b00001;
-}
+def FCVT_D_WU : FPUnaryOp_r<0b1101001, 0b00001, 0b000, FPR64, GPR, "fcvt.d.wu">,
+                Sched<[WriteFCvtI32ToF64, ReadFCvtI32ToF64]>;
 } // Predicates = [HasStdExtD]
 
 let Predicates = [HasStdExtD, IsRV64] in {
-def FCVT_L_D : FPUnaryOp_r_frm<0b1100001, GPR, FPR64, "fcvt.l.d">,
-               Sched<[WriteFCvtF64ToI64, ReadFCvtF64ToI64]> {
-  let rs2 = 0b00010;
-}
+def FCVT_L_D : FPUnaryOp_r_frm<0b1100001, 0b00010, GPR, FPR64, "fcvt.l.d">,
+               Sched<[WriteFCvtF64ToI64, ReadFCvtF64ToI64]>;
 def          : FPUnaryOpDynFrmAlias<FCVT_L_D, "fcvt.l.d", GPR, FPR64>;
 
-def FCVT_LU_D : FPUnaryOp_r_frm<0b1100001, GPR, FPR64, "fcvt.lu.d">,
-                Sched<[WriteFCvtF64ToI64, ReadFCvtF64ToI64]> {
-  let rs2 = 0b00011;
-}
+def FCVT_LU_D : FPUnaryOp_r_frm<0b1100001, 0b00011, GPR, FPR64, "fcvt.lu.d">,
+                Sched<[WriteFCvtF64ToI64, ReadFCvtF64ToI64]>;
 def           : FPUnaryOpDynFrmAlias<FCVT_LU_D, "fcvt.lu.d", GPR, FPR64>;
 
-def FMV_X_D : FPUnaryOp_r<0b1110001, 0b000, GPR, FPR64, "fmv.x.d">,
-              Sched<[WriteFMovF64ToI64, ReadFMovF64ToI64]> {
-  let rs2 = 0b00000;
-}
+let mayRaiseFPException = 0 in
+def FMV_X_D : FPUnaryOp_r<0b1110001, 0b00000, 0b000, GPR, FPR64, "fmv.x.d">,
+              Sched<[WriteFMovF64ToI64, ReadFMovF64ToI64]>;
 
-def FCVT_D_L : FPUnaryOp_r_frm<0b1101001, FPR64, GPR, "fcvt.d.l">,
-               Sched<[WriteFCvtI64ToF64, ReadFCvtI64ToF64]> {
-  let rs2 = 0b00010;
-}
+def FCVT_D_L : FPUnaryOp_r_frm<0b1101001, 0b00010, FPR64, GPR, "fcvt.d.l">,
+               Sched<[WriteFCvtI64ToF64, ReadFCvtI64ToF64]>;
 def          : FPUnaryOpDynFrmAlias<FCVT_D_L, "fcvt.d.l", FPR64, GPR>;
 
-def FCVT_D_LU : FPUnaryOp_r_frm<0b1101001, FPR64, GPR, "fcvt.d.lu">,
-                Sched<[WriteFCvtI64ToF64, ReadFCvtI64ToF64]> {
-  let rs2 = 0b00011;
-}
+def FCVT_D_LU : FPUnaryOp_r_frm<0b1101001, 0b00011, FPR64, GPR, "fcvt.d.lu">,
+                Sched<[WriteFCvtI64ToF64, ReadFCvtI64ToF64]>;
 def           : FPUnaryOpDynFrmAlias<FCVT_D_LU, "fcvt.d.lu", FPR64, GPR>;
 
-def FMV_D_X : FPUnaryOp_r<0b1111001, 0b000, FPR64, GPR, "fmv.d.x">,
-              Sched<[WriteFMovI64ToF64, ReadFMovI64ToF64]> {
-  let rs2 = 0b00000;
-}
+let mayRaiseFPException = 0 in
+def FMV_D_X : FPUnaryOp_r<0b1111001, 0b00000, 0b000, FPR64, GPR, "fmv.d.x">,
+              Sched<[WriteFMovI64ToF64, ReadFMovI64ToF64]>;
 } // Predicates = [HasStdExtD, IsRV64]
 
 //===----------------------------------------------------------------------===//
@@ -241,20 +184,20 @@ let Predicates = [HasStdExtD] in {
 /// Float conversion operations
 
 // f64 -> f32, f32 -> f64
-def : Pat<(fpround FPR64:$rs1), (FCVT_S_D FPR64:$rs1, 0b111)>;
-def : Pat<(fpextend FPR32:$rs1), (FCVT_D_S FPR32:$rs1)>;
+def : Pat<(any_fpround FPR64:$rs1), (FCVT_S_D FPR64:$rs1, 0b111)>;
+def : Pat<(any_fpextend FPR32:$rs1), (FCVT_D_S FPR32:$rs1)>;
 
 // [u]int<->double conversion patterns must be gated on IsRV32 or IsRV64, so
 // are defined later.
 
 /// Float arithmetic operations
 
-def : PatFpr64Fpr64DynFrm<fadd, FADD_D>;
-def : PatFpr64Fpr64DynFrm<fsub, FSUB_D>;
-def : PatFpr64Fpr64DynFrm<fmul, FMUL_D>;
-def : PatFpr64Fpr64DynFrm<fdiv, FDIV_D>;
+def : PatFpr64Fpr64DynFrm<any_fadd, FADD_D>;
+def : PatFpr64Fpr64DynFrm<any_fsub, FSUB_D>;
+def : PatFpr64Fpr64DynFrm<any_fmul, FMUL_D>;
+def : PatFpr64Fpr64DynFrm<any_fdiv, FDIV_D>;
 
-def : Pat<(fsqrt FPR64:$rs1), (FSQRT_D FPR64:$rs1, 0b111)>;
+def : Pat<(any_fsqrt FPR64:$rs1), (FSQRT_D FPR64:$rs1, 0b111)>;
 
 def : Pat<(fneg FPR64:$rs1), (FSGNJN_D $rs1, $rs1)>;
 def : Pat<(fabs FPR64:$rs1), (FSGNJX_D $rs1, $rs1)>;
@@ -266,19 +209,19 @@ def : Pat<(fcopysign FPR32:$rs1, FPR64:$rs2), (FSGNJ_S $rs1, (FCVT_S_D $rs2,
                                                               0b111))>;
 
 // fmadd: rs1 * rs2 + rs3
-def : Pat<(fma FPR64:$rs1, FPR64:$rs2, FPR64:$rs3),
+def : Pat<(any_fma FPR64:$rs1, FPR64:$rs2, FPR64:$rs3),
           (FMADD_D $rs1, $rs2, $rs3, 0b111)>;
 
 // fmsub: rs1 * rs2 - rs3
-def : Pat<(fma FPR64:$rs1, FPR64:$rs2, (fneg FPR64:$rs3)),
+def : Pat<(any_fma FPR64:$rs1, FPR64:$rs2, (fneg FPR64:$rs3)),
           (FMSUB_D FPR64:$rs1, FPR64:$rs2, FPR64:$rs3, 0b111)>;
 
 // fnmsub: -rs1 * rs2 + rs3
-def : Pat<(fma (fneg FPR64:$rs1), FPR64:$rs2, FPR64:$rs3),
+def : Pat<(any_fma (fneg FPR64:$rs1), FPR64:$rs2, FPR64:$rs3),
           (FNMSUB_D FPR64:$rs1, FPR64:$rs2, FPR64:$rs3, 0b111)>;
 
 // fnmadd: -rs1 * rs2 - rs3
-def : Pat<(fma (fneg FPR64:$rs1), FPR64:$rs2, (fneg FPR64:$rs3)),
+def : Pat<(any_fma (fneg FPR64:$rs1), FPR64:$rs2, (fneg FPR64:$rs3)),
           (FNMADD_D FPR64:$rs1, FPR64:$rs2, FPR64:$rs3, 0b111)>;
 
 // The ratified 20191213 ISA spec defines fmin and fmax in a way that matches
@@ -328,8 +271,8 @@ let Predicates = [HasStdExtD, IsRV32] in {
 def : Pat<(f64 (fpimm0)), (FCVT_D_W (i32 X0))>;
 
 // double->[u]int. Round-to-zero must be used.
-def : Pat<(i32 (fp_to_sint FPR64:$rs1)), (FCVT_W_D FPR64:$rs1, 0b001)>;
-def : Pat<(i32 (fp_to_uint FPR64:$rs1)), (FCVT_WU_D FPR64:$rs1, 0b001)>;
+def : Pat<(i32 (any_fp_to_sint FPR64:$rs1)), (FCVT_W_D FPR64:$rs1, 0b001)>;
+def : Pat<(i32 (any_fp_to_uint FPR64:$rs1)), (FCVT_WU_D FPR64:$rs1, 0b001)>;
 
 // Saturating double->[u]int32.
 def : Pat<(i32 (riscv_fcvt_x_rtz FPR64:$rs1)), (FCVT_W_D $rs1, 0b001)>;
@@ -342,8 +285,8 @@ def : Pat<(i32 (lrint FPR64:$rs1)), (FCVT_W_D $rs1, 0b111)>;
 def : Pat<(i32 (lround FPR64:$rs1)), (FCVT_W_D $rs1, 0b100)>;
 
 // [u]int->double.
-def : Pat<(sint_to_fp (i32 GPR:$rs1)), (FCVT_D_W GPR:$rs1)>;
-def : Pat<(uint_to_fp (i32 GPR:$rs1)), (FCVT_D_WU GPR:$rs1)>;
+def : Pat<(any_sint_to_fp (i32 GPR:$rs1)), (FCVT_D_W GPR:$rs1)>;
+def : Pat<(any_uint_to_fp (i32 GPR:$rs1)), (FCVT_D_WU GPR:$rs1)>;
 } // Predicates = [HasStdExtD, IsRV32]
 
 let Predicates = [HasStdExtD, IsRV64] in {
@@ -358,20 +301,20 @@ def : Pat<(i64 (bitconvert FPR64:$rs1)), (FMV_X_D FPR64:$rs1)>;
 // Use target specific isd nodes to help us remember the result is sign
 // extended. Matching sext_inreg+fptoui/fptosi may cause the conversion to be
 // duplicated if it has another user that didn't need the sign_extend.
-def : Pat<(riscv_fcvt_w_rtz_rv64 FPR64:$rs1),  (FCVT_W_D $rs1, 0b001)>;
-def : Pat<(riscv_fcvt_wu_rtz_rv64 FPR64:$rs1), (FCVT_WU_D $rs1, 0b001)>;
+def : Pat<(riscv_any_fcvt_w_rtz_rv64 FPR64:$rs1),  (FCVT_W_D $rs1, 0b001)>;
+def : Pat<(riscv_any_fcvt_wu_rtz_rv64 FPR64:$rs1), (FCVT_WU_D $rs1, 0b001)>;
 
 // [u]int32->fp
-def : Pat<(sint_to_fp (i64 (sexti32 (i64 GPR:$rs1)))), (FCVT_D_W $rs1)>;
-def : Pat<(uint_to_fp (i64 (zexti32 (i64 GPR:$rs1)))), (FCVT_D_WU $rs1)>;
+def : Pat<(any_sint_to_fp (i64 (sexti32 (i64 GPR:$rs1)))), (FCVT_D_W $rs1)>;
+def : Pat<(any_uint_to_fp (i64 (zexti32 (i64 GPR:$rs1)))), (FCVT_D_WU $rs1)>;
 
 // Saturating double->[u]int64.
 def : Pat<(i64 (riscv_fcvt_x_rtz FPR64:$rs1)), (FCVT_L_D $rs1, 0b001)>;
 def : Pat<(i64 (riscv_fcvt_xu_rtz FPR64:$rs1)), (FCVT_LU_D $rs1, 0b001)>;
 
 // double->[u]int64. Round-to-zero must be used.
-def : Pat<(i64 (fp_to_sint FPR64:$rs1)), (FCVT_L_D FPR64:$rs1, 0b001)>;
-def : Pat<(i64 (fp_to_uint FPR64:$rs1)), (FCVT_LU_D FPR64:$rs1, 0b001)>;
+def : Pat<(i64 (any_fp_to_sint FPR64:$rs1)), (FCVT_L_D FPR64:$rs1, 0b001)>;
+def : Pat<(i64 (any_fp_to_uint FPR64:$rs1)), (FCVT_LU_D FPR64:$rs1, 0b001)>;
 
 // double->int64 with current rounding mode.
 def : Pat<(i64 (lrint FPR64:$rs1)), (FCVT_L_D $rs1, 0b111)>;
@@ -382,6 +325,6 @@ def : Pat<(i64 (lround FPR64:$rs1)), (FCVT_L_D $rs1, 0b100)>;
 def : Pat<(i64 (llround FPR64:$rs1)), (FCVT_L_D $rs1, 0b100)>;
 
 // [u]int64->fp. Match GCC and default to using dynamic rounding mode.
-def : Pat<(sint_to_fp (i64 GPR:$rs1)), (FCVT_D_L GPR:$rs1, 0b111)>;
-def : Pat<(uint_to_fp (i64 GPR:$rs1)), (FCVT_D_LU GPR:$rs1, 0b111)>;
+def : Pat<(any_sint_to_fp (i64 GPR:$rs1)), (FCVT_D_L GPR:$rs1, 0b111)>;
+def : Pat<(any_uint_to_fp (i64 GPR:$rs1)), (FCVT_D_LU GPR:$rs1, 0b111)>;
 } // Predicates = [HasStdExtD, IsRV64]
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoF.td b/llvm/lib/Target/RISCV/RISCVInstrInfoF.td
index 3400c3be52bf..bb45ed859442 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoF.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoF.td
@@ -19,9 +19,9 @@ def SDT_RISCVFMV_W_X_RV64
     : SDTypeProfile<1, 1, [SDTCisVT<0, f32>, SDTCisVT<1, i64>]>;
 def SDT_RISCVFMV_X_ANYEXTW_RV64
     : SDTypeProfile<1, 1, [SDTCisVT<0, i64>, SDTCisVT<1, f32>]>;
-def STD_RISCVFCVT_W_RV64
+def SDT_RISCVFCVT_W_RV64
     : SDTypeProfile<1, 1, [SDTCisVT<0, i64>, SDTCisFP<1>]>;
-def STD_RISCVFCVT_X
+def SDT_RISCVFCVT_X
     : SDTypeProfile<1, 1, [SDTCisVT<0, XLenVT>, SDTCisFP<1>]>;
 
 def riscv_fmv_w_x_rv64
@@ -29,13 +29,27 @@ def riscv_fmv_w_x_rv64
 def riscv_fmv_x_anyextw_rv64
     : SDNode<"RISCVISD::FMV_X_ANYEXTW_RV64", SDT_RISCVFMV_X_ANYEXTW_RV64>;
 def riscv_fcvt_w_rtz_rv64
-    : SDNode<"RISCVISD::FCVT_W_RTZ_RV64", STD_RISCVFCVT_W_RV64>;
+    : SDNode<"RISCVISD::FCVT_W_RTZ_RV64", SDT_RISCVFCVT_W_RV64>;
 def riscv_fcvt_wu_rtz_rv64
-    : SDNode<"RISCVISD::FCVT_WU_RTZ_RV64", STD_RISCVFCVT_W_RV64>;
+    : SDNode<"RISCVISD::FCVT_WU_RTZ_RV64", SDT_RISCVFCVT_W_RV64>;
 def riscv_fcvt_x_rtz
-    : SDNode<"RISCVISD::FCVT_X_RTZ", STD_RISCVFCVT_X>;
+    : SDNode<"RISCVISD::FCVT_X_RTZ", SDT_RISCVFCVT_X>;
 def riscv_fcvt_xu_rtz
-    : SDNode<"RISCVISD::FCVT_XU_RTZ", STD_RISCVFCVT_X>;
+    : SDNode<"RISCVISD::FCVT_XU_RTZ", SDT_RISCVFCVT_X>;
+
+def riscv_strict_fcvt_w_rtz_rv64
+    : SDNode<"RISCVISD::STRICT_FCVT_W_RTZ_RV64", SDT_RISCVFCVT_W_RV64,
+             [SDNPHasChain]>;
+def riscv_strict_fcvt_wu_rtz_rv64
+    : SDNode<"RISCVISD::STRICT_FCVT_WU_RTZ_RV64", SDT_RISCVFCVT_W_RV64,
+             [SDNPHasChain]>;
+
+def riscv_any_fcvt_w_rtz_rv64 : PatFrags<(ops node:$src),
+                                         [(riscv_strict_fcvt_w_rtz_rv64 node:$src),
+                                          (riscv_fcvt_w_rtz_rv64 node:$src)]>;
+def riscv_any_fcvt_wu_rtz_rv64 : PatFrags<(ops node:$src),
+                                          [(riscv_strict_fcvt_wu_rtz_rv64 node:$src),
+                                           (riscv_fcvt_wu_rtz_rv64 node:$src)]>;
 
 //===----------------------------------------------------------------------===//
 // Operand and SDNode transformation definitions.
@@ -59,54 +73,65 @@ def frmarg : Operand<XLenVT> {
 // Instruction class templates
 //===----------------------------------------------------------------------===//
 
-let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
-class FPFMAS_rrr_frm<RISCVOpcode opcode, string opcodestr>
-    : RVInstR4Frm<0b00, opcode, (outs FPR32:$rd),
-                  (ins FPR32:$rs1, FPR32:$rs2, FPR32:$rs3, frmarg:$funct3),
-                  opcodestr, "$rd, $rs1, $rs2, $rs3, $funct3">;
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0, mayRaiseFPException = 1,
+    UseNamedOperandTable = 1, hasPostISelHook = 1 in
+class FPFMA_rrr_frm<RISCVOpcode opcode, bits<2> funct2, string opcodestr,
+                    RegisterClass rty>
+    : RVInstR4Frm<funct2, opcode, (outs rty:$rd),
+                  (ins rty:$rs1, rty:$rs2, rty:$rs3, frmarg:$frm),
+                  opcodestr, "$rd, $rs1, $rs2, $rs3, $frm">;
 
-class FPFMASDynFrmAlias<FPFMAS_rrr_frm Inst, string OpcodeStr>
+class FPFMADynFrmAlias<FPFMA_rrr_frm Inst, string OpcodeStr,
+                       RegisterClass rty>
     : InstAlias<OpcodeStr#" $rd, $rs1, $rs2, $rs3",
-                (Inst FPR32:$rd, FPR32:$rs1, FPR32:$rs2, FPR32:$rs3, 0b111)>;
-
-let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
-class FPALUS_rr<bits<7> funct7, bits<3> funct3, string opcodestr>
-    : RVInstR<funct7, funct3, OPC_OP_FP, (outs FPR32:$rd),
-              (ins FPR32:$rs1, FPR32:$rs2), opcodestr, "$rd, $rs1, $rs2">;
-
-let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
-class FPALUS_rr_frm<bits<7> funct7, string opcodestr>
-    : RVInstRFrm<funct7, OPC_OP_FP, (outs FPR32:$rd),
-                 (ins FPR32:$rs1, FPR32:$rs2, frmarg:$funct3), opcodestr,
-                  "$rd, $rs1, $rs2, $funct3">;
-
-class FPALUSDynFrmAlias<FPALUS_rr_frm Inst, string OpcodeStr>
+                (Inst rty:$rd, rty:$rs1, rty:$rs2, rty:$rs3, 0b111)>;
+
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0, mayRaiseFPException = 1 in
+class FPALU_rr<bits<7> funct7, bits<3> funct3, string opcodestr,
+               RegisterClass rty>
+    : RVInstR<funct7, funct3, OPC_OP_FP, (outs rty:$rd),
+              (ins rty:$rs1, rty:$rs2), opcodestr, "$rd, $rs1, $rs2">;
+
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0, mayRaiseFPException = 1,
+    UseNamedOperandTable = 1, hasPostISelHook = 1 in
+class FPALU_rr_frm<bits<7> funct7, string opcodestr, RegisterClass rty>
+    : RVInstRFrm<funct7, OPC_OP_FP, (outs rty:$rd),
+                 (ins rty:$rs1, rty:$rs2, frmarg:$frm), opcodestr,
+                  "$rd, $rs1, $rs2, $frm">;
+
+class FPALUDynFrmAlias<FPALU_rr_frm Inst, string OpcodeStr,
+                       RegisterClass rty>
     : InstAlias<OpcodeStr#" $rd, $rs1, $rs2",
-                (Inst FPR32:$rd, FPR32:$rs1, FPR32:$rs2, 0b111)>;
+                (Inst rty:$rd, rty:$rs1, rty:$rs2, 0b111)>;
 
-let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
-class FPUnaryOp_r<bits<7> funct7, bits<3> funct3, RegisterClass rdty,
-                RegisterClass rs1ty, string opcodestr>
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0, mayRaiseFPException = 1 in
+class FPUnaryOp_r<bits<7> funct7, bits<5> rs2val, bits<3> funct3,
+                  RegisterClass rdty, RegisterClass rs1ty, string opcodestr>
     : RVInstR<funct7, funct3, OPC_OP_FP, (outs rdty:$rd), (ins rs1ty:$rs1),
-              opcodestr, "$rd, $rs1">;
+              opcodestr, "$rd, $rs1"> {
+  let rs2 = rs2val;
+}
 
-let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
-class FPUnaryOp_r_frm<bits<7> funct7, RegisterClass rdty, RegisterClass rs1ty,
-                      string opcodestr>
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0, mayRaiseFPException = 1,
+    UseNamedOperandTable = 1, hasPostISelHook = 1 in
+class FPUnaryOp_r_frm<bits<7> funct7, bits<5> rs2val, RegisterClass rdty,
+                      RegisterClass rs1ty, string opcodestr>
     : RVInstRFrm<funct7, OPC_OP_FP, (outs rdty:$rd),
-                 (ins rs1ty:$rs1, frmarg:$funct3), opcodestr,
-                  "$rd, $rs1, $funct3">;
+                 (ins rs1ty:$rs1, frmarg:$frm), opcodestr,
+                  "$rd, $rs1, $frm"> {
+  let rs2 = rs2val;
+}
 
 class FPUnaryOpDynFrmAlias<FPUnaryOp_r_frm Inst, string OpcodeStr,
                            RegisterClass rdty, RegisterClass rs1ty>
     : InstAlias<OpcodeStr#" $rd, $rs1",
                 (Inst rdty:$rd, rs1ty:$rs1, 0b111)>;
 
-let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
-class FPCmpS_rr<bits<3> funct3, string opcodestr>
-    : RVInstR<0b1010000, funct3, OPC_OP_FP, (outs GPR:$rd),
-              (ins FPR32:$rs1, FPR32:$rs2), opcodestr, "$rd, $rs1, $rs2">,
-      Sched<[WriteFCmp32, ReadFCmp32, ReadFCmp32]>;
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0, mayRaiseFPException = 1 in
+class FPCmp_rr<bits<7> funct7, bits<3> funct3, string opcodestr,
+               RegisterClass rty>
+    : RVInstR<funct7, funct3, OPC_OP_FP, (outs GPR:$rd),
+              (ins rty:$rs1, rty:$rs2), opcodestr, "$rd, $rs1, $rs2">;
 
 //===----------------------------------------------------------------------===//
 // Instructions
@@ -128,116 +153,98 @@ def FSW : RVInstS<0b010, OPC_STORE_FP, (outs),
                    "fsw", "$rs2, ${imm12}(${rs1})">,
           Sched<[WriteFST32, ReadStoreData, ReadFMemBase]>;
 
-def FMADD_S  : FPFMAS_rrr_frm<OPC_MADD, "fmadd.s">,
-               Sched<[WriteFMA32, ReadFMA32, ReadFMA32, ReadFMA32]>;
-def          : FPFMASDynFrmAlias<FMADD_S, "fmadd.s">;
-def FMSUB_S  : FPFMAS_rrr_frm<OPC_MSUB, "fmsub.s">,
-               Sched<[WriteFMA32, ReadFMA32, ReadFMA32, ReadFMA32]>;
-def          : FPFMASDynFrmAlias<FMSUB_S, "fmsub.s">;
-def FNMSUB_S : FPFMAS_rrr_frm<OPC_NMSUB, "fnmsub.s">,
-               Sched<[WriteFMA32, ReadFMA32, ReadFMA32, ReadFMA32]>;
-def          : FPFMASDynFrmAlias<FNMSUB_S, "fnmsub.s">;
-def FNMADD_S : FPFMAS_rrr_frm<OPC_NMADD, "fnmadd.s">,
-               Sched<[WriteFMA32, ReadFMA32, ReadFMA32, ReadFMA32]>;
-def          : FPFMASDynFrmAlias<FNMADD_S, "fnmadd.s">;
-
-def FADD_S : FPALUS_rr_frm<0b0000000, "fadd.s">,
+let SchedRW = [WriteFMA32, ReadFMA32, ReadFMA32, ReadFMA32] in {
+def FMADD_S  : FPFMA_rrr_frm<OPC_MADD,  0b00, "fmadd.s",  FPR32>;
+def FMSUB_S  : FPFMA_rrr_frm<OPC_MSUB,  0b00, "fmsub.s",  FPR32>;
+def FNMSUB_S : FPFMA_rrr_frm<OPC_NMSUB, 0b00, "fnmsub.s", FPR32>;
+def FNMADD_S : FPFMA_rrr_frm<OPC_NMADD, 0b00, "fnmadd.s", FPR32>;
+}
+
+def : FPFMADynFrmAlias<FMADD_S,  "fmadd.s",  FPR32>;
+def : FPFMADynFrmAlias<FMSUB_S,  "fmsub.s",  FPR32>;
+def : FPFMADynFrmAlias<FNMSUB_S, "fnmsub.s", FPR32>;
+def : FPFMADynFrmAlias<FNMADD_S, "fnmadd.s", FPR32>;
+
+def FADD_S : FPALU_rr_frm<0b0000000, "fadd.s", FPR32>,
              Sched<[WriteFALU32, ReadFALU32, ReadFALU32]>;
-def        : FPALUSDynFrmAlias<FADD_S, "fadd.s">;
-def FSUB_S : FPALUS_rr_frm<0b0000100, "fsub.s">,
+def FSUB_S : FPALU_rr_frm<0b0000100, "fsub.s", FPR32>,
              Sched<[WriteFALU32, ReadFALU32, ReadFALU32]>;
-def        : FPALUSDynFrmAlias<FSUB_S, "fsub.s">;
-def FMUL_S : FPALUS_rr_frm<0b0001000, "fmul.s">,
+def FMUL_S : FPALU_rr_frm<0b0001000, "fmul.s", FPR32>,
              Sched<[WriteFMul32, ReadFMul32, ReadFMul32]>;
-def        : FPALUSDynFrmAlias<FMUL_S, "fmul.s">;
-def FDIV_S : FPALUS_rr_frm<0b0001100, "fdiv.s">,
+def FDIV_S : FPALU_rr_frm<0b0001100, "fdiv.s", FPR32>,
              Sched<[WriteFDiv32, ReadFDiv32, ReadFDiv32]>;
-def        : FPALUSDynFrmAlias<FDIV_S, "fdiv.s">;
 
-def FSQRT_S : FPUnaryOp_r_frm<0b0101100, FPR32, FPR32, "fsqrt.s">,
-              Sched<[WriteFSqrt32, ReadFSqrt32]> {
-  let rs2 = 0b00000;
-}
+def        : FPALUDynFrmAlias<FADD_S, "fadd.s", FPR32>;
+def        : FPALUDynFrmAlias<FSUB_S, "fsub.s", FPR32>;
+def        : FPALUDynFrmAlias<FMUL_S, "fmul.s", FPR32>;
+def        : FPALUDynFrmAlias<FDIV_S, "fdiv.s", FPR32>;
+
+def FSQRT_S : FPUnaryOp_r_frm<0b0101100, 0b00000, FPR32, FPR32, "fsqrt.s">,
+              Sched<[WriteFSqrt32, ReadFSqrt32]>;
 def         : FPUnaryOpDynFrmAlias<FSQRT_S, "fsqrt.s", FPR32, FPR32>;
 
-def FSGNJ_S  : FPALUS_rr<0b0010000, 0b000, "fsgnj.s">,
-               Sched<[WriteFSGNJ32, ReadFSGNJ32, ReadFSGNJ32]>;
-def FSGNJN_S : FPALUS_rr<0b0010000, 0b001, "fsgnjn.s">,
-               Sched<[WriteFSGNJ32, ReadFSGNJ32, ReadFSGNJ32]>;
-def FSGNJX_S : FPALUS_rr<0b0010000, 0b010, "fsgnjx.s">,
-               Sched<[WriteFSGNJ32, ReadFSGNJ32, ReadFSGNJ32]>;
-def FMIN_S   : FPALUS_rr<0b0010100, 0b000, "fmin.s">,
-               Sched<[WriteFMinMax32, ReadFMinMax32, ReadFMinMax32]>;
-def FMAX_S   : FPALUS_rr<0b0010100, 0b001, "fmax.s">,
-               Sched<[WriteFMinMax32, ReadFMinMax32, ReadFMinMax32]>;
-
-def FCVT_W_S : FPUnaryOp_r_frm<0b1100000, GPR, FPR32, "fcvt.w.s">,
-               Sched<[WriteFCvtF32ToI32, ReadFCvtF32ToI32]> {
-  let rs2 = 0b00000;
+let SchedRW = [WriteFSGNJ32, ReadFSGNJ32, ReadFSGNJ32],
+    mayRaiseFPException = 0 in {
+def FSGNJ_S  : FPALU_rr<0b0010000, 0b000, "fsgnj.s", FPR32>;
+def FSGNJN_S : FPALU_rr<0b0010000, 0b001, "fsgnjn.s", FPR32>;
+def FSGNJX_S : FPALU_rr<0b0010000, 0b010, "fsgnjx.s", FPR32>;
 }
-def          : FPUnaryOpDynFrmAlias<FCVT_W_S, "fcvt.w.s", GPR, FPR32>;
 
-def FCVT_WU_S : FPUnaryOp_r_frm<0b1100000, GPR, FPR32, "fcvt.wu.s">,
-                Sched<[WriteFCvtF32ToI32, ReadFCvtF32ToI32]> {
-  let rs2 = 0b00001;
+let SchedRW = [WriteFMinMax32, ReadFMinMax32, ReadFMinMax32] in {
+def FMIN_S   : FPALU_rr<0b0010100, 0b000, "fmin.s", FPR32>;
+def FMAX_S   : FPALU_rr<0b0010100, 0b001, "fmax.s", FPR32>;
 }
-def           : FPUnaryOpDynFrmAlias<FCVT_WU_S, "fcvt.wu.s", GPR, FPR32>;
 
-def FMV_X_W : FPUnaryOp_r<0b1110000, 0b000, GPR, FPR32, "fmv.x.w">,
-              Sched<[WriteFMovF32ToI32, ReadFMovF32ToI32]> {
-  let rs2 = 0b00000;
-}
+def FCVT_W_S : FPUnaryOp_r_frm<0b1100000, 0b00000, GPR, FPR32, "fcvt.w.s">,
+               Sched<[WriteFCvtF32ToI32, ReadFCvtF32ToI32]>;
+def          : FPUnaryOpDynFrmAlias<FCVT_W_S, "fcvt.w.s", GPR, FPR32>;
+
+def FCVT_WU_S : FPUnaryOp_r_frm<0b1100000, 0b00001, GPR, FPR32, "fcvt.wu.s">,
+                Sched<[WriteFCvtF32ToI32, ReadFCvtF32ToI32]>;
+def           : FPUnaryOpDynFrmAlias<FCVT_WU_S, "fcvt.wu.s", GPR, FPR32>;
 
-def FEQ_S : FPCmpS_rr<0b010, "feq.s">;
-def FLT_S : FPCmpS_rr<0b001, "flt.s">;
-def FLE_S : FPCmpS_rr<0b000, "fle.s">;
+let mayRaiseFPException = 0 in
+def FMV_X_W : FPUnaryOp_r<0b1110000, 0b00000, 0b000, GPR, FPR32, "fmv.x.w">,
+              Sched<[WriteFMovF32ToI32, ReadFMovF32ToI32]>;
 
-def FCLASS_S : FPUnaryOp_r<0b1110000, 0b001, GPR, FPR32, "fclass.s">,
-               Sched<[WriteFClass32, ReadFClass32]> {
-  let rs2 = 0b00000;
+let SchedRW = [WriteFCmp32, ReadFCmp32, ReadFCmp32] in {
+def FEQ_S : FPCmp_rr<0b1010000, 0b010, "feq.s", FPR32>;
+def FLT_S : FPCmp_rr<0b1010000, 0b001, "flt.s", FPR32>;
+def FLE_S : FPCmp_rr<0b1010000, 0b000, "fle.s", FPR32>;
 }
 
-def FCVT_S_W : FPUnaryOp_r_frm<0b1101000, FPR32, GPR, "fcvt.s.w">,
-               Sched<[WriteFCvtI32ToF32, ReadFCvtI32ToF32]> {
-  let rs2 = 0b00000;
-}
+let mayRaiseFPException = 0 in
+def FCLASS_S : FPUnaryOp_r<0b1110000, 0b00000, 0b001, GPR, FPR32, "fclass.s">,
+               Sched<[WriteFClass32, ReadFClass32]>;
+
+def FCVT_S_W : FPUnaryOp_r_frm<0b1101000, 0b00000, FPR32, GPR, "fcvt.s.w">,
+               Sched<[WriteFCvtI32ToF32, ReadFCvtI32ToF32]>;
 def          : FPUnaryOpDynFrmAlias<FCVT_S_W, "fcvt.s.w", FPR32, GPR>;
 
-def FCVT_S_WU : FPUnaryOp_r_frm<0b1101000, FPR32, GPR, "fcvt.s.wu">,
-                Sched<[WriteFCvtI32ToF32, ReadFCvtI32ToF32]> {
-  let rs2 = 0b00001;
-}
+def FCVT_S_WU : FPUnaryOp_r_frm<0b1101000, 0b00001, FPR32, GPR, "fcvt.s.wu">,
+                Sched<[WriteFCvtI32ToF32, ReadFCvtI32ToF32]>;
 def           : FPUnaryOpDynFrmAlias<FCVT_S_WU, "fcvt.s.wu", FPR32, GPR>;
 
-def FMV_W_X : FPUnaryOp_r<0b1111000, 0b000, FPR32, GPR, "fmv.w.x">,
-              Sched<[WriteFMovI32ToF32, ReadFMovI32ToF32]> {
-  let rs2 = 0b00000;
-}
+let mayRaiseFPException = 0 in
+def FMV_W_X : FPUnaryOp_r<0b1111000, 0b00000, 0b000, FPR32, GPR, "fmv.w.x">,
+              Sched<[WriteFMovI32ToF32, ReadFMovI32ToF32]>;
 } // Predicates = [HasStdExtF]
 
 let Predicates = [HasStdExtF, IsRV64] in {
-def FCVT_L_S  : FPUnaryOp_r_frm<0b1100000, GPR, FPR32, "fcvt.l.s">,
-                Sched<[WriteFCvtF32ToI64, ReadFCvtF32ToI64]> {
-  let rs2 = 0b00010;
-}
+def FCVT_L_S  : FPUnaryOp_r_frm<0b1100000, 0b00010, GPR, FPR32, "fcvt.l.s">,
+                Sched<[WriteFCvtF32ToI64, ReadFCvtF32ToI64]>;
 def           : FPUnaryOpDynFrmAlias<FCVT_L_S, "fcvt.l.s", GPR, FPR32>;
 
-def FCVT_LU_S  : FPUnaryOp_r_frm<0b1100000, GPR, FPR32, "fcvt.lu.s">,
-                 Sched<[WriteFCvtF32ToI64, ReadFCvtF32ToI64]> {
-  let rs2 = 0b00011;
-}
+def FCVT_LU_S  : FPUnaryOp_r_frm<0b1100000, 0b00011, GPR, FPR32, "fcvt.lu.s">,
+                 Sched<[WriteFCvtF32ToI64, ReadFCvtF32ToI64]>;
 def            : FPUnaryOpDynFrmAlias<FCVT_LU_S, "fcvt.lu.s", GPR, FPR32>;
 
-def FCVT_S_L : FPUnaryOp_r_frm<0b1101000, FPR32, GPR, "fcvt.s.l">,
-               Sched<[WriteFCvtI64ToF32, ReadFCvtI64ToF32]> {
-  let rs2 = 0b00010;
-}
+def FCVT_S_L : FPUnaryOp_r_frm<0b1101000, 0b00010, FPR32, GPR, "fcvt.s.l">,
+               Sched<[WriteFCvtI64ToF32, ReadFCvtI64ToF32]>;
 def          : FPUnaryOpDynFrmAlias<FCVT_S_L, "fcvt.s.l", FPR32, GPR>;
 
-def FCVT_S_LU : FPUnaryOp_r_frm<0b1101000, FPR32, GPR, "fcvt.s.lu">,
-                Sched<[WriteFCvtI64ToF32, ReadFCvtI64ToF32]> {
-  let rs2 = 0b00011;
-}
+def FCVT_S_LU : FPUnaryOp_r_frm<0b1101000, 0b00011, FPR32, GPR, "fcvt.s.lu">,
+                Sched<[WriteFCvtI64ToF32, ReadFCvtI64ToF32]>;
 def           : FPUnaryOpDynFrmAlias<FCVT_S_LU, "fcvt.s.lu", FPR32, GPR>;
 } // Predicates = [HasStdExtF, IsRV64]
 
@@ -320,12 +327,12 @@ def : Pat<(f32 (fpimm0)), (FMV_W_X X0)>;
 
 /// Float arithmetic operations
 
-def : PatFpr32Fpr32DynFrm<fadd, FADD_S>;
-def : PatFpr32Fpr32DynFrm<fsub, FSUB_S>;
-def : PatFpr32Fpr32DynFrm<fmul, FMUL_S>;
-def : PatFpr32Fpr32DynFrm<fdiv, FDIV_S>;
+def : PatFpr32Fpr32DynFrm<any_fadd, FADD_S>;
+def : PatFpr32Fpr32DynFrm<any_fsub, FSUB_S>;
+def : PatFpr32Fpr32DynFrm<any_fmul, FMUL_S>;
+def : PatFpr32Fpr32DynFrm<any_fdiv, FDIV_S>;
 
-def : Pat<(fsqrt FPR32:$rs1), (FSQRT_S FPR32:$rs1, 0b111)>;
+def : Pat<(any_fsqrt FPR32:$rs1), (FSQRT_S FPR32:$rs1, 0b111)>;
 
 def : Pat<(fneg FPR32:$rs1), (FSGNJN_S $rs1, $rs1)>;
 def : Pat<(fabs FPR32:$rs1), (FSGNJX_S $rs1, $rs1)>;
@@ -334,19 +341,19 @@ def : PatFpr32Fpr32<fcopysign, FSGNJ_S>;
 def : Pat<(fcopysign FPR32:$rs1, (fneg FPR32:$rs2)), (FSGNJN_S $rs1, $rs2)>;
 
 // fmadd: rs1 * rs2 + rs3
-def : Pat<(fma FPR32:$rs1, FPR32:$rs2, FPR32:$rs3),
+def : Pat<(any_fma FPR32:$rs1, FPR32:$rs2, FPR32:$rs3),
           (FMADD_S $rs1, $rs2, $rs3, 0b111)>;
 
 // fmsub: rs1 * rs2 - rs3
-def : Pat<(fma FPR32:$rs1, FPR32:$rs2, (fneg FPR32:$rs3)),
+def : Pat<(any_fma FPR32:$rs1, FPR32:$rs2, (fneg FPR32:$rs3)),
           (FMSUB_S FPR32:$rs1, FPR32:$rs2, FPR32:$rs3, 0b111)>;
 
 // fnmsub: -rs1 * rs2 + rs3
-def : Pat<(fma (fneg FPR32:$rs1), FPR32:$rs2, FPR32:$rs3),
+def : Pat<(any_fma (fneg FPR32:$rs1), FPR32:$rs2, FPR32:$rs3),
           (FNMSUB_S FPR32:$rs1, FPR32:$rs2, FPR32:$rs3, 0b111)>;
 
 // fnmadd: -rs1 * rs2 - rs3
-def : Pat<(fma (fneg FPR32:$rs1), FPR32:$rs2, (fneg FPR32:$rs3)),
+def : Pat<(any_fma (fneg FPR32:$rs1), FPR32:$rs2, (fneg FPR32:$rs3)),
           (FNMADD_S FPR32:$rs1, FPR32:$rs2, FPR32:$rs3, 0b111)>;
 
 // The ratified 20191213 ISA spec defines fmin and fmax in a way that matches
@@ -382,8 +389,8 @@ def : Pat<(bitconvert (i32 GPR:$rs1)), (FMV_W_X GPR:$rs1)>;
 def : Pat<(i32 (bitconvert FPR32:$rs1)), (FMV_X_W FPR32:$rs1)>;
 
 // float->[u]int. Round-to-zero must be used.
-def : Pat<(i32 (fp_to_sint FPR32:$rs1)), (FCVT_W_S $rs1, 0b001)>;
-def : Pat<(i32 (fp_to_uint FPR32:$rs1)), (FCVT_WU_S $rs1, 0b001)>;
+def : Pat<(i32 (any_fp_to_sint FPR32:$rs1)), (FCVT_W_S $rs1, 0b001)>;
+def : Pat<(i32 (any_fp_to_uint FPR32:$rs1)), (FCVT_WU_S $rs1, 0b001)>;
 
 // Saturating float->[u]int32.
 def : Pat<(i32 (riscv_fcvt_x_rtz FPR32:$rs1)), (FCVT_W_S $rs1, 0b001)>;
@@ -396,8 +403,8 @@ def : Pat<(i32 (lrint FPR32:$rs1)), (FCVT_W_S $rs1, 0b111)>;
 def : Pat<(i32 (lround FPR32:$rs1)), (FCVT_W_S $rs1, 0b100)>;
 
 // [u]int->float. Match GCC and default to using dynamic rounding mode.
-def : Pat<(sint_to_fp (i32 GPR:$rs1)), (FCVT_S_W $rs1, 0b111)>;
-def : Pat<(uint_to_fp (i32 GPR:$rs1)), (FCVT_S_WU $rs1, 0b111)>;
+def : Pat<(any_sint_to_fp (i32 GPR:$rs1)), (FCVT_S_W $rs1, 0b111)>;
+def : Pat<(any_uint_to_fp (i32 GPR:$rs1)), (FCVT_S_WU $rs1, 0b111)>;
 } // Predicates = [HasStdExtF, IsRV32]
 
 let Predicates = [HasStdExtF, IsRV64] in {
@@ -410,12 +417,12 @@ def : Pat<(sext_inreg (riscv_fmv_x_anyextw_rv64 FPR32:$src), i32),
 // Use target specific isd nodes to help us remember the result is sign
 // extended. Matching sext_inreg+fptoui/fptosi may cause the conversion to be
 // duplicated if it has another user that didn't need the sign_extend.
-def : Pat<(riscv_fcvt_w_rtz_rv64 FPR32:$rs1),  (FCVT_W_S $rs1, 0b001)>;
-def : Pat<(riscv_fcvt_wu_rtz_rv64 FPR32:$rs1), (FCVT_WU_S $rs1, 0b001)>;
+def : Pat<(riscv_any_fcvt_w_rtz_rv64 FPR32:$rs1),  (FCVT_W_S $rs1, 0b001)>;
+def : Pat<(riscv_any_fcvt_wu_rtz_rv64 FPR32:$rs1), (FCVT_WU_S $rs1, 0b001)>;
 
 // float->[u]int64. Round-to-zero must be used.
-def : Pat<(i64 (fp_to_sint FPR32:$rs1)), (FCVT_L_S $rs1, 0b001)>;
-def : Pat<(i64 (fp_to_uint FPR32:$rs1)), (FCVT_LU_S $rs1, 0b001)>;
+def : Pat<(i64 (any_fp_to_sint FPR32:$rs1)), (FCVT_L_S $rs1, 0b001)>;
+def : Pat<(i64 (any_fp_to_uint FPR32:$rs1)), (FCVT_LU_S $rs1, 0b001)>;
 
 // Saturating float->[u]int64.
 def : Pat<(i64 (riscv_fcvt_x_rtz FPR32:$rs1)), (FCVT_L_S $rs1, 0b001)>;
@@ -430,8 +437,8 @@ def : Pat<(i64 (lround FPR32:$rs1)), (FCVT_L_S $rs1, 0b100)>;
 def : Pat<(i64 (llround FPR32:$rs1)), (FCVT_L_S $rs1, 0b100)>;
 
 // [u]int->fp. Match GCC and default to using dynamic rounding mode.
-def : Pat<(sint_to_fp (i64 (sexti32 (i64 GPR:$rs1)))), (FCVT_S_W $rs1, 0b111)>;
-def : Pat<(uint_to_fp (i64 (zexti32 (i64 GPR:$rs1)))), (FCVT_S_WU $rs1, 0b111)>;
-def : Pat<(sint_to_fp (i64 GPR:$rs1)), (FCVT_S_L $rs1, 0b111)>;
-def : Pat<(uint_to_fp (i64 GPR:$rs1)), (FCVT_S_LU $rs1, 0b111)>;
+def : Pat<(any_sint_to_fp (i64 (sexti32 (i64 GPR:$rs1)))), (FCVT_S_W $rs1, 0b111)>;
+def : Pat<(any_uint_to_fp (i64 (zexti32 (i64 GPR:$rs1)))), (FCVT_S_WU $rs1, 0b111)>;
+def : Pat<(any_sint_to_fp (i64 GPR:$rs1)), (FCVT_S_L $rs1, 0b111)>;
+def : Pat<(any_uint_to_fp (i64 GPR:$rs1)), (FCVT_S_LU $rs1, 0b111)>;
 } // Predicates = [HasStdExtF, IsRV64]
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoM.td b/llvm/lib/Target/RISCV/RISCVInstrInfoM.td
index a037dbf585ce..b62e23d3b0fa 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoM.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoM.td
@@ -96,14 +96,6 @@ def : Pat<(srem (sexti32 (i64 GPR:$rs1)), (sexti32 (i64 GPR:$rs2))),
           (REMW GPR:$rs1, GPR:$rs2)>;
 } // Predicates = [HasStdExtM, IsRV64]
 
-// Pattern to detect constants with no more than 32 active bits that can't
-// be materialized with lui+addiw.
-def uimm32_not_simm32 : PatLeaf<(XLenVT GPR:$a), [{
-  auto *C = dyn_cast<ConstantSDNode>(N);
-  return C && C->hasOneUse() && isUInt<32>(C->getZExtValue()) &&
-         !isInt<32>(C->getSExtValue());
-}]>;
-
 let Predicates = [HasStdExtM, IsRV64, NotHasStdExtZba] in {
 // Special case for calculating the full 64-bit product of a 32x32 unsigned
 // multiply where the inputs aren't known to be zero extended. We can shift the
@@ -111,9 +103,4 @@ let Predicates = [HasStdExtM, IsRV64, NotHasStdExtZba] in {
 // zeroing the upper 32 bits.
 def : Pat<(i64 (mul (and GPR:$rs1, 0xffffffff), (and GPR:$rs2, 0xffffffff))),
           (MULHU (SLLI GPR:$rs1, 32), (SLLI GPR:$rs2, 32))>;
-// The RHS could also be a constant that is hard to materialize. By shifting
-// left we can allow constant materialization to use LUI+ADDIW via
-// hasAllWUsers.
-def : Pat<(i64 (mul (and GPR:$rs1, 0xffffffff), uimm32_not_simm32:$rs2)),
-          (MULHU (SLLI GPR:$rs1, 32), (SLLI GPR:$rs2, 32))>;
 } // Predicates = [HasStdExtM, IsRV64, NotHasStdExtZba]
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
index 3d5f9bc54731..173ae43a08d6 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
@@ -338,29 +338,6 @@ class VALUVs2<bits<6> funct6, bits<5> vs1, RISCVVFormat opv, string opcodestr>
                opcodestr, "$vd, $vs2$vm">;
 } // hasSideEffects = 0, mayLoad = 0, mayStore = 0
 
-let hasSideEffects = 0, mayLoad = 1, mayStore = 1 in {
-// vamo vd, (rs1), vs2, vd, vm
-class VAMOWd<RISCVAMOOP amoop, RISCVWidth width, string opcodestr>
-    : RVInstVAMO<amoop, width.Value{2-0}, (outs VR:$vd_wd),
-            (ins GPR:$rs1, VR:$vs2, VR:$vd, VMaskOp:$vm),
-            opcodestr, "$vd_wd, (${rs1}), $vs2, $vd$vm"> {
-    let Constraints = "$vd_wd = $vd";
-    let wd = 1;
-    bits<5> vd;
-    let Inst{11-7} = vd;
-}
-
-// vamo x0, (rs1), vs2, vs3, vm
-class VAMONoWd<RISCVAMOOP amoop, RISCVWidth width, string opcodestr>
-    : RVInstVAMO<amoop, width.Value{2-0}, (outs),
-            (ins GPR:$rs1, VR:$vs2, VR:$vs3, VMaskOp:$vm),
-            opcodestr, "x0, (${rs1}), $vs2, $vs3$vm"> {
-    bits<5> vs3;
-    let Inst{11-7} = vs3;
-}
-
-} // hasSideEffects = 0, mayLoad = 1, mayStore = 1
-
 //===----------------------------------------------------------------------===//
 // Combination of instruction classes.
 // Use these multiclasses to define instructions more easily.
@@ -779,11 +756,6 @@ multiclass VCPR_MV_Mask<string opcodestr, bits<6> funct6, string vm = "v"> {
            Sched<[WriteVCompressV, ReadVCompressV, ReadVCompressV]>;
 }
 
-multiclass VAMO<RISCVAMOOP amoop, RISCVWidth width, string opcodestr> {
-  def _WD : VAMOWd<amoop, width, opcodestr>;
-  def _UNWD : VAMONoWd<amoop, width, opcodestr>;
-}
-
 multiclass VWholeLoadN<bits<3> nf, string opcodestr, RegisterClass VRC> {
   foreach l = [8, 16, 32, 64] in {
     defvar w = !cast<RISCVWidth>("LSWidth" # l);
@@ -822,7 +794,7 @@ foreach eew = [8, 16, 32, 64] in {
   // Vector Strided Instructions
   def VLSE#eew#_V  : VStridedLoad<w,  "vlse"#eew#".v">, VLSSched<eew>;
   def VSSE#eew#_V  : VStridedStore<w,  "vsse"#eew#".v">, VSSSched<eew>;
-  
+
   // Vector Indexed Instructions
   def VLUXEI#eew#_V :
     VIndexedLoad<MOPLDIndexedUnord, w, "vluxei"#eew#".v">, VLXSched<eew, "U">;
@@ -1416,13 +1388,20 @@ defm VCOMPRESS_V : VCPR_MV_Mask<"vcompress", 0b010111>;
 
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0,
     RVVConstraint = NoConstraint in {
-foreach n = [1, 2, 4, 8] in {
-  def VMV#n#R_V  : RVInstV<0b100111, !add(n, -1), OPIVI, (outs VR:$vd),
-                           (ins VR:$vs2), "vmv" # n # "r.v", "$vd, $vs2">,
-                   VMVRSched<n> {
+def VMV1R_V  : RVInstV<0b100111, 0, OPIVI, (outs VR:$vd), (ins VR:$vs2),
+                       "vmv1r.v", "$vd, $vs2">, VMVRSched<1> {
   let Uses = [];
   let vm = 1;
 }
+// A future extension may relax the vector register alignment restrictions.
+foreach n = [2, 4, 8] in {
+  defvar vrc = !cast<VReg>("VRM"#n);
+  def VMV#n#R_V  : RVInstV<0b100111, !add(n, -1), OPIVI, (outs vrc:$vd),
+                           (ins vrc:$vs2), "vmv" # n # "r.v", "$vd, $vs2">,
+                   VMVRSched<n> {
+    let Uses = [];
+    let vm = 1;
+  }
 }
 } // hasSideEffects = 0, mayLoad = 0, mayStore = 0
 } // Predicates = [HasStdExtV]
@@ -1462,31 +1441,4 @@ let Predicates = [HasStdExtZvlsseg] in {
   }
 } // Predicates = [HasStdExtZvlsseg]
 
-let Predicates = [HasStdExtZvamo, HasStdExtA] in {
-  foreach eew = [8, 16, 32] in {
-    defvar w = !cast<RISCVWidth>("LSWidth"#eew);
-    defm VAMOSWAPEI#eew : VAMO<AMOOPVamoSwap, w, "vamoswapei"#eew#".v">;
-    defm VAMOADDEI#eew : VAMO<AMOOPVamoAdd, w, "vamoaddei"#eew#".v">;
-    defm VAMOXOREI#eew : VAMO<AMOOPVamoXor, w, "vamoxorei"#eew#".v">;
-    defm VAMOANDEI#eew : VAMO<AMOOPVamoAnd, w, "vamoandei"#eew#".v">;
-    defm VAMOOREI#eew : VAMO<AMOOPVamoOr, w, "vamoorei"#eew#".v">;
-    defm VAMOMINEI#eew : VAMO<AMOOPVamoMin, w, "vamominei"#eew#".v">;
-    defm VAMOMAXEI#eew : VAMO<AMOOPVamoMax, w, "vamomaxei"#eew#".v">;
-    defm VAMOMINUEI#eew : VAMO<AMOOPVamoMinu, w, "vamominuei"#eew#".v">;
-    defm VAMOMAXUEI#eew : VAMO<AMOOPVamoMaxu, w, "vamomaxuei"#eew#".v">;
-  }
-} // Predicates = [HasStdExtZvamo, HasStdExtA]
-
-let Predicates = [HasStdExtZvamo, HasStdExtA, IsRV64] in {
-  defm VAMOSWAPEI64 : VAMO<AMOOPVamoSwap, LSWidth64, "vamoswapei64.v">;
-  defm VAMOADDEI64 : VAMO<AMOOPVamoAdd, LSWidth64, "vamoaddei64.v">;
-  defm VAMOXOREI64 : VAMO<AMOOPVamoXor, LSWidth64, "vamoxorei64.v">;
-  defm VAMOANDEI64 : VAMO<AMOOPVamoAnd, LSWidth64, "vamoandei64.v">;
-  defm VAMOOREI64 : VAMO<AMOOPVamoOr, LSWidth64, "vamoorei64.v">;
-  defm VAMOMINEI64 : VAMO<AMOOPVamoMin, LSWidth64, "vamominei64.v">;
-  defm VAMOMAXEI64 : VAMO<AMOOPVamoMax, LSWidth64, "vamomaxei64.v">;
-  defm VAMOMINUEI64 : VAMO<AMOOPVamoMinu, LSWidth64, "vamominuei64.v">;
-  defm VAMOMAXUEI64 : VAMO<AMOOPVamoMaxu, LSWidth64, "vamomaxuei64.v">;
-} // Predicates = [HasStdExtZvamo, HasStdExtA, IsRV64]
-
 include "RISCVInstrInfoVPseudos.td"
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
index a82e333e6bab..073fa605e0fb 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
@@ -1124,68 +1124,6 @@ class VPseudoTernaryNoMaskWithPolicy<VReg RetClass,
   let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst);
 }
 
-class VPseudoAMOWDNoMask<VReg RetClass,
-                         VReg Op1Class> :
-        Pseudo<(outs GetVRegNoV0<RetClass>.R:$vd_wd),
-               (ins GPR:$rs1,
-                    Op1Class:$vs2,
-                    GetVRegNoV0<RetClass>.R:$vd,
-                    AVL:$vl, ixlenimm:$sew), []>,
-        RISCVVPseudo {
-  let mayLoad = 1;
-  let mayStore = 1;
-  let hasSideEffects = 1;
-  let Constraints = "$vd_wd = $vd";
-  let HasVLOp = 1;
-  let HasSEWOp = 1;
-  let HasDummyMask = 1;
-  let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst);
-}
-
-class VPseudoAMOWDMask<VReg RetClass,
-                       VReg Op1Class> :
-        Pseudo<(outs GetVRegNoV0<RetClass>.R:$vd_wd),
-               (ins GPR:$rs1,
-                    Op1Class:$vs2,
-                    GetVRegNoV0<RetClass>.R:$vd,
-                    VMaskOp:$vm, AVL:$vl, ixlenimm:$sew), []>,
-        RISCVVPseudo {
-  let mayLoad = 1;
-  let mayStore = 1;
-  let hasSideEffects = 1;
-  let Constraints = "$vd_wd = $vd";
-  let HasVLOp = 1;
-  let HasSEWOp = 1;
-  let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst);
-}
-
-multiclass VPseudoAMOEI<int eew> {
-  // Standard scalar AMO supports 32, 64, and 128 Mem data bits,
-  // and in the base vector "V" extension, only SEW up to ELEN = max(XLEN, FLEN)
-  // are required to be supported.
-  // therefore only [32, 64] is allowed here.
-  foreach sew = [32, 64] in {
-    foreach lmul = MxSet<sew>.m in {
-      defvar octuple_lmul = lmul.octuple;
-      // Calculate emul = eew * lmul / sew
-      defvar octuple_emul = !srl(!mul(eew, octuple_lmul), log2<sew>.val);
-      if !and(!ge(octuple_emul, 1), !le(octuple_emul, 64)) then {
-        defvar emulMX = octuple_to_str<octuple_emul>.ret;
-        defvar emul= !cast<LMULInfo>("V_" # emulMX);
-        let VLMul = lmul.value in {
-          def "_WD_" # lmul.MX # "_" # emulMX : VPseudoAMOWDNoMask<lmul.vrclass, emul.vrclass>;
-          def "_WD_" # lmul.MX # "_" # emulMX # "_MASK" : VPseudoAMOWDMask<lmul.vrclass, emul.vrclass>;
-        }
-      }
-    }
-  }
-}
-
-multiclass VPseudoAMO {
-  foreach eew = EEWList in
-  defm "EI" # eew : VPseudoAMOEI<eew>;
-}
-
 class VPseudoUSSegLoadNoMask<VReg RetClass, int EEW, bits<4> NF, bit isFF>:
       Pseudo<(outs RetClass:$rd),
              (ins GPR:$rs1, AVL:$vl, ixlenimm:$sew),[]>,
@@ -1376,17 +1314,35 @@ class VPseudoISegStoreMask<VReg ValClass, VReg IdxClass, int EEW, bits<3> LMUL,
   let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst);
 }
 
-multiclass VPseudoUSLoad<bit isFF> {
+multiclass VPseudoUSLoad {
   foreach eew = EEWList in {
     foreach lmul = MxSet<eew>.m in {
       defvar LInfo = lmul.MX;
       defvar vreg = lmul.vrclass;
-      defvar FFStr = !if(isFF, "FF", "");
       let VLMul = lmul.value in {
-        def "E" # eew # FFStr # "_V_" # LInfo :
-          VPseudoUSLoadNoMask<vreg, eew, isFF>;
-        def "E" # eew # FFStr # "_V_" # LInfo # "_MASK" :
-          VPseudoUSLoadMask<vreg, eew, isFF>;
+        def "E" # eew # "_V_" # LInfo :
+          VPseudoUSLoadNoMask<vreg, eew, false>,
+          VLESched<eew>;
+        def "E" # eew # "_V_" # LInfo # "_MASK" :
+          VPseudoUSLoadMask<vreg, eew, false>,
+          VLESched<eew>;
+      }
+    }
+  }
+}
+
+multiclass VPseudoFFLoad {
+  foreach eew = EEWList in {
+    foreach lmul = MxSet<eew>.m in {
+      defvar LInfo = lmul.MX;
+      defvar vreg = lmul.vrclass;
+      let VLMul = lmul.value in {
+        def "E" # eew # "FF_V_" # LInfo :
+          VPseudoUSLoadNoMask<vreg, eew, true>,
+          VLFSched<eew>;
+        def "E" # eew # "FF_V_" # LInfo # "_MASK" :
+          VPseudoUSLoadMask<vreg, eew, true>,
+          VLFSched<eew>;
       }
     }
   }
@@ -1406,8 +1362,10 @@ multiclass VPseudoSLoad {
       defvar LInfo = lmul.MX;
       defvar vreg = lmul.vrclass;
       let VLMul = lmul.value in {
-        def "E" # eew # "_V_" # LInfo : VPseudoSLoadNoMask<vreg, eew>;
-        def "E" # eew # "_V_" # LInfo # "_MASK" : VPseudoSLoadMask<vreg, eew>;
+        def "E" # eew # "_V_" # LInfo : VPseudoSLoadNoMask<vreg, eew>,
+                                        VLSSched<eew>;
+        def "E" # eew # "_V_" # LInfo # "_MASK" : VPseudoSLoadMask<vreg, eew>,
+                                                  VLSSched<eew>;
       }
     }
   }
@@ -1427,11 +1385,14 @@ multiclass VPseudoILoad<bit Ordered> {
           defvar Vreg = lmul.vrclass;
           defvar IdxVreg = idx_lmul.vrclass;
           defvar HasConstraint = !ne(sew, eew);
+          defvar Order = !if(Ordered, "O", "U");
           let VLMul = lmul.value in {
             def "EI" # eew # "_V_" # IdxLInfo # "_" # LInfo :
-              VPseudoILoadNoMask<Vreg, IdxVreg, eew, idx_lmul.value, Ordered, HasConstraint>;
+              VPseudoILoadNoMask<Vreg, IdxVreg, eew, idx_lmul.value, Ordered, HasConstraint>,
+              VLXSched<eew, Order>;
             def "EI" # eew # "_V_" # IdxLInfo # "_" # LInfo # "_MASK" :
-              VPseudoILoadMask<Vreg, IdxVreg, eew, idx_lmul.value, Ordered, HasConstraint>;
+              VPseudoILoadMask<Vreg, IdxVreg, eew, idx_lmul.value, Ordered, HasConstraint>,
+              VLXSched<eew, Order>;
           }
         }
       }
@@ -1445,8 +1406,10 @@ multiclass VPseudoUSStore {
       defvar LInfo = lmul.MX;
       defvar vreg = lmul.vrclass;
       let VLMul = lmul.value in {
-        def "E" # eew # "_V_" # LInfo : VPseudoUSStoreNoMask<vreg, eew>;
-        def "E" # eew # "_V_" # LInfo # "_MASK" : VPseudoUSStoreMask<vreg, eew>;
+        def "E" # eew # "_V_" # LInfo : VPseudoUSStoreNoMask<vreg, eew>,
+                                        VSESched<eew>;
+        def "E" # eew # "_V_" # LInfo # "_MASK" : VPseudoUSStoreMask<vreg, eew>,
+                                                  VSESched<eew>;
       }
     }
   }
@@ -1466,8 +1429,10 @@ multiclass VPseudoSStore {
       defvar LInfo = lmul.MX;
       defvar vreg = lmul.vrclass;
       let VLMul = lmul.value in {
-        def "E" # eew # "_V_" # LInfo : VPseudoSStoreNoMask<vreg, eew>;
-        def "E" # eew # "_V_" # LInfo # "_MASK" : VPseudoSStoreMask<vreg, eew>;
+        def "E" # eew # "_V_" # LInfo : VPseudoSStoreNoMask<vreg, eew>,
+                                        VSSSched<eew>;
+        def "E" # eew # "_V_" # LInfo # "_MASK" : VPseudoSStoreMask<vreg, eew>,
+                                                  VSSSched<eew>;
       }
     }
   }
@@ -1486,11 +1451,14 @@ multiclass VPseudoIStore<bit Ordered> {
           defvar idx_lmul = !cast<LMULInfo>("V_" # IdxLInfo);
           defvar Vreg = lmul.vrclass;
           defvar IdxVreg = idx_lmul.vrclass;
+          defvar Order = !if(Ordered, "O", "U");
           let VLMul = lmul.value in {
             def "EI" # eew # "_V_" # IdxLInfo # "_" # LInfo :
-              VPseudoIStoreNoMask<Vreg, IdxVreg, eew, idx_lmul.value, Ordered>;
+              VPseudoIStoreNoMask<Vreg, IdxVreg, eew, idx_lmul.value, Ordered>,
+              VSXSched<eew, Order>;
             def "EI" # eew # "_V_" # IdxLInfo # "_" # LInfo # "_MASK" :
-              VPseudoIStoreMask<Vreg, IdxVreg, eew, idx_lmul.value, Ordered>;
+              VPseudoIStoreMask<Vreg, IdxVreg, eew, idx_lmul.value, Ordered>,
+              VSXSched<eew, Order>;
           }
         }
       }
@@ -1498,32 +1466,50 @@ multiclass VPseudoIStore<bit Ordered> {
   }
 }
 
-multiclass VPseudoUnaryS_M {
+multiclass VPseudoVPOP_M {
   foreach mti = AllMasks in
   {
     let VLMul = mti.LMul.value in {
-      def "_M_" # mti.BX : VPseudoUnaryNoMask<GPR, VR>;
-      def "_M_" # mti.BX # "_MASK" : VPseudoMaskUnarySOutMask;
+      def "_M_" # mti.BX : VPseudoUnaryNoMask<GPR, VR>,
+                           Sched<[WriteVMPopV, ReadVMPopV, ReadVMPopV]>;
+      def "_M_" # mti.BX # "_MASK" : VPseudoMaskUnarySOutMask,
+                                     Sched<[WriteVMPopV, ReadVMPopV, ReadVMPopV]>;
     }
   }
 }
 
-multiclass VPseudoUnaryM_M {
+multiclass VPseudoV1ST_M {
+  foreach mti = AllMasks in
+  {
+    let VLMul = mti.LMul.value in {
+      def "_M_" # mti.BX : VPseudoUnaryNoMask<GPR, VR>,
+                           Sched<[WriteVMFFSV, ReadVMFFSV, ReadVMFFSV]>;
+      def "_M_" # mti.BX # "_MASK" : VPseudoMaskUnarySOutMask,
+                                     Sched<[WriteVMFFSV, ReadVMFFSV, ReadVMFFSV]>;
+    }
+  }
+}
+
+multiclass VPseudoVSFS_M {
   defvar constraint = "@earlyclobber $rd";
   foreach mti = AllMasks in
   {
     let VLMul = mti.LMul.value in {
-      def "_M_" # mti.BX : VPseudoUnaryNoMask<VR, VR, constraint>;
-      def "_M_" # mti.BX # "_MASK" : VPseudoUnaryMask<VR, VR, constraint>;
+      def "_M_" # mti.BX : VPseudoUnaryNoMask<VR, VR, constraint>,
+                           Sched<[WriteVMSFSV, ReadVMSFSV, ReadVMask]>;
+      def "_M_" # mti.BX # "_MASK" : VPseudoUnaryMask<VR, VR, constraint>,
+                                     Sched<[WriteVMSFSV, ReadVMSFSV, ReadVMask]>;
     }
   }
 }
 
-multiclass VPseudoMaskNullaryV {
+multiclass VPseudoVID_V {
   foreach m = MxList.m in {
     let VLMul = m.value in {
-      def "_V_" # m.MX : VPseudoNullaryNoMask<m.vrclass>;
-      def "_V_" # m.MX # "_MASK" : VPseudoNullaryMask<m.vrclass>;
+      def "_V_" # m.MX : VPseudoNullaryNoMask<m.vrclass>,
+                         Sched<[WriteVMIdxV, ReadVMask]>;
+      def "_V_" # m.MX # "_MASK" : VPseudoNullaryMask<m.vrclass>,
+                                   Sched<[WriteVMIdxV, ReadVMask]>;
     }
   }
 }
@@ -1536,20 +1522,23 @@ multiclass VPseudoNullaryPseudoM <string BaseInst> {
   }
 }
 
-multiclass VPseudoUnaryV_M {
+multiclass VPseudoVIOT_M {
   defvar constraint = "@earlyclobber $rd";
   foreach m = MxList.m in {
     let VLMul = m.value in {
-      def "_" # m.MX : VPseudoUnaryNoMask<m.vrclass, VR, constraint>;
-      def "_" # m.MX # "_MASK" : VPseudoUnaryMask<m.vrclass, VR, constraint>;
+      def "_" # m.MX : VPseudoUnaryNoMask<m.vrclass, VR, constraint>,
+                       Sched<[WriteVMIotV, ReadVMIotV, ReadVMask]>;
+      def "_" # m.MX # "_MASK" : VPseudoUnaryMask<m.vrclass, VR, constraint>,
+                                 Sched<[WriteVMIotV, ReadVMIotV, ReadVMask]>;
     }
   }
 }
 
-multiclass VPseudoUnaryV_V_AnyMask {
+multiclass VPseudoVCPR_V {
   foreach m = MxList.m in {
     let VLMul = m.value in
-      def _VM # "_" # m.MX : VPseudoUnaryAnyMask<m.vrclass, m.vrclass>;
+      def _VM # "_" # m.MX : VPseudoUnaryAnyMask<m.vrclass, m.vrclass>,
+                             Sched<[WriteVCompressV, ReadVCompressV, ReadVCompressV]>;
   }
 }
 
@@ -1611,7 +1600,7 @@ multiclass VPseudoBinaryV_VV<string Constraint = ""> {
     defm _VV : VPseudoBinary<m.vrclass, m.vrclass, m.vrclass, m, Constraint>;
 }
 
-multiclass VPseudoBinaryV_VV_EEW<int eew, string Constraint = ""> {
+multiclass VPseudoVGTR_VV_EEW<int eew, string Constraint = ""> {
   foreach m = MxList.m in {
     foreach sew = EEWList in {
       defvar octuple_lmul = m.octuple;
@@ -1620,7 +1609,8 @@ multiclass VPseudoBinaryV_VV_EEW<int eew, string Constraint = ""> {
       if !and(!ge(octuple_emul, 1), !le(octuple_emul, 64)) then {
         defvar emulMX = octuple_to_str<octuple_emul>.ret;
         defvar emul = !cast<LMULInfo>("V_" # emulMX);
-        defm _VV : VPseudoBinaryEmul<m.vrclass, m.vrclass, emul.vrclass, m, emul, Constraint>;
+        defm _VV : VPseudoBinaryEmul<m.vrclass, m.vrclass, emul.vrclass, m, emul, Constraint>,
+                   Sched<[WriteVGatherV, ReadVGatherV, ReadVGatherV]>;
       }
     }
   }
@@ -1631,6 +1621,12 @@ multiclass VPseudoBinaryV_VX<string Constraint = ""> {
     defm "_VX" : VPseudoBinary<m.vrclass, m.vrclass, GPR, m, Constraint>;
 }
 
+multiclass VPseudoVSLD1_VX<string Constraint = ""> {
+  foreach m = MxList.m in
+    defm "_VX" : VPseudoBinary<m.vrclass, m.vrclass, GPR, m, Constraint>,
+                 Sched<[WriteVISlide1X, ReadVISlideV, ReadVISlideX, ReadVMask]>;
+}
+
 multiclass VPseudoBinaryV_VF<string Constraint = ""> {
   foreach m = MxList.m in
     foreach f = FPList.fpinfo in
@@ -1638,15 +1634,24 @@ multiclass VPseudoBinaryV_VF<string Constraint = ""> {
                                        f.fprclass, m, Constraint>;
 }
 
+multiclass VPseudoVSLD1_VF<string Constraint = ""> {
+  foreach m = MxList.m in
+    foreach f = FPList.fpinfo in
+      defm "_V" # f.FX :
+        VPseudoBinary<m.vrclass, m.vrclass, f.fprclass, m, Constraint>,
+        Sched<[WriteVFSlide1F, ReadVFSlideV, ReadVFSlideF, ReadVMask]>;
+}
+
 multiclass VPseudoBinaryV_VI<Operand ImmType = simm5, string Constraint = ""> {
   foreach m = MxList.m in
     defm _VI : VPseudoBinary<m.vrclass, m.vrclass, ImmType, m, Constraint>;
 }
 
-multiclass VPseudoBinaryM_MM {
+multiclass VPseudoVALU_MM {
   foreach m = MxList.m in
     let VLMul = m.value in {
-      def "_MM_" # m.MX : VPseudoBinaryNoMask<VR, VR, VR, "">;
+      def "_MM_" # m.MX : VPseudoBinaryNoMask<VR, VR, VR, "">,
+                          Sched<[WriteVMALUV, ReadVMALUV, ReadVMALUV]>;
     }
 }
 
@@ -1744,12 +1749,13 @@ multiclass VPseudoBinaryV_XM<bit CarryOut = 0, bit CarryIn = 1,
                            m.vrclass, GPR, m, CarryIn, Constraint>;
 }
 
-multiclass VPseudoBinaryV_FM {
+multiclass VPseudoVMRG_FM {
   foreach m = MxList.m in
     foreach f = FPList.fpinfo in
       def "_V" # f.FX # "M_" # m.MX :
         VPseudoBinaryCarryIn<GetVRegNoV0<m.vrclass>.R,
-                             m.vrclass, f.fprclass, m, /*CarryIn=*/1, "">;
+                             m.vrclass, f.fprclass, m, /*CarryIn=*/1, "">,
+        Sched<[WriteVFMergeV, ReadVFMergeV, ReadVFMergeF, ReadVMask]>;
 }
 
 multiclass VPseudoBinaryV_IM<bit CarryOut = 0, bit CarryIn = 1,
@@ -1762,76 +1768,102 @@ multiclass VPseudoBinaryV_IM<bit CarryOut = 0, bit CarryIn = 1,
                            m.vrclass, simm5, m, CarryIn, Constraint>;
 }
 
-multiclass VPseudoUnaryV_V_X_I_NoDummyMask {
+multiclass VPseudoUnaryVMV_V_X_I {
   foreach m = MxList.m in {
     let VLMul = m.value in {
-      def "_V_" # m.MX : VPseudoUnaryNoDummyMask<m.vrclass, m.vrclass>;
-      def "_X_" # m.MX : VPseudoUnaryNoDummyMask<m.vrclass, GPR>;
-      def "_I_" # m.MX : VPseudoUnaryNoDummyMask<m.vrclass, simm5>;
+      def "_V_" # m.MX : VPseudoUnaryNoDummyMask<m.vrclass, m.vrclass>,
+                         Sched<[WriteVIMovV, ReadVIMovV]>;
+      def "_X_" # m.MX : VPseudoUnaryNoDummyMask<m.vrclass, GPR>,
+                         Sched<[WriteVIMovX, ReadVIMovX]>;
+      def "_I_" # m.MX : VPseudoUnaryNoDummyMask<m.vrclass, simm5>,
+                         Sched<[WriteVIMovI]>;
     }
   }
 }
 
-multiclass VPseudoUnaryV_F_NoDummyMask {
+multiclass VPseudoVMV_F {
   foreach m = MxList.m in {
     foreach f = FPList.fpinfo in {
       let VLMul = m.value in {
-        def "_" # f.FX # "_" # m.MX : VPseudoUnaryNoDummyMask<m.vrclass, f.fprclass>;
+        def "_" # f.FX # "_" # m.MX :
+          VPseudoUnaryNoDummyMask<m.vrclass, f.fprclass>,
+          Sched<[WriteVFMovV, ReadVFMovF]>;
       }
     }
   }
 }
 
-multiclass VPseudoUnaryTAV_V {
+multiclass VPseudoVCLS_V {
   foreach m = MxList.m in {
     let VLMul = m.value in {
-      def "_V_" # m.MX : VPseudoUnaryNoMask<m.vrclass, m.vrclass>;
-      def "_V_" # m.MX # "_MASK" : VPseudoUnaryMaskTA<m.vrclass, m.vrclass>;
+      def "_V_" # m.MX : VPseudoUnaryNoMask<m.vrclass, m.vrclass>,
+                         Sched<[WriteVFClassV, ReadVFClassV, ReadVMask]>;
+      def "_V_" # m.MX # "_MASK" : VPseudoUnaryMask<m.vrclass, m.vrclass>,
+                                   Sched<[WriteVFClassV, ReadVFClassV, ReadVMask]>;
     }
   }
 }
 
-multiclass VPseudoUnaryV_V {
+multiclass VPseudoVSQR_V {
   foreach m = MxList.m in {
     let VLMul = m.value in {
-      def "_V_" # m.MX : VPseudoUnaryNoMask<m.vrclass, m.vrclass>;
-      def "_V_" # m.MX # "_MASK" : VPseudoUnaryMask<m.vrclass, m.vrclass>;
+      def "_V_" # m.MX : VPseudoUnaryNoMask<m.vrclass, m.vrclass>,
+                         Sched<[WriteVFSqrtV, ReadVFSqrtV, ReadVMask]>;
+      def "_V_" # m.MX # "_MASK" : VPseudoUnaryMaskTA<m.vrclass, m.vrclass>,
+                                   Sched<[WriteVFSqrtV, ReadVFSqrtV, ReadVMask]>;
     }
   }
 }
 
-multiclass PseudoUnaryV_VF2 {
+multiclass VPseudoVRCP_V {
+  foreach m = MxList.m in {
+    let VLMul = m.value in {
+      def "_V_" # m.MX : VPseudoUnaryNoMask<m.vrclass, m.vrclass>,
+                         Sched<[WriteVFRecpV, ReadVFRecpV, ReadVMask]>;
+      def "_V_" # m.MX # "_MASK" : VPseudoUnaryMaskTA<m.vrclass, m.vrclass>,
+                                   Sched<[WriteVFRecpV, ReadVFRecpV, ReadVMask]>;
+    }
+  }
+}
+
+multiclass PseudoVEXT_VF2 {
   defvar constraints = "@earlyclobber $rd";
   foreach m = MxListVF2.m in
   {
     let VLMul = m.value in {
-      def "_" # m.MX : VPseudoUnaryNoMask<m.vrclass, m.f2vrclass, constraints>;
-      def "_" # m.MX # "_MASK" : VPseudoUnaryMaskTA<m.vrclass, m.f2vrclass,
-                                                    constraints>;
+      def "_" # m.MX : VPseudoUnaryNoMask<m.vrclass, m.f2vrclass, constraints>,
+                       Sched<[WriteVExtV, ReadVExtV, ReadVMask]>;
+      def "_" # m.MX # "_MASK" :
+        VPseudoUnaryMaskTA<m.vrclass, m.f2vrclass, constraints>,
+        Sched<[WriteVExtV, ReadVExtV, ReadVMask]>;
     }
   }
 }
 
-multiclass PseudoUnaryV_VF4 {
+multiclass PseudoVEXT_VF4 {
   defvar constraints = "@earlyclobber $rd";
   foreach m = MxListVF4.m in
   {
     let VLMul = m.value in {
-      def "_" # m.MX : VPseudoUnaryNoMask<m.vrclass, m.f4vrclass, constraints>;
-      def "_" # m.MX # "_MASK" : VPseudoUnaryMaskTA<m.vrclass, m.f4vrclass,
-                                                    constraints>;
+      def "_" # m.MX : VPseudoUnaryNoMask<m.vrclass, m.f4vrclass, constraints>,
+                       Sched<[WriteVExtV, ReadVExtV, ReadVMask]>;
+      def "_" # m.MX # "_MASK" :
+        VPseudoUnaryMaskTA<m.vrclass, m.f4vrclass, constraints>,
+        Sched<[WriteVExtV, ReadVExtV, ReadVMask]>;
     }
   }
 }
 
-multiclass PseudoUnaryV_VF8 {
+multiclass PseudoVEXT_VF8 {
   defvar constraints = "@earlyclobber $rd";
   foreach m = MxListVF8.m in
   {
     let VLMul = m.value in {
-      def "_" # m.MX : VPseudoUnaryNoMask<m.vrclass, m.f8vrclass, constraints>;
-      def "_" # m.MX # "_MASK" : VPseudoUnaryMaskTA<m.vrclass, m.f8vrclass,
-                                                    constraints>;
+      def "_" # m.MX : VPseudoUnaryNoMask<m.vrclass, m.f8vrclass, constraints>,
+                       Sched<[WriteVExtV, ReadVExtV, ReadVMask]>;
+      def "_" # m.MX # "_MASK" :
+        VPseudoUnaryMaskTA<m.vrclass, m.f8vrclass, constraints>,
+        Sched<[WriteVExtV, ReadVExtV, ReadVMask]>;
     }
   }
 }
@@ -1874,30 +1906,172 @@ multiclass VPseudoBinaryM_VI {
                               !if(!ge(m.octuple, 16), "@earlyclobber $rd", "")>;
 }
 
-multiclass VPseudoBinaryV_VV_VX_VI<Operand ImmType = simm5, string Constraint = ""> {
-  defm "" : VPseudoBinaryV_VV<Constraint>;
-  defm "" : VPseudoBinaryV_VX<Constraint>;
-  defm "" : VPseudoBinaryV_VI<ImmType, Constraint>;
+multiclass VPseudoVGTR_VV_VX_VI<Operand ImmType = simm5, string Constraint = ""> {
+  defm "" : VPseudoBinaryV_VV<Constraint>,
+            Sched<[WriteVGatherV, ReadVGatherV, ReadVGatherV, ReadVMask]>;
+  defm "" : VPseudoBinaryV_VX<Constraint>,
+            Sched<[WriteVGatherX, ReadVGatherV, ReadVGatherX, ReadVMask]>;
+  defm "" : VPseudoBinaryV_VI<ImmType, Constraint>,
+            Sched<[WriteVGatherI, ReadVGatherV, ReadVMask]>;
 }
 
-multiclass VPseudoBinaryV_VV_VX {
-  defm "" : VPseudoBinaryV_VV;
-  defm "" : VPseudoBinaryV_VX;
+multiclass VPseudoVSALU_VV_VX_VI<Operand ImmType = simm5, string Constraint = ""> {
+  defm "" : VPseudoBinaryV_VV<Constraint>,
+            Sched<[WriteVSALUV, ReadVSALUV, ReadVSALUV, ReadVMask]>;
+  defm "" : VPseudoBinaryV_VX<Constraint>,
+            Sched<[WriteVSALUX, ReadVSALUV, ReadVSALUX, ReadVMask]>;
+  defm "" : VPseudoBinaryV_VI<ImmType, Constraint>,
+            Sched<[WriteVSALUI, ReadVSALUV, ReadVMask]>;
 }
 
-multiclass VPseudoBinaryV_VV_VF {
-  defm "" : VPseudoBinaryV_VV;
-  defm "" : VPseudoBinaryV_VF;
+
+multiclass VPseudoVSHT_VV_VX_VI<Operand ImmType = simm5, string Constraint = ""> {
+  defm "" : VPseudoBinaryV_VV<Constraint>,
+            Sched<[WriteVShiftV, ReadVShiftV, ReadVShiftV, ReadVMask]>;
+  defm "" : VPseudoBinaryV_VX<Constraint>,
+            Sched<[WriteVShiftX, ReadVShiftV, ReadVShiftX, ReadVMask]>;
+  defm "" : VPseudoBinaryV_VI<ImmType, Constraint>,
+            Sched<[WriteVShiftI, ReadVShiftV, ReadVMask]>;
 }
 
-multiclass VPseudoBinaryV_VX_VI<Operand ImmType = simm5> {
-  defm "" : VPseudoBinaryV_VX;
-  defm "" : VPseudoBinaryV_VI<ImmType>;
+multiclass VPseudoVSSHT_VV_VX_VI<Operand ImmType = simm5, string Constraint = ""> {
+  defm "" : VPseudoBinaryV_VV<Constraint>,
+            Sched<[WriteVSShiftV, ReadVSShiftV, ReadVSShiftV, ReadVMask]>;
+  defm "" : VPseudoBinaryV_VX<Constraint>,
+            Sched<[WriteVSShiftX, ReadVSShiftV, ReadVSShiftX, ReadVMask]>;
+  defm "" : VPseudoBinaryV_VI<ImmType, Constraint>,
+            Sched<[WriteVSShiftI, ReadVSShiftV, ReadVMask]>;
 }
 
-multiclass VPseudoBinaryW_VV_VX {
-  defm "" : VPseudoBinaryW_VV;
-  defm "" : VPseudoBinaryW_VX;
+multiclass VPseudoVALU_VV_VX_VI<Operand ImmType = simm5, string Constraint = ""> {
+  defm "" : VPseudoBinaryV_VV<Constraint>,
+            Sched<[WriteVIALUV, ReadVIALUV, ReadVIALUV, ReadVMask]>;
+  defm "" : VPseudoBinaryV_VX<Constraint>,
+            Sched<[WriteVIALUX, ReadVIALUV, ReadVIALUX, ReadVMask]>;
+  defm "" : VPseudoBinaryV_VI<ImmType, Constraint>,
+            Sched<[WriteVIALUI, ReadVIALUV, ReadVMask]>;
+}
+
+multiclass VPseudoVSALU_VV_VX {
+  defm "" : VPseudoBinaryV_VV,
+            Sched<[WriteVSALUV, ReadVSALUV, ReadVSALUV, ReadVMask]>;
+  defm "" : VPseudoBinaryV_VX,
+            Sched<[WriteVSALUX, ReadVSALUV, ReadVSALUX, ReadVMask]>;
+}
+
+multiclass VPseudoVSMUL_VV_VX {
+  defm "" : VPseudoBinaryV_VV,
+            Sched<[WriteVSMulV, ReadVSMulV, ReadVSMulV, ReadVMask]>;
+  defm "" : VPseudoBinaryV_VX,
+            Sched<[WriteVSMulX, ReadVSMulV, ReadVSMulX, ReadVMask]>;
+}
+
+multiclass VPseudoVAALU_VV_VX {
+  defm "" : VPseudoBinaryV_VV,
+            Sched<[WriteVAALUV, ReadVAALUV, ReadVAALUV, ReadVMask]>;
+  defm "" : VPseudoBinaryV_VX,
+            Sched<[WriteVAALUX, ReadVAALUV, ReadVAALUX, ReadVMask]>;
+}
+
+multiclass VPseudoVMINMAX_VV_VX {
+  defm "" : VPseudoBinaryV_VV,
+            Sched<[WriteVICmpV, ReadVICmpV, ReadVICmpV, ReadVMask]>;
+  defm "" : VPseudoBinaryV_VX,
+            Sched<[WriteVICmpX, ReadVICmpV, ReadVICmpX, ReadVMask]>;
+}
+
+multiclass VPseudoVMUL_VV_VX {
+  defm "" : VPseudoBinaryV_VV,
+            Sched<[WriteVIMulV, ReadVIMulV, ReadVIMulV, ReadVMask]>;
+  defm "" : VPseudoBinaryV_VX,
+            Sched<[WriteVIMulX, ReadVIMulV, ReadVIMulX, ReadVMask]>;
+}
+
+multiclass VPseudoVDIV_VV_VX {
+  defm "" : VPseudoBinaryV_VV,
+            Sched<[WriteVIDivV, ReadVIDivV, ReadVIDivV, ReadVMask]>;
+  defm "" : VPseudoBinaryV_VX,
+            Sched<[WriteVIDivX, ReadVIDivV, ReadVIDivX, ReadVMask]>;
+}
+
+multiclass VPseudoVFMUL_VV_VF {
+  defm "" : VPseudoBinaryV_VV,
+            Sched<[WriteVFMulV, ReadVFMulV, ReadVFMulV, ReadVMask]>;
+  defm "" : VPseudoBinaryV_VF,
+            Sched<[WriteVFMulF, ReadVFMulV, ReadVFMulF, ReadVMask]>;
+}
+
+multiclass VPseudoVFDIV_VV_VF {
+  defm "" : VPseudoBinaryV_VV,
+            Sched<[WriteVFDivV, ReadVFDivV, ReadVFDivV, ReadVMask]>;
+  defm "" : VPseudoBinaryV_VF,
+            Sched<[WriteVFDivF, ReadVFDivV, ReadVFDivF, ReadVMask]>;
+}
+
+multiclass VPseudoVFRDIV_VF {
+  defm "" : VPseudoBinaryV_VF,
+            Sched<[WriteVFDivF, ReadVFDivV, ReadVFDivF, ReadVMask]>;
+}
+
+multiclass VPseudoVALU_VV_VX {
+  defm "" : VPseudoBinaryV_VV,
+            Sched<[WriteVIALUV, ReadVIALUV, ReadVIALUV, ReadVMask]>;
+  defm "" : VPseudoBinaryV_VX,
+            Sched<[WriteVIALUX, ReadVIALUV, ReadVIALUX, ReadVMask]>;
+}
+
+multiclass VPseudoVSGNJ_VV_VF {
+  defm "" : VPseudoBinaryV_VV,
+            Sched<[WriteVFSgnjV, ReadVFSgnjV, ReadVFSgnjV, ReadVMask]>;
+  defm "" : VPseudoBinaryV_VF,
+            Sched<[WriteVFSgnjF, ReadVFSgnjV, ReadVFSgnjF, ReadVMask]>;
+}
+
+multiclass VPseudoVMAX_VV_VF {
+  defm "" : VPseudoBinaryV_VV,
+            Sched<[WriteVFCmpV, ReadVFCmpV, ReadVFCmpV, ReadVMask]>;
+  defm "" : VPseudoBinaryV_VF,
+            Sched<[WriteVFCmpF, ReadVFCmpV, ReadVFCmpF, ReadVMask]>;
+}
+
+multiclass VPseudoVALU_VV_VF {
+  defm "" : VPseudoBinaryV_VV,
+            Sched<[WriteVFALUV, ReadVFALUV, ReadVFALUV, ReadVMask]>;
+  defm "" : VPseudoBinaryV_VF,
+            Sched<[WriteVFALUF, ReadVFALUV, ReadVFALUF, ReadVMask]>;
+}
+
+multiclass VPseudoVALU_VF {
+  defm "" : VPseudoBinaryV_VF,
+            Sched<[WriteVFALUF, ReadVFALUV, ReadVFALUF, ReadVMask]>;
+}
+
+multiclass VPseudoVALU_VX_VI<Operand ImmType = simm5> {
+  defm "" : VPseudoBinaryV_VX,
+            Sched<[WriteVIALUX, ReadVIALUV, ReadVIALUX, ReadVMask]>;
+  defm "" : VPseudoBinaryV_VI<ImmType>,
+            Sched<[WriteVIALUI, ReadVIALUV, ReadVMask]>;
+}
+
+multiclass VPseudoVWALU_VV_VX {
+  defm "" : VPseudoBinaryW_VV,
+            Sched<[WriteVIWALUV, ReadVIWALUV, ReadVIWALUV, ReadVMask]>;
+  defm "" : VPseudoBinaryW_VX,
+            Sched<[WriteVIWALUX, ReadVIWALUV, ReadVIWALUX, ReadVMask]>;
+}
+
+multiclass VPseudoVWMUL_VV_VX {
+  defm "" : VPseudoBinaryW_VV,
+            Sched<[WriteVIWMulV, ReadVIWMulV, ReadVIWMulV, ReadVMask]>;
+  defm "" : VPseudoBinaryW_VX,
+            Sched<[WriteVIWMulX, ReadVIWMulV, ReadVIWMulX, ReadVMask]>;
+}
+
+multiclass VPseudoVWMUL_VV_VF {
+  defm "" : VPseudoBinaryW_VV,
+            Sched<[WriteVFWMulV, ReadVFWMulV, ReadVFWMulV, ReadVMask]>;
+  defm "" : VPseudoBinaryW_VF,
+            Sched<[WriteVFWMulF, ReadVFWMulV, ReadVFWMulF, ReadVMask]>;
 }
 
 multiclass VPseudoBinaryW_VV_VF {
@@ -1905,53 +2079,100 @@ multiclass VPseudoBinaryW_VV_VF {
   defm "" : VPseudoBinaryW_VF;
 }
 
-multiclass VPseudoBinaryW_WV_WX {
-  defm "" : VPseudoBinaryW_WV;
-  defm "" : VPseudoBinaryW_WX;
+multiclass VPseudoVWALU_WV_WX {
+  defm "" : VPseudoBinaryW_WV,
+            Sched<[WriteVIWALUV, ReadVIWALUV, ReadVIWALUV, ReadVMask]>;
+  defm "" : VPseudoBinaryW_WX,
+            Sched<[WriteVIWALUX, ReadVIWALUV, ReadVIWALUX, ReadVMask]>;
+}
+
+multiclass VPseudoVFWALU_VV_VF {
+  defm "" : VPseudoBinaryW_VV,
+            Sched<[WriteVFWALUV, ReadVFWALUV, ReadVFWALUV, ReadVMask]>;
+  defm "" : VPseudoBinaryW_VF,
+            Sched<[WriteVFWALUF, ReadVFWALUV, ReadVFWALUF, ReadVMask]>;
+}
+
+multiclass VPseudoVFWALU_WV_WF {
+  defm "" : VPseudoBinaryW_WV,
+            Sched<[WriteVFWALUV, ReadVFWALUV, ReadVFWALUV, ReadVMask]>;
+  defm "" : VPseudoBinaryW_WF,
+            Sched<[WriteVFWALUF, ReadVFWALUV, ReadVFWALUF, ReadVMask]>;
+}
+
+multiclass VPseudoVMRG_VM_XM_IM {
+  defm "" : VPseudoBinaryV_VM,
+            Sched<[WriteVIMergeV, ReadVIMergeV, ReadVIMergeV, ReadVMask]>;
+  defm "" : VPseudoBinaryV_XM,
+            Sched<[WriteVIMergeX, ReadVIMergeV, ReadVIMergeX, ReadVMask]>;
+  defm "" : VPseudoBinaryV_IM,
+            Sched<[WriteVIMergeI, ReadVIMergeV, ReadVMask]>;
 }
 
-multiclass VPseudoBinaryW_WV_WF {
-  defm "" : VPseudoBinaryW_WV;
-  defm "" : VPseudoBinaryW_WF;
+multiclass VPseudoVCALU_VM_XM_IM {
+  defm "" : VPseudoBinaryV_VM,
+            Sched<[WriteVICALUV, ReadVIALUCV, ReadVIALUCV, ReadVMask]>;
+  defm "" : VPseudoBinaryV_XM,
+            Sched<[WriteVICALUX, ReadVIALUCV, ReadVIALUCX, ReadVMask]>;
+  defm "" : VPseudoBinaryV_IM,
+            Sched<[WriteVICALUI, ReadVIALUCV, ReadVMask]>;
 }
 
-multiclass VPseudoBinaryV_VM_XM_IM {
-  defm "" : VPseudoBinaryV_VM;
-  defm "" : VPseudoBinaryV_XM;
-  defm "" : VPseudoBinaryV_IM;
+multiclass VPseudoVCALU_VM_XM {
+  defm "" : VPseudoBinaryV_VM,
+            Sched<[WriteVICALUV, ReadVIALUCV, ReadVIALUCV, ReadVMask]>;
+  defm "" : VPseudoBinaryV_XM,
+            Sched<[WriteVICALUX, ReadVIALUCV, ReadVIALUCX, ReadVMask]>;
 }
 
-multiclass VPseudoBinaryV_VM_XM {
-  defm "" : VPseudoBinaryV_VM;
-  defm "" : VPseudoBinaryV_XM;
+multiclass VPseudoVCALUM_VM_XM_IM<string Constraint> {
+  defm "" : VPseudoBinaryV_VM</*CarryOut=*/1, /*CarryIn=*/1, Constraint>,
+            Sched<[WriteVICALUV, ReadVIALUCV, ReadVIALUCV, ReadVMask]>;
+  defm "" : VPseudoBinaryV_XM</*CarryOut=*/1, /*CarryIn=*/1, Constraint>,
+            Sched<[WriteVICALUX, ReadVIALUCV, ReadVIALUCX, ReadVMask]>;
+  defm "" : VPseudoBinaryV_IM</*CarryOut=*/1, /*CarryIn=*/1, Constraint>,
+            Sched<[WriteVICALUI, ReadVIALUCV, ReadVMask]>;
 }
 
-multiclass VPseudoBinaryM_VM_XM_IM<string Constraint> {
-  defm "" : VPseudoBinaryV_VM</*CarryOut=*/1, /*CarryIn=*/1, Constraint>;
-  defm "" : VPseudoBinaryV_XM</*CarryOut=*/1, /*CarryIn=*/1, Constraint>;
-  defm "" : VPseudoBinaryV_IM</*CarryOut=*/1, /*CarryIn=*/1, Constraint>;
+multiclass VPseudoVCALUM_VM_XM<string Constraint> {
+  defm "" : VPseudoBinaryV_VM</*CarryOut=*/1, /*CarryIn=*/1, Constraint>,
+            Sched<[WriteVICALUV, ReadVIALUCV, ReadVIALUCV, ReadVMask]>;
+  defm "" : VPseudoBinaryV_XM</*CarryOut=*/1, /*CarryIn=*/1, Constraint>,
+            Sched<[WriteVICALUX, ReadVIALUCV, ReadVIALUCX, ReadVMask]>;
 }
 
-multiclass VPseudoBinaryM_VM_XM<string Constraint> {
-  defm "" : VPseudoBinaryV_VM</*CarryOut=*/1, /*CarryIn=*/1, Constraint>;
-  defm "" : VPseudoBinaryV_XM</*CarryOut=*/1, /*CarryIn=*/1, Constraint>;
+multiclass VPseudoVCALUM_V_X_I<string Constraint> {
+  defm "" : VPseudoBinaryV_VM</*CarryOut=*/1, /*CarryIn=*/0, Constraint>,
+            Sched<[WriteVICALUV, ReadVIALUCV, ReadVIALUCV]>;
+  defm "" : VPseudoBinaryV_XM</*CarryOut=*/1, /*CarryIn=*/0, Constraint>,
+            Sched<[WriteVICALUX, ReadVIALUCV, ReadVIALUCX]>;
+  defm "" : VPseudoBinaryV_IM</*CarryOut=*/1, /*CarryIn=*/0, Constraint>,
+            Sched<[WriteVICALUI, ReadVIALUCV]>;
 }
 
-multiclass VPseudoBinaryM_V_X_I<string Constraint> {
-  defm "" : VPseudoBinaryV_VM</*CarryOut=*/1, /*CarryIn=*/0, Constraint>;
-  defm "" : VPseudoBinaryV_XM</*CarryOut=*/1, /*CarryIn=*/0, Constraint>;
-  defm "" : VPseudoBinaryV_IM</*CarryOut=*/1, /*CarryIn=*/0, Constraint>;
+multiclass VPseudoVCALUM_V_X<string Constraint> {
+  defm "" : VPseudoBinaryV_VM</*CarryOut=*/1, /*CarryIn=*/0, Constraint>,
+            Sched<[WriteVICALUV, ReadVIALUCV, ReadVIALUCV]>;
+  defm "" : VPseudoBinaryV_XM</*CarryOut=*/1, /*CarryIn=*/0, Constraint>,
+            Sched<[WriteVICALUX, ReadVIALUCV, ReadVIALUCX]>;
 }
 
-multiclass VPseudoBinaryM_V_X<string Constraint> {
-  defm "" : VPseudoBinaryV_VM</*CarryOut=*/1, /*CarryIn=*/0, Constraint>;
-  defm "" : VPseudoBinaryV_XM</*CarryOut=*/1, /*CarryIn=*/0, Constraint>;
+multiclass VPseudoVNCLP_WV_WX_WI {
+  defm "" : VPseudoBinaryV_WV,
+            Sched<[WriteVNClipV, ReadVNClipV, ReadVNClipV, ReadVMask]>;
+  defm "" : VPseudoBinaryV_WX,
+            Sched<[WriteVNClipX, ReadVNClipV, ReadVNClipX, ReadVMask]>;
+  defm "" : VPseudoBinaryV_WI,
+            Sched<[WriteVNClipI, ReadVNClipV, ReadVMask]>;
 }
 
-multiclass VPseudoBinaryV_WV_WX_WI {
-  defm "" : VPseudoBinaryV_WV;
-  defm "" : VPseudoBinaryV_WX;
-  defm "" : VPseudoBinaryV_WI;
+multiclass VPseudoVNSHT_WV_WX_WI {
+  defm "" : VPseudoBinaryV_WV,
+            Sched<[WriteVNShiftV, ReadVNShiftV, ReadVNShiftV, ReadVMask]>;
+  defm "" : VPseudoBinaryV_WX,
+            Sched<[WriteVNShiftX, ReadVNShiftV, ReadVNShiftX, ReadVMask]>;
+  defm "" : VPseudoBinaryV_WI,
+            Sched<[WriteVNShiftI, ReadVNShiftV, ReadVMask]>;
 }
 
 multiclass VPseudoTernary<VReg RetClass,
@@ -2031,55 +2252,113 @@ multiclass VPseudoTernaryV_VI<Operand ImmType = simm5, string Constraint = ""> {
     defm _VI : VPseudoTernary<m.vrclass, m.vrclass, ImmType, m, Constraint>;
 }
 
-multiclass VPseudoTernaryV_VV_VX_AAXA<string Constraint = ""> {
-  defm "" : VPseudoTernaryV_VV_AAXA<Constraint>;
-  defm "" : VPseudoTernaryV_VX_AAXA<Constraint>;
+multiclass VPseudoVMAC_VV_VX_AAXA<string Constraint = ""> {
+  defm "" : VPseudoTernaryV_VV_AAXA<Constraint>,
+            Sched<[WriteVIMulAddV, ReadVIMulAddV, ReadVIMulAddV, ReadVIMulAddV, ReadVMask]>;
+  defm "" : VPseudoTernaryV_VX_AAXA<Constraint>,
+            Sched<[WriteVIMulAddX, ReadVIMulAddV, ReadVIMulAddV, ReadVIMulAddX, ReadVMask]>;
 }
 
-multiclass VPseudoTernaryV_VV_VF_AAXA<string Constraint = ""> {
-  defm "" : VPseudoTernaryV_VV_AAXA<Constraint>;
-  defm "" : VPseudoTernaryV_VF_AAXA<Constraint>;
+multiclass VPseudoVMAC_VV_VF_AAXA<string Constraint = ""> {
+  defm "" : VPseudoTernaryV_VV_AAXA<Constraint>,
+            Sched<[WriteVFMulAddV, ReadVFMulAddV, ReadVFMulAddV, ReadVFMulAddV, ReadVMask]>;
+  defm "" : VPseudoTernaryV_VF_AAXA<Constraint>,
+            Sched<[WriteVFMulAddF, ReadVFMulAddV, ReadVFMulAddV, ReadVFMulAddF, ReadVMask]>;
 }
 
-multiclass VPseudoTernaryV_VX_VI<Operand ImmType = simm5, string Constraint = ""> {
-  defm "" : VPseudoTernaryV_VX<Constraint>;
-  defm "" : VPseudoTernaryV_VI<ImmType, Constraint>;
+multiclass VPseudoVSLD_VX_VI<Operand ImmType = simm5, string Constraint = ""> {
+  defm "" : VPseudoTernaryV_VX<Constraint>,
+            Sched<[WriteVISlideX, ReadVISlideV, ReadVISlideV, ReadVISlideX, ReadVMask]>;
+  defm "" : VPseudoTernaryV_VI<ImmType, Constraint>,
+            Sched<[WriteVISlideI, ReadVISlideV, ReadVISlideV, ReadVMask]>;
 }
 
-multiclass VPseudoTernaryW_VV_VX {
-  defm "" : VPseudoTernaryW_VV;
-  defm "" : VPseudoTernaryW_VX;
+multiclass VPseudoVWMAC_VV_VX {
+  defm "" : VPseudoTernaryW_VV,
+            Sched<[WriteVIWMulAddV, ReadVIWMulAddV, ReadVIWMulAddV, ReadVIWMulAddV, ReadVMask]>;
+  defm "" : VPseudoTernaryW_VX,
+            Sched<[WriteVIWMulAddX, ReadVIWMulAddV, ReadVIWMulAddV, ReadVIWMulAddX, ReadVMask]>;
 }
 
-multiclass VPseudoTernaryW_VV_VF {
-  defm "" : VPseudoTernaryW_VV;
-  defm "" : VPseudoTernaryW_VF;
+multiclass VPseudoVWMAC_VX {
+  defm "" : VPseudoTernaryW_VX,
+            Sched<[WriteVIWMulAddX, ReadVIWMulAddV, ReadVIWMulAddV, ReadVIWMulAddX, ReadVMask]>;
 }
 
-multiclass VPseudoBinaryM_VV_VX_VI {
-  defm "" : VPseudoBinaryM_VV;
-  defm "" : VPseudoBinaryM_VX;
-  defm "" : VPseudoBinaryM_VI;
+multiclass VPseudoVWMAC_VV_VF {
+  defm "" : VPseudoTernaryW_VV,
+            Sched<[WriteVFWMulAddV, ReadVFWMulAddV, ReadVFWMulAddV, ReadVFWMulAddV, ReadVMask]>;
+  defm "" : VPseudoTernaryW_VF,
+            Sched<[WriteVFWMulAddF, ReadVFWMulAddV, ReadVFWMulAddV, ReadVFWMulAddF, ReadVMask]>;
 }
 
-multiclass VPseudoBinaryM_VV_VX {
-  defm "" : VPseudoBinaryM_VV;
-  defm "" : VPseudoBinaryM_VX;
+multiclass VPseudoVCMPM_VV_VX_VI {
+  defm "" : VPseudoBinaryM_VV,
+            Sched<[WriteVICmpV, ReadVICmpV, ReadVICmpV, ReadVMask]>;
+  defm "" : VPseudoBinaryM_VX,
+            Sched<[WriteVICmpX, ReadVICmpV, ReadVICmpX, ReadVMask]>;
+  defm "" : VPseudoBinaryM_VI,
+            Sched<[WriteVICmpI, ReadVICmpV, ReadVMask]>;
 }
 
-multiclass VPseudoBinaryM_VV_VF {
-  defm "" : VPseudoBinaryM_VV;
-  defm "" : VPseudoBinaryM_VF;
+multiclass VPseudoVCMPM_VV_VX {
+  defm "" : VPseudoBinaryM_VV,
+            Sched<[WriteVICmpV, ReadVICmpV, ReadVICmpV, ReadVMask]>;
+  defm "" : VPseudoBinaryM_VX,
+            Sched<[WriteVICmpX, ReadVICmpV, ReadVICmpX, ReadVMask]>;
 }
 
-multiclass VPseudoBinaryM_VX_VI {
-  defm "" : VPseudoBinaryM_VX;
-  defm "" : VPseudoBinaryM_VI;
+multiclass VPseudoVCMPM_VV_VF {
+  defm "" : VPseudoBinaryM_VV,
+            Sched<[WriteVFCmpV, ReadVFCmpV, ReadVFCmpV, ReadVMask]>;
+  defm "" : VPseudoBinaryM_VF,
+            Sched<[WriteVFCmpF, ReadVFCmpV, ReadVFCmpF, ReadVMask]>;
 }
 
-multiclass VPseudoReductionV_VS {
+multiclass VPseudoVCMPM_VF {
+  defm "" : VPseudoBinaryM_VF,
+            Sched<[WriteVFCmpF, ReadVFCmpV, ReadVFCmpF, ReadVMask]>;
+}
+
+multiclass VPseudoVCMPM_VX_VI {
+  defm "" : VPseudoBinaryM_VX,
+            Sched<[WriteVICmpX, ReadVICmpV, ReadVICmpX, ReadVMask]>;
+  defm "" : VPseudoBinaryM_VI,
+            Sched<[WriteVICmpI, ReadVICmpV, ReadVMask]>;
+}
+
+multiclass VPseudoVRED_VS {
   foreach m = MxList.m in {
-    defm _VS : VPseudoTernary<V_M1.vrclass, m.vrclass, V_M1.vrclass, m>;
+    defm _VS : VPseudoTernary<V_M1.vrclass, m.vrclass, V_M1.vrclass, m>,
+               Sched<[WriteVIRedV, ReadVIRedV, ReadVIRedV, ReadVIRedV, ReadVMask]>;
+  }
+}
+
+multiclass VPseudoVWRED_VS {
+  foreach m = MxList.m in {
+    defm _VS : VPseudoTernary<V_M1.vrclass, m.vrclass, V_M1.vrclass, m>,
+               Sched<[WriteVIWRedV, ReadVIWRedV, ReadVIWRedV, ReadVIWRedV, ReadVMask]>;
+  }
+}
+
+multiclass VPseudoVFRED_VS {
+  foreach m = MxList.m in {
+    defm _VS : VPseudoTernary<V_M1.vrclass, m.vrclass, V_M1.vrclass, m>,
+               Sched<[WriteVFRedV, ReadVFRedV, ReadVFRedV, ReadVFRedV, ReadVMask]>;
+  }
+}
+
+multiclass VPseudoVFREDO_VS {
+  foreach m = MxList.m in {
+    defm _VS : VPseudoTernary<V_M1.vrclass, m.vrclass, V_M1.vrclass, m>,
+               Sched<[WriteVFRedOV, ReadVFRedOV, ReadVFRedOV, ReadVFRedOV, ReadVMask]>;
+  }
+}
+
+multiclass VPseudoVFWRED_VS {
+  foreach m = MxList.m in {
+    defm _VS : VPseudoTernary<V_M1.vrclass, m.vrclass, V_M1.vrclass, m>,
+               Sched<[WriteVFWRedV, ReadVFWRedV, ReadVFWRedV, ReadVFWRedV, ReadVMask]>;
   }
 }
 
@@ -2094,9 +2373,16 @@ multiclass VPseudoConversion<VReg RetClass,
   }
 }
 
-multiclass VPseudoConversionV_V {
+multiclass VPseudoVCVTI_V {
+  foreach m = MxList.m in
+    defm _V : VPseudoConversion<m.vrclass, m.vrclass, m>,
+              Sched<[WriteVFCvtFToIV, ReadVFCvtFToIV, ReadVMask]>;
+}
+
+multiclass VPseudoVCVTF_V {
   foreach m = MxList.m in
-    defm _V : VPseudoConversion<m.vrclass, m.vrclass, m>;
+    defm _V : VPseudoConversion<m.vrclass, m.vrclass, m>,
+              Sched<[WriteVFCvtIToFV, ReadVFCvtIToFV, ReadVMask]>;
 }
 
 multiclass VPseudoConversionW_V {
@@ -2105,10 +2391,46 @@ multiclass VPseudoConversionW_V {
     defm _V : VPseudoConversion<m.wvrclass, m.vrclass, m, constraint>;
 }
 
-multiclass VPseudoConversionV_W {
+multiclass VPseudoVWCVTI_V {
+  defvar constraint = "@earlyclobber $rd";
+  foreach m = MxList.m[0-5] in
+    defm _V : VPseudoConversion<m.wvrclass, m.vrclass, m, constraint>,
+              Sched<[WriteVFWCvtFToIV, ReadVFWCvtFToIV, ReadVMask]>;
+}
+
+multiclass VPseudoVWCVTF_V {
+  defvar constraint = "@earlyclobber $rd";
+  foreach m = MxList.m[0-5] in
+    defm _V : VPseudoConversion<m.wvrclass, m.vrclass, m, constraint>,
+              Sched<[WriteVFWCvtIToFV, ReadVFWCvtIToFV, ReadVMask]>;
+}
+
+multiclass VPseudoVWCVTD_V {
+  defvar constraint = "@earlyclobber $rd";
+  foreach m = MxList.m[0-5] in
+    defm _V : VPseudoConversion<m.wvrclass, m.vrclass, m, constraint>,
+              Sched<[WriteVFWCvtFToFV, ReadVFWCvtFToFV, ReadVMask]>;
+}
+
+multiclass VPseudoVNCVTI_W {
+  defvar constraint = "@earlyclobber $rd";
+  foreach m = MxList.m[0-5] in
+    defm _W : VPseudoConversion<m.vrclass, m.wvrclass, m, constraint>,
+              Sched<[WriteVFNCvtFToIV, ReadVFNCvtFToIV, ReadVMask]>;
+}
+
+multiclass VPseudoVNCVTF_W {
+  defvar constraint = "@earlyclobber $rd";
+  foreach m = MxList.m[0-5] in
+    defm _W : VPseudoConversion<m.vrclass, m.wvrclass, m, constraint>,
+              Sched<[WriteVFNCvtIToFV, ReadVFNCvtIToFV, ReadVMask]>;
+}
+
+multiclass VPseudoVNCVTD_W {
   defvar constraint = "@earlyclobber $rd";
   foreach m = MxListW.m in
-    defm _W : VPseudoConversion<m.vrclass, m.wvrclass, m, constraint>;
+    defm _W : VPseudoConversion<m.vrclass, m.wvrclass, m, constraint>,
+              Sched<[WriteVFNCvtFToFV, ReadVFNCvtFToFV, ReadVMask]>;
 }
 
 multiclass VPseudoUSSegLoad<bit isFF> {
@@ -2543,42 +2865,6 @@ class VPatTernaryMask<string intrinsic,
                     (mask_type V0),
                     GPR:$vl, sew)>;
 
-class VPatAMOWDNoMask<string intrinsic_name,
-                    string inst,
-                    ValueType result_type,
-                    ValueType op1_type,
-                    int sew,
-                    LMULInfo vlmul,
-                    LMULInfo emul,
-                    VReg op1_reg_class> :
-  Pat<(result_type (!cast<Intrinsic>(intrinsic_name)
-                    GPR:$rs1,
-                    (op1_type op1_reg_class:$vs2),
-                    (result_type vlmul.vrclass:$vd),
-                    VLOpFrag)),
-                   (!cast<Instruction>(inst # "_WD_" # vlmul.MX # "_" # emul.MX)
-                    $rs1, $vs2, $vd,
-                    GPR:$vl, sew)>;
-
-class VPatAMOWDMask<string intrinsic_name,
-                    string inst,
-                    ValueType result_type,
-                    ValueType op1_type,
-                    ValueType mask_type,
-                    int sew,
-                    LMULInfo vlmul,
-                    LMULInfo emul,
-                    VReg op1_reg_class> :
-  Pat<(result_type (!cast<Intrinsic>(intrinsic_name # "_mask")
-                    GPR:$rs1,
-                    (op1_type op1_reg_class:$vs2),
-                    (result_type vlmul.vrclass:$vd),
-                    (mask_type V0),
-                    VLOpFrag)),
-                   (!cast<Instruction>(inst # "_WD_" # vlmul.MX # "_" # emul.MX # "_MASK")
-                    $rs1, $vs2, $vd,
-                    (mask_type V0), GPR:$vl, sew)>;
-
 multiclass VPatUnaryS_M<string intrinsic_name,
                              string inst>
 {
@@ -3416,44 +3702,6 @@ multiclass VPatConversionVF_WF <string intrinsic, string instruction> {
   }
 }
 
-multiclass VPatAMOWD<string intrinsic,
-                     string inst,
-                     ValueType result_type,
-                     ValueType offset_type,
-                     ValueType mask_type,
-                     int sew,
-                     LMULInfo vlmul,
-                     LMULInfo emul,
-                     VReg op1_reg_class>
-{
-  def : VPatAMOWDNoMask<intrinsic, inst, result_type, offset_type,
-                        sew, vlmul, emul, op1_reg_class>;
-  def : VPatAMOWDMask<intrinsic, inst, result_type, offset_type,
-                      mask_type, sew, vlmul, emul, op1_reg_class>;
-}
-
-multiclass VPatAMOV_WD<string intrinsic,
-                       string inst,
-                       list<VTypeInfo> vtilist> {
-  foreach eew = EEWList in {
-    foreach vti = vtilist in {
-      if !or(!eq(vti.SEW, 32), !eq(vti.SEW, 64)) then {
-        defvar octuple_lmul = vti.LMul.octuple;
-        // Calculate emul = eew * lmul / sew
-        defvar octuple_emul = !srl(!mul(eew, octuple_lmul), vti.Log2SEW);
-        if !and(!ge(octuple_emul, 1), !le(octuple_emul, 64)) then {
-          defvar emulMX = octuple_to_str<octuple_emul>.ret;
-          defvar offsetVti = !cast<VTypeInfo>("VI" # eew # emulMX);
-          defvar inst_ei = inst # "EI" # eew;
-          defm : VPatAMOWD<intrinsic, inst_ei,
-                           vti.Vector, offsetVti.Vector,
-                           vti.Mask, vti.Log2SEW, vti.LMul, offsetVti.LMul, offsetVti.RegClass>;
-        }
-      }
-    }
-  }
-}
-
 //===----------------------------------------------------------------------===//
 // Pseudo instructions
 //===----------------------------------------------------------------------===//
@@ -3531,11 +3779,13 @@ def PseudoVSETIVLI : Pseudo<(outs GPR:$rd), (ins uimm5:$rs1, VTypeIOp:$vtypei),
 //===----------------------------------------------------------------------===//
 
 // Pseudos Unit-Stride Loads and Stores
-defm PseudoVL : VPseudoUSLoad</*isFF=*/false>;
+defm PseudoVL : VPseudoUSLoad;
 defm PseudoVS : VPseudoUSStore;
 
-defm PseudoVLM : VPseudoLoadMask;
-defm PseudoVSM : VPseudoStoreMask;
+defm PseudoVLM : VPseudoLoadMask,
+                 Sched<[WriteVLDM, ReadVLDX]>;
+defm PseudoVSM : VPseudoStoreMask,
+                 Sched<[WriteVSTM, ReadVSTX]>;
 
 //===----------------------------------------------------------------------===//
 // 7.5 Vector Strided Instructions
@@ -3561,7 +3811,7 @@ defm PseudoVSUX : VPseudoIStore</*Ordered=*/false>;
 
 // vleff may update VL register
 let hasSideEffects = 1, Defs = [VL] in
-defm PseudoVL : VPseudoUSLoad</*isFF=*/true>;
+defm PseudoVL : VPseudoFFLoad;
 
 //===----------------------------------------------------------------------===//
 // 7.8. Vector Load/Store Segment Instructions
@@ -3580,28 +3830,15 @@ let hasSideEffects = 1, Defs = [VL] in
 defm PseudoVLSEG : VPseudoUSSegLoad</*isFF=*/true>;
 
 //===----------------------------------------------------------------------===//
-// 8. Vector AMO Operations
-//===----------------------------------------------------------------------===//
-defm PseudoVAMOSWAP : VPseudoAMO;
-defm PseudoVAMOADD : VPseudoAMO;
-defm PseudoVAMOXOR : VPseudoAMO;
-defm PseudoVAMOAND : VPseudoAMO;
-defm PseudoVAMOOR : VPseudoAMO;
-defm PseudoVAMOMIN : VPseudoAMO;
-defm PseudoVAMOMAX : VPseudoAMO;
-defm PseudoVAMOMINU : VPseudoAMO;
-defm PseudoVAMOMAXU : VPseudoAMO;
-
-//===----------------------------------------------------------------------===//
 // 12. Vector Integer Arithmetic Instructions
 //===----------------------------------------------------------------------===//
 
 //===----------------------------------------------------------------------===//
 // 12.1. Vector Single-Width Integer Add and Subtract
 //===----------------------------------------------------------------------===//
-defm PseudoVADD        : VPseudoBinaryV_VV_VX_VI;
-defm PseudoVSUB        : VPseudoBinaryV_VV_VX;
-defm PseudoVRSUB       : VPseudoBinaryV_VX_VI;
+defm PseudoVADD   : VPseudoVALU_VV_VX_VI;
+defm PseudoVSUB   : VPseudoVALU_VV_VX;
+defm PseudoVRSUB  : VPseudoVALU_VX_VI;
 
 foreach vti = AllIntegerVectors in {
   // Match vrsub with 2 vector operands to vsub.vv by swapping operands. This
@@ -3657,166 +3894,166 @@ foreach vti = AllIntegerVectors in {
 //===----------------------------------------------------------------------===//
 // 12.2. Vector Widening Integer Add/Subtract
 //===----------------------------------------------------------------------===//
-defm PseudoVWADDU    : VPseudoBinaryW_VV_VX;
-defm PseudoVWSUBU    : VPseudoBinaryW_VV_VX;
-defm PseudoVWADD     : VPseudoBinaryW_VV_VX;
-defm PseudoVWSUB     : VPseudoBinaryW_VV_VX;
-defm PseudoVWADDU    : VPseudoBinaryW_WV_WX;
-defm PseudoVWSUBU    : VPseudoBinaryW_WV_WX;
-defm PseudoVWADD     : VPseudoBinaryW_WV_WX;
-defm PseudoVWSUB     : VPseudoBinaryW_WV_WX;
+defm PseudoVWADDU : VPseudoVWALU_VV_VX;
+defm PseudoVWSUBU : VPseudoVWALU_VV_VX;
+defm PseudoVWADD  : VPseudoVWALU_VV_VX;
+defm PseudoVWSUB  : VPseudoVWALU_VV_VX;
+defm PseudoVWADDU : VPseudoVWALU_WV_WX;
+defm PseudoVWSUBU : VPseudoVWALU_WV_WX;
+defm PseudoVWADD  : VPseudoVWALU_WV_WX;
+defm PseudoVWSUB  : VPseudoVWALU_WV_WX;
 
 //===----------------------------------------------------------------------===//
 // 12.3. Vector Integer Extension
 //===----------------------------------------------------------------------===//
-defm PseudoVZEXT_VF2 : PseudoUnaryV_VF2;
-defm PseudoVZEXT_VF4 : PseudoUnaryV_VF4;
-defm PseudoVZEXT_VF8 : PseudoUnaryV_VF8;
-defm PseudoVSEXT_VF2 : PseudoUnaryV_VF2;
-defm PseudoVSEXT_VF4 : PseudoUnaryV_VF4;
-defm PseudoVSEXT_VF8 : PseudoUnaryV_VF8;
+defm PseudoVZEXT_VF2 : PseudoVEXT_VF2;
+defm PseudoVZEXT_VF4 : PseudoVEXT_VF4;
+defm PseudoVZEXT_VF8 : PseudoVEXT_VF8;
+defm PseudoVSEXT_VF2 : PseudoVEXT_VF2;
+defm PseudoVSEXT_VF4 : PseudoVEXT_VF4;
+defm PseudoVSEXT_VF8 : PseudoVEXT_VF8;
 
 //===----------------------------------------------------------------------===//
 // 12.4. Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions
 //===----------------------------------------------------------------------===//
-defm PseudoVADC        : VPseudoBinaryV_VM_XM_IM;
-defm PseudoVMADC       : VPseudoBinaryM_VM_XM_IM<"@earlyclobber $rd">;
-defm PseudoVMADC       : VPseudoBinaryM_V_X_I<"@earlyclobber $rd">;
+defm PseudoVADC  : VPseudoVCALU_VM_XM_IM;
+defm PseudoVMADC : VPseudoVCALUM_VM_XM_IM<"@earlyclobber $rd">;
+defm PseudoVMADC : VPseudoVCALUM_V_X_I<"@earlyclobber $rd">;
 
-defm PseudoVSBC        : VPseudoBinaryV_VM_XM;
-defm PseudoVMSBC       : VPseudoBinaryM_VM_XM<"@earlyclobber $rd">;
-defm PseudoVMSBC       : VPseudoBinaryM_V_X<"@earlyclobber $rd">;
+defm PseudoVSBC  : VPseudoVCALU_VM_XM;
+defm PseudoVMSBC : VPseudoVCALUM_VM_XM<"@earlyclobber $rd">;
+defm PseudoVMSBC : VPseudoVCALUM_V_X<"@earlyclobber $rd">;
 
 //===----------------------------------------------------------------------===//
 // 12.5. Vector Bitwise Logical Instructions
 //===----------------------------------------------------------------------===//
-defm PseudoVAND        : VPseudoBinaryV_VV_VX_VI;
-defm PseudoVOR         : VPseudoBinaryV_VV_VX_VI;
-defm PseudoVXOR        : VPseudoBinaryV_VV_VX_VI;
+defm PseudoVAND : VPseudoVALU_VV_VX_VI;
+defm PseudoVOR  : VPseudoVALU_VV_VX_VI;
+defm PseudoVXOR : VPseudoVALU_VV_VX_VI;
 
 //===----------------------------------------------------------------------===//
 // 12.6. Vector Single-Width Bit Shift Instructions
 //===----------------------------------------------------------------------===//
-defm PseudoVSLL        : VPseudoBinaryV_VV_VX_VI<uimm5>;
-defm PseudoVSRL        : VPseudoBinaryV_VV_VX_VI<uimm5>;
-defm PseudoVSRA        : VPseudoBinaryV_VV_VX_VI<uimm5>;
+defm PseudoVSLL : VPseudoVSHT_VV_VX_VI<uimm5>;
+defm PseudoVSRL : VPseudoVSHT_VV_VX_VI<uimm5>;
+defm PseudoVSRA : VPseudoVSHT_VV_VX_VI<uimm5>;
 
 //===----------------------------------------------------------------------===//
 // 12.7. Vector Narrowing Integer Right Shift Instructions
 //===----------------------------------------------------------------------===//
-defm PseudoVNSRL       : VPseudoBinaryV_WV_WX_WI;
-defm PseudoVNSRA       : VPseudoBinaryV_WV_WX_WI;
+defm PseudoVNSRL : VPseudoVNSHT_WV_WX_WI;
+defm PseudoVNSRA : VPseudoVNSHT_WV_WX_WI;
 
 //===----------------------------------------------------------------------===//
 // 12.8. Vector Integer Comparison Instructions
 //===----------------------------------------------------------------------===//
-defm PseudoVMSEQ       : VPseudoBinaryM_VV_VX_VI;
-defm PseudoVMSNE       : VPseudoBinaryM_VV_VX_VI;
-defm PseudoVMSLTU      : VPseudoBinaryM_VV_VX;
-defm PseudoVMSLT       : VPseudoBinaryM_VV_VX;
-defm PseudoVMSLEU      : VPseudoBinaryM_VV_VX_VI;
-defm PseudoVMSLE       : VPseudoBinaryM_VV_VX_VI;
-defm PseudoVMSGTU      : VPseudoBinaryM_VX_VI;
-defm PseudoVMSGT       : VPseudoBinaryM_VX_VI;
+defm PseudoVMSEQ  : VPseudoVCMPM_VV_VX_VI;
+defm PseudoVMSNE  : VPseudoVCMPM_VV_VX_VI;
+defm PseudoVMSLTU : VPseudoVCMPM_VV_VX;
+defm PseudoVMSLT  : VPseudoVCMPM_VV_VX;
+defm PseudoVMSLEU : VPseudoVCMPM_VV_VX_VI;
+defm PseudoVMSLE  : VPseudoVCMPM_VV_VX_VI;
+defm PseudoVMSGTU : VPseudoVCMPM_VX_VI;
+defm PseudoVMSGT  : VPseudoVCMPM_VX_VI;
 
 //===----------------------------------------------------------------------===//
 // 12.9. Vector Integer Min/Max Instructions
 //===----------------------------------------------------------------------===//
-defm PseudoVMINU       : VPseudoBinaryV_VV_VX;
-defm PseudoVMIN        : VPseudoBinaryV_VV_VX;
-defm PseudoVMAXU       : VPseudoBinaryV_VV_VX;
-defm PseudoVMAX        : VPseudoBinaryV_VV_VX;
+defm PseudoVMINU : VPseudoVMINMAX_VV_VX;
+defm PseudoVMIN  : VPseudoVMINMAX_VV_VX;
+defm PseudoVMAXU : VPseudoVMINMAX_VV_VX;
+defm PseudoVMAX  : VPseudoVMINMAX_VV_VX;
 
 //===----------------------------------------------------------------------===//
 // 12.10. Vector Single-Width Integer Multiply Instructions
 //===----------------------------------------------------------------------===//
-defm PseudoVMUL        : VPseudoBinaryV_VV_VX;
-defm PseudoVMULH       : VPseudoBinaryV_VV_VX;
-defm PseudoVMULHU      : VPseudoBinaryV_VV_VX;
-defm PseudoVMULHSU     : VPseudoBinaryV_VV_VX;
+defm PseudoVMUL    : VPseudoVMUL_VV_VX;
+defm PseudoVMULH   : VPseudoVMUL_VV_VX;
+defm PseudoVMULHU  : VPseudoVMUL_VV_VX;
+defm PseudoVMULHSU : VPseudoVMUL_VV_VX;
 
 //===----------------------------------------------------------------------===//
 // 12.11. Vector Integer Divide Instructions
 //===----------------------------------------------------------------------===//
-defm PseudoVDIVU       : VPseudoBinaryV_VV_VX;
-defm PseudoVDIV        : VPseudoBinaryV_VV_VX;
-defm PseudoVREMU       : VPseudoBinaryV_VV_VX;
-defm PseudoVREM        : VPseudoBinaryV_VV_VX;
+defm PseudoVDIVU : VPseudoVDIV_VV_VX;
+defm PseudoVDIV  : VPseudoVDIV_VV_VX;
+defm PseudoVREMU : VPseudoVDIV_VV_VX;
+defm PseudoVREM  : VPseudoVDIV_VV_VX;
 
 //===----------------------------------------------------------------------===//
 // 12.12. Vector Widening Integer Multiply Instructions
 //===----------------------------------------------------------------------===//
-defm PseudoVWMUL       : VPseudoBinaryW_VV_VX;
-defm PseudoVWMULU      : VPseudoBinaryW_VV_VX;
-defm PseudoVWMULSU     : VPseudoBinaryW_VV_VX;
+defm PseudoVWMUL   : VPseudoVWMUL_VV_VX;
+defm PseudoVWMULU  : VPseudoVWMUL_VV_VX;
+defm PseudoVWMULSU : VPseudoVWMUL_VV_VX;
 
 //===----------------------------------------------------------------------===//
 // 12.13. Vector Single-Width Integer Multiply-Add Instructions
 //===----------------------------------------------------------------------===//
-defm PseudoVMACC       : VPseudoTernaryV_VV_VX_AAXA;
-defm PseudoVNMSAC      : VPseudoTernaryV_VV_VX_AAXA;
-defm PseudoVMADD       : VPseudoTernaryV_VV_VX_AAXA;
-defm PseudoVNMSUB      : VPseudoTernaryV_VV_VX_AAXA;
+defm PseudoVMACC  : VPseudoVMAC_VV_VX_AAXA;
+defm PseudoVNMSAC : VPseudoVMAC_VV_VX_AAXA;
+defm PseudoVMADD  : VPseudoVMAC_VV_VX_AAXA;
+defm PseudoVNMSUB : VPseudoVMAC_VV_VX_AAXA;
 
 //===----------------------------------------------------------------------===//
 // 12.14. Vector Widening Integer Multiply-Add Instructions
 //===----------------------------------------------------------------------===//
-defm PseudoVWMACCU     : VPseudoTernaryW_VV_VX;
-defm PseudoVWMACC      : VPseudoTernaryW_VV_VX;
-defm PseudoVWMACCSU    : VPseudoTernaryW_VV_VX;
-defm PseudoVWMACCUS    : VPseudoTernaryW_VX;
+defm PseudoVWMACCU  : VPseudoVWMAC_VV_VX;
+defm PseudoVWMACC   : VPseudoVWMAC_VV_VX;
+defm PseudoVWMACCSU : VPseudoVWMAC_VV_VX;
+defm PseudoVWMACCUS : VPseudoVWMAC_VX;
 
 //===----------------------------------------------------------------------===//
 // 12.15. Vector Integer Merge Instructions
 //===----------------------------------------------------------------------===//
-defm PseudoVMERGE      : VPseudoBinaryV_VM_XM_IM;
+defm PseudoVMERGE : VPseudoVMRG_VM_XM_IM;
 
 //===----------------------------------------------------------------------===//
 // 12.16. Vector Integer Move Instructions
 //===----------------------------------------------------------------------===//
-defm PseudoVMV_V       : VPseudoUnaryV_V_X_I_NoDummyMask;
+defm PseudoVMV_V : VPseudoUnaryVMV_V_X_I;
 
 //===----------------------------------------------------------------------===//
 // 13.1. Vector Single-Width Saturating Add and Subtract
 //===----------------------------------------------------------------------===//
 let Defs = [VXSAT], hasSideEffects = 1 in {
-  defm PseudoVSADDU      : VPseudoBinaryV_VV_VX_VI;
-  defm PseudoVSADD       : VPseudoBinaryV_VV_VX_VI;
-  defm PseudoVSSUBU      : VPseudoBinaryV_VV_VX;
-  defm PseudoVSSUB       : VPseudoBinaryV_VV_VX;
+  defm PseudoVSADDU : VPseudoVSALU_VV_VX_VI;
+  defm PseudoVSADD  : VPseudoVSALU_VV_VX_VI;
+  defm PseudoVSSUBU : VPseudoVSALU_VV_VX;
+  defm PseudoVSSUB  : VPseudoVSALU_VV_VX;
 }
 
 //===----------------------------------------------------------------------===//
 // 13.2. Vector Single-Width Averaging Add and Subtract
 //===----------------------------------------------------------------------===//
 let Uses = [VXRM], hasSideEffects = 1 in {
-  defm PseudoVAADDU      : VPseudoBinaryV_VV_VX;
-  defm PseudoVAADD       : VPseudoBinaryV_VV_VX;
-  defm PseudoVASUBU      : VPseudoBinaryV_VV_VX;
-  defm PseudoVASUB       : VPseudoBinaryV_VV_VX;
+  defm PseudoVAADDU : VPseudoVAALU_VV_VX;
+  defm PseudoVAADD  : VPseudoVAALU_VV_VX;
+  defm PseudoVASUBU : VPseudoVAALU_VV_VX;
+  defm PseudoVASUB  : VPseudoVAALU_VV_VX;
 }
 
 //===----------------------------------------------------------------------===//
 // 13.3. Vector Single-Width Fractional Multiply with Rounding and Saturation
 //===----------------------------------------------------------------------===//
 let Uses = [VXRM], Defs = [VXSAT], hasSideEffects = 1 in {
-  defm PseudoVSMUL      : VPseudoBinaryV_VV_VX;
+  defm PseudoVSMUL : VPseudoVSMUL_VV_VX;
 }
 
 //===----------------------------------------------------------------------===//
 // 13.4. Vector Single-Width Scaling Shift Instructions
 //===----------------------------------------------------------------------===//
 let Uses = [VXRM], hasSideEffects = 1 in {
-  defm PseudoVSSRL        : VPseudoBinaryV_VV_VX_VI<uimm5>;
-  defm PseudoVSSRA        : VPseudoBinaryV_VV_VX_VI<uimm5>;
+  defm PseudoVSSRL : VPseudoVSSHT_VV_VX_VI<uimm5>;
+  defm PseudoVSSRA : VPseudoVSSHT_VV_VX_VI<uimm5>;
 }
 
 //===----------------------------------------------------------------------===//
 // 13.5. Vector Narrowing Fixed-Point Clip Instructions
 //===----------------------------------------------------------------------===//
 let Uses = [VXRM], Defs = [VXSAT], hasSideEffects = 1 in {
-  defm PseudoVNCLIP     : VPseudoBinaryV_WV_WX_WI;
-  defm PseudoVNCLIPU    : VPseudoBinaryV_WV_WX_WI;
+  defm PseudoVNCLIP  : VPseudoVNCLP_WV_WX_WI;
+  defm PseudoVNCLIPU : VPseudoVNCLP_WV_WX_WI;
 }
 
 } // Predicates = [HasVInstructions]
@@ -3825,156 +4062,156 @@ let Predicates = [HasVInstructionsAnyF] in {
 //===----------------------------------------------------------------------===//
 // 14.2. Vector Single-Width Floating-Point Add/Subtract Instructions
 //===----------------------------------------------------------------------===//
-defm PseudoVFADD       : VPseudoBinaryV_VV_VF;
-defm PseudoVFSUB       : VPseudoBinaryV_VV_VF;
-defm PseudoVFRSUB      : VPseudoBinaryV_VF;
+defm PseudoVFADD  : VPseudoVALU_VV_VF;
+defm PseudoVFSUB  : VPseudoVALU_VV_VF;
+defm PseudoVFRSUB : VPseudoVALU_VF;
 
 //===----------------------------------------------------------------------===//
 // 14.3. Vector Widening Floating-Point Add/Subtract Instructions
 //===----------------------------------------------------------------------===//
-defm PseudoVFWADD     : VPseudoBinaryW_VV_VF;
-defm PseudoVFWSUB     : VPseudoBinaryW_VV_VF;
-defm PseudoVFWADD     : VPseudoBinaryW_WV_WF;
-defm PseudoVFWSUB     : VPseudoBinaryW_WV_WF;
+defm PseudoVFWADD : VPseudoVFWALU_VV_VF;
+defm PseudoVFWSUB : VPseudoVFWALU_VV_VF;
+defm PseudoVFWADD : VPseudoVFWALU_WV_WF;
+defm PseudoVFWSUB : VPseudoVFWALU_WV_WF;
 
 //===----------------------------------------------------------------------===//
 // 14.4. Vector Single-Width Floating-Point Multiply/Divide Instructions
 //===----------------------------------------------------------------------===//
-defm PseudoVFMUL       : VPseudoBinaryV_VV_VF;
-defm PseudoVFDIV       : VPseudoBinaryV_VV_VF;
-defm PseudoVFRDIV      : VPseudoBinaryV_VF;
+defm PseudoVFMUL  : VPseudoVFMUL_VV_VF;
+defm PseudoVFDIV  : VPseudoVFDIV_VV_VF;
+defm PseudoVFRDIV : VPseudoVFRDIV_VF;
 
 //===----------------------------------------------------------------------===//
 // 14.5. Vector Widening Floating-Point Multiply
 //===----------------------------------------------------------------------===//
-defm PseudoVFWMUL      : VPseudoBinaryW_VV_VF;
+defm PseudoVFWMUL : VPseudoVWMUL_VV_VF;
 
 //===----------------------------------------------------------------------===//
 // 14.6. Vector Single-Width Floating-Point Fused Multiply-Add Instructions
 //===----------------------------------------------------------------------===//
-defm PseudoVFMACC      : VPseudoTernaryV_VV_VF_AAXA;
-defm PseudoVFNMACC     : VPseudoTernaryV_VV_VF_AAXA;
-defm PseudoVFMSAC      : VPseudoTernaryV_VV_VF_AAXA;
-defm PseudoVFNMSAC     : VPseudoTernaryV_VV_VF_AAXA;
-defm PseudoVFMADD      : VPseudoTernaryV_VV_VF_AAXA;
-defm PseudoVFNMADD     : VPseudoTernaryV_VV_VF_AAXA;
-defm PseudoVFMSUB      : VPseudoTernaryV_VV_VF_AAXA;
-defm PseudoVFNMSUB     : VPseudoTernaryV_VV_VF_AAXA;
+defm PseudoVFMACC  : VPseudoVMAC_VV_VF_AAXA;
+defm PseudoVFNMACC : VPseudoVMAC_VV_VF_AAXA;
+defm PseudoVFMSAC  : VPseudoVMAC_VV_VF_AAXA;
+defm PseudoVFNMSAC : VPseudoVMAC_VV_VF_AAXA;
+defm PseudoVFMADD  : VPseudoVMAC_VV_VF_AAXA;
+defm PseudoVFNMADD : VPseudoVMAC_VV_VF_AAXA;
+defm PseudoVFMSUB  : VPseudoVMAC_VV_VF_AAXA;
+defm PseudoVFNMSUB : VPseudoVMAC_VV_VF_AAXA;
 
 //===----------------------------------------------------------------------===//
 // 14.7. Vector Widening Floating-Point Fused Multiply-Add Instructions
 //===----------------------------------------------------------------------===//
-defm PseudoVFWMACC     : VPseudoTernaryW_VV_VF;
-defm PseudoVFWNMACC    : VPseudoTernaryW_VV_VF;
-defm PseudoVFWMSAC     : VPseudoTernaryW_VV_VF;
-defm PseudoVFWNMSAC    : VPseudoTernaryW_VV_VF;
+defm PseudoVFWMACC  : VPseudoVWMAC_VV_VF;
+defm PseudoVFWNMACC : VPseudoVWMAC_VV_VF;
+defm PseudoVFWMSAC  : VPseudoVWMAC_VV_VF;
+defm PseudoVFWNMSAC : VPseudoVWMAC_VV_VF;
 
 //===----------------------------------------------------------------------===//
 // 14.8. Vector Floating-Point Square-Root Instruction
 //===----------------------------------------------------------------------===//
-defm PseudoVFSQRT      : VPseudoUnaryTAV_V;
+defm PseudoVFSQRT : VPseudoVSQR_V;
 
 //===----------------------------------------------------------------------===//
 // 14.9. Vector Floating-Point Reciprocal Square-Root Estimate Instruction
 //===----------------------------------------------------------------------===//
-defm PseudoVFRSQRT7    : VPseudoUnaryTAV_V;
+defm PseudoVFRSQRT7 : VPseudoVRCP_V;
 
 //===----------------------------------------------------------------------===//
 // 14.10. Vector Floating-Point Reciprocal Estimate Instruction
 //===----------------------------------------------------------------------===//
-defm PseudoVFREC7      : VPseudoUnaryTAV_V;
+defm PseudoVFREC7 : VPseudoVRCP_V;
 
 //===----------------------------------------------------------------------===//
 // 14.11. Vector Floating-Point Min/Max Instructions
 //===----------------------------------------------------------------------===//
-defm PseudoVFMIN       : VPseudoBinaryV_VV_VF;
-defm PseudoVFMAX       : VPseudoBinaryV_VV_VF;
+defm PseudoVFMIN : VPseudoVMAX_VV_VF;
+defm PseudoVFMAX : VPseudoVMAX_VV_VF;
 
 //===----------------------------------------------------------------------===//
 // 14.12. Vector Floating-Point Sign-Injection Instructions
 //===----------------------------------------------------------------------===//
-defm PseudoVFSGNJ      : VPseudoBinaryV_VV_VF;
-defm PseudoVFSGNJN     : VPseudoBinaryV_VV_VF;
-defm PseudoVFSGNJX     : VPseudoBinaryV_VV_VF;
+defm PseudoVFSGNJ  : VPseudoVSGNJ_VV_VF;
+defm PseudoVFSGNJN : VPseudoVSGNJ_VV_VF;
+defm PseudoVFSGNJX : VPseudoVSGNJ_VV_VF;
 
 //===----------------------------------------------------------------------===//
 // 14.13. Vector Floating-Point Compare Instructions
 //===----------------------------------------------------------------------===//
-defm PseudoVMFEQ       : VPseudoBinaryM_VV_VF;
-defm PseudoVMFNE       : VPseudoBinaryM_VV_VF;
-defm PseudoVMFLT       : VPseudoBinaryM_VV_VF;
-defm PseudoVMFLE       : VPseudoBinaryM_VV_VF;
-defm PseudoVMFGT       : VPseudoBinaryM_VF;
-defm PseudoVMFGE       : VPseudoBinaryM_VF;
+defm PseudoVMFEQ : VPseudoVCMPM_VV_VF;
+defm PseudoVMFNE : VPseudoVCMPM_VV_VF;
+defm PseudoVMFLT : VPseudoVCMPM_VV_VF;
+defm PseudoVMFLE : VPseudoVCMPM_VV_VF;
+defm PseudoVMFGT : VPseudoVCMPM_VF;
+defm PseudoVMFGE : VPseudoVCMPM_VF;
 
 //===----------------------------------------------------------------------===//
 // 14.14. Vector Floating-Point Classify Instruction
 //===----------------------------------------------------------------------===//
-defm PseudoVFCLASS     : VPseudoUnaryV_V;
+defm PseudoVFCLASS : VPseudoVCLS_V;
 
 //===----------------------------------------------------------------------===//
 // 14.15. Vector Floating-Point Merge Instruction
 //===----------------------------------------------------------------------===//
-defm PseudoVFMERGE     : VPseudoBinaryV_FM;
+defm PseudoVFMERGE : VPseudoVMRG_FM;
 
 //===----------------------------------------------------------------------===//
 // 14.16. Vector Floating-Point Move Instruction
 //===----------------------------------------------------------------------===//
-defm PseudoVFMV_V      : VPseudoUnaryV_F_NoDummyMask;
+defm PseudoVFMV_V : VPseudoVMV_F;
 
 //===----------------------------------------------------------------------===//
 // 14.17. Single-Width Floating-Point/Integer Type-Convert Instructions
 //===----------------------------------------------------------------------===//
-defm PseudoVFCVT_XU_F : VPseudoConversionV_V;
-defm PseudoVFCVT_X_F : VPseudoConversionV_V;
-defm PseudoVFCVT_RTZ_XU_F : VPseudoConversionV_V;
-defm PseudoVFCVT_RTZ_X_F : VPseudoConversionV_V;
-defm PseudoVFCVT_F_XU : VPseudoConversionV_V;
-defm PseudoVFCVT_F_X : VPseudoConversionV_V;
+defm PseudoVFCVT_XU_F : VPseudoVCVTI_V;
+defm PseudoVFCVT_X_F : VPseudoVCVTI_V;
+defm PseudoVFCVT_RTZ_XU_F : VPseudoVCVTI_V;
+defm PseudoVFCVT_RTZ_X_F : VPseudoVCVTI_V;
+defm PseudoVFCVT_F_XU : VPseudoVCVTF_V;
+defm PseudoVFCVT_F_X : VPseudoVCVTF_V;
 
 //===----------------------------------------------------------------------===//
 // 14.18. Widening Floating-Point/Integer Type-Convert Instructions
 //===----------------------------------------------------------------------===//
-defm PseudoVFWCVT_XU_F : VPseudoConversionW_V;
-defm PseudoVFWCVT_X_F : VPseudoConversionW_V;
-defm PseudoVFWCVT_RTZ_XU_F : VPseudoConversionW_V;
-defm PseudoVFWCVT_RTZ_X_F : VPseudoConversionW_V;
-defm PseudoVFWCVT_F_XU : VPseudoConversionW_V;
-defm PseudoVFWCVT_F_X : VPseudoConversionW_V;
-defm PseudoVFWCVT_F_F : VPseudoConversionW_V;
+defm PseudoVFWCVT_XU_F     : VPseudoVWCVTI_V;
+defm PseudoVFWCVT_X_F      : VPseudoVWCVTI_V;
+defm PseudoVFWCVT_RTZ_XU_F : VPseudoVWCVTI_V;
+defm PseudoVFWCVT_RTZ_X_F  : VPseudoVWCVTI_V;
+defm PseudoVFWCVT_F_XU     : VPseudoVWCVTF_V;
+defm PseudoVFWCVT_F_X      : VPseudoVWCVTF_V;
+defm PseudoVFWCVT_F_F      : VPseudoVWCVTD_V;
 
 //===----------------------------------------------------------------------===//
 // 14.19. Narrowing Floating-Point/Integer Type-Convert Instructions
 //===----------------------------------------------------------------------===//
-defm PseudoVFNCVT_XU_F : VPseudoConversionV_W;
-defm PseudoVFNCVT_X_F : VPseudoConversionV_W;
-defm PseudoVFNCVT_RTZ_XU_F : VPseudoConversionV_W;
-defm PseudoVFNCVT_RTZ_X_F : VPseudoConversionV_W;
-defm PseudoVFNCVT_F_XU : VPseudoConversionV_W;
-defm PseudoVFNCVT_F_X : VPseudoConversionV_W;
-defm PseudoVFNCVT_F_F : VPseudoConversionV_W;
-defm PseudoVFNCVT_ROD_F_F : VPseudoConversionV_W;
+defm PseudoVFNCVT_XU_F     : VPseudoVNCVTI_W;
+defm PseudoVFNCVT_X_F      : VPseudoVNCVTI_W;
+defm PseudoVFNCVT_RTZ_XU_F : VPseudoVNCVTI_W;
+defm PseudoVFNCVT_RTZ_X_F  : VPseudoVNCVTI_W;
+defm PseudoVFNCVT_F_XU     : VPseudoVNCVTF_W;
+defm PseudoVFNCVT_F_X      : VPseudoVNCVTF_W;
+defm PseudoVFNCVT_F_F      : VPseudoVNCVTD_W;
+defm PseudoVFNCVT_ROD_F_F  : VPseudoVNCVTD_W;
 } // Predicates = [HasVInstructionsAnyF]
 
 let Predicates = [HasVInstructions] in {
 //===----------------------------------------------------------------------===//
 // 15.1. Vector Single-Width Integer Reduction Instructions
 //===----------------------------------------------------------------------===//
-defm PseudoVREDSUM     : VPseudoReductionV_VS;
-defm PseudoVREDAND     : VPseudoReductionV_VS;
-defm PseudoVREDOR      : VPseudoReductionV_VS;
-defm PseudoVREDXOR     : VPseudoReductionV_VS;
-defm PseudoVREDMINU    : VPseudoReductionV_VS;
-defm PseudoVREDMIN     : VPseudoReductionV_VS;
-defm PseudoVREDMAXU    : VPseudoReductionV_VS;
-defm PseudoVREDMAX     : VPseudoReductionV_VS;
+defm PseudoVREDSUM  : VPseudoVRED_VS;
+defm PseudoVREDAND  : VPseudoVRED_VS;
+defm PseudoVREDOR   : VPseudoVRED_VS;
+defm PseudoVREDXOR  : VPseudoVRED_VS;
+defm PseudoVREDMINU : VPseudoVRED_VS;
+defm PseudoVREDMIN  : VPseudoVRED_VS;
+defm PseudoVREDMAXU : VPseudoVRED_VS;
+defm PseudoVREDMAX  : VPseudoVRED_VS;
 
 //===----------------------------------------------------------------------===//
 // 15.2. Vector Widening Integer Reduction Instructions
 //===----------------------------------------------------------------------===//
 let IsRVVWideningReduction = 1 in {
-defm PseudoVWREDSUMU   : VPseudoReductionV_VS;
-defm PseudoVWREDSUM    : VPseudoReductionV_VS;
+defm PseudoVWREDSUMU   : VPseudoVWRED_VS;
+defm PseudoVWREDSUM    : VPseudoVWRED_VS;
 }
 } // Predicates = [HasVInstructions]
 
@@ -3982,17 +4219,17 @@ let Predicates = [HasVInstructionsAnyF] in {
 //===----------------------------------------------------------------------===//
 // 15.3. Vector Single-Width Floating-Point Reduction Instructions
 //===----------------------------------------------------------------------===//
-defm PseudoVFREDOSUM   : VPseudoReductionV_VS;
-defm PseudoVFREDUSUM   : VPseudoReductionV_VS;
-defm PseudoVFREDMIN    : VPseudoReductionV_VS;
-defm PseudoVFREDMAX    : VPseudoReductionV_VS;
+defm PseudoVFREDOSUM : VPseudoVFREDO_VS;
+defm PseudoVFREDUSUM : VPseudoVFRED_VS;
+defm PseudoVFREDMIN  : VPseudoVFRED_VS;
+defm PseudoVFREDMAX  : VPseudoVFRED_VS;
 
 //===----------------------------------------------------------------------===//
 // 15.4. Vector Widening Floating-Point Reduction Instructions
 //===----------------------------------------------------------------------===//
 let IsRVVWideningReduction = 1 in {
-defm PseudoVFWREDUSUM  : VPseudoReductionV_VS;
-defm PseudoVFWREDOSUM  : VPseudoReductionV_VS;
+defm PseudoVFWREDUSUM  : VPseudoVFWRED_VS;
+defm PseudoVFWREDOSUM  : VPseudoVFWRED_VS;
 }
 
 } // Predicates = [HasVInstructionsAnyF]
@@ -4005,55 +4242,57 @@ defm PseudoVFWREDOSUM  : VPseudoReductionV_VS;
 // 16.1 Vector Mask-Register Logical Instructions
 //===----------------------------------------------------------------------===//
 
-defm PseudoVMAND: VPseudoBinaryM_MM;
-defm PseudoVMNAND: VPseudoBinaryM_MM;
-defm PseudoVMANDN: VPseudoBinaryM_MM;
-defm PseudoVMXOR: VPseudoBinaryM_MM;
-defm PseudoVMOR: VPseudoBinaryM_MM;
-defm PseudoVMNOR: VPseudoBinaryM_MM;
-defm PseudoVMORN: VPseudoBinaryM_MM;
-defm PseudoVMXNOR: VPseudoBinaryM_MM;
+defm PseudoVMAND: VPseudoVALU_MM;
+defm PseudoVMNAND: VPseudoVALU_MM;
+defm PseudoVMANDN: VPseudoVALU_MM;
+defm PseudoVMXOR: VPseudoVALU_MM;
+defm PseudoVMOR: VPseudoVALU_MM;
+defm PseudoVMNOR: VPseudoVALU_MM;
+defm PseudoVMORN: VPseudoVALU_MM;
+defm PseudoVMXNOR: VPseudoVALU_MM;
 
 // Pseudo instructions
-defm PseudoVMCLR : VPseudoNullaryPseudoM<"VMXOR">;
-defm PseudoVMSET : VPseudoNullaryPseudoM<"VMXNOR">;
+defm PseudoVMCLR : VPseudoNullaryPseudoM<"VMXOR">,
+                   Sched<[WriteVMALUV, ReadVMALUV, ReadVMALUV]>;
+defm PseudoVMSET : VPseudoNullaryPseudoM<"VMXNOR">,
+                   Sched<[WriteVMALUV, ReadVMALUV, ReadVMALUV]>;
 
 //===----------------------------------------------------------------------===//
 // 16.2. Vector mask population count vcpop
 //===----------------------------------------------------------------------===//
 
-defm PseudoVCPOP: VPseudoUnaryS_M;
+defm PseudoVCPOP: VPseudoVPOP_M;
 
 //===----------------------------------------------------------------------===//
 // 16.3. vfirst find-first-set mask bit
 //===----------------------------------------------------------------------===//
 
-defm PseudoVFIRST: VPseudoUnaryS_M;
+defm PseudoVFIRST: VPseudoV1ST_M;
 
 //===----------------------------------------------------------------------===//
 // 16.4. vmsbf.m set-before-first mask bit
 //===----------------------------------------------------------------------===//
-defm PseudoVMSBF: VPseudoUnaryM_M;
+defm PseudoVMSBF: VPseudoVSFS_M;
 
 //===----------------------------------------------------------------------===//
 // 16.5. vmsif.m set-including-first mask bit
 //===----------------------------------------------------------------------===//
-defm PseudoVMSIF: VPseudoUnaryM_M;
+defm PseudoVMSIF: VPseudoVSFS_M;
 
 //===----------------------------------------------------------------------===//
 // 16.6. vmsof.m set-only-first mask bit
 //===----------------------------------------------------------------------===//
-defm PseudoVMSOF: VPseudoUnaryM_M;
+defm PseudoVMSOF: VPseudoVSFS_M;
 
 //===----------------------------------------------------------------------===//
 // 16.8.  Vector Iota Instruction
 //===----------------------------------------------------------------------===//
-defm PseudoVIOTA_M: VPseudoUnaryV_M;
+defm PseudoVIOTA_M: VPseudoVIOT_M;
 
 //===----------------------------------------------------------------------===//
 // 16.9. Vector Element Index Instruction
 //===----------------------------------------------------------------------===//
-defm PseudoVID : VPseudoMaskNullaryV;
+defm PseudoVID : VPseudoVID_V;
 
 //===----------------------------------------------------------------------===//
 // 17. Vector Permutation Instructions
@@ -4068,15 +4307,18 @@ let mayLoad = 0, mayStore = 0, hasSideEffects = 0 in {
   foreach m = MxList.m in {
     let VLMul = m.value in {
       let HasSEWOp = 1, BaseInstr = VMV_X_S in
-      def PseudoVMV_X_S # "_" # m.MX: Pseudo<(outs GPR:$rd),
-                                             (ins m.vrclass:$rs2, ixlenimm:$sew),
-                                             []>, RISCVVPseudo;
+      def PseudoVMV_X_S # "_" # m.MX:
+        Pseudo<(outs GPR:$rd), (ins m.vrclass:$rs2, ixlenimm:$sew), []>,
+        Sched<[WriteVIMovVX, ReadVIMovVX]>,
+        RISCVVPseudo;
       let HasVLOp = 1, HasSEWOp = 1, BaseInstr = VMV_S_X,
           Constraints = "$rd = $rs1" in
       def PseudoVMV_S_X # "_" # m.MX: Pseudo<(outs m.vrclass:$rd),
                                              (ins m.vrclass:$rs1, GPR:$rs2,
                                                   AVL:$vl, ixlenimm:$sew),
-                                             []>, RISCVVPseudo;
+                                             []>,
+        Sched<[WriteVIMovXV, ReadVIMovXV, ReadVIMovXX]>,
+        RISCVVPseudo;
     }
   }
 }
@@ -4093,17 +4335,19 @@ let mayLoad = 0, mayStore = 0, hasSideEffects = 0 in {
       let VLMul = m.value in {
         let HasSEWOp = 1, BaseInstr = VFMV_F_S in
         def "PseudoVFMV_" # f.FX # "_S_" # m.MX :
-                                          Pseudo<(outs f.fprclass:$rd),
-                                                 (ins m.vrclass:$rs2,
-                                                      ixlenimm:$sew),
-                                                 []>, RISCVVPseudo;
+          Pseudo<(outs f.fprclass:$rd),
+                 (ins m.vrclass:$rs2, ixlenimm:$sew), []>,
+          Sched<[WriteVFMovVF, ReadVFMovVF]>,
+          RISCVVPseudo;
         let HasVLOp = 1, HasSEWOp = 1, BaseInstr = VFMV_S_F,
             Constraints = "$rd = $rs1" in
         def "PseudoVFMV_S_" # f.FX # "_" # m.MX :
                                           Pseudo<(outs m.vrclass:$rd),
                                                  (ins m.vrclass:$rs1, f.fprclass:$rs2,
                                                       AVL:$vl, ixlenimm:$sew),
-                                                 []>, RISCVVPseudo;
+                                                 []>,
+          Sched<[WriteVFMovFV, ReadVFMovFV, ReadVFMovFX]>,
+          RISCVVPseudo;
       }
     }
   }
@@ -4114,52 +4358,33 @@ let mayLoad = 0, mayStore = 0, hasSideEffects = 0 in {
 // 17.3. Vector Slide Instructions
 //===----------------------------------------------------------------------===//
 let Predicates = [HasVInstructions] in {
-  defm PseudoVSLIDEUP    : VPseudoTernaryV_VX_VI<uimm5, "@earlyclobber $rd">;
-  defm PseudoVSLIDEDOWN  : VPseudoTernaryV_VX_VI<uimm5>;
-  defm PseudoVSLIDE1UP   : VPseudoBinaryV_VX<"@earlyclobber $rd">;
-  defm PseudoVSLIDE1DOWN : VPseudoBinaryV_VX;
+  defm PseudoVSLIDEUP    : VPseudoVSLD_VX_VI<uimm5, "@earlyclobber $rd">;
+  defm PseudoVSLIDEDOWN  : VPseudoVSLD_VX_VI<uimm5>;
+  defm PseudoVSLIDE1UP   : VPseudoVSLD1_VX<"@earlyclobber $rd">;
+  defm PseudoVSLIDE1DOWN : VPseudoVSLD1_VX;
 } // Predicates = [HasVInstructions]
 
 let Predicates = [HasVInstructionsAnyF] in {
-  defm PseudoVFSLIDE1UP  : VPseudoBinaryV_VF<"@earlyclobber $rd">;
-  defm PseudoVFSLIDE1DOWN : VPseudoBinaryV_VF;
+  defm PseudoVFSLIDE1UP  : VPseudoVSLD1_VF<"@earlyclobber $rd">;
+  defm PseudoVFSLIDE1DOWN : VPseudoVSLD1_VF;
 } // Predicates = [HasVInstructionsAnyF]
 
 //===----------------------------------------------------------------------===//
 // 17.4. Vector Register Gather Instructions
 //===----------------------------------------------------------------------===//
-defm PseudoVRGATHER    : VPseudoBinaryV_VV_VX_VI<uimm5, "@earlyclobber $rd">;
-defm PseudoVRGATHEREI16 : VPseudoBinaryV_VV_EEW</* eew */ 16, "@earlyclobber $rd">;
+defm PseudoVRGATHER     : VPseudoVGTR_VV_VX_VI<uimm5, "@earlyclobber $rd">;
+defm PseudoVRGATHEREI16 : VPseudoVGTR_VV_EEW</* eew */ 16, "@earlyclobber $rd">;
 
 //===----------------------------------------------------------------------===//
 // 17.5. Vector Compress Instruction
 //===----------------------------------------------------------------------===//
-defm PseudoVCOMPRESS : VPseudoUnaryV_V_AnyMask;
+defm PseudoVCOMPRESS : VPseudoVCPR_V;
 
 //===----------------------------------------------------------------------===//
 // Patterns.
 //===----------------------------------------------------------------------===//
 
 //===----------------------------------------------------------------------===//
-// 8. Vector AMO Operations
-//===----------------------------------------------------------------------===//
-let Predicates = [HasStdExtZvamo] in {
-  defm : VPatAMOV_WD<"int_riscv_vamoswap", "PseudoVAMOSWAP", AllIntegerVectors>;
-  defm : VPatAMOV_WD<"int_riscv_vamoadd", "PseudoVAMOADD", AllIntegerVectors>;
-  defm : VPatAMOV_WD<"int_riscv_vamoxor", "PseudoVAMOXOR", AllIntegerVectors>;
-  defm : VPatAMOV_WD<"int_riscv_vamoand", "PseudoVAMOAND", AllIntegerVectors>;
-  defm : VPatAMOV_WD<"int_riscv_vamoor", "PseudoVAMOOR", AllIntegerVectors>;
-  defm : VPatAMOV_WD<"int_riscv_vamomin", "PseudoVAMOMIN", AllIntegerVectors>;
-  defm : VPatAMOV_WD<"int_riscv_vamomax", "PseudoVAMOMAX", AllIntegerVectors>;
-  defm : VPatAMOV_WD<"int_riscv_vamominu", "PseudoVAMOMINU", AllIntegerVectors>;
-  defm : VPatAMOV_WD<"int_riscv_vamomaxu", "PseudoVAMOMAXU", AllIntegerVectors>;
-} // Predicates = [HasStdExtZvamo]
-
-let Predicates = [HasStdExtZvamo, HasVInstructionsAnyF] in {
-  defm : VPatAMOV_WD<"int_riscv_vamoswap", "PseudoVAMOSWAP", AllFloatVectors>;
-} // Predicates = [HasStdExtZvamo, HasVInstructionsAnyF]
-
-//===----------------------------------------------------------------------===//
 // 12. Vector Integer Arithmetic Instructions
 //===----------------------------------------------------------------------===//
 
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
index 461bdd348934..7eb8ae7d4193 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
@@ -382,50 +382,50 @@ def FSRI : RVBTernaryImm6<0b101, OPC_OP_IMM, "fsri",
 } // Predicates = [HasStdExtZbt]
 
 let Predicates = [HasStdExtZbb] in {
-def CLZ  : RVBUnary<0b0110000, 0b00000, 0b001, RISCVOpcode<0b0010011>, "clz">,
+def CLZ  : RVBUnary<0b0110000, 0b00000, 0b001, OPC_OP_IMM, "clz">,
            Sched<[WriteCLZ, ReadCLZ]>;
-def CTZ  : RVBUnary<0b0110000, 0b00001, 0b001, RISCVOpcode<0b0010011>, "ctz">,
+def CTZ  : RVBUnary<0b0110000, 0b00001, 0b001, OPC_OP_IMM, "ctz">,
            Sched<[WriteCTZ, ReadCTZ]>;
-def CPOP : RVBUnary<0b0110000, 0b00010, 0b001, RISCVOpcode<0b0010011>, "cpop">,
+def CPOP : RVBUnary<0b0110000, 0b00010, 0b001, OPC_OP_IMM, "cpop">,
            Sched<[WriteCPOP, ReadCPOP]>;
 } // Predicates = [HasStdExtZbb]
 
 let Predicates = [HasStdExtZbm, IsRV64] in
-def BMATFLIP : RVBUnary<0b0110000, 0b00011, 0b001, RISCVOpcode<0b0010011>,
-                        "bmatflip">, Sched<[]>;
+def BMATFLIP : RVBUnary<0b0110000, 0b00011, 0b001, OPC_OP_IMM, "bmatflip">,
+               Sched<[]>;
 
 let Predicates = [HasStdExtZbb] in {
-def SEXTB : RVBUnary<0b0110000, 0b00100, 0b001, RISCVOpcode<0b0010011>,
-                     "sext.b">, Sched<[WriteIALU, ReadIALU]>;
-def SEXTH : RVBUnary<0b0110000, 0b00101, 0b001, RISCVOpcode<0b0010011>,
-                     "sext.h">, Sched<[WriteIALU, ReadIALU]>;
+def SEXTB : RVBUnary<0b0110000, 0b00100, 0b001, OPC_OP_IMM, "sext.b">,
+            Sched<[WriteIALU, ReadIALU]>;
+def SEXTH : RVBUnary<0b0110000, 0b00101, 0b001, OPC_OP_IMM, "sext.h">,
+            Sched<[WriteIALU, ReadIALU]>;
 } // Predicates = [HasStdExtZbb]
 
 let Predicates = [HasStdExtZbr] in {
-def CRC32B : RVBUnary<0b0110000, 0b10000, 0b001, RISCVOpcode<0b0010011>,
-                      "crc32.b">, Sched<[]>;
-def CRC32H : RVBUnary<0b0110000, 0b10001, 0b001, RISCVOpcode<0b0010011>,
-                      "crc32.h">, Sched<[]>;
-def CRC32W : RVBUnary<0b0110000, 0b10010, 0b001, RISCVOpcode<0b0010011>,
-                      "crc32.w">, Sched<[]>;
+def CRC32B : RVBUnary<0b0110000, 0b10000, 0b001, OPC_OP_IMM, "crc32.b">,
+             Sched<[]>;
+def CRC32H : RVBUnary<0b0110000, 0b10001, 0b001, OPC_OP_IMM, "crc32.h">,
+             Sched<[]>;
+def CRC32W : RVBUnary<0b0110000, 0b10010, 0b001, OPC_OP_IMM, "crc32.w">,
+             Sched<[]>;
 } // Predicates = [HasStdExtZbr]
 
 let Predicates = [HasStdExtZbr, IsRV64] in
-def CRC32D  : RVBUnary<0b0110000, 0b10011, 0b001, RISCVOpcode<0b0010011>,
-                       "crc32.d">, Sched<[]>;
+def CRC32D  : RVBUnary<0b0110000, 0b10011, 0b001, OPC_OP_IMM, "crc32.d">,
+              Sched<[]>;
 
 let Predicates = [HasStdExtZbr] in {
-def CRC32CB : RVBUnary<0b0110000, 0b11000, 0b001, RISCVOpcode<0b0010011>,
-                       "crc32c.b">, Sched<[]>;
-def CRC32CH : RVBUnary<0b0110000, 0b11001, 0b001, RISCVOpcode<0b0010011>,
-                       "crc32c.h">, Sched<[]>;
-def CRC32CW : RVBUnary<0b0110000, 0b11010, 0b001, RISCVOpcode<0b0010011>,
-                       "crc32c.w">, Sched<[]>;
+def CRC32CB : RVBUnary<0b0110000, 0b11000, 0b001, OPC_OP_IMM, "crc32c.b">,
+              Sched<[]>;
+def CRC32CH : RVBUnary<0b0110000, 0b11001, 0b001, OPC_OP_IMM, "crc32c.h">,
+              Sched<[]>;
+def CRC32CW : RVBUnary<0b0110000, 0b11010, 0b001, OPC_OP_IMM, "crc32c.w">,
+              Sched<[]>;
 } // Predicates = [HasStdExtZbr]
 
 let Predicates = [HasStdExtZbr, IsRV64] in
-def CRC32CD : RVBUnary<0b0110000, 0b11011, 0b001, RISCVOpcode<0b0010011>,
-                       "crc32c.d">, Sched<[]>;
+def CRC32CD : RVBUnary<0b0110000, 0b11011, 0b001, OPC_OP_IMM, "crc32c.d">,
+              Sched<[]>;
 
 let Predicates = [HasStdExtZbc] in {
 def CLMUL  : ALU_rr<0b0000101, 0b001, "clmul">, Sched<[]>;
@@ -523,12 +523,12 @@ def FSRIW : RVBTernaryImm5<0b10, 0b101, OPC_OP_IMM_32,
 } // Predicates = [HasStdExtZbt, IsRV64]
 
 let Predicates = [HasStdExtZbb, IsRV64] in {
-def CLZW   : RVBUnary<0b0110000, 0b00000, 0b001, RISCVOpcode<0b0011011>,
-                      "clzw">, Sched<[WriteCLZ32, ReadCLZ32]>;
-def CTZW   : RVBUnary<0b0110000, 0b00001, 0b001, RISCVOpcode<0b0011011>,
-                      "ctzw">, Sched<[WriteCTZ32, ReadCTZ32]>;
-def CPOPW  : RVBUnary<0b0110000, 0b00010, 0b001, RISCVOpcode<0b0011011>,
-                      "cpopw">, Sched<[WriteCPOP32, ReadCPOP32]>;
+def CLZW   : RVBUnary<0b0110000, 0b00000, 0b001, OPC_OP_IMM_32, "clzw">,
+             Sched<[WriteCLZ32, ReadCLZ32]>;
+def CTZW   : RVBUnary<0b0110000, 0b00001, 0b001, OPC_OP_IMM_32, "ctzw">,
+             Sched<[WriteCTZ32, ReadCTZ32]>;
+def CPOPW  : RVBUnary<0b0110000, 0b00010, 0b001, OPC_OP_IMM_32, "cpopw">,
+             Sched<[WriteCPOP32, ReadCPOP32]>;
 } // Predicates = [HasStdExtZbb, IsRV64]
 
 let Predicates = [HasStdExtZbp, IsRV64] in {
@@ -791,6 +791,9 @@ def : Pat<(xor GPR:$rs1, BSETINVMask:$mask),
 def : Pat<(and (srl GPR:$rs1, uimmlog2xlen:$shamt), (XLenVT 1)),
           (BEXTI GPR:$rs1, uimmlog2xlen:$shamt)>;
 
+def : Pat<(and (not (srl GPR:$rs1, uimmlog2xlen:$shamt)), (XLenVT 1)),
+          (XORI (BEXTI GPR:$rs1, uimmlog2xlen:$shamt), (XLenVT 1))>;
+
 def : Pat<(or GPR:$r, BSETINVTwoBitsMask:$i),
           (BSETI (BSETI GPR:$r, (TrailingZerosXForm BSETINVTwoBitsMask:$i)),
                  (BSETINVTwoBitsMaskHigh BSETINVTwoBitsMask:$i))>;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td
index a33494461869..663e44813899 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td
@@ -28,41 +28,6 @@ def riscv_fmv_x_anyexth
     : SDNode<"RISCVISD::FMV_X_ANYEXTH", SDT_RISCVFMV_X_ANYEXTH>;
 
 //===----------------------------------------------------------------------===//
-// Instruction class templates
-//===----------------------------------------------------------------------===//
-
-let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
-class FPFMAH_rrr_frm<RISCVOpcode opcode, string opcodestr>
-    : RVInstR4Frm<0b10, opcode, (outs FPR16:$rd),
-                  (ins FPR16:$rs1, FPR16:$rs2, FPR16:$rs3, frmarg:$funct3),
-                  opcodestr, "$rd, $rs1, $rs2, $rs3, $funct3">;
-
-class FPFMAHDynFrmAlias<FPFMAH_rrr_frm Inst, string OpcodeStr>
-    : InstAlias<OpcodeStr#" $rd, $rs1, $rs2, $rs3",
-                (Inst FPR16:$rd, FPR16:$rs1, FPR16:$rs2, FPR16:$rs3, 0b111)>;
-
-let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
-class FPALUH_rr<bits<7> funct7, bits<3> funct3, string opcodestr>
-    : RVInstR<funct7, funct3, OPC_OP_FP, (outs FPR16:$rd),
-              (ins FPR16:$rs1, FPR16:$rs2), opcodestr, "$rd, $rs1, $rs2">;
-
-let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
-class FPALUH_rr_frm<bits<7> funct7, string opcodestr>
-    : RVInstRFrm<funct7, OPC_OP_FP, (outs FPR16:$rd),
-                 (ins FPR16:$rs1, FPR16:$rs2, frmarg:$funct3), opcodestr,
-                  "$rd, $rs1, $rs2, $funct3">;
-
-class FPALUHDynFrmAlias<FPALUH_rr_frm Inst, string OpcodeStr>
-    : InstAlias<OpcodeStr#" $rd, $rs1, $rs2",
-                (Inst FPR16:$rd, FPR16:$rs1, FPR16:$rs2, 0b111)>;
-
-let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
-class FPCmpH_rr<bits<3> funct3, string opcodestr>
-    : RVInstR<0b1010010, funct3, OPC_OP_FP, (outs GPR:$rd),
-              (ins FPR16:$rs1, FPR16:$rs2), opcodestr, "$rd, $rs1, $rs2">,
-      Sched<[WriteFCmp16, ReadFCmp16, ReadFCmp16]>;
-
-//===----------------------------------------------------------------------===//
 // Instructions
 //===----------------------------------------------------------------------===//
 
@@ -84,145 +49,120 @@ def FSH : RVInstS<0b001, OPC_STORE_FP, (outs),
 } // Predicates = [HasStdExtZfhmin]
 
 let Predicates = [HasStdExtZfh] in {
-def FMADD_H  : FPFMAH_rrr_frm<OPC_MADD, "fmadd.h">,
-               Sched<[WriteFMA16, ReadFMA16, ReadFMA16, ReadFMA16]>;
-def          : FPFMAHDynFrmAlias<FMADD_H, "fmadd.h">;
-def FMSUB_H  : FPFMAH_rrr_frm<OPC_MSUB, "fmsub.h">,
-               Sched<[WriteFMA16, ReadFMA16, ReadFMA16, ReadFMA16]>;
-def          : FPFMAHDynFrmAlias<FMSUB_H, "fmsub.h">;
-def FNMSUB_H : FPFMAH_rrr_frm<OPC_NMSUB, "fnmsub.h">,
-               Sched<[WriteFMA16, ReadFMA16, ReadFMA16, ReadFMA16]>;
-def          : FPFMAHDynFrmAlias<FNMSUB_H, "fnmsub.h">;
-def FNMADD_H : FPFMAH_rrr_frm<OPC_NMADD, "fnmadd.h">,
-               Sched<[WriteFMA16, ReadFMA16, ReadFMA16, ReadFMA16]>;
-def          : FPFMAHDynFrmAlias<FNMADD_H, "fnmadd.h">;
-
-def FADD_H : FPALUH_rr_frm<0b0000010, "fadd.h">,
+let SchedRW = [WriteFMA16, ReadFMA16, ReadFMA16, ReadFMA16] in {
+def FMADD_H  : FPFMA_rrr_frm<OPC_MADD,  0b10, "fmadd.h",  FPR16>;
+def FMSUB_H  : FPFMA_rrr_frm<OPC_MSUB,  0b10, "fmsub.h",  FPR16>;
+def FNMSUB_H : FPFMA_rrr_frm<OPC_NMSUB, 0b10, "fnmsub.h", FPR16>;
+def FNMADD_H : FPFMA_rrr_frm<OPC_NMADD, 0b10, "fnmadd.h", FPR16>;
+}
+
+def : FPFMADynFrmAlias<FMADD_H,  "fmadd.h",  FPR16>;
+def : FPFMADynFrmAlias<FMSUB_H,  "fmsub.h",  FPR16>;
+def : FPFMADynFrmAlias<FNMSUB_H, "fnmsub.h", FPR16>;
+def : FPFMADynFrmAlias<FNMADD_H, "fnmadd.h", FPR16>;
+
+def FADD_H : FPALU_rr_frm<0b0000010, "fadd.h", FPR16>,
              Sched<[WriteFALU16, ReadFALU16, ReadFALU16]>;
-def        : FPALUHDynFrmAlias<FADD_H, "fadd.h">;
-def FSUB_H : FPALUH_rr_frm<0b0000110, "fsub.h">,
+def FSUB_H : FPALU_rr_frm<0b0000110, "fsub.h", FPR16>,
              Sched<[WriteFALU16, ReadFALU16, ReadFALU16]>;
-def        : FPALUHDynFrmAlias<FSUB_H, "fsub.h">;
-def FMUL_H : FPALUH_rr_frm<0b0001010, "fmul.h">,
+def FMUL_H : FPALU_rr_frm<0b0001010, "fmul.h", FPR16>,
              Sched<[WriteFMul16, ReadFMul16, ReadFMul16]>;
-def        : FPALUHDynFrmAlias<FMUL_H, "fmul.h">;
-def FDIV_H : FPALUH_rr_frm<0b0001110, "fdiv.h">,
+def FDIV_H : FPALU_rr_frm<0b0001110, "fdiv.h", FPR16>,
              Sched<[WriteFDiv16, ReadFDiv16, ReadFDiv16]>;
-def        : FPALUHDynFrmAlias<FDIV_H, "fdiv.h">;
 
-def FSQRT_H : FPUnaryOp_r_frm<0b0101110, FPR16, FPR16, "fsqrt.h">,
-              Sched<[WriteFSqrt16, ReadFSqrt16]> {
-  let rs2 = 0b00000;
-}
+def        : FPALUDynFrmAlias<FADD_H, "fadd.h", FPR16>;
+def        : FPALUDynFrmAlias<FSUB_H, "fsub.h", FPR16>;
+def        : FPALUDynFrmAlias<FMUL_H, "fmul.h", FPR16>;
+def        : FPALUDynFrmAlias<FDIV_H, "fdiv.h", FPR16>;
+
+def FSQRT_H : FPUnaryOp_r_frm<0b0101110, 0b00000, FPR16, FPR16, "fsqrt.h">,
+              Sched<[WriteFSqrt16, ReadFSqrt16]>;
 def         : FPUnaryOpDynFrmAlias<FSQRT_H, "fsqrt.h", FPR16, FPR16>;
 
-def FSGNJ_H  : FPALUH_rr<0b0010010, 0b000, "fsgnj.h">,
-               Sched<[WriteFSGNJ16, ReadFSGNJ16, ReadFSGNJ16]>;
-def FSGNJN_H : FPALUH_rr<0b0010010, 0b001, "fsgnjn.h">,
-               Sched<[WriteFSGNJ16, ReadFSGNJ16, ReadFSGNJ16]>;
-def FSGNJX_H : FPALUH_rr<0b0010010, 0b010, "fsgnjx.h">,
-               Sched<[WriteFSGNJ16, ReadFSGNJ16, ReadFSGNJ16]>;
-def FMIN_H   : FPALUH_rr<0b0010110, 0b000, "fmin.h">,
-               Sched<[WriteFMinMax16, ReadFMinMax16, ReadFMinMax16]>;
-def FMAX_H   : FPALUH_rr<0b0010110, 0b001, "fmax.h">,
-               Sched<[WriteFMinMax16, ReadFMinMax16, ReadFMinMax16]>;
-
-def FCVT_W_H : FPUnaryOp_r_frm<0b1100010, GPR, FPR16, "fcvt.w.h">,
-               Sched<[WriteFCvtF16ToI32, ReadFCvtF16ToI32]> {
-  let rs2 = 0b00000;
+let SchedRW = [WriteFSGNJ16, ReadFSGNJ16, ReadFSGNJ16],
+    mayRaiseFPException = 0 in {
+def FSGNJ_H  : FPALU_rr<0b0010010, 0b000, "fsgnj.h", FPR16>;
+def FSGNJN_H : FPALU_rr<0b0010010, 0b001, "fsgnjn.h", FPR16>;
+def FSGNJX_H : FPALU_rr<0b0010010, 0b010, "fsgnjx.h", FPR16>;
 }
-def          : FPUnaryOpDynFrmAlias<FCVT_W_H, "fcvt.w.h", GPR, FPR16>;
 
-def FCVT_WU_H : FPUnaryOp_r_frm<0b1100010, GPR, FPR16, "fcvt.wu.h">,
-                Sched<[WriteFCvtF16ToI32, ReadFCvtF16ToI32]> {
-  let rs2 = 0b00001;
+let SchedRW = [WriteFMinMax16, ReadFMinMax16, ReadFMinMax16] in {
+def FMIN_H   : FPALU_rr<0b0010110, 0b000, "fmin.h", FPR16>;
+def FMAX_H   : FPALU_rr<0b0010110, 0b001, "fmax.h", FPR16>;
 }
+
+def FCVT_W_H : FPUnaryOp_r_frm<0b1100010, 0b00000, GPR, FPR16, "fcvt.w.h">,
+               Sched<[WriteFCvtF16ToI32, ReadFCvtF16ToI32]>;
+def          : FPUnaryOpDynFrmAlias<FCVT_W_H, "fcvt.w.h", GPR, FPR16>;
+
+def FCVT_WU_H : FPUnaryOp_r_frm<0b1100010, 0b00001, GPR, FPR16, "fcvt.wu.h">,
+                Sched<[WriteFCvtF16ToI32, ReadFCvtF16ToI32]>;
 def           : FPUnaryOpDynFrmAlias<FCVT_WU_H, "fcvt.wu.h", GPR, FPR16>;
 
-def FCVT_H_W : FPUnaryOp_r_frm<0b1101010, FPR16, GPR, "fcvt.h.w">,
-               Sched<[WriteFCvtI32ToF16, ReadFCvtI32ToF16]> {
-  let rs2 = 0b00000;
-}
+def FCVT_H_W : FPUnaryOp_r_frm<0b1101010, 0b00000, FPR16, GPR, "fcvt.h.w">,
+               Sched<[WriteFCvtI32ToF16, ReadFCvtI32ToF16]>;
 def          : FPUnaryOpDynFrmAlias<FCVT_H_W, "fcvt.h.w", FPR16, GPR>;
 
-def FCVT_H_WU : FPUnaryOp_r_frm<0b1101010, FPR16, GPR, "fcvt.h.wu">,
-                Sched<[WriteFCvtI32ToF16, ReadFCvtI32ToF16]> {
-  let rs2 = 0b00001;
-}
+def FCVT_H_WU : FPUnaryOp_r_frm<0b1101010, 0b00001, FPR16, GPR, "fcvt.h.wu">,
+                Sched<[WriteFCvtI32ToF16, ReadFCvtI32ToF16]>;
 def           : FPUnaryOpDynFrmAlias<FCVT_H_WU, "fcvt.h.wu", FPR16, GPR>;
 } // Predicates = [HasStdExtZfh]
 
 let Predicates = [HasStdExtZfhmin] in {
-def FCVT_H_S : FPUnaryOp_r_frm<0b0100010, FPR16, FPR32, "fcvt.h.s">,
-               Sched<[WriteFCvtF32ToF16, ReadFCvtF32ToF16]> {
-  let rs2 = 0b00000;
-}
+def FCVT_H_S : FPUnaryOp_r_frm<0b0100010, 0b00000, FPR16, FPR32, "fcvt.h.s">,
+               Sched<[WriteFCvtF32ToF16, ReadFCvtF32ToF16]>;
 def          : FPUnaryOpDynFrmAlias<FCVT_H_S, "fcvt.h.s", FPR16, FPR32>;
 
-def FCVT_S_H : FPUnaryOp_r<0b0100000, 0b000, FPR32, FPR16, "fcvt.s.h">,
-               Sched<[WriteFCvtF16ToF32, ReadFCvtF16ToF32]> {
-  let rs2 = 0b00010;
-}
+def FCVT_S_H : FPUnaryOp_r<0b0100000, 0b00010, 0b000, FPR32, FPR16, "fcvt.s.h">,
+               Sched<[WriteFCvtF16ToF32, ReadFCvtF16ToF32]>;
 
-def FMV_X_H : FPUnaryOp_r<0b1110010, 0b000, GPR, FPR16, "fmv.x.h">,
-              Sched<[WriteFMovF16ToI16, ReadFMovF16ToI16]> {
-  let rs2 = 0b00000;
-}
+let mayRaiseFPException = 0 in
+def FMV_X_H : FPUnaryOp_r<0b1110010, 0b00000, 0b000, GPR, FPR16, "fmv.x.h">,
+              Sched<[WriteFMovF16ToI16, ReadFMovF16ToI16]>;
 
-def FMV_H_X : FPUnaryOp_r<0b1111010, 0b000, FPR16, GPR, "fmv.h.x">,
-              Sched<[WriteFMovI16ToF16, ReadFMovI16ToF16]> {
-  let rs2 = 0b00000;
-}
+let mayRaiseFPException = 0 in
+def FMV_H_X : FPUnaryOp_r<0b1111010, 0b00000, 0b000, FPR16, GPR, "fmv.h.x">,
+              Sched<[WriteFMovI16ToF16, ReadFMovI16ToF16]>;
 } // Predicates = [HasStdExtZfhmin]
 
 let Predicates = [HasStdExtZfh] in {
-def FEQ_H : FPCmpH_rr<0b010, "feq.h">;
-def FLT_H : FPCmpH_rr<0b001, "flt.h">;
-def FLE_H : FPCmpH_rr<0b000, "fle.h">;
 
-def FCLASS_H : FPUnaryOp_r<0b1110010, 0b001, GPR, FPR16, "fclass.h">,
-               Sched<[WriteFClass16, ReadFClass16]> {
-  let rs2 = 0b00000;
+let SchedRW = [WriteFCmp16, ReadFCmp16, ReadFCmp16] in {
+def FEQ_H : FPCmp_rr<0b1010010, 0b010, "feq.h", FPR16>;
+def FLT_H : FPCmp_rr<0b1010010, 0b001, "flt.h", FPR16>;
+def FLE_H : FPCmp_rr<0b1010010, 0b000, "fle.h", FPR16>;
 }
+
+let mayRaiseFPException = 0 in
+def FCLASS_H : FPUnaryOp_r<0b1110010, 0b00000, 0b001, GPR, FPR16, "fclass.h">,
+               Sched<[WriteFClass16, ReadFClass16]>;
 } // Predicates = [HasStdExtZfh]
 
 let Predicates = [HasStdExtZfh, IsRV64] in {
-def FCVT_L_H  : FPUnaryOp_r_frm<0b1100010, GPR, FPR16, "fcvt.l.h">,
-                Sched<[WriteFCvtF16ToI64, ReadFCvtF16ToI64]> {
-  let rs2 = 0b00010;
-}
+def FCVT_L_H  : FPUnaryOp_r_frm<0b1100010, 0b00010, GPR, FPR16, "fcvt.l.h">,
+                Sched<[WriteFCvtF16ToI64, ReadFCvtF16ToI64]>;
 def           : FPUnaryOpDynFrmAlias<FCVT_L_H, "fcvt.l.h", GPR, FPR16>;
 
-def FCVT_LU_H  : FPUnaryOp_r_frm<0b1100010, GPR, FPR16, "fcvt.lu.h">,
-                 Sched<[WriteFCvtF16ToI64, ReadFCvtF16ToI64]> {
-  let rs2 = 0b00011;
-}
+def FCVT_LU_H  : FPUnaryOp_r_frm<0b1100010, 0b00011, GPR, FPR16, "fcvt.lu.h">,
+                 Sched<[WriteFCvtF16ToI64, ReadFCvtF16ToI64]>;
 def            : FPUnaryOpDynFrmAlias<FCVT_LU_H, "fcvt.lu.h", GPR, FPR16>;
 
-def FCVT_H_L : FPUnaryOp_r_frm<0b1101010, FPR16, GPR, "fcvt.h.l">,
-               Sched<[WriteFCvtI64ToF16, ReadFCvtI64ToF16]> {
-  let rs2 = 0b00010;
-}
+def FCVT_H_L : FPUnaryOp_r_frm<0b1101010, 0b00010, FPR16, GPR, "fcvt.h.l">,
+               Sched<[WriteFCvtI64ToF16, ReadFCvtI64ToF16]>;
 def          : FPUnaryOpDynFrmAlias<FCVT_H_L, "fcvt.h.l", FPR16, GPR>;
 
-def FCVT_H_LU : FPUnaryOp_r_frm<0b1101010, FPR16, GPR, "fcvt.h.lu">,
-                Sched<[WriteFCvtI64ToF16, ReadFCvtI64ToF16]> {
-  let rs2 = 0b00011;
-}
+def FCVT_H_LU : FPUnaryOp_r_frm<0b1101010, 0b00011, FPR16, GPR, "fcvt.h.lu">,
+                Sched<[WriteFCvtI64ToF16, ReadFCvtI64ToF16]>;
 def           : FPUnaryOpDynFrmAlias<FCVT_H_LU, "fcvt.h.lu", FPR16, GPR>;
 } // Predicates = [HasStdExtZfh, IsRV64]
 
 let Predicates = [HasStdExtZfhmin, HasStdExtD] in {
-def FCVT_H_D : FPUnaryOp_r_frm<0b0100010, FPR16, FPR64, "fcvt.h.d">,
-               Sched<[WriteFCvtF64ToF16, ReadFCvtF64ToF16]> {
-  let rs2 = 0b00001;
-}
+def FCVT_H_D : FPUnaryOp_r_frm<0b0100010, 0b00001, FPR16, FPR64, "fcvt.h.d">,
+               Sched<[WriteFCvtF64ToF16, ReadFCvtF64ToF16]>;
 def          : FPUnaryOpDynFrmAlias<FCVT_H_D, "fcvt.h.d", FPR16, FPR64>;
 
-def FCVT_D_H : FPUnaryOp_r<0b0100001, 0b000, FPR64, FPR16, "fcvt.d.h">,
-               Sched<[WriteFCvtF16ToF64, ReadFCvtF16ToF64]> {
-  let rs2 = 0b00010;
-}
+def FCVT_D_H : FPUnaryOp_r<0b0100001, 0b00010, 0b000, FPR64, FPR16, "fcvt.d.h">,
+               Sched<[WriteFCvtF16ToF64, ReadFCvtF16ToF64]>;
 } // Predicates = [HasStdExtZfhmin, HasStdExtD]
 
 //===----------------------------------------------------------------------===//
@@ -275,12 +215,12 @@ def : Pat<(f16 (fpimm0)), (FMV_H_X X0)>;
 
 /// Float arithmetic operations
 
-def : PatFpr16Fpr16DynFrm<fadd, FADD_H>;
-def : PatFpr16Fpr16DynFrm<fsub, FSUB_H>;
-def : PatFpr16Fpr16DynFrm<fmul, FMUL_H>;
-def : PatFpr16Fpr16DynFrm<fdiv, FDIV_H>;
+def : PatFpr16Fpr16DynFrm<any_fadd, FADD_H>;
+def : PatFpr16Fpr16DynFrm<any_fsub, FSUB_H>;
+def : PatFpr16Fpr16DynFrm<any_fmul, FMUL_H>;
+def : PatFpr16Fpr16DynFrm<any_fdiv, FDIV_H>;
 
-def : Pat<(fsqrt FPR16:$rs1), (FSQRT_H FPR16:$rs1, 0b111)>;
+def : Pat<(any_fsqrt FPR16:$rs1), (FSQRT_H FPR16:$rs1, 0b111)>;
 
 def : Pat<(fneg FPR16:$rs1), (FSGNJN_H $rs1, $rs1)>;
 def : Pat<(fabs FPR16:$rs1), (FSGNJX_H $rs1, $rs1)>;
@@ -292,19 +232,19 @@ def : Pat<(fcopysign FPR16:$rs1, FPR32:$rs2),
 def : Pat<(fcopysign FPR32:$rs1, FPR16:$rs2), (FSGNJ_S $rs1, (FCVT_S_H $rs2))>;
 
 // fmadd: rs1 * rs2 + rs3
-def : Pat<(fma FPR16:$rs1, FPR16:$rs2, FPR16:$rs3),
+def : Pat<(any_fma FPR16:$rs1, FPR16:$rs2, FPR16:$rs3),
           (FMADD_H $rs1, $rs2, $rs3, 0b111)>;
 
 // fmsub: rs1 * rs2 - rs3
-def : Pat<(fma FPR16:$rs1, FPR16:$rs2, (fneg FPR16:$rs3)),
+def : Pat<(any_fma FPR16:$rs1, FPR16:$rs2, (fneg FPR16:$rs3)),
           (FMSUB_H FPR16:$rs1, FPR16:$rs2, FPR16:$rs3, 0b111)>;
 
 // fnmsub: -rs1 * rs2 + rs3
-def : Pat<(fma (fneg FPR16:$rs1), FPR16:$rs2, FPR16:$rs3),
+def : Pat<(any_fma (fneg FPR16:$rs1), FPR16:$rs2, FPR16:$rs3),
           (FNMSUB_H FPR16:$rs1, FPR16:$rs2, FPR16:$rs3, 0b111)>;
 
 // fnmadd: -rs1 * rs2 - rs3
-def : Pat<(fma (fneg FPR16:$rs1), FPR16:$rs2, (fneg FPR16:$rs3)),
+def : Pat<(any_fma (fneg FPR16:$rs1), FPR16:$rs2, (fneg FPR16:$rs3)),
           (FNMADD_H FPR16:$rs1, FPR16:$rs2, FPR16:$rs3, 0b111)>;
 
 // The ratified 20191213 ISA spec defines fmin and fmax in a way that matches
@@ -337,8 +277,8 @@ defm : StPat<store, FSH, FPR16, f16>;
 /// Float conversion operations
 
 // f32 -> f16, f16 -> f32
-def : Pat<(fpround FPR32:$rs1), (FCVT_H_S FPR32:$rs1, 0b111)>;
-def : Pat<(fpextend FPR16:$rs1), (FCVT_S_H FPR16:$rs1)>;
+def : Pat<(any_fpround FPR32:$rs1), (FCVT_H_S FPR32:$rs1, 0b111)>;
+def : Pat<(any_fpextend FPR16:$rs1), (FCVT_S_H FPR16:$rs1)>;
 
 // Moves (no conversion)
 def : Pat<(riscv_fmv_h_x GPR:$src), (FMV_H_X GPR:$src)>;
@@ -347,8 +287,8 @@ def : Pat<(riscv_fmv_x_anyexth FPR16:$src), (FMV_X_H FPR16:$src)>;
 
 let Predicates = [HasStdExtZfh, IsRV32] in {
 // half->[u]int. Round-to-zero must be used.
-def : Pat<(i32 (fp_to_sint FPR16:$rs1)), (FCVT_W_H $rs1, 0b001)>;
-def : Pat<(i32 (fp_to_uint FPR16:$rs1)), (FCVT_WU_H $rs1, 0b001)>;
+def : Pat<(i32 (any_fp_to_sint FPR16:$rs1)), (FCVT_W_H $rs1, 0b001)>;
+def : Pat<(i32 (any_fp_to_uint FPR16:$rs1)), (FCVT_WU_H $rs1, 0b001)>;
 
 // Saturating float->[u]int32.
 def : Pat<(i32 (riscv_fcvt_x_rtz FPR16:$rs1)), (FCVT_W_H $rs1, 0b001)>;
@@ -361,20 +301,20 @@ def : Pat<(i32 (lrint FPR16:$rs1)), (FCVT_W_H $rs1, 0b111)>;
 def : Pat<(i32 (lround FPR16:$rs1)), (FCVT_W_H $rs1, 0b100)>;
 
 // [u]int->half. Match GCC and default to using dynamic rounding mode.
-def : Pat<(sint_to_fp (i32 GPR:$rs1)), (FCVT_H_W $rs1, 0b111)>;
-def : Pat<(uint_to_fp (i32 GPR:$rs1)), (FCVT_H_WU $rs1, 0b111)>;
+def : Pat<(any_sint_to_fp (i32 GPR:$rs1)), (FCVT_H_W $rs1, 0b111)>;
+def : Pat<(any_uint_to_fp (i32 GPR:$rs1)), (FCVT_H_WU $rs1, 0b111)>;
 } // Predicates = [HasStdExtZfh, IsRV32]
 
 let Predicates = [HasStdExtZfh, IsRV64] in {
 // Use target specific isd nodes to help us remember the result is sign
 // extended. Matching sext_inreg+fptoui/fptosi may cause the conversion to be
 // duplicated if it has another user that didn't need the sign_extend.
-def : Pat<(riscv_fcvt_w_rtz_rv64 FPR16:$rs1),  (FCVT_W_H $rs1, 0b001)>;
-def : Pat<(riscv_fcvt_wu_rtz_rv64 FPR16:$rs1), (FCVT_WU_H $rs1, 0b001)>;
+def : Pat<(riscv_any_fcvt_w_rtz_rv64 FPR16:$rs1),  (FCVT_W_H $rs1, 0b001)>;
+def : Pat<(riscv_any_fcvt_wu_rtz_rv64 FPR16:$rs1), (FCVT_WU_H $rs1, 0b001)>;
 
 // half->[u]int64. Round-to-zero must be used.
-def : Pat<(i64 (fp_to_sint FPR16:$rs1)), (FCVT_L_H $rs1, 0b001)>;
-def : Pat<(i64 (fp_to_uint FPR16:$rs1)), (FCVT_LU_H $rs1, 0b001)>;
+def : Pat<(i64 (any_fp_to_sint FPR16:$rs1)), (FCVT_L_H $rs1, 0b001)>;
+def : Pat<(i64 (any_fp_to_uint FPR16:$rs1)), (FCVT_LU_H $rs1, 0b001)>;
 
 // Saturating float->[u]int64.
 def : Pat<(i64 (riscv_fcvt_x_rtz FPR16:$rs1)), (FCVT_L_H $rs1, 0b001)>;
@@ -389,17 +329,17 @@ def : Pat<(i64 (lround FPR16:$rs1)), (FCVT_L_H $rs1, 0b100)>;
 def : Pat<(i64 (llround FPR16:$rs1)), (FCVT_L_H $rs1, 0b100)>;
 
 // [u]int->fp. Match GCC and default to using dynamic rounding mode.
-def : Pat<(sint_to_fp (i64 (sexti32 (i64 GPR:$rs1)))), (FCVT_H_W $rs1, 0b111)>;
-def : Pat<(uint_to_fp (i64 (zexti32 (i64 GPR:$rs1)))), (FCVT_H_WU $rs1, 0b111)>;
-def : Pat<(sint_to_fp (i64 GPR:$rs1)), (FCVT_H_L $rs1, 0b111)>;
-def : Pat<(uint_to_fp (i64 GPR:$rs1)), (FCVT_H_LU $rs1, 0b111)>;
+def : Pat<(any_sint_to_fp (i64 (sexti32 (i64 GPR:$rs1)))), (FCVT_H_W $rs1, 0b111)>;
+def : Pat<(any_uint_to_fp (i64 (zexti32 (i64 GPR:$rs1)))), (FCVT_H_WU $rs1, 0b111)>;
+def : Pat<(any_sint_to_fp (i64 GPR:$rs1)), (FCVT_H_L $rs1, 0b111)>;
+def : Pat<(any_uint_to_fp (i64 GPR:$rs1)), (FCVT_H_LU $rs1, 0b111)>;
 } // Predicates = [HasStdExtZfh, IsRV64]
 
 let Predicates = [HasStdExtZfhmin, HasStdExtD] in {
 /// Float conversion operations
 // f64 -> f16, f16 -> f64
-def : Pat<(fpround FPR64:$rs1), (FCVT_H_D FPR64:$rs1, 0b111)>;
-def : Pat<(fpextend FPR16:$rs1), (FCVT_D_H FPR16:$rs1)>;
+def : Pat<(any_fpround FPR64:$rs1), (FCVT_H_D FPR64:$rs1, 0b111)>;
+def : Pat<(any_fpextend FPR16:$rs1), (FCVT_D_H FPR16:$rs1)>;
 
 /// Float arithmetic operations
 def : Pat<(fcopysign FPR16:$rs1, FPR64:$rs2),
diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
index 798532d5bc44..9094dff1dda1 100644
--- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
@@ -105,7 +105,6 @@ BitVector RISCVRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
   // Floating point environment registers.
   markSuperRegs(Reserved, RISCV::FRM);
   markSuperRegs(Reserved, RISCV::FFLAGS);
-  markSuperRegs(Reserved, RISCV::FCSR);
 
   assert(checkAllSuperRegsMarked(Reserved));
   return Reserved;
diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.td b/llvm/lib/Target/RISCV/RISCVRegisterInfo.td
index a56f992d320e..20903b317180 100644
--- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.td
+++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.td
@@ -550,16 +550,15 @@ def VRM8NoV0 : VReg<[vint8m8_t, vint16m8_t, vint32m8_t, vint64m8_t,
                      vfloat16m8_t, vfloat32m8_t, vfloat64m8_t],
              (add V8M8, V16M8, V24M8), 8>;
 
-defvar VMaskVTs = [vbool64_t, vbool32_t, vbool16_t, vbool8_t,
-                   vbool4_t, vbool2_t, vbool1_t];
+defvar VMaskVTs = [vbool1_t, vbool2_t, vbool4_t, vbool8_t, vbool16_t,
+                   vbool32_t, vbool64_t];
 
 def VMV0 : RegisterClass<"RISCV", VMaskVTs, 64, (add V0)> {
   let Size = 64;
 }
 
 // The register class is added for inline assembly for vector mask types.
-def VM : VReg<[vbool1_t, vbool2_t, vbool4_t, vbool8_t, vbool16_t,
-               vbool32_t, vbool64_t],
+def VM : VReg<VMaskVTs,
            (add (sequence "V%u", 8, 31),
                 (sequence "V%u", 0, 7)), 1>;
 
@@ -578,7 +577,6 @@ foreach m = LMULList.m in {
 // Special registers
 def FFLAGS : RISCVReg<0, "fflags">;
 def FRM    : RISCVReg<0, "frm">;
-def FCSR   : RISCVReg<0, "fcsr">;
 
 // Any type register. Used for .insn directives when we don't know what the
 // register types could be.
diff --git a/llvm/lib/Target/RISCV/RISCVSchedRocket.td b/llvm/lib/Target/RISCV/RISCVSchedRocket.td
index 14f59152ed42..d5a0932c8778 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedRocket.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedRocket.td
@@ -16,7 +16,8 @@ def RocketModel : SchedMachineModel {
   let IssueWidth = 1;        // 1 micro-op is dispatched per cycle.
   let LoadLatency = 3;
   let MispredictPenalty = 3;
-  let UnsupportedFeatures = [HasStdExtV, HasStdExtZvamo, HasStdExtZvlsseg];
+  let CompleteModel = false;
+  let UnsupportedFeatures = [HasStdExtV, HasStdExtZvlsseg];
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
index 5b435fcb16a2..7f9d0aabc4ed 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
@@ -15,7 +15,7 @@ def SiFive7Model : SchedMachineModel {
   let LoadLatency = 3;
   let MispredictPenalty = 3;
   let CompleteModel = 0;
-  let UnsupportedFeatures = [HasStdExtV, HasStdExtZvamo, HasStdExtZvlsseg];
+  let UnsupportedFeatures = [HasStdExtV, HasStdExtZvlsseg];
 }
 
 // The SiFive7 microarchitecture has two pipelines: A and B.
diff --git a/llvm/lib/Target/RISCV/RISCVSubtarget.h b/llvm/lib/Target/RISCV/RISCVSubtarget.h
index deb2a11f98f1..d0330e6984a5 100644
--- a/llvm/lib/Target/RISCV/RISCVSubtarget.h
+++ b/llvm/lib/Target/RISCV/RISCVSubtarget.h
@@ -51,7 +51,6 @@ class RISCVSubtarget : public RISCVGenSubtargetInfo {
   bool HasStdExtZbt = false;
   bool HasStdExtV = false;
   bool HasStdExtZvlsseg = false;
-  bool HasStdExtZvamo = false;
   bool HasStdExtZfhmin = false;
   bool HasStdExtZfh = false;
   bool HasRV64 = false;
@@ -118,7 +117,6 @@ public:
   bool hasStdExtZbt() const { return HasStdExtZbt; }
   bool hasStdExtV() const { return HasStdExtV; }
   bool hasStdExtZvlsseg() const { return HasStdExtZvlsseg; }
-  bool hasStdExtZvamo() const { return HasStdExtZvamo; }
   bool hasStdExtZfhmin() const { return HasStdExtZfhmin; }
   bool hasStdExtZfh() const { return HasStdExtZfh; }
   bool is64Bit() const { return HasRV64; }
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 56f0952fafc9..c435430a1288 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -162,3 +162,94 @@ InstructionCost RISCVTTIImpl::getGatherScatterOpCost(
       getMemoryOpCost(Opcode, VTy->getElementType(), Alignment, 0, CostKind, I);
   return NumLoads * MemOpCost;
 }
+
+void RISCVTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
+                                           TTI::UnrollingPreferences &UP,
+                                           OptimizationRemarkEmitter *ORE) {
+  // TODO: More tuning on benchmarks and metrics with changes as needed
+  //       would apply to all settings below to enable performance.
+
+  // Support explicit targets enabled for SiFive with the unrolling preferences
+  // below
+  bool UseDefaultPreferences = true;
+  if (ST->getTuneCPU().contains("sifive-e76") ||
+      ST->getTuneCPU().contains("sifive-s76") ||
+      ST->getTuneCPU().contains("sifive-u74") ||
+      ST->getTuneCPU().contains("sifive-7"))
+    UseDefaultPreferences = false;
+
+  if (UseDefaultPreferences)
+    return BasicTTIImplBase::getUnrollingPreferences(L, SE, UP, ORE);
+
+  // Enable Upper bound unrolling universally, not dependant upon the conditions
+  // below.
+  UP.UpperBound = true;
+
+  // Disable loop unrolling for Oz and Os.
+  UP.OptSizeThreshold = 0;
+  UP.PartialOptSizeThreshold = 0;
+  if (L->getHeader()->getParent()->hasOptSize())
+    return;
+
+  SmallVector<BasicBlock *, 4> ExitingBlocks;
+  L->getExitingBlocks(ExitingBlocks);
+  LLVM_DEBUG(dbgs() << "Loop has:\n"
+                    << "Blocks: " << L->getNumBlocks() << "\n"
+                    << "Exit blocks: " << ExitingBlocks.size() << "\n");
+
+  // Only allow another exit other than the latch. This acts as an early exit
+  // as it mirrors the profitability calculation of the runtime unroller.
+  if (ExitingBlocks.size() > 2)
+    return;
+
+  // Limit the CFG of the loop body for targets with a branch predictor.
+  // Allowing 4 blocks permits if-then-else diamonds in the body.
+  if (L->getNumBlocks() > 4)
+    return;
+
+  // Don't unroll vectorized loops, including the remainder loop
+  if (getBooleanLoopAttribute(L, "llvm.loop.isvectorized"))
+    return;
+
+  // Scan the loop: don't unroll loops with calls as this could prevent
+  // inlining.
+  InstructionCost Cost = 0;
+  for (auto *BB : L->getBlocks()) {
+    for (auto &I : *BB) {
+      // Initial setting - Don't unroll loops containing vectorized
+      // instructions.
+      if (I.getType()->isVectorTy())
+        return;
+
+      if (isa<CallInst>(I) || isa<InvokeInst>(I)) {
+        if (const Function *F = cast<CallBase>(I).getCalledFunction()) {
+          if (!isLoweredToCall(F))
+            continue;
+        }
+        return;
+      }
+
+      SmallVector<const Value *> Operands(I.operand_values());
+      Cost +=
+          getUserCost(&I, Operands, TargetTransformInfo::TCK_SizeAndLatency);
+    }
+  }
+
+  LLVM_DEBUG(dbgs() << "Cost of loop: " << Cost << "\n");
+
+  UP.Partial = true;
+  UP.Runtime = true;
+  UP.UnrollRemainder = true;
+  UP.UnrollAndJam = true;
+  UP.UnrollAndJamInnerLoopThreshold = 60;
+
+  // Force unrolling small loops can be very useful because of the branch
+  // taken cost of the backedge.
+  if (Cost < 12)
+    UP.Force = true;
+}
+
+void RISCVTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
+                                         TTI::PeelingPreferences &PP) {
+  BaseT::getPeelingPreferences(L, SE, PP);
+}
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index 675681616d6e..7353496f4684 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -73,6 +73,13 @@ public:
     llvm_unreachable("Unsupported register kind");
   }
 
+  void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
+                               TTI::UnrollingPreferences &UP,
+                               OptimizationRemarkEmitter *ORE);
+
+  void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
+                             TTI::PeelingPreferences &PP);
+
   unsigned getMinVectorRegisterBitWidth() const {
     return ST->hasVInstructions() ? ST->getMinRVVVectorSizeInBits() : 0;
   }
@@ -178,7 +185,9 @@ public:
   }
 
   unsigned getMaxInterleaveFactor(unsigned VF) {
-    return ST->getMaxInterleaveFactor();
+    // If the loop will not be vectorized, don't interleave the loop.
+    // Let regular unroll to unroll the loop.
+    return VF == 1 ? 1 : ST->getMaxInterleaveFactor();
   }
 };
 
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp
index 0f5e0b9672a9..538380263c3c 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp
@@ -28,25 +28,43 @@ static uint64_t extractBitsForFixup(MCFixupKind Kind, uint64_t Value,
   if (Kind < FirstTargetFixupKind)
     return Value;
 
+  auto checkFixupInRange = [&](int64_t Min, int64_t Max) -> bool {
+    int64_t SVal = int64_t(Value);
+    if (SVal < Min || SVal > Max) {
+      Ctx.reportError(Fixup.getLoc(), "operand out of range (" + Twine(SVal) +
+                                          " not between " + Twine(Min) +
+                                          " and " + Twine(Max) + ")");
+      return false;
+    }
+    return true;
+  };
+
+  auto handlePCRelFixupValue = [&](unsigned W) -> uint64_t {
+    if (Value % 2 != 0)
+      Ctx.reportError(Fixup.getLoc(), "Non-even PC relative offset.");
+    if (!checkFixupInRange(minIntN(W) * 2, maxIntN(W) * 2))
+      return 0;
+    return (int64_t)Value / 2;
+  };
+
   switch (unsigned(Kind)) {
   case SystemZ::FK_390_PC12DBL:
+    return handlePCRelFixupValue(12);
   case SystemZ::FK_390_PC16DBL:
+    return handlePCRelFixupValue(16);
   case SystemZ::FK_390_PC24DBL:
+    return handlePCRelFixupValue(24);
   case SystemZ::FK_390_PC32DBL:
-    return (int64_t)Value / 2;
+    return handlePCRelFixupValue(32);
 
   case SystemZ::FK_390_12:
-    if (!isUInt<12>(Value)) {
-      Ctx.reportError(Fixup.getLoc(), "displacement exceeds uint12");
+    if (!checkFixupInRange(0, maxUIntN(12)))
       return 0;
-    }
     return Value;
 
   case SystemZ::FK_390_20: {
-    if (!isInt<20>(Value)) {
-      Ctx.reportError(Fixup.getLoc(), "displacement exceeds int20");
+    if (!checkFixupInRange(minIntN(20), maxIntN(20)))
       return 0;
-    }
     // The high byte of a 20 bit displacement value comes first.
     uint64_t DLo = Value & 0xfff;
     uint64_t DHi = (Value >> 12) & 0xff;
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCCodeEmitter.cpp b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCCodeEmitter.cpp
index e280e4aaf3d8..c83796b8579b 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCCodeEmitter.cpp
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCCodeEmitter.cpp
@@ -197,7 +197,8 @@ getDispOpValue(const MCInst &MI, unsigned OpNum,
     // All instructions follow the pattern where the first displacement has a
     // 2 bytes offset, and the second one 4 bytes.
     unsigned ByteOffs = Fixups.size() == 0 ? 2 : 4;
-    Fixups.push_back(MCFixup::create(ByteOffs, MO.getExpr(), (MCFixupKind)Kind));
+    Fixups.push_back(MCFixup::create(ByteOffs, MO.getExpr(), (MCFixupKind)Kind,
+                                     MI.getLoc()));
     assert(Fixups.size() <= 2 && "More than two memory operands in MI?");
     return 0;
   }
@@ -296,6 +297,7 @@ SystemZMCCodeEmitter::getPCRelEncoding(const MCInst &MI, unsigned OpNum,
                                        SmallVectorImpl<MCFixup> &Fixups,
                                        unsigned Kind, int64_t Offset,
                                        bool AllowTLS) const {
+  SMLoc Loc = MI.getLoc();
   const MCOperand &MO = MI.getOperand(OpNum);
   const MCExpr *Expr;
   if (MO.isImm())
@@ -311,13 +313,13 @@ SystemZMCCodeEmitter::getPCRelEncoding(const MCInst &MI, unsigned OpNum,
       Expr = MCBinaryExpr::createAdd(Expr, OffsetExpr, Ctx);
     }
   }
-  Fixups.push_back(MCFixup::create(Offset, Expr, (MCFixupKind)Kind));
+  Fixups.push_back(MCFixup::create(Offset, Expr, (MCFixupKind)Kind, Loc));
 
   // Output the fixup for the TLS marker if present.
   if (AllowTLS && OpNum + 1 < MI.getNumOperands()) {
     const MCOperand &MOTLS = MI.getOperand(OpNum + 1);
-    Fixups.push_back(MCFixup::create(0, MOTLS.getExpr(),
-                                     (MCFixupKind)SystemZ::FK_390_TLS_CALL));
+    Fixups.push_back(MCFixup::create(
+        0, MOTLS.getExpr(), (MCFixupKind)SystemZ::FK_390_TLS_CALL, Loc));
   }
   return 0;
 }
diff --git a/llvm/lib/Target/SystemZ/SystemZCallingConv.td b/llvm/lib/Target/SystemZ/SystemZCallingConv.td
index 373023effb4a..a7ea5e1e4bf8 100644
--- a/llvm/lib/Target/SystemZ/SystemZCallingConv.td
+++ b/llvm/lib/Target/SystemZ/SystemZCallingConv.td
@@ -166,6 +166,7 @@ def CSR_SystemZ_NoRegs : CalleeSavedRegs<(add)>;
 // any non-leaf function and restored in the epilogue for use by the
 // return instruction so it functions exactly like a callee-saved register.
 def CSR_SystemZ_XPLINK64 : CalleeSavedRegs<(add (sequence "R%dD", 7, 15),
+                                                (sequence "R%dD", 4, 4),
                                                 (sequence "F%dD", 15, 8))>;
 
 def CSR_SystemZ_XPLINK64_Vector : CalleeSavedRegs<(add CSR_SystemZ_XPLINK64,
diff --git a/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp b/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp
index 2f7cdfcf7bde..99ab4c5455d6 100644
--- a/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp
@@ -818,7 +818,7 @@ bool SystemZELFFrameLowering::usePackedStack(MachineFunction &MF) const {
 }
 
 SystemZXPLINKFrameLowering::SystemZXPLINKFrameLowering()
-    : SystemZFrameLowering(TargetFrameLowering::StackGrowsUp, Align(32), 128,
+    : SystemZFrameLowering(TargetFrameLowering::StackGrowsDown, Align(32), 0,
                            Align(32), /* StackRealignable */ false),
       RegSpillOffsets(-1) {
 
@@ -990,12 +990,184 @@ bool SystemZXPLINKFrameLowering::spillCalleeSavedRegisters(
   return true;
 }
 
+bool SystemZXPLINKFrameLowering::restoreCalleeSavedRegisters(
+    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
+    MutableArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
+
+  if (CSI.empty())
+    return false;
+
+  MachineFunction &MF = *MBB.getParent();
+  SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>();
+  const SystemZSubtarget &Subtarget = MF.getSubtarget<SystemZSubtarget>();
+  const TargetInstrInfo *TII = Subtarget.getInstrInfo();
+  auto &Regs = Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>();
+
+  DebugLoc DL = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc();
+
+  // Restore FPRs in the normal TargetInstrInfo way.
+  for (unsigned I = 0, E = CSI.size(); I != E; ++I) {
+    unsigned Reg = CSI[I].getReg();
+    if (SystemZ::FP64BitRegClass.contains(Reg))
+      TII->loadRegFromStackSlot(MBB, MBBI, Reg, CSI[I].getFrameIdx(),
+                                &SystemZ::FP64BitRegClass, TRI);
+    if (SystemZ::VR128BitRegClass.contains(Reg))
+      TII->loadRegFromStackSlot(MBB, MBBI, Reg, CSI[I].getFrameIdx(),
+                                &SystemZ::VR128BitRegClass, TRI);
+  }
+
+  // Restore call-saved GPRs (but not call-clobbered varargs, which at
+  // this point might hold return values).
+  SystemZ::GPRRegs RestoreGPRs = ZFI->getRestoreGPRRegs();
+  if (RestoreGPRs.LowGPR) {
+    assert(isInt<20>(Regs.getStackPointerBias() + RestoreGPRs.GPROffset));
+    if (RestoreGPRs.LowGPR == RestoreGPRs.HighGPR)
+      // Build an LG/L instruction.
+      BuildMI(MBB, MBBI, DL, TII->get(SystemZ::LG), RestoreGPRs.LowGPR)
+          .addReg(Regs.getStackPointerRegister())
+          .addImm(Regs.getStackPointerBias() + RestoreGPRs.GPROffset)
+          .addReg(0);
+    else {
+      // Build an LMG/LM instruction.
+      MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(SystemZ::LMG));
+
+      // Add the explicit register operands.
+      MIB.addReg(RestoreGPRs.LowGPR, RegState::Define);
+      MIB.addReg(RestoreGPRs.HighGPR, RegState::Define);
+
+      // Add the address.
+      MIB.addReg(Regs.getStackPointerRegister());
+      MIB.addImm(Regs.getStackPointerBias() + RestoreGPRs.GPROffset);
+
+      // Do a second scan adding regs as being defined by instruction
+      for (unsigned I = 0, E = CSI.size(); I != E; ++I) {
+        unsigned Reg = CSI[I].getReg();
+        if (Reg > RestoreGPRs.LowGPR && Reg < RestoreGPRs.HighGPR)
+          MIB.addReg(Reg, RegState::ImplicitDefine);
+      }
+    }
+  }
+
+  return true;
+}
+
 void SystemZXPLINKFrameLowering::emitPrologue(MachineFunction &MF,
-                                              MachineBasicBlock &MBB) const {}
+                                              MachineBasicBlock &MBB) const {
+  assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported");
+  const SystemZSubtarget &Subtarget = MF.getSubtarget<SystemZSubtarget>();
+  SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>();
+  MachineBasicBlock::iterator MBBI = MBB.begin();
+  auto *ZII = static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo());
+  auto &Regs = Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>();
+  MachineFrameInfo &MFFrame = MF.getFrameInfo();
+  MachineInstr *StoreInstr = nullptr;
+  bool HasFP = hasFP(MF);
+  // Debug location must be unknown since the first debug location is used
+  // to determine the end of the prologue.
+  DebugLoc DL;
+  uint64_t Offset = 0;
+
+  // TODO: Support leaf functions; only add size of save+reserved area when
+  // function is non-leaf.
+  MFFrame.setStackSize(MFFrame.getStackSize() + Regs.getCallFrameSize());
+  uint64_t StackSize = MFFrame.getStackSize();
+
+  // FIXME: Implement support for large stack sizes, when the stack extension
+  // routine needs to be called.
+  if (StackSize > 1024 * 1024) {
+    llvm_unreachable("Huge Stack Frame not yet supported on z/OS");
+  }
+
+  if (ZFI->getSpillGPRRegs().LowGPR) {
+    // Skip over the GPR saves.
+    if ((MBBI != MBB.end()) && ((MBBI->getOpcode() == SystemZ::STMG))) {
+      const int Operand = 3;
+      // Now we can set the offset for the operation, since now the Stack
+      // has been finalized.
+      Offset = Regs.getStackPointerBias() + MBBI->getOperand(Operand).getImm();
+      // Maximum displacement for STMG instruction.
+      if (isInt<20>(Offset - StackSize))
+        Offset -= StackSize;
+      else
+        StoreInstr = &*MBBI;
+      MBBI->getOperand(Operand).setImm(Offset);
+      ++MBBI;
+    } else
+      llvm_unreachable("Couldn't skip over GPR saves");
+  }
+
+  if (StackSize) {
+    MachineBasicBlock::iterator InsertPt = StoreInstr ? StoreInstr : MBBI;
+    // Allocate StackSize bytes.
+    int64_t Delta = -int64_t(StackSize);
+
+    // In case the STM(G) instruction also stores SP (R4), but the displacement
+    // is too large, the SP register is manipulated first before storing,
+    // resulting in the wrong value stored and retrieved later. In this case, we
+    // need to temporarily save the value of SP, and store it later to memory.
+    if (StoreInstr && HasFP) {
+      // Insert LR r0,r4 before STMG instruction.
+      BuildMI(MBB, InsertPt, DL, ZII->get(SystemZ::LGR))
+          .addReg(SystemZ::R0D, RegState::Define)
+          .addReg(SystemZ::R4D);
+      // Insert ST r0,xxx(,r4) after STMG instruction.
+      BuildMI(MBB, MBBI, DL, ZII->get(SystemZ::STG))
+          .addReg(SystemZ::R0D, RegState::Kill)
+          .addReg(SystemZ::R4D)
+          .addImm(Offset)
+          .addReg(0);
+    }
+
+    emitIncrement(MBB, InsertPt, DL, Regs.getStackPointerRegister(), Delta,
+                  ZII);
+  }
+
+  if (HasFP) {
+    // Copy the base of the frame to Frame Pointer Register.
+    BuildMI(MBB, MBBI, DL, ZII->get(SystemZ::LGR),
+            Regs.getFramePointerRegister())
+        .addReg(Regs.getStackPointerRegister());
+
+    // Mark the FramePtr as live at the beginning of every block except
+    // the entry block.  (We'll have marked R8 as live on entry when
+    // saving the GPRs.)
+    for (auto I = std::next(MF.begin()), E = MF.end(); I != E; ++I)
+      I->addLiveIn(Regs.getFramePointerRegister());
+  }
+}
 
 void SystemZXPLINKFrameLowering::emitEpilogue(MachineFunction &MF,
-                                              MachineBasicBlock &MBB) const {}
+                                              MachineBasicBlock &MBB) const {
+  const SystemZSubtarget &Subtarget = MF.getSubtarget<SystemZSubtarget>();
+  MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
+  SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>();
+  MachineFrameInfo &MFFrame = MF.getFrameInfo();
+  auto *ZII = static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo());
+  auto &Regs = Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>();
+
+  // Skip the return instruction.
+  assert(MBBI->isReturn() && "Can only insert epilogue into returning blocks");
+
+  uint64_t StackSize = MFFrame.getStackSize();
+  if (StackSize) {
+    unsigned SPReg = Regs.getStackPointerRegister();
+    if (ZFI->getRestoreGPRRegs().LowGPR != SPReg) {
+      DebugLoc DL = MBBI->getDebugLoc();
+      emitIncrement(MBB, MBBI, DL, SPReg, StackSize, ZII);
+    }
+  }
+}
 
 bool SystemZXPLINKFrameLowering::hasFP(const MachineFunction &MF) const {
-  return false;
+  return (MF.getFrameInfo().hasVarSizedObjects());
+}
+
+void SystemZXPLINKFrameLowering::processFunctionBeforeFrameFinalized(
+    MachineFunction &MF, RegScavenger *RS) const {
+  MachineFrameInfo &MFFrame = MF.getFrameInfo();
+  const SystemZSubtarget &Subtarget = MF.getSubtarget<SystemZSubtarget>();
+  auto &Regs = Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>();
+
+  // Setup stack frame offset
+  MFFrame.setOffsetAdjustment(Regs.getStackPointerBias());
 }
diff --git a/llvm/lib/Target/SystemZ/SystemZFrameLowering.h b/llvm/lib/Target/SystemZ/SystemZFrameLowering.h
index af219da79c32..106b9e8ebe06 100644
--- a/llvm/lib/Target/SystemZ/SystemZFrameLowering.h
+++ b/llvm/lib/Target/SystemZ/SystemZFrameLowering.h
@@ -115,11 +115,20 @@ public:
                                  ArrayRef<CalleeSavedInfo> CSI,
                                  const TargetRegisterInfo *TRI) const override;
 
+  bool
+  restoreCalleeSavedRegisters(MachineBasicBlock &MBB,
+                              MachineBasicBlock::iterator MBBII,
+                              MutableArrayRef<CalleeSavedInfo> CSI,
+                              const TargetRegisterInfo *TRI) const override;
+
   void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
 
   void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
 
   bool hasFP(const MachineFunction &MF) const override;
+
+  void processFunctionBeforeFrameFinalized(MachineFunction &MF,
+                                           RegScavenger *RS) const override;
 };
 } // end namespace llvm
 
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index 71432218068e..24de52850771 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -1500,8 +1500,16 @@ SDValue SystemZTargetLowering::LowerFormalArguments(
       assert(VA.isMemLoc() && "Argument not register or memory");
 
       // Create the frame index object for this incoming parameter.
-      int FI = MFI.CreateFixedObject(LocVT.getSizeInBits() / 8,
-                                     VA.getLocMemOffset(), true);
+      // FIXME: Pre-include call frame size in the offset, should not
+      // need to manually add it here.
+      int64_t ArgSPOffset = VA.getLocMemOffset();
+      if (Subtarget.isTargetXPLINK64()) {
+        auto &XPRegs =
+            Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>();
+        ArgSPOffset += XPRegs.getCallFrameSize();
+      }
+      int FI =
+          MFI.CreateFixedObject(LocVT.getSizeInBits() / 8, ArgSPOffset, true);
 
       // Create the SelectionDAG nodes corresponding to a load
       // from this parameter.  Unpromoted ints and floats are
@@ -5714,6 +5722,7 @@ const char *SystemZTargetLowering::getTargetNodeName(unsigned Opcode) const {
     OPCODE(OC);
     OPCODE(XC);
     OPCODE(CLC);
+    OPCODE(MEMSET_MVC);
     OPCODE(STPCPY);
     OPCODE(STRCMP);
     OPCODE(SEARCH_STRING);
@@ -7860,8 +7869,10 @@ MachineBasicBlock *SystemZTargetLowering::emitExt128(MachineInstr &MI,
   return MBB;
 }
 
-MachineBasicBlock *SystemZTargetLowering::emitMemMemWrapper(
-    MachineInstr &MI, MachineBasicBlock *MBB, unsigned Opcode) const {
+MachineBasicBlock *
+SystemZTargetLowering::emitMemMemWrapper(MachineInstr &MI,
+                                         MachineBasicBlock *MBB,
+                                         unsigned Opcode, bool IsMemset) const {
   MachineFunction &MF = *MBB->getParent();
   const SystemZInstrInfo *TII =
       static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo());
@@ -7870,18 +7881,64 @@ MachineBasicBlock *SystemZTargetLowering::emitMemMemWrapper(
 
   MachineOperand DestBase = earlyUseOperand(MI.getOperand(0));
   uint64_t DestDisp = MI.getOperand(1).getImm();
-  MachineOperand SrcBase = earlyUseOperand(MI.getOperand(2));
-  uint64_t SrcDisp = MI.getOperand(3).getImm();
-  MachineOperand &LengthMO = MI.getOperand(4);
+  MachineOperand SrcBase = MachineOperand::CreateReg(0U, false);
+  uint64_t SrcDisp;
+
+  // Fold the displacement Disp if it is out of range.
+  auto foldDisplIfNeeded = [&](MachineOperand &Base, uint64_t &Disp) -> void {
+    if (!isUInt<12>(Disp)) {
+      Register Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
+      unsigned Opcode = TII->getOpcodeForOffset(SystemZ::LA, Disp);
+      BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), TII->get(Opcode), Reg)
+        .add(Base).addImm(Disp).addReg(0);
+      Base = MachineOperand::CreateReg(Reg, false);
+      Disp = 0;
+    }
+  };
+
+  if (!IsMemset) {
+    SrcBase = earlyUseOperand(MI.getOperand(2));
+    SrcDisp = MI.getOperand(3).getImm();
+  } else {
+    SrcBase = DestBase;
+    SrcDisp = DestDisp++;
+    foldDisplIfNeeded(DestBase, DestDisp);
+  }
+
+  MachineOperand &LengthMO = MI.getOperand(IsMemset ? 2 : 4);
   bool IsImmForm = LengthMO.isImm();
   bool IsRegForm = !IsImmForm;
 
+  // Build and insert one Opcode of Length, with special treatment for memset.
+  auto insertMemMemOp = [&](MachineBasicBlock *InsMBB,
+                            MachineBasicBlock::iterator InsPos,
+                            MachineOperand DBase, uint64_t DDisp,
+                            MachineOperand SBase, uint64_t SDisp,
+                            unsigned Length) -> void {
+    assert(Length > 0 && Length <= 256 && "Building memory op with bad length.");
+    if (IsMemset) {
+      MachineOperand ByteMO = earlyUseOperand(MI.getOperand(3));
+      if (ByteMO.isImm())
+        BuildMI(*InsMBB, InsPos, DL, TII->get(SystemZ::MVI))
+          .add(SBase).addImm(SDisp).add(ByteMO);
+      else
+        BuildMI(*InsMBB, InsPos, DL, TII->get(SystemZ::STC))
+          .add(ByteMO).add(SBase).addImm(SDisp).addReg(0);
+      if (--Length == 0)
+        return;
+    }
+    BuildMI(*MBB, InsPos, DL, TII->get(Opcode))
+      .add(DBase).addImm(DDisp).addImm(Length)
+      .add(SBase).addImm(SDisp)
+      .setMemRefs(MI.memoperands());
+  };
+
   bool NeedsLoop = false;
   uint64_t ImmLength = 0;
-  Register LenMinus1Reg = SystemZ::NoRegister;
+  Register LenAdjReg = SystemZ::NoRegister;
   if (IsImmForm) {
     ImmLength = LengthMO.getImm();
-    ImmLength++; // Add back the '1' subtracted originally.
+    ImmLength += IsMemset ? 2 : 1; // Add back the subtracted adjustment.
     if (ImmLength == 0) {
       MI.eraseFromParent();
       return MBB;
@@ -7905,7 +7962,7 @@ MachineBasicBlock *SystemZTargetLowering::emitMemMemWrapper(
       NeedsLoop = true;
   } else {
     NeedsLoop = true;
-    LenMinus1Reg = LengthMO.getReg();
+    LenAdjReg = LengthMO.getReg();
   }
 
   // When generating more than one CLC, all but the last will need to
@@ -7923,17 +7980,17 @@ MachineBasicBlock *SystemZTargetLowering::emitMemMemWrapper(
       ImmLength &= 255;
     } else {
       BuildMI(*MBB, MI, DL, TII->get(SystemZ::SRLG), StartCountReg)
-        .addReg(LenMinus1Reg)
+        .addReg(LenAdjReg)
         .addReg(0)
         .addImm(8);
     }
 
+    bool HaveSingleBase = DestBase.isIdenticalTo(SrcBase);
     auto loadZeroAddress = [&]() -> MachineOperand {
       Register Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
       BuildMI(*MBB, MI, DL, TII->get(SystemZ::LGHI), Reg).addImm(0);
       return MachineOperand::CreateReg(Reg, false);
     };
-    bool HaveSingleBase = DestBase.isIdenticalTo(SrcBase);
     if (DestBase.isReg() && DestBase.getReg() == SystemZ::NoRegister)
       DestBase = loadZeroAddress();
     if (SrcBase.isReg() && SrcBase.getReg() == SystemZ::NoRegister)
@@ -7968,14 +8025,41 @@ MachineBasicBlock *SystemZTargetLowering::emitMemMemWrapper(
       DoneMBB = SystemZ::emitBlockAfter(NextMBB);
 
       //  MBB:
-      //   # Jump to AllDoneMBB if LenMinus1Reg is -1, or fall thru to StartMBB.
+      //   # Jump to AllDoneMBB if LenAdjReg means 0, or fall thru to StartMBB.
       BuildMI(MBB, DL, TII->get(SystemZ::CGHI))
-        .addReg(LenMinus1Reg).addImm(-1);
+        .addReg(LenAdjReg).addImm(IsMemset ? -2 : -1);
       BuildMI(MBB, DL, TII->get(SystemZ::BRC))
         .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_EQ)
         .addMBB(AllDoneMBB);
       MBB->addSuccessor(AllDoneMBB);
-      MBB->addSuccessor(StartMBB);
+      if (!IsMemset)
+        MBB->addSuccessor(StartMBB);
+      else {
+        // MemsetOneCheckMBB:
+        // # Jump to MemsetOneMBB for a memset of length 1, or
+        // # fall thru to StartMBB.
+        MachineBasicBlock *MemsetOneCheckMBB = SystemZ::emitBlockAfter(MBB);
+        MachineBasicBlock *MemsetOneMBB = SystemZ::emitBlockAfter(&*MF.rbegin());
+        MBB->addSuccessor(MemsetOneCheckMBB);
+        MBB = MemsetOneCheckMBB;
+        BuildMI(MBB, DL, TII->get(SystemZ::CGHI))
+          .addReg(LenAdjReg).addImm(-1);
+        BuildMI(MBB, DL, TII->get(SystemZ::BRC))
+          .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_EQ)
+          .addMBB(MemsetOneMBB);
+        MBB->addSuccessor(MemsetOneMBB, {10, 100});
+        MBB->addSuccessor(StartMBB, {90, 100});
+
+        // MemsetOneMBB:
+        // # Jump back to AllDoneMBB after a single MVI or STC.
+        MBB = MemsetOneMBB;
+        insertMemMemOp(MBB, MBB->end(),
+                       MachineOperand::CreateReg(StartDestReg, false), DestDisp,
+                       MachineOperand::CreateReg(StartSrcReg, false), SrcDisp,
+                       1);
+        BuildMI(MBB, DL, TII->get(SystemZ::J)).addMBB(AllDoneMBB);
+        MBB->addSuccessor(AllDoneMBB);
+      }
 
       // StartMBB:
       // # Jump to DoneMBB if %StartCountReg is zero, or fall through to LoopMBB.
@@ -8032,10 +8116,10 @@ MachineBasicBlock *SystemZTargetLowering::emitMemMemWrapper(
     if (Opcode == SystemZ::MVC)
       BuildMI(MBB, DL, TII->get(SystemZ::PFD))
         .addImm(SystemZ::PFD_WRITE)
-        .addReg(ThisDestReg).addImm(DestDisp + 768).addReg(0);
-    BuildMI(MBB, DL, TII->get(Opcode))
-      .addReg(ThisDestReg).addImm(DestDisp).addImm(256)
-      .addReg(ThisSrcReg).addImm(SrcDisp);
+        .addReg(ThisDestReg).addImm(DestDisp - IsMemset + 768).addReg(0);
+    insertMemMemOp(MBB, MBB->end(),
+                   MachineOperand::CreateReg(ThisDestReg, false), DestDisp,
+                   MachineOperand::CreateReg(ThisSrcReg, false), SrcDisp, 256);
     if (EndMBB) {
       BuildMI(MBB, DL, TII->get(SystemZ::BRC))
         .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_NE)
@@ -8075,7 +8159,7 @@ MachineBasicBlock *SystemZTargetLowering::emitMemMemWrapper(
       // # Make PHIs for RemDestReg/RemSrcReg as the loop may or may not run.
       // # Use EXecute Relative Long for the remainder of the bytes. The target
       //   instruction of the EXRL will have a length field of 1 since 0 is an
-      //   illegal value. The number of bytes processed becomes (%LenMinus1Reg &
+      //   illegal value. The number of bytes processed becomes (%LenAdjReg &
       //   0xff) + 1.
       // # Fall through to AllDoneMBB.
       Register RemSrcReg  = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
@@ -8088,10 +8172,14 @@ MachineBasicBlock *SystemZTargetLowering::emitMemMemWrapper(
         BuildMI(MBB, DL, TII->get(SystemZ::PHI), RemSrcReg)
           .addReg(StartSrcReg).addMBB(StartMBB)
           .addReg(NextSrcReg).addMBB(NextMBB);
+      if (IsMemset)
+        insertMemMemOp(MBB, MBB->end(),
+                       MachineOperand::CreateReg(RemDestReg, false), DestDisp,
+                       MachineOperand::CreateReg(RemSrcReg, false), SrcDisp, 1);
       MachineInstrBuilder EXRL_MIB =
         BuildMI(MBB, DL, TII->get(SystemZ::EXRL_Pseudo))
           .addImm(Opcode)
-          .addReg(LenMinus1Reg)
+          .addReg(LenAdjReg)
           .addReg(RemDestReg).addImm(DestDisp)
           .addReg(RemSrcReg).addImm(SrcDisp);
       MBB->addSuccessor(AllDoneMBB);
@@ -8107,32 +8195,10 @@ MachineBasicBlock *SystemZTargetLowering::emitMemMemWrapper(
   while (ImmLength > 0) {
     uint64_t ThisLength = std::min(ImmLength, uint64_t(256));
     // The previous iteration might have created out-of-range displacements.
-    // Apply them using LAY if so.
-    if (!isUInt<12>(DestDisp)) {
-      Register Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
-      BuildMI(*MBB, MI, MI.getDebugLoc(), TII->get(SystemZ::LAY), Reg)
-          .add(DestBase)
-          .addImm(DestDisp)
-          .addReg(0);
-      DestBase = MachineOperand::CreateReg(Reg, false);
-      DestDisp = 0;
-    }
-    if (!isUInt<12>(SrcDisp)) {
-      Register Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
-      BuildMI(*MBB, MI, MI.getDebugLoc(), TII->get(SystemZ::LAY), Reg)
-          .add(SrcBase)
-          .addImm(SrcDisp)
-          .addReg(0);
-      SrcBase = MachineOperand::CreateReg(Reg, false);
-      SrcDisp = 0;
-    }
-    BuildMI(*MBB, MI, DL, TII->get(Opcode))
-        .add(DestBase)
-        .addImm(DestDisp)
-        .addImm(ThisLength)
-        .add(SrcBase)
-        .addImm(SrcDisp)
-        .setMemRefs(MI.memoperands());
+    // Apply them using LA/LAY if so.
+    foldDisplIfNeeded(DestBase, DestDisp);
+    foldDisplIfNeeded(SrcBase, SrcDisp);
+    insertMemMemOp(MBB, MI, DestBase, DestDisp, SrcBase, SrcDisp, ThisLength);
     DestDisp += ThisLength;
     SrcDisp += ThisLength;
     ImmLength -= ThisLength;
@@ -8630,6 +8696,11 @@ MachineBasicBlock *SystemZTargetLowering::EmitInstrWithCustomInserter(
   case SystemZ::CLCImm:
   case SystemZ::CLCReg:
     return emitMemMemWrapper(MI, MBB, SystemZ::CLC);
+  case SystemZ::MemsetImmImm:
+  case SystemZ::MemsetImmReg:
+  case SystemZ::MemsetRegImm:
+  case SystemZ::MemsetRegReg:
+    return emitMemMemWrapper(MI, MBB, SystemZ::MVC, true/*IsMemset*/);
   case SystemZ::CLSTLoop:
     return emitStringWrapper(MI, MBB, SystemZ::CLST);
   case SystemZ::MVSTLoop:
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.h b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
index 461f804ca55e..940c0a857ea4 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.h
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
@@ -126,6 +126,9 @@ enum NodeType : unsigned {
   // as for MVC.
   CLC,
 
+  // Use MVC to set a block of memory after storing the first byte.
+  MEMSET_MVC,
+
   // Use an MVST-based sequence to implement stpcpy().
   STPCPY,
 
@@ -709,7 +712,8 @@ private:
   MachineBasicBlock *emitAtomicCmpSwapW(MachineInstr &MI,
                                         MachineBasicBlock *BB) const;
   MachineBasicBlock *emitMemMemWrapper(MachineInstr &MI, MachineBasicBlock *BB,
-                                       unsigned Opcode) const;
+                                       unsigned Opcode,
+                                       bool IsMemset = false) const;
   MachineBasicBlock *emitStringWrapper(MachineInstr &MI, MachineBasicBlock *BB,
                                        unsigned Opcode) const;
   MachineBasicBlock *emitTransactionBegin(MachineInstr &MI,
diff --git a/llvm/lib/Target/SystemZ/SystemZInstrFormats.td b/llvm/lib/Target/SystemZ/SystemZInstrFormats.td
index cd60fff1ab11..e513befd0d6f 100644
--- a/llvm/lib/Target/SystemZ/SystemZInstrFormats.td
+++ b/llvm/lib/Target/SystemZ/SystemZInstrFormats.td
@@ -5256,6 +5256,16 @@ class RotateSelectAliasRIEf<RegisterOperand cls1, RegisterOperand cls2>
   let Constraints = "$R1 = $R1src";
 }
 
+class MemsetPseudo<DAGOperand lenop, DAGOperand byteop>
+  : Pseudo<(outs), (ins bdaddr12only:$dest, lenop:$length, byteop:$B),
+           [(z_memset_mvc bdaddr12only:$dest, lenop:$length, byteop:$B)]> {
+  let Defs = [CC];
+  let mayLoad = 1;
+  let mayStore = 1;
+  let usesCustomInserter = 1;
+  let hasNoSchedulingInfo = 1;
+}
+
 //===----------------------------------------------------------------------===//
 // Multiclasses that emit both real and pseudo instructions
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/SystemZ/SystemZInstrInfo.td b/llvm/lib/Target/SystemZ/SystemZInstrInfo.td
index e4760229fd6b..84f1e0fb428c 100644
--- a/llvm/lib/Target/SystemZ/SystemZInstrInfo.td
+++ b/llvm/lib/Target/SystemZ/SystemZInstrInfo.td
@@ -510,6 +510,12 @@ let mayLoad = 1, mayStore = 1, Defs = [CC] in {
   def MVCLU : SideEffectTernaryMemMemRSY<"mvclu", 0xEB8E, GR128, GR128>;
 }
 
+// Memset[Length][Byte] pseudos.
+def MemsetImmImm : MemsetPseudo<imm64, imm32zx8trunc>;
+def MemsetImmReg : MemsetPseudo<imm64, GR32>;
+def MemsetRegImm : MemsetPseudo<ADDR64, imm32zx8trunc>;
+def MemsetRegReg : MemsetPseudo<ADDR64, GR32>;
+
 // Move right.
 let Predicates = [FeatureMiscellaneousExtensions3],
     mayLoad = 1, mayStore = 1, Uses = [R0L] in
diff --git a/llvm/lib/Target/SystemZ/SystemZOperators.td b/llvm/lib/Target/SystemZ/SystemZOperators.td
index 927d97233286..9935416559bc 100644
--- a/llvm/lib/Target/SystemZ/SystemZOperators.td
+++ b/llvm/lib/Target/SystemZ/SystemZOperators.td
@@ -102,6 +102,10 @@ def SDT_ZMemMemLengthCC     : SDTypeProfile<1, 3,
                                              SDTCisPtrTy<1>,
                                              SDTCisPtrTy<2>,
                                              SDTCisVT<3, i64>]>;
+def SDT_ZMemsetMVC          : SDTypeProfile<0, 3,
+                                            [SDTCisPtrTy<0>,
+                                             SDTCisVT<1, i64>,
+                                             SDTCisVT<2, i32>]>;
 def SDT_ZString             : SDTypeProfile<1, 3,
                                             [SDTCisPtrTy<0>,
                                              SDTCisPtrTy<1>,
@@ -413,6 +417,8 @@ def z_xc                : SDNode<"SystemZISD::XC", SDT_ZMemMemLength,
                                   [SDNPHasChain, SDNPMayStore, SDNPMayLoad]>;
 def z_clc               : SDNode<"SystemZISD::CLC", SDT_ZMemMemLengthCC,
                                  [SDNPHasChain, SDNPMayLoad]>;
+def z_memset_mvc        : SDNode<"SystemZISD::MEMSET_MVC", SDT_ZMemsetMVC,
+                                 [SDNPHasChain, SDNPMayStore, SDNPMayLoad]>;
 def z_strcmp            : SDNode<"SystemZISD::STRCMP", SDT_ZStringCC,
                                  [SDNPHasChain, SDNPMayLoad]>;
 def z_stpcpy            : SDNode<"SystemZISD::STPCPY", SDT_ZString,
diff --git a/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp b/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp
index f38e93109967..db4b4879b33a 100644
--- a/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp
@@ -17,29 +17,44 @@ using namespace llvm;
 
 #define DEBUG_TYPE "systemz-selectiondag-info"
 
-static SDVTList getMemMemVTs(unsigned Op, SelectionDAG &DAG) {
-  return Op == SystemZISD::CLC ? DAG.getVTList(MVT::i32, MVT::Other)
-                               : DAG.getVTList(MVT::Other);
+static unsigned getMemMemLenAdj(unsigned Op) {
+  return Op == SystemZISD::MEMSET_MVC ? 2 : 1;
 }
 
-// Emit a mem-mem operation after subtracting one from size, which will be
-// added back during pseudo expansion. As the Reg case emitted here may be
-// converted by DAGCombiner into having an Imm length, they are both emitted
-// the same way.
+static SDValue createMemMemNode(SelectionDAG &DAG, const SDLoc &DL, unsigned Op,
+                                SDValue Chain, SDValue Dst, SDValue Src,
+                                SDValue LenAdj, SDValue Byte) {
+  SDVTList VTs = Op == SystemZISD::CLC ? DAG.getVTList(MVT::i32, MVT::Other)
+                                       : DAG.getVTList(MVT::Other);
+  SmallVector<SDValue, 6> Ops;
+  if (Op == SystemZISD::MEMSET_MVC)
+    Ops = { Chain, Dst, LenAdj, Byte };
+  else
+    Ops = { Chain, Dst, Src, LenAdj };
+  return DAG.getNode(Op, DL, VTs, Ops);
+}
+
+// Emit a mem-mem operation after subtracting one (or two for memset) from
+// size, which will be added back during pseudo expansion. As the Reg case
+// emitted here may be converted by DAGCombiner into having an Imm length,
+// they are both emitted the same way.
 static SDValue emitMemMemImm(SelectionDAG &DAG, const SDLoc &DL, unsigned Op,
                              SDValue Chain, SDValue Dst, SDValue Src,
-                             uint64_t Size) {
-  return DAG.getNode(Op, DL, getMemMemVTs(Op, DAG), Chain, Dst, Src,
-                     DAG.getConstant(Size - 1, DL, Src.getValueType()));
+                             uint64_t Size, SDValue Byte = SDValue()) {
+  unsigned Adj = getMemMemLenAdj(Op);
+  assert(Size >= Adj && "Adjusted length overflow.");
+  SDValue LenAdj = DAG.getConstant(Size - Adj, DL, Dst.getValueType());
+  return createMemMemNode(DAG, DL, Op, Chain, Dst, Src, LenAdj, Byte);
 }
 
 static SDValue emitMemMemReg(SelectionDAG &DAG, const SDLoc &DL, unsigned Op,
                              SDValue Chain, SDValue Dst, SDValue Src,
-                             SDValue Size) {
-  SDValue LenMinus1 = DAG.getNode(ISD::ADD, DL, MVT::i64,
-                                  DAG.getZExtOrTrunc(Size, DL, MVT::i64),
-                                  DAG.getConstant(-1, DL, MVT::i64));
-  return DAG.getNode(Op, DL, getMemMemVTs(Op, DAG), Chain, Dst, Src, LenMinus1);
+                             SDValue Size, SDValue Byte = SDValue()) {
+  int64_t Adj = getMemMemLenAdj(Op);
+  SDValue LenAdj = DAG.getNode(ISD::ADD, DL, MVT::i64,
+                               DAG.getZExtOrTrunc(Size, DL, MVT::i64),
+                               DAG.getConstant(0 - Adj, DL, MVT::i64));
+  return createMemMemNode(DAG, DL, Op, Chain, Dst, Src, LenAdj, Byte);
 }
 
 SDValue SystemZSelectionDAGInfo::EmitTargetCodeForMemcpy(
@@ -127,13 +142,8 @@ SDValue SystemZSelectionDAGInfo::EmitTargetCodeForMemset(
     if (CByte && CByte->getZExtValue() == 0)
       return emitMemMemImm(DAG, DL, SystemZISD::XC, Chain, Dst, Dst, Bytes);
 
-    // Copy the byte to the first location and then use MVC to copy
-    // it to the rest.
-    Chain = DAG.getStore(Chain, DL, Byte, Dst, DstPtrInfo, Alignment);
-    SDValue DstPlus1 = DAG.getNode(ISD::ADD, DL, PtrVT, Dst,
-                                   DAG.getConstant(1, DL, PtrVT));
-    return emitMemMemImm(DAG, DL, SystemZISD::MVC, Chain, DstPlus1, Dst,
-                         Bytes - 1);
+    return emitMemMemImm(DAG, DL, SystemZISD::MEMSET_MVC, Chain, Dst, SDValue(),
+                         Bytes, DAG.getAnyExtOrTrunc(Byte, DL, MVT::i32));
   }
 
   // Variable length
@@ -141,7 +151,8 @@ SDValue SystemZSelectionDAGInfo::EmitTargetCodeForMemset(
     // Handle the special case of a variable length memset of 0 with XC.
     return emitMemMemReg(DAG, DL, SystemZISD::XC, Chain, Dst, Dst, Size);
 
-  return SDValue();
+  return emitMemMemReg(DAG, DL, SystemZISD::MEMSET_MVC, Chain, Dst, SDValue(),
+                       Size, DAG.getAnyExtOrTrunc(Byte, DL, MVT::i32));
 }
 
 // Convert the current CC value into an integer that is 0 if CC == 0,
diff --git a/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp b/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp
index 7e92e4b33812..fd9dc32b04f5 100644
--- a/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp
+++ b/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp
@@ -84,6 +84,8 @@ class VEAsmParser : public MCTargetAsmParser {
   StringRef splitMnemonic(StringRef Name, SMLoc NameLoc,
                           OperandVector *Operands);
 
+  bool parseLiteralValues(unsigned Size, SMLoc L);
+
 public:
   VEAsmParser(const MCSubtargetInfo &sti, MCAsmParser &parser,
               const MCInstrInfo &MII, const MCTargetOptions &Options)
@@ -994,10 +996,43 @@ bool VEAsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
 }
 
 bool VEAsmParser::ParseDirective(AsmToken DirectiveID) {
+  std::string IDVal = DirectiveID.getIdentifier().lower();
+
+  // Defines VE specific directives.  Reference is "Vector Engine Assembly
+  // Language Reference Manual":
+  // https://www.hpc.nec/documents/sdk/pdfs/VectorEngine-as-manual-v1.3.pdf
+
+  // The .word is 4 bytes long on VE.
+  if (IDVal == ".word")
+    return parseLiteralValues(4, DirectiveID.getLoc());
+
+  // The .long is 8 bytes long on VE.
+  if (IDVal == ".long")
+    return parseLiteralValues(8, DirectiveID.getLoc());
+
+  // The .llong is 8 bytes long on VE.
+  if (IDVal == ".llong")
+    return parseLiteralValues(8, DirectiveID.getLoc());
+
   // Let the MC layer to handle other directives.
   return true;
 }
 
+/// parseLiteralValues
+///  ::= .word expression [, expression]*
+///  ::= .long expression [, expression]*
+///  ::= .llong expression [, expression]*
+bool VEAsmParser::parseLiteralValues(unsigned Size, SMLoc L) {
+  auto parseOne = [&]() -> bool {
+    const MCExpr *Value;
+    if (getParser().parseExpression(Value))
+      return true;
+    getParser().getStreamer().emitValue(Value, Size, L);
+    return false;
+  };
+  return (parseMany(parseOne));
+}
+
 /// Extract \code @lo32/@hi32/etc \endcode modifier from expression.
 /// Recursively scan the expression and check for VK_VE_HI32/LO32/etc
 /// symbol variants.  If all symbols with modifier use the same
diff --git a/llvm/lib/Target/VE/MCTargetDesc/VEAsmBackend.cpp b/llvm/lib/Target/VE/MCTargetDesc/VEAsmBackend.cpp
index 29c209934680..38d163b37080 100644
--- a/llvm/lib/Target/VE/MCTargetDesc/VEAsmBackend.cpp
+++ b/llvm/lib/Target/VE/MCTargetDesc/VEAsmBackend.cpp
@@ -42,6 +42,7 @@ static uint64_t adjustFixupValue(unsigned Kind, uint64_t Value) {
   case VE::fixup_ve_tpoff_hi32:
     return (Value >> 32) & 0xffffffff;
   case VE::fixup_ve_reflong:
+  case VE::fixup_ve_srel32:
   case VE::fixup_ve_lo32:
   case VE::fixup_ve_pc_lo32:
   case VE::fixup_ve_got_lo32:
@@ -68,6 +69,7 @@ static unsigned getFixupKindNumBytes(unsigned Kind) {
   case FK_Data_4:
   case FK_PCRel_4:
   case VE::fixup_ve_reflong:
+  case VE::fixup_ve_srel32:
   case VE::fixup_ve_hi32:
   case VE::fixup_ve_lo32:
   case VE::fixup_ve_pc_hi32:
@@ -103,6 +105,7 @@ public:
     const static MCFixupKindInfo Infos[VE::NumTargetFixupKinds] = {
         // name, offset, bits, flags
         {"fixup_ve_reflong", 0, 32, 0},
+        {"fixup_ve_srel32", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
         {"fixup_ve_hi32", 0, 32, 0},
         {"fixup_ve_lo32", 0, 32, 0},
         {"fixup_ve_pc_hi32", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
diff --git a/llvm/lib/Target/VE/MCTargetDesc/VEELFObjectWriter.cpp b/llvm/lib/Target/VE/MCTargetDesc/VEELFObjectWriter.cpp
index 741e8320a941..ae065407409a 100644
--- a/llvm/lib/Target/VE/MCTargetDesc/VEELFObjectWriter.cpp
+++ b/llvm/lib/Target/VE/MCTargetDesc/VEELFObjectWriter.cpp
@@ -9,6 +9,7 @@
 #include "VEFixupKinds.h"
 #include "VEMCExpr.h"
 #include "VEMCTargetDesc.h"
+#include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCELFObjectWriter.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCObjectWriter.h"
@@ -46,16 +47,29 @@ unsigned VEELFObjectWriter::getRelocType(MCContext &Ctx, const MCValue &Target,
   if (IsPCRel) {
     switch (Fixup.getTargetKind()) {
     default:
-      llvm_unreachable("Unimplemented fixup -> relocation");
+      Ctx.reportError(Fixup.getLoc(), "Unsupported pc-relative fixup kind");
+      return ELF::R_VE_NONE;
+    case FK_Data_1:
     case FK_PCRel_1:
-      llvm_unreachable("Unimplemented fixup fk_data_1 -> relocation");
+      Ctx.reportError(Fixup.getLoc(),
+                      "1-byte pc-relative data relocation is not supported");
+      return ELF::R_VE_NONE;
+    case FK_Data_2:
     case FK_PCRel_2:
-      llvm_unreachable("Unimplemented fixup fk_data_2 -> relocation");
-    // FIXME: relative kind?
+      Ctx.reportError(Fixup.getLoc(),
+                      "2-byte pc-relative data relocation is not supported");
+      return ELF::R_VE_NONE;
+    case FK_Data_4:
     case FK_PCRel_4:
-      return ELF::R_VE_REFLONG;
+      return ELF::R_VE_SREL32;
+    case FK_Data_8:
     case FK_PCRel_8:
-      return ELF::R_VE_REFQUAD;
+      Ctx.reportError(Fixup.getLoc(),
+                      "8-byte pc-relative data relocation is not supported");
+      return ELF::R_VE_NONE;
+    case VE::fixup_ve_reflong:
+    case VE::fixup_ve_srel32:
+      return ELF::R_VE_SREL32;
     case VE::fixup_ve_pc_hi32:
       return ELF::R_VE_PC_HI32;
     case VE::fixup_ve_pc_lo32:
@@ -65,25 +79,36 @@ unsigned VEELFObjectWriter::getRelocType(MCContext &Ctx, const MCValue &Target,
 
   switch (Fixup.getTargetKind()) {
   default:
-    llvm_unreachable("Unimplemented fixup -> relocation");
+    Ctx.reportError(Fixup.getLoc(), "Unknown ELF relocation type");
+    return ELF::R_VE_NONE;
   case FK_Data_1:
-    llvm_unreachable("Unimplemented fixup fk_data_1 -> relocation");
+    Ctx.reportError(Fixup.getLoc(), "1-byte data relocation is not supported");
+    return ELF::R_VE_NONE;
   case FK_Data_2:
-    llvm_unreachable("Unimplemented fixup fk_data_2 -> relocation");
+    Ctx.reportError(Fixup.getLoc(), "2-byte data relocation is not supported");
+    return ELF::R_VE_NONE;
   case FK_Data_4:
     return ELF::R_VE_REFLONG;
   case FK_Data_8:
     return ELF::R_VE_REFQUAD;
   case VE::fixup_ve_reflong:
     return ELF::R_VE_REFLONG;
+  case VE::fixup_ve_srel32:
+    Ctx.reportError(Fixup.getLoc(),
+                    "A non pc-relative srel32 relocation is not supported");
+    return ELF::R_VE_NONE;
   case VE::fixup_ve_hi32:
     return ELF::R_VE_HI32;
   case VE::fixup_ve_lo32:
     return ELF::R_VE_LO32;
   case VE::fixup_ve_pc_hi32:
-    llvm_unreachable("Unimplemented fixup pc_hi32 -> relocation");
+    Ctx.reportError(Fixup.getLoc(),
+                    "A non pc-relative pc_hi32 relocation is not supported");
+    return ELF::R_VE_NONE;
   case VE::fixup_ve_pc_lo32:
-    llvm_unreachable("Unimplemented fixup pc_lo32 -> relocation");
+    Ctx.reportError(Fixup.getLoc(),
+                    "A non pc-relative pc_lo32 relocation is not supported");
+    return ELF::R_VE_NONE;
   case VE::fixup_ve_got_hi32:
     return ELF::R_VE_GOT_HI32;
   case VE::fixup_ve_got_lo32:
diff --git a/llvm/lib/Target/VE/MCTargetDesc/VEFixupKinds.h b/llvm/lib/Target/VE/MCTargetDesc/VEFixupKinds.h
index 5d5dc1c5c891..46b995cee840 100644
--- a/llvm/lib/Target/VE/MCTargetDesc/VEFixupKinds.h
+++ b/llvm/lib/Target/VE/MCTargetDesc/VEFixupKinds.h
@@ -17,6 +17,9 @@ enum Fixups {
   /// fixup_ve_reflong - 32-bit fixup corresponding to foo
   fixup_ve_reflong = FirstTargetFixupKind,
 
+  /// fixup_ve_srel32 - 32-bit fixup corresponding to foo for relative branch
+  fixup_ve_srel32,
+
   /// fixup_ve_hi32 - 32-bit fixup corresponding to foo@hi
   fixup_ve_hi32,
 
diff --git a/llvm/lib/Target/VE/MCTargetDesc/VEMCCodeEmitter.cpp b/llvm/lib/Target/VE/MCTargetDesc/VEMCCodeEmitter.cpp
index d50d8fcae9da..65bb0cf8b0d7 100644
--- a/llvm/lib/Target/VE/MCTargetDesc/VEMCCodeEmitter.cpp
+++ b/llvm/lib/Target/VE/MCTargetDesc/VEMCCodeEmitter.cpp
@@ -102,11 +102,11 @@ unsigned VEMCCodeEmitter::getMachineOpValue(const MCInst &MI,
                                             const MCSubtargetInfo &STI) const {
   if (MO.isReg())
     return Ctx.getRegisterInfo()->getEncodingValue(MO.getReg());
-
   if (MO.isImm())
-    return MO.getImm();
+    return static_cast<unsigned>(MO.getImm());
 
   assert(MO.isExpr());
+
   const MCExpr *Expr = MO.getExpr();
   if (const VEMCExpr *SExpr = dyn_cast<VEMCExpr>(Expr)) {
     MCFixupKind Kind = (MCFixupKind)SExpr->getFixupKind();
@@ -131,7 +131,7 @@ VEMCCodeEmitter::getBranchTargetOpValue(const MCInst &MI, unsigned OpNo,
     return getMachineOpValue(MI, MO, Fixups, STI);
 
   Fixups.push_back(
-      MCFixup::create(0, MO.getExpr(), (MCFixupKind)VE::fixup_ve_pc_lo32));
+      MCFixup::create(0, MO.getExpr(), (MCFixupKind)VE::fixup_ve_srel32));
   return 0;
 }
 
diff --git a/llvm/lib/Target/VE/MCTargetDesc/VEMCExpr.cpp b/llvm/lib/Target/VE/MCTargetDesc/VEMCExpr.cpp
index a3ce3b3309be..4d45918ad0aa 100644
--- a/llvm/lib/Target/VE/MCTargetDesc/VEMCExpr.cpp
+++ b/llvm/lib/Target/VE/MCTargetDesc/VEMCExpr.cpp
@@ -12,11 +12,12 @@
 //===----------------------------------------------------------------------===//
 
 #include "VEMCExpr.h"
+#include "llvm/BinaryFormat/ELF.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCObjectStreamer.h"
 #include "llvm/MC/MCSymbolELF.h"
-#include "llvm/BinaryFormat/ELF.h"
+#include "llvm/MC/MCValue.h"
 
 using namespace llvm;
 
@@ -174,7 +175,13 @@ VE::Fixups VEMCExpr::getFixupKind(VEMCExpr::VariantKind Kind) {
 bool VEMCExpr::evaluateAsRelocatableImpl(MCValue &Res,
                                          const MCAsmLayout *Layout,
                                          const MCFixup *Fixup) const {
-  return getSubExpr()->evaluateAsRelocatable(Res, Layout, Fixup);
+  if (!getSubExpr()->evaluateAsRelocatable(Res, Layout, Fixup))
+    return false;
+
+  Res =
+      MCValue::get(Res.getSymA(), Res.getSymB(), Res.getConstant(), getKind());
+
+  return true;
 }
 
 static void fixELFSymbolsInTLSFixupsImpl(const MCExpr *Expr, MCAssembler &Asm) {
diff --git a/llvm/lib/Target/VE/VEISelLowering.cpp b/llvm/lib/Target/VE/VEISelLowering.cpp
index 32315543826a..5ef223d6030b 100644
--- a/llvm/lib/Target/VE/VEISelLowering.cpp
+++ b/llvm/lib/Target/VE/VEISelLowering.cpp
@@ -1720,7 +1720,7 @@ SDValue VETargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::EXTRACT_VECTOR_ELT:
     return lowerEXTRACT_VECTOR_ELT(Op, DAG);
 
-#define ADD_BINARY_VVP_OP(VVP_NAME, ISD_NAME) case ISD::ISD_NAME:
+#define ADD_BINARY_VVP_OP(VVP_NAME, VP_NAME, ISD_NAME) case ISD::ISD_NAME:
 #include "VVPNodes.def"
     return lowerToVVP(Op, DAG);
   }
diff --git a/llvm/lib/Target/VE/VVPInstrInfo.td b/llvm/lib/Target/VE/VVPInstrInfo.td
index 2c88d5099a7b..99566e91ec11 100644
--- a/llvm/lib/Target/VE/VVPInstrInfo.td
+++ b/llvm/lib/Target/VE/VVPInstrInfo.td
@@ -29,6 +29,16 @@ def SDTIntBinOpVVP : SDTypeProfile<1, 4, [     // vp_add, vp_and, etc.
   IsVLVT<4>
 ]>;
 
+// BinaryFPOp(x,y,mask,vl)
+def SDTFPBinOpVVP : SDTypeProfile<1, 4, [      // vvp_fadd, etc.
+  SDTCisSameAs<0, 1>,
+  SDTCisSameAs<0, 2>,
+  SDTCisFP<0>,
+  SDTCisInt<3>,
+  SDTCisSameNumEltsAs<0, 3>,
+  IsVLVT<4>
+]>;
+
 // Binary operator commutative pattern.
 class vvp_commutative<SDNode RootOp> :
   PatFrags<
@@ -40,7 +50,32 @@ class vvp_commutative<SDNode RootOp> :
 def vvp_add    : SDNode<"VEISD::VVP_ADD",  SDTIntBinOpVVP>;
 def c_vvp_add  : vvp_commutative<vvp_add>;
 
+def vvp_sub    : SDNode<"VEISD::VVP_SUB",  SDTIntBinOpVVP>;
+
+def vvp_mul    : SDNode<"VEISD::VVP_MUL",  SDTIntBinOpVVP>;
+def c_vvp_mul  : vvp_commutative<vvp_mul>;
+
+def vvp_sdiv   : SDNode<"VEISD::VVP_SDIV", SDTIntBinOpVVP>;
+def vvp_udiv   : SDNode<"VEISD::VVP_UDIV", SDTIntBinOpVVP>;
+
 def vvp_and    : SDNode<"VEISD::VVP_AND",  SDTIntBinOpVVP>;
 def c_vvp_and  : vvp_commutative<vvp_and>;
 
+def vvp_or     : SDNode<"VEISD::VVP_OR",  SDTIntBinOpVVP>;
+def c_vvp_or   : vvp_commutative<vvp_or>;
+
+def vvp_xor    : SDNode<"VEISD::VVP_XOR",  SDTIntBinOpVVP>;
+def c_vvp_xor  : vvp_commutative<vvp_xor>;
+
+def vvp_srl    : SDNode<"VEISD::VVP_SRL",  SDTIntBinOpVVP>;
+def vvp_sra    : SDNode<"VEISD::VVP_SRA",  SDTIntBinOpVVP>;
+def vvp_shl    : SDNode<"VEISD::VVP_SHL",  SDTIntBinOpVVP>;
+
+def vvp_fadd    : SDNode<"VEISD::VVP_FADD",  SDTFPBinOpVVP>;
+def c_vvp_fadd  : vvp_commutative<vvp_fadd>;
+def vvp_fsub    : SDNode<"VEISD::VVP_FSUB",  SDTFPBinOpVVP>;
+def vvp_fmul    : SDNode<"VEISD::VVP_FMUL",  SDTFPBinOpVVP>;
+def c_vvp_fmul  : vvp_commutative<vvp_fmul>;
+def vvp_fdiv    : SDNode<"VEISD::VVP_FDIV",  SDTFPBinOpVVP>;
+
 // } Binary Operators
diff --git a/llvm/lib/Target/VE/VVPInstrPatternsVec.td b/llvm/lib/Target/VE/VVPInstrPatternsVec.td
index ac03e0bf627e..8d5d9d103547 100644
--- a/llvm/lib/Target/VE/VVPInstrPatternsVec.td
+++ b/llvm/lib/Target/VE/VVPInstrPatternsVec.td
@@ -17,54 +17,177 @@
 //===----------------------------------------------------------------------===//
 include "VVPInstrInfo.td"
 
-multiclass VectorBinaryArith<
-    SDPatternOperator OpNode,
-    ValueType ScalarVT, ValueType DataVT, ValueType MaskVT,
-    string OpBaseName> {
-  // No mask.
+multiclass Binary_rv<SDPatternOperator OpNode,
+    ValueType ScalarVT, ValueType DataVT,
+    ValueType MaskVT, string OpBaseName> {
+  // Masked with select, broadcast.
+  // TODO
+
+  // Unmasked, broadcast.
   def : Pat<(OpNode
-                (any_broadcast ScalarVT:$sx),
-                DataVT:$vy, (MaskVT true_mask), i32:$avl),
+                (any_broadcast ScalarVT:$sx), DataVT:$vy,
+                (MaskVT true_mask),
+                i32:$avl),
             (!cast<Instruction>(OpBaseName#"rvl")
                 ScalarVT:$sx, $vy, $avl)>;
-  def : Pat<(OpNode DataVT:$vx, DataVT:$vy, (MaskVT true_mask), i32:$avl),
+  // Masked, broadcast.
+  def : Pat<(OpNode
+                (any_broadcast ScalarVT:$sx), DataVT:$vy,
+                MaskVT:$mask,
+                i32:$avl),
+            (!cast<Instruction>(OpBaseName#"rvml")
+                ScalarVT:$sx, $vy, $mask, $avl)>;
+}
+
+multiclass Binary_vr<SDPatternOperator OpNode,
+    ValueType ScalarVT, ValueType DataVT,
+    ValueType MaskVT, string OpBaseName> {
+  // Masked with select, broadcast.
+  // TODO
+
+  // Unmasked, broadcast.
+  def : Pat<(OpNode
+                DataVT:$vx, (any_broadcast ScalarVT:$sy),
+                (MaskVT true_mask),
+                i32:$avl),
+            (!cast<Instruction>(OpBaseName#"vrl")
+                $vx, ScalarVT:$sy, $avl)>;
+  // Masked, broadcast.
+  def : Pat<(OpNode
+                DataVT:$vx, (any_broadcast ScalarVT:$sy),
+                MaskVT:$mask,
+                i32:$avl),
+            (!cast<Instruction>(OpBaseName#"vrml")
+                $vx, ScalarVT:$sy, $mask, $avl)>;
+}
+
+multiclass Binary_vv<SDPatternOperator OpNode,
+    ValueType DataVT,
+    ValueType MaskVT, string OpBaseName> {
+  // Masked with select.
+  // TODO
+
+  // Unmasked.
+  def : Pat<(OpNode
+                DataVT:$vx, DataVT:$vy,
+                (MaskVT true_mask),
+                i32:$avl),
             (!cast<Instruction>(OpBaseName#"vvl")
                 $vx, $vy, $avl)>;
 
-  // Mask.
+  // Masked.
   def : Pat<(OpNode
-                (any_broadcast ScalarVT:$sx),
-                DataVT:$vy, MaskVT:$mask, i32:$avl),
-            (!cast<Instruction>(OpBaseName#"rvml")
-                ScalarVT:$sx, $vy, $mask, $avl)>;
-  def : Pat<(OpNode DataVT:$vx, DataVT:$vy, MaskVT:$mask, i32:$avl),
+                DataVT:$vx, DataVT:$vy,
+                MaskVT:$mask,
+                i32:$avl),
             (!cast<Instruction>(OpBaseName#"vvml")
                 $vx, $vy, $mask, $avl)>;
+}
 
-  // TODO We do not specify patterns for the immediate variants here. There
-  // will be an immediate folding pass that takes care of switching to the
-  // immediate variant where applicable.
+multiclass Binary_rv_vv<
+    SDPatternOperator OpNode,
+    ValueType ScalarVT, ValueType DataVT, ValueType MaskVT,
+    string OpBaseName> {
+  defm : Binary_rv<OpNode, ScalarVT, DataVT, MaskVT, OpBaseName>;
+  defm : Binary_vv<OpNode, DataVT, MaskVT, OpBaseName>;
+}
+
+multiclass Binary_vr_vv<
+    SDPatternOperator OpNode,
+    ValueType ScalarVT, ValueType DataVT, ValueType MaskVT,
+    string OpBaseName> {
+  defm : Binary_vr<OpNode, ScalarVT, DataVT, MaskVT, OpBaseName>;
+  defm : Binary_vv<OpNode, DataVT, MaskVT, OpBaseName>;
+}
 
-  // TODO Fold vvp_select into passthru.
+multiclass Binary_rv_vr_vv<
+    SDPatternOperator OpNode,
+    ValueType ScalarVT, ValueType DataVT, ValueType MaskVT,
+    string OpBaseName> {
+  defm : Binary_rv<OpNode, ScalarVT, DataVT, MaskVT, OpBaseName>;
+  defm : Binary_vr_vv<OpNode, ScalarVT, DataVT, MaskVT, OpBaseName>;
 }
 
 // Expand both 64bit and 32 bit variant (256 elements)
-multiclass VectorBinaryArith_ShortLong<
+multiclass Binary_rv_vv_ShortLong<
+    SDPatternOperator OpNode,
+    ValueType LongScalarVT, ValueType LongDataVT, string LongOpBaseName,
+    ValueType ShortScalarVT, ValueType ShortDataVT, string ShortOpBaseName> {
+  defm : Binary_rv_vv<OpNode,
+                      LongScalarVT, LongDataVT, v256i1,
+                      LongOpBaseName>;
+  defm : Binary_rv_vv<OpNode,
+                      ShortScalarVT, ShortDataVT, v256i1,
+                      ShortOpBaseName>;
+}
+
+multiclass Binary_vr_vv_ShortLong<
+    SDPatternOperator OpNode,
+    ValueType LongScalarVT, ValueType LongDataVT, string LongOpBaseName,
+    ValueType ShortScalarVT, ValueType ShortDataVT, string ShortOpBaseName> {
+  defm : Binary_vr_vv<OpNode,
+                      LongScalarVT, LongDataVT, v256i1,
+                      LongOpBaseName>;
+  defm : Binary_vr_vv<OpNode,
+                      ShortScalarVT, ShortDataVT, v256i1,
+                      ShortOpBaseName>;
+}
+
+multiclass Binary_rv_vr_vv_ShortLong<
     SDPatternOperator OpNode,
     ValueType LongScalarVT, ValueType LongDataVT, string LongOpBaseName,
     ValueType ShortScalarVT, ValueType ShortDataVT, string ShortOpBaseName> {
-  defm : VectorBinaryArith<OpNode,
-                           LongScalarVT, LongDataVT, v256i1,
-                           LongOpBaseName>;
-  defm : VectorBinaryArith<OpNode,
-                           ShortScalarVT, ShortDataVT, v256i1,
-                           ShortOpBaseName>;
+  defm : Binary_rv_vr_vv<OpNode,
+                      LongScalarVT, LongDataVT, v256i1,
+                      LongOpBaseName>;
+  defm : Binary_rv_vr_vv<OpNode,
+                      ShortScalarVT, ShortDataVT, v256i1,
+                      ShortOpBaseName>;
 }
 
+defm : Binary_rv_vv_ShortLong<c_vvp_add,
+                              i64, v256i64, "VADDSL",
+                              i32, v256i32, "VADDSWSX">;
+defm : Binary_rv_vv_ShortLong<vvp_sub,
+                              i64, v256i64, "VSUBSL",
+                              i32, v256i32, "VSUBSWSX">;
+defm : Binary_rv_vv_ShortLong<c_vvp_mul,
+                              i64, v256i64, "VMULSL",
+                              i32, v256i32, "VMULSWSX">;
+defm : Binary_rv_vr_vv_ShortLong<vvp_sdiv,
+                              i64, v256i64, "VDIVSL",
+                              i32, v256i32, "VDIVSWSX">;
+defm : Binary_rv_vr_vv_ShortLong<vvp_udiv,
+                              i64, v256i64, "VDIVUL",
+                              i32, v256i32, "VDIVUW">;
+defm : Binary_rv_vv_ShortLong<c_vvp_and,
+                              i64, v256i64, "VAND",
+                              i32, v256i32, "PVANDLO">;
+defm : Binary_rv_vv_ShortLong<c_vvp_or,
+                              i64, v256i64, "VOR",
+                              i32, v256i32, "PVORLO">;
+defm : Binary_rv_vv_ShortLong<c_vvp_xor,
+                              i64, v256i64, "VXOR",
+                              i32, v256i32, "PVXORLO">;
+defm : Binary_vr_vv_ShortLong<vvp_shl,
+                              i64, v256i64, "VSLL",
+                              i32, v256i32, "PVSLLLO">;
+defm : Binary_vr_vv_ShortLong<vvp_sra,
+                              i64, v256i64, "VSRAL",
+                              i32, v256i32, "PVSRALO">;
+defm : Binary_vr_vv_ShortLong<vvp_srl,
+                              i64, v256i64, "VSRL",
+                              i32, v256i32, "PVSRLLO">;
 
-defm : VectorBinaryArith_ShortLong<c_vvp_add,
-                                   i64, v256i64, "VADDSL",
-                                   i32, v256i32, "VADDSWSX">;
-defm : VectorBinaryArith_ShortLong<c_vvp_and,
-                                   i64, v256i64, "VAND",
-                                   i32, v256i32, "PVANDLO">;
+defm : Binary_rv_vv_ShortLong<c_vvp_fadd,
+                              f64, v256f64, "VFADDD",
+                              f32, v256f32, "PVFADDUP">;
+defm : Binary_rv_vv_ShortLong<c_vvp_fmul,
+                              f64, v256f64, "VFMULD",
+                              f32, v256f32, "PVFMULUP">;
+defm : Binary_rv_vv_ShortLong<vvp_fsub,
+                              f64, v256f64, "VFSUBD",
+                              f32, v256f32, "PVFSUBUP">;
+defm : Binary_rv_vr_vv_ShortLong<vvp_fdiv,
+                              f64, v256f64, "VFDIVD",
+                              f32, v256f32, "VFDIVS">;
diff --git a/llvm/lib/Target/VE/VVPNodes.def b/llvm/lib/Target/VE/VVPNodes.def
index a68402e9ea10..8a9231f7d3e6 100644
--- a/llvm/lib/Target/VE/VVPNodes.def
+++ b/llvm/lib/Target/VE/VVPNodes.def
@@ -28,14 +28,38 @@
 /// \p VVPName is a VVP Binary operator.
 /// \p SDNAME is the generic SD opcode corresponding to \p VVPName.
 #ifndef ADD_BINARY_VVP_OP
-#define ADD_BINARY_VVP_OP(X,Y) ADD_VVP_OP(X,Y) HANDLE_VP_TO_VVP(VP_##Y, X)
+#define ADD_BINARY_VVP_OP(VVPNAME,VPNAME,SDNAME) \
+            ADD_VVP_OP(VVPNAME,SDNAME) \
+            HANDLE_VP_TO_VVP(VPNAME, VVPNAME)
+#endif
+
+#ifndef ADD_BINARY_VVP_OP_COMPACT
+#define ADD_BINARY_VVP_OP_COMPACT(NAME) \
+    ADD_BINARY_VVP_OP(VVP_##NAME,VP_##NAME,NAME)
 #endif
 
 // Integer arithmetic.
-ADD_BINARY_VVP_OP(VVP_ADD,ADD)
+ADD_BINARY_VVP_OP_COMPACT(ADD)
+ADD_BINARY_VVP_OP_COMPACT(SUB)
+ADD_BINARY_VVP_OP_COMPACT(MUL)
+ADD_BINARY_VVP_OP_COMPACT(UDIV)
+ADD_BINARY_VVP_OP_COMPACT(SDIV)
 
-ADD_BINARY_VVP_OP(VVP_AND,AND)
+ADD_BINARY_VVP_OP(VVP_SRA,VP_ASHR,SRA)
+ADD_BINARY_VVP_OP(VVP_SRL,VP_LSHR,SRL)
+ADD_BINARY_VVP_OP_COMPACT(SHL)
+
+ADD_BINARY_VVP_OP_COMPACT(AND)
+ADD_BINARY_VVP_OP_COMPACT(OR)
+ADD_BINARY_VVP_OP_COMPACT(XOR)
+
+// FP arithmetic.
+ADD_BINARY_VVP_OP_COMPACT(FADD)
+ADD_BINARY_VVP_OP_COMPACT(FSUB)
+ADD_BINARY_VVP_OP_COMPACT(FMUL)
+ADD_BINARY_VVP_OP_COMPACT(FDIV)
 
-#undef HANDLE_VP_TO_VVP
 #undef ADD_BINARY_VVP_OP
+#undef ADD_BINARY_VVP_OP_COMPACT
 #undef ADD_VVP_OP
+#undef HANDLE_VP_TO_VVP
diff --git a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
index 7d1e6c553f81..56689d3ee06b 100644
--- a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
+++ b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
@@ -571,7 +571,6 @@ public:
     // proper nesting.
     bool ExpectBlockType = false;
     bool ExpectFuncType = false;
-    bool ExpectHeapType = false;
     std::unique_ptr<WebAssemblyOperand> FunctionTable;
     if (Name == "block") {
       push(Block);
@@ -624,8 +623,6 @@ public:
       if (parseFunctionTableOperand(&FunctionTable))
         return true;
       ExpectFuncType = true;
-    } else if (Name == "ref.null") {
-      ExpectHeapType = true;
     }
 
     if (ExpectFuncType || (ExpectBlockType && Lexer.is(AsmToken::LParen))) {
@@ -670,15 +667,6 @@ public:
             return error("Unknown block type: ", Id);
           addBlockTypeOperand(Operands, NameLoc, BT);
           Parser.Lex();
-        } else if (ExpectHeapType) {
-          auto HeapType = WebAssembly::parseHeapType(Id.getString());
-          if (HeapType == WebAssembly::HeapType::Invalid) {
-            return error("Expected a heap type: ", Id);
-          }
-          Operands.push_back(std::make_unique<WebAssemblyOperand>(
-              WebAssemblyOperand::Integer, Id.getLoc(), Id.getEndLoc(),
-              WebAssemblyOperand::IntOp{static_cast<int64_t>(HeapType)}));
-          Parser.Lex();
         } else {
           // Assume this identifier is a label.
           const MCExpr *Val;
diff --git a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.cpp b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.cpp
index a6b5d4252f2f..128ce5c4fec0 100644
--- a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.cpp
+++ b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.cpp
@@ -112,9 +112,18 @@ bool WebAssemblyAsmTypeCheck::getLocal(SMLoc ErrorLoc, const MCInst &Inst,
   return false;
 }
 
-bool WebAssemblyAsmTypeCheck::checkEnd(SMLoc ErrorLoc) {
+bool WebAssemblyAsmTypeCheck::checkEnd(SMLoc ErrorLoc, bool PopVals) {
   if (LastSig.Returns.size() > Stack.size())
     return typeError(ErrorLoc, "end: insufficient values on the type stack");
+  
+  if (PopVals) {
+    for (auto VT : llvm::reverse(LastSig.Returns)) {
+      if (popType(ErrorLoc, VT)) 
+        return true;
+    }
+    return false;
+  }
+  
   for (size_t i = 0; i < LastSig.Returns.size(); i++) {
     auto EVT = LastSig.Returns[i];
     auto PVT = Stack[Stack.size() - LastSig.Returns.size() + i];
@@ -221,7 +230,7 @@ bool WebAssemblyAsmTypeCheck::typeCheck(SMLoc ErrorLoc, const MCInst &Inst) {
       return true;
   } else if (Name == "end_block" || Name == "end_loop" || Name == "end_if" ||
              Name == "else" || Name == "end_try") {
-    if (checkEnd(ErrorLoc))
+    if (checkEnd(ErrorLoc, Name == "else"))
       return true;
     if (Name == "end_block")
       Unreachable = false;
diff --git a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.h b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.h
index aa35213ccca3..2b07faf67a18 100644
--- a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.h
+++ b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.h
@@ -39,7 +39,7 @@ class WebAssemblyAsmTypeCheck final {
   bool typeError(SMLoc ErrorLoc, const Twine &Msg);
   bool popType(SMLoc ErrorLoc, Optional<wasm::ValType> EVT);
   bool getLocal(SMLoc ErrorLoc, const MCInst &Inst, wasm::ValType &Type);
-  bool checkEnd(SMLoc ErrorLoc);
+  bool checkEnd(SMLoc ErrorLoc, bool PopVals = false);
   bool checkSig(SMLoc ErrorLoc, const wasm::WasmSignature &Sig);
   bool getSymRef(SMLoc ErrorLoc, const MCInst &Inst,
                  const MCSymbolRefExpr *&SymRef);
diff --git a/llvm/lib/Target/WebAssembly/Disassembler/WebAssemblyDisassembler.cpp b/llvm/lib/Target/WebAssembly/Disassembler/WebAssemblyDisassembler.cpp
index 2e1e4f061219..5d38145559da 100644
--- a/llvm/lib/Target/WebAssembly/Disassembler/WebAssemblyDisassembler.cpp
+++ b/llvm/lib/Target/WebAssembly/Disassembler/WebAssemblyDisassembler.cpp
@@ -241,28 +241,6 @@ MCDisassembler::DecodeStatus WebAssemblyDisassembler::getInstruction(
       }
       break;
     }
-    // heap_type operands, for e.g. ref.null:
-    case WebAssembly::OPERAND_HEAPTYPE: {
-      int64_t Val;
-      uint64_t PrevSize = Size;
-      if (!nextLEB(Val, Bytes, Size, true))
-        return MCDisassembler::Fail;
-      if (Val < 0 && Size == PrevSize + 1) {
-        // The HeapType encoding is like BlockType, in that encodings that
-        // decode as negative values indicate ValTypes.  In practice we expect
-        // either wasm::ValType::EXTERNREF or wasm::ValType::FUNCREF here.
-        //
-        // The positive SLEB values are reserved for future expansion and are
-        // expected to be type indices in the typed function references
-        // proposal, and should disassemble as MCSymbolRefExpr as in BlockType
-        // above.
-        MI.addOperand(MCOperand::createImm(Val & 0x7f));
-      } else {
-        MI.addOperand(
-            MCOperand::createImm(int64_t(WebAssembly::HeapType::Invalid)));
-      }
-      break;
-    }
     // FP operands.
     case WebAssembly::OPERAND_F32IMM: {
       if (!parseImmediate<float>(MI, Size, Bytes))
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.cpp b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.cpp
index 2967aaa00ad4..d72bfdbbfb99 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.cpp
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.cpp
@@ -366,26 +366,3 @@ void WebAssemblyInstPrinter::printWebAssemblySignatureOperand(const MCInst *MI,
     }
   }
 }
-
-void WebAssemblyInstPrinter::printWebAssemblyHeapTypeOperand(const MCInst *MI,
-                                                             unsigned OpNo,
-                                                             raw_ostream &O) {
-  const MCOperand &Op = MI->getOperand(OpNo);
-  if (Op.isImm()) {
-    switch (Op.getImm()) {
-    case long(wasm::ValType::EXTERNREF):
-      O << "extern";
-      break;
-    case long(wasm::ValType::FUNCREF):
-      O << "func";
-      break;
-    default:
-      O << "unsupported_heap_type_value";
-      break;
-    }
-  } else {
-    // Typed function references and other subtypes of funcref and externref
-    // currently unimplemented.
-    O << "unsupported_heap_type_operand";
-  }
-}
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.h b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.h
index 7d980c78c3c9..fe104cbca12e 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.h
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.h
@@ -47,8 +47,6 @@ public:
                                       raw_ostream &O);
   void printWebAssemblySignatureOperand(const MCInst *MI, unsigned OpNo,
                                         raw_ostream &O);
-  void printWebAssemblyHeapTypeOperand(const MCInst *MI, unsigned OpNo,
-                                       raw_ostream &O);
 
   // Autogenerated by tblgen.
   std::pair<const char *, uint64_t> getMnemonic(const MCInst *MI) override;
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCAsmInfo.cpp b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCAsmInfo.cpp
index c3d259e6ff20..d8122950e061 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCAsmInfo.cpp
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCAsmInfo.cpp
@@ -13,6 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "WebAssemblyMCAsmInfo.h"
+#include "Utils/WebAssemblyUtilities.h"
 #include "llvm/ADT/Triple.h"
 
 using namespace llvm;
@@ -44,5 +45,13 @@ WebAssemblyMCAsmInfo::WebAssemblyMCAsmInfo(const Triple &T,
 
   SupportsDebugInformation = true;
 
+  // When compilation is done on a cpp file by clang, the exception model info
+  // is stored in LangOptions, which is later used to set the info in
+  // TargetOptions and then MCAsmInfo in LLVMTargetMachine::initAsmInfo(). But
+  // this process does not happen when compiling bitcode directly with clang, so
+  // we make sure this info is set correctly.
+  if (WebAssembly::WasmEnableEH || WebAssembly::WasmEnableSjLj)
+    ExceptionsType = ExceptionHandling::Wasm;
+
   // TODO: UseIntegratedAssembler?
 }
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCCodeEmitter.cpp b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCCodeEmitter.cpp
index 4961c2ef9529..6e494b9430f7 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCCodeEmitter.cpp
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCCodeEmitter.cpp
@@ -106,9 +106,6 @@ void WebAssemblyMCCodeEmitter::encodeInstruction(
           encodeSLEB128(int64_t(MO.getImm()), OS);
           break;
         case WebAssembly::OPERAND_SIGNATURE:
-        case WebAssembly::OPERAND_HEAPTYPE:
-          OS << uint8_t(MO.getImm());
-          break;
         case WebAssembly::OPERAND_VEC_I8IMM:
           support::endian::write<uint8_t>(OS, MO.getImm(), support::little);
           break;
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h
index d07bfce9abc1..b2f10ca93a4f 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h
@@ -78,8 +78,6 @@ enum OperandType {
   OPERAND_BRLIST,
   /// 32-bit unsigned table number.
   OPERAND_TABLE,
-  /// heap type immediate for ref.null.
-  OPERAND_HEAPTYPE,
 };
 } // end namespace WebAssembly
 
diff --git a/llvm/lib/Target/WebAssembly/Utils/WebAssemblyTypeUtilities.cpp b/llvm/lib/Target/WebAssembly/Utils/WebAssemblyTypeUtilities.cpp
index 6f81431bba2d..0412e524f800 100644
--- a/llvm/lib/Target/WebAssembly/Utils/WebAssemblyTypeUtilities.cpp
+++ b/llvm/lib/Target/WebAssembly/Utils/WebAssemblyTypeUtilities.cpp
@@ -41,13 +41,6 @@ Optional<wasm::ValType> WebAssembly::parseType(StringRef Type) {
   return Optional<wasm::ValType>();
 }
 
-WebAssembly::HeapType WebAssembly::parseHeapType(StringRef Type) {
-  return StringSwitch<WebAssembly::HeapType>(Type)
-      .Case("extern", WebAssembly::HeapType::Externref)
-      .Case("func", WebAssembly::HeapType::Funcref)
-      .Default(WebAssembly::HeapType::Invalid);
-}
-
 WebAssembly::BlockType WebAssembly::parseBlockType(StringRef Type) {
   // Multivalue block types are handled separately in parseSignature
   return StringSwitch<WebAssembly::BlockType>(Type)
diff --git a/llvm/lib/Target/WebAssembly/Utils/WebAssemblyTypeUtilities.h b/llvm/lib/Target/WebAssembly/Utils/WebAssemblyTypeUtilities.h
index 8d757df27b34..042d51c7d6cb 100644
--- a/llvm/lib/Target/WebAssembly/Utils/WebAssemblyTypeUtilities.h
+++ b/llvm/lib/Target/WebAssembly/Utils/WebAssemblyTypeUtilities.h
@@ -41,17 +41,9 @@ enum class BlockType : unsigned {
   Multivalue = 0xffff,
 };
 
-/// Used as immediate MachineOperands for heap types, e.g. for ref.null.
-enum class HeapType : unsigned {
-  Invalid = 0x00,
-  Externref = unsigned(wasm::ValType::EXTERNREF),
-  Funcref = unsigned(wasm::ValType::FUNCREF),
-};
-
 // Convert StringRef to ValType / HealType / BlockType
 
 Optional<wasm::ValType> parseType(StringRef Type);
-HeapType parseHeapType(StringRef Type);
 BlockType parseBlockType(StringRef Type);
 MVT parseMVT(StringRef Type);
 
diff --git a/llvm/lib/Target/WebAssembly/Utils/WebAssemblyUtilities.cpp b/llvm/lib/Target/WebAssembly/Utils/WebAssemblyUtilities.cpp
index 3da80f4fc875..b87c884c9e4a 100644
--- a/llvm/lib/Target/WebAssembly/Utils/WebAssemblyUtilities.cpp
+++ b/llvm/lib/Target/WebAssembly/Utils/WebAssemblyUtilities.cpp
@@ -18,6 +18,31 @@
 #include "llvm/MC/MCContext.h"
 using namespace llvm;
 
+// Exception handling & setjmp-longjmp handling related options. These are
+// defined here to be shared between WebAssembly and its subdirectories.
+
+// Emscripten's asm.js-style exception handling
+cl::opt<bool> WebAssembly::WasmEnableEmEH(
+    "enable-emscripten-cxx-exceptions",
+    cl::desc("WebAssembly Emscripten-style exception handling"),
+    cl::init(false));
+// Emscripten's asm.js-style setjmp/longjmp handling
+cl::opt<bool> WebAssembly::WasmEnableEmSjLj(
+    "enable-emscripten-sjlj",
+    cl::desc("WebAssembly Emscripten-style setjmp/longjmp handling"),
+    cl::init(false));
+// Exception handling using wasm EH instructions
+cl::opt<bool>
+    WebAssembly::WasmEnableEH("wasm-enable-eh",
+                              cl::desc("WebAssembly exception handling"),
+                              cl::init(false));
+// setjmp/longjmp handling using wasm EH instrutions
+cl::opt<bool>
+    WebAssembly::WasmEnableSjLj("wasm-enable-sjlj",
+                                cl::desc("WebAssembly setjmp/longjmp handling"),
+                                cl::init(false));
+
+// Function names in libc++abi and libunwind
 const char *const WebAssembly::CxaBeginCatchFn = "__cxa_begin_catch";
 const char *const WebAssembly::CxaRethrowFn = "__cxa_rethrow";
 const char *const WebAssembly::StdTerminateFn = "_ZSt9terminatev";
diff --git a/llvm/lib/Target/WebAssembly/Utils/WebAssemblyUtilities.h b/llvm/lib/Target/WebAssembly/Utils/WebAssemblyUtilities.h
index f6e96d9b2877..d024185defb4 100644
--- a/llvm/lib/Target/WebAssembly/Utils/WebAssemblyUtilities.h
+++ b/llvm/lib/Target/WebAssembly/Utils/WebAssemblyUtilities.h
@@ -16,6 +16,7 @@
 #define LLVM_LIB_TARGET_WEBASSEMBLY_UTILS_WEBASSEMBLYUTILITIES_H
 
 #include "llvm/IR/DerivedTypes.h"
+#include "llvm/Support/CommandLine.h"
 
 namespace llvm {
 
@@ -70,6 +71,12 @@ inline bool isRefType(const Type *Ty) {
 bool isChild(const MachineInstr &MI, const WebAssemblyFunctionInfo &MFI);
 bool mayThrow(const MachineInstr &MI);
 
+// Exception handling / setjmp-longjmp handling command-line options
+extern cl::opt<bool> WasmEnableEmEH;   // asm.js-style EH
+extern cl::opt<bool> WasmEnableEmSjLj; // asm.js-style SjLJ
+extern cl::opt<bool> WasmEnableEH;     // EH using Wasm EH instructions
+extern cl::opt<bool> WasmEnableSjLj;   // SjLj using Wasm EH instructions
+
 // Exception-related function names
 extern const char *const ClangCallTerminateFn;
 extern const char *const CxaBeginCatchFn;
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp
index 0d3f51693261..e3af6b2662ef 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp
@@ -51,8 +51,6 @@ using namespace llvm;
 #define DEBUG_TYPE "asm-printer"
 
 extern cl::opt<bool> WasmKeepRegisters;
-extern cl::opt<bool> WasmEnableEmEH;
-extern cl::opt<bool> WasmEnableEmSjLj;
 
 //===----------------------------------------------------------------------===//
 // Helpers.
@@ -196,6 +194,13 @@ void WebAssemblyAsmPrinter::emitGlobalVariable(const GlobalVariable *GV) {
     Sym->setGlobalType(wasm::WasmGlobalType{uint8_t(Type), Mutable});
   }
 
+  // If the GlobalVariable refers to a table, we handle it here instead of
+  // in emitExternalDecls
+  if (Sym->isTable()) {
+    getTargetStreamer()->emitTableType(Sym);
+    return;
+  }
+
   emitVisibility(Sym, GV->getVisibility(), !GV->isDeclaration());
   if (GV->hasInitializer()) {
     assert(getSymbolPreferLocal(*GV) == Sym);
@@ -315,8 +320,9 @@ void WebAssemblyAsmPrinter::emitExternalDecls(const Module &M) {
       // will discard it later if it turns out not to be necessary.
       auto Signature = signatureFromMVTs(Results, Params);
       bool InvokeDetected = false;
-      auto *Sym = getMCSymbolForFunction(&F, WasmEnableEmEH || WasmEnableEmSjLj,
-                                         Signature.get(), InvokeDetected);
+      auto *Sym = getMCSymbolForFunction(
+          &F, WebAssembly::WasmEnableEmEH || WebAssembly::WasmEnableEmSjLj,
+          Signature.get(), InvokeDetected);
 
       // Multiple functions can be mapped to the same invoke symbol. For
       // example, two IR functions '__invoke_void_i8*' and '__invoke_void_i32'
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp
index 7832f199a2cc..17e867e4c7d8 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp
@@ -1741,7 +1741,7 @@ void WebAssemblyCFGStackify::rewriteDepthImmediates(MachineFunction &MF) {
 
 void WebAssemblyCFGStackify::cleanupFunctionData(MachineFunction &MF) {
   if (FakeCallerBB)
-    MF.DeleteMachineBasicBlock(FakeCallerBB);
+    MF.deleteMachineBasicBlock(FakeCallerBB);
   AppendixBB = FakeCallerBB = nullptr;
 }
 
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISD.def b/llvm/lib/Target/WebAssembly/WebAssemblyISD.def
index 1fa0ea3867c7..a3a33f4a5b3a 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISD.def
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISD.def
@@ -31,6 +31,7 @@ HANDLE_NODETYPE(SWIZZLE)
 HANDLE_NODETYPE(VEC_SHL)
 HANDLE_NODETYPE(VEC_SHR_S)
 HANDLE_NODETYPE(VEC_SHR_U)
+HANDLE_NODETYPE(NARROW_U)
 HANDLE_NODETYPE(EXTEND_LOW_S)
 HANDLE_NODETYPE(EXTEND_LOW_U)
 HANDLE_NODETYPE(EXTEND_HIGH_S)
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
index 0df8f3e0e09c..38ed4c73fb93 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
@@ -176,6 +176,8 @@ WebAssemblyTargetLowering::WebAssemblyTargetLowering(
     setTargetDAGCombine(ISD::FP_ROUND);
     setTargetDAGCombine(ISD::CONCAT_VECTORS);
 
+    setTargetDAGCombine(ISD::TRUNCATE);
+
     // Support saturating add for i8x16 and i16x8
     for (auto Op : {ISD::SADDSAT, ISD::UADDSAT})
       for (auto T : {MVT::v16i8, MVT::v8i16})
@@ -644,8 +646,7 @@ LowerCallResults(MachineInstr &CallResults, DebugLoc DL, MachineBasicBlock *BB,
     Register RegFuncref =
         MF.getRegInfo().createVirtualRegister(&WebAssembly::FUNCREFRegClass);
     MachineInstr *RefNull =
-        BuildMI(MF, DL, TII.get(WebAssembly::REF_NULL_FUNCREF), RegFuncref)
-            .addImm(static_cast<int32_t>(WebAssembly::HeapType::Funcref));
+        BuildMI(MF, DL, TII.get(WebAssembly::REF_NULL_FUNCREF), RegFuncref);
     BB->insertAfter(Const0->getIterator(), RefNull);
 
     MachineInstr *TableSet =
@@ -2610,6 +2611,114 @@ performVectorTruncZeroCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
   return DAG.getNode(Op, SDLoc(N), ResVT, Source);
 }
 
+// Helper to extract VectorWidth bits from Vec, starting from IdxVal.
+static SDValue extractSubVector(SDValue Vec, unsigned IdxVal, SelectionDAG &DAG,
+                                const SDLoc &DL, unsigned VectorWidth) {
+  EVT VT = Vec.getValueType();
+  EVT ElVT = VT.getVectorElementType();
+  unsigned Factor = VT.getSizeInBits() / VectorWidth;
+  EVT ResultVT = EVT::getVectorVT(*DAG.getContext(), ElVT,
+                                  VT.getVectorNumElements() / Factor);
+
+  // Extract the relevant VectorWidth bits.  Generate an EXTRACT_SUBVECTOR
+  unsigned ElemsPerChunk = VectorWidth / ElVT.getSizeInBits();
+  assert(isPowerOf2_32(ElemsPerChunk) && "Elements per chunk not power of 2");
+
+  // This is the index of the first element of the VectorWidth-bit chunk
+  // we want. Since ElemsPerChunk is a power of 2 just need to clear bits.
+  IdxVal &= ~(ElemsPerChunk - 1);
+
+  // If the input is a buildvector just emit a smaller one.
+  if (Vec.getOpcode() == ISD::BUILD_VECTOR)
+    return DAG.getBuildVector(ResultVT, DL,
+                              Vec->ops().slice(IdxVal, ElemsPerChunk));
+
+  SDValue VecIdx = DAG.getIntPtrConstant(IdxVal, DL);
+  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ResultVT, Vec, VecIdx);
+}
+
+// Helper to recursively truncate vector elements in half with NARROW_U. DstVT
+// is the expected destination value type after recursion. In is the initial
+// input. Note that the input should have enough leading zero bits to prevent
+// NARROW_U from saturating results.
+static SDValue truncateVectorWithNARROW(EVT DstVT, SDValue In, const SDLoc &DL,
+                                        SelectionDAG &DAG) {
+  EVT SrcVT = In.getValueType();
+
+  // No truncation required, we might get here due to recursive calls.
+  if (SrcVT == DstVT)
+    return In;
+
+  unsigned SrcSizeInBits = SrcVT.getSizeInBits();
+  unsigned NumElems = SrcVT.getVectorNumElements();
+  if (!isPowerOf2_32(NumElems))
+    return SDValue();
+  assert(DstVT.getVectorNumElements() == NumElems && "Illegal truncation");
+  assert(SrcSizeInBits > DstVT.getSizeInBits() && "Illegal truncation");
+
+  LLVMContext &Ctx = *DAG.getContext();
+  EVT PackedSVT = EVT::getIntegerVT(Ctx, SrcVT.getScalarSizeInBits() / 2);
+
+  // Narrow to the largest type possible:
+  // vXi64/vXi32 -> i16x8.narrow_i32x4_u and vXi16 -> i8x16.narrow_i16x8_u.
+  EVT InVT = MVT::i16, OutVT = MVT::i8;
+  if (SrcVT.getScalarSizeInBits() > 16) {
+    InVT = MVT::i32;
+    OutVT = MVT::i16;
+  }
+  unsigned SubSizeInBits = SrcSizeInBits / 2;
+  InVT = EVT::getVectorVT(Ctx, InVT, SubSizeInBits / InVT.getSizeInBits());
+  OutVT = EVT::getVectorVT(Ctx, OutVT, SubSizeInBits / OutVT.getSizeInBits());
+
+  // Split lower/upper subvectors.
+  SDValue Lo = extractSubVector(In, 0, DAG, DL, SubSizeInBits);
+  SDValue Hi = extractSubVector(In, NumElems / 2, DAG, DL, SubSizeInBits);
+
+  // 256bit -> 128bit truncate - Narrow lower/upper 128-bit subvectors.
+  if (SrcVT.is256BitVector() && DstVT.is128BitVector()) {
+    Lo = DAG.getBitcast(InVT, Lo);
+    Hi = DAG.getBitcast(InVT, Hi);
+    SDValue Res = DAG.getNode(WebAssemblyISD::NARROW_U, DL, OutVT, Lo, Hi);
+    return DAG.getBitcast(DstVT, Res);
+  }
+
+  // Recursively narrow lower/upper subvectors, concat result and narrow again.
+  EVT PackedVT = EVT::getVectorVT(Ctx, PackedSVT, NumElems / 2);
+  Lo = truncateVectorWithNARROW(PackedVT, Lo, DL, DAG);
+  Hi = truncateVectorWithNARROW(PackedVT, Hi, DL, DAG);
+
+  PackedVT = EVT::getVectorVT(Ctx, PackedSVT, NumElems);
+  SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, PackedVT, Lo, Hi);
+  return truncateVectorWithNARROW(DstVT, Res, DL, DAG);
+}
+
+static SDValue performTruncateCombine(SDNode *N,
+                                      TargetLowering::DAGCombinerInfo &DCI) {
+  auto &DAG = DCI.DAG;
+
+  SDValue In = N->getOperand(0);
+  EVT InVT = In.getValueType();
+  if (!InVT.isSimple())
+    return SDValue();
+
+  EVT OutVT = N->getValueType(0);
+  if (!OutVT.isVector())
+    return SDValue();
+
+  EVT OutSVT = OutVT.getVectorElementType();
+  EVT InSVT = InVT.getVectorElementType();
+  // Currently only cover truncate to v16i8 or v8i16.
+  if (!((InSVT == MVT::i16 || InSVT == MVT::i32 || InSVT == MVT::i64) &&
+        (OutSVT == MVT::i8 || OutSVT == MVT::i16) && OutVT.is128BitVector()))
+    return SDValue();
+
+  SDLoc DL(N);
+  APInt Mask = APInt::getLowBitsSet(InVT.getScalarSizeInBits(),
+                                    OutVT.getScalarSizeInBits());
+  In = DAG.getNode(ISD::AND, DL, InVT, In, DAG.getConstant(Mask, DL, InVT));
+  return truncateVectorWithNARROW(OutVT, In, DL, DAG);
+}
+
 SDValue
 WebAssemblyTargetLowering::PerformDAGCombine(SDNode *N,
                                              DAGCombinerInfo &DCI) const {
@@ -2626,5 +2735,7 @@ WebAssemblyTargetLowering::PerformDAGCombine(SDNode *N,
   case ISD::FP_ROUND:
   case ISD::CONCAT_VECTORS:
     return performVectorTruncZeroCombine(N, DCI);
+  case ISD::TRUNCATE:
+    return performTruncateCombine(N, DCI);
   }
 }
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrInfo.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrInfo.td
index ee9247a8bef9..3fb0af1d47a0 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrInfo.td
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrInfo.td
@@ -202,11 +202,6 @@ def Signature : Operand<i32> {
   let PrintMethod = "printWebAssemblySignatureOperand";
 }
 
-let OperandType = "OPERAND_HEAPTYPE" in
-def HeapType : Operand<i32> {
-  let PrintMethod = "printWebAssemblyHeapTypeOperand";
-}
-
 let OperandType = "OPERAND_TYPEINDEX" in
 def TypeIndex : Operand<i32>;
 
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrRef.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrRef.td
index ef9bd35d004a..76a88caafc47 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrRef.td
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrRef.td
@@ -11,13 +11,14 @@
 ///
 //===----------------------------------------------------------------------===//
 
-multiclass REF_I<WebAssemblyRegClass rc, ValueType vt> {
-  defm REF_NULL_#rc : I<(outs rc:$res), (ins HeapType:$heaptype),
-                        (outs), (ins HeapType:$heaptype),
-                        [],
-                        "ref.null\t$res, $heaptype",
-                        "ref.null\t$heaptype",
-                        0xd0>,
+multiclass REF_I<WebAssemblyRegClass rc, ValueType vt, string ht> {
+  defm REF_NULL_#rc : I<(outs rc:$dst), (ins),
+                        (outs), (ins),
+                        [(set rc:$dst, (!cast<Intrinsic>("int_wasm_ref_null_" # ht)))],
+                        "ref.null_" # ht # "$dst",
+                        "ref.null_" # ht,
+                        !cond(!eq(ht, "func")   : 0xd070, 
+                              !eq(ht, "extern") : 0xd06f)>,
                       Requires<[HasReferenceTypes]>;
   defm SELECT_#rc: I<(outs rc:$dst), (ins rc:$lhs, rc:$rhs, I32:$cond),
                      (outs), (ins),
@@ -28,8 +29,8 @@ multiclass REF_I<WebAssemblyRegClass rc, ValueType vt> {
                    Requires<[HasReferenceTypes]>;
 }
 
-defm "" : REF_I<FUNCREF, funcref>;
-defm "" : REF_I<EXTERNREF, externref>;
+defm "" : REF_I<FUNCREF, funcref, "func">;
+defm "" : REF_I<EXTERNREF, externref, "extern">;
 
 foreach rc = [FUNCREF, EXTERNREF] in {
 def : Pat<(select (i32 (setne I32:$cond, 0)), rc:$lhs, rc:$rhs),
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td
index 30b99c3a69a9..5bb12c7fbdc7 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td
@@ -1278,6 +1278,14 @@ multiclass SIMDNarrow<Vec vec, bits<32> baseInst> {
 defm "" : SIMDNarrow<I16x8, 101>;
 defm "" : SIMDNarrow<I32x4, 133>;
 
+// WebAssemblyISD::NARROW_U
+def wasm_narrow_t : SDTypeProfile<1, 2, []>;
+def wasm_narrow_u : SDNode<"WebAssemblyISD::NARROW_U", wasm_narrow_t>;
+def : Pat<(v16i8 (wasm_narrow_u (v8i16 V128:$left), (v8i16 V128:$right))),
+          (NARROW_U_I8x16 $left, $right)>;
+def : Pat<(v8i16 (wasm_narrow_u (v4i32 V128:$left), (v4i32 V128:$right))),
+          (NARROW_U_I16x8 $left, $right)>;
+
 // Bitcasts are nops
 // Matching bitcast t1 to t1 causes strange errors, so avoid repeating types
 foreach t1 = AllVecs in
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrTable.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrTable.td
index e44c2073eaeb..1fd00bf1cbc8 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrTable.td
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrTable.td
@@ -20,7 +20,7 @@ def WebAssemblyTableGet : SDNode<"WebAssemblyISD::TABLE_GET", WebAssemblyTableGe
                                  [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
 
 
-multiclass TABLE<WebAssemblyRegClass rc> {
+multiclass TABLE<WebAssemblyRegClass rc, string suffix> {
   let mayLoad = 1 in
   defm TABLE_GET_#rc : I<(outs rc:$res), (ins table32_op:$table, I32:$i),
                          (outs), (ins table32_op:$table),
@@ -39,14 +39,14 @@ multiclass TABLE<WebAssemblyRegClass rc> {
 
   defm TABLE_GROW_#rc : I<(outs I32:$sz), (ins table32_op:$table, rc:$val, I32:$n),
                           (outs), (ins table32_op:$table),
-                          [],
+                          [(set I32:$sz, (!cast<Intrinsic>("int_wasm_table_grow_" # suffix) (WebAssemblyWrapper tglobaladdr:$table), rc:$val, I32:$n))],
                           "table.grow\t$sz, $table, $val, $n",
                           "table.grow\t$table",
                           0xfc0f>;
 
   defm TABLE_FILL_#rc : I<(outs), (ins table32_op:$table, I32:$i, rc:$val, I32:$n),
                           (outs), (ins table32_op:$table),
-                          [],
+                          [(!cast<Intrinsic>("int_wasm_table_fill_" # suffix) (WebAssemblyWrapper tglobaladdr:$table), I32:$i, rc:$val, I32:$n)],
                           "table.fill\t$table, $i, $val, $n",
                           "table.fill\t$table",
                           0xfc11>;
@@ -62,8 +62,8 @@ multiclass TABLE<WebAssemblyRegClass rc> {
   }
 }
 
-defm "" : TABLE<FUNCREF>, Requires<[HasReferenceTypes]>;
-defm "" : TABLE<EXTERNREF>, Requires<[HasReferenceTypes]>;
+defm "" : TABLE<FUNCREF, "funcref">, Requires<[HasReferenceTypes]>;
+defm "" : TABLE<EXTERNREF, "externref">, Requires<[HasReferenceTypes]>;
 
 def : Pat<(WebAssemblyTableSet mcsym:$table, i32:$idx, funcref:$r),
           (TABLE_SET_FUNCREF mcsym:$table, i32:$idx, funcref:$r)>,
@@ -71,7 +71,7 @@ def : Pat<(WebAssemblyTableSet mcsym:$table, i32:$idx, funcref:$r),
 
 defm TABLE_SIZE : I<(outs I32:$sz), (ins table32_op:$table),
                     (outs), (ins table32_op:$table),
-                    [],
+                    [(set I32:$sz, (int_wasm_table_size (WebAssemblyWrapper tglobaladdr:$table)))],
                     "table.size\t$sz, $table",
                     "table.size\t$table",
                     0xfc10>,
@@ -80,7 +80,9 @@ defm TABLE_SIZE : I<(outs I32:$sz), (ins table32_op:$table),
 
 defm TABLE_COPY : I<(outs), (ins table32_op:$table1, table32_op:$table2, I32:$d, I32:$s, I32:$n),
                     (outs), (ins table32_op:$table1, table32_op:$table2),
-                    [],
+                    [(int_wasm_table_copy (WebAssemblyWrapper tglobaladdr:$table1),
+                                          (WebAssemblyWrapper tglobaladdr:$table2),
+                                          I32:$d, I32:$s, I32:$n)],
                     "table.copy\t$table1, $table2, $d, $s, $n",
                     "table.copy\t$table1, $table2",
                     0xfc0e>,
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp
index 4eacc921b6cd..23aaa5160abd 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp
@@ -267,6 +267,7 @@
 ///
 ///===----------------------------------------------------------------------===//
 
+#include "Utils/WebAssemblyUtilities.h"
 #include "WebAssembly.h"
 #include "WebAssemblyTargetMachine.h"
 #include "llvm/ADT/StringExtras.h"
@@ -285,13 +286,6 @@ using namespace llvm;
 
 #define DEBUG_TYPE "wasm-lower-em-ehsjlj"
 
-// Emscripten's asm.js-style exception handling
-extern cl::opt<bool> WasmEnableEmEH;
-// Emscripten's asm.js-style setjmp/longjmp handling
-extern cl::opt<bool> WasmEnableEmSjLj;
-// Wasm setjmp/longjmp handling using wasm EH instructions
-extern cl::opt<bool> WasmEnableSjLj;
-
 static cl::list<std::string>
     EHAllowlist("emscripten-cxx-exceptions-allowed",
                 cl::desc("The list of function names in which Emscripten-style "
@@ -370,8 +364,9 @@ public:
   static char ID;
 
   WebAssemblyLowerEmscriptenEHSjLj()
-      : ModulePass(ID), EnableEmEH(WasmEnableEmEH),
-        EnableEmSjLj(WasmEnableEmSjLj), EnableWasmSjLj(WasmEnableSjLj) {
+      : ModulePass(ID), EnableEmEH(WebAssembly::WasmEnableEmEH),
+        EnableEmSjLj(WebAssembly::WasmEnableEmSjLj),
+        EnableWasmSjLj(WebAssembly::WasmEnableSjLj) {
     assert(!(EnableEmSjLj && EnableWasmSjLj) &&
            "Two SjLj modes cannot be turned on at the same time");
     assert(!(EnableEmEH && EnableWasmSjLj) &&
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp
index 0b953a90aeab..09bccef17ab0 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp
@@ -40,9 +40,6 @@ cl::opt<bool>
                                " instruction output for test purposes only."),
                       cl::init(false));
 
-extern cl::opt<bool> WasmEnableEmEH;
-extern cl::opt<bool> WasmEnableEmSjLj;
-
 static void removeRegisterOperands(const MachineInstr *MI, MCInst &OutMI);
 
 MCSymbol *
@@ -66,9 +63,11 @@ WebAssemblyMCInstLower::GetGlobalAddressSymbol(const MachineOperand &MO) const {
       // they reach this point as aggregate Array types with an element type
       // that is a reference type.
       wasm::ValType Type;
+      bool IsTable = false;
       if (GlobalVT->isArrayTy() &&
           WebAssembly::isRefType(GlobalVT->getArrayElementType())) {
         MVT VT;
+        IsTable = true;
         switch (GlobalVT->getArrayElementType()->getPointerAddressSpace()) {
         case WebAssembly::WasmAddressSpace::WASM_ADDRESS_SPACE_FUNCREF:
           VT = MVT::funcref;
@@ -85,9 +84,14 @@ WebAssemblyMCInstLower::GetGlobalAddressSymbol(const MachineOperand &MO) const {
       } else
         report_fatal_error("Aggregate globals not yet implemented");
 
-      WasmSym->setType(wasm::WASM_SYMBOL_TYPE_GLOBAL);
-      WasmSym->setGlobalType(
-          wasm::WasmGlobalType{uint8_t(Type), /*Mutable=*/true});
+      if (IsTable) {
+        WasmSym->setType(wasm::WASM_SYMBOL_TYPE_TABLE);
+        WasmSym->setTableType(Type);
+      } else {
+        WasmSym->setType(wasm::WASM_SYMBOL_TYPE_GLOBAL);
+        WasmSym->setGlobalType(
+            wasm::WasmGlobalType{uint8_t(Type), /*Mutable=*/true});
+      }
     }
     return WasmSym;
   }
@@ -105,7 +109,8 @@ WebAssemblyMCInstLower::GetGlobalAddressSymbol(const MachineOperand &MO) const {
 
   bool InvokeDetected = false;
   auto *WasmSym = Printer.getMCSymbolForFunction(
-      F, WasmEnableEmEH || WasmEnableEmSjLj, Signature.get(), InvokeDetected);
+      F, WebAssembly::WasmEnableEmEH || WebAssembly::WasmEnableEmSjLj,
+      Signature.get(), InvokeDetected);
   WasmSym->setSignature(Signature.get());
   Printer.addSignature(std::move(Signature));
   WasmSym->setType(wasm::WASM_SYMBOL_TYPE_FUNCTION);
@@ -275,11 +280,6 @@ void WebAssemblyMCInstLower::lower(const MachineInstr *MI,
                                          SmallVector<wasm::ValType, 4>());
             break;
           }
-        } else if (Info.OperandType == WebAssembly::OPERAND_HEAPTYPE) {
-          assert(static_cast<WebAssembly::HeapType>(MO.getImm()) !=
-                 WebAssembly::HeapType::Invalid);
-          // With typed function references, this will need a case for type
-          // index operands.  Otherwise, fall through.
         }
       }
       MCOp = MCOperand::createImm(MO.getImm());
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp
index 7b70d99b5f52..482837178f3d 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp
@@ -14,6 +14,7 @@
 #include "WebAssemblyTargetMachine.h"
 #include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
 #include "TargetInfo/WebAssemblyTargetInfo.h"
+#include "Utils/WebAssemblyUtilities.h"
 #include "WebAssembly.h"
 #include "WebAssemblyMachineFunctionInfo.h"
 #include "WebAssemblyTargetObjectFile.h"
@@ -24,6 +25,7 @@
 #include "llvm/CodeGen/RegAllocRegistry.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/IR/Function.h"
+#include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/TargetRegistry.h"
 #include "llvm/Target/TargetOptions.h"
 #include "llvm/Transforms/Scalar.h"
@@ -33,28 +35,6 @@ using namespace llvm;
 
 #define DEBUG_TYPE "wasm"
 
-// Emscripten's asm.js-style exception handling
-cl::opt<bool>
-    WasmEnableEmEH("enable-emscripten-cxx-exceptions",
-                   cl::desc("WebAssembly Emscripten-style exception handling"),
-                   cl::init(false));
-
-// Emscripten's asm.js-style setjmp/longjmp handling
-cl::opt<bool> WasmEnableEmSjLj(
-    "enable-emscripten-sjlj",
-    cl::desc("WebAssembly Emscripten-style setjmp/longjmp handling"),
-    cl::init(false));
-
-// Exception handling using wasm EH instructions
-cl::opt<bool> WasmEnableEH("wasm-enable-eh",
-                           cl::desc("WebAssembly exception handling"),
-                           cl::init(false));
-
-// setjmp/longjmp handling using wasm EH instrutions
-cl::opt<bool> WasmEnableSjLj("wasm-enable-sjlj",
-                             cl::desc("WebAssembly setjmp/longjmp handling"),
-                             cl::init(false));
-
 // A command-line option to keep implicit locals
 // for the purpose of testing with lit/llc ONLY.
 // This produces output which is not valid WebAssembly, and is not supported
@@ -368,7 +348,23 @@ FunctionPass *WebAssemblyPassConfig::createTargetRegisterAllocator(bool) {
   return nullptr; // No reg alloc
 }
 
-static void basicCheckForEHAndSjLj(const TargetMachine *TM) {
+using WebAssembly::WasmEnableEH;
+using WebAssembly::WasmEnableEmEH;
+using WebAssembly::WasmEnableEmSjLj;
+using WebAssembly::WasmEnableSjLj;
+
+static void basicCheckForEHAndSjLj(TargetMachine *TM) {
+  // Before checking, we make sure TargetOptions.ExceptionModel is the same as
+  // MCAsmInfo.ExceptionsType. Normally these have to be the same, because clang
+  // stores the exception model info in LangOptions, which is later transferred
+  // to TargetOptions and MCAsmInfo. But when clang compiles bitcode directly,
+  // clang's LangOptions is not used and thus the exception model info is not
+  // correctly transferred to TargetOptions and MCAsmInfo, so we make sure we
+  // have the correct exception model in in WebAssemblyMCAsmInfo constructor.
+  // But in this case TargetOptions is still not updated, so we make sure they
+  // are the same.
+  TM->Options.ExceptionModel = TM->getMCAsmInfo()->getExceptionHandlingType();
+
   // Basic Correctness checking related to -exception-model
   if (TM->Options.ExceptionModel != ExceptionHandling::None &&
       TM->Options.ExceptionModel != ExceptionHandling::Wasm)
diff --git a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
index 8ce6b47d10e8..2ba0b97229cc 100644
--- a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
+++ b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
@@ -1759,7 +1759,8 @@ bool X86AsmParser::CreateMemForMSInlineAsm(
   // registers in a mmory expression, and though unaccessible via rip/eip.
   if (IsGlobalLV && (BaseReg || IndexReg)) {
     Operands.push_back(X86Operand::CreateMem(getPointerWidth(), Disp, Start,
-                                             End, Size, Identifier, Decl));
+                                             End, Size, Identifier, Decl,
+                                             FrontendSize));
     return false;
   }
   // Otherwise, we set the base register to a non-zero value
@@ -2551,8 +2552,6 @@ bool X86AsmParser::ParseIntelOperand(OperandVector &Operands) {
   StringRef ErrMsg;
   unsigned BaseReg = SM.getBaseReg();
   unsigned IndexReg = SM.getIndexReg();
-  if (IndexReg && BaseReg == X86::RIP)
-    BaseReg = 0;
   unsigned Scale = SM.getScale();
   if (!PtrInOperand)
     Size = SM.getElementSize() << 3;
@@ -4430,8 +4429,7 @@ bool X86AsmParser::MatchAndEmitATTInstruction(SMLoc IDLoc, unsigned &Opcode,
   // If exactly one matched, then we treat that as a successful match (and the
   // instruction will already have been filled in correctly, since the failing
   // matches won't have modified it).
-  unsigned NumSuccessfulMatches =
-      std::count(std::begin(Match), std::end(Match), Match_Success);
+  unsigned NumSuccessfulMatches = llvm::count(Match, Match_Success);
   if (NumSuccessfulMatches == 1) {
     if (!MatchingInlineAsm && validateInstruction(Inst, Operands))
       return true;
@@ -4479,7 +4477,7 @@ bool X86AsmParser::MatchAndEmitATTInstruction(SMLoc IDLoc, unsigned &Opcode,
 
   // If all of the instructions reported an invalid mnemonic, then the original
   // mnemonic was invalid.
-  if (std::count(std::begin(Match), std::end(Match), Match_MnemonicFail) == 4) {
+  if (llvm::count(Match, Match_MnemonicFail) == 4) {
     if (OriginalError == Match_MnemonicFail)
       return Error(IDLoc, "invalid instruction mnemonic '" + Base + "'",
                    Op.getLocRange(), MatchingInlineAsm);
@@ -4508,16 +4506,14 @@ bool X86AsmParser::MatchAndEmitATTInstruction(SMLoc IDLoc, unsigned &Opcode,
   }
 
   // If one instruction matched as unsupported, report this as unsupported.
-  if (std::count(std::begin(Match), std::end(Match),
-                 Match_Unsupported) == 1) {
+  if (llvm::count(Match, Match_Unsupported) == 1) {
     return Error(IDLoc, "unsupported instruction", EmptyRange,
                  MatchingInlineAsm);
   }
 
   // If one instruction matched with a missing feature, report this as a
   // missing feature.
-  if (std::count(std::begin(Match), std::end(Match),
-                 Match_MissingFeature) == 1) {
+  if (llvm::count(Match, Match_MissingFeature) == 1) {
     ErrorInfo = Match_MissingFeature;
     return ErrorMissingFeature(IDLoc, ErrorInfoMissingFeatures,
                                MatchingInlineAsm);
@@ -4525,8 +4521,7 @@ bool X86AsmParser::MatchAndEmitATTInstruction(SMLoc IDLoc, unsigned &Opcode,
 
   // If one instruction matched with an invalid operand, report this as an
   // operand failure.
-  if (std::count(std::begin(Match), std::end(Match),
-                 Match_InvalidOperand) == 1) {
+  if (llvm::count(Match, Match_InvalidOperand) == 1) {
     return Error(IDLoc, "invalid operand for instruction", EmptyRange,
                  MatchingInlineAsm);
   }
@@ -4674,8 +4669,7 @@ bool X86AsmParser::MatchAndEmitIntelInstruction(SMLoc IDLoc, unsigned &Opcode,
                  Op.getLocRange(), MatchingInlineAsm);
   }
 
-  unsigned NumSuccessfulMatches =
-      std::count(std::begin(Match), std::end(Match), Match_Success);
+  unsigned NumSuccessfulMatches = llvm::count(Match, Match_Success);
 
   // If matching was ambiguous and we had size information from the frontend,
   // try again with that. This handles cases like "movxz eax, m8/m16".
@@ -4721,16 +4715,14 @@ bool X86AsmParser::MatchAndEmitIntelInstruction(SMLoc IDLoc, unsigned &Opcode,
   }
 
   // If one instruction matched as unsupported, report this as unsupported.
-  if (std::count(std::begin(Match), std::end(Match),
-                 Match_Unsupported) == 1) {
+  if (llvm::count(Match, Match_Unsupported) == 1) {
     return Error(IDLoc, "unsupported instruction", EmptyRange,
                  MatchingInlineAsm);
   }
 
   // If one instruction matched with a missing feature, report this as a
   // missing feature.
-  if (std::count(std::begin(Match), std::end(Match),
-                 Match_MissingFeature) == 1) {
+  if (llvm::count(Match, Match_MissingFeature) == 1) {
     ErrorInfo = Match_MissingFeature;
     return ErrorMissingFeature(IDLoc, ErrorInfoMissingFeatures,
                                MatchingInlineAsm);
@@ -4738,14 +4730,12 @@ bool X86AsmParser::MatchAndEmitIntelInstruction(SMLoc IDLoc, unsigned &Opcode,
 
   // If one instruction matched with an invalid operand, report this as an
   // operand failure.
-  if (std::count(std::begin(Match), std::end(Match),
-                 Match_InvalidOperand) == 1) {
+  if (llvm::count(Match, Match_InvalidOperand) == 1) {
     return Error(IDLoc, "invalid operand for instruction", EmptyRange,
                  MatchingInlineAsm);
   }
 
-  if (std::count(std::begin(Match), std::end(Match),
-                 Match_InvalidImmUnsignedi4) == 1) {
+  if (llvm::count(Match, Match_InvalidImmUnsignedi4) == 1) {
     SMLoc ErrorLoc = ((X86Operand &)*Operands[ErrorInfo]).getStartLoc();
     if (ErrorLoc == SMLoc())
       ErrorLoc = IDLoc;
diff --git a/llvm/lib/Target/X86/AsmParser/X86Operand.h b/llvm/lib/Target/X86/AsmParser/X86Operand.h
index 9164c699b569..67b1244708a8 100644
--- a/llvm/lib/Target/X86/AsmParser/X86Operand.h
+++ b/llvm/lib/Target/X86/AsmParser/X86Operand.h
@@ -285,6 +285,12 @@ struct X86Operand final : public MCParsedAsmOperand {
 
   bool isOffsetOfLocal() const override { return isImm() && Imm.LocalRef; }
 
+  bool isMemPlaceholder(const MCInstrDesc &Desc) const override {
+    // Only MS InlineAsm uses global variables with registers rather than
+    // rip/eip.
+    return isMem() && !Mem.DefaultBaseReg && Mem.FrontendSize;
+  }
+
   bool needAddressOf() const override { return AddressOf; }
 
   bool isMem() const override { return Kind == Memory; }
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86InstComments.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86InstComments.cpp
index b51011e2c52f..a903c5f455a2 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86InstComments.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86InstComments.cpp
@@ -948,39 +948,39 @@ bool llvm::EmitAnyX86InstComments(const MCInst *MI, raw_ostream &OS,
     break;
 
   CASE_UNPCK(PUNPCKHBW, r)
-  case X86::MMX_PUNPCKHBWirr:
+  case X86::MMX_PUNPCKHBWrr:
     Src2Name = getRegName(MI->getOperand(NumOperands - 1).getReg());
     RegForm = true;
     LLVM_FALLTHROUGH;
 
   CASE_UNPCK(PUNPCKHBW, m)
-  case X86::MMX_PUNPCKHBWirm:
+  case X86::MMX_PUNPCKHBWrm:
     Src1Name = getRegName(MI->getOperand(NumOperands-(RegForm?2:6)).getReg());
     DestName = getRegName(MI->getOperand(0).getReg());
     DecodeUNPCKHMask(getRegOperandNumElts(MI, 8, 0), 8, ShuffleMask);
     break;
 
   CASE_UNPCK(PUNPCKHWD, r)
-  case X86::MMX_PUNPCKHWDirr:
+  case X86::MMX_PUNPCKHWDrr:
     Src2Name = getRegName(MI->getOperand(NumOperands - 1).getReg());
     RegForm = true;
     LLVM_FALLTHROUGH;
 
   CASE_UNPCK(PUNPCKHWD, m)
-  case X86::MMX_PUNPCKHWDirm:
+  case X86::MMX_PUNPCKHWDrm:
     Src1Name = getRegName(MI->getOperand(NumOperands-(RegForm?2:6)).getReg());
     DestName = getRegName(MI->getOperand(0).getReg());
     DecodeUNPCKHMask(getRegOperandNumElts(MI, 16, 0), 16, ShuffleMask);
     break;
 
   CASE_UNPCK(PUNPCKHDQ, r)
-  case X86::MMX_PUNPCKHDQirr:
+  case X86::MMX_PUNPCKHDQrr:
     Src2Name = getRegName(MI->getOperand(NumOperands - 1).getReg());
     RegForm = true;
     LLVM_FALLTHROUGH;
 
   CASE_UNPCK(PUNPCKHDQ, m)
-  case X86::MMX_PUNPCKHDQirm:
+  case X86::MMX_PUNPCKHDQrm:
     Src1Name = getRegName(MI->getOperand(NumOperands-(RegForm?2:6)).getReg());
     DestName = getRegName(MI->getOperand(0).getReg());
     DecodeUNPCKHMask(getRegOperandNumElts(MI, 32, 0), 32, ShuffleMask);
@@ -998,39 +998,39 @@ bool llvm::EmitAnyX86InstComments(const MCInst *MI, raw_ostream &OS,
     break;
 
   CASE_UNPCK(PUNPCKLBW, r)
-  case X86::MMX_PUNPCKLBWirr:
+  case X86::MMX_PUNPCKLBWrr:
     Src2Name = getRegName(MI->getOperand(NumOperands - 1).getReg());
     RegForm = true;
     LLVM_FALLTHROUGH;
 
   CASE_UNPCK(PUNPCKLBW, m)
-  case X86::MMX_PUNPCKLBWirm:
+  case X86::MMX_PUNPCKLBWrm:
     Src1Name = getRegName(MI->getOperand(NumOperands-(RegForm?2:6)).getReg());
     DestName = getRegName(MI->getOperand(0).getReg());
     DecodeUNPCKLMask(getRegOperandNumElts(MI, 8, 0), 8, ShuffleMask);
     break;
 
   CASE_UNPCK(PUNPCKLWD, r)
-  case X86::MMX_PUNPCKLWDirr:
+  case X86::MMX_PUNPCKLWDrr:
     Src2Name = getRegName(MI->getOperand(NumOperands - 1).getReg());
     RegForm = true;
     LLVM_FALLTHROUGH;
 
   CASE_UNPCK(PUNPCKLWD, m)
-  case X86::MMX_PUNPCKLWDirm:
+  case X86::MMX_PUNPCKLWDrm:
     Src1Name = getRegName(MI->getOperand(NumOperands-(RegForm?2:6)).getReg());
     DestName = getRegName(MI->getOperand(0).getReg());
     DecodeUNPCKLMask(getRegOperandNumElts(MI, 16, 0), 16, ShuffleMask);
     break;
 
   CASE_UNPCK(PUNPCKLDQ, r)
-  case X86::MMX_PUNPCKLDQirr:
+  case X86::MMX_PUNPCKLDQrr:
     Src2Name = getRegName(MI->getOperand(NumOperands - 1).getReg());
     RegForm = true;
     LLVM_FALLTHROUGH;
 
   CASE_UNPCK(PUNPCKLDQ, m)
-  case X86::MMX_PUNPCKLDQirm:
+  case X86::MMX_PUNPCKLDQrm:
     Src1Name = getRegName(MI->getOperand(NumOperands-(RegForm?2:6)).getReg());
     DestName = getRegName(MI->getOperand(0).getReg());
     DecodeUNPCKLMask(getRegOperandNumElts(MI, 32, 0), 32, ShuffleMask);
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86WinCOFFTargetStreamer.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86WinCOFFTargetStreamer.cpp
index 11251fb2b2ba..bf3f4e990ecc 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86WinCOFFTargetStreamer.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86WinCOFFTargetStreamer.cpp
@@ -236,7 +236,7 @@ bool X86WinCOFFTargetStreamer::emitFPOStackAlloc(unsigned StackAlloc, SMLoc L) {
 bool X86WinCOFFTargetStreamer::emitFPOStackAlign(unsigned Align, SMLoc L) {
   if (checkInFPOPrologue(L))
     return true;
-  if (!llvm::any_of(CurFPOData->Instructions, [](const FPOInstruction &Inst) {
+  if (llvm::none_of(CurFPOData->Instructions, [](const FPOInstruction &Inst) {
         return Inst.Op == FPOInstruction::SetFrame;
       })) {
     getContext().reportError(
diff --git a/llvm/lib/Target/X86/X86AsmPrinter.cpp b/llvm/lib/Target/X86/X86AsmPrinter.cpp
index 2e08482e4ff6..d48b8e458219 100644
--- a/llvm/lib/Target/X86/X86AsmPrinter.cpp
+++ b/llvm/lib/Target/X86/X86AsmPrinter.cpp
@@ -754,8 +754,6 @@ static void emitNonLazyStubs(MachineModuleInfo *MMI, MCStreamer &OutStreamer) {
 void X86AsmPrinter::emitEndOfAsmFile(Module &M) {
   const Triple &TT = TM.getTargetTriple();
 
-  emitAsanMemaccessSymbols(M);
-
   if (TT.isOSBinFormatMachO()) {
     // Mach-O uses non-lazy symbol stubs to encode per-TU information into
     // global table for symbol lookup.
diff --git a/llvm/lib/Target/X86/X86AsmPrinter.h b/llvm/lib/Target/X86/X86AsmPrinter.h
index 3b0983a7d935..b22f25af26cf 100644
--- a/llvm/lib/Target/X86/X86AsmPrinter.h
+++ b/llvm/lib/Target/X86/X86AsmPrinter.h
@@ -31,6 +31,7 @@ class LLVM_LIBRARY_VISIBILITY X86AsmPrinter : public AsmPrinter {
   FaultMaps FM;
   std::unique_ptr<MCCodeEmitter> CodeEmitter;
   bool EmitFPOData = false;
+  bool ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = false;
 
   // This utility class tracks the length of a stackmap instruction's 'shadow'.
   // It is used by the X86AsmPrinter to ensure that the stackmap shadow
@@ -100,20 +101,6 @@ class LLVM_LIBRARY_VISIBILITY X86AsmPrinter : public AsmPrinter {
 
   // Address sanitizer specific lowering for X86.
   void LowerASAN_CHECK_MEMACCESS(const MachineInstr &MI);
-  void emitAsanMemaccessSymbols(Module &M);
-  void emitAsanMemaccessPartial(Module &M, unsigned Reg,
-                                const ASanAccessInfo &AccessInfo,
-                                MCSubtargetInfo &STI);
-  void emitAsanMemaccessFull(Module &M, unsigned Reg,
-                             const ASanAccessInfo &AccessInfo,
-                             MCSubtargetInfo &STI);
-  void emitAsanReportError(Module &M, unsigned Reg,
-                           const ASanAccessInfo &AccessInfo,
-                           MCSubtargetInfo &STI);
-
-  typedef std::tuple<unsigned /*Reg*/, uint32_t /*AccessInfo*/>
-      AsanMemaccessTuple;
-  std::map<AsanMemaccessTuple, MCSymbol *> AsanMemaccessSymbols;
 
   // Choose between emitting .seh_ directives and .cv_fpo_ directives.
   void EmitSEHInstruction(const MachineInstr *MI);
@@ -165,6 +152,10 @@ public:
   bool runOnMachineFunction(MachineFunction &MF) override;
   void emitFunctionBodyStart() override;
   void emitFunctionBodyEnd() override;
+
+  bool shouldEmitWeakSwiftAsyncExtendedFramePointerFlags() const override {
+    return ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags;
+  }
 };
 
 } // end namespace llvm
diff --git a/llvm/lib/Target/X86/X86CmovConversion.cpp b/llvm/lib/Target/X86/X86CmovConversion.cpp
index 863438793acf..96d3d1390a59 100644
--- a/llvm/lib/Target/X86/X86CmovConversion.cpp
+++ b/llvm/lib/Target/X86/X86CmovConversion.cpp
@@ -186,7 +186,7 @@ bool X86CmovConverterPass::runOnMachineFunction(MachineFunction &MF) {
     if (collectCmovCandidates(Blocks, AllCmovGroups, /*IncludeLoads*/ true)) {
       for (auto &Group : AllCmovGroups) {
         // Skip any group that doesn't do at least one memory operand cmov.
-        if (!llvm::any_of(Group, [&](MachineInstr *I) { return I->mayLoad(); }))
+        if (llvm::none_of(Group, [&](MachineInstr *I) { return I->mayLoad(); }))
           continue;
 
         // For CMOV groups which we can rewrite and which contain a memory load,
diff --git a/llvm/lib/Target/X86/X86ExpandPseudo.cpp b/llvm/lib/Target/X86/X86ExpandPseudo.cpp
index 93bc23006dc4..6a047838f0b5 100644
--- a/llvm/lib/Target/X86/X86ExpandPseudo.cpp
+++ b/llvm/lib/Target/X86/X86ExpandPseudo.cpp
@@ -191,8 +191,6 @@ void X86ExpandPseudo::expandCALL_RVMARKER(MachineBasicBlock &MBB,
                                           MachineBasicBlock::iterator MBBI) {
   // Expand CALL_RVMARKER pseudo to call instruction, followed by the special
   //"movq %rax, %rdi" marker.
-  // TODO: Mark the sequence as bundle, to avoid passes moving other code
-  // in between.
   MachineInstr &MI = *MBBI;
 
   MachineInstr *OriginalCall;
@@ -236,15 +234,23 @@ void X86ExpandPseudo::expandCALL_RVMARKER(MachineBasicBlock &MBB,
   // Emit call to ObjC runtime.
   const uint32_t *RegMask =
       TRI->getCallPreservedMask(*MBB.getParent(), CallingConv::C);
-  BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(X86::CALL64pcrel32))
-      .addGlobalAddress(MI.getOperand(0).getGlobal(), 0, 0)
-      .addRegMask(RegMask)
-      .addReg(X86::RAX,
-              RegState::Implicit |
-                  (RAXImplicitDead ? (RegState::Dead | RegState::Define)
-                                   : RegState::Define))
-      .getInstr();
+  MachineInstr *RtCall =
+      BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(X86::CALL64pcrel32))
+          .addGlobalAddress(MI.getOperand(0).getGlobal(), 0, 0)
+          .addRegMask(RegMask)
+          .addReg(X86::RAX,
+                  RegState::Implicit |
+                      (RAXImplicitDead ? (RegState::Dead | RegState::Define)
+                                       : RegState::Define))
+          .getInstr();
   MI.eraseFromParent();
+
+  auto &TM = MBB.getParent()->getTarget();
+  // On Darwin platforms, wrap the expanded sequence in a bundle to prevent
+  // later optimizations from breaking up the sequence.
+  if (TM.getTargetTriple().isOSDarwin())
+    finalizeBundle(MBB, OriginalCall->getIterator(),
+                   std::next(RtCall->getIterator()));
 }
 
 /// If \p MBBI is a pseudo instruction, this method expands
diff --git a/llvm/lib/Target/X86/X86FastTileConfig.cpp b/llvm/lib/Target/X86/X86FastTileConfig.cpp
index 87c04a07cd13..47874e82ff3b 100644
--- a/llvm/lib/Target/X86/X86FastTileConfig.cpp
+++ b/llvm/lib/Target/X86/X86FastTileConfig.cpp
@@ -134,11 +134,7 @@ bool X86FastTileConfig::isAMXInstr(MachineInstr &MI) {
   if (MI.getOpcode() == X86::PLDTILECFGV || MI.isDebugInstr())
     return false;
 
-  for (MachineOperand &MO : MI.operands())
-    if (isTilePhysReg(MO))
-      return true;
-
-  return false;
+  return llvm::any_of(MI.operands(), isTilePhysReg);
 }
 
 MachineInstr *X86FastTileConfig::getKeyAMXInstr(MachineInstr *MI) {
diff --git a/llvm/lib/Target/X86/X86FixupBWInsts.cpp b/llvm/lib/Target/X86/X86FixupBWInsts.cpp
index e1d4b4c34772..16bff201dd03 100644
--- a/llvm/lib/Target/X86/X86FixupBWInsts.cpp
+++ b/llvm/lib/Target/X86/X86FixupBWInsts.cpp
@@ -457,14 +457,12 @@ void FixupBWInstPass::processBasicBlock(MachineFunction &MF,
   OptForSize = MF.getFunction().hasOptSize() ||
                llvm::shouldOptimizeForSize(&MBB, PSI, MBFI);
 
-  for (auto I = MBB.rbegin(); I != MBB.rend(); ++I) {
-    MachineInstr *MI = &*I;
-
-    if (MachineInstr *NewMI = tryReplaceInstr(MI, MBB))
-      MIReplacements.push_back(std::make_pair(MI, NewMI));
+  for (MachineInstr &MI : llvm::reverse(MBB)) {
+    if (MachineInstr *NewMI = tryReplaceInstr(&MI, MBB))
+      MIReplacements.push_back(std::make_pair(&MI, NewMI));
 
     // We're done with this instruction, update liveness for the next one.
-    LiveRegs.stepBackward(*MI);
+    LiveRegs.stepBackward(MI);
   }
 
   while (!MIReplacements.empty()) {
diff --git a/llvm/lib/Target/X86/X86FloatingPoint.cpp b/llvm/lib/Target/X86/X86FloatingPoint.cpp
index 4d9160f35226..2f0ab4ca9de4 100644
--- a/llvm/lib/Target/X86/X86FloatingPoint.cpp
+++ b/llvm/lib/Target/X86/X86FloatingPoint.cpp
@@ -1442,7 +1442,7 @@ void FPS::handleTwoArgFP(MachineBasicBlock::iterator &I) {
   assert(UpdatedSlot < StackTop && Dest < 7);
   Stack[UpdatedSlot]   = Dest;
   RegMap[Dest]         = UpdatedSlot;
-  MBB->getParent()->DeleteMachineInstr(&MI); // Remove the old instruction
+  MBB->getParent()->deleteMachineInstr(&MI); // Remove the old instruction
 }
 
 /// handleCompareFP - Handle FUCOM and FUCOMI instructions, which have two FP
diff --git a/llvm/lib/Target/X86/X86FrameLowering.cpp b/llvm/lib/Target/X86/X86FrameLowering.cpp
index c29ae9f6af4c..0a7aea467809 100644
--- a/llvm/lib/Target/X86/X86FrameLowering.cpp
+++ b/llvm/lib/Target/X86/X86FrameLowering.cpp
@@ -2496,8 +2496,8 @@ bool X86FrameLowering::assignCalleeSavedSpillSlots(
   }
 
   // Assign slots for GPRs. It increases frame size.
-  for (unsigned i = CSI.size(); i != 0; --i) {
-    unsigned Reg = CSI[i - 1].getReg();
+  for (CalleeSavedInfo &I : llvm::reverse(CSI)) {
+    unsigned Reg = I.getReg();
 
     if (!X86::GR64RegClass.contains(Reg) && !X86::GR32RegClass.contains(Reg))
       continue;
@@ -2506,15 +2506,15 @@ bool X86FrameLowering::assignCalleeSavedSpillSlots(
     CalleeSavedFrameSize += SlotSize;
 
     int SlotIndex = MFI.CreateFixedSpillStackObject(SlotSize, SpillSlotOffset);
-    CSI[i - 1].setFrameIdx(SlotIndex);
+    I.setFrameIdx(SlotIndex);
   }
 
   X86FI->setCalleeSavedFrameSize(CalleeSavedFrameSize);
   MFI.setCVBytesOfCalleeSavedRegisters(CalleeSavedFrameSize);
 
   // Assign slots for XMMs.
-  for (unsigned i = CSI.size(); i != 0; --i) {
-    unsigned Reg = CSI[i - 1].getReg();
+  for (CalleeSavedInfo &I : llvm::reverse(CSI)) {
+    unsigned Reg = I.getReg();
     if (X86::GR64RegClass.contains(Reg) || X86::GR32RegClass.contains(Reg))
       continue;
 
@@ -2533,7 +2533,7 @@ bool X86FrameLowering::assignCalleeSavedSpillSlots(
     // spill into slot
     SpillSlotOffset -= Size;
     int SlotIndex = MFI.CreateFixedSpillStackObject(Size, SpillSlotOffset);
-    CSI[i - 1].setFrameIdx(SlotIndex);
+    I.setFrameIdx(SlotIndex);
     MFI.ensureMaxAlignment(Alignment);
 
     // Save the start offset and size of XMM in stack frame for funclets.
@@ -2559,8 +2559,8 @@ bool X86FrameLowering::spillCalleeSavedRegisters(
   // Push GPRs. It increases frame size.
   const MachineFunction &MF = *MBB.getParent();
   unsigned Opc = STI.is64Bit() ? X86::PUSH64r : X86::PUSH32r;
-  for (unsigned i = CSI.size(); i != 0; --i) {
-    unsigned Reg = CSI[i - 1].getReg();
+  for (const CalleeSavedInfo &I : llvm::reverse(CSI)) {
+    unsigned Reg = I.getReg();
 
     if (!X86::GR64RegClass.contains(Reg) && !X86::GR32RegClass.contains(Reg))
       continue;
@@ -2593,8 +2593,8 @@ bool X86FrameLowering::spillCalleeSavedRegisters(
 
   // Make XMM regs spilled. X86 does not have ability of push/pop XMM.
   // It can be done by spilling XMMs to stack frame.
-  for (unsigned i = CSI.size(); i != 0; --i) {
-    unsigned Reg = CSI[i-1].getReg();
+  for (const CalleeSavedInfo &I : llvm::reverse(CSI)) {
+    unsigned Reg = I.getReg();
     if (X86::GR64RegClass.contains(Reg) || X86::GR32RegClass.contains(Reg))
       continue;
 
@@ -2607,8 +2607,7 @@ bool X86FrameLowering::spillCalleeSavedRegisters(
     MBB.addLiveIn(Reg);
     const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg, VT);
 
-    TII.storeRegToStackSlot(MBB, MI, Reg, true, CSI[i - 1].getFrameIdx(), RC,
-                            TRI);
+    TII.storeRegToStackSlot(MBB, MI, Reg, true, I.getFrameIdx(), RC, TRI);
     --MI;
     MI->setFlag(MachineInstr::FrameSetup);
     ++MI;
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 62b2387396be..6f6361b6757b 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -1091,17 +1091,11 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
       setOperationAction(ISD::SRL,              VT, Custom);
       setOperationAction(ISD::SHL,              VT, Custom);
       setOperationAction(ISD::SRA,              VT, Custom);
+      if (VT == MVT::v2i64) continue;
+      setOperationAction(ISD::ROTL,             VT, Custom);
+      setOperationAction(ISD::ROTR,             VT, Custom);
     }
 
-    setOperationAction(ISD::ROTL,               MVT::v4i32, Custom);
-    setOperationAction(ISD::ROTL,               MVT::v8i16, Custom);
-
-    // With 512-bit registers or AVX512VL+BW, expanding (and promoting the
-    // shifts) is better.
-    if (!Subtarget.useAVX512Regs() &&
-        !(Subtarget.hasBWI() && Subtarget.hasVLX()))
-      setOperationAction(ISD::ROTL,             MVT::v16i8, Custom);
-
     setOperationAction(ISD::STRICT_FSQRT,       MVT::v2f64, Legal);
     setOperationAction(ISD::STRICT_FADD,        MVT::v2f64, Legal);
     setOperationAction(ISD::STRICT_FSUB,        MVT::v2f64, Legal);
@@ -1199,8 +1193,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
 
   if (!Subtarget.useSoftFloat() && Subtarget.hasXOP()) {
     for (auto VT : { MVT::v16i8, MVT::v8i16,  MVT::v4i32, MVT::v2i64,
-                     MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 })
+                     MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
       setOperationAction(ISD::ROTL, VT, Custom);
+      setOperationAction(ISD::ROTR, VT, Custom);
+    }
 
     // XOP can efficiently perform BITREVERSE with VPPERM.
     for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 })
@@ -1283,6 +1279,9 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
       setOperationAction(ISD::SRL, VT, Custom);
       setOperationAction(ISD::SHL, VT, Custom);
       setOperationAction(ISD::SRA, VT, Custom);
+      if (VT == MVT::v4i64) continue;
+      setOperationAction(ISD::ROTL, VT, Custom);
+      setOperationAction(ISD::ROTR, VT, Custom);
     }
 
     // These types need custom splitting if their input is a 128-bit vector.
@@ -1291,13 +1290,6 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::ZERO_EXTEND,       MVT::v8i64,  Custom);
     setOperationAction(ISD::ZERO_EXTEND,       MVT::v16i32, Custom);
 
-    setOperationAction(ISD::ROTL,              MVT::v8i32,  Custom);
-    setOperationAction(ISD::ROTL,              MVT::v16i16, Custom);
-
-    // With BWI, expanding (and promoting the shifts) is the better.
-    if (!Subtarget.useBWIRegs())
-      setOperationAction(ISD::ROTL,            MVT::v32i8,  Custom);
-
     setOperationAction(ISD::SELECT,            MVT::v4f64, Custom);
     setOperationAction(ISD::SELECT,            MVT::v4i64, Custom);
     setOperationAction(ISD::SELECT,            MVT::v8i32, Custom);
@@ -1662,6 +1654,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
       setOperationAction(ISD::SRL,              VT, Custom);
       setOperationAction(ISD::SHL,              VT, Custom);
       setOperationAction(ISD::SRA,              VT, Custom);
+      setOperationAction(ISD::ROTL,             VT, Custom);
+      setOperationAction(ISD::ROTR,             VT, Custom);
       setOperationAction(ISD::SETCC,            VT, Custom);
 
       // The condition codes aren't legal in SSE/AVX and under AVX512 we use
@@ -1676,16 +1670,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
       setOperationAction(ISD::UMIN,             VT, Legal);
       setOperationAction(ISD::ABS,              VT, Legal);
       setOperationAction(ISD::CTPOP,            VT, Custom);
-      setOperationAction(ISD::ROTL,             VT, Custom);
-      setOperationAction(ISD::ROTR,             VT, Custom);
       setOperationAction(ISD::STRICT_FSETCC,    VT, Custom);
       setOperationAction(ISD::STRICT_FSETCCS,   VT, Custom);
     }
 
-    // With BWI, expanding (and promoting the shifts) is the better.
-    if (!Subtarget.useBWIRegs())
-      setOperationAction(ISD::ROTL, MVT::v32i16, Custom);
-
     for (auto VT : { MVT::v64i8, MVT::v32i16 }) {
       setOperationAction(ISD::ABS,     VT, HasBWI ? Legal : Custom);
       setOperationAction(ISD::CTPOP,   VT, Subtarget.hasBITALG() ? Legal : Custom);
@@ -5926,8 +5914,7 @@ static bool isSequentialOrUndefOrZeroInRange(ArrayRef<int> Mask, unsigned Pos,
 /// from position Pos and ending in Pos+Size is undef or is zero.
 static bool isUndefOrZeroInRange(ArrayRef<int> Mask, unsigned Pos,
                                  unsigned Size) {
-  return llvm::all_of(Mask.slice(Pos, Size),
-                      [](int M) { return isUndefOrZero(M); });
+  return llvm::all_of(Mask.slice(Pos, Size), isUndefOrZero);
 }
 
 /// Helper function to test whether a shuffle mask could be
@@ -6788,12 +6775,33 @@ void llvm::createSplat2ShuffleMask(MVT VT, SmallVectorImpl<int> &Mask,
   }
 }
 
+// Attempt to constant fold, else just create a VECTOR_SHUFFLE.
+static SDValue getVectorShuffle(SelectionDAG &DAG, EVT VT, const SDLoc &dl,
+                                SDValue V1, SDValue V2, ArrayRef<int> Mask) {
+  if ((ISD::isBuildVectorOfConstantSDNodes(V1.getNode()) || V1.isUndef()) &&
+      (ISD::isBuildVectorOfConstantSDNodes(V2.getNode()) || V2.isUndef())) {
+    SmallVector<SDValue> Ops(Mask.size(), DAG.getUNDEF(VT.getScalarType()));
+    for (int I = 0, NumElts = Mask.size(); I != NumElts; ++I) {
+      int M = Mask[I];
+      if (M < 0)
+        continue;
+      SDValue V = (M < NumElts) ? V1 : V2;
+      if (V.isUndef())
+        continue;
+      Ops[I] = V.getOperand(M % NumElts);
+    }
+    return DAG.getBuildVector(VT, dl, Ops);
+  }
+
+  return DAG.getVectorShuffle(VT, dl, V1, V2, Mask);
+}
+
 /// Returns a vector_shuffle node for an unpackl operation.
 static SDValue getUnpackl(SelectionDAG &DAG, const SDLoc &dl, EVT VT,
                           SDValue V1, SDValue V2) {
   SmallVector<int, 8> Mask;
   createUnpackShuffleMask(VT, Mask, /* Lo = */ true, /* Unary = */ false);
-  return DAG.getVectorShuffle(VT, dl, V1, V2, Mask);
+  return getVectorShuffle(DAG, VT, dl, V1, V2, Mask);
 }
 
 /// Returns a vector_shuffle node for an unpackh operation.
@@ -6801,12 +6809,11 @@ static SDValue getUnpackh(SelectionDAG &DAG, const SDLoc &dl, EVT VT,
                           SDValue V1, SDValue V2) {
   SmallVector<int, 8> Mask;
   createUnpackShuffleMask(VT, Mask, /* Lo = */ false, /* Unary = */ false);
-  return DAG.getVectorShuffle(VT, dl, V1, V2, Mask);
+  return getVectorShuffle(DAG, VT, dl, V1, V2, Mask);
 }
 
 /// Returns a node that packs the LHS + RHS nodes together at half width.
 /// May return X86ISD::PACKSS/PACKUS, packing the top/bottom half.
-/// TODO: Add vXi64 -> vXi32 pack support with vector_shuffle node.
 /// TODO: Add subvector splitting if/when we have a need for it.
 static SDValue getPack(SelectionDAG &DAG, const X86Subtarget &Subtarget,
                        const SDLoc &dl, MVT VT, SDValue LHS, SDValue RHS,
@@ -6818,9 +6825,24 @@ static SDValue getPack(SelectionDAG &DAG, const X86Subtarget &Subtarget,
          VT.getSizeInBits() == OpVT.getSizeInBits() &&
          (EltSizeInBits * 2) == OpVT.getScalarSizeInBits() &&
          "Unexpected PACK operand types");
-  assert((EltSizeInBits == 8 || EltSizeInBits == 16) &&
+  assert((EltSizeInBits == 8 || EltSizeInBits == 16 || EltSizeInBits == 32) &&
          "Unexpected PACK result type");
 
+  // Rely on vector shuffles for vXi64 -> vXi32 packing.
+  if (EltSizeInBits == 32) {
+    SmallVector<int> PackMask;
+    int Offset = PackHiHalf ? 1 : 0;
+    int NumElts = VT.getVectorNumElements();
+    for (int I = 0; I != NumElts; I += 4) {
+      PackMask.push_back(I + Offset);
+      PackMask.push_back(I + Offset + 2);
+      PackMask.push_back(I + Offset + NumElts);
+      PackMask.push_back(I + Offset + NumElts + 2);
+    }
+    return DAG.getVectorShuffle(VT, dl, DAG.getBitcast(VT, LHS),
+                                DAG.getBitcast(VT, RHS), PackMask);
+  }
+
   // See if we already have sufficient leading bits for PACKSS/PACKUS.
   if (!PackHiHalf) {
     if (UsePackUS &&
@@ -15192,12 +15214,10 @@ static SDValue lowerV8I16GeneralSingleInputShuffle(
       // need
       // to balance this to ensure we don't form a 3-1 shuffle in the other
       // half.
-      int NumFlippedAToBInputs =
-          std::count(AToBInputs.begin(), AToBInputs.end(), 2 * ADWord) +
-          std::count(AToBInputs.begin(), AToBInputs.end(), 2 * ADWord + 1);
-      int NumFlippedBToBInputs =
-          std::count(BToBInputs.begin(), BToBInputs.end(), 2 * BDWord) +
-          std::count(BToBInputs.begin(), BToBInputs.end(), 2 * BDWord + 1);
+      int NumFlippedAToBInputs = llvm::count(AToBInputs, 2 * ADWord) +
+                                 llvm::count(AToBInputs, 2 * ADWord + 1);
+      int NumFlippedBToBInputs = llvm::count(BToBInputs, 2 * BDWord) +
+                                 llvm::count(BToBInputs, 2 * BDWord + 1);
       if ((NumFlippedAToBInputs == 1 &&
            (NumFlippedBToBInputs == 0 || NumFlippedBToBInputs == 2)) ||
           (NumFlippedBToBInputs == 1 &&
@@ -25599,6 +25619,7 @@ static SDValue getTargetVShiftByConstNode(unsigned Opc, const SDLoc &dl, MVT VT,
 
 /// Handle vector element shifts where the shift amount may or may not be a
 /// constant. Takes immediate version of shift as input.
+/// TODO: Replace with vector + (splat) idx to avoid extract_element nodes.
 static SDValue getTargetVShiftNode(unsigned Opc, const SDLoc &dl, MVT VT,
                                    SDValue SrcOp, SDValue ShAmt,
                                    const X86Subtarget &Subtarget,
@@ -25606,11 +25627,6 @@ static SDValue getTargetVShiftNode(unsigned Opc, const SDLoc &dl, MVT VT,
   MVT SVT = ShAmt.getSimpleValueType();
   assert((SVT == MVT::i32 || SVT == MVT::i64) && "Unexpected value type!");
 
-  // Catch shift-by-constant.
-  if (ConstantSDNode *CShAmt = dyn_cast<ConstantSDNode>(ShAmt))
-    return getTargetVShiftByConstNode(Opc, dl, VT, SrcOp,
-                                      CShAmt->getZExtValue(), DAG);
-
   // Change opcode to non-immediate version.
   Opc = getTargetVShiftUniformOpcode(Opc, true);
 
@@ -26342,10 +26358,19 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
       return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32,
                          DAG.getBitcast(MVT::i16, Ins));
     }
-    case VSHIFT:
+    case VSHIFT: {
+      SDValue SrcOp = Op.getOperand(1);
+      SDValue ShAmt = Op.getOperand(2);
+
+      // Catch shift-by-constant.
+      if (auto *CShAmt = dyn_cast<ConstantSDNode>(ShAmt))
+        return getTargetVShiftByConstNode(IntrData->Opc0, dl,
+                                          Op.getSimpleValueType(), SrcOp,
+                                          CShAmt->getZExtValue(), DAG);
+
       return getTargetVShiftNode(IntrData->Opc0, dl, Op.getSimpleValueType(),
-                                 Op.getOperand(1), Op.getOperand(2), Subtarget,
-                                 DAG);
+                                 SrcOp, ShAmt, Subtarget, DAG);
+    }
     case COMPRESS_EXPAND_IN_REG: {
       SDValue Mask = Op.getOperand(3);
       SDValue DataToCompress = Op.getOperand(1);
@@ -26638,7 +26663,7 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
       X86CC = X86::COND_E;
       break;
     }
-    SmallVector<SDValue, 5> NewOps(Op->op_begin()+1, Op->op_end());
+    SmallVector<SDValue, 5> NewOps(llvm::drop_begin(Op->ops()));
     SDVTList VTs = DAG.getVTList(MVT::i32, MVT::v16i8, MVT::i32);
     SDValue PCMP = DAG.getNode(Opcode, dl, VTs, NewOps).getValue(2);
     SDValue SetCC = getSETCC(X86CC, PCMP, dl, DAG);
@@ -26653,7 +26678,7 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
     else
       Opcode = X86ISD::PCMPESTR;
 
-    SmallVector<SDValue, 5> NewOps(Op->op_begin()+1, Op->op_end());
+    SmallVector<SDValue, 5> NewOps(llvm::drop_begin(Op->ops()));
     SDVTList VTs = DAG.getVTList(MVT::i32, MVT::v16i8, MVT::i32);
     return DAG.getNode(Opcode, dl, VTs, NewOps);
   }
@@ -26666,7 +26691,7 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
     else
       Opcode = X86ISD::PCMPESTR;
 
-    SmallVector<SDValue, 5> NewOps(Op->op_begin()+1, Op->op_end());
+    SmallVector<SDValue, 5> NewOps(llvm::drop_begin(Op->ops()));
     SDVTList VTs = DAG.getVTList(MVT::i32, MVT::v16i8, MVT::i32);
     return DAG.getNode(Opcode, dl, VTs, NewOps).getValue(1);
   }
@@ -28892,10 +28917,13 @@ SDValue X86TargetLowering::LowerWin64_INT128_TO_FP(SDValue Op,
 // supported by the Subtarget
 static bool supportedVectorShiftWithImm(MVT VT, const X86Subtarget &Subtarget,
                                         unsigned Opcode) {
+  if (!(VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()))
+    return false;
+
   if (VT.getScalarSizeInBits() < 16)
     return false;
 
-  if (VT.is512BitVector() && Subtarget.hasAVX512() &&
+  if (VT.is512BitVector() && Subtarget.useAVX512Regs() &&
       (VT.getScalarSizeInBits() > 16 || Subtarget.hasBWI()))
     return true;
 
@@ -28919,6 +28947,8 @@ bool supportedVectorShiftWithBaseAmnt(MVT VT, const X86Subtarget &Subtarget,
 // natively supported by the Subtarget
 static bool supportedVectorVarShift(MVT VT, const X86Subtarget &Subtarget,
                                     unsigned Opcode) {
+  if (!(VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()))
+    return false;
 
   if (!Subtarget.hasInt256() || VT.getScalarSizeInBits() < 16)
     return false;
@@ -28927,7 +28957,8 @@ static bool supportedVectorVarShift(MVT VT, const X86Subtarget &Subtarget,
   if (VT.getScalarSizeInBits() == 16 && !Subtarget.hasBWI())
     return false;
 
-  if (Subtarget.hasAVX512())
+  if (Subtarget.hasAVX512() &&
+      (Subtarget.useAVX512Regs() || !VT.is512BitVector()))
     return true;
 
   bool LShift = VT.is128BitVector() || VT.is256BitVector();
@@ -28935,8 +28966,8 @@ static bool supportedVectorVarShift(MVT VT, const X86Subtarget &Subtarget,
   return (Opcode == ISD::SRA) ? AShift : LShift;
 }
 
-static SDValue LowerScalarImmediateShift(SDValue Op, SelectionDAG &DAG,
-                                         const X86Subtarget &Subtarget) {
+static SDValue LowerShiftByScalarImmediate(SDValue Op, SelectionDAG &DAG,
+                                           const X86Subtarget &Subtarget) {
   MVT VT = Op.getSimpleValueType();
   SDLoc dl(Op);
   SDValue R = Op.getOperand(0);
@@ -29066,8 +29097,8 @@ static SDValue LowerScalarImmediateShift(SDValue Op, SelectionDAG &DAG,
   return SDValue();
 }
 
-static SDValue LowerScalarVariableShift(SDValue Op, SelectionDAG &DAG,
-                                        const X86Subtarget &Subtarget) {
+static SDValue LowerShiftByScalarVariable(SDValue Op, SelectionDAG &DAG,
+                                          const X86Subtarget &Subtarget) {
   MVT VT = Op.getSimpleValueType();
   SDLoc dl(Op);
   SDValue R = Op.getOperand(0);
@@ -29166,28 +29197,20 @@ static SDValue convertShiftLeftToScale(SDValue Amt, const SDLoc &dl,
         (Subtarget.hasBWI() && VT == MVT::v64i8)))
     return SDValue();
 
-  if (ISD::isBuildVectorOfConstantSDNodes(Amt.getNode())) {
-    SmallVector<SDValue, 8> Elts;
-    MVT SVT = VT.getVectorElementType();
-    unsigned SVTBits = SVT.getSizeInBits();
-    APInt One(SVTBits, 1);
-    unsigned NumElems = VT.getVectorNumElements();
-
-    for (unsigned i = 0; i != NumElems; ++i) {
-      SDValue Op = Amt->getOperand(i);
-      if (Op->isUndef()) {
-        Elts.push_back(Op);
-        continue;
-      }
+  MVT SVT = VT.getVectorElementType();
+  unsigned SVTBits = SVT.getSizeInBits();
+  unsigned NumElems = VT.getVectorNumElements();
 
-      ConstantSDNode *ND = cast<ConstantSDNode>(Op);
-      APInt C(SVTBits, ND->getZExtValue());
-      uint64_t ShAmt = C.getZExtValue();
-      if (ShAmt >= SVTBits) {
-        Elts.push_back(DAG.getUNDEF(SVT));
+  APInt UndefElts;
+  SmallVector<APInt> EltBits;
+  if (getTargetConstantBitsFromNode(Amt, SVTBits, UndefElts, EltBits)) {
+    APInt One(SVTBits, 1);
+    SmallVector<SDValue> Elts(NumElems, DAG.getUNDEF(SVT));
+    for (unsigned I = 0; I != NumElems; ++I) {
+      if (UndefElts[I] || EltBits[I].uge(SVTBits))
         continue;
-      }
-      Elts.push_back(DAG.getConstant(One.shl(ShAmt), dl, SVT));
+      uint64_t ShAmt = EltBits[I].getZExtValue();
+      Elts[I] = DAG.getConstant(One.shl(ShAmt), dl, SVT);
     }
     return DAG.getBuildVector(VT, dl, Elts);
   }
@@ -29233,10 +29256,10 @@ static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget,
   assert(VT.isVector() && "Custom lowering only for vector shifts!");
   assert(Subtarget.hasSSE2() && "Only custom lower when we have SSE2!");
 
-  if (SDValue V = LowerScalarImmediateShift(Op, DAG, Subtarget))
+  if (SDValue V = LowerShiftByScalarImmediate(Op, DAG, Subtarget))
     return V;
 
-  if (SDValue V = LowerScalarVariableShift(Op, DAG, Subtarget))
+  if (SDValue V = LowerShiftByScalarVariable(Op, DAG, Subtarget))
     return V;
 
   if (supportedVectorVarShift(VT, Subtarget, Opc))
@@ -29818,14 +29841,29 @@ static SDValue LowerRotate(SDValue Op, const X86Subtarget &Subtarget,
     return DAG.getNode(FunnelOpc, DL, VT, R, R, Amt);
   }
 
-  assert(IsROTL && "Only ROTL supported");
+  SDValue Z = DAG.getConstant(0, DL, VT);
+
+  if (!IsROTL) {
+    // If the ISD::ROTR amount is constant, we're always better converting to
+    // ISD::ROTL.
+    if (SDValue NegAmt = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {Z, Amt}))
+      return DAG.getNode(ISD::ROTL, DL, VT, R, NegAmt);
+
+    // XOP targets always prefers ISD::ROTL.
+    if (Subtarget.hasXOP())
+      return DAG.getNode(ISD::ROTL, DL, VT, R,
+                         DAG.getNode(ISD::SUB, DL, VT, Z, Amt));
+  }
+
+  // Split 256-bit integers on XOP/pre-AVX2 targets.
+  if (VT.is256BitVector() && (Subtarget.hasXOP() || !Subtarget.hasAVX2()))
+    return splitVectorIntBinary(Op, DAG);
 
   // XOP has 128-bit vector variable + immediate rotates.
   // +ve/-ve Amt = rotate left/right - just need to handle ISD::ROTL.
   // XOP implicitly uses modulo rotation amounts.
   if (Subtarget.hasXOP()) {
-    if (VT.is256BitVector())
-      return splitVectorIntBinary(Op, DAG);
+    assert(IsROTL && "Only ROTL expected");
     assert(VT.is128BitVector() && "Only rotate 128-bit vectors!");
 
     // Attempt to rotate by immediate.
@@ -29839,54 +29877,88 @@ static SDValue LowerRotate(SDValue Op, const X86Subtarget &Subtarget,
     return Op;
   }
 
-  // Split 256-bit integers on pre-AVX2 targets.
-  if (VT.is256BitVector() && !Subtarget.hasAVX2())
-    return splitVectorIntBinary(Op, DAG);
-
-  assert((VT == MVT::v4i32 || VT == MVT::v8i16 || VT == MVT::v16i8 ||
-          ((VT == MVT::v8i32 || VT == MVT::v16i16 || VT == MVT::v32i8 ||
-            VT == MVT::v32i16) &&
-           Subtarget.hasAVX2())) &&
-         "Only vXi32/vXi16/vXi8 vector rotates supported");
-
   // Rotate by an uniform constant - expand back to shifts.
   if (IsCstSplat)
     return SDValue();
 
-  bool IsSplatAmt = DAG.isSplatValue(Amt);
-  SDValue AmtMask = DAG.getConstant(EltSizeInBits - 1, DL, VT);
-
-  // v16i8/v32i8: Split rotation into rot4/rot2/rot1 stages and select by
-  // the amount bit.
-  if (EltSizeInBits == 8) {
-    if (ISD::isBuildVectorOfConstantSDNodes(Amt.getNode()))
-      return SDValue();
+  // Split 512-bit integers on non 512-bit BWI targets.
+  if (VT.is512BitVector() && !Subtarget.useBWIRegs())
+    return splitVectorIntBinary(Op, DAG);
 
-    // Check for a hidden ISD::ROTR, vXi8 lowering can handle both, but we
-    // currently hit infinite loops in legalization if we allow ISD::ROTR.
-    // FIXME: Infinite ROTL<->ROTR legalization in TargetLowering::expandROT.
-    SDValue HiddenROTRAmt;
-    if (Amt.getOpcode() == ISD::SUB &&
-        ISD::isBuildVectorAllZeros(Amt.getOperand(0).getNode()))
-      HiddenROTRAmt = Amt.getOperand(1);
+  assert(
+      (VT == MVT::v4i32 || VT == MVT::v8i16 || VT == MVT::v16i8 ||
+       ((VT == MVT::v8i32 || VT == MVT::v16i16 || VT == MVT::v32i8) &&
+        Subtarget.hasAVX2()) ||
+       ((VT == MVT::v32i16 || VT == MVT::v64i8) && Subtarget.useBWIRegs())) &&
+      "Only vXi32/vXi16/vXi8 vector rotates supported");
 
-    MVT ExtVT = MVT::getVectorVT(MVT::i16, NumElts / 2);
+  MVT ExtSVT = MVT::getIntegerVT(2 * EltSizeInBits);
+  MVT ExtVT = MVT::getVectorVT(ExtSVT, NumElts / 2);
 
-    // If the amount is a splat, attempt to fold as unpack(x,x) << zext(y):
-    // rotl(x,y) -> (((aext(x) << bw) | zext(x)) << (y & (bw-1))) >> bw.
-    // rotr(x,y) -> (((aext(x) << bw) | zext(x)) >> (y & (bw-1))).
-    if (SDValue BaseRotAmt = DAG.getSplatValue(DAG.getNode(
-            ISD::AND, DL, VT, HiddenROTRAmt ? HiddenROTRAmt : Amt, AmtMask))) {
-      unsigned ShiftX86Opc = HiddenROTRAmt ? X86ISD::VSRLI : X86ISD::VSHLI;
-      BaseRotAmt = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, BaseRotAmt);
+  SDValue AmtMask = DAG.getConstant(EltSizeInBits - 1, DL, VT);
+  SDValue AmtMod = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask);
+
+  // Attempt to fold as unpack(x,x) << zext(splat(y)):
+  // rotl(x,y) -> (unpack(x,x) << (y & (bw-1))) >> bw.
+  // rotr(x,y) -> (unpack(x,x) >> (y & (bw-1))).
+  // TODO: Handle vXi16 cases.
+  if (EltSizeInBits == 8 || EltSizeInBits == 32) {
+    if (SDValue BaseRotAmt = DAG.getSplatValue(AmtMod)) {
+      unsigned ShiftX86Opc = IsROTL ? X86ISD::VSHLI : X86ISD::VSRLI;
       SDValue Lo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, R, R));
       SDValue Hi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, R, R));
+      BaseRotAmt = DAG.getZExtOrTrunc(BaseRotAmt, DL, MVT::i32);
       Lo = getTargetVShiftNode(ShiftX86Opc, DL, ExtVT, Lo, BaseRotAmt,
                                Subtarget, DAG);
       Hi = getTargetVShiftNode(ShiftX86Opc, DL, ExtVT, Hi, BaseRotAmt,
                                Subtarget, DAG);
-      return getPack(DAG, Subtarget, DL, VT, Lo, Hi, !HiddenROTRAmt);
+      return getPack(DAG, Subtarget, DL, VT, Lo, Hi, IsROTL);
     }
+  }
+
+  // v16i8/v32i8/v64i8: Split rotation into rot4/rot2/rot1 stages and select by
+  // the amount bit.
+  // TODO: We're doing nothing here that we couldn't do for funnel shifts.
+  if (EltSizeInBits == 8) {
+    bool IsConstAmt = ISD::isBuildVectorOfConstantSDNodes(Amt.getNode());
+    MVT WideVT =
+        MVT::getVectorVT(Subtarget.hasBWI() ? MVT::i16 : MVT::i32, NumElts);
+    unsigned ShiftOpc = IsROTL ? ISD::SHL : ISD::SRL;
+
+    // Attempt to fold as:
+    // rotl(x,y) -> (((aext(x) << bw) | zext(x)) << (y & (bw-1))) >> bw.
+    // rotr(x,y) -> (((aext(x) << bw) | zext(x)) >> (y & (bw-1))).
+    if (supportedVectorVarShift(WideVT, Subtarget, ShiftOpc) &&
+        supportedVectorShiftWithImm(WideVT, Subtarget, ShiftOpc)) {
+      // If we're rotating by constant, just use default promotion.
+      if (IsConstAmt)
+        return SDValue();
+      // See if we can perform this by widening to vXi16 or vXi32.
+      R = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, R);
+      R = DAG.getNode(
+          ISD::OR, DL, WideVT, R,
+          getTargetVShiftByConstNode(X86ISD::VSHLI, DL, WideVT, R, 8, DAG));
+      Amt = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, AmtMod);
+      R = DAG.getNode(ShiftOpc, DL, WideVT, R, Amt);
+      if (IsROTL)
+        R = getTargetVShiftByConstNode(X86ISD::VSRLI, DL, WideVT, R, 8, DAG);
+      return DAG.getNode(ISD::TRUNCATE, DL, VT, R);
+    }
+
+    // Attempt to fold as unpack(x,x) << zext(y):
+    // rotl(x,y) -> (unpack(x,x) << (y & (bw-1))) >> bw.
+    // rotr(x,y) -> (unpack(x,x) >> (y & (bw-1))).
+    if (IsConstAmt || supportedVectorVarShift(ExtVT, Subtarget, ShiftOpc)) {
+      // See if we can perform this by unpacking to lo/hi vXi16.
+      SDValue RLo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, R, R));
+      SDValue RHi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, R, R));
+      SDValue ALo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, AmtMod, Z));
+      SDValue AHi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, AmtMod, Z));
+      SDValue Lo = DAG.getNode(ShiftOpc, DL, ExtVT, RLo, ALo);
+      SDValue Hi = DAG.getNode(ShiftOpc, DL, ExtVT, RHi, AHi);
+      return getPack(DAG, Subtarget, DL, VT, Lo, Hi, IsROTL);
+    }
+    assert((VT == MVT::v16i8 || VT == MVT::v32i8) && "Unsupported vXi8 type");
 
     // We don't need ModuloAmt here as we just peek at individual bits.
     auto SignBitSelect = [&](MVT SelVT, SDValue Sel, SDValue V0, SDValue V1) {
@@ -29907,15 +29979,15 @@ static SDValue LowerRotate(SDValue Op, const X86Subtarget &Subtarget,
       return DAG.getSelect(DL, SelVT, C, V0, V1);
     };
 
-    // 'Hidden' ROTR is currently only profitable on AVX512 targets where we
-    // have VPTERNLOG.
-    unsigned ShiftLHS = ISD::SHL;
-    unsigned ShiftRHS = ISD::SRL;
-    if (HiddenROTRAmt && useVPTERNLOG(Subtarget, VT)) {
-      std::swap(ShiftLHS, ShiftRHS);
-      Amt = HiddenROTRAmt;
+    // ISD::ROTR is currently only profitable on AVX512 targets with VPTERNLOG.
+    if (!IsROTL && !useVPTERNLOG(Subtarget, VT)) {
+      Amt = DAG.getNode(ISD::SUB, DL, VT, Z, Amt);
+      IsROTL = true;
     }
 
+    unsigned ShiftLHS = IsROTL ? ISD::SHL : ISD::SRL;
+    unsigned ShiftRHS = IsROTL ? ISD::SRL : ISD::SHL;
+
     // Turn 'a' into a mask suitable for VSELECT: a = a << 5;
     // We can safely do this using i16 shifts as we're only interested in
     // the 3 lower bits of each byte.
@@ -29952,18 +30024,7 @@ static SDValue LowerRotate(SDValue Op, const X86Subtarget &Subtarget,
     return SignBitSelect(VT, Amt, M, R);
   }
 
-  // ISD::ROT* uses modulo rotate amounts.
-  if (SDValue BaseRotAmt = DAG.getSplatValue(Amt)) {
-    // If the amount is a splat, perform the modulo BEFORE the splat,
-    // this helps LowerScalarVariableShift to remove the splat later.
-    Amt = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, BaseRotAmt);
-    Amt = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask);
-    Amt = DAG.getVectorShuffle(VT, DL, Amt, DAG.getUNDEF(VT),
-                               SmallVector<int>(NumElts, 0));
-  } else {
-    Amt = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask);
-  }
-
+  bool IsSplatAmt = DAG.isSplatValue(Amt);
   bool ConstantAmt = ISD::isBuildVectorOfConstantSDNodes(Amt.getNode());
   bool LegalVarShifts = supportedVectorVarShift(VT, Subtarget, ISD::SHL) &&
                         supportedVectorVarShift(VT, Subtarget, ISD::SRL);
@@ -29971,13 +30032,25 @@ static SDValue LowerRotate(SDValue Op, const X86Subtarget &Subtarget,
   // Fallback for splats + all supported variable shifts.
   // Fallback for non-constants AVX2 vXi16 as well.
   if (IsSplatAmt || LegalVarShifts || (Subtarget.hasAVX2() && !ConstantAmt)) {
+    Amt = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask);
     SDValue AmtR = DAG.getConstant(EltSizeInBits, DL, VT);
     AmtR = DAG.getNode(ISD::SUB, DL, VT, AmtR, Amt);
-    SDValue SHL = DAG.getNode(ISD::SHL, DL, VT, R, Amt);
-    SDValue SRL = DAG.getNode(ISD::SRL, DL, VT, R, AmtR);
+    SDValue SHL = DAG.getNode(IsROTL ? ISD::SHL : ISD::SRL, DL, VT, R, Amt);
+    SDValue SRL = DAG.getNode(IsROTL ? ISD::SRL : ISD::SHL, DL, VT, R, AmtR);
     return DAG.getNode(ISD::OR, DL, VT, SHL, SRL);
   }
 
+  // Everything below assumes ISD::ROTL.
+  if (!IsROTL) {
+    Amt = DAG.getNode(ISD::SUB, DL, VT, Z, Amt);
+    IsROTL = true;
+  }
+
+  // ISD::ROT* uses modulo rotate amounts.
+  Amt = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask);
+
+  assert(IsROTL && "Only ROTL supported");
+
   // As with shifts, attempt to convert the rotation amount to a multiplication
   // factor, fallback to general expansion.
   SDValue Scale = convertShiftLeftToScale(Amt, DL, Subtarget, DAG);
@@ -32927,11 +33000,6 @@ bool X86TargetLowering::isLegalAddressingMode(const DataLayout &DL,
 bool X86TargetLowering::isVectorShiftByScalarCheap(Type *Ty) const {
   unsigned Bits = Ty->getScalarSizeInBits();
 
-  // 8-bit shifts are always expensive, but versions with a scalar amount aren't
-  // particularly cheaper than those without.
-  if (Bits == 8)
-    return false;
-
   // XOP has v16i8/v8i16/v4i32/v2i64 variable vector shifts.
   // Splitting for v32i8/v16i16 on XOP+AVX2 targets is still preferred.
   if (Subtarget.hasXOP() &&
@@ -36249,9 +36317,10 @@ static bool matchUnaryShuffle(MVT MaskVT, ArrayRef<int> Mask,
         (V1.getOpcode() == ISD::SCALAR_TO_VECTOR &&
          isUndefOrZeroInRange(Mask, 1, NumMaskElts - 1))) {
       Shuffle = X86ISD::VZEXT_MOVL;
-      SrcVT = DstVT = MaskEltSize == 16      ? MVT::v8f16
-                      : !Subtarget.hasSSE2() ? MVT::v4f32
-                                             : MaskVT;
+      if (MaskEltSize == 16)
+        SrcVT = DstVT = MaskVT.changeVectorElementType(MVT::f16);
+      else
+        SrcVT = DstVT = !Subtarget.hasSSE2() ? MVT::v4f32 : MaskVT;
       return true;
     }
   }
@@ -36300,9 +36369,10 @@ static bool matchUnaryShuffle(MVT MaskVT, ArrayRef<int> Mask,
       isUndefOrEqual(Mask[0], 0) &&
       isUndefOrZeroInRange(Mask, 1, NumMaskElts - 1)) {
     Shuffle = X86ISD::VZEXT_MOVL;
-    SrcVT = DstVT = MaskEltSize == 16      ? MVT::v8f16
-                    : !Subtarget.hasSSE2() ? MVT::v4f32
-                                           : MaskVT;
+    if (MaskEltSize == 16)
+      SrcVT = DstVT = MaskVT.changeVectorElementType(MVT::f16);
+    else
+      SrcVT = DstVT = !Subtarget.hasSSE2() ? MVT::v4f32 : MaskVT;
     return true;
   }
 
@@ -40981,6 +41051,28 @@ SDValue X86TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(
       Op, DemandedBits, DemandedElts, DAG, Depth);
 }
 
+bool X86TargetLowering::isSplatValueForTargetNode(SDValue Op,
+                                                  const APInt &DemandedElts,
+                                                  APInt &UndefElts,
+                                                  unsigned Depth) const {
+  unsigned NumElts = DemandedElts.getBitWidth();
+  unsigned Opc = Op.getOpcode();
+
+  switch (Opc) {
+  case X86ISD::VBROADCAST:
+  case X86ISD::VBROADCAST_LOAD:
+    // TODO: Permit vXi64 types on 32-bit targets.
+    if (isTypeLegal(Op.getValueType().getVectorElementType())) {
+      UndefElts = APInt::getNullValue(NumElts);
+      return true;
+    }
+    return false;
+  }
+
+  return TargetLowering::isSplatValueForTargetNode(Op, DemandedElts, UndefElts,
+                                                   Depth);
+}
+
 // Helper to peek through bitops/trunc/setcc to determine size of source vector.
 // Allows combineBitcastvxi1 to determine what size vector generated a <X x i1>.
 static bool checkBitcastSrcVectorSize(SDValue Src, unsigned Size,
@@ -46204,25 +46296,27 @@ static SDValue combineScalarAndWithMaskSetcc(SDNode *N, SelectionDAG &DAG,
 static SDValue combineAnd(SDNode *N, SelectionDAG &DAG,
                           TargetLowering::DAGCombinerInfo &DCI,
                           const X86Subtarget &Subtarget) {
+  SDValue N0 = N->getOperand(0);
+  SDValue N1 = N->getOperand(1);
   EVT VT = N->getValueType(0);
+  SDLoc dl(N);
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
 
   // If this is SSE1 only convert to FAND to avoid scalarization.
   if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32) {
-    return DAG.getBitcast(
-        MVT::v4i32, DAG.getNode(X86ISD::FAND, SDLoc(N), MVT::v4f32,
-                                DAG.getBitcast(MVT::v4f32, N->getOperand(0)),
-                                DAG.getBitcast(MVT::v4f32, N->getOperand(1))));
+    return DAG.getBitcast(MVT::v4i32,
+                          DAG.getNode(X86ISD::FAND, dl, MVT::v4f32,
+                                      DAG.getBitcast(MVT::v4f32, N0),
+                                      DAG.getBitcast(MVT::v4f32, N1)));
   }
 
   // Use a 32-bit and+zext if upper bits known zero.
-  if (VT == MVT::i64 && Subtarget.is64Bit() &&
-      !isa<ConstantSDNode>(N->getOperand(1))) {
+  if (VT == MVT::i64 && Subtarget.is64Bit() && !isa<ConstantSDNode>(N1)) {
     APInt HiMask = APInt::getHighBitsSet(64, 32);
-    if (DAG.MaskedValueIsZero(N->getOperand(1), HiMask) ||
-        DAG.MaskedValueIsZero(N->getOperand(0), HiMask)) {
-      SDLoc dl(N);
-      SDValue LHS = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, N->getOperand(0));
-      SDValue RHS = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, N->getOperand(1));
+    if (DAG.MaskedValueIsZero(N1, HiMask) ||
+        DAG.MaskedValueIsZero(N0, HiMask)) {
+      SDValue LHS = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, N0);
+      SDValue RHS = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, N1);
       return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64,
                          DAG.getNode(ISD::AND, dl, MVT::i32, LHS, RHS));
     }
@@ -46235,8 +46329,6 @@ static SDValue combineAnd(SDNode *N, SelectionDAG &DAG,
     SmallVector<APInt, 2> SrcPartials;
     if (matchScalarReduction(SDValue(N, 0), ISD::AND, SrcOps, &SrcPartials) &&
         SrcOps.size() == 1) {
-      SDLoc dl(N);
-      const TargetLowering &TLI = DAG.getTargetLoweringInfo();
       unsigned NumElts = SrcOps[0].getValueType().getVectorNumElements();
       EVT MaskVT = EVT::getIntegerVT(*DAG.getContext(), NumElts);
       SDValue Mask = combineBitcastvxi1(DAG, MaskVT, SrcOps[0], dl, Subtarget);
@@ -46276,33 +46368,57 @@ static SDValue combineAnd(SDNode *N, SelectionDAG &DAG,
   if (SDValue R = combineAndLoadToBZHI(N, DAG, Subtarget))
     return R;
 
-  // Attempt to recursively combine a bitmask AND with shuffles.
   if (VT.isVector() && (VT.getScalarSizeInBits() % 8) == 0) {
+    // Attempt to recursively combine a bitmask AND with shuffles.
     SDValue Op(N, 0);
     if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
       return Res;
+
+    // If either operand is a constant mask, then only the elements that aren't
+    // zero are actually demanded by the other operand.
+    auto SimplifyUndemandedElts = [&](SDValue Op, SDValue OtherOp) {
+      APInt UndefElts;
+      SmallVector<APInt> EltBits;
+      int NumElts = VT.getVectorNumElements();
+      int EltSizeInBits = VT.getScalarSizeInBits();
+      if (!getTargetConstantBitsFromNode(Op, EltSizeInBits, UndefElts, EltBits))
+        return false;
+
+      APInt DemandedElts = APInt::getZero(NumElts);
+      for (int I = 0; I != NumElts; ++I)
+        if (!EltBits[I].isZero())
+          DemandedElts.setBit(I);
+
+      APInt KnownUndef, KnownZero;
+      return TLI.SimplifyDemandedVectorElts(OtherOp, DemandedElts, KnownUndef,
+                                            KnownZero, DCI);
+    };
+    if (SimplifyUndemandedElts(N0, N1) || SimplifyUndemandedElts(N1, N0)) {
+      if (N->getOpcode() != ISD::DELETED_NODE)
+        DCI.AddToWorklist(N);
+      return SDValue(N, 0);
+    }
   }
 
   // Attempt to combine a scalar bitmask AND with an extracted shuffle.
   if ((VT.getScalarSizeInBits() % 8) == 0 &&
-      N->getOperand(0).getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
-      isa<ConstantSDNode>(N->getOperand(0).getOperand(1))) {
-    SDValue BitMask = N->getOperand(1);
-    SDValue SrcVec = N->getOperand(0).getOperand(0);
+      N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
+      isa<ConstantSDNode>(N0.getOperand(1))) {
+    SDValue BitMask = N1;
+    SDValue SrcVec = N0.getOperand(0);
     EVT SrcVecVT = SrcVec.getValueType();
 
     // Check that the constant bitmask masks whole bytes.
     APInt UndefElts;
     SmallVector<APInt, 64> EltBits;
-    if (VT == SrcVecVT.getScalarType() &&
-        N->getOperand(0)->isOnlyUserOf(SrcVec.getNode()) &&
+    if (VT == SrcVecVT.getScalarType() && N0->isOnlyUserOf(SrcVec.getNode()) &&
         getTargetConstantBitsFromNode(BitMask, 8, UndefElts, EltBits) &&
         llvm::all_of(EltBits, [](const APInt &M) {
           return M.isZero() || M.isAllOnes();
         })) {
       unsigned NumElts = SrcVecVT.getVectorNumElements();
       unsigned Scale = SrcVecVT.getScalarSizeInBits() / 8;
-      unsigned Idx = N->getOperand(0).getConstantOperandVal(1);
+      unsigned Idx = N0.getConstantOperandVal(1);
 
       // Create a root shuffle mask from the byte mask and the extracted index.
       SmallVector<int, 16> ShuffleMask(NumElts * Scale, SM_SentinelUndef);
@@ -46318,8 +46434,8 @@ static SDValue combineAnd(SDNode *N, SelectionDAG &DAG,
               X86::MaxShuffleCombineDepth,
               /*HasVarMask*/ false, /*AllowVarCrossLaneMask*/ true,
               /*AllowVarPerLaneMask*/ true, DAG, Subtarget))
-        return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), VT, Shuffle,
-                           N->getOperand(0).getOperand(1));
+        return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Shuffle,
+                           N0.getOperand(1));
     }
   }
 
@@ -46644,11 +46760,13 @@ static SDValue combineOr(SDNode *N, SelectionDAG &DAG,
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
   EVT VT = N->getValueType(0);
+  SDLoc dl(N);
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
 
   // If this is SSE1 only convert to FOR to avoid scalarization.
   if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32) {
     return DAG.getBitcast(MVT::v4i32,
-                          DAG.getNode(X86ISD::FOR, SDLoc(N), MVT::v4f32,
+                          DAG.getNode(X86ISD::FOR, dl, MVT::v4f32,
                                       DAG.getBitcast(MVT::v4f32, N0),
                                       DAG.getBitcast(MVT::v4f32, N1)));
   }
@@ -46660,8 +46778,6 @@ static SDValue combineOr(SDNode *N, SelectionDAG &DAG,
     SmallVector<APInt, 2> SrcPartials;
     if (matchScalarReduction(SDValue(N, 0), ISD::OR, SrcOps, &SrcPartials) &&
         SrcOps.size() == 1) {
-      SDLoc dl(N);
-      const TargetLowering &TLI = DAG.getTargetLoweringInfo();
       unsigned NumElts = SrcOps[0].getValueType().getVectorNumElements();
       EVT MaskVT = EVT::getIntegerVT(*DAG.getContext(), NumElts);
       SDValue Mask = combineBitcastvxi1(DAG, MaskVT, SrcOps[0], dl, Subtarget);
@@ -46707,7 +46823,6 @@ static SDValue combineOr(SDNode *N, SelectionDAG &DAG,
     if (NumElts >= 16 && N1.getOpcode() == X86ISD::KSHIFTL &&
         N1.getConstantOperandAPInt(1) == HalfElts &&
         DAG.MaskedValueIsZero(N0, APInt(1, 1), UpperElts)) {
-      SDLoc dl(N);
       return DAG.getNode(
           ISD::CONCAT_VECTORS, dl, VT,
           extractSubVector(N0, 0, DAG, dl, HalfElts),
@@ -46716,7 +46831,6 @@ static SDValue combineOr(SDNode *N, SelectionDAG &DAG,
     if (NumElts >= 16 && N0.getOpcode() == X86ISD::KSHIFTL &&
         N0.getConstantOperandAPInt(1) == HalfElts &&
         DAG.MaskedValueIsZero(N1, APInt(1, 1), UpperElts)) {
-      SDLoc dl(N);
       return DAG.getNode(
           ISD::CONCAT_VECTORS, dl, VT,
           extractSubVector(N1, 0, DAG, dl, HalfElts),
@@ -46724,11 +46838,36 @@ static SDValue combineOr(SDNode *N, SelectionDAG &DAG,
     }
   }
 
-  // Attempt to recursively combine an OR of shuffles.
   if (VT.isVector() && (VT.getScalarSizeInBits() % 8) == 0) {
+    // Attempt to recursively combine an OR of shuffles.
     SDValue Op(N, 0);
     if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
       return Res;
+
+    // If either operand is a constant mask, then only the elements that aren't
+    // allones are actually demanded by the other operand.
+    auto SimplifyUndemandedElts = [&](SDValue Op, SDValue OtherOp) {
+      APInt UndefElts;
+      SmallVector<APInt> EltBits;
+      int NumElts = VT.getVectorNumElements();
+      int EltSizeInBits = VT.getScalarSizeInBits();
+      if (!getTargetConstantBitsFromNode(Op, EltSizeInBits, UndefElts, EltBits))
+        return false;
+
+      APInt DemandedElts = APInt::getZero(NumElts);
+      for (int I = 0; I != NumElts; ++I)
+        if (!EltBits[I].isAllOnes())
+          DemandedElts.setBit(I);
+
+      APInt KnownUndef, KnownZero;
+      return TLI.SimplifyDemandedVectorElts(OtherOp, DemandedElts, KnownUndef,
+                                            KnownZero, DCI);
+    };
+    if (SimplifyUndemandedElts(N0, N1) || SimplifyUndemandedElts(N1, N0)) {
+      if (N->getOpcode() != ISD::DELETED_NODE)
+        DCI.AddToWorklist(N);
+      return SDValue(N, 0);
+    }
   }
 
   // We should fold "masked merge" patterns when `andn` is not available.
@@ -52111,7 +52250,7 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
     case X86ISD::VSHLI:
     case X86ISD::VSRLI:
       // Special case: SHL/SRL AVX1 V4i64 by 32-bits can lower as a shuffle.
-      // TODO: Move this to LowerScalarImmediateShift?
+      // TODO: Move this to LowerShiftByScalarImmediate?
       if (VT == MVT::v4i64 && !Subtarget.hasInt256() &&
           llvm::all_of(Ops, [](SDValue Op) {
             return Op.getConstantOperandAPInt(1) == 32;
diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h
index 6805cb75f0f2..d1d6e319f16b 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.h
+++ b/llvm/lib/Target/X86/X86ISelLowering.h
@@ -1100,6 +1100,12 @@ namespace llvm {
 
     bool shouldSplatInsEltVarIndex(EVT VT) const override;
 
+    bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override {
+      // Converting to sat variants holds little benefit on X86 as we will just
+      // need to saturate the value back using fp arithmatic.
+      return Op != ISD::FP_TO_UINT_SAT && isOperationLegalOrCustom(Op, VT);
+    }
+
     bool convertSetCCLogicToBitwiseLogic(EVT VT) const override {
       return VT.isScalarInteger();
     }
@@ -1153,6 +1159,10 @@ namespace llvm {
         SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
         SelectionDAG &DAG, unsigned Depth) const override;
 
+    bool isSplatValueForTargetNode(SDValue Op, const APInt &DemandedElts,
+                                   APInt &UndefElts,
+                                   unsigned Depth) const override;
+
     const Constant *getTargetConstantFromLoad(LoadSDNode *LD) const override;
 
     SDValue unwrapAddress(SDValue N) const override;
diff --git a/llvm/lib/Target/X86/X86IndirectBranchTracking.cpp b/llvm/lib/Target/X86/X86IndirectBranchTracking.cpp
index 732b2b1a5ada..6642f46e64b2 100644
--- a/llvm/lib/Target/X86/X86IndirectBranchTracking.cpp
+++ b/llvm/lib/Target/X86/X86IndirectBranchTracking.cpp
@@ -137,8 +137,10 @@ bool X86IndirectBranchTrackingPass::runOnMachineFunction(MachineFunction &MF) {
       Changed |= addENDBR(MBB, MBB.begin());
 
     for (MachineBasicBlock::iterator I = MBB.begin(); I != MBB.end(); ++I) {
-      if (I->isCall() && IsCallReturnTwice(I->getOperand(0)))
+      if (I->isCall() && I->getNumOperands() > 0 &&
+          IsCallReturnTwice(I->getOperand(0))) {
         Changed |= addENDBR(MBB, std::next(I));
+      }
     }
 
     // Exception handle may indirectly jump to catch pad, So we should add
diff --git a/llvm/lib/Target/X86/X86InstrAVX512.td b/llvm/lib/Target/X86/X86InstrAVX512.td
index 1db83033ba35..ecd4777c3533 100644
--- a/llvm/lib/Target/X86/X86InstrAVX512.td
+++ b/llvm/lib/Target/X86/X86InstrAVX512.td
@@ -9958,74 +9958,74 @@ multiclass avx512_trunc_wb<bits<8> opc, string OpcodeStr, SDNode OpNode,
 }
 
 defm VPMOVQB    : avx512_trunc_qb<0x32, "vpmovqb",
-                                  WriteShuffle256, truncstorevi8,
+                                  WriteVPMOV256, truncstorevi8,
                                   masked_truncstorevi8, X86vtrunc, X86vmtrunc>;
 defm VPMOVSQB   : avx512_trunc_qb<0x22, "vpmovsqb",
-                                  WriteShuffle256, truncstore_s_vi8,
+                                  WriteVPMOV256, truncstore_s_vi8,
                                   masked_truncstore_s_vi8, X86vtruncs,
                                   X86vmtruncs>;
 defm VPMOVUSQB  : avx512_trunc_qb<0x12, "vpmovusqb",
-                                  WriteShuffle256, truncstore_us_vi8,
+                                  WriteVPMOV256, truncstore_us_vi8,
                                   masked_truncstore_us_vi8, X86vtruncus, X86vmtruncus>;
 
 defm VPMOVQW    : avx512_trunc_qw<0x34, "vpmovqw", trunc, select_trunc,
-                                  WriteShuffle256, truncstorevi16,
+                                  WriteVPMOV256, truncstorevi16,
                                   masked_truncstorevi16, X86vtrunc, X86vmtrunc>;
 defm VPMOVSQW   : avx512_trunc_qw<0x24, "vpmovsqw",  X86vtruncs, select_truncs,
-                                  WriteShuffle256, truncstore_s_vi16,
+                                  WriteVPMOV256, truncstore_s_vi16,
                                   masked_truncstore_s_vi16, X86vtruncs,
                                   X86vmtruncs>;
 defm VPMOVUSQW  : avx512_trunc_qw<0x14, "vpmovusqw", X86vtruncus,
-                                  select_truncus, WriteShuffle256,
+                                  select_truncus, WriteVPMOV256,
                                   truncstore_us_vi16, masked_truncstore_us_vi16,
                                   X86vtruncus, X86vmtruncus>;
 
 defm VPMOVQD    : avx512_trunc_qd<0x35, "vpmovqd", trunc, select_trunc,
-                                  WriteShuffle256, truncstorevi32,
+                                  WriteVPMOV256, truncstorevi32,
                                   masked_truncstorevi32, X86vtrunc, X86vmtrunc>;
 defm VPMOVSQD   : avx512_trunc_qd<0x25, "vpmovsqd",  X86vtruncs, select_truncs,
-                                  WriteShuffle256, truncstore_s_vi32,
+                                  WriteVPMOV256, truncstore_s_vi32,
                                   masked_truncstore_s_vi32, X86vtruncs,
                                   X86vmtruncs>;
 defm VPMOVUSQD  : avx512_trunc_qd<0x15, "vpmovusqd", X86vtruncus,
-                                  select_truncus, WriteShuffle256,
+                                  select_truncus, WriteVPMOV256,
                                   truncstore_us_vi32, masked_truncstore_us_vi32,
                                   X86vtruncus, X86vmtruncus>;
 
 defm VPMOVDB    : avx512_trunc_db<0x31, "vpmovdb", trunc, select_trunc,
-                                  WriteShuffle256, truncstorevi8,
+                                  WriteVPMOV256, truncstorevi8,
                                   masked_truncstorevi8, X86vtrunc, X86vmtrunc>;
 defm VPMOVSDB   : avx512_trunc_db<0x21, "vpmovsdb", X86vtruncs, select_truncs,
-                                  WriteShuffle256, truncstore_s_vi8,
+                                  WriteVPMOV256, truncstore_s_vi8,
                                   masked_truncstore_s_vi8, X86vtruncs,
                                   X86vmtruncs>;
 defm VPMOVUSDB  : avx512_trunc_db<0x11, "vpmovusdb",  X86vtruncus,
-                                  select_truncus, WriteShuffle256,
+                                  select_truncus, WriteVPMOV256,
                                   truncstore_us_vi8, masked_truncstore_us_vi8,
                                   X86vtruncus, X86vmtruncus>;
 
 defm VPMOVDW    : avx512_trunc_dw<0x33, "vpmovdw", trunc, select_trunc,
-                                  WriteShuffle256, truncstorevi16,
+                                  WriteVPMOV256, truncstorevi16,
                                   masked_truncstorevi16, X86vtrunc, X86vmtrunc>;
 defm VPMOVSDW   : avx512_trunc_dw<0x23, "vpmovsdw", X86vtruncs, select_truncs,
-                                  WriteShuffle256, truncstore_s_vi16,
+                                  WriteVPMOV256, truncstore_s_vi16,
                                   masked_truncstore_s_vi16, X86vtruncs,
                                   X86vmtruncs>;
 defm VPMOVUSDW  : avx512_trunc_dw<0x13, "vpmovusdw", X86vtruncus,
-                                  select_truncus, WriteShuffle256,
+                                  select_truncus, WriteVPMOV256,
                                   truncstore_us_vi16, masked_truncstore_us_vi16,
                                   X86vtruncus, X86vmtruncus>;
 
 defm VPMOVWB    : avx512_trunc_wb<0x30, "vpmovwb", trunc, select_trunc,
-                                  WriteShuffle256, truncstorevi8,
+                                  WriteVPMOV256, truncstorevi8,
                                   masked_truncstorevi8, X86vtrunc,
                                   X86vmtrunc>;
 defm VPMOVSWB   : avx512_trunc_wb<0x20, "vpmovswb", X86vtruncs, select_truncs,
-                                  WriteShuffle256, truncstore_s_vi8,
+                                  WriteVPMOV256, truncstore_s_vi8,
                                   masked_truncstore_s_vi8, X86vtruncs,
                                   X86vmtruncs>;
 defm VPMOVUSWB  : avx512_trunc_wb<0x10, "vpmovuswb", X86vtruncus,
-                                  select_truncus, WriteShuffle256,
+                                  select_truncus, WriteVPMOV256,
                                   truncstore_us_vi8, masked_truncstore_us_vi8,
                                   X86vtruncus, X86vmtruncus>;
 
@@ -10084,7 +10084,7 @@ defm : mtrunc_lowering<"VPMOVSQWZ", X86vmtruncs, v8i16x_info, v8i64_info>;
 defm : mtrunc_lowering<"VPMOVUSQWZ", X86vmtruncus, v8i16x_info, v8i64_info>;
 }
 
-multiclass WriteShuffle256_common<bits<8> opc, string OpcodeStr, X86FoldableSchedWrite sched,
+multiclass avx512_pmovx_common<bits<8> opc, string OpcodeStr, X86FoldableSchedWrite sched,
               X86VectorVTInfo DestInfo, X86VectorVTInfo SrcInfo,
               X86MemOperand x86memop, PatFrag LdFrag, SDNode OpNode>{
   let ExeDomain = DestInfo.ExeDomain in {
@@ -10100,135 +10100,140 @@ multiclass WriteShuffle256_common<bits<8> opc, string OpcodeStr, X86FoldableSche
   }
 }
 
-multiclass WriteShuffle256_BW<bits<8> opc, string OpcodeStr,
+multiclass avx512_pmovx_bw<bits<8> opc, string OpcodeStr,
           SDNode OpNode, SDNode InVecNode, string ExtTy,
-          X86FoldableSchedWrite sched, PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi8")> {
+          X86FoldableSchedWrite schedX, X86FoldableSchedWrite schedYZ,
+          PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi8")> {
   let Predicates = [HasVLX, HasBWI] in {
-    defm Z128:  WriteShuffle256_common<opc, OpcodeStr, sched, v8i16x_info,
+    defm Z128:  avx512_pmovx_common<opc, OpcodeStr, schedX, v8i16x_info,
                     v16i8x_info, i64mem, LdFrag, InVecNode>,
                      EVEX_CD8<8, CD8VH>, T8PD, EVEX_V128, VEX_WIG;
 
-    defm Z256:  WriteShuffle256_common<opc, OpcodeStr, sched, v16i16x_info,
+    defm Z256:  avx512_pmovx_common<opc, OpcodeStr, schedYZ, v16i16x_info,
                     v16i8x_info, i128mem, LdFrag, OpNode>,
                      EVEX_CD8<8, CD8VH>, T8PD, EVEX_V256, VEX_WIG;
   }
   let Predicates = [HasBWI] in {
-    defm Z   :  WriteShuffle256_common<opc, OpcodeStr, sched, v32i16_info,
+    defm Z   :  avx512_pmovx_common<opc, OpcodeStr, schedYZ, v32i16_info,
                     v32i8x_info, i256mem, LdFrag, OpNode>,
                      EVEX_CD8<8, CD8VH>, T8PD, EVEX_V512, VEX_WIG;
   }
 }
 
-multiclass WriteShuffle256_BD<bits<8> opc, string OpcodeStr,
+multiclass avx512_pmovx_bd<bits<8> opc, string OpcodeStr,
           SDNode OpNode, SDNode InVecNode, string ExtTy,
-          X86FoldableSchedWrite sched, PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi8")> {
+          X86FoldableSchedWrite schedX, X86FoldableSchedWrite schedYZ,
+          PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi8")> {
   let Predicates = [HasVLX, HasAVX512] in {
-    defm Z128:  WriteShuffle256_common<opc, OpcodeStr, sched, v4i32x_info,
+    defm Z128:  avx512_pmovx_common<opc, OpcodeStr, schedX, v4i32x_info,
                    v16i8x_info, i32mem, LdFrag, InVecNode>,
                          EVEX_CD8<8, CD8VQ>, T8PD, EVEX_V128, VEX_WIG;
 
-    defm Z256:  WriteShuffle256_common<opc, OpcodeStr, sched, v8i32x_info,
+    defm Z256:  avx512_pmovx_common<opc, OpcodeStr, schedYZ, v8i32x_info,
                    v16i8x_info, i64mem, LdFrag, InVecNode>,
                          EVEX_CD8<8, CD8VQ>, T8PD, EVEX_V256, VEX_WIG;
   }
   let Predicates = [HasAVX512] in {
-    defm Z   :  WriteShuffle256_common<opc, OpcodeStr, sched, v16i32_info,
+    defm Z   :  avx512_pmovx_common<opc, OpcodeStr, schedYZ, v16i32_info,
                    v16i8x_info, i128mem, LdFrag, OpNode>,
                          EVEX_CD8<8, CD8VQ>, T8PD, EVEX_V512, VEX_WIG;
   }
 }
 
-multiclass WriteShuffle256_BQ<bits<8> opc, string OpcodeStr,
+multiclass avx512_pmovx_bq<bits<8> opc, string OpcodeStr,
                               SDNode InVecNode, string ExtTy,
-                              X86FoldableSchedWrite sched,
+                              X86FoldableSchedWrite schedX, X86FoldableSchedWrite schedYZ,
                               PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi8")> {
   let Predicates = [HasVLX, HasAVX512] in {
-    defm Z128:  WriteShuffle256_common<opc, OpcodeStr, sched, v2i64x_info,
+    defm Z128:  avx512_pmovx_common<opc, OpcodeStr, schedX, v2i64x_info,
                    v16i8x_info, i16mem, LdFrag, InVecNode>,
                      EVEX_CD8<8, CD8VO>, T8PD, EVEX_V128, VEX_WIG;
 
-    defm Z256:  WriteShuffle256_common<opc, OpcodeStr, sched, v4i64x_info,
+    defm Z256:  avx512_pmovx_common<opc, OpcodeStr, schedYZ, v4i64x_info,
                    v16i8x_info, i32mem, LdFrag, InVecNode>,
                      EVEX_CD8<8, CD8VO>, T8PD, EVEX_V256, VEX_WIG;
   }
   let Predicates = [HasAVX512] in {
-    defm Z   :  WriteShuffle256_common<opc, OpcodeStr, sched, v8i64_info,
+    defm Z   :  avx512_pmovx_common<opc, OpcodeStr, schedYZ, v8i64_info,
                    v16i8x_info, i64mem, LdFrag, InVecNode>,
                      EVEX_CD8<8, CD8VO>, T8PD, EVEX_V512, VEX_WIG;
   }
 }
 
-multiclass WriteShuffle256_WD<bits<8> opc, string OpcodeStr,
+multiclass avx512_pmovx_wd<bits<8> opc, string OpcodeStr,
          SDNode OpNode, SDNode InVecNode, string ExtTy,
-         X86FoldableSchedWrite sched, PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi16")> {
+         X86FoldableSchedWrite schedX, X86FoldableSchedWrite schedYZ,
+         PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi16")> {
   let Predicates = [HasVLX, HasAVX512] in {
-    defm Z128:  WriteShuffle256_common<opc, OpcodeStr, sched, v4i32x_info,
+    defm Z128:  avx512_pmovx_common<opc, OpcodeStr, schedX, v4i32x_info,
                    v8i16x_info, i64mem, LdFrag, InVecNode>,
                      EVEX_CD8<16, CD8VH>, T8PD, EVEX_V128, VEX_WIG;
 
-    defm Z256:  WriteShuffle256_common<opc, OpcodeStr, sched, v8i32x_info,
+    defm Z256:  avx512_pmovx_common<opc, OpcodeStr, schedYZ, v8i32x_info,
                    v8i16x_info, i128mem, LdFrag, OpNode>,
                      EVEX_CD8<16, CD8VH>, T8PD, EVEX_V256, VEX_WIG;
   }
   let Predicates = [HasAVX512] in {
-    defm Z   :  WriteShuffle256_common<opc, OpcodeStr, sched, v16i32_info,
+    defm Z   :  avx512_pmovx_common<opc, OpcodeStr, schedYZ, v16i32_info,
                    v16i16x_info, i256mem, LdFrag, OpNode>,
                      EVEX_CD8<16, CD8VH>, T8PD, EVEX_V512, VEX_WIG;
   }
 }
 
-multiclass WriteShuffle256_WQ<bits<8> opc, string OpcodeStr,
+multiclass avx512_pmovx_wq<bits<8> opc, string OpcodeStr,
          SDNode OpNode, SDNode InVecNode, string ExtTy,
-         X86FoldableSchedWrite sched, PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi16")> {
+         X86FoldableSchedWrite schedX, X86FoldableSchedWrite schedYZ,
+         PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi16")> {
   let Predicates = [HasVLX, HasAVX512] in {
-    defm Z128:  WriteShuffle256_common<opc, OpcodeStr, sched, v2i64x_info,
+    defm Z128:  avx512_pmovx_common<opc, OpcodeStr, schedX, v2i64x_info,
                    v8i16x_info, i32mem, LdFrag, InVecNode>,
                      EVEX_CD8<16, CD8VQ>, T8PD, EVEX_V128, VEX_WIG;
 
-    defm Z256:  WriteShuffle256_common<opc, OpcodeStr, sched, v4i64x_info,
+    defm Z256:  avx512_pmovx_common<opc, OpcodeStr, schedYZ, v4i64x_info,
                    v8i16x_info, i64mem, LdFrag, InVecNode>,
                      EVEX_CD8<16, CD8VQ>, T8PD, EVEX_V256, VEX_WIG;
   }
   let Predicates = [HasAVX512] in {
-    defm Z   :  WriteShuffle256_common<opc, OpcodeStr, sched, v8i64_info,
+    defm Z   :  avx512_pmovx_common<opc, OpcodeStr, schedYZ, v8i64_info,
                    v8i16x_info, i128mem, LdFrag, OpNode>,
                      EVEX_CD8<16, CD8VQ>, T8PD, EVEX_V512, VEX_WIG;
   }
 }
 
-multiclass WriteShuffle256_DQ<bits<8> opc, string OpcodeStr,
+multiclass avx512_pmovx_dq<bits<8> opc, string OpcodeStr,
          SDNode OpNode, SDNode InVecNode, string ExtTy,
-         X86FoldableSchedWrite sched, PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi32")> {
+         X86FoldableSchedWrite schedX, X86FoldableSchedWrite schedYZ,
+         PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi32")> {
 
   let Predicates = [HasVLX, HasAVX512] in {
-    defm Z128:  WriteShuffle256_common<opc, OpcodeStr, sched, v2i64x_info,
+    defm Z128:  avx512_pmovx_common<opc, OpcodeStr, schedX, v2i64x_info,
                    v4i32x_info, i64mem, LdFrag, InVecNode>,
                      EVEX_CD8<32, CD8VH>, T8PD, EVEX_V128;
 
-    defm Z256:  WriteShuffle256_common<opc, OpcodeStr, sched, v4i64x_info,
+    defm Z256:  avx512_pmovx_common<opc, OpcodeStr, schedYZ, v4i64x_info,
                    v4i32x_info, i128mem, LdFrag, OpNode>,
                      EVEX_CD8<32, CD8VH>, T8PD, EVEX_V256;
   }
   let Predicates = [HasAVX512] in {
-    defm Z   :  WriteShuffle256_common<opc, OpcodeStr, sched, v8i64_info,
+    defm Z   :  avx512_pmovx_common<opc, OpcodeStr, schedYZ, v8i64_info,
                    v8i32x_info, i256mem, LdFrag, OpNode>,
                      EVEX_CD8<32, CD8VH>, T8PD, EVEX_V512;
   }
 }
 
-defm VPMOVZXBW : WriteShuffle256_BW<0x30, "vpmovzxbw", zext, zext_invec, "z", WriteShuffle256>;
-defm VPMOVZXBD : WriteShuffle256_BD<0x31, "vpmovzxbd", zext, zext_invec, "z", WriteShuffle256>;
-defm VPMOVZXBQ : WriteShuffle256_BQ<0x32, "vpmovzxbq",       zext_invec, "z", WriteShuffle256>;
-defm VPMOVZXWD : WriteShuffle256_WD<0x33, "vpmovzxwd", zext, zext_invec, "z", WriteShuffle256>;
-defm VPMOVZXWQ : WriteShuffle256_WQ<0x34, "vpmovzxwq", zext, zext_invec, "z", WriteShuffle256>;
-defm VPMOVZXDQ : WriteShuffle256_DQ<0x35, "vpmovzxdq", zext, zext_invec, "z", WriteShuffle256>;
+defm VPMOVZXBW : avx512_pmovx_bw<0x30, "vpmovzxbw", zext, zext_invec, "z", SchedWriteShuffle.XMM, WriteVPMOV256>;
+defm VPMOVZXBD : avx512_pmovx_bd<0x31, "vpmovzxbd", zext, zext_invec, "z", SchedWriteShuffle.XMM, WriteVPMOV256>;
+defm VPMOVZXBQ : avx512_pmovx_bq<0x32, "vpmovzxbq",       zext_invec, "z", SchedWriteShuffle.XMM, WriteVPMOV256>;
+defm VPMOVZXWD : avx512_pmovx_wd<0x33, "vpmovzxwd", zext, zext_invec, "z", SchedWriteShuffle.XMM, WriteVPMOV256>;
+defm VPMOVZXWQ : avx512_pmovx_wq<0x34, "vpmovzxwq", zext, zext_invec, "z", SchedWriteShuffle.XMM, WriteVPMOV256>;
+defm VPMOVZXDQ : avx512_pmovx_dq<0x35, "vpmovzxdq", zext, zext_invec, "z", SchedWriteShuffle.XMM, WriteVPMOV256>;
 
-defm VPMOVSXBW: WriteShuffle256_BW<0x20, "vpmovsxbw", sext, sext_invec, "s", WriteShuffle256>;
-defm VPMOVSXBD: WriteShuffle256_BD<0x21, "vpmovsxbd", sext, sext_invec, "s", WriteShuffle256>;
-defm VPMOVSXBQ: WriteShuffle256_BQ<0x22, "vpmovsxbq",       sext_invec, "s", WriteShuffle256>;
-defm VPMOVSXWD: WriteShuffle256_WD<0x23, "vpmovsxwd", sext, sext_invec, "s", WriteShuffle256>;
-defm VPMOVSXWQ: WriteShuffle256_WQ<0x24, "vpmovsxwq", sext, sext_invec, "s", WriteShuffle256>;
-defm VPMOVSXDQ: WriteShuffle256_DQ<0x25, "vpmovsxdq", sext, sext_invec, "s", WriteShuffle256>;
+defm VPMOVSXBW: avx512_pmovx_bw<0x20, "vpmovsxbw", sext, sext_invec, "s", SchedWriteShuffle.XMM, WriteVPMOV256>;
+defm VPMOVSXBD: avx512_pmovx_bd<0x21, "vpmovsxbd", sext, sext_invec, "s", SchedWriteShuffle.XMM, WriteVPMOV256>;
+defm VPMOVSXBQ: avx512_pmovx_bq<0x22, "vpmovsxbq",       sext_invec, "s", SchedWriteShuffle.XMM, WriteVPMOV256>;
+defm VPMOVSXWD: avx512_pmovx_wd<0x23, "vpmovsxwd", sext, sext_invec, "s", SchedWriteShuffle.XMM, WriteVPMOV256>;
+defm VPMOVSXWQ: avx512_pmovx_wq<0x24, "vpmovsxwq", sext, sext_invec, "s", SchedWriteShuffle.XMM, WriteVPMOV256>;
+defm VPMOVSXDQ: avx512_pmovx_dq<0x25, "vpmovsxdq", sext, sext_invec, "s", SchedWriteShuffle.XMM, WriteVPMOV256>;
 
 
 // Patterns that we also need any extend versions of. aext_vector_inreg
@@ -10523,21 +10528,22 @@ defm VSCATTERPF1DPD: avx512_gather_scatter_prefetch<0xC6, MRM6m, "vscatterpf1dpd
 defm VSCATTERPF1QPD: avx512_gather_scatter_prefetch<0xC7, MRM6m, "vscatterpf1qpd",
                      VK8WM, vz512mem>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
 
-multiclass cvt_by_vec_width<bits<8> opc, X86VectorVTInfo Vec, string OpcodeStr > {
+multiclass cvt_by_vec_width<bits<8> opc, X86VectorVTInfo Vec, string OpcodeStr, SchedWrite Sched> {
 def rr : AVX512XS8I<opc, MRMSrcReg, (outs Vec.RC:$dst), (ins Vec.KRC:$src),
                   !strconcat(OpcodeStr#Vec.Suffix, "\t{$src, $dst|$dst, $src}"),
                   [(set Vec.RC:$dst, (Vec.VT (sext Vec.KRC:$src)))]>,
-                  EVEX, Sched<[WriteMove]>; // TODO - WriteVecTrunc?
+                  EVEX, Sched<[Sched]>;
 }
 
 multiclass cvt_mask_by_elt_width<bits<8> opc, AVX512VLVectorVTInfo VTInfo,
                                  string OpcodeStr, Predicate prd> {
+// TODO - Replace WriteMove with WriteVecTrunc?
 let Predicates = [prd] in
-  defm Z : cvt_by_vec_width<opc, VTInfo.info512, OpcodeStr>, EVEX_V512;
+  defm Z : cvt_by_vec_width<opc, VTInfo.info512, OpcodeStr, WriteMove>, EVEX_V512;
 
   let Predicates = [prd, HasVLX] in {
-    defm Z256 : cvt_by_vec_width<opc, VTInfo.info256, OpcodeStr>, EVEX_V256;
-    defm Z128 : cvt_by_vec_width<opc, VTInfo.info128, OpcodeStr>, EVEX_V128;
+    defm Z256 : cvt_by_vec_width<opc, VTInfo.info256, OpcodeStr, WriteMove>, EVEX_V256;
+    defm Z128 : cvt_by_vec_width<opc, VTInfo.info128, OpcodeStr, WriteMove>, EVEX_V128;
   }
 }
 
diff --git a/llvm/lib/Target/X86/X86InstrCompiler.td b/llvm/lib/Target/X86/X86InstrCompiler.td
index ba52283b570d..7288ce812138 100644
--- a/llvm/lib/Target/X86/X86InstrCompiler.td
+++ b/llvm/lib/Target/X86/X86InstrCompiler.td
@@ -260,10 +260,10 @@ let isPseudo = 1, SchedRW = [WriteSystem] in {
 // Pseudo instructions used by address sanitizer.
 //===----------------------------------------------------------------------===//
 let
-  Defs = [R8, EFLAGS] in {
+  Defs = [R10, R11, EFLAGS] in {
 def ASAN_CHECK_MEMACCESS : PseudoI<
-  (outs), (ins GR64NoR8:$addr, i32imm:$accessinfo),
-  [(int_asan_check_memaccess GR64NoR8:$addr, (i32 timm:$accessinfo))]>,
+  (outs), (ins GR64PLTSafe:$addr, i32imm:$accessinfo),
+  [(int_asan_check_memaccess GR64PLTSafe:$addr, (i32 timm:$accessinfo))]>,
   Sched<[]>;
 }
 
diff --git a/llvm/lib/Target/X86/X86InstrFoldTables.cpp b/llvm/lib/Target/X86/X86InstrFoldTables.cpp
index 6d4ad08842c7..226349485238 100644
--- a/llvm/lib/Target/X86/X86InstrFoldTables.cpp
+++ b/llvm/lib/Target/X86/X86InstrFoldTables.cpp
@@ -529,11 +529,11 @@ static const X86MemoryFoldTableEntry MemoryFoldTable1[] = {
   { X86::LZCNT16rr,            X86::LZCNT16rm,            0 },
   { X86::LZCNT32rr,            X86::LZCNT32rm,            0 },
   { X86::LZCNT64rr,            X86::LZCNT64rm,            0 },
-  { X86::MMX_CVTPD2PIirr,      X86::MMX_CVTPD2PIirm,      TB_ALIGN_16 },
-  { X86::MMX_CVTPI2PDirr,      X86::MMX_CVTPI2PDirm,      0 },
-  { X86::MMX_CVTPS2PIirr,      X86::MMX_CVTPS2PIirm,      TB_NO_REVERSE },
-  { X86::MMX_CVTTPD2PIirr,     X86::MMX_CVTTPD2PIirm,     TB_ALIGN_16 },
-  { X86::MMX_CVTTPS2PIirr,     X86::MMX_CVTTPS2PIirm,     TB_NO_REVERSE },
+  { X86::MMX_CVTPD2PIrr,       X86::MMX_CVTPD2PIrm,       TB_ALIGN_16 },
+  { X86::MMX_CVTPI2PDrr,       X86::MMX_CVTPI2PDrm,       0 },
+  { X86::MMX_CVTPS2PIrr,       X86::MMX_CVTPS2PIrm,       TB_NO_REVERSE },
+  { X86::MMX_CVTTPD2PIrr,      X86::MMX_CVTTPD2PIrm,      TB_ALIGN_16 },
+  { X86::MMX_CVTTPS2PIrr,      X86::MMX_CVTTPS2PIrm,      TB_NO_REVERSE },
   { X86::MMX_MOVD64to64rr,     X86::MMX_MOVQ64rm,         0 },
   { X86::MMX_PABSBrr,          X86::MMX_PABSBrm,          0 },
   { X86::MMX_PABSDrr,          X86::MMX_PABSDrm,          0 },
@@ -1339,29 +1339,29 @@ static const X86MemoryFoldTableEntry MemoryFoldTable2[] = {
   { X86::MINSDrr_Int,              X86::MINSDrm_Int,              TB_NO_REVERSE },
   { X86::MINSSrr,                  X86::MINSSrm,                  0 },
   { X86::MINSSrr_Int,              X86::MINSSrm_Int,              TB_NO_REVERSE },
-  { X86::MMX_CVTPI2PSirr,          X86::MMX_CVTPI2PSirm,          0 },
-  { X86::MMX_PACKSSDWirr,          X86::MMX_PACKSSDWirm,          0 },
-  { X86::MMX_PACKSSWBirr,          X86::MMX_PACKSSWBirm,          0 },
-  { X86::MMX_PACKUSWBirr,          X86::MMX_PACKUSWBirm,          0 },
-  { X86::MMX_PADDBirr,             X86::MMX_PADDBirm,             0 },
-  { X86::MMX_PADDDirr,             X86::MMX_PADDDirm,             0 },
-  { X86::MMX_PADDQirr,             X86::MMX_PADDQirm,             0 },
-  { X86::MMX_PADDSBirr,            X86::MMX_PADDSBirm,            0 },
-  { X86::MMX_PADDSWirr,            X86::MMX_PADDSWirm,            0 },
-  { X86::MMX_PADDUSBirr,           X86::MMX_PADDUSBirm,           0 },
-  { X86::MMX_PADDUSWirr,           X86::MMX_PADDUSWirm,           0 },
-  { X86::MMX_PADDWirr,             X86::MMX_PADDWirm,             0 },
+  { X86::MMX_CVTPI2PSrr,           X86::MMX_CVTPI2PSrm,           0 },
+  { X86::MMX_PACKSSDWrr,           X86::MMX_PACKSSDWrm,           0 },
+  { X86::MMX_PACKSSWBrr,           X86::MMX_PACKSSWBrm,           0 },
+  { X86::MMX_PACKUSWBrr,           X86::MMX_PACKUSWBrm,           0 },
+  { X86::MMX_PADDBrr,              X86::MMX_PADDBrm,              0 },
+  { X86::MMX_PADDDrr,              X86::MMX_PADDDrm,              0 },
+  { X86::MMX_PADDQrr,              X86::MMX_PADDQrm,              0 },
+  { X86::MMX_PADDSBrr,             X86::MMX_PADDSBrm,             0 },
+  { X86::MMX_PADDSWrr,             X86::MMX_PADDSWrm,             0 },
+  { X86::MMX_PADDUSBrr,            X86::MMX_PADDUSBrm,            0 },
+  { X86::MMX_PADDUSWrr,            X86::MMX_PADDUSWrm,            0 },
+  { X86::MMX_PADDWrr,              X86::MMX_PADDWrm,              0 },
   { X86::MMX_PALIGNRrri,           X86::MMX_PALIGNRrmi,           0 },
-  { X86::MMX_PANDNirr,             X86::MMX_PANDNirm,             0 },
-  { X86::MMX_PANDirr,              X86::MMX_PANDirm,              0 },
-  { X86::MMX_PAVGBirr,             X86::MMX_PAVGBirm,             0 },
-  { X86::MMX_PAVGWirr,             X86::MMX_PAVGWirm,             0 },
-  { X86::MMX_PCMPEQBirr,           X86::MMX_PCMPEQBirm,           0 },
-  { X86::MMX_PCMPEQDirr,           X86::MMX_PCMPEQDirm,           0 },
-  { X86::MMX_PCMPEQWirr,           X86::MMX_PCMPEQWirm,           0 },
-  { X86::MMX_PCMPGTBirr,           X86::MMX_PCMPGTBirm,           0 },
-  { X86::MMX_PCMPGTDirr,           X86::MMX_PCMPGTDirm,           0 },
-  { X86::MMX_PCMPGTWirr,           X86::MMX_PCMPGTWirm,           0 },
+  { X86::MMX_PANDNrr,              X86::MMX_PANDNrm,              0 },
+  { X86::MMX_PANDrr,               X86::MMX_PANDrm,               0 },
+  { X86::MMX_PAVGBrr,              X86::MMX_PAVGBrm,              0 },
+  { X86::MMX_PAVGWrr,              X86::MMX_PAVGWrm,              0 },
+  { X86::MMX_PCMPEQBrr,            X86::MMX_PCMPEQBrm,            0 },
+  { X86::MMX_PCMPEQDrr,            X86::MMX_PCMPEQDrm,            0 },
+  { X86::MMX_PCMPEQWrr,            X86::MMX_PCMPEQWrm,            0 },
+  { X86::MMX_PCMPGTBrr,            X86::MMX_PCMPGTBrm,            0 },
+  { X86::MMX_PCMPGTDrr,            X86::MMX_PCMPGTDrm,            0 },
+  { X86::MMX_PCMPGTWrr,            X86::MMX_PCMPGTWrm,            0 },
   { X86::MMX_PHADDDrr,             X86::MMX_PHADDDrm,             0 },
   { X86::MMX_PHADDSWrr,            X86::MMX_PHADDSWrm,            0 },
   { X86::MMX_PHADDWrr,             X86::MMX_PHADDWrm,             0 },
@@ -1370,18 +1370,18 @@ static const X86MemoryFoldTableEntry MemoryFoldTable2[] = {
   { X86::MMX_PHSUBWrr,             X86::MMX_PHSUBWrm,             0 },
   { X86::MMX_PINSRWrr,             X86::MMX_PINSRWrm,             TB_NO_REVERSE },
   { X86::MMX_PMADDUBSWrr,          X86::MMX_PMADDUBSWrm,          0 },
-  { X86::MMX_PMADDWDirr,           X86::MMX_PMADDWDirm,           0 },
-  { X86::MMX_PMAXSWirr,            X86::MMX_PMAXSWirm,            0 },
-  { X86::MMX_PMAXUBirr,            X86::MMX_PMAXUBirm,            0 },
-  { X86::MMX_PMINSWirr,            X86::MMX_PMINSWirm,            0 },
-  { X86::MMX_PMINUBirr,            X86::MMX_PMINUBirm,            0 },
+  { X86::MMX_PMADDWDrr,            X86::MMX_PMADDWDrm,            0 },
+  { X86::MMX_PMAXSWrr,             X86::MMX_PMAXSWrm,             0 },
+  { X86::MMX_PMAXUBrr,             X86::MMX_PMAXUBrm,             0 },
+  { X86::MMX_PMINSWrr,             X86::MMX_PMINSWrm,             0 },
+  { X86::MMX_PMINUBrr,             X86::MMX_PMINUBrm,             0 },
   { X86::MMX_PMULHRSWrr,           X86::MMX_PMULHRSWrm,           0 },
-  { X86::MMX_PMULHUWirr,           X86::MMX_PMULHUWirm,           0 },
-  { X86::MMX_PMULHWirr,            X86::MMX_PMULHWirm,            0 },
-  { X86::MMX_PMULLWirr,            X86::MMX_PMULLWirm,            0 },
-  { X86::MMX_PMULUDQirr,           X86::MMX_PMULUDQirm,           0 },
-  { X86::MMX_PORirr,               X86::MMX_PORirm,               0 },
-  { X86::MMX_PSADBWirr,            X86::MMX_PSADBWirm,            0 },
+  { X86::MMX_PMULHUWrr,            X86::MMX_PMULHUWrm,            0 },
+  { X86::MMX_PMULHWrr,             X86::MMX_PMULHWrm,             0 },
+  { X86::MMX_PMULLWrr,             X86::MMX_PMULLWrm,             0 },
+  { X86::MMX_PMULUDQrr,            X86::MMX_PMULUDQrm,            0 },
+  { X86::MMX_PORrr,                X86::MMX_PORrm,                0 },
+  { X86::MMX_PSADBWrr,             X86::MMX_PSADBWrm,             0 },
   { X86::MMX_PSHUFBrr,             X86::MMX_PSHUFBrm,             0 },
   { X86::MMX_PSIGNBrr,             X86::MMX_PSIGNBrm,             0 },
   { X86::MMX_PSIGNDrr,             X86::MMX_PSIGNDrm,             0 },
@@ -1394,21 +1394,21 @@ static const X86MemoryFoldTableEntry MemoryFoldTable2[] = {
   { X86::MMX_PSRLDrr,              X86::MMX_PSRLDrm,              0 },
   { X86::MMX_PSRLQrr,              X86::MMX_PSRLQrm,              0 },
   { X86::MMX_PSRLWrr,              X86::MMX_PSRLWrm,              0 },
-  { X86::MMX_PSUBBirr,             X86::MMX_PSUBBirm,             0 },
-  { X86::MMX_PSUBDirr,             X86::MMX_PSUBDirm,             0 },
-  { X86::MMX_PSUBQirr,             X86::MMX_PSUBQirm,             0 },
-  { X86::MMX_PSUBSBirr,            X86::MMX_PSUBSBirm,            0 },
-  { X86::MMX_PSUBSWirr,            X86::MMX_PSUBSWirm,            0 },
-  { X86::MMX_PSUBUSBirr,           X86::MMX_PSUBUSBirm,           0 },
-  { X86::MMX_PSUBUSWirr,           X86::MMX_PSUBUSWirm,           0 },
-  { X86::MMX_PSUBWirr,             X86::MMX_PSUBWirm,             0 },
-  { X86::MMX_PUNPCKHBWirr,         X86::MMX_PUNPCKHBWirm,         0 },
-  { X86::MMX_PUNPCKHDQirr,         X86::MMX_PUNPCKHDQirm,         0 },
-  { X86::MMX_PUNPCKHWDirr,         X86::MMX_PUNPCKHWDirm,         0 },
-  { X86::MMX_PUNPCKLBWirr,         X86::MMX_PUNPCKLBWirm,         TB_NO_REVERSE },
-  { X86::MMX_PUNPCKLDQirr,         X86::MMX_PUNPCKLDQirm,         TB_NO_REVERSE },
-  { X86::MMX_PUNPCKLWDirr,         X86::MMX_PUNPCKLWDirm,         TB_NO_REVERSE },
-  { X86::MMX_PXORirr,              X86::MMX_PXORirm,              0 },
+  { X86::MMX_PSUBBrr,              X86::MMX_PSUBBrm,              0 },
+  { X86::MMX_PSUBDrr,              X86::MMX_PSUBDrm,              0 },
+  { X86::MMX_PSUBQrr,              X86::MMX_PSUBQrm,              0 },
+  { X86::MMX_PSUBSBrr,             X86::MMX_PSUBSBrm,             0 },
+  { X86::MMX_PSUBSWrr,             X86::MMX_PSUBSWrm,             0 },
+  { X86::MMX_PSUBUSBrr,            X86::MMX_PSUBUSBrm,            0 },
+  { X86::MMX_PSUBUSWrr,            X86::MMX_PSUBUSWrm,            0 },
+  { X86::MMX_PSUBWrr,              X86::MMX_PSUBWrm,              0 },
+  { X86::MMX_PUNPCKHBWrr,          X86::MMX_PUNPCKHBWrm,          0 },
+  { X86::MMX_PUNPCKHDQrr,          X86::MMX_PUNPCKHDQrm,          0 },
+  { X86::MMX_PUNPCKHWDrr,          X86::MMX_PUNPCKHWDrm,          0 },
+  { X86::MMX_PUNPCKLBWrr,          X86::MMX_PUNPCKLBWrm,          TB_NO_REVERSE },
+  { X86::MMX_PUNPCKLDQrr,          X86::MMX_PUNPCKLDQrm,          TB_NO_REVERSE },
+  { X86::MMX_PUNPCKLWDrr,          X86::MMX_PUNPCKLWDrm,          TB_NO_REVERSE },
+  { X86::MMX_PXORrr,               X86::MMX_PXORrm,               0 },
   { X86::MOVLHPSrr,                X86::MOVHPSrm,                 TB_NO_REVERSE },
   { X86::MOVSDrr,                  X86::MOVLPDrm,                 TB_NO_REVERSE },
   { X86::MPSADBWrri,               X86::MPSADBWrmi,               TB_ALIGN_16 },
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index bb5637a31947..c379aa8d9258 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -4088,8 +4088,8 @@ bool X86InstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
 bool X86InstrInfo::isRedundantFlagInstr(const MachineInstr &FlagI,
                                         Register SrcReg, Register SrcReg2,
                                         int64_t ImmMask, int64_t ImmValue,
-                                        const MachineInstr &OI, bool *IsSwapped,
-                                        int64_t *ImmDelta) const {
+                                        const MachineInstr &OI,
+                                        bool *IsSwapped) const {
   switch (OI.getOpcode()) {
   case X86::CMP64rr:
   case X86::CMP32rr:
@@ -4140,21 +4140,10 @@ bool X86InstrInfo::isRedundantFlagInstr(const MachineInstr &FlagI,
       int64_t OIMask;
       int64_t OIValue;
       if (analyzeCompare(OI, OISrcReg, OISrcReg2, OIMask, OIValue) &&
-          SrcReg == OISrcReg && ImmMask == OIMask) {
-        if (OIValue == ImmValue) {
-          *ImmDelta = 0;
-          return true;
-        } else if (static_cast<uint64_t>(ImmValue) ==
-                   static_cast<uint64_t>(OIValue) - 1) {
-          *ImmDelta = -1;
-          return true;
-        } else if (static_cast<uint64_t>(ImmValue) ==
-                   static_cast<uint64_t>(OIValue) + 1) {
-          *ImmDelta = 1;
-          return true;
-        } else {
-          return false;
-        }
+          SrcReg == OISrcReg && ImmMask == OIMask && OIValue == ImmValue) {
+        assert(SrcReg2 == X86::NoRegister && OISrcReg2 == X86::NoRegister &&
+               "should not have 2nd register");
+        return true;
       }
     }
     return FlagI.isIdenticalTo(OI);
@@ -4404,7 +4393,6 @@ bool X86InstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
   bool ShouldUpdateCC = false;
   bool IsSwapped = false;
   X86::CondCode NewCC = X86::COND_INVALID;
-  int64_t ImmDelta = 0;
 
   // Search backward from CmpInstr for the next instruction defining EFLAGS.
   const TargetRegisterInfo *TRI = &getRegisterInfo();
@@ -4451,7 +4439,7 @@ bool X86InstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
         //     ...           // EFLAGS not changed
         //     cmp x, y      // <-- can be removed
         if (isRedundantFlagInstr(CmpInstr, SrcReg, SrcReg2, CmpMask, CmpValue,
-                                 Inst, &IsSwapped, &ImmDelta)) {
+                                 Inst, &IsSwapped)) {
           Sub = &Inst;
           break;
         }
@@ -4485,7 +4473,7 @@ bool X86InstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
   // It is safe to remove CmpInstr if EFLAGS is redefined or killed.
   // If we are done with the basic block, we need to check whether EFLAGS is
   // live-out.
-  bool FlagsMayLiveOut = true;
+  bool IsSafe = false;
   SmallVector<std::pair<MachineInstr*, X86::CondCode>, 4> OpsToUpdate;
   MachineBasicBlock::iterator AfterCmpInstr =
       std::next(MachineBasicBlock::iterator(CmpInstr));
@@ -4495,7 +4483,7 @@ bool X86InstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
     // We should check the usage if this instruction uses and updates EFLAGS.
     if (!UseEFLAGS && ModifyEFLAGS) {
       // It is safe to remove CmpInstr if EFLAGS is updated again.
-      FlagsMayLiveOut = false;
+      IsSafe = true;
       break;
     }
     if (!UseEFLAGS && !ModifyEFLAGS)
@@ -4503,7 +4491,7 @@ bool X86InstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
 
     // EFLAGS is used by this instruction.
     X86::CondCode OldCC = X86::COND_INVALID;
-    if (MI || IsSwapped || ImmDelta != 0) {
+    if (MI || IsSwapped) {
       // We decode the condition code from opcode.
       if (Instr.isBranch())
         OldCC = X86::getCondFromBranch(Instr);
@@ -4555,60 +4543,11 @@ bool X86InstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
       // to be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc.
       // We swap the condition code and synthesize the new opcode.
       ReplacementCC = getSwappedCondition(OldCC);
-      if (ReplacementCC == X86::COND_INVALID) return false;
-      ShouldUpdateCC = true;
-    } else if (ImmDelta != 0) {
-      unsigned BitWidth = TRI->getRegSizeInBits(*MRI->getRegClass(SrcReg));
-      // Shift amount for min/max constants to adjust for 8/16/32 instruction
-      // sizes.
-      switch (OldCC) {
-      case X86::COND_L: // x <s (C + 1)  -->  x <=s C
-        if (ImmDelta != 1 || APInt::getSignedMinValue(BitWidth) == CmpValue)
-          return false;
-        ReplacementCC = X86::COND_LE;
-        break;
-      case X86::COND_B: // x <u (C + 1)  -->  x <=u C
-        if (ImmDelta != 1 || CmpValue == 0)
-          return false;
-        ReplacementCC = X86::COND_BE;
-        break;
-      case X86::COND_GE: // x >=s (C + 1)  -->  x >s C
-        if (ImmDelta != 1 || APInt::getSignedMinValue(BitWidth) == CmpValue)
-          return false;
-        ReplacementCC = X86::COND_G;
-        break;
-      case X86::COND_AE: // x >=u (C + 1)  -->  x >u C
-        if (ImmDelta != 1 || CmpValue == 0)
-          return false;
-        ReplacementCC = X86::COND_A;
-        break;
-      case X86::COND_G: // x >s (C - 1)  -->  x >=s C
-        if (ImmDelta != -1 || APInt::getSignedMaxValue(BitWidth) == CmpValue)
-          return false;
-        ReplacementCC = X86::COND_GE;
-        break;
-      case X86::COND_A: // x >u (C - 1)  -->  x >=u C
-        if (ImmDelta != -1 || APInt::getMaxValue(BitWidth) == CmpValue)
-          return false;
-        ReplacementCC = X86::COND_AE;
-        break;
-      case X86::COND_LE: // x <=s (C - 1)  -->  x <s C
-        if (ImmDelta != -1 || APInt::getSignedMaxValue(BitWidth) == CmpValue)
-          return false;
-        ReplacementCC = X86::COND_L;
-        break;
-      case X86::COND_BE: // x <=u (C - 1)  -->  x <u C
-        if (ImmDelta != -1 || APInt::getMaxValue(BitWidth) == CmpValue)
-          return false;
-        ReplacementCC = X86::COND_B;
-        break;
-      default:
+      if (ReplacementCC == X86::COND_INVALID)
         return false;
-      }
-      ShouldUpdateCC = true;
     }
 
-    if (ShouldUpdateCC && ReplacementCC != OldCC) {
+    if ((ShouldUpdateCC || IsSwapped) && ReplacementCC != OldCC) {
       // Push the MachineInstr to OpsToUpdate.
       // If it is safe to remove CmpInstr, the condition code of these
       // instructions will be modified.
@@ -4616,14 +4555,14 @@ bool X86InstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
     }
     if (ModifyEFLAGS || Instr.killsRegister(X86::EFLAGS, TRI)) {
       // It is safe to remove CmpInstr if EFLAGS is updated again or killed.
-      FlagsMayLiveOut = false;
+      IsSafe = true;
       break;
     }
   }
 
-  // If we have to update users but EFLAGS is live-out abort, since we cannot
-  // easily find all of the users.
-  if (ShouldUpdateCC && FlagsMayLiveOut) {
+  // If EFLAGS is not killed nor re-defined, we should check whether it is
+  // live-out. If it is live-out, do not optimize.
+  if ((MI || IsSwapped) && !IsSafe) {
     for (MachineBasicBlock *Successor : CmpMBB.successors())
       if (Successor->isLiveIn(X86::EFLAGS))
         return false;
@@ -4944,7 +4883,7 @@ bool X86InstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
   case X86::SETB_C64r:
     return Expand2AddrUndef(MIB, get(X86::SBB64rr));
   case X86::MMX_SET0:
-    return Expand2AddrUndef(MIB, get(X86::MMX_PXORirr));
+    return Expand2AddrUndef(MIB, get(X86::MMX_PXORrr));
   case X86::V_SET0:
   case X86::FsFLD0SS:
   case X86::FsFLD0SD:
@@ -5217,12 +5156,12 @@ static bool hasUndefRegUpdate(unsigned Opcode, unsigned OpNum,
                               bool ForLoadFold = false) {
   // Set the OpNum parameter to the first source operand.
   switch (Opcode) {
-  case X86::MMX_PUNPCKHBWirr:
-  case X86::MMX_PUNPCKHWDirr:
-  case X86::MMX_PUNPCKHDQirr:
-  case X86::MMX_PUNPCKLBWirr:
-  case X86::MMX_PUNPCKLWDirr:
-  case X86::MMX_PUNPCKLDQirr:
+  case X86::MMX_PUNPCKHBWrr:
+  case X86::MMX_PUNPCKHWDrr:
+  case X86::MMX_PUNPCKHDQrr:
+  case X86::MMX_PUNPCKLBWrr:
+  case X86::MMX_PUNPCKLWDrr:
+  case X86::MMX_PUNPCKLDQrr:
   case X86::MOVHLPSrr:
   case X86::PACKSSWBrr:
   case X86::PACKUSWBrr:
diff --git a/llvm/lib/Target/X86/X86InstrInfo.h b/llvm/lib/Target/X86/X86InstrInfo.h
index 33ce55bbdb2b..537ada6222bf 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.h
+++ b/llvm/lib/Target/X86/X86InstrInfo.h
@@ -643,8 +643,7 @@ private:
   ///   CMP %1, %2   and  %3 = SUB %2, %1  ; IsSwapped=true
   bool isRedundantFlagInstr(const MachineInstr &FlagI, Register SrcReg,
                             Register SrcReg2, int64_t ImmMask, int64_t ImmValue,
-                            const MachineInstr &OI, bool *IsSwapped,
-                            int64_t *ImmDelta) const;
+                            const MachineInstr &OI, bool *IsSwapped) const;
 };
 
 } // namespace llvm
diff --git a/llvm/lib/Target/X86/X86InstrMMX.td b/llvm/lib/Target/X86/X86InstrMMX.td
index bb3e6df3bf3e..aeecc25ddea2 100644
--- a/llvm/lib/Target/X86/X86InstrMMX.td
+++ b/llvm/lib/Target/X86/X86InstrMMX.td
@@ -34,14 +34,14 @@ let Constraints = "$src1 = $dst" in {
   multiclass MMXI_binop_rm_int<bits<8> opc, string OpcodeStr, Intrinsic IntId,
                                X86FoldableSchedWrite sched, bit Commutable = 0,
                                X86MemOperand OType = i64mem> {
-    def irr : MMXI<opc, MRMSrcReg, (outs VR64:$dst),
+    def rr : MMXI<opc, MRMSrcReg, (outs VR64:$dst),
                  (ins VR64:$src1, VR64:$src2),
                  !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),
                  [(set VR64:$dst, (IntId VR64:$src1, VR64:$src2))]>,
               Sched<[sched]> {
       let isCommutable = Commutable;
     }
-    def irm : MMXI<opc, MRMSrcMem, (outs VR64:$dst),
+    def rm : MMXI<opc, MRMSrcMem, (outs VR64:$dst),
                  (ins VR64:$src1, OType:$src2),
                  !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),
                  [(set VR64:$dst, (IntId VR64:$src1, (load_mmx addr:$src2)))]>,
@@ -123,25 +123,25 @@ multiclass ssse3_palign_mm<string asm, Intrinsic IntId,
 multiclass sse12_cvt_pint<bits<8> opc, RegisterClass SrcRC, RegisterClass DstRC,
                          Intrinsic Int, X86MemOperand x86memop, PatFrag ld_frag,
                          string asm, X86FoldableSchedWrite sched, Domain d> {
-  def irr : MMXPI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src), asm,
-                  [(set DstRC:$dst, (Int SrcRC:$src))], d>,
-            Sched<[sched]>;
-  def irm : MMXPI<opc, MRMSrcMem, (outs DstRC:$dst), (ins x86memop:$src), asm,
-                  [(set DstRC:$dst, (Int (ld_frag addr:$src)))], d>,
-            Sched<[sched.Folded]>;
+  def rr : MMXPI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src), asm,
+                 [(set DstRC:$dst, (Int SrcRC:$src))], d>,
+           Sched<[sched]>;
+  def rm : MMXPI<opc, MRMSrcMem, (outs DstRC:$dst), (ins x86memop:$src), asm,
+                 [(set DstRC:$dst, (Int (ld_frag addr:$src)))], d>,
+           Sched<[sched.Folded]>;
 }
 
 multiclass sse12_cvt_pint_3addr<bits<8> opc, RegisterClass SrcRC,
                     RegisterClass DstRC, Intrinsic Int, X86MemOperand x86memop,
                     PatFrag ld_frag, string asm, Domain d> {
-  def irr : MMXPI<opc, MRMSrcReg, (outs DstRC:$dst),
-                  (ins DstRC:$src1, SrcRC:$src2), asm,
-                  [(set DstRC:$dst, (Int DstRC:$src1, SrcRC:$src2))], d>,
-                  Sched<[WriteCvtI2PS]>;
-  def irm : MMXPI<opc, MRMSrcMem, (outs DstRC:$dst),
-                  (ins DstRC:$src1, x86memop:$src2), asm,
-                  [(set DstRC:$dst, (Int DstRC:$src1, (ld_frag addr:$src2)))], d>,
-                  Sched<[WriteCvtI2PS.Folded]>;
+  def rr : MMXPI<opc, MRMSrcReg, (outs DstRC:$dst),
+                 (ins DstRC:$src1, SrcRC:$src2), asm,
+                 [(set DstRC:$dst, (Int DstRC:$src1, SrcRC:$src2))], d>,
+                 Sched<[WriteCvtI2PS]>;
+  def rm : MMXPI<opc, MRMSrcMem, (outs DstRC:$dst),
+                 (ins DstRC:$src1, x86memop:$src2), asm,
+                 [(set DstRC:$dst, (Int DstRC:$src1, (ld_frag addr:$src2)))], d>,
+                 Sched<[WriteCvtI2PS.Folded]>;
 }
 
 //===----------------------------------------------------------------------===//
@@ -569,14 +569,14 @@ def : Pat<(x86mmx (bitconvert (f64 FR64:$src))),
           (MMX_MOVFR642Qrr FR64:$src)>;
 def : Pat<(x86mmx (MMX_X86movdq2q
                    (bc_v2i64 (v4i32 (X86cvtp2Int (v4f32 VR128:$src)))))),
-          (MMX_CVTPS2PIirr VR128:$src)>;
+          (MMX_CVTPS2PIrr VR128:$src)>;
 def : Pat<(x86mmx (MMX_X86movdq2q
                    (bc_v2i64 (v4i32 (X86cvttp2si (v4f32 VR128:$src)))))),
-          (MMX_CVTTPS2PIirr VR128:$src)>;
+          (MMX_CVTTPS2PIrr VR128:$src)>;
 def : Pat<(x86mmx (MMX_X86movdq2q
                    (bc_v2i64 (v4i32 (X86cvtp2Int (v2f64 VR128:$src)))))),
-          (MMX_CVTPD2PIirr VR128:$src)>;
+          (MMX_CVTPD2PIrr VR128:$src)>;
 def : Pat<(x86mmx (MMX_X86movdq2q
                    (bc_v2i64 (v4i32 (X86cvttp2si (v2f64 VR128:$src)))))),
-          (MMX_CVTTPD2PIirr VR128:$src)>;
+          (MMX_CVTTPD2PIrr VR128:$src)>;
 }
diff --git a/llvm/lib/Target/X86/X86MCInstLower.cpp b/llvm/lib/Target/X86/X86MCInstLower.cpp
index c3cd634612a4..9044f10ec630 100644
--- a/llvm/lib/Target/X86/X86MCInstLower.cpp
+++ b/llvm/lib/Target/X86/X86MCInstLower.cpp
@@ -48,6 +48,7 @@
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
 #include "llvm/Transforms/Instrumentation/AddressSanitizerCommon.h"
+#include <string>
 
 using namespace llvm;
 
@@ -1336,235 +1337,29 @@ void X86AsmPrinter::LowerASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
     return;
   }
 
-  unsigned Reg = MI.getOperand(0).getReg().id();
+  const auto &Reg = MI.getOperand(0).getReg();
   ASanAccessInfo AccessInfo(MI.getOperand(1).getImm());
 
-  MCSymbol *&Sym =
-      AsanMemaccessSymbols[AsanMemaccessTuple(Reg, AccessInfo.Packed)];
-  if (!Sym) {
-    std::string Name = AccessInfo.IsWrite ? "store" : "load";
-    std::string SymName = "__asan_check_" + Name +
-                          utostr(1ULL << AccessInfo.AccessSizeIndex) + "_rn" +
-                          utostr(Reg);
-    Sym = OutContext.getOrCreateSymbol(SymName);
-  }
-
-  EmitAndCountInstruction(
-      MCInstBuilder(X86::CALL64pcrel32)
-          .addExpr(MCSymbolRefExpr::create(Sym, OutContext)));
-}
-
-void X86AsmPrinter::emitAsanMemaccessPartial(Module &M, unsigned Reg,
-                                             const ASanAccessInfo &AccessInfo,
-                                             MCSubtargetInfo &STI) {
-  assert(AccessInfo.AccessSizeIndex == 0 || AccessInfo.AccessSizeIndex == 1 ||
-         AccessInfo.AccessSizeIndex == 2);
-  assert(Reg != X86::R8);
-
   uint64_t ShadowBase;
   int MappingScale;
   bool OrShadowOffset;
-  getAddressSanitizerParams(
-      Triple(M.getTargetTriple()), M.getDataLayout().getPointerSizeInBits(),
-      AccessInfo.CompileKernel, &ShadowBase, &MappingScale, &OrShadowOffset);
-
-  OutStreamer->emitInstruction(
-      MCInstBuilder(X86::MOV64rr).addReg(X86::R8).addReg(X86::NoRegister + Reg),
-      STI);
-  OutStreamer->emitInstruction(MCInstBuilder(X86::SHR64ri)
-                                   .addReg(X86::R8)
-                                   .addReg(X86::R8)
-                                   .addImm(MappingScale),
-                               STI);
-  if (OrShadowOffset) {
-    OutStreamer->emitInstruction(MCInstBuilder(X86::OR64ri32)
-                                     .addReg(X86::R8)
-                                     .addReg(X86::R8)
-                                     .addImm(ShadowBase),
-                                 STI);
-    OutStreamer->emitInstruction(MCInstBuilder(X86::MOV8rm)
-                                     .addReg(X86::R8B)
-                                     .addReg(X86::R8)
-                                     .addImm(1)
-                                     .addReg(X86::NoRegister)
-                                     .addImm(0)
-                                     .addReg(X86::NoRegister),
-                                 STI);
-    OutStreamer->emitInstruction(
-        MCInstBuilder(X86::TEST8rr).addReg(X86::R8B).addReg(X86::R8B), STI);
-  } else {
-    OutStreamer->emitInstruction(MCInstBuilder(X86::MOVSX32rm8)
-                                     .addReg(X86::R8D)
-                                     .addReg(X86::R8)
-                                     .addImm(1)
-                                     .addReg(X86::NoRegister)
-                                     .addImm(ShadowBase)
-                                     .addReg(X86::NoRegister),
-                                 STI);
-    OutStreamer->emitInstruction(
-        MCInstBuilder(X86::TEST32rr).addReg(X86::R8D).addReg(X86::R8D), STI);
-  }
-  MCSymbol *AdditionalCheck = OutContext.createTempSymbol();
-  OutStreamer->emitInstruction(
-      MCInstBuilder(X86::JCC_1)
-          .addExpr(MCSymbolRefExpr::create(AdditionalCheck, OutContext))
-          .addImm(X86::COND_NE),
-      STI);
-  MCSymbol *ReturnSym = OutContext.createTempSymbol();
-  OutStreamer->emitLabel(ReturnSym);
-  OutStreamer->emitInstruction(MCInstBuilder(getRetOpcode(*Subtarget)), STI);
-
-  // Shadow byte is non-zero so we need to perform additional checks.
-  OutStreamer->emitLabel(AdditionalCheck);
-  OutStreamer->emitInstruction(MCInstBuilder(X86::PUSH64r).addReg(X86::RCX),
-                               STI);
-  OutStreamer->emitInstruction(MCInstBuilder(X86::MOV64rr)
-                                   .addReg(X86::RCX)
-                                   .addReg(X86::NoRegister + Reg),
-                               STI);
-  const size_t Granularity = 1ULL << MappingScale;
-  OutStreamer->emitInstruction(MCInstBuilder(X86::AND32ri8)
-                                   .addReg(X86::NoRegister)
-                                   .addReg(X86::ECX)
-                                   .addImm(Granularity - 1),
-                               STI);
-  if (AccessInfo.AccessSizeIndex == 1) {
-    OutStreamer->emitInstruction(MCInstBuilder(X86::ADD32ri8)
-                                     .addReg(X86::NoRegister)
-                                     .addReg(X86::ECX)
-                                     .addImm(1),
-                                 STI);
-  } else if (AccessInfo.AccessSizeIndex == 2) {
-    OutStreamer->emitInstruction(MCInstBuilder(X86::ADD32ri8)
-                                     .addReg(X86::NoRegister)
-                                     .addReg(X86::ECX)
-                                     .addImm(3),
-                                 STI);
-  }
-
-  OutStreamer->emitInstruction(
-      MCInstBuilder(X86::CMP32rr).addReg(X86::ECX).addReg(X86::R8D).addImm(1),
-      STI);
-  OutStreamer->emitInstruction(MCInstBuilder(X86::POP64r).addReg(X86::RCX),
-                               STI);
-  OutStreamer->emitInstruction(
-      MCInstBuilder(X86::JCC_1)
-          .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext))
-          .addImm(X86::COND_L),
-      STI);
-
-  emitAsanReportError(M, Reg, AccessInfo, STI);
-}
-
-void X86AsmPrinter::emitAsanMemaccessFull(Module &M, unsigned Reg,
-                                          const ASanAccessInfo &AccessInfo,
-                                          MCSubtargetInfo &STI) {
-  assert(AccessInfo.AccessSizeIndex == 3 || AccessInfo.AccessSizeIndex == 4);
-  assert(Reg != X86::R8);
+  getAddressSanitizerParams(Triple(TM.getTargetTriple()), 64,
+                            AccessInfo.CompileKernel, &ShadowBase,
+                            &MappingScale, &OrShadowOffset);
 
-  uint64_t ShadowBase;
-  int MappingScale;
-  bool OrShadowOffset;
-  getAddressSanitizerParams(
-      Triple(M.getTargetTriple()), M.getDataLayout().getPointerSizeInBits(),
-      AccessInfo.CompileKernel, &ShadowBase, &MappingScale, &OrShadowOffset);
-
-  OutStreamer->emitInstruction(
-      MCInstBuilder(X86::MOV64rr).addReg(X86::R8).addReg(X86::NoRegister + Reg),
-      STI);
-  OutStreamer->emitInstruction(MCInstBuilder(X86::SHR64ri)
-                                   .addReg(X86::R8)
-                                   .addReg(X86::R8)
-                                   .addImm(MappingScale),
-                               STI);
-  if (OrShadowOffset) {
-    OutStreamer->emitInstruction(MCInstBuilder(X86::OR64ri32)
-                                     .addReg(X86::R8)
-                                     .addReg(X86::R8)
-                                     .addImm(ShadowBase),
-                                 STI);
-    auto OpCode = AccessInfo.AccessSizeIndex == 3 ? X86::CMP8mi : X86::CMP16mi8;
-    OutStreamer->emitInstruction(MCInstBuilder(OpCode)
-                                     .addReg(X86::R8)
-                                     .addImm(1)
-                                     .addReg(X86::NoRegister)
-                                     .addImm(0)
-                                     .addReg(X86::NoRegister)
-                                     .addImm(0),
-                                 STI);
-  } else {
-    auto OpCode = AccessInfo.AccessSizeIndex == 3 ? X86::CMP8mi : X86::CMP16mi8;
-    OutStreamer->emitInstruction(MCInstBuilder(OpCode)
-                                     .addReg(X86::R8)
-                                     .addImm(1)
-                                     .addReg(X86::NoRegister)
-                                     .addImm(ShadowBase)
-                                     .addReg(X86::NoRegister)
-                                     .addImm(0),
-                                 STI);
-  }
-  MCSymbol *ReportCode = OutContext.createTempSymbol();
-  OutStreamer->emitInstruction(
-      MCInstBuilder(X86::JCC_1)
-          .addExpr(MCSymbolRefExpr::create(ReportCode, OutContext))
-          .addImm(X86::COND_NE),
-      STI);
-  MCSymbol *ReturnSym = OutContext.createTempSymbol();
-  OutStreamer->emitLabel(ReturnSym);
-  OutStreamer->emitInstruction(MCInstBuilder(getRetOpcode(*Subtarget)), STI);
-
-  OutStreamer->emitLabel(ReportCode);
-  emitAsanReportError(M, Reg, AccessInfo, STI);
-}
-
-void X86AsmPrinter::emitAsanReportError(Module &M, unsigned Reg,
-                                        const ASanAccessInfo &AccessInfo,
-                                        MCSubtargetInfo &STI) {
   std::string Name = AccessInfo.IsWrite ? "store" : "load";
-  MCSymbol *ReportError = OutContext.getOrCreateSymbol(
-      "__asan_report_" + Name + utostr(1ULL << AccessInfo.AccessSizeIndex));
-  OutStreamer->emitInstruction(MCInstBuilder(X86::MOV64rr)
-                                   .addReg(X86::RDI)
-                                   .addReg(X86::NoRegister + Reg),
-                               STI);
-  OutStreamer->emitInstruction(
-      MCInstBuilder(X86::JMP_4)
-          .addExpr(MCSymbolRefExpr::create(ReportError, MCSymbolRefExpr::VK_PLT,
-                                           OutContext)),
-      STI);
-}
+  std::string Op = OrShadowOffset ? "or" : "add";
+  std::string SymName = "__asan_check_" + Name + "_" + Op + "_" +
+                        utostr(1ULL << AccessInfo.AccessSizeIndex) + "_" +
+                        TM.getMCRegisterInfo()->getName(Reg.asMCReg());
+  if (OrShadowOffset)
+    report_fatal_error(
+        "OrShadowOffset is not supported with optimized callbacks");
 
-void X86AsmPrinter::emitAsanMemaccessSymbols(Module &M) {
-  if (AsanMemaccessSymbols.empty())
-    return;
-
-  const Triple &TT = TM.getTargetTriple();
-  assert(TT.isOSBinFormatELF());
-  std::unique_ptr<MCSubtargetInfo> STI(
-      TM.getTarget().createMCSubtargetInfo(TT.str(), "", ""));
-  assert(STI && "Unable to create subtarget info");
-
-  for (auto &P : AsanMemaccessSymbols) {
-    MCSymbol *Sym = P.second;
-    OutStreamer->SwitchSection(OutContext.getELFSection(
-        ".text.hot", ELF::SHT_PROGBITS,
-        ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, 0, Sym->getName(),
-        /*IsComdat=*/true));
-
-    OutStreamer->emitSymbolAttribute(Sym, MCSA_ELF_TypeFunction);
-    OutStreamer->emitSymbolAttribute(Sym, MCSA_Weak);
-    OutStreamer->emitSymbolAttribute(Sym, MCSA_Hidden);
-    OutStreamer->emitLabel(Sym);
-
-    unsigned Reg = std::get<0>(P.first);
-    ASanAccessInfo AccessInfo(std::get<1>(P.first));
-
-    if (AccessInfo.AccessSizeIndex < 3) {
-      emitAsanMemaccessPartial(M, Reg, AccessInfo, *STI);
-    } else {
-      emitAsanMemaccessFull(M, Reg, AccessInfo, *STI);
-    }
-  }
+  EmitAndCountInstruction(
+      MCInstBuilder(X86::CALL64pcrel32)
+          .addExpr(MCSymbolRefExpr::create(
+              OutContext.getOrCreateSymbol(SymName), OutContext)));
 }
 
 void X86AsmPrinter::LowerPATCHABLE_OP(const MachineInstr &MI,
@@ -2615,6 +2410,15 @@ void X86AsmPrinter::emitInstruction(const MachineInstr *MI) {
   const X86RegisterInfo *RI =
       MF->getSubtarget<X86Subtarget>().getRegisterInfo();
 
+  if (MI->getOpcode() == X86::OR64rm) {
+    for (auto &Opd : MI->operands()) {
+      if (Opd.isSymbol() && StringRef(Opd.getSymbolName()) ==
+                                "swift_async_extendedFramePointerFlags") {
+        ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = true;
+      }
+    }
+  }
+
   // Add a comment about EVEX-2-VEX compression for AVX-512 instrs that
   // are compressed from EVEX encoding to VEX encoding.
   if (TM.Options.MCOptions.ShowMCEncoding) {
diff --git a/llvm/lib/Target/X86/X86RegisterInfo.td b/llvm/lib/Target/X86/X86RegisterInfo.td
index d835f452b67e..1b704bcb8e08 100644
--- a/llvm/lib/Target/X86/X86RegisterInfo.td
+++ b/llvm/lib/Target/X86/X86RegisterInfo.td
@@ -430,11 +430,11 @@ def GR64 : RegisterClass<"X86", [i64], 64,
                          (add RAX, RCX, RDX, RSI, RDI, R8, R9, R10, R11,
                               RBX, R14, R15, R12, R13, RBP, RSP, RIP)>;
 
-// GR64 - 64-bit GPRs without R8 and RIP. Could be used when emitting code for
-// intrinsics, which use implict input registers.
-def GR64NoR8 : RegisterClass<"X86", [i64], 64,
-                              (add RAX, RCX, RDX, RSI, RDI, R9, R10, R11,
-                                   RBX, R14, R15, R12, R13, RBP, RSP)>;
+// GR64PLTSafe - 64-bit GPRs without R10, R11, RSP and RIP. Could be used when
+// emitting code for intrinsics, which use implict input registers.
+def GR64PLTSafe : RegisterClass<"X86", [i64], 64,
+                              (add RAX, RCX, RDX, RSI, RDI, R8, R9,
+                                   RBX, R14, R15, R12, R13, RBP)>;
 
 // Segment registers for use by MOV instructions (and others) that have a
 //   segment register as one operand.  Always contain a 16-bit segment
diff --git a/llvm/lib/Target/X86/X86SchedBroadwell.td b/llvm/lib/Target/X86/X86SchedBroadwell.td
index 2827981b7fb0..a6ff472aac6f 100644
--- a/llvm/lib/Target/X86/X86SchedBroadwell.td
+++ b/llvm/lib/Target/X86/X86SchedBroadwell.td
@@ -783,7 +783,7 @@ def BWWriteResGroup27 : SchedWriteRes<[BWPort1]> {
   let NumMicroOps = 1;
   let ResourceCycles = [1];
 }
-def: InstRW<[BWWriteResGroup27], (instrs MMX_CVTPI2PSirr)>;
+def: InstRW<[BWWriteResGroup27], (instrs MMX_CVTPI2PSrr)>;
 def: InstRW<[BWWriteResGroup27], (instregex "P(DEP|EXT)(32|64)rr",
                                             "(V?)CVTDQ2PS(Y?)rr")>;
 
@@ -800,9 +800,9 @@ def BWWriteResGroup33 : SchedWriteRes<[BWPort5,BWPort0156]> {
   let NumMicroOps = 3;
   let ResourceCycles = [2,1];
 }
-def: InstRW<[BWWriteResGroup33], (instrs MMX_PACKSSDWirr,
-                                         MMX_PACKSSWBirr,
-                                         MMX_PACKUSWBirr)>;
+def: InstRW<[BWWriteResGroup33], (instrs MMX_PACKSSDWrr,
+                                         MMX_PACKSSWBrr,
+                                         MMX_PACKUSWBrr)>;
 
 def BWWriteResGroup34 : SchedWriteRes<[BWPort6,BWPort0156]> {
   let Latency = 3;
@@ -862,9 +862,9 @@ def BWWriteResGroup42 : SchedWriteRes<[BWPort1,BWPort5]> {
   let NumMicroOps = 2;
   let ResourceCycles = [1,1];
 }
-def: InstRW<[BWWriteResGroup42], (instrs MMX_CVTPI2PDirr)>;
-def: InstRW<[BWWriteResGroup42], (instregex "MMX_CVT(T?)PD2PIirr",
-                                            "MMX_CVT(T?)PS2PIirr",
+def: InstRW<[BWWriteResGroup42], (instrs MMX_CVTPI2PDrr)>;
+def: InstRW<[BWWriteResGroup42], (instregex "MMX_CVT(T?)PD2PIrr",
+                                            "MMX_CVT(T?)PS2PIrr",
                                             "(V?)CVTDQ2PDrr",
                                             "(V?)CVTPD2PSrr",
                                             "(V?)CVTSD2SSrr",
@@ -1086,9 +1086,9 @@ def BWWriteResGroup79 : SchedWriteRes<[BWPort5,BWPort23]> {
   let NumMicroOps = 3;
   let ResourceCycles = [2,1];
 }
-def: InstRW<[BWWriteResGroup79], (instrs MMX_PACKSSDWirm,
-                                         MMX_PACKSSWBirm,
-                                         MMX_PACKUSWBirm)>;
+def: InstRW<[BWWriteResGroup79], (instrs MMX_PACKSSDWrm,
+                                         MMX_PACKSSWBrm,
+                                         MMX_PACKUSWBrm)>;
 
 def BWWriteResGroup80 : SchedWriteRes<[BWPort23,BWPort0156]> {
   let Latency = 7;
@@ -1155,7 +1155,7 @@ def BWWriteResGroup91 : SchedWriteRes<[BWPort1,BWPort23]> {
   let NumMicroOps = 2;
   let ResourceCycles = [1,1];
 }
-def: InstRW<[BWWriteResGroup91], (instrs MMX_CVTPI2PSirm,
+def: InstRW<[BWWriteResGroup91], (instrs MMX_CVTPI2PSrm,
                                          CVTDQ2PSrm,
                                          VCVTDQ2PSrm)>;
 def: InstRW<[BWWriteResGroup91], (instregex "P(DEP|EXT)(32|64)rm")>;
@@ -1236,8 +1236,8 @@ def BWWriteResGroup107 : SchedWriteRes<[BWPort1,BWPort5,BWPort23]> {
 def: InstRW<[BWWriteResGroup107], (instrs CVTPD2PSrm,
                                           CVTPD2DQrm,
                                           CVTTPD2DQrm,
-                                          MMX_CVTPI2PDirm)>;
-def: InstRW<[BWWriteResGroup107], (instregex "MMX_CVT(T?)PD2PIirm",
+                                          MMX_CVTPI2PDrm)>;
+def: InstRW<[BWWriteResGroup107], (instregex "MMX_CVT(T?)PD2PIrm",
                                              "(V?)CVTDQ2PDrm",
                                              "(V?)CVTSD2SSrm")>;
 
diff --git a/llvm/lib/Target/X86/X86SchedHaswell.td b/llvm/lib/Target/X86/X86SchedHaswell.td
index 68961d6245ab..371a9571ae39 100644
--- a/llvm/lib/Target/X86/X86SchedHaswell.td
+++ b/llvm/lib/Target/X86/X86SchedHaswell.td
@@ -995,7 +995,7 @@ def HWWriteResGroup12 : SchedWriteRes<[HWPort1,HWPort23]> {
   let NumMicroOps = 2;
   let ResourceCycles = [1,1];
 }
-def: InstRW<[HWWriteResGroup12], (instrs MMX_CVTPI2PSirm)>;
+def: InstRW<[HWWriteResGroup12], (instrs MMX_CVTPI2PSrm)>;
 def: InstRW<[HWWriteResGroup12], (instregex "P(DEP|EXT)(32|64)rm")>;
 
 def HWWriteResGroup13 : SchedWriteRes<[HWPort5,HWPort23]> {
@@ -1164,9 +1164,9 @@ def HWWriteResGroup36_2 : SchedWriteRes<[HWPort5,HWPort23]> {
   let NumMicroOps = 3;
   let ResourceCycles = [2,1];
 }
-def: InstRW<[HWWriteResGroup36_2], (instrs MMX_PACKSSDWirm,
-                                           MMX_PACKSSWBirm,
-                                           MMX_PACKUSWBirm)>;
+def: InstRW<[HWWriteResGroup36_2], (instrs MMX_PACKSSDWrm,
+                                           MMX_PACKSSWBrm,
+                                           MMX_PACKUSWBrm)>;
 
 def HWWriteResGroup37 : SchedWriteRes<[HWPort23,HWPort0156]> {
   let Latency = 7;
@@ -1240,7 +1240,7 @@ def HWWriteResGroup50 : SchedWriteRes<[HWPort1]> {
   let NumMicroOps = 1;
   let ResourceCycles = [1];
 }
-def: InstRW<[HWWriteResGroup50], (instrs MMX_CVTPI2PSirr)>;
+def: InstRW<[HWWriteResGroup50], (instrs MMX_CVTPI2PSrr)>;
 def: InstRW<[HWWriteResGroup50], (instregex "P(DEP|EXT)(32|64)rr",
                                             "(V?)CVTDQ2PS(Y?)rr")>;
 
@@ -1285,9 +1285,9 @@ def HWWriteResGroup57 : SchedWriteRes<[HWPort5,HWPort0156]> {
   let NumMicroOps = 3;
   let ResourceCycles = [2,1];
 }
-def: InstRW<[HWWriteResGroup57], (instrs MMX_PACKSSDWirr,
-                                         MMX_PACKSSWBirr,
-                                         MMX_PACKUSWBirr)>;
+def: InstRW<[HWWriteResGroup57], (instrs MMX_PACKSSDWrr,
+                                         MMX_PACKSSWBrr,
+                                         MMX_PACKUSWBrr)>;
 
 def HWWriteResGroup58 : SchedWriteRes<[HWPort6,HWPort0156]> {
   let Latency = 3;
@@ -1373,11 +1373,11 @@ def HWWriteResGroup73 : SchedWriteRes<[HWPort1,HWPort5]> {
   let NumMicroOps = 2;
   let ResourceCycles = [1,1];
 }
-def: InstRW<[HWWriteResGroup73], (instrs MMX_CVTPI2PDirr,
-                                         MMX_CVTPD2PIirr,
-                                         MMX_CVTPS2PIirr,
-                                         MMX_CVTTPD2PIirr,
-                                         MMX_CVTTPS2PIirr)>;
+def: InstRW<[HWWriteResGroup73], (instrs MMX_CVTPI2PDrr,
+                                         MMX_CVTPD2PIrr,
+                                         MMX_CVTPS2PIrr,
+                                         MMX_CVTTPD2PIrr,
+                                         MMX_CVTTPS2PIrr)>;
 def: InstRW<[HWWriteResGroup73], (instregex "(V?)CVTDQ2PDrr",
                                             "(V?)CVTPD2PSrr",
                                             "(V?)CVTSD2SSrr",
@@ -1418,8 +1418,8 @@ def HWWriteResGroup78 : SchedWriteRes<[HWPort1,HWPort5,HWPort23]> {
 def: InstRW<[HWWriteResGroup78], (instrs CVTPD2PSrm,
                                          CVTPD2DQrm,
                                          CVTTPD2DQrm,
-                                         MMX_CVTPD2PIirm,
-                                         MMX_CVTTPD2PIirm,
+                                         MMX_CVTPD2PIrm,
+                                         MMX_CVTTPD2PIrm,
                                          CVTDQ2PDrm,
                                          VCVTDQ2PDrm)>;
 
@@ -1428,7 +1428,7 @@ def HWWriteResGroup78_1 : SchedWriteRes<[HWPort1,HWPort5,HWPort23]> {
   let NumMicroOps = 3;
   let ResourceCycles = [1,1,1];
 }
-def: InstRW<[HWWriteResGroup78_1], (instrs MMX_CVTPI2PDirm,
+def: InstRW<[HWWriteResGroup78_1], (instrs MMX_CVTPI2PDrm,
                                            CVTSD2SSrm, CVTSD2SSrm_Int,
                                            VCVTSD2SSrm, VCVTSD2SSrm_Int)>;
 
diff --git a/llvm/lib/Target/X86/X86SchedIceLake.td b/llvm/lib/Target/X86/X86SchedIceLake.td
index 889b9b7fa666..789de9eb5751 100644
--- a/llvm/lib/Target/X86/X86SchedIceLake.td
+++ b/llvm/lib/Target/X86/X86SchedIceLake.td
@@ -331,12 +331,12 @@ defm : ICXWriteResPair<WriteFLogicZ, [ICXPort05], 1, [1], 1, 7>;
 defm : ICXWriteResPair<WriteFTest,  [ICXPort0], 2, [1], 1, 6>; // Floating point TEST instructions.
 defm : ICXWriteResPair<WriteFTestY, [ICXPort0], 2, [1], 1, 7>;
 defm : ICXWriteResPair<WriteFTestZ, [ICXPort0], 2, [1], 1, 7>;
-defm : ICXWriteResPair<WriteFShuffle,  [ICXPort5], 1, [1], 1, 6>; // Floating point vector shuffles.
-defm : ICXWriteResPair<WriteFShuffleY, [ICXPort5], 1, [1], 1, 7>;
-defm : ICXWriteResPair<WriteFShuffleZ, [ICXPort5], 1, [1], 1, 7>;
-defm : ICXWriteResPair<WriteFVarShuffle,  [ICXPort5], 1, [1], 1, 6>; // Floating point vector variable shuffles.
-defm : ICXWriteResPair<WriteFVarShuffleY, [ICXPort5], 1, [1], 1, 7>;
-defm : ICXWriteResPair<WriteFVarShuffleZ, [ICXPort5], 1, [1], 1, 7>;
+defm : ICXWriteResPair<WriteFShuffle,  [ICXPort15], 1, [1], 1, 6>; // Floating point vector shuffles.
+defm : ICXWriteResPair<WriteFShuffleY, [ICXPort15], 1, [1], 1, 7>;
+defm : ICXWriteResPair<WriteFShuffleZ, [ICXPort5],  1, [1], 1, 7>;
+defm : ICXWriteResPair<WriteFVarShuffle,  [ICXPort15], 1, [1], 1, 6>; // Floating point vector variable shuffles.
+defm : ICXWriteResPair<WriteFVarShuffleY, [ICXPort15], 1, [1], 1, 7>;
+defm : ICXWriteResPair<WriteFVarShuffleZ, [ICXPort5],  1, [1], 1, 7>;
 defm : ICXWriteResPair<WriteFBlend, [ICXPort015], 1, [1], 1, 6>; // Floating point vector blends.
 defm : ICXWriteResPair<WriteFBlendY,[ICXPort015], 1, [1], 1, 7>;
 defm : ICXWriteResPair<WriteFBlendZ,[ICXPort015], 1, [1], 1, 7>;
@@ -388,14 +388,14 @@ defm : ICXWriteResPair<WriteVecIMulZ, [ICXPort05],  5, [1], 1, 7>;
 defm : ICXWriteResPair<WritePMULLD,   [ICXPort01], 10, [2], 2, 6>; // Vector PMULLD.
 defm : ICXWriteResPair<WritePMULLDY,  [ICXPort01], 10, [2], 2, 7>;
 defm : ICXWriteResPair<WritePMULLDZ,  [ICXPort05], 10, [2], 2, 7>;
-defm : ICXWriteResPair<WriteShuffle,  [ICXPort5], 1, [1], 1, 5>; // Vector shuffles.
-defm : ICXWriteResPair<WriteShuffleX, [ICXPort5], 1, [1], 1, 6>;
-defm : ICXWriteResPair<WriteShuffleY, [ICXPort5], 1, [1], 1, 7>;
-defm : ICXWriteResPair<WriteShuffleZ, [ICXPort5], 1, [1], 1, 7>;
-defm : ICXWriteResPair<WriteVarShuffle,  [ICXPort5], 1, [1], 1, 5>; // Vector variable shuffles.
-defm : ICXWriteResPair<WriteVarShuffleX, [ICXPort5], 1, [1], 1, 6>;
-defm : ICXWriteResPair<WriteVarShuffleY, [ICXPort5], 1, [1], 1, 7>;
-defm : ICXWriteResPair<WriteVarShuffleZ, [ICXPort5], 1, [1], 1, 7>;
+defm : ICXWriteResPair<WriteShuffle,  [ICXPort5],  1, [1], 1, 5>; // Vector shuffles.
+defm : ICXWriteResPair<WriteShuffleX, [ICXPort15], 1, [1], 1, 6>;
+defm : ICXWriteResPair<WriteShuffleY, [ICXPort15], 1, [1], 1, 7>;
+defm : ICXWriteResPair<WriteShuffleZ, [ICXPort5],  1, [1], 1, 7>;
+defm : ICXWriteResPair<WriteVarShuffle,  [ICXPort5],  1, [1], 1, 5>; // Vector variable shuffles.
+defm : ICXWriteResPair<WriteVarShuffleX, [ICXPort15], 1, [1], 1, 6>;
+defm : ICXWriteResPair<WriteVarShuffleY, [ICXPort15], 1, [1], 1, 7>;
+defm : ICXWriteResPair<WriteVarShuffleZ, [ICXPort5],  1, [1], 1, 7>;
 defm : ICXWriteResPair<WriteBlend, [ICXPort5], 1, [1], 1, 6>; // Vector blends.
 defm : ICXWriteResPair<WriteBlendY,[ICXPort5], 1, [1], 1, 7>;
 defm : ICXWriteResPair<WriteBlendZ,[ICXPort5], 1, [1], 1, 7>;
@@ -642,15 +642,15 @@ def: InstRW<[ICXWriteResGroup1], (instregex "KAND(B|D|Q|W)rr",
                                             "KXOR(B|D|Q|W)rr",
                                             "KSET0(B|D|Q|W)", // Same as KXOR
                                             "KSET1(B|D|Q|W)", // Same as KXNOR
-                                            "MMX_PADDS(B|W)irr",
-                                            "MMX_PADDUS(B|W)irr",
-                                            "MMX_PAVG(B|W)irr",
-                                            "MMX_PCMPEQ(B|D|W)irr",
-                                            "MMX_PCMPGT(B|D|W)irr",
-                                            "MMX_P(MAX|MIN)SWirr",
-                                            "MMX_P(MAX|MIN)UBirr",
-                                            "MMX_PSUBS(B|W)irr",
-                                            "MMX_PSUBUS(B|W)irr",
+                                            "MMX_PADDS(B|W)rr",
+                                            "MMX_PADDUS(B|W)rr",
+                                            "MMX_PAVG(B|W)rr",
+                                            "MMX_PCMPEQ(B|D|W)rr",
+                                            "MMX_PCMPGT(B|D|W)rr",
+                                            "MMX_P(MAX|MIN)SWrr",
+                                            "MMX_P(MAX|MIN)UBrr",
+                                            "MMX_PSUBS(B|W)rr",
+                                            "MMX_PSUBUS(B|W)rr",
                                             "VPMOVB2M(Z|Z128|Z256)rr",
                                             "VPMOVD2M(Z|Z128|Z256)rr",
                                             "VPMOVQ2M(Z|Z128|Z256)rr",
@@ -663,7 +663,16 @@ def ICXWriteResGroup3 : SchedWriteRes<[ICXPort5]> {
 }
 def: InstRW<[ICXWriteResGroup3], (instregex "COM(P?)_FST0r",
                                             "KMOV(B|D|Q|W)kr",
-                                            "UCOM_F(P?)r")>;
+                                            "UCOM_F(P?)r",
+                                            "VPBROADCAST(D|Q)rr",
+                                            "(V?)INSERTPS(Z?)rr",
+                                            "(V?)MOV(HL|LH)PS(Z?)rr",
+                                            "(V?)MOVDDUP(Y|Z|Z128|Z256)?rr",
+                                            "(V?)PALIGNR(Y|Z|Z128|Z256)?rri",
+                                            "(V?)PERMIL(PD|PS)(Y|Z|Z128|Z256)?ri",
+                                            "(V?)PERMIL(PD|PS)(Y|Z|Z128|Z256)?rr",
+                                            "(V?)PACK(U|S)S(DW|WB)(Y|Z|Z128|Z256)?rr",
+                                            "(V?)UNPCK(L|H)(PD|PS)(Y|Z|Z128|Z256)?rr")>;
 
 def ICXWriteResGroup4 : SchedWriteRes<[ICXPort6]> {
   let Latency = 1;
@@ -702,6 +711,7 @@ def: InstRW<[ICXWriteResGroup9], (instregex "VBLENDMPD(Z128|Z256)rr",
                                             "VBLENDMPS(Z128|Z256)rr",
                                             "VPADD(B|D|Q|W)(Y|Z|Z128|Z256)rr",
                                             "(V?)PADD(B|D|Q|W)rr",
+                                            "(V?)MOV(SD|SS)(Z?)rr",
                                             "VPBLENDD(Y?)rri",
                                             "VPBLENDMB(Z128|Z256)rr",
                                             "VPBLENDMD(Z128|Z256)rr",
@@ -892,9 +902,9 @@ def ICXWriteResGroup41 : SchedWriteRes<[ICXPort5,ICXPort0156]> {
   let NumMicroOps = 3;
   let ResourceCycles = [2,1];
 }
-def: InstRW<[ICXWriteResGroup41], (instrs MMX_PACKSSDWirr,
-                                          MMX_PACKSSWBirr,
-                                          MMX_PACKUSWBirr)>;
+def: InstRW<[ICXWriteResGroup41], (instrs MMX_PACKSSDWrr,
+                                          MMX_PACKSSWBrr,
+                                          MMX_PACKUSWBrr)>;
 
 def ICXWriteResGroup42 : SchedWriteRes<[ICXPort6,ICXPort0156]> {
   let Latency = 3;
@@ -1055,8 +1065,8 @@ def ICXWriteResGroup61 : SchedWriteRes<[ICXPort5,ICXPort015]> {
   let NumMicroOps = 2;
   let ResourceCycles = [1,1];
 }
-def: InstRW<[ICXWriteResGroup61], (instregex "MMX_CVT(T?)PD2PIirr",
-                                             "MMX_CVT(T?)PS2PIirr",
+def: InstRW<[ICXWriteResGroup61], (instregex "MMX_CVT(T?)PD2PIrr",
+                                             "MMX_CVT(T?)PS2PIrr",
                                              "VCVTDQ2PDZ128rr",
                                              "VCVTPD2DQZ128rr",
                                              "(V?)CVT(T?)PD2DQrr",
@@ -1162,7 +1172,7 @@ def ICXWriteResGroup72 : SchedWriteRes<[ICXPort5]> {
   let NumMicroOps = 2;
   let ResourceCycles = [2];
 }
-def: InstRW<[ICXWriteResGroup72], (instrs MMX_CVTPI2PSirr)>;
+def: InstRW<[ICXWriteResGroup72], (instrs MMX_CVTPI2PSrr)>;
 def: InstRW<[ICXWriteResGroup72], (instregex "VCOMPRESSPD(Z|Z128|Z256)rr",
                                              "VCOMPRESSPS(Z|Z128|Z256)rr",
                                              "VPCOMPRESSD(Z|Z128|Z256)rr",
@@ -1174,26 +1184,26 @@ def ICXWriteResGroup73 : SchedWriteRes<[ICXPort0,ICXPort23]> {
   let NumMicroOps = 2;
   let ResourceCycles = [1,1];
 }
-def: InstRW<[ICXWriteResGroup73], (instrs MMX_PADDSBirm,
-                                          MMX_PADDSWirm,
-                                          MMX_PADDUSBirm,
-                                          MMX_PADDUSWirm,
-                                          MMX_PAVGBirm,
-                                          MMX_PAVGWirm,
-                                          MMX_PCMPEQBirm,
-                                          MMX_PCMPEQDirm,
-                                          MMX_PCMPEQWirm,
-                                          MMX_PCMPGTBirm,
-                                          MMX_PCMPGTDirm,
-                                          MMX_PCMPGTWirm,
-                                          MMX_PMAXSWirm,
-                                          MMX_PMAXUBirm,
-                                          MMX_PMINSWirm,
-                                          MMX_PMINUBirm,
-                                          MMX_PSUBSBirm,
-                                          MMX_PSUBSWirm,
-                                          MMX_PSUBUSBirm,
-                                          MMX_PSUBUSWirm)>;
+def: InstRW<[ICXWriteResGroup73], (instrs MMX_PADDSBrm,
+                                          MMX_PADDSWrm,
+                                          MMX_PADDUSBrm,
+                                          MMX_PADDUSWrm,
+                                          MMX_PAVGBrm,
+                                          MMX_PAVGWrm,
+                                          MMX_PCMPEQBrm,
+                                          MMX_PCMPEQDrm,
+                                          MMX_PCMPEQWrm,
+                                          MMX_PCMPGTBrm,
+                                          MMX_PCMPGTDrm,
+                                          MMX_PCMPGTWrm,
+                                          MMX_PMAXSWrm,
+                                          MMX_PMAXUBrm,
+                                          MMX_PMINSWrm,
+                                          MMX_PMINUBrm,
+                                          MMX_PSUBSBrm,
+                                          MMX_PSUBSWrm,
+                                          MMX_PSUBUSBrm,
+                                          MMX_PSUBUSWrm)>;
 
 def ICXWriteResGroup76 : SchedWriteRes<[ICXPort6,ICXPort23]> {
   let Latency = 6;
@@ -1295,20 +1305,14 @@ def ICXWriteResGroup92 : SchedWriteRes<[ICXPort5,ICXPort23]> {
   let NumMicroOps = 2;
   let ResourceCycles = [1,1];
 }
-def: InstRW<[ICXWriteResGroup92], (instregex "VMOVSDZrm(b?)",
-                                             "VMOVSSZrm(b?)")>;
-
-def ICXWriteResGroup92a : SchedWriteRes<[ICXPort5,ICXPort23]> {
-  let Latency = 6;
-  let NumMicroOps = 2;
-  let ResourceCycles = [1,1];
-}
-def: InstRW<[ICXWriteResGroup92a], (instregex "(V?)PMOV(SX|ZX)BDrm",
-                                              "(V?)PMOV(SX|ZX)BQrm",
-                                              "(V?)PMOV(SX|ZX)BWrm",
-                                              "(V?)PMOV(SX|ZX)DQrm",
-                                              "(V?)PMOV(SX|ZX)WDrm",
-                                              "(V?)PMOV(SX|ZX)WQrm")>;
+def: InstRW<[ICXWriteResGroup92], (instregex "VMOV(SD|SS)Zrm(b?)",
+                                              "VPBROADCAST(B|W)(Z128)?rm",
+                                             "(V?)INSERTPS(Z?)rm",
+                                             "(V?)PALIGNR(Z128)?rmi",
+                                             "(V?)PERMIL(PD|PS)(Z128)?m(b?)i",
+                                             "(V?)PERMIL(PD|PS)(Z128)?rm",
+                                             "(V?)PACK(U|S)S(DW|WB)(Z128)?rm",
+                                             "(V?)UNPCK(L|H)(PD|PS)(Z128)?rm")>;
 
 def ICXWriteResGroup93 : SchedWriteRes<[ICXPort5,ICXPort015]> {
   let Latency = 7;
@@ -1391,9 +1395,9 @@ def ICXWriteResGroup96 : SchedWriteRes<[ICXPort5,ICXPort23]> {
   let NumMicroOps = 3;
   let ResourceCycles = [2,1];
 }
-def: InstRW<[ICXWriteResGroup96], (instrs MMX_PACKSSDWirm,
-                                          MMX_PACKSSWBirm,
-                                          MMX_PACKUSWBirm)>;
+def: InstRW<[ICXWriteResGroup96], (instrs MMX_PACKSSDWrm,
+                                          MMX_PACKSSWBrm,
+                                          MMX_PACKUSWBrm)>;
 
 def ICXWriteResGroup97 : SchedWriteRes<[ICXPort5,ICXPort015]> {
   let Latency = 7;
@@ -1546,7 +1550,12 @@ def ICXWriteResGroup119 : SchedWriteRes<[ICXPort5,ICXPort23]> {
 }
 def: InstRW<[ICXWriteResGroup119], (instregex "FCOM(P?)(32|64)m",
                                               "VPBROADCASTB(Z|Z256)rm(b?)",
-                                              "VPBROADCASTW(Z|Z256)rm(b?)")>;
+                                              "VPBROADCASTW(Z|Z256)rm(b?)",
+                                              "(V?)PALIGNR(Y|Z|Z256)rmi",
+                                              "(V?)PERMIL(PD|PS)(Y|Z|Z256)m(b?)i",
+                                              "(V?)PERMIL(PD|PS)(Y|Z|Z256)rm",
+                                              "(V?)PACK(U|S)S(DW|WB)(Y|Z|Z256)rm",
+                                              "(V?)UNPCK(L|H)(PD|PS)(Y|Z|Z256)rm")>;
 def: InstRW<[ICXWriteResGroup119], (instrs VPBROADCASTBYrm,
                                            VPBROADCASTWYrm,
                                            VPMOVSXBDYrm,
@@ -1683,7 +1692,7 @@ def ICXWriteResGroup135 : SchedWriteRes<[ICXPort0,ICXPort23]> {
   let NumMicroOps = 2;
   let ResourceCycles = [1,1];
 }
-def: InstRW<[ICXWriteResGroup135], (instrs MMX_CVTPI2PSirm)>;
+def: InstRW<[ICXWriteResGroup135], (instrs MMX_CVTPI2PSrm)>;
 
 def ICXWriteResGroup136 : SchedWriteRes<[ICXPort5,ICXPort23]> {
   let Latency = 9;
@@ -1709,19 +1718,7 @@ def: InstRW<[ICXWriteResGroup136], (instregex "VALIGN(D|Q)Z128rm(b?)i",
                                               "VPMAXSQZ128rm(b?)",
                                               "VPMAXUQZ128rm(b?)",
                                               "VPMINSQZ128rm(b?)",
-                                              "VPMINUQZ128rm(b?)",
-                                              "VPMOVSXBDZ128rm(b?)",
-                                              "VPMOVSXBQZ128rm(b?)",
-                                              "VPMOVSXBWZ128rm(b?)",
-                                              "VPMOVSXDQZ128rm(b?)",
-                                              "VPMOVSXWDZ128rm(b?)",
-                                              "VPMOVSXWQZ128rm(b?)",
-                                              "VPMOVZXBDZ128rm(b?)",
-                                              "VPMOVZXBQZ128rm(b?)",
-                                              "VPMOVZXBWZ128rm(b?)",
-                                              "VPMOVZXDQZ128rm(b?)",
-                                              "VPMOVZXWDZ128rm(b?)",
-                                              "VPMOVZXWQZ128rm(b?)")>;
+                                              "VPMINUQZ128rm(b?)")>;
 
 def ICXWriteResGroup136_2 : SchedWriteRes<[ICXPort5,ICXPort23]> {
   let Latency = 10;
@@ -1753,7 +1750,7 @@ def ICXWriteResGroup137 : SchedWriteRes<[ICXPort23,ICXPort015]> {
   let NumMicroOps = 2;
   let ResourceCycles = [1,1];
 }
-def: InstRW<[ICXWriteResGroup137], (instregex "MMX_CVT(T?)PS2PIirm",
+def: InstRW<[ICXWriteResGroup137], (instregex "MMX_CVT(T?)PS2PIrm",
                                               "(V?)CVTPS2PDrm")>;
 
 def ICXWriteResGroup143 : SchedWriteRes<[ICXPort5,ICXPort01,ICXPort23]> {
@@ -1950,8 +1947,8 @@ def ICXWriteResGroup166 : SchedWriteRes<[ICXPort5,ICXPort23,ICXPort015]> {
 def: InstRW<[ICXWriteResGroup166], (instrs CVTPD2PSrm,
                                            CVTPD2DQrm,
                                            CVTTPD2DQrm,
-                                           MMX_CVTPD2PIirm,
-                                           MMX_CVTTPD2PIirm)>;
+                                           MMX_CVTPD2PIrm,
+                                           MMX_CVTTPD2PIrm)>;
 
 def ICXWriteResGroup167 : SchedWriteRes<[ICXPort5,ICXPort23,ICXPort015]> {
   let Latency = 11;
diff --git a/llvm/lib/Target/X86/X86SchedSandyBridge.td b/llvm/lib/Target/X86/X86SchedSandyBridge.td
index c8d7b0f72c1c..af5c0540deb5 100644
--- a/llvm/lib/Target/X86/X86SchedSandyBridge.td
+++ b/llvm/lib/Target/X86/X86SchedSandyBridge.td
@@ -623,7 +623,7 @@ def SBWriteResGroup5 : SchedWriteRes<[SBPort15]> {
 def: InstRW<[SBWriteResGroup5], (instrs MMX_PABSBrr,
                                         MMX_PABSDrr,
                                         MMX_PABSWrr,
-                                        MMX_PADDQirr,
+                                        MMX_PADDQrr,
                                         MMX_PALIGNRrri,
                                         MMX_PSIGNBrr,
                                         MMX_PSIGNDrr,
@@ -870,7 +870,7 @@ def SBWriteResGroup59 : SchedWriteRes<[SBPort23,SBPort15]> {
   let NumMicroOps = 2;
   let ResourceCycles = [1,1];
 }
-def: InstRW<[SBWriteResGroup59], (instrs MMX_PADDQirm)>;
+def: InstRW<[SBWriteResGroup59], (instrs MMX_PADDQrm)>;
 
 def SBWriteResGroup62 : SchedWriteRes<[SBPort5,SBPort23]> {
   let Latency = 7;
diff --git a/llvm/lib/Target/X86/X86SchedSkylakeClient.td b/llvm/lib/Target/X86/X86SchedSkylakeClient.td
index 7d3229c3b023..b3c13c72dd01 100644
--- a/llvm/lib/Target/X86/X86SchedSkylakeClient.td
+++ b/llvm/lib/Target/X86/X86SchedSkylakeClient.td
@@ -624,15 +624,15 @@ def SKLWriteResGroup1 : SchedWriteRes<[SKLPort0]> {
   let NumMicroOps = 1;
   let ResourceCycles = [1];
 }
-def: InstRW<[SKLWriteResGroup1], (instregex "MMX_PADDS(B|W)irr",
-                                            "MMX_PADDUS(B|W)irr",
-                                            "MMX_PAVG(B|W)irr",
-                                            "MMX_PCMPEQ(B|D|W)irr",
-                                            "MMX_PCMPGT(B|D|W)irr",
-                                            "MMX_P(MAX|MIN)SWirr",
-                                            "MMX_P(MAX|MIN)UBirr",
-                                            "MMX_PSUBS(B|W)irr",
-                                            "MMX_PSUBUS(B|W)irr")>;
+def: InstRW<[SKLWriteResGroup1], (instregex "MMX_PADDS(B|W)rr",
+                                            "MMX_PADDUS(B|W)rr",
+                                            "MMX_PAVG(B|W)rr",
+                                            "MMX_PCMPEQ(B|D|W)rr",
+                                            "MMX_PCMPGT(B|D|W)rr",
+                                            "MMX_P(MAX|MIN)SWrr",
+                                            "MMX_P(MAX|MIN)UBrr",
+                                            "MMX_PSUBS(B|W)rr",
+                                            "MMX_PSUBUS(B|W)rr")>;
 
 def SKLWriteResGroup3 : SchedWriteRes<[SKLPort5]> {
   let Latency = 1;
@@ -815,9 +815,9 @@ def SKLWriteResGroup39 : SchedWriteRes<[SKLPort5,SKLPort0156]> {
   let NumMicroOps = 3;
   let ResourceCycles = [2,1];
 }
-def: InstRW<[SKLWriteResGroup39], (instrs MMX_PACKSSDWirr,
-                                          MMX_PACKSSWBirr,
-                                          MMX_PACKUSWBirr)>;
+def: InstRW<[SKLWriteResGroup39], (instrs MMX_PACKSSDWrr,
+                                          MMX_PACKSSWBrr,
+                                          MMX_PACKUSWBrr)>;
 
 def SKLWriteResGroup40 : SchedWriteRes<[SKLPort6,SKLPort0156]> {
   let Latency = 3;
@@ -927,7 +927,7 @@ def SKLWriteResGroup59 : SchedWriteRes<[SKLPort0,SKLPort5]> {
   let NumMicroOps = 2;
   let ResourceCycles = [1,1];
 }
-def: InstRW<[SKLWriteResGroup59], (instrs MMX_CVTPI2PDirr,
+def: InstRW<[SKLWriteResGroup59], (instrs MMX_CVTPI2PDrr,
                                           CVTDQ2PDrr,
                                           VCVTDQ2PDrr)>;
 
@@ -936,8 +936,8 @@ def SKLWriteResGroup60 : SchedWriteRes<[SKLPort5,SKLPort015]> {
   let NumMicroOps = 2;
   let ResourceCycles = [1,1];
 }
-def: InstRW<[SKLWriteResGroup60], (instregex "MMX_CVT(T?)PD2PIirr",
-                                             "MMX_CVT(T?)PS2PIirr",
+def: InstRW<[SKLWriteResGroup60], (instregex "MMX_CVT(T?)PD2PIrr",
+                                             "MMX_CVT(T?)PS2PIrr",
                                              "(V?)CVT(T?)PD2DQrr",
                                              "(V?)CVTPD2PSrr",
                                              "(V?)CVTPS2PDrr",
@@ -984,33 +984,33 @@ def SKLWriteResGroup68 : SchedWriteRes<[SKLPort0]> {
   let NumMicroOps = 2;
   let ResourceCycles = [2];
 }
-def: InstRW<[SKLWriteResGroup68], (instrs MMX_CVTPI2PSirr)>;
+def: InstRW<[SKLWriteResGroup68], (instrs MMX_CVTPI2PSrr)>;
 
 def SKLWriteResGroup69 : SchedWriteRes<[SKLPort0,SKLPort23]> {
   let Latency = 6;
   let NumMicroOps = 2;
   let ResourceCycles = [1,1];
 }
-def: InstRW<[SKLWriteResGroup69], (instrs MMX_PADDSBirm,
-                                          MMX_PADDSWirm,
-                                          MMX_PADDUSBirm,
-                                          MMX_PADDUSWirm,
-                                          MMX_PAVGBirm,
-                                          MMX_PAVGWirm,
-                                          MMX_PCMPEQBirm,
-                                          MMX_PCMPEQDirm,
-                                          MMX_PCMPEQWirm,
-                                          MMX_PCMPGTBirm,
-                                          MMX_PCMPGTDirm,
-                                          MMX_PCMPGTWirm,
-                                          MMX_PMAXSWirm,
-                                          MMX_PMAXUBirm,
-                                          MMX_PMINSWirm,
-                                          MMX_PMINUBirm,
-                                          MMX_PSUBSBirm,
-                                          MMX_PSUBSWirm,
-                                          MMX_PSUBUSBirm,
-                                          MMX_PSUBUSWirm)>;
+def: InstRW<[SKLWriteResGroup69], (instrs MMX_PADDSBrm,
+                                          MMX_PADDSWrm,
+                                          MMX_PADDUSBrm,
+                                          MMX_PADDUSWrm,
+                                          MMX_PAVGBrm,
+                                          MMX_PAVGWrm,
+                                          MMX_PCMPEQBrm,
+                                          MMX_PCMPEQDrm,
+                                          MMX_PCMPEQWrm,
+                                          MMX_PCMPGTBrm,
+                                          MMX_PCMPGTDrm,
+                                          MMX_PCMPGTWrm,
+                                          MMX_PMAXSWrm,
+                                          MMX_PMAXUBrm,
+                                          MMX_PMINSWrm,
+                                          MMX_PMINUBrm,
+                                          MMX_PSUBSBrm,
+                                          MMX_PSUBSWrm,
+                                          MMX_PSUBUSBrm,
+                                          MMX_PSUBUSWrm)>;
 
 def SKLWriteResGroup70 : SchedWriteRes<[SKLPort0,SKLPort01]> {
   let Latency = 6;
@@ -1144,9 +1144,9 @@ def SKLWriteResGroup92 : SchedWriteRes<[SKLPort5,SKLPort23]> {
   let NumMicroOps = 3;
   let ResourceCycles = [2,1];
 }
-def: InstRW<[SKLWriteResGroup92], (instrs MMX_PACKSSDWirm,
-                                          MMX_PACKSSWBirm,
-                                          MMX_PACKUSWBirm)>;
+def: InstRW<[SKLWriteResGroup92], (instrs MMX_PACKSSDWrm,
+                                          MMX_PACKSSWBrm,
+                                          MMX_PACKUSWBrm)>;
 
 def SKLWriteResGroup94 : SchedWriteRes<[SKLPort23,SKLPort0156]> {
   let Latency = 7;
@@ -1283,7 +1283,7 @@ def SKLWriteResGroup120 : SchedWriteRes<[SKLPort0,SKLPort23]> {
   let NumMicroOps = 2;
   let ResourceCycles = [1,1];
 }
-def: InstRW<[SKLWriteResGroup120], (instrs MMX_CVTPI2PSirm)>;
+def: InstRW<[SKLWriteResGroup120], (instrs MMX_CVTPI2PSrm)>;
 
 def SKLWriteResGroup121 : SchedWriteRes<[SKLPort5,SKLPort23]> {
   let Latency = 9;
@@ -1302,7 +1302,7 @@ def SKLWriteResGroup123 : SchedWriteRes<[SKLPort23,SKLPort01]> {
   let NumMicroOps = 2;
   let ResourceCycles = [1,1];
 }
-def: InstRW<[SKLWriteResGroup123], (instregex "MMX_CVT(T?)PS2PIirm",
+def: InstRW<[SKLWriteResGroup123], (instregex "MMX_CVT(T?)PS2PIrm",
                                               "(V?)CVTPS2PDrm")>;
 
 def SKLWriteResGroup128 : SchedWriteRes<[SKLPort5,SKLPort01,SKLPort23]> {
@@ -1345,7 +1345,7 @@ def SKLWriteResGroup138 : SchedWriteRes<[SKLPort0,SKLPort5,SKLPort23]> {
   let NumMicroOps = 3;
   let ResourceCycles = [1,1,1];
 }
-def: InstRW<[SKLWriteResGroup138], (instrs MMX_CVTPI2PDirm)>;
+def: InstRW<[SKLWriteResGroup138], (instrs MMX_CVTPI2PDrm)>;
 
 def SKLWriteResGroup139 : SchedWriteRes<[SKLPort5,SKLPort23,SKLPort01]> {
   let Latency = 10;
@@ -1425,8 +1425,8 @@ def SKLWriteResGroup152 : SchedWriteRes<[SKLPort5,SKLPort23,SKLPort01]> {
 def: InstRW<[SKLWriteResGroup152], (instrs CVTPD2PSrm,
                                            CVTPD2DQrm,
                                            CVTTPD2DQrm,
-                                           MMX_CVTPD2PIirm,
-                                           MMX_CVTTPD2PIirm)>;
+                                           MMX_CVTPD2PIrm,
+                                           MMX_CVTTPD2PIrm)>;
 
 def SKLWriteResGroup154 : SchedWriteRes<[SKLPort1,SKLPort06,SKLPort0156]> {
   let Latency = 11;
diff --git a/llvm/lib/Target/X86/X86SchedSkylakeServer.td b/llvm/lib/Target/X86/X86SchedSkylakeServer.td
index 1d8417aef41e..74f9da158353 100644
--- a/llvm/lib/Target/X86/X86SchedSkylakeServer.td
+++ b/llvm/lib/Target/X86/X86SchedSkylakeServer.td
@@ -634,15 +634,15 @@ def: InstRW<[SKXWriteResGroup1], (instregex "KAND(B|D|Q|W)rr",
                                             "KXOR(B|D|Q|W)rr",
                                             "KSET0(B|D|Q|W)", // Same as KXOR
                                             "KSET1(B|D|Q|W)", // Same as KXNOR
-                                            "MMX_PADDS(B|W)irr",
-                                            "MMX_PADDUS(B|W)irr",
-                                            "MMX_PAVG(B|W)irr",
-                                            "MMX_PCMPEQ(B|D|W)irr",
-                                            "MMX_PCMPGT(B|D|W)irr",
-                                            "MMX_P(MAX|MIN)SWirr",
-                                            "MMX_P(MAX|MIN)UBirr",
-                                            "MMX_PSUBS(B|W)irr",
-                                            "MMX_PSUBUS(B|W)irr",
+                                            "MMX_PADDS(B|W)rr",
+                                            "MMX_PADDUS(B|W)rr",
+                                            "MMX_PAVG(B|W)rr",
+                                            "MMX_PCMPEQ(B|D|W)rr",
+                                            "MMX_PCMPGT(B|D|W)rr",
+                                            "MMX_P(MAX|MIN)SWrr",
+                                            "MMX_P(MAX|MIN)UBrr",
+                                            "MMX_PSUBS(B|W)rr",
+                                            "MMX_PSUBUS(B|W)rr",
                                             "VPMOVB2M(Z|Z128|Z256)rr",
                                             "VPMOVD2M(Z|Z128|Z256)rr",
                                             "VPMOVQ2M(Z|Z128|Z256)rr",
@@ -884,9 +884,9 @@ def SKXWriteResGroup41 : SchedWriteRes<[SKXPort5,SKXPort0156]> {
   let NumMicroOps = 3;
   let ResourceCycles = [2,1];
 }
-def: InstRW<[SKXWriteResGroup41], (instrs MMX_PACKSSDWirr,
-                                          MMX_PACKSSWBirr,
-                                          MMX_PACKUSWBirr)>;
+def: InstRW<[SKXWriteResGroup41], (instrs MMX_PACKSSDWrr,
+                                          MMX_PACKSSWBrr,
+                                          MMX_PACKUSWBrr)>;
 
 def SKXWriteResGroup42 : SchedWriteRes<[SKXPort6,SKXPort0156]> {
   let Latency = 3;
@@ -1047,8 +1047,8 @@ def SKXWriteResGroup61 : SchedWriteRes<[SKXPort5,SKXPort015]> {
   let NumMicroOps = 2;
   let ResourceCycles = [1,1];
 }
-def: InstRW<[SKXWriteResGroup61], (instregex "MMX_CVT(T?)PD2PIirr",
-                                             "MMX_CVT(T?)PS2PIirr",
+def: InstRW<[SKXWriteResGroup61], (instregex "MMX_CVT(T?)PD2PIrr",
+                                             "MMX_CVT(T?)PS2PIrr",
                                              "VCVTDQ2PDZ128rr",
                                              "VCVTPD2DQZ128rr",
                                              "(V?)CVT(T?)PD2DQrr",
@@ -1154,7 +1154,7 @@ def SKXWriteResGroup72 : SchedWriteRes<[SKXPort5]> {
   let NumMicroOps = 2;
   let ResourceCycles = [2];
 }
-def: InstRW<[SKXWriteResGroup72], (instrs MMX_CVTPI2PSirr)>;
+def: InstRW<[SKXWriteResGroup72], (instrs MMX_CVTPI2PSrr)>;
 def: InstRW<[SKXWriteResGroup72], (instregex "VCOMPRESSPD(Z|Z128|Z256)rr",
                                              "VCOMPRESSPS(Z|Z128|Z256)rr",
                                              "VPCOMPRESSD(Z|Z128|Z256)rr",
@@ -1166,26 +1166,26 @@ def SKXWriteResGroup73 : SchedWriteRes<[SKXPort0,SKXPort23]> {
   let NumMicroOps = 2;
   let ResourceCycles = [1,1];
 }
-def: InstRW<[SKXWriteResGroup73], (instrs MMX_PADDSBirm,
-                                          MMX_PADDSWirm,
-                                          MMX_PADDUSBirm,
-                                          MMX_PADDUSWirm,
-                                          MMX_PAVGBirm,
-                                          MMX_PAVGWirm,
-                                          MMX_PCMPEQBirm,
-                                          MMX_PCMPEQDirm,
-                                          MMX_PCMPEQWirm,
-                                          MMX_PCMPGTBirm,
-                                          MMX_PCMPGTDirm,
-                                          MMX_PCMPGTWirm,
-                                          MMX_PMAXSWirm,
-                                          MMX_PMAXUBirm,
-                                          MMX_PMINSWirm,
-                                          MMX_PMINUBirm,
-                                          MMX_PSUBSBirm,
-                                          MMX_PSUBSWirm,
-                                          MMX_PSUBUSBirm,
-                                          MMX_PSUBUSWirm)>;
+def: InstRW<[SKXWriteResGroup73], (instrs MMX_PADDSBrm,
+                                          MMX_PADDSWrm,
+                                          MMX_PADDUSBrm,
+                                          MMX_PADDUSWrm,
+                                          MMX_PAVGBrm,
+                                          MMX_PAVGWrm,
+                                          MMX_PCMPEQBrm,
+                                          MMX_PCMPEQDrm,
+                                          MMX_PCMPEQWrm,
+                                          MMX_PCMPGTBrm,
+                                          MMX_PCMPGTDrm,
+                                          MMX_PCMPGTWrm,
+                                          MMX_PMAXSWrm,
+                                          MMX_PMAXUBrm,
+                                          MMX_PMINSWrm,
+                                          MMX_PMINUBrm,
+                                          MMX_PSUBSBrm,
+                                          MMX_PSUBSWrm,
+                                          MMX_PSUBUSBrm,
+                                          MMX_PSUBUSWrm)>;
 
 def SKXWriteResGroup76 : SchedWriteRes<[SKXPort6,SKXPort23]> {
   let Latency = 6;
@@ -1383,9 +1383,9 @@ def SKXWriteResGroup96 : SchedWriteRes<[SKXPort5,SKXPort23]> {
   let NumMicroOps = 3;
   let ResourceCycles = [2,1];
 }
-def: InstRW<[SKXWriteResGroup96], (instrs MMX_PACKSSDWirm,
-                                          MMX_PACKSSWBirm,
-                                          MMX_PACKUSWBirm)>;
+def: InstRW<[SKXWriteResGroup96], (instrs MMX_PACKSSDWrm,
+                                          MMX_PACKSSWBrm,
+                                          MMX_PACKUSWBrm)>;
 
 def SKXWriteResGroup97 : SchedWriteRes<[SKXPort5,SKXPort015]> {
   let Latency = 7;
@@ -1675,7 +1675,7 @@ def SKXWriteResGroup135 : SchedWriteRes<[SKXPort0,SKXPort23]> {
   let NumMicroOps = 2;
   let ResourceCycles = [1,1];
 }
-def: InstRW<[SKXWriteResGroup135], (instrs MMX_CVTPI2PSirm)>;
+def: InstRW<[SKXWriteResGroup135], (instrs MMX_CVTPI2PSrm)>;
 
 def SKXWriteResGroup136 : SchedWriteRes<[SKXPort5,SKXPort23]> {
   let Latency = 9;
@@ -1701,19 +1701,7 @@ def: InstRW<[SKXWriteResGroup136], (instregex "VALIGN(D|Q)Z128rm(b?)i",
                                               "VPMAXSQZ128rm(b?)",
                                               "VPMAXUQZ128rm(b?)",
                                               "VPMINSQZ128rm(b?)",
-                                              "VPMINUQZ128rm(b?)",
-                                              "VPMOVSXBDZ128rm(b?)",
-                                              "VPMOVSXBQZ128rm(b?)",
-                                              "VPMOVSXBWZ128rm(b?)",
-                                              "VPMOVSXDQZ128rm(b?)",
-                                              "VPMOVSXWDZ128rm(b?)",
-                                              "VPMOVSXWQZ128rm(b?)",
-                                              "VPMOVZXBDZ128rm(b?)",
-                                              "VPMOVZXBQZ128rm(b?)",
-                                              "VPMOVZXBWZ128rm(b?)",
-                                              "VPMOVZXDQZ128rm(b?)",
-                                              "VPMOVZXWDZ128rm(b?)",
-                                              "VPMOVZXWQZ128rm(b?)")>;
+                                              "VPMINUQZ128rm(b?)")>;
 
 def SKXWriteResGroup136_2 : SchedWriteRes<[SKXPort5,SKXPort23]> {
   let Latency = 10;
@@ -1745,7 +1733,7 @@ def SKXWriteResGroup137 : SchedWriteRes<[SKXPort23,SKXPort015]> {
   let NumMicroOps = 2;
   let ResourceCycles = [1,1];
 }
-def: InstRW<[SKXWriteResGroup137], (instregex "MMX_CVT(T?)PS2PIirm",
+def: InstRW<[SKXWriteResGroup137], (instregex "MMX_CVT(T?)PS2PIrm",
                                               "(V?)CVTPS2PDrm")>;
 
 def SKXWriteResGroup143 : SchedWriteRes<[SKXPort5,SKXPort01,SKXPort23]> {
@@ -1942,8 +1930,8 @@ def SKXWriteResGroup166 : SchedWriteRes<[SKXPort5,SKXPort23,SKXPort015]> {
 def: InstRW<[SKXWriteResGroup166], (instrs CVTPD2PSrm,
                                            CVTPD2DQrm,
                                            CVTTPD2DQrm,
-                                           MMX_CVTPD2PIirm,
-                                           MMX_CVTTPD2PIirm)>;
+                                           MMX_CVTPD2PIrm,
+                                           MMX_CVTTPD2PIrm)>;
 
 def SKXWriteResGroup167 : SchedWriteRes<[SKXPort5,SKXPort23,SKXPort015]> {
   let Latency = 11;
diff --git a/llvm/lib/Target/X86/X86ScheduleAtom.td b/llvm/lib/Target/X86/X86ScheduleAtom.td
index 6fd98280f560..0fedfc01092c 100644
--- a/llvm/lib/Target/X86/X86ScheduleAtom.td
+++ b/llvm/lib/Target/X86/X86ScheduleAtom.td
@@ -320,30 +320,30 @@ defm : X86WriteResPairUnsupported<WriteFVarShuffle256>;
 // Conversions.
 ////////////////////////////////////////////////////////////////////////////////
 
-defm : AtomWriteResPair<WriteCvtSS2I,   [AtomPort0,AtomPort1], [AtomPort0,AtomPort1],  8,  9,  [7,7],  [6,6]>;
-defm : AtomWriteResPair<WriteCvtPS2I,   [AtomPort0,AtomPort1], [AtomPort0,AtomPort1],  6,  7,  [5,5],  [6,6]>;
+defm : AtomWriteResPair<WriteCvtSS2I,   [AtomPort0,AtomPort1], [AtomPort0,AtomPort1],  8,  9,  [8,8],  [9,9], 3, 4>;
+defm : AtomWriteResPair<WriteCvtPS2I,   [AtomPort0,AtomPort1], [AtomPort0,AtomPort1],  6,  7,  [6,6],  [7,7], 3, 4>;
 defm : X86WriteResPairUnsupported<WriteCvtPS2IY>;
 defm : X86WriteResPairUnsupported<WriteCvtPS2IZ>;
-defm : AtomWriteResPair<WriteCvtSD2I,   [AtomPort0,AtomPort1], [AtomPort0,AtomPort1],  8,  9,  [7,7],  [6,6]>;
-defm : AtomWriteResPair<WriteCvtPD2I,   [AtomPort0,AtomPort1], [AtomPort0,AtomPort1],  7,  8,  [6,6],  [7,7]>;
+defm : AtomWriteResPair<WriteCvtSD2I,   [AtomPort0,AtomPort1], [AtomPort0,AtomPort1],  8,  9,  [8,8],[10,10], 3, 4>;
+defm : AtomWriteResPair<WriteCvtPD2I,   [AtomPort0,AtomPort1], [AtomPort0,AtomPort1],  7,  8,  [7,7],  [8,8], 4, 5>;
 defm : X86WriteResPairUnsupported<WriteCvtPD2IY>;
 defm : X86WriteResPairUnsupported<WriteCvtPD2IZ>;
 
-defm : AtomWriteResPair<WriteCvtI2SS,   [AtomPort0,AtomPort1], [AtomPort0,AtomPort1],  6,  7,  [5,5],  [6,6]>;
-defm : AtomWriteResPair<WriteCvtI2PS,   [AtomPort0,AtomPort1], [AtomPort0,AtomPort1],  6,  7,  [5,5],  [6,6]>;
+defm : AtomWriteResPair<WriteCvtI2SS,   [AtomPort0,AtomPort1], [AtomPort0,AtomPort1],  6,  7,  [6,6],  [6,6], 3, 1>;
+defm : AtomWriteResPair<WriteCvtI2PS,   [AtomPort0,AtomPort1], [AtomPort0,AtomPort1],  6,  7,  [6,6],  [7,7], 3, 4>;
 defm : X86WriteResPairUnsupported<WriteCvtI2PSY>;
 defm : X86WriteResPairUnsupported<WriteCvtI2PSZ>;
-defm : AtomWriteResPair<WriteCvtI2SD,   [AtomPort0,AtomPort1], [AtomPort0,AtomPort1],  6,  7,  [5,5],  [6,6]>;
-defm : AtomWriteResPair<WriteCvtI2PD,   [AtomPort0,AtomPort1], [AtomPort0,AtomPort1],  7,  8,  [6,6],  [7,7]>;
+defm : AtomWriteResPair<WriteCvtI2SD,   [AtomPort0,AtomPort1], [AtomPort0,AtomPort1],  6,  7,  [6,6],  [7,7], 3, 3>;
+defm : AtomWriteResPair<WriteCvtI2PD,   [AtomPort0,AtomPort1], [AtomPort0,AtomPort1],  7,  8,  [6,6],  [7,7], 3, 4>;
 defm : X86WriteResPairUnsupported<WriteCvtI2PDY>;
 defm : X86WriteResPairUnsupported<WriteCvtI2PDZ>;
 
-defm : AtomWriteResPair<WriteCvtSS2SD,  [AtomPort0,AtomPort1], [AtomPort0,AtomPort1],  6,  7,  [5,5],  [6,6]>;
-defm : AtomWriteResPair<WriteCvtPS2PD,  [AtomPort0,AtomPort1], [AtomPort0,AtomPort1],  7,  8,  [6,6],  [7,7]>;
+defm : AtomWriteResPair<WriteCvtSS2SD,  [AtomPort0,AtomPort1], [AtomPort0,AtomPort1],  6,  7,  [6,6],  [7,7], 3, 4>;
+defm : AtomWriteResPair<WriteCvtPS2PD,  [AtomPort0,AtomPort1], [AtomPort0,AtomPort1],  7,  8,  [6,6],  [7,7], 4, 5>;
 defm : X86WriteResPairUnsupported<WriteCvtPS2PDY>;
 defm : X86WriteResPairUnsupported<WriteCvtPS2PDZ>;
-defm : AtomWriteResPair<WriteCvtSD2SS,  [AtomPort0,AtomPort1], [AtomPort0,AtomPort1],  6,  7,  [5,5],  [6,6]>;
-defm : AtomWriteResPair<WriteCvtPD2PS,  [AtomPort0,AtomPort1], [AtomPort0,AtomPort1],  7,  8,  [6,6],  [7,7]>;
+defm : AtomWriteResPair<WriteCvtSD2SS,  [AtomPort0,AtomPort1], [AtomPort0,AtomPort1], 10, 11,[10,10],[12,12], 3, 4>;
+defm : AtomWriteResPair<WriteCvtPD2PS,  [AtomPort0,AtomPort1], [AtomPort0,AtomPort1], 11, 12,[11,11],[12,12], 4, 5>;
 defm : X86WriteResPairUnsupported<WriteCvtPD2PSY>;
 defm : X86WriteResPairUnsupported<WriteCvtPD2PSZ>;
 
@@ -525,8 +525,8 @@ def AtomWrite1_5 : SchedWriteRes<[AtomPort1]> {
   let Latency = 5;
   let ResourceCycles = [5];
 }
-def : InstRW<[AtomWrite1_5], (instrs MMX_CVTPI2PSirr, MMX_CVTPI2PSirm,
-                                     MMX_CVTPS2PIirr, MMX_CVTTPS2PIirr)>;
+def : InstRW<[AtomWrite1_5], (instrs MMX_CVTPI2PSrr, MMX_CVTPI2PSrm,
+                                     MMX_CVTPS2PIrr, MMX_CVTTPS2PIrr)>;
 
 // Port0 and Port1
 def AtomWrite0_1_1 : SchedWriteRes<[AtomPort0, AtomPort1]> {
@@ -547,9 +547,43 @@ def AtomWrite0_1_5 : SchedWriteRes<[AtomPort0, AtomPort1]> {
   let Latency = 5;
   let ResourceCycles = [5, 5];
 }
-def : InstRW<[AtomWrite0_1_5], (instrs MMX_CVTPS2PIirm, MMX_CVTTPS2PIirm)>;
+def : InstRW<[AtomWrite0_1_5], (instrs MMX_CVTPS2PIrm, MMX_CVTTPS2PIrm)>;
 def : InstRW<[AtomWrite0_1_5], (instregex "ILD_F(16|32|64)")>;
 
+def AtomWrite0_1_7 : SchedWriteRes<[AtomPort0,AtomPort1]> {
+  let Latency = 7;
+  let ResourceCycles = [6,6];
+}
+def : InstRW<[AtomWrite0_1_7], (instregex "CVTSI642SDrm(_Int)?")>;
+
+def AtomWrite0_1_7_4 : SchedWriteRes<[AtomPort0,AtomPort1]> {
+  let Latency = 7;
+  let ResourceCycles = [8,8];
+  let NumMicroOps = 4;
+}
+def : InstRW<[AtomWrite0_1_7_4], (instregex "CVTSI642SSrr(_Int)?")>;
+
+def AtomWrite0_1_8_4 : SchedWriteRes<[AtomPort0,AtomPort1]> {
+  let Latency = 8;
+  let ResourceCycles = [8,8];
+  let NumMicroOps = 4;
+}
+def : InstRW<[AtomWrite0_1_7_4], (instregex "CVTSI642SSrm(_Int)?")>;
+
+def AtomWrite0_1_9 : SchedWriteRes<[AtomPort0,AtomPort1]> {
+  let Latency = 9;
+  let ResourceCycles = [9,9];
+  let NumMicroOps = 4;
+}
+def : InstRW<[AtomWrite0_1_9], (instregex "CVT(T)?SS2SI64rr(_Int)?")>;
+
+def AtomWrite0_1_10 : SchedWriteRes<[AtomPort0,AtomPort1]> {
+  let Latency = 10;
+  let ResourceCycles = [11,11];
+  let NumMicroOps = 5;
+}
+def : InstRW<[AtomWrite0_1_10], (instregex "CVT(T)?SS2SI64rm(_Int)?")>;
+
 // Port0 or Port1
 def AtomWrite01_1 : SchedWriteRes<[AtomPort01]> {
   let Latency = 1;
@@ -570,7 +604,7 @@ def : InstRW<[AtomWrite01_2], (instrs LEAVE, LEAVE64, POP16r,
                                       SCASB, SCASL, SCASQ, SCASW)>;
 def : InstRW<[AtomWrite01_2], (instregex "PUSH(CS|DS|ES|FS|GS|SS)(16|32|64)",
                                          "(ST|ISTT)_F(P)?(16|32|64)?(m|rr)",
-                                         "MMX_P(ADD|SUB)Qirr",
+                                         "MMX_P(ADD|SUB)Qrr",
                                          "MOV(S|Z)X16rr8",
                                          "MOV(UPS|UPD|DQU)mr",
                                          "MASKMOVDQU(64)?",
@@ -589,7 +623,7 @@ def : InstRW<[AtomWrite01_3], (instregex "XADD(8|16|32|64)rm",
                                          "XCHG(8|16|32|64)rm",
                                          "PH(ADD|SUB)Drr",
                                          "MOV(S|Z)X16rm8",
-                                         "MMX_P(ADD|SUB)Qirm",
+                                         "MMX_P(ADD|SUB)Qrm",
                                          "MOV(UPS|UPD|DQU)rm",
                                          "P(ADD|SUB)Qrm")>;
 
@@ -647,15 +681,13 @@ def : InstRW<[AtomWrite01_9], (instrs POPA16, POPA32,
                                       SHLD64mri8, SHRD64mri8,
                                       SHLD64rri8, SHRD64rri8,
                                       CMPXCHG8rr)>;
-def : InstRW<[AtomWrite01_9], (instregex "(U)?COM_FI", "TST_F",
-                                         "CVT(T)?SS2SI64rr(_Int)?")>;
+def : InstRW<[AtomWrite01_9], (instregex "(U)?COM_FI", "TST_F")>;
 
 def AtomWrite01_10 : SchedWriteRes<[AtomPort01]> {
   let Latency = 10;
   let ResourceCycles = [10];
 }
 def : SchedAlias<WriteFLDC, AtomWrite01_10>;
-def : InstRW<[AtomWrite01_10], (instregex "CVT(T)?SS2SI64rm(_Int)?")>;
 
 def AtomWrite01_11 : SchedWriteRes<[AtomPort01]> {
   let Latency = 11;
diff --git a/llvm/lib/Target/X86/X86ScheduleBdVer2.td b/llvm/lib/Target/X86/X86ScheduleBdVer2.td
index 4c16b5b52b1d..0f6f24f9f1fe 100644
--- a/llvm/lib/Target/X86/X86ScheduleBdVer2.td
+++ b/llvm/lib/Target/X86/X86ScheduleBdVer2.td
@@ -1008,11 +1008,11 @@ defm : PdWriteResXMMPair<WriteCvtPD2I,   [PdFPU0, PdFPCVT, PdFPSTO],          8,
 defm : PdWriteResYMMPair<WriteCvtPD2IY,  [PdFPU0, PdFPCVT, PdFPSTO, PdFPFMA], 8, [1, 2, 1, 1], 4>;
 defm : X86WriteResPairUnsupported<WriteCvtPD2IZ>;
 
-def PdWriteMMX_CVTTPD2PIirr : SchedWriteRes<[PdFPU0, PdFPCVT, PdFPSTO]> {
+def PdWriteMMX_CVTTPD2PIrr : SchedWriteRes<[PdFPU0, PdFPCVT, PdFPSTO]> {
   let Latency = 6;
   let NumMicroOps = 2;
 }
-def : InstRW<[PdWriteMMX_CVTTPD2PIirr], (instrs MMX_CVTTPD2PIirr)>;
+def : InstRW<[PdWriteMMX_CVTTPD2PIrr], (instrs MMX_CVTTPD2PIrr)>;
 
 // FIXME: f+3 ST, LD+STC latency
 defm : PdWriteResXMMPair<WriteCvtI2SS,   [PdFPU0, PdFPCVT, PdFPSTO], 4, [], 2>;
@@ -1048,18 +1048,18 @@ defm : PdWriteResXMMPair<WriteCvtPD2PS,  [PdFPU0, PdFPCVT, PdFPSTO],          8,
 defm : PdWriteResYMMPair<WriteCvtPD2PSY, [PdFPU0, PdFPCVT, PdFPSTO, PdFPFMA], 8, [1, 2, 1, 1], 4>;
 defm : X86WriteResPairUnsupported<WriteCvtPD2PSZ>;
 
-def PdWriteMMX_CVTPD2PIirrMMX_CVTPI2PDirr : SchedWriteRes<[PdFPU0, PdFPCVT, PdFPSTO]> {
+def PdWriteMMX_CVTPD2PIrrMMX_CVTPI2PDrr : SchedWriteRes<[PdFPU0, PdFPCVT, PdFPSTO]> {
   let Latency = 6;
   let NumMicroOps = 2;
 }
-def : InstRW<[PdWriteMMX_CVTPD2PIirrMMX_CVTPI2PDirr], (instrs MMX_CVTPD2PIirr,
-                                                            MMX_CVTPI2PDirr)>;
+def : InstRW<[PdWriteMMX_CVTPD2PIrrMMX_CVTPI2PDrr], (instrs MMX_CVTPD2PIrr,
+                                                            MMX_CVTPI2PDrr)>;
 
-def PdWriteMMX_CVTPI2PSirr : SchedWriteRes<[PdFPU0, PdFPCVT, PdFPSTO]> {
+def PdWriteMMX_CVTPI2PSrr : SchedWriteRes<[PdFPU0, PdFPCVT, PdFPSTO]> {
   let Latency = 4;
   let NumMicroOps = 2;
 }
-def : InstRW<[PdWriteMMX_CVTPI2PSirr], (instrs MMX_CVTPI2PSirr)>;
+def : InstRW<[PdWriteMMX_CVTPI2PSrr], (instrs MMX_CVTPI2PSrr)>;
 
 defm : PdWriteResXMMPair<WriteCvtPH2PS,  [PdFPU0, PdFPCVT, PdFPSTO], 8, [1, 2, 1], 2, 1>;
 defm : PdWriteResYMMPair<WriteCvtPH2PSY, [PdFPU0, PdFPCVT, PdFPSTO], 8, [1, 2, 1], 4, 3>;
@@ -1365,7 +1365,7 @@ def PdWriteVZeroIdiomLogic : SchedWriteVariant<[
   SchedVar<MCSchedPredicate<ZeroIdiomPredicate>, [PdWriteZeroLatency]>,
   SchedVar<MCSchedPredicate<TruePred>,           [WriteVecLogic]>
 ]>;
-def : InstRW<[PdWriteVZeroIdiomLogic], (instrs MMX_PXORirr, MMX_PANDNirr)>;
+def : InstRW<[PdWriteVZeroIdiomLogic], (instrs MMX_PXORrr, MMX_PANDNrr)>;
 
 def PdWriteVZeroIdiomLogicX : SchedWriteVariant<[
   SchedVar<MCSchedPredicate<ZeroIdiomPredicate>, [PdWriteZeroLatency]>,
@@ -1378,11 +1378,11 @@ def PdWriteVZeroIdiomALU : SchedWriteVariant<[
   SchedVar<MCSchedPredicate<ZeroIdiomPredicate>, [PdWriteZeroLatency]>,
   SchedVar<MCSchedPredicate<TruePred>,           [WriteVecALU]>
 ]>;
-def : InstRW<[PdWriteVZeroIdiomALU], (instrs MMX_PSUBBirr,   MMX_PSUBDirr,
-                                             MMX_PSUBQirr,   MMX_PSUBWirr,
-                                             MMX_PCMPGTBirr,
-                                             MMX_PCMPGTDirr,
-                                             MMX_PCMPGTWirr)>;
+def : InstRW<[PdWriteVZeroIdiomALU], (instrs MMX_PSUBBrr,   MMX_PSUBDrr,
+                                             MMX_PSUBQrr,   MMX_PSUBWrr,
+                                             MMX_PCMPGTBrr,
+                                             MMX_PCMPGTDrr,
+                                             MMX_PCMPGTWrr)>;
 
 def PdWriteVZeroIdiomALUX : SchedWriteVariant<[
     SchedVar<MCSchedPredicate<ZeroIdiomPredicate>, [PdWriteZeroLatency]>,
@@ -1408,10 +1408,10 @@ def : IsZeroIdiomFunction<[
 
   // MMX Zero-idioms.
   DepBreakingClass<[
-    MMX_PXORirr, MMX_PANDNirr, MMX_PSUBBirr,
-    MMX_PSUBDirr, MMX_PSUBQirr, MMX_PSUBWirr,
-    MMX_PSUBSBirr, MMX_PSUBSWirr, MMX_PSUBUSBirr, MMX_PSUBUSWirr,
-    MMX_PCMPGTBirr, MMX_PCMPGTDirr, MMX_PCMPGTWirr
+    MMX_PXORrr, MMX_PANDNrr, MMX_PSUBBrr,
+    MMX_PSUBDrr, MMX_PSUBQrr, MMX_PSUBWrr,
+    MMX_PSUBSBrr, MMX_PSUBSWrr, MMX_PSUBUSBrr, MMX_PSUBUSWrr,
+    MMX_PCMPGTBrr, MMX_PCMPGTDrr, MMX_PCMPGTWrr
   ], ZeroIdiomPredicate>,
 
   // SSE Zero-idioms.
@@ -1449,7 +1449,7 @@ def : IsDepBreakingFunction<[
 
   // MMX
   DepBreakingClass<[
-    MMX_PCMPEQBirr, MMX_PCMPEQDirr, MMX_PCMPEQWirr
+    MMX_PCMPEQBrr, MMX_PCMPEQDrr, MMX_PCMPEQWrr
   ], ZeroIdiomPredicate>,
 
   // SSE
diff --git a/llvm/lib/Target/X86/X86ScheduleBtVer2.td b/llvm/lib/Target/X86/X86ScheduleBtVer2.td
index 68ebaa244acf..a070da34cab5 100644
--- a/llvm/lib/Target/X86/X86ScheduleBtVer2.td
+++ b/llvm/lib/Target/X86/X86ScheduleBtVer2.td
@@ -888,7 +888,7 @@ def JWriteVZeroIdiomLogic : SchedWriteVariant<[
     SchedVar<MCSchedPredicate<ZeroIdiomPredicate>, [JWriteZeroLatency]>,
     SchedVar<NoSchedPred,                          [WriteVecLogic]>
 ]>;
-def : InstRW<[JWriteVZeroIdiomLogic], (instrs MMX_PXORirr, MMX_PANDNirr)>;
+def : InstRW<[JWriteVZeroIdiomLogic], (instrs MMX_PXORrr, MMX_PANDNrr)>;
 
 def JWriteVZeroIdiomLogicX : SchedWriteVariant<[
     SchedVar<MCSchedPredicate<ZeroIdiomPredicate>, [JWriteZeroLatency]>,
@@ -901,12 +901,12 @@ def JWriteVZeroIdiomALU : SchedWriteVariant<[
     SchedVar<MCSchedPredicate<ZeroIdiomPredicate>, [JWriteZeroLatency]>,
     SchedVar<NoSchedPred,                          [WriteVecALU]>
 ]>;
-def : InstRW<[JWriteVZeroIdiomALU], (instrs MMX_PSUBBirr, MMX_PSUBDirr,
-                                            MMX_PSUBQirr, MMX_PSUBWirr,
-                                            MMX_PSUBSBirr, MMX_PSUBSWirr,
-                                            MMX_PSUBUSBirr, MMX_PSUBUSWirr,
-                                            MMX_PCMPGTBirr, MMX_PCMPGTDirr,
-                                            MMX_PCMPGTWirr)>;
+def : InstRW<[JWriteVZeroIdiomALU], (instrs MMX_PSUBBrr, MMX_PSUBDrr,
+                                            MMX_PSUBQrr, MMX_PSUBWrr,
+                                            MMX_PSUBSBrr, MMX_PSUBSWrr,
+                                            MMX_PSUBUSBrr, MMX_PSUBUSWrr,
+                                            MMX_PCMPGTBrr, MMX_PCMPGTDrr,
+                                            MMX_PCMPGTWrr)>;
 
 def JWriteVZeroIdiomALUX : SchedWriteVariant<[
     SchedVar<MCSchedPredicate<ZeroIdiomPredicate>, [JWriteZeroLatency]>,
@@ -974,10 +974,10 @@ def : IsZeroIdiomFunction<[
 
   // MMX Zero-idioms.
   DepBreakingClass<[
-    MMX_PXORirr, MMX_PANDNirr, MMX_PSUBBirr,
-    MMX_PSUBDirr, MMX_PSUBQirr, MMX_PSUBWirr,
-    MMX_PSUBSBirr, MMX_PSUBSWirr, MMX_PSUBUSBirr, MMX_PSUBUSWirr,
-    MMX_PCMPGTBirr, MMX_PCMPGTDirr, MMX_PCMPGTWirr
+    MMX_PXORrr, MMX_PANDNrr, MMX_PSUBBrr,
+    MMX_PSUBDrr, MMX_PSUBQrr, MMX_PSUBWrr,
+    MMX_PSUBSBrr, MMX_PSUBSWrr, MMX_PSUBUSBrr, MMX_PSUBUSWrr,
+    MMX_PCMPGTBrr, MMX_PCMPGTDrr, MMX_PCMPGTWrr
   ], ZeroIdiomPredicate>,
 
   // SSE Zero-idioms.
@@ -1017,7 +1017,7 @@ def : IsDepBreakingFunction<[
 
   // MMX
   DepBreakingClass<[
-    MMX_PCMPEQBirr, MMX_PCMPEQDirr, MMX_PCMPEQWirr
+    MMX_PCMPEQBrr, MMX_PCMPEQDrr, MMX_PCMPEQWrr
   ], ZeroIdiomPredicate>,
 
   // SSE
diff --git a/llvm/lib/Target/X86/X86ScheduleSLM.td b/llvm/lib/Target/X86/X86ScheduleSLM.td
index 5af9835f75a7..36e5b55a4194 100644
--- a/llvm/lib/Target/X86/X86ScheduleSLM.td
+++ b/llvm/lib/Target/X86/X86ScheduleSLM.td
@@ -467,8 +467,8 @@ def SLMWriteResGroup1rr : SchedWriteRes<[SLM_FPC_RSV01]> {
   let NumMicroOps = 2;
   let ResourceCycles = [8];
 }
-def: InstRW<[SLMWriteResGroup1rr], (instrs MMX_PADDQirr, PADDQrr,
-                                           MMX_PSUBQirr, PSUBQrr,
+def: InstRW<[SLMWriteResGroup1rr], (instrs MMX_PADDQrr, PADDQrr,
+                                           MMX_PSUBQrr, PSUBQrr,
                                            PCMPEQQrr)>;
 
 def SLMWriteResGroup1rm : SchedWriteRes<[SLM_MEC_RSV,SLM_FPC_RSV01]> {
@@ -476,8 +476,8 @@ def SLMWriteResGroup1rm : SchedWriteRes<[SLM_MEC_RSV,SLM_FPC_RSV01]> {
   let NumMicroOps = 3;
   let ResourceCycles = [1,8];
 }
-def: InstRW<[SLMWriteResGroup1rm], (instrs MMX_PADDQirm, PADDQrm,
-                                           MMX_PSUBQirm, PSUBQrm,
+def: InstRW<[SLMWriteResGroup1rm], (instrs MMX_PADDQrm, PADDQrm,
+                                           MMX_PSUBQrm, PSUBQrm,
                                            PCMPEQQrm)>;
 
 } // SchedModel
diff --git a/llvm/lib/Target/X86/X86ScheduleZnver1.td b/llvm/lib/Target/X86/X86ScheduleZnver1.td
index 8e30e5e10ca8..4343e1ed45d1 100644
--- a/llvm/lib/Target/X86/X86ScheduleZnver1.td
+++ b/llvm/lib/Target/X86/X86ScheduleZnver1.td
@@ -1000,12 +1000,12 @@ def ZnWriteFPU12Ym : SchedWriteRes<[ZnAGU, ZnFPU12]> {
   let NumMicroOps = 2;
 }
 
-def : InstRW<[ZnWriteFPU12], (instrs MMX_PACKSSDWirr,
-                                     MMX_PACKSSWBirr,
-                                     MMX_PACKUSWBirr)>;
-def : InstRW<[ZnWriteFPU12m], (instrs MMX_PACKSSDWirm,
-                                      MMX_PACKSSWBirm,
-                                      MMX_PACKUSWBirm)>;
+def : InstRW<[ZnWriteFPU12], (instrs MMX_PACKSSDWrr,
+                                     MMX_PACKSSWBrr,
+                                     MMX_PACKUSWBrr)>;
+def : InstRW<[ZnWriteFPU12m], (instrs MMX_PACKSSDWrm,
+                                      MMX_PACKSSWBrm,
+                                      MMX_PACKUSWBrm)>;
 
 def ZnWriteFPU013 : SchedWriteRes<[ZnFPU013]> ;
 def ZnWriteFPU013Y : SchedWriteRes<[ZnFPU013]> {
@@ -1305,15 +1305,15 @@ def ZnWriteCVTPS2PIr: SchedWriteRes<[ZnFPU3]> {
 }
 // CVT(T)PS2PI.
 // mm,x.
-def : InstRW<[ZnWriteCVTPS2PIr], (instregex "MMX_CVT(T?)PS2PIirr")>;
+def : InstRW<[ZnWriteCVTPS2PIr], (instregex "MMX_CVT(T?)PS2PIrr")>;
 
 // CVTPI2PD.
 // x,mm.
-def : InstRW<[ZnWriteCVTPS2PDr], (instrs MMX_CVTPI2PDirr)>;
+def : InstRW<[ZnWriteCVTPS2PDr], (instrs MMX_CVTPI2PDrr)>;
 
 // CVT(T)PD2PI.
 // mm,x.
-def : InstRW<[ZnWriteCVTPS2PIr], (instregex "MMX_CVT(T?)PD2PIirr")>;
+def : InstRW<[ZnWriteCVTPS2PIr], (instregex "MMX_CVT(T?)PD2PIrr")>;
 
 def ZnWriteCVSTSI2SSr: SchedWriteRes<[ZnFPU3]> {
   let Latency = 5;
diff --git a/llvm/lib/Target/X86/X86ScheduleZnver2.td b/llvm/lib/Target/X86/X86ScheduleZnver2.td
index a83c89e2f28a..96d2837880c7 100644
--- a/llvm/lib/Target/X86/X86ScheduleZnver2.td
+++ b/llvm/lib/Target/X86/X86ScheduleZnver2.td
@@ -1012,12 +1012,12 @@ def Zn2WriteFPU12Ym : SchedWriteRes<[Zn2AGU, Zn2FPU12]> {
   let NumMicroOps = 2;
 }
 
-def : InstRW<[Zn2WriteFPU12], (instrs MMX_PACKSSDWirr,
-                                     MMX_PACKSSWBirr,
-                                     MMX_PACKUSWBirr)>;
-def : InstRW<[Zn2WriteFPU12m], (instrs MMX_PACKSSDWirm,
-                                      MMX_PACKSSWBirm,
-                                      MMX_PACKUSWBirm)>;
+def : InstRW<[Zn2WriteFPU12], (instrs MMX_PACKSSDWrr,
+                                     MMX_PACKSSWBrr,
+                                     MMX_PACKUSWBrr)>;
+def : InstRW<[Zn2WriteFPU12m], (instrs MMX_PACKSSDWrm,
+                                      MMX_PACKSSWBrm,
+                                      MMX_PACKUSWBrm)>;
 
 def Zn2WriteFPU013 : SchedWriteRes<[Zn2FPU013]> ;
 def Zn2WriteFPU013Y : SchedWriteRes<[Zn2FPU013]> ;
@@ -1304,15 +1304,15 @@ def Zn2WriteCVTPS2PIr: SchedWriteRes<[Zn2FPU3]> {
 }
 // CVT(T)PS2PI.
 // mm,x.
-def : InstRW<[Zn2WriteCVTPS2PIr], (instregex "MMX_CVT(T?)PS2PIirr")>;
+def : InstRW<[Zn2WriteCVTPS2PIr], (instregex "MMX_CVT(T?)PS2PIrr")>;
 
 // CVTPI2PD.
 // x,mm.
-def : InstRW<[Zn2WriteCVTPS2PDr], (instrs MMX_CVTPI2PDirr)>;
+def : InstRW<[Zn2WriteCVTPS2PDr], (instrs MMX_CVTPI2PDrr)>;
 
 // CVT(T)PD2PI.
 // mm,x.
-def : InstRW<[Zn2WriteCVTPS2PIr], (instregex "MMX_CVT(T?)PD2PIirr")>;
+def : InstRW<[Zn2WriteCVTPS2PIr], (instregex "MMX_CVT(T?)PD2PIrr")>;
 
 def Zn2WriteCVSTSI2SSr: SchedWriteRes<[Zn2FPU3]> {
   let Latency = 3;
diff --git a/llvm/lib/Target/X86/X86ScheduleZnver3.td b/llvm/lib/Target/X86/X86ScheduleZnver3.td
index be07c069aae1..f4e03ac11f0b 100644
--- a/llvm/lib/Target/X86/X86ScheduleZnver3.td
+++ b/llvm/lib/Target/X86/X86ScheduleZnver3.td
@@ -1075,9 +1075,9 @@ def Zn3WriteVecALUXMMX : SchedWriteRes<[Zn3FPVAdd01]> {
 }
 def : InstRW<[Zn3WriteVecALUXMMX], (instrs MMX_PABSBrr, MMX_PABSDrr, MMX_PABSWrr,
                                            MMX_PSIGNBrr, MMX_PSIGNDrr, MMX_PSIGNWrr,
-                                           MMX_PADDSBirr, MMX_PADDSWirr, MMX_PADDUSBirr, MMX_PADDUSWirr,
-                                           MMX_PAVGBirr, MMX_PAVGWirr,
-                                           MMX_PSUBSBirr, MMX_PSUBSWirr, MMX_PSUBUSBirr, MMX_PSUBUSWirr)>;
+                                           MMX_PADDSBrr, MMX_PADDSWrr, MMX_PADDUSBrr, MMX_PADDUSWrr,
+                                           MMX_PAVGBrr, MMX_PAVGWrr,
+                                           MMX_PSUBSBrr, MMX_PSUBSWrr, MMX_PSUBUSBrr, MMX_PSUBUSWrr)>;
 
 defm : Zn3WriteResYMMPair<WriteVecALUY, [Zn3FPVAdd0123], 1, [1], 1>; // Vector integer ALU op, no logicals (YMM).
 
@@ -1161,7 +1161,7 @@ def Zn3WriteCvtPD2IMMX : SchedWriteRes<[Zn3FPFCvt01]> {
   let ResourceCycles = [2];
   let NumMicroOps = 2;
 }
-def : InstRW<[Zn3WriteCvtPD2IMMX], (instrs MMX_CVTPD2PIirm, MMX_CVTTPD2PIirm, MMX_CVTPD2PIirr, MMX_CVTTPD2PIirr)>;
+def : InstRW<[Zn3WriteCvtPD2IMMX], (instrs MMX_CVTPD2PIrm, MMX_CVTTPD2PIrm, MMX_CVTPD2PIrr, MMX_CVTTPD2PIrr)>;
 
 defm : Zn3WriteResXMMPair<WriteCvtSS2I, [Zn3FPFCvt01], 2, [2], 2>;  // Float -> Integer.
 
@@ -1179,7 +1179,7 @@ def Zn3WriteCvtI2PDMMX : SchedWriteRes<[Zn3FPFCvt01]> {
   let ResourceCycles = [6];
   let NumMicroOps = 2;
 }
-def : InstRW<[Zn3WriteCvtI2PDMMX], (instrs MMX_CVTPI2PDirm, MMX_CVTPI2PDirr)>;
+def : InstRW<[Zn3WriteCvtI2PDMMX], (instrs MMX_CVTPI2PDrm, MMX_CVTPI2PDrr)>;
 
 defm : Zn3WriteResXMMPair<WriteCvtI2SS, [Zn3FPFCvt01], 3, [2], 2, /*LoadUOps=*/-1>;  // Integer -> Float.
 defm : Zn3WriteResXMMPair<WriteCvtI2PS, [Zn3FPFCvt01], 3, [1], 1>; // Integer -> Float (XMM).
@@ -1191,7 +1191,7 @@ def Zn3WriteCvtI2PSMMX : SchedWriteRes<[Zn3FPFCvt01]> {
   let ResourceCycles = [1];
   let NumMicroOps = 2;
 }
-def : InstRW<[Zn3WriteCvtI2PSMMX], (instrs MMX_CVTPI2PSirr)>;
+def : InstRW<[Zn3WriteCvtI2PSMMX], (instrs MMX_CVTPI2PSrr)>;
 
 defm : Zn3WriteResXMMPair<WriteCvtSS2SD, [Zn3FPFCvt01], 3, [1], 1>;  // Float -> Double size conversion.
 defm : Zn3WriteResXMMPair<WriteCvtPS2PD, [Zn3FPFCvt01], 3, [1], 1>; // Float -> Double size conversion (XMM).
@@ -1621,7 +1621,7 @@ def : IsDepBreakingFunction<[
 
   // MMX
   DepBreakingClass<[
-    MMX_PCMPEQBirr, MMX_PCMPEQWirr, MMX_PCMPEQDirr
+    MMX_PCMPEQBrr, MMX_PCMPEQWrr, MMX_PCMPEQDrr
   ], ZeroIdiomPredicate>,
 
   // SSE
diff --git a/llvm/lib/Target/X86/X86Subtarget.h b/llvm/lib/Target/X86/X86Subtarget.h
index 9da54dc2e9b7..5d773f0c57df 100644
--- a/llvm/lib/Target/X86/X86Subtarget.h
+++ b/llvm/lib/Target/X86/X86Subtarget.h
@@ -958,8 +958,7 @@ public:
     // extended frames should be flagged as present.
     const Triple &TT = getTargetTriple();
 
-    unsigned Major, Minor, Micro;
-    TT.getOSVersion(Major, Minor, Micro);
+    unsigned Major = TT.getOSVersion().getMajor();
     switch(TT.getOS()) {
     default:
       return false;
diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp
index 336985f3bf9d..78bc5519c23f 100644
--- a/llvm/lib/Target/X86/X86TargetMachine.cpp
+++ b/llvm/lib/Target/X86/X86TargetMachine.cpp
@@ -588,6 +588,18 @@ void X86PassConfig::addPreEmitPass2() {
 
   // Insert pseudo probe annotation for callsite profiling
   addPass(createPseudoProbeInserter());
+
+  // On Darwin platforms, BLR_RVMARKER pseudo instructions are lowered to
+  // bundles.
+  if (TT.isOSDarwin())
+    addPass(createUnpackMachineBundles([](const MachineFunction &MF) {
+      // Only run bundle expansion if there are relevant ObjC runtime functions
+      // present in the module.
+      const Function &F = MF.getFunction();
+      const Module *M = F.getParent();
+      return M->getFunction("objc_retainAutoreleasedReturnValue") ||
+             M->getFunction("objc_unsafeClaimAutoreleasedReturnValue");
+    }));
 }
 
 bool X86PassConfig::addPostFastRegAllocRewrite() {
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 869762b35196..d8cd7311a0d5 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -236,47 +236,50 @@ InstructionCost X86TTIImpl::getArithmeticInstrCost(
     }
   }
 
-  if ((ISD == ISD::MUL || ISD == ISD::SDIV || ISD == ISD::SREM ||
-       ISD == ISD::UDIV || ISD == ISD::UREM) &&
+  // Vector multiply by pow2 will be simplified to shifts.
+  if (ISD == ISD::MUL &&
+      (Op2Info == TargetTransformInfo::OK_UniformConstantValue ||
+       Op2Info == TargetTransformInfo::OK_NonUniformConstantValue) &&
+      Opd2PropInfo == TargetTransformInfo::OP_PowerOf2)
+    return getArithmeticInstrCost(Instruction::Shl, Ty, CostKind, Op1Info,
+                                  Op2Info, TargetTransformInfo::OP_None,
+                                  TargetTransformInfo::OP_None);
+
+  // On X86, vector signed division by constants power-of-two are
+  // normally expanded to the sequence SRA + SRL + ADD + SRA.
+  // The OperandValue properties may not be the same as that of the previous
+  // operation; conservatively assume OP_None.
+  if ((ISD == ISD::SDIV || ISD == ISD::SREM) &&
       (Op2Info == TargetTransformInfo::OK_UniformConstantValue ||
        Op2Info == TargetTransformInfo::OK_NonUniformConstantValue) &&
       Opd2PropInfo == TargetTransformInfo::OP_PowerOf2) {
-    // Vector multiply by pow2 will be simplified to shifts.
-    if (ISD == ISD::MUL) {
-      InstructionCost Cost = getArithmeticInstrCost(
-          Instruction::Shl, Ty, CostKind, Op1Info, Op2Info,
-          TargetTransformInfo::OP_None, TargetTransformInfo::OP_None);
-      return Cost;
+    InstructionCost Cost =
+        2 * getArithmeticInstrCost(Instruction::AShr, Ty, CostKind, Op1Info,
+                                   Op2Info, TargetTransformInfo::OP_None,
+                                   TargetTransformInfo::OP_None);
+    Cost += getArithmeticInstrCost(Instruction::LShr, Ty, CostKind, Op1Info,
+                                   Op2Info, TargetTransformInfo::OP_None,
+                                   TargetTransformInfo::OP_None);
+    Cost += getArithmeticInstrCost(Instruction::Add, Ty, CostKind, Op1Info,
+                                   Op2Info, TargetTransformInfo::OP_None,
+                                   TargetTransformInfo::OP_None);
+
+    if (ISD == ISD::SREM) {
+      // For SREM: (X % C) is the equivalent of (X - (X/C)*C)
+      Cost += getArithmeticInstrCost(Instruction::Mul, Ty, CostKind, Op1Info,
+                                     Op2Info);
+      Cost += getArithmeticInstrCost(Instruction::Sub, Ty, CostKind, Op1Info,
+                                     Op2Info);
     }
 
-    if (ISD == ISD::SDIV || ISD == ISD::SREM) {
-      // On X86, vector signed division by constants power-of-two are
-      // normally expanded to the sequence SRA + SRL + ADD + SRA.
-      // The OperandValue properties may not be the same as that of the previous
-      // operation; conservatively assume OP_None.
-      InstructionCost Cost =
-          2 * getArithmeticInstrCost(Instruction::AShr, Ty, CostKind, Op1Info,
-                                     Op2Info, TargetTransformInfo::OP_None,
-                                     TargetTransformInfo::OP_None);
-      Cost += getArithmeticInstrCost(Instruction::LShr, Ty, CostKind, Op1Info,
-                                     Op2Info, TargetTransformInfo::OP_None,
-                                     TargetTransformInfo::OP_None);
-      Cost += getArithmeticInstrCost(Instruction::Add, Ty, CostKind, Op1Info,
-                                     Op2Info, TargetTransformInfo::OP_None,
-                                     TargetTransformInfo::OP_None);
-
-      if (ISD == ISD::SREM) {
-        // For SREM: (X % C) is the equivalent of (X - (X/C)*C)
-        Cost += getArithmeticInstrCost(Instruction::Mul, Ty, CostKind, Op1Info,
-                                       Op2Info);
-        Cost += getArithmeticInstrCost(Instruction::Sub, Ty, CostKind, Op1Info,
-                                       Op2Info);
-      }
-
-      return Cost;
-    }
+    return Cost;
+  }
 
-    // Vector unsigned division/remainder will be simplified to shifts/masks.
+  // Vector unsigned division/remainder will be simplified to shifts/masks.
+  if ((ISD == ISD::UDIV || ISD == ISD::UREM) &&
+      (Op2Info == TargetTransformInfo::OK_UniformConstantValue ||
+       Op2Info == TargetTransformInfo::OK_NonUniformConstantValue) &&
+      Opd2PropInfo == TargetTransformInfo::OP_PowerOf2) {
     if (ISD == ISD::UDIV)
       return getArithmeticInstrCost(Instruction::LShr, Ty, CostKind, Op1Info,
                                     Op2Info, TargetTransformInfo::OP_None,
@@ -660,6 +663,7 @@ InstructionCost X86TTIImpl::getArithmeticInstrCost(
     { ISD::MUL,     MVT::v8i32,      1 }, // pmulld (Skylake from agner.org)
     { ISD::MUL,     MVT::v4i32,      1 }, // pmulld (Skylake from agner.org)
     { ISD::MUL,     MVT::v8i64,      6 }, // 3*pmuludq/3*shift/2*add
+    { ISD::MUL,     MVT::i64,        1 }, // Skylake from http://www.agner.org/
 
     { ISD::FNEG,    MVT::v8f64,      1 }, // Skylake from http://www.agner.org/
     { ISD::FADD,    MVT::v8f64,      1 }, // Skylake from http://www.agner.org/
@@ -5188,10 +5192,10 @@ bool X86TTIImpl::areInlineCompatible(const Function *Caller,
   return (RealCallerBits & RealCalleeBits) == RealCalleeBits;
 }
 
-bool X86TTIImpl::areFunctionArgsABICompatible(
-    const Function *Caller, const Function *Callee,
-    SmallPtrSetImpl<Argument *> &Args) const {
-  if (!BaseT::areFunctionArgsABICompatible(Caller, Callee, Args))
+bool X86TTIImpl::areTypesABICompatible(const Function *Caller,
+                                       const Function *Callee,
+                                       const ArrayRef<Type *> &Types) const {
+  if (!BaseT::areTypesABICompatible(Caller, Callee, Types))
     return false;
 
   // If we get here, we know the target features match. If one function
@@ -5206,13 +5210,8 @@ bool X86TTIImpl::areFunctionArgsABICompatible(
   // Consider the arguments compatible if they aren't vectors or aggregates.
   // FIXME: Look at the size of vectors.
   // FIXME: Look at the element types of aggregates to see if there are vectors.
-  // FIXME: The API of this function seems intended to allow arguments
-  // to be removed from the set, but the caller doesn't check if the set
-  // becomes empty so that may not work in practice.
-  return llvm::none_of(Args, [](Argument *A) {
-    auto *EltTy = cast<PointerType>(A->getType())->getElementType();
-    return EltTy->isVectorTy() || EltTy->isAggregateType();
-  });
+  return llvm::none_of(Types,
+      [](Type *T) { return T->isVectorTy() || T->isAggregateType(); });
 }
 
 X86TTIImpl::TTI::MemCmpExpansionOptions
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index c53424ec0026..11e9cb09c7d5 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -234,9 +234,8 @@ public:
   bool isFCmpOrdCheaperThanFCmpZero(Type *Ty);
   bool areInlineCompatible(const Function *Caller,
                            const Function *Callee) const;
-  bool areFunctionArgsABICompatible(const Function *Caller,
-                                    const Function *Callee,
-                                    SmallPtrSetImpl<Argument *> &Args) const;
+  bool areTypesABICompatible(const Function *Caller, const Function *Callee,
+                             const ArrayRef<Type *> &Type) const;
   TTI::MemCmpExpansionOptions enableMemCmpExpansion(bool OptSize,
                                                     bool IsZeroCmp) const;
   bool prefersVectorizedAddressing() const;
diff --git a/llvm/lib/Transforms/AggressiveInstCombine/TruncInstCombine.cpp b/llvm/lib/Transforms/AggressiveInstCombine/TruncInstCombine.cpp
index abac3f801a22..4624b735bef8 100644
--- a/llvm/lib/Transforms/AggressiveInstCombine/TruncInstCombine.cpp
+++ b/llvm/lib/Transforms/AggressiveInstCombine/TruncInstCombine.cpp
@@ -475,12 +475,12 @@ void TruncInstCombine::ReduceExpressionDag(Type *SclTy) {
   // any of its operands, this way, when we get to the operand, we already
   // removed the instructions (from the expression dag) that uses it.
   CurrentTruncInst->eraseFromParent();
-  for (auto I = InstInfoMap.rbegin(), E = InstInfoMap.rend(); I != E; ++I) {
+  for (auto &I : llvm::reverse(InstInfoMap)) {
     // We still need to check that the instruction has no users before we erase
     // it, because {SExt, ZExt}Inst Instruction might have other users that was
     // not reduced, in such case, we need to keep that instruction.
-    if (I->first->use_empty())
-      I->first->eraseFromParent();
+    if (I.first->use_empty())
+      I.first->eraseFromParent();
   }
 }
 
diff --git a/llvm/lib/Transforms/CFGuard/CFGuard.cpp b/llvm/lib/Transforms/CFGuard/CFGuard.cpp
index 96c083a144b2..5fc5295969d0 100644
--- a/llvm/lib/Transforms/CFGuard/CFGuard.cpp
+++ b/llvm/lib/Transforms/CFGuard/CFGuard.cpp
@@ -165,6 +165,12 @@ void CFGuard::insertCFGuardCheck(CallBase *CB) {
   IRBuilder<> B(CB);
   Value *CalledOperand = CB->getCalledOperand();
 
+  // If the indirect call is called within catchpad or cleanuppad,
+  // we need to copy "funclet" bundle of the call.
+  SmallVector<llvm::OperandBundleDef, 1> Bundles;
+  if (auto Bundle = CB->getOperandBundle(LLVMContext::OB_funclet))
+    Bundles.push_back(OperandBundleDef(*Bundle));
+
   // Load the global symbol as a pointer to the check function.
   LoadInst *GuardCheckLoad = B.CreateLoad(GuardFnPtrType, GuardFnGlobal);
 
@@ -172,7 +178,7 @@ void CFGuard::insertCFGuardCheck(CallBase *CB) {
   // even if the original CallBase is an Invoke or CallBr instruction.
   CallInst *GuardCheck =
       B.CreateCall(GuardFnType, GuardCheckLoad,
-                   {B.CreateBitCast(CalledOperand, B.getInt8PtrTy())});
+                   {B.CreateBitCast(CalledOperand, B.getInt8PtrTy())}, Bundles);
 
   // Ensure that the first argument is passed in the correct register
   // (e.g. ECX on 32-bit X86 targets).
diff --git a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
index ac3d078714ce..a0d12865bd3a 100644
--- a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
@@ -1237,8 +1237,10 @@ namespace {
 struct AllocaUseVisitor : PtrUseVisitor<AllocaUseVisitor> {
   using Base = PtrUseVisitor<AllocaUseVisitor>;
   AllocaUseVisitor(const DataLayout &DL, const DominatorTree &DT,
-                   const CoroBeginInst &CB, const SuspendCrossingInfo &Checker)
-      : PtrUseVisitor(DL), DT(DT), CoroBegin(CB), Checker(Checker) {}
+                   const CoroBeginInst &CB, const SuspendCrossingInfo &Checker,
+                   bool ShouldUseLifetimeStartInfo)
+      : PtrUseVisitor(DL), DT(DT), CoroBegin(CB), Checker(Checker),
+        ShouldUseLifetimeStartInfo(ShouldUseLifetimeStartInfo) {}
 
   void visit(Instruction &I) {
     Users.insert(&I);
@@ -1390,6 +1392,7 @@ private:
   SmallPtrSet<Instruction *, 4> Users{};
   SmallPtrSet<IntrinsicInst *, 2> LifetimeStarts{};
   bool MayWriteBeforeCoroBegin{false};
+  bool ShouldUseLifetimeStartInfo{true};
 
   mutable llvm::Optional<bool> ShouldLiveOnFrame{};
 
@@ -1398,7 +1401,7 @@ private:
     // more precise. We look at every pair of lifetime.start intrinsic and
     // every basic block that uses the pointer to see if they cross suspension
     // points. The uses cover both direct uses as well as indirect uses.
-    if (!LifetimeStarts.empty()) {
+    if (ShouldUseLifetimeStartInfo && !LifetimeStarts.empty()) {
       for (auto *I : Users)
         for (auto *S : LifetimeStarts)
           if (Checker.isDefinitionAcrossSuspend(*S, I))
@@ -2484,8 +2487,15 @@ static void collectFrameAllocas(Function &F, coro::Shape &Shape,
       continue;
     }
     DominatorTree DT(F);
+    // The code that uses lifetime.start intrinsic does not work for functions
+    // with loops without exit. Disable it on ABIs we know to generate such
+    // code.
+    bool ShouldUseLifetimeStartInfo =
+        (Shape.ABI != coro::ABI::Async && Shape.ABI != coro::ABI::Retcon &&
+         Shape.ABI != coro::ABI::RetconOnce);
     AllocaUseVisitor Visitor{F.getParent()->getDataLayout(), DT,
-                             *Shape.CoroBegin, Checker};
+                             *Shape.CoroBegin, Checker,
+                             ShouldUseLifetimeStartInfo};
     Visitor.visitPtr(*AI);
     if (!Visitor.getShouldLiveOnFrame())
       continue;
@@ -2572,9 +2582,15 @@ void coro::salvageDebugInfo(
   DVI->setExpression(Expr);
   /// It makes no sense to move the dbg.value intrinsic.
   if (!isa<DbgValueInst>(DVI)) {
-    if (auto *InsertPt = dyn_cast<Instruction>(Storage))
+    if (auto *II = dyn_cast<InvokeInst>(Storage))
+      DVI->moveBefore(II->getNormalDest()->getFirstNonPHI());
+    else if (auto *CBI = dyn_cast<CallBrInst>(Storage))
+      DVI->moveBefore(CBI->getDefaultDest()->getFirstNonPHI());
+    else if (auto *InsertPt = dyn_cast<Instruction>(Storage)) {
+      assert(!InsertPt->isTerminator() &&
+             "Unimaged terminator that could return a storage.");
       DVI->moveAfter(InsertPt);
-    else if (isa<Argument>(Storage))
+    } else if (isa<Argument>(Storage))
       DVI->moveAfter(F->getEntryBlock().getFirstNonPHI());
   }
 }
@@ -2664,7 +2680,10 @@ void coro::buildCoroutineFrame(Function &F, Shape &Shape) {
     }
   }
 
-  sinkLifetimeStartMarkers(F, Shape, Checker);
+  if (Shape.ABI != coro::ABI::Async && Shape.ABI != coro::ABI::Retcon &&
+      Shape.ABI != coro::ABI::RetconOnce)
+    sinkLifetimeStartMarkers(F, Shape, Checker);
+
   if (Shape.ABI != coro::ABI::Async || !Shape.CoroSuspends.empty())
     collectFrameAllocas(F, Shape, Checker, FrameData.Allocas);
   LLVM_DEBUG(dumpAllocas(FrameData.Allocas));
diff --git a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
index fa1d92f439b8..12c1829524ef 100644
--- a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
@@ -280,6 +280,27 @@ static void replaceFallthroughCoroEnd(AnyCoroEndInst *End,
   BB->getTerminator()->eraseFromParent();
 }
 
+// Mark a coroutine as done, which implies that the coroutine is finished and
+// never get resumed.
+//
+// In resume-switched ABI, the done state is represented by storing zero in
+// ResumeFnAddr.
+//
+// NOTE: We couldn't omit the argument `FramePtr`. It is necessary because the
+// pointer to the frame in splitted function is not stored in `Shape`.
+static void markCoroutineAsDone(IRBuilder<> &Builder, const coro::Shape &Shape,
+                                Value *FramePtr) {
+  assert(
+      Shape.ABI == coro::ABI::Switch &&
+      "markCoroutineAsDone is only supported for Switch-Resumed ABI for now.");
+  auto *GepIndex = Builder.CreateStructGEP(
+      Shape.FrameTy, FramePtr, coro::Shape::SwitchFieldIndex::Resume,
+      "ResumeFn.addr");
+  auto *NullPtr = ConstantPointerNull::get(cast<PointerType>(
+      Shape.FrameTy->getTypeAtIndex(coro::Shape::SwitchFieldIndex::Resume)));
+  Builder.CreateStore(NullPtr, GepIndex);
+}
+
 /// Replace an unwind call to llvm.coro.end.
 static void replaceUnwindCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape,
                                  Value *FramePtr, bool InResume,
@@ -288,10 +309,18 @@ static void replaceUnwindCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape,
 
   switch (Shape.ABI) {
   // In switch-lowering, this does nothing in the main function.
-  case coro::ABI::Switch:
+  case coro::ABI::Switch: {
+    // In C++'s specification, the coroutine should be marked as done
+    // if promise.unhandled_exception() throws.  The frontend will
+    // call coro.end(true) along this path.
+    //
+    // FIXME: We should refactor this once there is other language
+    // which uses Switch-Resumed style other than C++.
+    markCoroutineAsDone(Builder, Shape, FramePtr);
     if (!InResume)
       return;
     break;
+  }
   // In async lowering this does nothing.
   case coro::ABI::Async:
     break;
@@ -364,13 +393,9 @@ static void createResumeEntryBlock(Function &F, coro::Shape &Shape) {
     auto *Save = S->getCoroSave();
     Builder.SetInsertPoint(Save);
     if (S->isFinal()) {
-      // Final suspend point is represented by storing zero in ResumeFnAddr.
-      auto *GepIndex = Builder.CreateStructGEP(FrameTy, FramePtr,
-                                 coro::Shape::SwitchFieldIndex::Resume,
-                                  "ResumeFn.addr");
-      auto *NullPtr = ConstantPointerNull::get(cast<PointerType>(
-          FrameTy->getTypeAtIndex(coro::Shape::SwitchFieldIndex::Resume)));
-      Builder.CreateStore(NullPtr, GepIndex);
+      // The coroutine should be marked done if it reaches the final suspend
+      // point.
+      markCoroutineAsDone(Builder, Shape, FramePtr);
     } else {
       auto *GepIndex = Builder.CreateStructGEP(
           FrameTy, FramePtr, Shape.getSwitchIndexField(), "index.addr");
diff --git a/llvm/lib/Transforms/Coroutines/Coroutines.cpp b/llvm/lib/Transforms/Coroutines/Coroutines.cpp
index e4883ef89db7..fba8b03e44ba 100644
--- a/llvm/lib/Transforms/Coroutines/Coroutines.cpp
+++ b/llvm/lib/Transforms/Coroutines/Coroutines.cpp
@@ -141,7 +141,6 @@ static bool isCoroutineIntrinsicName(StringRef Name) {
       "llvm.coro.id.retcon",
       "llvm.coro.id.retcon.once",
       "llvm.coro.noop",
-      "llvm.coro.param",
       "llvm.coro.prepare.async",
       "llvm.coro.prepare.retcon",
       "llvm.coro.promise",
diff --git a/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp b/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp
index 93bb11433775..3a42a2cac928 100644
--- a/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp
+++ b/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp
@@ -835,14 +835,20 @@ bool ArgumentPromotionPass::areFunctionArgsABICompatible(
     const Function &F, const TargetTransformInfo &TTI,
     SmallPtrSetImpl<Argument *> &ArgsToPromote,
     SmallPtrSetImpl<Argument *> &ByValArgsToTransform) {
+  // TODO: Check individual arguments so we can promote a subset?
+  SmallVector<Type *, 32> Types;
+  for (Argument *Arg : ArgsToPromote)
+    Types.push_back(Arg->getType()->getPointerElementType());
+  for (Argument *Arg : ByValArgsToTransform)
+    Types.push_back(Arg->getParamByValType());
+
   for (const Use &U : F.uses()) {
     CallBase *CB = dyn_cast<CallBase>(U.getUser());
     if (!CB)
       return false;
     const Function *Caller = CB->getCaller();
     const Function *Callee = CB->getCalledFunction();
-    if (!TTI.areFunctionArgsABICompatible(Caller, Callee, ArgsToPromote) ||
-        !TTI.areFunctionArgsABICompatible(Caller, Callee, ByValArgsToTransform))
+    if (!TTI.areTypesABICompatible(Caller, Callee, Types))
       return false;
   }
   return true;
diff --git a/llvm/lib/Transforms/IPO/Attributor.cpp b/llvm/lib/Transforms/IPO/Attributor.cpp
index edadc79e3a9f..7e729e57153c 100644
--- a/llvm/lib/Transforms/IPO/Attributor.cpp
+++ b/llvm/lib/Transforms/IPO/Attributor.cpp
@@ -2139,12 +2139,10 @@ bool Attributor::shouldSeedAttribute(AbstractAttribute &AA) {
   bool Result = true;
 #ifndef NDEBUG
   if (SeedAllowList.size() != 0)
-    Result =
-        std::count(SeedAllowList.begin(), SeedAllowList.end(), AA.getName());
+    Result = llvm::is_contained(SeedAllowList, AA.getName());
   Function *Fn = AA.getAnchorScope();
   if (FunctionSeedAllowList.size() != 0 && Fn)
-    Result &= std::count(FunctionSeedAllowList.begin(),
-                         FunctionSeedAllowList.end(), Fn->getName());
+    Result &= llvm::is_contained(FunctionSeedAllowList, Fn->getName());
 #endif
   return Result;
 }
diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
index ec08287393de..b977821bcaa6 100644
--- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
+++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
@@ -417,7 +417,7 @@ const Value *stripAndAccumulateMinimalOffsets(
                                                 AttributorAnalysis);
 }
 
-static const Value *getMinimalBaseOfAccsesPointerOperand(
+static const Value *getMinimalBaseOfAccessPointerOperand(
     Attributor &A, const AbstractAttribute &QueryingAA, const Instruction *I,
     int64_t &BytesOffset, const DataLayout &DL, bool AllowNonInbounds = false) {
   const Value *Ptr = getPointerOperand(I, /* AllowVolatile */ false);
@@ -2129,7 +2129,7 @@ static int64_t getKnownNonNullAndDerefBytesForUse(
 
   int64_t Offset;
   const Value *Base =
-      getMinimalBaseOfAccsesPointerOperand(A, QueryingAA, I, Offset, DL);
+      getMinimalBaseOfAccessPointerOperand(A, QueryingAA, I, Offset, DL);
   if (Base) {
     if (Base == &AssociatedValue &&
         getPointerOperand(I, /* AllowVolatile */ false) == UseV) {
@@ -6414,31 +6414,36 @@ struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl {
       return indicatePessimisticFixpoint();
     }
 
+    // Collect the types that will replace the privatizable type in the function
+    // signature.
+    SmallVector<Type *, 16> ReplacementTypes;
+    identifyReplacementTypes(PrivatizableType.getValue(), ReplacementTypes);
+
     // Verify callee and caller agree on how the promoted argument would be
     // passed.
-    // TODO: The use of the ArgumentPromotion interface here is ugly, we need a
-    // specialized form of TargetTransformInfo::areFunctionArgsABICompatible
-    // which doesn't require the arguments ArgumentPromotion wanted to pass.
     Function &Fn = *getIRPosition().getAnchorScope();
-    SmallPtrSet<Argument *, 1> ArgsToPromote, Dummy;
-    ArgsToPromote.insert(getAssociatedArgument());
     const auto *TTI =
         A.getInfoCache().getAnalysisResultForFunction<TargetIRAnalysis>(Fn);
-    if (!TTI ||
-        !ArgumentPromotionPass::areFunctionArgsABICompatible(
-            Fn, *TTI, ArgsToPromote, Dummy) ||
-        ArgsToPromote.empty()) {
+    if (!TTI) {
+      LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Missing TTI for function "
+                        << Fn.getName() << "\n");
+      return indicatePessimisticFixpoint();
+    }
+
+    auto CallSiteCheck = [&](AbstractCallSite ACS) {
+      CallBase *CB = ACS.getInstruction();
+      return TTI->areTypesABICompatible(
+          CB->getCaller(), CB->getCalledFunction(), ReplacementTypes);
+    };
+    bool AllCallSitesKnown;
+    if (!A.checkForAllCallSites(CallSiteCheck, *this, true,
+                                AllCallSitesKnown)) {
       LLVM_DEBUG(
           dbgs() << "[AAPrivatizablePtr] ABI incompatibility detected for "
                  << Fn.getName() << "\n");
       return indicatePessimisticFixpoint();
     }
 
-    // Collect the types that will replace the privatizable type in the function
-    // signature.
-    SmallVector<Type *, 16> ReplacementTypes;
-    identifyReplacementTypes(PrivatizableType.getValue(), ReplacementTypes);
-
     // Register a rewrite of the argument.
     Argument *Arg = getAssociatedArgument();
     if (!A.isValidFunctionSignatureRewrite(*Arg, ReplacementTypes)) {
@@ -6558,7 +6563,6 @@ struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl {
       return false;
     };
 
-    bool AllCallSitesKnown;
     if (!A.checkForAllCallSites(IsCompatiblePrivArgOfOtherCallSite, *this, true,
                                 AllCallSitesKnown))
       return indicatePessimisticFixpoint();
diff --git a/llvm/lib/Transforms/IPO/FunctionAttrs.cpp b/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
index cde78713b554..321d4a19a585 100644
--- a/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
+++ b/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
@@ -76,6 +76,7 @@ STATISTIC(NumNoCapture, "Number of arguments marked nocapture");
 STATISTIC(NumReturned, "Number of arguments marked returned");
 STATISTIC(NumReadNoneArg, "Number of arguments marked readnone");
 STATISTIC(NumReadOnlyArg, "Number of arguments marked readonly");
+STATISTIC(NumWriteOnlyArg, "Number of arguments marked writeonly");
 STATISTIC(NumNoAlias, "Number of function returns marked noalias");
 STATISTIC(NumNonNullReturn, "Number of function returns marked nonnull");
 STATISTIC(NumNoRecurse, "Number of functions marked as norecurse");
@@ -580,16 +581,8 @@ struct ArgumentUsesTracker : public CaptureTracker {
       return true;
     }
 
-    // Note: the callee and the two successor blocks *follow* the argument
-    // operands.  This means there is no need to adjust UseIndex to account for
-    // these.
-
-    unsigned UseIndex =
-        std::distance(const_cast<const Use *>(CB->arg_begin()), U);
-
-    assert(UseIndex < CB->data_operands_size() &&
-           "Indirect function calls should have been filtered above!");
-
+    assert(!CB->isCallee(U) && "callee operand reported captured?");
+    const unsigned UseIndex = CB->getDataOperandNo(U);
     if (UseIndex >= CB->arg_size()) {
       // Data operand, but not a argument operand -- must be a bundle operand
       assert(CB->hasOperandBundles() && "Must be!");
@@ -649,8 +642,8 @@ struct GraphTraits<ArgumentGraph *> : public GraphTraits<ArgumentGraphNode *> {
 
 /// Returns Attribute::None, Attribute::ReadOnly or Attribute::ReadNone.
 static Attribute::AttrKind
-determinePointerReadAttrs(Argument *A,
-                          const SmallPtrSet<Argument *, 8> &SCCNodes) {
+determinePointerAccessAttrs(Argument *A,
+                            const SmallPtrSet<Argument *, 8> &SCCNodes) {
   SmallVector<Use *, 32> Worklist;
   SmallPtrSet<Use *, 32> Visited;
 
@@ -659,7 +652,7 @@ determinePointerReadAttrs(Argument *A,
     return Attribute::None;
 
   bool IsRead = false;
-  // We don't need to track IsWritten. If A is written to, return immediately.
+  bool IsWrite = false;
 
   for (Use &U : A->uses()) {
     Visited.insert(&U);
@@ -667,6 +660,10 @@ determinePointerReadAttrs(Argument *A,
   }
 
   while (!Worklist.empty()) {
+    if (IsWrite && IsRead)
+      // No point in searching further..
+      return Attribute::None;
+
     Use *U = Worklist.pop_back_val();
     Instruction *I = cast<Instruction>(U->getUser());
 
@@ -684,73 +681,49 @@ determinePointerReadAttrs(Argument *A,
 
     case Instruction::Call:
     case Instruction::Invoke: {
-      bool Captures = true;
+      CallBase &CB = cast<CallBase>(*I);
+      if (CB.isCallee(U)) {
+        IsRead = true;
+        // Note that indirect calls do not capture, see comment in
+        // CaptureTracking for context
+        continue;
+      }
 
-      if (I->getType()->isVoidTy())
-        Captures = false;
+      // Given we've explictily handled the callee operand above, what's left
+      // must be a data operand (e.g. argument or operand bundle)
+      const unsigned UseIndex = CB.getDataOperandNo(U);
 
-      auto AddUsersToWorklistIfCapturing = [&] {
-        if (Captures)
+      if (!CB.doesNotCapture(UseIndex)) {
+        if (!CB.onlyReadsMemory())
+          // If the callee can save a copy into other memory, then simply
+          // scanning uses of the call is insufficient.  We have no way
+          // of tracking copies of the pointer through memory to see
+          // if a reloaded copy is written to, thus we must give up.
+          return Attribute::None;
+        // Push users for processing once we finish this one
+        if (!I->getType()->isVoidTy())
           for (Use &UU : I->uses())
             if (Visited.insert(&UU).second)
               Worklist.push_back(&UU);
-      };
-
-      CallBase &CB = cast<CallBase>(*I);
-      if (CB.doesNotAccessMemory()) {
-        AddUsersToWorklistIfCapturing();
-        continue;
       }
+      
+      if (CB.doesNotAccessMemory())
+        continue;
 
-      Function *F = CB.getCalledFunction();
-      if (!F) {
-        if (CB.onlyReadsMemory()) {
-          IsRead = true;
-          AddUsersToWorklistIfCapturing();
-          continue;
-        }
-        return Attribute::None;
-      }
-
-      // Note: the callee and the two successor blocks *follow* the argument
-      // operands.  This means there is no need to adjust UseIndex to account
-      // for these.
-
-      unsigned UseIndex = std::distance(CB.arg_begin(), U);
-
-      // U cannot be the callee operand use: since we're exploring the
-      // transitive uses of an Argument, having such a use be a callee would
-      // imply the call site is an indirect call or invoke; and we'd take the
-      // early exit above.
-      assert(UseIndex < CB.data_operands_size() &&
-             "Data operand use expected!");
-
-      bool IsOperandBundleUse = UseIndex >= CB.arg_size();
+      if (Function *F = CB.getCalledFunction())
+        if (CB.isArgOperand(U) && UseIndex < F->arg_size() &&
+            SCCNodes.count(F->getArg(UseIndex)))
+          // This is an argument which is part of the speculative SCC.  Note
+          // that only operands corresponding to formal arguments of the callee
+          // can participate in the speculation.
+          break;
 
-      if (UseIndex >= F->arg_size() && !IsOperandBundleUse) {
-        assert(F->isVarArg() && "More params than args in non-varargs call");
+      // The accessors used on call site here do the right thing for calls and
+      // invokes with operand bundles.
+      if (!CB.onlyReadsMemory() && !CB.onlyReadsMemory(UseIndex))
         return Attribute::None;
-      }
-
-      Captures &= !CB.doesNotCapture(UseIndex);
-
-      // Since the optimizer (by design) cannot see the data flow corresponding
-      // to a operand bundle use, these cannot participate in the optimistic SCC
-      // analysis.  Instead, we model the operand bundle uses as arguments in
-      // call to a function external to the SCC.
-      if (IsOperandBundleUse ||
-          !SCCNodes.count(&*std::next(F->arg_begin(), UseIndex))) {
-
-        // The accessors used on call site here do the right thing for calls and
-        // invokes with operand bundles.
-
-        if (!CB.onlyReadsMemory() && !CB.onlyReadsMemory(UseIndex))
-          return Attribute::None;
-        if (!CB.doesNotAccessMemory(UseIndex))
-          IsRead = true;
-      }
-
-      AddUsersToWorklistIfCapturing();
+      if (!CB.doesNotAccessMemory(UseIndex))
+        IsRead = true;
       break;
     }
 
@@ -763,6 +736,19 @@ determinePointerReadAttrs(Argument *A,
       IsRead = true;
       break;
 
+    case Instruction::Store:
+      if (cast<StoreInst>(I)->getValueOperand() == *U)
+        // untrackable capture
+        return Attribute::None;
+
+      // A volatile store has side effects beyond what writeonly can be relied
+      // upon.
+      if (cast<StoreInst>(I)->isVolatile())
+        return Attribute::None;
+
+      IsWrite = true;
+      break;
+
     case Instruction::ICmp:
     case Instruction::Ret:
       break;
@@ -772,7 +758,14 @@ determinePointerReadAttrs(Argument *A,
     }
   }
 
-  return IsRead ? Attribute::ReadOnly : Attribute::ReadNone;
+  if (IsWrite && IsRead)
+    return Attribute::None;
+  else if (IsRead)
+    return Attribute::ReadOnly;
+  else if (IsWrite)
+    return Attribute::WriteOnly;
+  else
+    return Attribute::ReadNone;
 }
 
 /// Deduce returned attributes for the SCC.
@@ -865,9 +858,10 @@ static bool addArgumentAttrsFromCallsites(Function &F) {
   return Changed;
 }
 
-static bool addReadAttr(Argument *A, Attribute::AttrKind R) {
-  assert((R == Attribute::ReadOnly || R == Attribute::ReadNone)
-         && "Must be a Read attribute.");
+static bool addAccessAttr(Argument *A, Attribute::AttrKind R) {
+  assert((R == Attribute::ReadOnly || R == Attribute::ReadNone ||
+          R == Attribute::WriteOnly)
+         && "Must be an access attribute.");
   assert(A && "Argument must not be null.");
 
   // If the argument already has the attribute, nothing needs to be done.
@@ -880,7 +874,12 @@ static bool addReadAttr(Argument *A, Attribute::AttrKind R) {
   A->removeAttr(Attribute::ReadOnly);
   A->removeAttr(Attribute::ReadNone);
   A->addAttr(R);
-  R == Attribute::ReadOnly ? ++NumReadOnlyArg : ++NumReadNoneArg;
+  if (R == Attribute::ReadOnly)
+    ++NumReadOnlyArg;
+  else if (R == Attribute::WriteOnly)
+    ++NumWriteOnlyArg;
+  else
+    ++NumReadNoneArg;
   return true;
 }
 
@@ -945,15 +944,15 @@ static void addArgumentAttrs(const SCCNodeSet &SCCNodes,
         // Otherwise, it's captured. Don't bother doing SCC analysis on it.
       }
       if (!HasNonLocalUses && !A->onlyReadsMemory()) {
-        // Can we determine that it's readonly/readnone without doing an SCC?
-        // Note that we don't allow any calls at all here, or else our result
-        // will be dependent on the iteration order through the functions in the
-        // SCC.
+        // Can we determine that it's readonly/readnone/writeonly without doing
+        // an SCC? Note that we don't allow any calls at all here, or else our
+        // result will be dependent on the iteration order through the
+        // functions in the SCC.
         SmallPtrSet<Argument *, 8> Self;
         Self.insert(&*A);
-        Attribute::AttrKind R = determinePointerReadAttrs(&*A, Self);
+        Attribute::AttrKind R = determinePointerAccessAttrs(&*A, Self);
         if (R != Attribute::None)
-          if (addReadAttr(A, R))
+          if (addAccessAttr(A, R))
             Changed.insert(F);
       }
     }
@@ -979,6 +978,13 @@ static void addArgumentAttrs(const SCCNodeSet &SCCNodes,
         A->addAttr(Attribute::NoCapture);
         ++NumNoCapture;
         Changed.insert(A->getParent());
+
+        // Infer the access attributes given the new nocapture one
+        SmallPtrSet<Argument *, 8> Self;
+        Self.insert(&*A);
+        Attribute::AttrKind R = determinePointerAccessAttrs(&*A, Self);
+        if (R != Attribute::None)
+          addAccessAttr(A, R);
       }
       continue;
     }
@@ -1023,10 +1029,10 @@ static void addArgumentAttrs(const SCCNodeSet &SCCNodes,
       Changed.insert(A->getParent());
     }
 
-    // We also want to compute readonly/readnone. With a small number of false
-    // negatives, we can assume that any pointer which is captured isn't going
-    // to be provably readonly or readnone, since by definition we can't
-    // analyze all uses of a captured pointer.
+    // We also want to compute readonly/readnone/writeonly. With a small number
+    // of false negatives, we can assume that any pointer which is captured
+    // isn't going to be provably readonly or readnone, since by definition
+    // we can't analyze all uses of a captured pointer.
     //
     // The false negatives happen when the pointer is captured by a function
     // that promises readonly/readnone behaviour on the pointer, then the
@@ -1034,24 +1040,28 @@ static void addArgumentAttrs(const SCCNodeSet &SCCNodes,
     // Also, a readonly/readnone pointer may be returned, but returning a
     // pointer is capturing it.
 
-    Attribute::AttrKind ReadAttr = Attribute::ReadNone;
-    for (unsigned i = 0, e = ArgumentSCC.size(); i != e; ++i) {
+    auto meetAccessAttr = [](Attribute::AttrKind A, Attribute::AttrKind B) {
+      if (A == B)
+        return A;
+      if (A == Attribute::ReadNone)
+        return B;
+      if (B == Attribute::ReadNone)
+        return A;
+      return Attribute::None;
+    };
+
+    Attribute::AttrKind AccessAttr = Attribute::ReadNone;
+    for (unsigned i = 0, e = ArgumentSCC.size();
+         i != e && AccessAttr != Attribute::None; ++i) {
       Argument *A = ArgumentSCC[i]->Definition;
-      Attribute::AttrKind K = determinePointerReadAttrs(A, ArgumentSCCNodes);
-      if (K == Attribute::ReadNone)
-        continue;
-      if (K == Attribute::ReadOnly) {
-        ReadAttr = Attribute::ReadOnly;
-        continue;
-      }
-      ReadAttr = K;
-      break;
+      Attribute::AttrKind K = determinePointerAccessAttrs(A, ArgumentSCCNodes);
+      AccessAttr = meetAccessAttr(AccessAttr, K);
     }
 
-    if (ReadAttr != Attribute::None) {
+    if (AccessAttr != Attribute::None) {
       for (unsigned i = 0, e = ArgumentSCC.size(); i != e; ++i) {
         Argument *A = ArgumentSCC[i]->Definition;
-        if (addReadAttr(A, ReadAttr))
+        if (addAccessAttr(A, AccessAttr))
           Changed.insert(A->getParent());
       }
     }
diff --git a/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp b/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp
index fbd083bb9bbf..2425646455bd 100644
--- a/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp
+++ b/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp
@@ -64,8 +64,8 @@ static cl::opt<unsigned> FuncSpecializationMaxIters(
     cl::desc("The maximum number of iterations function specialization is run"),
     cl::init(1));
 
-static cl::opt<unsigned> MaxConstantsThreshold(
-    "func-specialization-max-constants", cl::Hidden,
+static cl::opt<unsigned> MaxClonesThreshold(
+    "func-specialization-max-clones", cl::Hidden,
     cl::desc("The maximum number of clones allowed for a single function "
              "specialization"),
     cl::init(3));
@@ -92,6 +92,28 @@ static cl::opt<bool> EnableSpecializationForLiteralConstant(
     cl::desc("Enable specialization of functions that take a literal constant "
              "as an argument."));
 
+namespace {
+// Bookkeeping struct to pass data from the analysis and profitability phase
+// to the actual transform helper functions.
+struct ArgInfo {
+  Function *Fn;         // The function to perform specialisation on.
+  Argument *Arg;        // The Formal argument being analysed.
+  Constant *Const;      // A corresponding actual constant argument.
+  InstructionCost Gain; // Profitability: Gain = Bonus - Cost.
+
+  // Flag if this will be a partial specialization, in which case we will need
+  // to keep the original function around in addition to the added
+  // specializations.
+  bool Partial = false;
+
+  ArgInfo(Function *F, Argument *A, Constant *C, InstructionCost G)
+      : Fn(F), Arg(A), Const(C), Gain(G){};
+};
+} // Anonymous namespace
+
+using FuncList = SmallVectorImpl<Function *>;
+using ConstList = SmallVectorImpl<Constant *>;
+
 // Helper to check if \p LV is either a constant or a constant
 // range with a single element. This should cover exactly the same cases as the
 // old ValueLatticeElement::isConstant() and is intended to be used in the
@@ -169,7 +191,7 @@ static Constant *getConstantStackValue(CallInst *Call, Value *Val,
 //       ret void
 //     }
 //
-static void constantArgPropagation(SmallVectorImpl<Function *> &WorkList,
+static void constantArgPropagation(FuncList &WorkList,
                                    Module &M, SCCPSolver &Solver) {
   // Iterate over the argument tracked functions see if there
   // are any new constant values for the call instruction via
@@ -254,40 +276,33 @@ public:
   ///
   /// \returns true if at least one function is specialized.
   bool
-  specializeFunctions(SmallVectorImpl<Function *> &FuncDecls,
-                      SmallVectorImpl<Function *> &CurrentSpecializations) {
-
-    // Attempt to specialize the argument-tracked functions.
+  specializeFunctions(FuncList &FuncDecls,
+                      FuncList &CurrentSpecializations) {
     bool Changed = false;
     for (auto *F : FuncDecls) {
-      if (specializeFunction(F, CurrentSpecializations)) {
-        Changed = true;
-        LLVM_DEBUG(dbgs() << "FnSpecialization: Can specialize this func.\n");
-      } else {
+      if (!isCandidateFunction(F, CurrentSpecializations))
+        continue;
+
+      auto Cost = getSpecializationCost(F);
+      if (!Cost.isValid()) {
         LLVM_DEBUG(
-            dbgs() << "FnSpecialization: Cannot specialize this func.\n");
+            dbgs() << "FnSpecialization: Invalid specialisation cost.\n");
+        continue;
       }
-    }
 
-    for (auto *SpecializedFunc : CurrentSpecializations) {
-      SpecializedFuncs.insert(SpecializedFunc);
-
-      // Initialize the state of the newly created functions, marking them
-      // argument-tracked and executable.
-      if (SpecializedFunc->hasExactDefinition() &&
-          !SpecializedFunc->hasFnAttribute(Attribute::Naked))
-        Solver.addTrackedFunction(SpecializedFunc);
-      Solver.addArgumentTrackedFunction(SpecializedFunc);
-      FuncDecls.push_back(SpecializedFunc);
-      Solver.markBlockExecutable(&SpecializedFunc->front());
+      auto ConstArgs = calculateGains(F, Cost);
+      if (ConstArgs.empty()) {
+        LLVM_DEBUG(dbgs() << "FnSpecialization: no possible constants found\n");
+        continue;
+      }
 
-      // Replace the function arguments for the specialized functions.
-      for (Argument &Arg : SpecializedFunc->args())
-        if (!Arg.use_empty() && tryToReplaceWithConstant(&Arg))
-          LLVM_DEBUG(dbgs() << "FnSpecialization: Replaced constant argument: "
-                            << Arg.getName() << "\n");
+      for (auto &CA : ConstArgs) {
+        specializeFunction(CA, CurrentSpecializations);
+        Changed = true;
+      }
     }
 
+    updateSpecializedFuncs(FuncDecls, CurrentSpecializations);
     NumFuncSpecialized += NbFunctionsSpecialized;
     return Changed;
   }
@@ -333,15 +348,83 @@ private:
     return Clone;
   }
 
-  /// This function decides whether to specialize function \p F based on the
-  /// known constant values its arguments can take on. Specialization is
-  /// performed on the first interesting argument. Specializations based on
-  /// additional arguments will be evaluated on following iterations of the
-  /// main IPSCCP solve loop. \returns true if the function is specialized and
-  /// false otherwise.
-  bool specializeFunction(Function *F,
-                          SmallVectorImpl<Function *> &Specializations) {
+  /// This function decides whether it's worthwhile to specialize function \p F
+  /// based on the known constant values its arguments can take on, i.e. it
+  /// calculates a gain and returns a list of actual arguments that are deemed
+  /// profitable to specialize. Specialization is performed on the first
+  /// interesting argument. Specializations based on additional arguments will
+  /// be evaluated on following iterations of the main IPSCCP solve loop.
+  SmallVector<ArgInfo> calculateGains(Function *F, InstructionCost Cost) {
+    SmallVector<ArgInfo> Worklist;
+    // Determine if we should specialize the function based on the values the
+    // argument can take on. If specialization is not profitable, we continue
+    // on to the next argument.
+    for (Argument &FormalArg : F->args()) {
+      LLVM_DEBUG(dbgs() << "FnSpecialization: Analysing arg: "
+                        << FormalArg.getName() << "\n");
+      // Determine if this argument is interesting. If we know the argument can
+      // take on any constant values, they are collected in Constants. If the
+      // argument can only ever equal a constant value in Constants, the
+      // function will be completely specialized, and the IsPartial flag will
+      // be set to false by isArgumentInteresting (that function only adds
+      // values to the Constants list that are deemed profitable).
+      bool IsPartial = true;
+      SmallVector<Constant *> ActualConstArg;
+      if (!isArgumentInteresting(&FormalArg, ActualConstArg, IsPartial)) {
+        LLVM_DEBUG(dbgs() << "FnSpecialization: Argument is not interesting\n");
+        continue;
+      }
+
+      for (auto *ActualArg : ActualConstArg) {
+        InstructionCost Gain =
+            ForceFunctionSpecialization
+                ? 1
+                : getSpecializationBonus(&FormalArg, ActualArg) - Cost;
 
+        if (Gain <= 0)
+          continue;
+        Worklist.push_back({F, &FormalArg, ActualArg, Gain});
+      }
+
+      if (Worklist.empty())
+        continue;
+
+      // Sort the candidates in descending order.
+      llvm::stable_sort(Worklist, [](const ArgInfo &L, const ArgInfo &R) {
+        return L.Gain > R.Gain;
+      });
+
+      // Truncate the worklist to 'MaxClonesThreshold' candidates if
+      // necessary.
+      if (Worklist.size() > MaxClonesThreshold) {
+        LLVM_DEBUG(dbgs() << "FnSpecialization: number of candidates exceed "
+                    << "the maximum number of clones threshold.\n"
+                    << "Truncating worklist to " << MaxClonesThreshold
+                    << " candidates.\n");
+        Worklist.erase(Worklist.begin() + MaxClonesThreshold,
+                       Worklist.end());
+      }
+
+      if (IsPartial || Worklist.size() < ActualConstArg.size())
+        for (auto &ActualArg : Worklist)
+          ActualArg.Partial = true;
+
+      LLVM_DEBUG(dbgs() << "Sorted list of candidates by gain:\n";
+                 for (auto &C
+                      : Worklist) {
+                   dbgs() << "- Function = " << C.Fn->getName() << ", ";
+                   dbgs() << "FormalArg = " << C.Arg->getName() << ", ";
+                   dbgs() << "ActualArg = " << C.Const->getName() << ", ";
+                   dbgs() << "Gain = " << C.Gain << "\n";
+                 });
+
+      // FIXME: Only one argument per function.
+      break;
+    }
+    return Worklist;
+  }
+
+  bool isCandidateFunction(Function *F, FuncList &Specializations) {
     // Do not specialize the cloned function again.
     if (SpecializedFuncs.contains(F))
       return false;
@@ -362,84 +445,32 @@ private:
 
     LLVM_DEBUG(dbgs() << "FnSpecialization: Try function: " << F->getName()
                       << "\n");
+    return true;
+  }
 
-    // Determine if it would be profitable to create a specialization of the
-    // function where the argument takes on the given constant value. If so,
-    // add the constant to Constants.
-    auto FnSpecCost = getSpecializationCost(F);
-    if (!FnSpecCost.isValid()) {
-      LLVM_DEBUG(dbgs() << "FnSpecialization: Invalid specialisation cost.\n");
-      return false;
-    }
-
-    LLVM_DEBUG(dbgs() << "FnSpecialization: func specialisation cost: ";
-               FnSpecCost.print(dbgs()); dbgs() << "\n");
+  void specializeFunction(ArgInfo &AI, FuncList &Specializations) {
+    Function *Clone = cloneCandidateFunction(AI.Fn);
+    Argument *ClonedArg = Clone->getArg(AI.Arg->getArgNo());
 
-    // Determine if we should specialize the function based on the values the
-    // argument can take on. If specialization is not profitable, we continue
-    // on to the next argument.
-    for (Argument &A : F->args()) {
-      LLVM_DEBUG(dbgs() << "FnSpecialization: Analysing arg: " << A.getName()
-                        << "\n");
-      // True if this will be a partial specialization. We will need to keep
-      // the original function around in addition to the added specializations.
-      bool IsPartial = true;
+    // Rewrite calls to the function so that they call the clone instead.
+    rewriteCallSites(AI.Fn, Clone, *ClonedArg, AI.Const);
 
-      // Determine if this argument is interesting. If we know the argument can
-      // take on any constant values, they are collected in Constants. If the
-      // argument can only ever equal a constant value in Constants, the
-      // function will be completely specialized, and the IsPartial flag will
-      // be set to false by isArgumentInteresting (that function only adds
-      // values to the Constants list that are deemed profitable).
-      SmallVector<Constant *, 4> Constants;
-      if (!isArgumentInteresting(&A, Constants, FnSpecCost, IsPartial)) {
-        LLVM_DEBUG(dbgs() << "FnSpecialization: Argument is not interesting\n");
-        continue;
-      }
-
-      assert(!Constants.empty() && "No constants on which to specialize");
-      LLVM_DEBUG(dbgs() << "FnSpecialization: Argument is interesting!\n"
-                        << "FnSpecialization: Specializing '" << F->getName()
-                        << "' on argument: " << A << "\n"
-                        << "FnSpecialization: Constants are:\n\n";
-                 for (unsigned I = 0; I < Constants.size(); ++I) dbgs()
-                 << *Constants[I] << "\n";
-                 dbgs() << "FnSpecialization: End of constants\n\n");
-
-      // Create a version of the function in which the argument is marked
-      // constant with the given value.
-      for (auto *C : Constants) {
-        // Clone the function. We leave the ValueToValueMap empty to allow
-        // IPSCCP to propagate the constant arguments.
-        Function *Clone = cloneCandidateFunction(F);
-        Argument *ClonedArg = Clone->arg_begin() + A.getArgNo();
-
-        // Rewrite calls to the function so that they call the clone instead.
-        rewriteCallSites(F, Clone, *ClonedArg, C);
-
-        // Initialize the lattice state of the arguments of the function clone,
-        // marking the argument on which we specialized the function constant
-        // with the given value.
-        Solver.markArgInFuncSpecialization(F, ClonedArg, C);
-
-        // Mark all the specialized functions
-        Specializations.push_back(Clone);
-        NbFunctionsSpecialized++;
-      }
+    // Initialize the lattice state of the arguments of the function clone,
+    // marking the argument on which we specialized the function constant
+    // with the given value.
+    Solver.markArgInFuncSpecialization(AI.Fn, ClonedArg, AI.Const);
 
-      // If the function has been completely specialized, the original function
-      // is no longer needed. Mark it unreachable.
-      if (!IsPartial)
-        Solver.markFunctionUnreachable(F);
-
-      // FIXME: Only one argument per function.
-      return true;
-    }
+    // Mark all the specialized functions
+    Specializations.push_back(Clone);
+    NbFunctionsSpecialized++;
 
-    return false;
+    // If the function has been completely specialized, the original function
+    // is no longer needed. Mark it unreachable.
+    if (!AI.Partial)
+      Solver.markFunctionUnreachable(AI.Fn);
   }
 
-  /// Compute the cost of specializing function \p F.
+  /// Compute and return the cost of specializing function \p F.
   InstructionCost getSpecializationCost(Function *F) {
     // Compute the code metrics for the function.
     SmallPtrSet<const Value *, 32> EphValues;
@@ -578,9 +609,7 @@ private:
   ///
   /// \returns true if the function should be specialized on the given
   /// argument.
-  bool isArgumentInteresting(Argument *A,
-                             SmallVectorImpl<Constant *> &Constants,
-                             const InstructionCost &FnSpecCost,
+  bool isArgumentInteresting(Argument *A, ConstList &Constants,
                              bool &IsPartial) {
     // For now, don't attempt to specialize functions based on the values of
     // composite types.
@@ -608,42 +637,8 @@ private:
     //
     // TODO 2: this currently does not support constants, i.e. integer ranges.
     //
-    SmallVector<Constant *, 4> PossibleConstants;
-    bool AllConstant = getPossibleConstants(A, PossibleConstants);
-    if (PossibleConstants.empty()) {
-      LLVM_DEBUG(dbgs() << "FnSpecialization: no possible constants found\n");
-      return false;
-    }
-    if (PossibleConstants.size() > MaxConstantsThreshold) {
-      LLVM_DEBUG(dbgs() << "FnSpecialization: number of constants found exceed "
-                        << "the maximum number of constants threshold.\n");
-      return false;
-    }
-
-    for (auto *C : PossibleConstants) {
-      LLVM_DEBUG(dbgs() << "FnSpecialization: Constant: " << *C << "\n");
-      if (ForceFunctionSpecialization) {
-        LLVM_DEBUG(dbgs() << "FnSpecialization: Forced!\n");
-        Constants.push_back(C);
-        continue;
-      }
-      if (getSpecializationBonus(A, C) > FnSpecCost) {
-        LLVM_DEBUG(dbgs() << "FnSpecialization: profitable!\n");
-        Constants.push_back(C);
-      } else {
-        LLVM_DEBUG(dbgs() << "FnSpecialization: not profitable\n");
-      }
-    }
-
-    // None of the constant values the argument can take on were deemed good
-    // candidates on which to specialize the function.
-    if (Constants.empty())
-      return false;
-
-    // This will be a partial specialization if some of the constants were
-    // rejected due to their profitability.
-    IsPartial = !AllConstant || PossibleConstants.size() != Constants.size();
-
+    IsPartial = !getPossibleConstants(A, Constants);
+    LLVM_DEBUG(dbgs() << "FnSpecialization: interesting arg: " << *A << "\n");
     return true;
   }
 
@@ -653,8 +648,7 @@ private:
   /// \returns true if all of the values the argument can take on are constant
   /// (e.g., the argument's parent function cannot be called with an
   /// overdefined value).
-  bool getPossibleConstants(Argument *A,
-                            SmallVectorImpl<Constant *> &Constants) {
+  bool getPossibleConstants(Argument *A, ConstList &Constants) {
     Function *F = A->getParent();
     bool AllConstant = true;
 
@@ -681,7 +675,7 @@ private:
 
       // For now, constant expressions are fine but only if they are function
       // calls.
-      if (auto *CE =  dyn_cast<ConstantExpr>(V))
+      if (auto *CE = dyn_cast<ConstantExpr>(V))
         if (!isa<Function>(CE->getOperand(0)))
           return false;
 
@@ -737,6 +731,29 @@ private:
       }
     }
   }
+
+  void updateSpecializedFuncs(FuncList &FuncDecls,
+                              FuncList &CurrentSpecializations) {
+    for (auto *SpecializedFunc : CurrentSpecializations) {
+      SpecializedFuncs.insert(SpecializedFunc);
+
+      // Initialize the state of the newly created functions, marking them
+      // argument-tracked and executable.
+      if (SpecializedFunc->hasExactDefinition() &&
+          !SpecializedFunc->hasFnAttribute(Attribute::Naked))
+        Solver.addTrackedFunction(SpecializedFunc);
+
+      Solver.addArgumentTrackedFunction(SpecializedFunc);
+      FuncDecls.push_back(SpecializedFunc);
+      Solver.markBlockExecutable(&SpecializedFunc->front());
+
+      // Replace the function arguments for the specialized functions.
+      for (Argument &Arg : SpecializedFunc->args())
+        if (!Arg.use_empty() && tryToReplaceWithConstant(&Arg))
+          LLVM_DEBUG(dbgs() << "FnSpecialization: Replaced constant argument: "
+                            << Arg.getName() << "\n");
+    }
+  }
 };
 } // namespace
 
diff --git a/llvm/lib/Transforms/IPO/GlobalOpt.cpp b/llvm/lib/Transforms/IPO/GlobalOpt.cpp
index ba7589c2bf60..b1f3ff15c97b 100644
--- a/llvm/lib/Transforms/IPO/GlobalOpt.cpp
+++ b/llvm/lib/Transforms/IPO/GlobalOpt.cpp
@@ -305,8 +305,9 @@ static bool CleanupConstantGlobalUsers(GlobalVariable *GV,
     else if (auto *LI = dyn_cast<LoadInst>(U)) {
       // A load from zeroinitializer is always zeroinitializer, regardless of
       // any applied offset.
-      if (Init->isNullValue()) {
-        LI->replaceAllUsesWith(Constant::getNullValue(LI->getType()));
+      Type *Ty = LI->getType();
+      if (Init->isNullValue() && !Ty->isX86_MMXTy() && !Ty->isX86_AMXTy()) {
+        LI->replaceAllUsesWith(Constant::getNullValue(Ty));
         EraseFromParent(LI);
         continue;
       }
@@ -316,8 +317,7 @@ static bool CleanupConstantGlobalUsers(GlobalVariable *GV,
       PtrOp = PtrOp->stripAndAccumulateConstantOffsets(
           DL, Offset, /* AllowNonInbounds */ true);
       if (PtrOp == GV) {
-        if (auto *Value = ConstantFoldLoadFromConst(Init, LI->getType(),
-                                                    Offset, DL)) {
+        if (auto *Value = ConstantFoldLoadFromConst(Init, Ty, Offset, DL)) {
           LI->replaceAllUsesWith(Value);
           EraseFromParent(LI);
         }
@@ -368,8 +368,7 @@ static bool isSafeSROAGEP(User *U) {
       return false;
   }
 
-  return llvm::all_of(U->users(),
-                      [](User *UU) { return isSafeSROAElementUse(UU); });
+  return llvm::all_of(U->users(), isSafeSROAElementUse);
 }
 
 /// Return true if the specified instruction is a safe user of a derived
diff --git a/llvm/lib/Transforms/IPO/HotColdSplitting.cpp b/llvm/lib/Transforms/IPO/HotColdSplitting.cpp
index 833049d6896f..a964fcde0396 100644
--- a/llvm/lib/Transforms/IPO/HotColdSplitting.cpp
+++ b/llvm/lib/Transforms/IPO/HotColdSplitting.cpp
@@ -294,7 +294,7 @@ static int getOutliningPenalty(ArrayRef<BasicBlock *> Region,
       // Find all incoming values from the outlining region.
       int NumIncomingVals = 0;
       for (unsigned i = 0; i < PN.getNumIncomingValues(); ++i)
-        if (find(Region, PN.getIncomingBlock(i)) != Region.end()) {
+        if (llvm::is_contained(Region, PN.getIncomingBlock(i))) {
           ++NumIncomingVals;
           if (NumIncomingVals > 1) {
             ++NumSplitExitPhis;
diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp
index 992c2b292e1e..4e3689f09536 100644
--- a/llvm/lib/Transforms/IPO/Inliner.cpp
+++ b/llvm/lib/Transforms/IPO/Inliner.cpp
@@ -856,6 +856,8 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
 
       if (InlineHistoryID != -1 &&
           inlineHistoryIncludes(&Callee, InlineHistoryID, InlineHistory)) {
+        LLVM_DEBUG(dbgs() << "Skipping inlining due to history: "
+                          << F.getName() << " -> " << Callee.getName() << "\n");
         setInlineRemark(*CB, "recursive");
         continue;
       }
diff --git a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
index f78971f0e586..c0bb19e184d6 100644
--- a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
+++ b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
@@ -1774,8 +1774,9 @@ void LowerTypeTestsModule::replaceCfiUses(Function *Old, Value *New,
                                           bool IsJumpTableCanonical) {
   SmallSetVector<Constant *, 4> Constants;
   for (Use &U : llvm::make_early_inc_range(Old->uses())) {
-    // Skip block addresses
-    if (isa<BlockAddress>(U.getUser()))
+    // Skip block addresses and no_cfi values, which refer to the function
+    // body instead of the jump table.
+    if (isa<BlockAddress, NoCFIValue>(U.getUser()))
       continue;
 
     // Skip direct calls to externally defined or non-dso_local functions
@@ -1802,7 +1803,7 @@ void LowerTypeTestsModule::replaceCfiUses(Function *Old, Value *New,
 }
 
 void LowerTypeTestsModule::replaceDirectCalls(Value *Old, Value *New) {
-  Old->replaceUsesWithIf(New, [](Use &U) { return isDirectCall(U); });
+  Old->replaceUsesWithIf(New, isDirectCall);
 }
 
 bool LowerTypeTestsModule::lower() {
diff --git a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
index 055ee6b50296..f289e3ecc979 100644
--- a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
+++ b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
@@ -3964,6 +3964,9 @@ struct AAKernelInfoCallSite : AAKernelInfo {
     case OMPRTL___kmpc_master:
     case OMPRTL___kmpc_end_master:
     case OMPRTL___kmpc_barrier:
+    case OMPRTL___kmpc_nvptx_parallel_reduce_nowait_v2:
+    case OMPRTL___kmpc_nvptx_teams_reduce_nowait_v2:
+    case OMPRTL___kmpc_nvptx_end_reduce_nowait:
       break;
     case OMPRTL___kmpc_distribute_static_init_4:
     case OMPRTL___kmpc_distribute_static_init_4u:
@@ -4010,6 +4013,7 @@ struct AAKernelInfoCallSite : AAKernelInfo {
       break;
     case OMPRTL___kmpc_omp_task:
       // We do not look into tasks right now, just give up.
+      SPMDCompatibilityTracker.indicatePessimisticFixpoint();
       SPMDCompatibilityTracker.insert(&CB);
       ReachedUnknownParallelRegions.insert(&CB);
       break;
@@ -4020,6 +4024,7 @@ struct AAKernelInfoCallSite : AAKernelInfo {
     default:
       // Unknown OpenMP runtime calls cannot be executed in SPMD-mode,
       // generally. However, they do not hide parallel regions.
+      SPMDCompatibilityTracker.indicatePessimisticFixpoint();
       SPMDCompatibilityTracker.insert(&CB);
       break;
     }
@@ -4079,6 +4084,7 @@ struct AAKernelInfoCallSite : AAKernelInfo {
         SPMDCompatibilityTracker.insert(&CB);
       break;
     default:
+      SPMDCompatibilityTracker.indicatePessimisticFixpoint();
       SPMDCompatibilityTracker.insert(&CB);
     }
 
diff --git a/llvm/lib/Transforms/IPO/SampleContextTracker.cpp b/llvm/lib/Transforms/IPO/SampleContextTracker.cpp
index bae9a1e27e75..7334bf695b67 100644
--- a/llvm/lib/Transforms/IPO/SampleContextTracker.cpp
+++ b/llvm/lib/Transforms/IPO/SampleContextTracker.cpp
@@ -32,7 +32,7 @@ ContextTrieNode *ContextTrieNode::getChildContext(const LineLocation &CallSite,
   if (CalleeName.empty())
     return getHottestChildContext(CallSite);
 
-  uint64_t Hash = nodeHash(CalleeName, CallSite);
+  uint64_t Hash = FunctionSamples::getCallSiteHash(CalleeName, CallSite);
   auto It = AllChildContext.find(Hash);
   if (It != AllChildContext.end())
     return &It->second;
@@ -65,7 +65,8 @@ ContextTrieNode::getHottestChildContext(const LineLocation &CallSite) {
 ContextTrieNode &ContextTrieNode::moveToChildContext(
     const LineLocation &CallSite, ContextTrieNode &&NodeToMove,
     uint32_t ContextFramesToRemove, bool DeleteNode) {
-  uint64_t Hash = nodeHash(NodeToMove.getFuncName(), CallSite);
+  uint64_t Hash =
+      FunctionSamples::getCallSiteHash(NodeToMove.getFuncName(), CallSite);
   assert(!AllChildContext.count(Hash) && "Node to remove must exist");
   LineLocation OldCallSite = NodeToMove.CallSiteLoc;
   ContextTrieNode &OldParentContext = *NodeToMove.getParentContext();
@@ -108,7 +109,7 @@ ContextTrieNode &ContextTrieNode::moveToChildContext(
 
 void ContextTrieNode::removeChildContext(const LineLocation &CallSite,
                                          StringRef CalleeName) {
-  uint64_t Hash = nodeHash(CalleeName, CallSite);
+  uint64_t Hash = FunctionSamples::getCallSiteHash(CalleeName, CallSite);
   // Note this essentially calls dtor and destroys that child context
   AllChildContext.erase(Hash);
 }
@@ -174,21 +175,9 @@ void ContextTrieNode::dumpTree() {
   }
 }
 
-uint64_t ContextTrieNode::nodeHash(StringRef ChildName,
-                                   const LineLocation &Callsite) {
-  // We still use child's name for child hash, this is
-  // because for children of root node, we don't have
-  // different line/discriminator, and we'll rely on name
-  // to differentiate children.
-  uint64_t NameHash = std::hash<std::string>{}(ChildName.str());
-  uint64_t LocId =
-      (((uint64_t)Callsite.LineOffset) << 32) | Callsite.Discriminator;
-  return NameHash + (LocId << 5) + LocId;
-}
-
 ContextTrieNode *ContextTrieNode::getOrCreateChildContext(
     const LineLocation &CallSite, StringRef CalleeName, bool AllowCreate) {
-  uint64_t Hash = nodeHash(CalleeName, CallSite);
+  uint64_t Hash = FunctionSamples::getCallSiteHash(CalleeName, CallSite);
   auto It = AllChildContext.find(Hash);
   if (It != AllChildContext.end()) {
     assert(It->second.getFuncName() == CalleeName &&
diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp
index b8fac9d47763..bc6051de90c4 100644
--- a/llvm/lib/Transforms/IPO/SampleProfile.cpp
+++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp
@@ -467,6 +467,9 @@ protected:
   void emitOptimizationRemarksForInlineCandidates(
       const SmallVectorImpl<CallBase *> &Candidates, const Function &F,
       bool Hot);
+  void promoteMergeNotInlinedContextSamples(
+      DenseMap<CallBase *, const FunctionSamples *> NonInlinedCallSites,
+      const Function &F);
   std::vector<Function *> buildFunctionOrder(Module &M, CallGraph *CG);
   std::unique_ptr<ProfiledCallGraph> buildProfiledCallGraph(CallGraph &CG);
   void generateMDProfMetadata(Function &F);
@@ -485,7 +488,7 @@ protected:
   std::unique_ptr<SampleContextTracker> ContextTracker;
 
   /// Flag indicating whether input profile is context-sensitive
-  bool ProfileIsCS = false;
+  bool ProfileIsCSFlat = false;
 
   /// Flag indicating which LTO/ThinLTO phase the pass is invoked in.
   ///
@@ -602,7 +605,7 @@ ErrorOr<uint64_t> SampleProfileLoader::getInstWeight(const Instruction &Inst) {
   // call instruction should have 0 count.
   // For CS profile, the callsite count of previously inlined callees is
   // populated with the entry count of the callees.
-  if (!ProfileIsCS)
+  if (!ProfileIsCSFlat)
     if (const auto *CB = dyn_cast<CallBase>(&Inst))
       if (!CB->isIndirectCall() && findCalleeFunctionSamples(*CB))
         return 0;
@@ -641,7 +644,7 @@ ErrorOr<uint64_t> SampleProfileLoader::getProbeWeight(const Instruction &Inst) {
   // call instruction should have 0 count.
   // For CS profile, the callsite count of previously inlined callees is
   // populated with the entry count of the callees.
-  if (!ProfileIsCS)
+  if (!ProfileIsCSFlat)
     if (const auto *CB = dyn_cast<CallBase>(&Inst))
       if (!CB->isIndirectCall() && findCalleeFunctionSamples(*CB))
         return 0;
@@ -695,7 +698,7 @@ SampleProfileLoader::findCalleeFunctionSamples(const CallBase &Inst) const {
   if (Function *Callee = Inst.getCalledFunction())
     CalleeName = Callee->getName();
 
-  if (ProfileIsCS)
+  if (ProfileIsCSFlat)
     return ContextTracker->getCalleeContextSamplesFor(Inst, CalleeName);
 
   const FunctionSamples *FS = findFunctionSamples(Inst);
@@ -727,7 +730,7 @@ SampleProfileLoader::findIndirectCallFunctionSamples(
            FunctionSamples::getGUID(R->getName());
   };
 
-  if (ProfileIsCS) {
+  if (ProfileIsCSFlat) {
     auto CalleeSamples =
         ContextTracker->getIndirectCalleeContextSamplesFor(DIL);
     if (CalleeSamples.empty())
@@ -780,7 +783,7 @@ SampleProfileLoader::findFunctionSamples(const Instruction &Inst) const {
 
   auto it = DILocation2SampleMap.try_emplace(DIL,nullptr);
   if (it.second) {
-    if (ProfileIsCS)
+    if (ProfileIsCSFlat)
       it.first->second = ContextTracker->getContextSamplesFor(DIL);
     else
       it.first->second =
@@ -1039,7 +1042,7 @@ void SampleProfileLoader::findExternalInlineCandidate(
 
   // For AutoFDO profile, retrieve candidate profiles by walking over
   // the nested inlinee profiles.
-  if (!ProfileIsCS) {
+  if (!ProfileIsCSFlat) {
     Samples->findInlinedFunctions(InlinedGUIDs, SymbolMap, Threshold);
     return;
   }
@@ -1134,7 +1137,7 @@ bool SampleProfileLoader::inlineHotFunctions(
               assert((!FunctionSamples::UseMD5 || FS->GUIDToFuncNameMap) &&
                      "GUIDToFuncNameMap has to be populated");
               AllCandidates.push_back(CB);
-              if (FS->getEntrySamples() > 0 || ProfileIsCS)
+              if (FS->getEntrySamples() > 0 || ProfileIsCSFlat)
                 LocalNotInlinedCallSites.try_emplace(CB, FS);
               if (callsiteIsHot(FS, PSI, ProfAccForSymsInList))
                 Hot = true;
@@ -1156,11 +1159,9 @@ bool SampleProfileLoader::inlineHotFunctions(
     }
     for (CallBase *I : CIS) {
       Function *CalledFunction = I->getCalledFunction();
-      InlineCandidate Candidate = {
-          I,
-          LocalNotInlinedCallSites.count(I) ? LocalNotInlinedCallSites[I]
-                                            : nullptr,
-          0 /* dummy count */, 1.0 /* dummy distribution factor */};
+      InlineCandidate Candidate = {I, LocalNotInlinedCallSites.lookup(I),
+                                   0 /* dummy count */,
+                                   1.0 /* dummy distribution factor */};
       // Do not inline recursive calls.
       if (CalledFunction == &F)
         continue;
@@ -1198,53 +1199,9 @@ bool SampleProfileLoader::inlineHotFunctions(
   }
 
   // For CS profile, profile for not inlined context will be merged when
-  // base profile is being trieved
-  if (ProfileIsCS)
-    return Changed;
-
-  // Accumulate not inlined callsite information into notInlinedSamples
-  for (const auto &Pair : LocalNotInlinedCallSites) {
-    CallBase *I = Pair.getFirst();
-    Function *Callee = I->getCalledFunction();
-    if (!Callee || Callee->isDeclaration())
-      continue;
-
-    ORE->emit(OptimizationRemarkAnalysis(CSINLINE_DEBUG, "NotInline",
-                                         I->getDebugLoc(), I->getParent())
-              << "previous inlining not repeated: '"
-              << ore::NV("Callee", Callee) << "' into '"
-              << ore::NV("Caller", &F) << "'");
-
-    ++NumCSNotInlined;
-    const FunctionSamples *FS = Pair.getSecond();
-    if (FS->getTotalSamples() == 0 && FS->getEntrySamples() == 0) {
-      continue;
-    }
-
-    if (ProfileMergeInlinee) {
-      // A function call can be replicated by optimizations like callsite
-      // splitting or jump threading and the replicates end up sharing the
-      // sample nested callee profile instead of slicing the original inlinee's
-      // profile. We want to do merge exactly once by filtering out callee
-      // profiles with a non-zero head sample count.
-      if (FS->getHeadSamples() == 0) {
-        // Use entry samples as head samples during the merge, as inlinees
-        // don't have head samples.
-        const_cast<FunctionSamples *>(FS)->addHeadSamples(
-            FS->getEntrySamples());
-
-        // Note that we have to do the merge right after processing function.
-        // This allows OutlineFS's profile to be used for annotation during
-        // top-down processing of functions' annotation.
-        FunctionSamples *OutlineFS = Reader->getOrCreateSamplesFor(*Callee);
-        OutlineFS->merge(*FS);
-      }
-    } else {
-      auto pair =
-          notInlinedCallInfo.try_emplace(Callee, NotInlinedProfileInfo{0});
-      pair.first->second.entryCount += FS->getEntrySamples();
-    }
-  }
+  // base profile is being retrieved.
+  if (!FunctionSamples::ProfileIsCSFlat)
+    promoteMergeNotInlinedContextSamples(LocalNotInlinedCallSites, F);
   return Changed;
 }
 
@@ -1285,7 +1242,7 @@ bool SampleProfileLoader::tryInlineCandidate(
         InlinedCallSites->push_back(I);
     }
 
-    if (ProfileIsCS)
+    if (ProfileIsCSFlat)
       ContextTracker->markContextSamplesInlined(Candidate.CalleeSamples);
     ++NumCSInlined;
 
@@ -1430,7 +1387,6 @@ SampleProfileLoader::shouldInlineCandidate(InlineCandidate &Candidate) {
 
 bool SampleProfileLoader::inlineHotFunctionsWithPriority(
     Function &F, DenseSet<GlobalValue::GUID> &InlinedGUIDs) {
-  assert(ProfileIsCS && "Prioritiy based inliner only works with CSSPGO now");
 
   // ProfAccForSymsInList is used in callsiteIsHot. The assertion makes sure
   // Profile symbol list is ignored when profile-sample-accurate is on.
@@ -1467,6 +1423,8 @@ bool SampleProfileLoader::inlineHotFunctionsWithPriority(
   if (ExternalInlineAdvisor)
     SizeLimit = std::numeric_limits<unsigned>::max();
 
+  DenseMap<CallBase *, const FunctionSamples *> LocalNotInlinedCallSites;
+
   // Perform iterative BFS call site prioritized inlining
   bool Changed = false;
   while (!CQueue.empty() && F.getInstructionCount() < SizeLimit) {
@@ -1521,6 +1479,8 @@ bool SampleProfileLoader::inlineHotFunctionsWithPriority(
           }
           ICPCount++;
           Changed = true;
+        } else if (!ContextTracker) {
+          LocalNotInlinedCallSites.try_emplace(I, FS);
         }
       }
     } else if (CalledFunction && CalledFunction->getSubprogram() &&
@@ -1532,6 +1492,8 @@ bool SampleProfileLoader::inlineHotFunctionsWithPriority(
             CQueue.emplace(NewCandidate);
         }
         Changed = true;
+      } else if (!ContextTracker) {
+        LocalNotInlinedCallSites.try_emplace(I, Candidate.CalleeSamples);
       }
     } else if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) {
       findExternalInlineCandidate(I, findCalleeFunctionSamples(*I),
@@ -1549,9 +1511,63 @@ bool SampleProfileLoader::inlineHotFunctionsWithPriority(
       ++NumCSInlinedHitGrowthLimit;
   }
 
+  // For CS profile, profile for not inlined context will be merged when
+  // base profile is being retrieved.
+  if (!FunctionSamples::ProfileIsCSFlat)
+    promoteMergeNotInlinedContextSamples(LocalNotInlinedCallSites, F);
   return Changed;
 }
 
+void SampleProfileLoader::promoteMergeNotInlinedContextSamples(
+    DenseMap<CallBase *, const FunctionSamples *> NonInlinedCallSites,
+    const Function &F) {
+  // Accumulate not inlined callsite information into notInlinedSamples
+  for (const auto &Pair : NonInlinedCallSites) {
+    CallBase *I = Pair.getFirst();
+    Function *Callee = I->getCalledFunction();
+    if (!Callee || Callee->isDeclaration())
+      continue;
+
+    ORE->emit(OptimizationRemarkAnalysis(CSINLINE_DEBUG, "NotInline",
+                                         I->getDebugLoc(), I->getParent())
+              << "previous inlining not repeated: '"
+              << ore::NV("Callee", Callee) << "' into '"
+              << ore::NV("Caller", &F) << "'");
+
+    ++NumCSNotInlined;
+    const FunctionSamples *FS = Pair.getSecond();
+    if (FS->getTotalSamples() == 0 && FS->getEntrySamples() == 0) {
+      continue;
+    }
+
+    if (ProfileMergeInlinee) {
+      // A function call can be replicated by optimizations like callsite
+      // splitting or jump threading and the replicates end up sharing the
+      // sample nested callee profile instead of slicing the original
+      // inlinee's profile. We want to do merge exactly once by filtering out
+      // callee profiles with a non-zero head sample count.
+      if (FS->getHeadSamples() == 0) {
+        // Use entry samples as head samples during the merge, as inlinees
+        // don't have head samples.
+        const_cast<FunctionSamples *>(FS)->addHeadSamples(
+            FS->getEntrySamples());
+
+        // Note that we have to do the merge right after processing function.
+        // This allows OutlineFS's profile to be used for annotation during
+        // top-down processing of functions' annotation.
+        FunctionSamples *OutlineFS = Reader->getOrCreateSamplesFor(*Callee);
+        OutlineFS->merge(*FS, 1);
+        // Set outlined profile to be synthetic to not bias the inliner.
+        OutlineFS->SetContextSynthetic();
+      }
+    } else {
+      auto pair =
+          notInlinedCallInfo.try_emplace(Callee, NotInlinedProfileInfo{0});
+      pair.first->second.entryCount += FS->getEntrySamples();
+    }
+  }
+}
+
 /// Returns the sorted CallTargetMap \p M by count in descending order.
 static SmallVector<InstrProfValueData, 2>
 GetSortedValueDataFromCallTargets(const SampleRecord::CallTargetMap &M) {
@@ -1607,7 +1623,7 @@ void SampleProfileLoader::generateMDProfMetadata(Function &F) {
           // With CSSPGO all indirect call targets are counted torwards the
           // original indirect call site in the profile, including both
           // inlined and non-inlined targets.
-          if (!FunctionSamples::ProfileIsCS) {
+          if (!FunctionSamples::ProfileIsCSFlat) {
             if (const FunctionSamplesMap *M =
                     FS->findFunctionSamplesMapAt(CallSite)) {
               for (const auto &NameFS : *M)
@@ -1754,7 +1770,7 @@ bool SampleProfileLoader::emitAnnotations(Function &F) {
   }
 
   DenseSet<GlobalValue::GUID> InlinedGUIDs;
-  if (ProfileIsCS && CallsitePrioritizedInline)
+  if (CallsitePrioritizedInline)
     Changed |= inlineHotFunctionsWithPriority(F, InlinedGUIDs);
   else
     Changed |= inlineHotFunctions(F, InlinedGUIDs);
@@ -1782,7 +1798,7 @@ INITIALIZE_PASS_END(SampleProfileLoaderLegacyPass, "sample-profile",
 std::unique_ptr<ProfiledCallGraph>
 SampleProfileLoader::buildProfiledCallGraph(CallGraph &CG) {
   std::unique_ptr<ProfiledCallGraph> ProfiledCG;
-  if (ProfileIsCS)
+  if (ProfileIsCSFlat)
     ProfiledCG = std::make_unique<ProfiledCallGraph>(*ContextTracker);
   else
     ProfiledCG = std::make_unique<ProfiledCallGraph>(Reader->getProfiles());
@@ -1828,7 +1844,7 @@ SampleProfileLoader::buildFunctionOrder(Module &M, CallGraph *CG) {
   assert(&CG->getModule() == &M);
 
   if (UseProfiledCallGraph ||
-      (ProfileIsCS && !UseProfiledCallGraph.getNumOccurrences())) {
+      (ProfileIsCSFlat && !UseProfiledCallGraph.getNumOccurrences())) {
     // Use profiled call edges to augment the top-down order. There are cases
     // that the top-down order computed based on the static call graph doesn't
     // reflect real execution order. For example
@@ -1961,10 +1977,8 @@ bool SampleProfileLoader::doInitialization(Module &M,
   }
 
   // Apply tweaks if context-sensitive profile is available.
-  if (Reader->profileIsCS()) {
-    ProfileIsCS = true;
-    FunctionSamples::ProfileIsCS = true;
-
+  if (Reader->profileIsCSFlat() || Reader->profileIsCSNested()) {
+    ProfileIsCSFlat = Reader->profileIsCSFlat();
     // Enable priority-base inliner and size inline by default for CSSPGO.
     if (!ProfileSizeInline.getNumOccurrences())
       ProfileSizeInline = true;
@@ -1982,10 +1996,15 @@ bool SampleProfileLoader::doInitialization(Module &M,
     // Enable iterative-BFI by default for CSSPGO.
     if (!UseIterativeBFIInference.getNumOccurrences())
       UseIterativeBFIInference = true;
-
-    // Tracker for profiles under different context
-    ContextTracker = std::make_unique<SampleContextTracker>(
-        Reader->getProfiles(), &GUIDToFuncNameMap);
+    // Enable Profi by default for CSSPGO.
+    if (!SampleProfileUseProfi.getNumOccurrences())
+      SampleProfileUseProfi = true;
+
+    if (FunctionSamples::ProfileIsCSFlat) {
+      // Tracker for profiles under different context
+      ContextTracker = std::make_unique<SampleContextTracker>(
+          Reader->getProfiles(), &GUIDToFuncNameMap);
+    }
   }
 
   // Load pseudo probe descriptors for probe-based function samples.
@@ -1994,7 +2013,8 @@ bool SampleProfileLoader::doInitialization(Module &M,
     if (!ProbeManager->moduleIsProbed(M)) {
       const char *Msg =
           "Pseudo-probe-based profile requires SampleProfileProbePass";
-      Ctx.diagnose(DiagnosticInfoSampleProfile(Filename, Msg));
+      Ctx.diagnose(DiagnosticInfoSampleProfile(M.getModuleIdentifier(), Msg,
+                                               DS_Warning));
       return false;
     }
   }
@@ -2062,7 +2082,7 @@ bool SampleProfileLoader::runOnModule(Module &M, ModuleAnalysisManager *AM,
   }
 
   // Account for cold calls not inlined....
-  if (!ProfileIsCS)
+  if (!ProfileIsCSFlat)
     for (const std::pair<Function *, NotInlinedProfileInfo> &pair :
          notInlinedCallInfo)
       updateProfileCallee(pair.first, pair.second.entryCount);
@@ -2138,7 +2158,7 @@ bool SampleProfileLoader::runOnFunction(Function &F, ModuleAnalysisManager *AM)
     ORE = OwnedORE.get();
   }
 
-  if (ProfileIsCS)
+  if (ProfileIsCSFlat)
     Samples = ContextTracker->getBaseSamplesFor(F);
   else
     Samples = Reader->getSamplesFor(F);
diff --git a/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp b/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp
index 0cc1b37844f6..daaf6cbeb3fd 100644
--- a/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp
+++ b/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp
@@ -87,7 +87,8 @@ void promoteInternals(Module &ExportM, Module &ImportM, StringRef ModuleId,
     if (isa<Function>(&ExportGV) && allowPromotionAlias(OldName)) {
       // Create a local alias with the original name to avoid breaking
       // references from inline assembly.
-      std::string Alias = ".set " + OldName + "," + NewName + "\n";
+      std::string Alias =
+          ".lto_set_conditional " + OldName + "," + NewName + "\n";
       ExportM.appendModuleInlineAsm(Alias);
     }
   }
diff --git a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
index 61054e7ae46f..6acace1d9fd4 100644
--- a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
+++ b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
@@ -359,6 +359,36 @@ template <> struct DenseMapInfo<VTableSlotSummary> {
 
 namespace {
 
+// Returns true if the function must be unreachable based on ValueInfo.
+//
+// In particular, identifies a function as unreachable in the following
+// conditions
+//   1) All summaries are live.
+//   2) All function summaries indicate it's unreachable
+bool mustBeUnreachableFunction(ValueInfo TheFnVI) {
+  if ((!TheFnVI) || TheFnVI.getSummaryList().empty()) {
+    // Returns false if ValueInfo is absent, or the summary list is empty
+    // (e.g., function declarations).
+    return false;
+  }
+
+  for (auto &Summary : TheFnVI.getSummaryList()) {
+    // Conservatively returns false if any non-live functions are seen.
+    // In general either all summaries should be live or all should be dead.
+    if (!Summary->isLive())
+      return false;
+    if (auto *FS = dyn_cast<FunctionSummary>(Summary.get())) {
+      if (!FS->fflags().MustBeUnreachable)
+        return false;
+    }
+    // Do nothing if a non-function has the same GUID (which is rare).
+    // This is correct since non-function summaries are not relevant.
+  }
+  // All function summaries are live and all of them agree that the function is
+  // unreachble.
+  return true;
+}
+
 // A virtual call site. VTable is the loaded virtual table pointer, and CS is
 // the indirect virtual call.
 struct VirtualCallSite {
@@ -562,10 +592,12 @@ struct DevirtModule {
   void buildTypeIdentifierMap(
       std::vector<VTableBits> &Bits,
       DenseMap<Metadata *, std::set<TypeMemberInfo>> &TypeIdMap);
+
   bool
   tryFindVirtualCallTargets(std::vector<VirtualCallTarget> &TargetsForSlot,
                             const std::set<TypeMemberInfo> &TypeMemberInfos,
-                            uint64_t ByteOffset);
+                            uint64_t ByteOffset,
+                            ModuleSummaryIndex *ExportSummary);
 
   void applySingleImplDevirt(VTableSlotInfo &SlotInfo, Constant *TheFn,
                              bool &IsExported);
@@ -640,6 +672,23 @@ struct DevirtModule {
 
   bool run();
 
+  // Look up the corresponding ValueInfo entry of `TheFn` in `ExportSummary`.
+  //
+  // Caller guarantees that `ExportSummary` is not nullptr.
+  static ValueInfo lookUpFunctionValueInfo(Function *TheFn,
+                                           ModuleSummaryIndex *ExportSummary);
+
+  // Returns true if the function definition must be unreachable.
+  //
+  // Note if this helper function returns true, `F` is guaranteed
+  // to be unreachable; if it returns false, `F` might still
+  // be unreachable but not covered by this helper function.
+  //
+  // Implementation-wise, if function definition is present, IR is analyzed; if
+  // not, look up function flags from ExportSummary as a fallback.
+  static bool mustBeUnreachableFunction(Function *const F,
+                                        ModuleSummaryIndex *ExportSummary);
+
   // Lower the module using the action and summary passed as command line
   // arguments. For testing purposes only.
   static bool
@@ -969,7 +1018,8 @@ void DevirtModule::buildTypeIdentifierMap(
 
 bool DevirtModule::tryFindVirtualCallTargets(
     std::vector<VirtualCallTarget> &TargetsForSlot,
-    const std::set<TypeMemberInfo> &TypeMemberInfos, uint64_t ByteOffset) {
+    const std::set<TypeMemberInfo> &TypeMemberInfos, uint64_t ByteOffset,
+    ModuleSummaryIndex *ExportSummary) {
   for (const TypeMemberInfo &TM : TypeMemberInfos) {
     if (!TM.Bits->GV->isConstant())
       return false;
@@ -997,6 +1047,11 @@ bool DevirtModule::tryFindVirtualCallTargets(
     if (Fn->getName() == "__cxa_pure_virtual")
       continue;
 
+    // We can disregard unreachable functions as possible call targets, as
+    // unreachable functions shouldn't be called.
+    if (mustBeUnreachableFunction(Fn, ExportSummary))
+      continue;
+
     TargetsForSlot.push_back({Fn, &TM});
   }
 
@@ -1053,6 +1108,9 @@ bool DevirtIndex::tryFindVirtualCallTargets(
       if (VTP.VTableOffset != P.AddressPointOffset + ByteOffset)
         continue;
 
+      if (mustBeUnreachableFunction(VTP.FuncVI))
+        continue;
+
       TargetsForSlot.push_back(VTP.FuncVI);
     }
   }
@@ -1744,7 +1802,7 @@ void DevirtModule::rebuildGlobal(VTableBits &B) {
                          GlobalVariable::PrivateLinkage, NewInit, "", B.GV);
   NewGV->setSection(B.GV->getSection());
   NewGV->setComdat(B.GV->getComdat());
-  NewGV->setAlignment(MaybeAlign(B.GV->getAlignment()));
+  NewGV->setAlignment(B.GV->getAlign());
 
   // Copy the original vtable's metadata to the anonymous global, adjusting
   // offsets as required.
@@ -2014,6 +2072,44 @@ void DevirtModule::removeRedundantTypeTests() {
   }
 }
 
+ValueInfo
+DevirtModule::lookUpFunctionValueInfo(Function *TheFn,
+                                      ModuleSummaryIndex *ExportSummary) {
+  assert((ExportSummary != nullptr) &&
+         "Caller guarantees ExportSummary is not nullptr");
+
+  const auto TheFnGUID = TheFn->getGUID();
+  const auto TheFnGUIDWithExportedName = GlobalValue::getGUID(TheFn->getName());
+  // Look up ValueInfo with the GUID in the current linkage.
+  ValueInfo TheFnVI = ExportSummary->getValueInfo(TheFnGUID);
+  // If no entry is found and GUID is different from GUID computed using
+  // exported name, look up ValueInfo with the exported name unconditionally.
+  // This is a fallback.
+  //
+  // The reason to have a fallback:
+  // 1. LTO could enable global value internalization via
+  // `enable-lto-internalization`.
+  // 2. The GUID in ExportedSummary is computed using exported name.
+  if ((!TheFnVI) && (TheFnGUID != TheFnGUIDWithExportedName)) {
+    TheFnVI = ExportSummary->getValueInfo(TheFnGUIDWithExportedName);
+  }
+  return TheFnVI;
+}
+
+bool DevirtModule::mustBeUnreachableFunction(
+    Function *const F, ModuleSummaryIndex *ExportSummary) {
+  // First, learn unreachability by analyzing function IR.
+  if (!F->isDeclaration()) {
+    // A function must be unreachable if its entry block ends with an
+    // 'unreachable'.
+    return isa<UnreachableInst>(F->getEntryBlock().getTerminator());
+  }
+  // Learn unreachability from ExportSummary if ExportSummary is present.
+  return ExportSummary &&
+         ::mustBeUnreachableFunction(
+             DevirtModule::lookUpFunctionValueInfo(F, ExportSummary));
+}
+
 bool DevirtModule::run() {
   // If only some of the modules were split, we cannot correctly perform
   // this transformation. We already checked for the presense of type tests
@@ -2137,7 +2233,7 @@ bool DevirtModule::run() {
                      cast<MDString>(S.first.TypeID)->getString())
                  .WPDRes[S.first.ByteOffset];
     if (tryFindVirtualCallTargets(TargetsForSlot, TypeMemberInfos,
-                                  S.first.ByteOffset)) {
+                                  S.first.ByteOffset, ExportSummary)) {
 
       if (!trySingleImplDevirt(ExportSummary, TargetsForSlot, S.second, Res)) {
         DidVirtualConstProp |=
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
index dc55b5a31596..de1034c910d5 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
@@ -1795,6 +1795,55 @@ static Instruction *foldComplexAndOrPatterns(BinaryOperator &I,
     }
   }
 
+  // (~A & B & C) | ... --> ...
+  // (~A | B | C) | ... --> ...
+  // TODO: One use checks are conservative. We just need to check that a total
+  //       number of multiple used values does not exceed reduction
+  //       in operations.
+  if (match(Op0,
+            m_OneUse(m_c_BinOp(FlippedOpcode,
+                               m_BinOp(FlippedOpcode, m_Value(B), m_Value(C)),
+                               m_CombineAnd(m_Value(X), m_Not(m_Value(A)))))) ||
+      match(Op0, m_OneUse(m_c_BinOp(
+                     FlippedOpcode,
+                     m_c_BinOp(FlippedOpcode, m_Value(C),
+                               m_CombineAnd(m_Value(X), m_Not(m_Value(A)))),
+                     m_Value(B))))) {
+    // X = ~A
+    // (~A & B & C) | ~(A | B | C) --> ~(A | (B ^ C))
+    // (~A | B | C) & ~(A & B & C) --> (~A | (B ^ C))
+    if (match(Op1, m_OneUse(m_Not(m_c_BinOp(
+                       Opcode, m_c_BinOp(Opcode, m_Specific(A), m_Specific(B)),
+                       m_Specific(C))))) ||
+        match(Op1, m_OneUse(m_Not(m_c_BinOp(
+                       Opcode, m_c_BinOp(Opcode, m_Specific(B), m_Specific(C)),
+                       m_Specific(A))))) ||
+        match(Op1, m_OneUse(m_Not(m_c_BinOp(
+                       Opcode, m_c_BinOp(Opcode, m_Specific(A), m_Specific(C)),
+                       m_Specific(B)))))) {
+      Value *Xor = Builder.CreateXor(B, C);
+      return (Opcode == Instruction::Or)
+                 ? BinaryOperator::CreateNot(Builder.CreateOr(Xor, A))
+                 : BinaryOperator::CreateOr(Xor, X);
+    }
+
+    // (~A & B & C) | ~(A | B) --> (C | ~B) & ~A
+    // (~A | B | C) & ~(A & B) --> (C & ~B) | ~A
+    if (match(Op1, m_OneUse(m_Not(m_OneUse(
+                       m_c_BinOp(Opcode, m_Specific(A), m_Specific(B)))))))
+      return BinaryOperator::Create(
+          FlippedOpcode, Builder.CreateBinOp(Opcode, C, Builder.CreateNot(B)),
+          X);
+
+    // (~A & B & C) | ~(A | C) --> (B | ~C) & ~A
+    // (~A | B | C) & ~(A & C) --> (B & ~C) | ~A
+    if (match(Op1, m_OneUse(m_Not(m_OneUse(
+                       m_c_BinOp(Opcode, m_Specific(A), m_Specific(C)))))))
+      return BinaryOperator::Create(
+          FlippedOpcode, Builder.CreateBinOp(Opcode, B, Builder.CreateNot(C)),
+          X);
+  }
+
   return nullptr;
 }
 
@@ -2102,6 +2151,15 @@ Instruction *InstCombinerImpl::visitAnd(BinaryOperator &I) {
     Value *Cmp = Builder.CreateICmpSLT(X, Zero, "isneg");
     return SelectInst::Create(Cmp, Y, Zero);
   }
+  // If there's a 'not' of the shifted value, swap the select operands:
+  // ~(iN X s>> (N-1)) & Y --> (X s< 0) ? 0 : Y
+  if (match(&I, m_c_And(m_OneUse(m_Not(
+                            m_AShr(m_Value(X), m_SpecificInt(FullShift)))),
+                        m_Value(Y)))) {
+    Constant *Zero = ConstantInt::getNullValue(Ty);
+    Value *Cmp = Builder.CreateICmpSLT(X, Zero, "isneg");
+    return SelectInst::Create(Cmp, Zero, Y);
+  }
 
   // (~x) & y  -->  ~(x | (~y))  iff that gets rid of inversions
   if (sinkNotIntoOtherHandOfAndOrOr(I))
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 7da2669e1d13..14427bd1f2f4 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -2472,6 +2472,12 @@ static bool isSafeToEliminateVarargsCast(const CallBase &Call,
 Instruction *InstCombinerImpl::tryOptimizeCall(CallInst *CI) {
   if (!CI->getCalledFunction()) return nullptr;
 
+  // Skip optimizing notail and musttail calls so
+  // LibCallSimplifier::optimizeCall doesn't have to preserve those invariants.
+  // LibCallSimplifier::optimizeCall should try to preseve tail calls though.
+  if (CI->isMustTailCall() || CI->isNoTailCall())
+    return nullptr;
+
   auto InstCombineRAUW = [this](Instruction *From, Value *With) {
     replaceInstUsesWith(*From, With);
   };
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
index 33f217659c01..8df4a4529f47 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
@@ -157,7 +157,7 @@ Instruction *InstCombinerImpl::PromoteCastOfAllocation(BitCastInst &CI,
     Amt = Builder.CreateAdd(Amt, Off);
   }
 
-  AllocaInst *New = Builder.CreateAlloca(CastElTy, Amt);
+  AllocaInst *New = Builder.CreateAlloca(CastElTy, AI.getAddressSpace(), Amt);
   New->setAlignment(AI.getAlign());
   New->takeName(&AI);
   New->setUsedWithInAlloca(AI.isUsedWithInAlloca());
@@ -965,13 +965,13 @@ Instruction *InstCombinerImpl::visitTrunc(TruncInst &Trunc) {
   if (match(Src, m_VScale(DL))) {
     if (Trunc.getFunction() &&
         Trunc.getFunction()->hasFnAttribute(Attribute::VScaleRange)) {
-      unsigned MaxVScale = Trunc.getFunction()
-                               ->getFnAttribute(Attribute::VScaleRange)
-                               .getVScaleRangeArgs()
-                               .second;
-      if (MaxVScale > 0 && Log2_32(MaxVScale) < DestWidth) {
-        Value *VScale = Builder.CreateVScale(ConstantInt::get(DestTy, 1));
-        return replaceInstUsesWith(Trunc, VScale);
+      Attribute Attr =
+          Trunc.getFunction()->getFnAttribute(Attribute::VScaleRange);
+      if (Optional<unsigned> MaxVScale = Attr.getVScaleRangeMax()) {
+        if (Log2_32(MaxVScale.getValue()) < DestWidth) {
+          Value *VScale = Builder.CreateVScale(ConstantInt::get(DestTy, 1));
+          return replaceInstUsesWith(Trunc, VScale);
+        }
       }
     }
   }
@@ -1337,14 +1337,13 @@ Instruction *InstCombinerImpl::visitZExt(ZExtInst &CI) {
   if (match(Src, m_VScale(DL))) {
     if (CI.getFunction() &&
         CI.getFunction()->hasFnAttribute(Attribute::VScaleRange)) {
-      unsigned MaxVScale = CI.getFunction()
-                               ->getFnAttribute(Attribute::VScaleRange)
-                               .getVScaleRangeArgs()
-                               .second;
-      unsigned TypeWidth = Src->getType()->getScalarSizeInBits();
-      if (MaxVScale > 0 && Log2_32(MaxVScale) < TypeWidth) {
-        Value *VScale = Builder.CreateVScale(ConstantInt::get(DestTy, 1));
-        return replaceInstUsesWith(CI, VScale);
+      Attribute Attr = CI.getFunction()->getFnAttribute(Attribute::VScaleRange);
+      if (Optional<unsigned> MaxVScale = Attr.getVScaleRangeMax()) {
+        unsigned TypeWidth = Src->getType()->getScalarSizeInBits();
+        if (Log2_32(MaxVScale.getValue()) < TypeWidth) {
+          Value *VScale = Builder.CreateVScale(ConstantInt::get(DestTy, 1));
+          return replaceInstUsesWith(CI, VScale);
+        }
       }
     }
   }
@@ -1608,13 +1607,12 @@ Instruction *InstCombinerImpl::visitSExt(SExtInst &CI) {
   if (match(Src, m_VScale(DL))) {
     if (CI.getFunction() &&
         CI.getFunction()->hasFnAttribute(Attribute::VScaleRange)) {
-      unsigned MaxVScale = CI.getFunction()
-                               ->getFnAttribute(Attribute::VScaleRange)
-                               .getVScaleRangeArgs()
-                               .second;
-      if (MaxVScale > 0 && Log2_32(MaxVScale) < (SrcBitSize - 1)) {
-        Value *VScale = Builder.CreateVScale(ConstantInt::get(DestTy, 1));
-        return replaceInstUsesWith(CI, VScale);
+      Attribute Attr = CI.getFunction()->getFnAttribute(Attribute::VScaleRange);
+      if (Optional<unsigned> MaxVScale = Attr.getVScaleRangeMax()) {
+        if (Log2_32(MaxVScale.getValue()) < (SrcBitSize - 1)) {
+          Value *VScale = Builder.CreateVScale(ConstantInt::get(DestTy, 1));
+          return replaceInstUsesWith(CI, VScale);
+        }
       }
     }
   }
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
index 20c75188ec9f..39b55b028110 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
+++ b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
@@ -600,6 +600,7 @@ public:
   /// Canonicalize the position of binops relative to shufflevector.
   Instruction *foldVectorBinop(BinaryOperator &Inst);
   Instruction *foldVectorSelect(SelectInst &Sel);
+  Instruction *foldSelectShuffle(ShuffleVectorInst &Shuf);
 
   /// Given a binary operator, cast instruction, or select which has a PHI node
   /// as operand #0, see if we can fold the instruction into the PHI (which is
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
index 79a8a065d02a..0dbfdba353c4 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
@@ -163,7 +163,7 @@ static bool isDereferenceableForAllocaSize(const Value *V, const AllocaInst *AI,
   uint64_t AllocaSize = DL.getTypeStoreSize(AI->getAllocatedType());
   if (!AllocaSize)
     return false;
-  return isDereferenceableAndAlignedPointer(V, Align(AI->getAlignment()),
+  return isDereferenceableAndAlignedPointer(V, AI->getAlign(),
                                             APInt(64, AllocaSize), DL);
 }
 
@@ -183,7 +183,8 @@ static Instruction *simplifyAllocaArraySize(InstCombinerImpl &IC,
   if (const ConstantInt *C = dyn_cast<ConstantInt>(AI.getArraySize())) {
     if (C->getValue().getActiveBits() <= 64) {
       Type *NewTy = ArrayType::get(AI.getAllocatedType(), C->getZExtValue());
-      AllocaInst *New = IC.Builder.CreateAlloca(NewTy, nullptr, AI.getName());
+      AllocaInst *New = IC.Builder.CreateAlloca(NewTy, AI.getAddressSpace(),
+                                                nullptr, AI.getName());
       New->setAlignment(AI.getAlign());
 
       // Scan to the end of the allocation instructions, to skip over a block of
@@ -199,21 +200,13 @@ static Instruction *simplifyAllocaArraySize(InstCombinerImpl &IC,
       Type *IdxTy = IC.getDataLayout().getIntPtrType(AI.getType());
       Value *NullIdx = Constant::getNullValue(IdxTy);
       Value *Idx[2] = {NullIdx, NullIdx};
-      Instruction *NewI = GetElementPtrInst::CreateInBounds(
+      Instruction *GEP = GetElementPtrInst::CreateInBounds(
           NewTy, New, Idx, New->getName() + ".sub");
-      IC.InsertNewInstBefore(NewI, *It);
-
-      // Gracefully handle allocas in other address spaces.
-      if (AI.getType()->getPointerAddressSpace() !=
-          NewI->getType()->getPointerAddressSpace()) {
-        NewI =
-            CastInst::CreatePointerBitCastOrAddrSpaceCast(NewI, AI.getType());
-        IC.InsertNewInstBefore(NewI, *It);
-      }
+      IC.InsertNewInstBefore(GEP, *It);
 
       // Now make everything use the getelementptr instead of the original
       // allocation.
-      return IC.replaceInstUsesWith(AI, NewI);
+      return IC.replaceInstUsesWith(AI, GEP);
     }
   }
 
@@ -640,7 +633,6 @@ static Instruction *unpackLoadToAggregate(InstCombinerImpl &IC, LoadInst &LI) {
     return nullptr;
 
   StringRef Name = LI.getName();
-  assert(LI.getAlignment() && "Alignment must be set at this point");
 
   if (auto *ST = dyn_cast<StructType>(T)) {
     // If the struct only have one element, we unpack.
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
index 779d298da7a4..aca7ec8d7325 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
@@ -755,6 +755,15 @@ Instruction *InstCombinerImpl::commonIDivTransforms(BinaryOperator &I) {
   if (simplifyDivRemOfSelectWithZeroOp(I))
     return &I;
 
+  // If the divisor is a select-of-constants, try to constant fold all div ops:
+  // C / (select Cond, TrueC, FalseC) --> select Cond, (C / TrueC), (C / FalseC)
+  // TODO: Adapt simplifyDivRemOfSelectWithZeroOp to allow this and other folds.
+  if (match(Op0, m_ImmConstant()) &&
+      match(Op1, m_Select(m_Value(), m_ImmConstant(), m_ImmConstant()))) {
+    if (Instruction *R = FoldOpIntoSelect(I, cast<SelectInst>(Op1)))
+      return R;
+  }
+
   const APInt *C2;
   if (match(Op1, m_APInt(C2))) {
     Value *X;
@@ -1461,6 +1470,15 @@ Instruction *InstCombinerImpl::commonIRemTransforms(BinaryOperator &I) {
   if (simplifyDivRemOfSelectWithZeroOp(I))
     return &I;
 
+  // If the divisor is a select-of-constants, try to constant fold all rem ops:
+  // C % (select Cond, TrueC, FalseC) --> select Cond, (C % TrueC), (C % FalseC)
+  // TODO: Adapt simplifyDivRemOfSelectWithZeroOp to allow this and other folds.
+  if (match(Op0, m_ImmConstant()) &&
+      match(Op1, m_Select(m_Value(), m_ImmConstant(), m_ImmConstant()))) {
+    if (Instruction *R = FoldOpIntoSelect(I, cast<SelectInst>(Op1)))
+      return R;
+  }
+
   if (isa<Constant>(Op1)) {
     if (Instruction *Op0I = dyn_cast<Instruction>(Op0)) {
       if (SelectInst *SI = dyn_cast<SelectInst>(Op0I)) {
diff --git a/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp b/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp
index 35739c3b9a21..30f6aab2114b 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp
@@ -664,10 +664,7 @@ Instruction *InstCombinerImpl::foldPHIArgLoadIntoPHI(PHINode &PN) {
     return nullptr;
 
   // When processing loads, we need to propagate two bits of information to the
-  // sunk load: whether it is volatile, and what its alignment is.  We currently
-  // don't sink loads when some have their alignment specified and some don't.
-  // visitLoadInst will propagate an alignment onto the load when TD is around,
-  // and if TD isn't around, we can't handle the mixed case.
+  // sunk load: whether it is volatile, and what its alignment is.
   bool isVolatile = FirstLI->isVolatile();
   Align LoadAlignment = FirstLI->getAlign();
   unsigned LoadAddrSpace = FirstLI->getPointerAddressSpace();
@@ -699,7 +696,7 @@ Instruction *InstCombinerImpl::foldPHIArgLoadIntoPHI(PHINode &PN) {
         !isSafeAndProfitableToSinkLoad(LI))
       return nullptr;
 
-    LoadAlignment = std::min(LoadAlignment, Align(LI->getAlign()));
+    LoadAlignment = std::min(LoadAlignment, LI->getAlign());
 
     // If the PHI is of volatile loads and the load block has multiple
     // successors, sinking it would remove a load of the volatile value from
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
index 518d3952dce5..a6d6b5199105 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
@@ -1482,7 +1482,12 @@ tryToReuseConstantFromSelectInComparison(SelectInst &Sel, ICmpInst &Cmp,
   if (C0->getType() != Sel.getType())
     return nullptr;
 
-  // FIXME: are there any magic icmp predicate+constant pairs we must not touch?
+  // ULT with 'add' of a constant is canonical. See foldICmpAddConstant().
+  // FIXME: Are there more magic icmp predicate+constant pairs we must avoid?
+  //        Or should we just abandon this transform entirely?
+  if (Pred == CmpInst::ICMP_ULT && match(X, m_Add(m_Value(), m_Constant())))
+    return nullptr;
+
 
   Value *SelVal0, *SelVal1; // We do not care which one is from where.
   match(&Sel, m_Select(m_Value(), m_Value(SelVal0), m_Value(SelVal1)));
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
index e357a9da8b12..4dc712f32536 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
@@ -1595,12 +1595,6 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V,
     simplifyAndSetOp(I, 0, DemandedElts, UndefElts);
     simplifyAndSetOp(I, 1, DemandedElts, UndefElts2);
 
-    // Any change to an instruction with potential poison must clear those flags
-    // because we can not guarantee those constraints now. Other analysis may
-    // determine that it is safe to re-apply the flags.
-    if (MadeChange)
-      BO->dropPoisonGeneratingFlags();
-
     // Output elements are undefined if both are undefined. Consider things
     // like undef & 0. The result is known zero, not undef.
     UndefElts &= UndefElts2;
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
index 32e537897140..c6a4602e59e3 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
@@ -363,6 +363,18 @@ static APInt findDemandedEltsByAllUsers(Value *V) {
   return UnionUsedElts;
 }
 
+/// Given a constant index for a extractelement or insertelement instruction,
+/// return it with the canonical type if it isn't already canonical.  We
+/// arbitrarily pick 64 bit as our canonical type.  The actual bitwidth doesn't
+/// matter, we just want a consistent type to simplify CSE.
+ConstantInt *getPreferredVectorIndex(ConstantInt *IndexC) {
+  const unsigned IndexBW = IndexC->getType()->getBitWidth();
+  if (IndexBW == 64 || IndexC->getValue().getActiveBits() > 64)
+    return nullptr;
+  return ConstantInt::get(IndexC->getContext(),
+                          IndexC->getValue().zextOrTrunc(64));
+}
+
 Instruction *InstCombinerImpl::visitExtractElementInst(ExtractElementInst &EI) {
   Value *SrcVec = EI.getVectorOperand();
   Value *Index = EI.getIndexOperand();
@@ -374,6 +386,10 @@ Instruction *InstCombinerImpl::visitExtractElementInst(ExtractElementInst &EI) {
   // find a previously computed scalar that was inserted into the vector.
   auto *IndexC = dyn_cast<ConstantInt>(Index);
   if (IndexC) {
+    // Canonicalize type of constant indices to i64 to simplify CSE
+    if (auto *NewIdx = getPreferredVectorIndex(IndexC))
+      return replaceOperand(EI, 1, NewIdx);
+
     ElementCount EC = EI.getVectorOperandType()->getElementCount();
     unsigned NumElts = EC.getKnownMinValue();
 
@@ -401,37 +417,6 @@ Instruction *InstCombinerImpl::visitExtractElementInst(ExtractElementInst &EI) {
     if (!EC.isScalable() && IndexC->getValue().uge(NumElts))
       return nullptr;
 
-    // This instruction only demands the single element from the input vector.
-    // Skip for scalable type, the number of elements is unknown at
-    // compile-time.
-    if (!EC.isScalable() && NumElts != 1) {
-      // If the input vector has a single use, simplify it based on this use
-      // property.
-      if (SrcVec->hasOneUse()) {
-        APInt UndefElts(NumElts, 0);
-        APInt DemandedElts(NumElts, 0);
-        DemandedElts.setBit(IndexC->getZExtValue());
-        if (Value *V =
-                SimplifyDemandedVectorElts(SrcVec, DemandedElts, UndefElts))
-          return replaceOperand(EI, 0, V);
-      } else {
-        // If the input vector has multiple uses, simplify it based on a union
-        // of all elements used.
-        APInt DemandedElts = findDemandedEltsByAllUsers(SrcVec);
-        if (!DemandedElts.isAllOnes()) {
-          APInt UndefElts(NumElts, 0);
-          if (Value *V = SimplifyDemandedVectorElts(
-                  SrcVec, DemandedElts, UndefElts, 0 /* Depth */,
-                  true /* AllowMultipleUsers */)) {
-            if (V != SrcVec) {
-              SrcVec->replaceAllUsesWith(V);
-              return &EI;
-            }
-          }
-        }
-      }
-    }
-
     if (Instruction *I = foldBitcastExtElt(EI))
       return I;
 
@@ -473,11 +458,9 @@ Instruction *InstCombinerImpl::visitExtractElementInst(ExtractElementInst &EI) {
 
   if (auto *I = dyn_cast<Instruction>(SrcVec)) {
     if (auto *IE = dyn_cast<InsertElementInst>(I)) {
-      // Extracting the inserted element?
-      if (IE->getOperand(2) == Index)
-        return replaceInstUsesWith(EI, IE->getOperand(1));
-      // If the inserted and extracted elements are constants, they must not
-      // be the same value, extract from the pre-inserted value instead.
+      // instsimplify already handled the case where the indices are constants
+      // and equal by value, if both are constants, they must not be the same
+      // value, extract from the pre-inserted value instead.
       if (isa<Constant>(IE->getOperand(2)) && IndexC)
         return replaceOperand(EI, 0, IE->getOperand(0));
     } else if (auto *GEP = dyn_cast<GetElementPtrInst>(I)) {
@@ -497,30 +480,27 @@ Instruction *InstCombinerImpl::visitExtractElementInst(ExtractElementInst &EI) {
             llvm::count_if(GEP->operands(), [](const Value *V) {
               return isa<VectorType>(V->getType());
             });
-        if (VectorOps > 1)
-          return nullptr;
-        assert(VectorOps == 1 && "Expected exactly one vector GEP operand!");
-
-        Value *NewPtr = GEP->getPointerOperand();
-        if (isa<VectorType>(NewPtr->getType()))
-          NewPtr = Builder.CreateExtractElement(NewPtr, IndexC);
-
-        SmallVector<Value *> NewOps;
-        for (unsigned I = 1; I != GEP->getNumOperands(); ++I) {
-          Value *Op = GEP->getOperand(I);
-          if (isa<VectorType>(Op->getType()))
-            NewOps.push_back(Builder.CreateExtractElement(Op, IndexC));
-          else
-            NewOps.push_back(Op);
-        }
+        if (VectorOps == 1) {
+          Value *NewPtr = GEP->getPointerOperand();
+          if (isa<VectorType>(NewPtr->getType()))
+            NewPtr = Builder.CreateExtractElement(NewPtr, IndexC);
+
+          SmallVector<Value *> NewOps;
+          for (unsigned I = 1; I != GEP->getNumOperands(); ++I) {
+            Value *Op = GEP->getOperand(I);
+            if (isa<VectorType>(Op->getType()))
+              NewOps.push_back(Builder.CreateExtractElement(Op, IndexC));
+            else
+              NewOps.push_back(Op);
+          }
 
-        GetElementPtrInst *NewGEP = GetElementPtrInst::Create(
-            cast<PointerType>(NewPtr->getType())->getElementType(), NewPtr,
-            NewOps);
-        NewGEP->setIsInBounds(GEP->isInBounds());
-        return NewGEP;
+          GetElementPtrInst *NewGEP = GetElementPtrInst::Create(
+              cast<PointerType>(NewPtr->getType())->getElementType(), NewPtr,
+              NewOps);
+          NewGEP->setIsInBounds(GEP->isInBounds());
+          return NewGEP;
+        }
       }
-      return nullptr;
     } else if (auto *SVI = dyn_cast<ShuffleVectorInst>(I)) {
       // If this is extracting an element from a shufflevector, figure out where
       // it came from and extract from the appropriate input element instead.
@@ -554,6 +534,44 @@ Instruction *InstCombinerImpl::visitExtractElementInst(ExtractElementInst &EI) {
       }
     }
   }
+
+  // Run demanded elements after other transforms as this can drop flags on
+  // binops.  If there's two paths to the same final result, we prefer the
+  // one which doesn't force us to drop flags.
+  if (IndexC) {
+    ElementCount EC = EI.getVectorOperandType()->getElementCount();
+    unsigned NumElts = EC.getKnownMinValue();
+    // This instruction only demands the single element from the input vector.
+    // Skip for scalable type, the number of elements is unknown at
+    // compile-time.
+    if (!EC.isScalable() && NumElts != 1) {
+      // If the input vector has a single use, simplify it based on this use
+      // property.
+      if (SrcVec->hasOneUse()) {
+        APInt UndefElts(NumElts, 0);
+        APInt DemandedElts(NumElts, 0);
+        DemandedElts.setBit(IndexC->getZExtValue());
+        if (Value *V =
+                SimplifyDemandedVectorElts(SrcVec, DemandedElts, UndefElts))
+          return replaceOperand(EI, 0, V);
+      } else {
+        // If the input vector has multiple uses, simplify it based on a union
+        // of all elements used.
+        APInt DemandedElts = findDemandedEltsByAllUsers(SrcVec);
+        if (!DemandedElts.isAllOnes()) {
+          APInt UndefElts(NumElts, 0);
+          if (Value *V = SimplifyDemandedVectorElts(
+                  SrcVec, DemandedElts, UndefElts, 0 /* Depth */,
+                  true /* AllowMultipleUsers */)) {
+            if (V != SrcVec) {
+              SrcVec->replaceAllUsesWith(V);
+              return &EI;
+            }
+          }
+        }
+      }
+    }
+  }
   return nullptr;
 }
 
@@ -1476,6 +1494,11 @@ Instruction *InstCombinerImpl::visitInsertElementInst(InsertElementInst &IE) {
           VecOp, ScalarOp, IdxOp, SQ.getWithInstruction(&IE)))
     return replaceInstUsesWith(IE, V);
 
+  // Canonicalize type of constant indices to i64 to simplify CSE
+  if (auto *IndexC = dyn_cast<ConstantInt>(IdxOp))
+    if (auto *NewIdx = getPreferredVectorIndex(IndexC))
+      return replaceOperand(IE, 2, NewIdx);
+
   // If the scalar is bitcast and inserted into undef, do the insert in the
   // source type followed by bitcast.
   // TODO: Generalize for insert into any constant, not just undef?
@@ -2008,9 +2031,7 @@ static Instruction *canonicalizeInsertSplat(ShuffleVectorInst &Shuf,
 }
 
 /// Try to fold shuffles that are the equivalent of a vector select.
-static Instruction *foldSelectShuffle(ShuffleVectorInst &Shuf,
-                                      InstCombiner::BuilderTy &Builder,
-                                      const DataLayout &DL) {
+Instruction *InstCombinerImpl::foldSelectShuffle(ShuffleVectorInst &Shuf) {
   if (!Shuf.isSelect())
     return nullptr;
 
@@ -2118,21 +2139,23 @@ static Instruction *foldSelectShuffle(ShuffleVectorInst &Shuf,
     V = Builder.CreateShuffleVector(X, Y, Mask);
   }
 
-  Instruction *NewBO = ConstantsAreOp1 ? BinaryOperator::Create(BOpc, V, NewC) :
-                                         BinaryOperator::Create(BOpc, NewC, V);
+  Value *NewBO = ConstantsAreOp1 ? Builder.CreateBinOp(BOpc, V, NewC) :
+                                   Builder.CreateBinOp(BOpc, NewC, V);
 
   // Flags are intersected from the 2 source binops. But there are 2 exceptions:
   // 1. If we changed an opcode, poison conditions might have changed.
   // 2. If the shuffle had undef mask elements, the new binop might have undefs
   //    where the original code did not. But if we already made a safe constant,
   //    then there's no danger.
-  NewBO->copyIRFlags(B0);
-  NewBO->andIRFlags(B1);
-  if (DropNSW)
-    NewBO->setHasNoSignedWrap(false);
-  if (is_contained(Mask, UndefMaskElem) && !MightCreatePoisonOrUB)
-    NewBO->dropPoisonGeneratingFlags();
-  return NewBO;
+  if (auto *NewI = dyn_cast<Instruction>(NewBO)) {
+    NewI->copyIRFlags(B0);
+    NewI->andIRFlags(B1);
+    if (DropNSW)
+      NewI->setHasNoSignedWrap(false);
+    if (is_contained(Mask, UndefMaskElem) && !MightCreatePoisonOrUB)
+      NewI->dropPoisonGeneratingFlags();
+  }
+  return replaceInstUsesWith(Shuf, NewBO);
 }
 
 /// Convert a narrowing shuffle of a bitcasted vector into a vector truncate.
@@ -2497,7 +2520,7 @@ Instruction *InstCombinerImpl::visitShuffleVectorInst(ShuffleVectorInst &SVI) {
   if (Instruction *I = canonicalizeInsertSplat(SVI, Builder))
     return I;
 
-  if (Instruction *I = foldSelectShuffle(SVI, Builder, DL))
+  if (Instruction *I = foldSelectShuffle(SVI))
     return I;
 
   if (Instruction *I = foldTruncShuffle(SVI, DL.isBigEndian()))
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index 1f81624f79e7..eb5eadba194d 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -2546,7 +2546,7 @@ Instruction *InstCombinerImpl::visitGetElementPtrInst(GetElementPtrInst &GEP) {
   return nullptr;
 }
 
-static bool isNeverEqualToUnescapedAlloc(Value *V, const TargetLibraryInfo *TLI,
+static bool isNeverEqualToUnescapedAlloc(Value *V, const TargetLibraryInfo &TLI,
                                          Instruction *AI) {
   if (isa<ConstantPointerNull>(V))
     return true;
@@ -2557,12 +2557,34 @@ static bool isNeverEqualToUnescapedAlloc(Value *V, const TargetLibraryInfo *TLI,
   // through bitcasts of V can cause
   // the result statement below to be true, even when AI and V (ex:
   // i8* ->i32* ->i8* of AI) are the same allocations.
-  return isAllocLikeFn(V, TLI) && V != AI;
+  return isAllocLikeFn(V, &TLI) && V != AI;
+}
+
+/// Given a call CB which uses an address UsedV, return true if we can prove the
+/// call's only possible effect is storing to V.
+static bool isRemovableWrite(CallBase &CB, Value *UsedV,
+                             const TargetLibraryInfo &TLI) {
+  if (!CB.use_empty())
+    // TODO: add recursion if returned attribute is present
+    return false;
+
+  if (CB.isTerminator())
+    // TODO: remove implementation restriction
+    return false;
+
+  if (!CB.willReturn() || !CB.doesNotThrow())
+    return false;
+
+  // If the only possible side effect of the call is writing to the alloca,
+  // and the result isn't used, we can safely remove any reads implied by the
+  // call including those which might read the alloca itself.
+  Optional<MemoryLocation> Dest = MemoryLocation::getForDest(&CB, TLI);
+  return Dest && Dest->Ptr == UsedV;
 }
 
 static bool isAllocSiteRemovable(Instruction *AI,
                                  SmallVectorImpl<WeakTrackingVH> &Users,
-                                 const TargetLibraryInfo *TLI) {
+                                 const TargetLibraryInfo &TLI) {
   SmallVector<Instruction*, 4> Worklist;
   Worklist.push_back(AI);
 
@@ -2627,12 +2649,17 @@ static bool isAllocSiteRemovable(Instruction *AI,
           }
         }
 
-        if (isFreeCall(I, TLI)) {
+        if (isRemovableWrite(*cast<CallBase>(I), PI, TLI)) {
+          Users.emplace_back(I);
+          continue;
+        }
+
+        if (isFreeCall(I, &TLI)) {
           Users.emplace_back(I);
           continue;
         }
 
-        if (isReallocLikeFn(I, TLI, true)) {
+        if (isReallocLikeFn(I, &TLI, true)) {
           Users.emplace_back(I);
           Worklist.push_back(I);
           continue;
@@ -2676,7 +2703,7 @@ Instruction *InstCombinerImpl::visitAllocSite(Instruction &MI) {
     DIB.reset(new DIBuilder(*MI.getModule(), /*AllowUnresolved=*/false));
   }
 
-  if (isAllocSiteRemovable(&MI, Users, &TLI)) {
+  if (isAllocSiteRemovable(&MI, Users, TLI)) {
     for (unsigned i = 0, e = Users.size(); i != e; ++i) {
       // Lowering all @llvm.objectsize calls first because they may
       // use a bitcast/GEP of the alloca we are removing.
diff --git a/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
index 38c219ce3465..9f26b37bbc79 100644
--- a/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
@@ -232,6 +232,12 @@ static cl::opt<int> ClTrackOrigins("dfsan-track-origins",
                                    cl::desc("Track origins of labels"),
                                    cl::Hidden, cl::init(0));
 
+static cl::opt<bool> ClIgnorePersonalityRoutine(
+    "dfsan-ignore-personality-routine",
+    cl::desc("If a personality routine is marked uninstrumented from the ABI "
+             "list, do not create a wrapper for it."),
+    cl::Hidden, cl::init(false));
+
 static StringRef getGlobalTypeString(const GlobalValue &G) {
   // Types of GlobalVariables are always pointer types.
   Type *GType = G.getValueType();
@@ -1115,7 +1121,7 @@ DataFlowSanitizer::buildWrapperFunction(Function *F, StringRef NewFName,
 
   BasicBlock *BB = BasicBlock::Create(*Ctx, "entry", NewF);
   if (F->isVarArg()) {
-    NewF->removeFnAttrs(AttrBuilder().addAttribute("split-stack"));
+    NewF->removeFnAttr("split-stack");
     CallInst::Create(DFSanVarargWrapperFn,
                      IRBuilder<>(BB).CreateGlobalStringPtr(F->getName()), "",
                      BB);
@@ -1357,9 +1363,24 @@ bool DataFlowSanitizer::runImpl(Module &M) {
   std::vector<Function *> FnsToInstrument;
   SmallPtrSet<Function *, 2> FnsWithNativeABI;
   SmallPtrSet<Function *, 2> FnsWithForceZeroLabel;
+  SmallPtrSet<Constant *, 1> PersonalityFns;
   for (Function &F : M)
-    if (!F.isIntrinsic() && !DFSanRuntimeFunctions.contains(&F))
+    if (!F.isIntrinsic() && !DFSanRuntimeFunctions.contains(&F)) {
       FnsToInstrument.push_back(&F);
+      if (F.hasPersonalityFn())
+        PersonalityFns.insert(F.getPersonalityFn()->stripPointerCasts());
+    }
+
+  if (ClIgnorePersonalityRoutine) {
+    for (auto *C : PersonalityFns) {
+      assert(isa<Function>(C) && "Personality routine is not a function!");
+      Function *F = cast<Function>(C);
+      if (!isInstrumented(F))
+        FnsToInstrument.erase(
+            std::remove(FnsToInstrument.begin(), FnsToInstrument.end(), F),
+            FnsToInstrument.end());
+    }
+  }
 
   // Give function aliases prefixes when necessary, and build wrappers where the
   // instrumentedness is inconsistent.
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index d1d3b8ffdf7a..de34348606ef 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -26,7 +26,9 @@
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Constant.h"
 #include "llvm/IR/Constants.h"
+#include "llvm/IR/DIBuilder.h"
 #include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/DiagnosticInfo.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/GlobalValue.h"
@@ -40,6 +42,7 @@
 #include "llvm/InitializePasses.h"
 #include "llvm/Pass.h"
 #include "llvm/ProfileData/InstrProf.h"
+#include "llvm/ProfileData/InstrProfCorrelator.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Error.h"
@@ -57,6 +60,13 @@ using namespace llvm;
 
 #define DEBUG_TYPE "instrprof"
 
+namespace llvm {
+cl::opt<bool>
+    DebugInfoCorrelate("debug-info-correlate", cl::ZeroOrMore,
+                       cl::desc("Use debug info to correlate profiles."),
+                       cl::init(false));
+} // namespace llvm
+
 namespace {
 
 cl::opt<bool> DoHashBasedCounterSplit(
@@ -641,6 +651,12 @@ void InstrProfiling::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) {
 }
 
 void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
+  // TODO: Value profiling heavily depends on the data section which is omitted
+  // in lightweight mode. We need to move the value profile pointer to the
+  // Counter struct to get this working.
+  assert(
+      !DebugInfoCorrelate &&
+      "Value profiling is not yet supported with lightweight instrumentation");
   GlobalVariable *Name = Ind->getName();
   auto It = ProfileDataMap.find(Name);
   assert(It != ProfileDataMap.end() && It->second.DataVar &&
@@ -855,6 +871,12 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
   GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage();
   GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();
 
+  // Use internal rather than private linkage so the counter variable shows up
+  // in the symbol table when using debug info for correlation.
+  if (DebugInfoCorrelate && TT.isOSBinFormatMachO() &&
+      Linkage == GlobalValue::PrivateLinkage)
+    Linkage = GlobalValue::InternalLinkage;
+
   // Due to the limitation of binder as of 2021/09/28, the duplicate weak
   // symbols in the same csect won't be discarded. When there are duplicate weak
   // symbols, we can NOT guarantee that the relocations get resolved to the
@@ -916,6 +938,42 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
   MaybeSetComdat(CounterPtr);
   CounterPtr->setLinkage(Linkage);
   PD.RegionCounters = CounterPtr;
+  if (DebugInfoCorrelate) {
+    if (auto *SP = Fn->getSubprogram()) {
+      DIBuilder DB(*M, true, SP->getUnit());
+      Metadata *FunctionNameAnnotation[] = {
+          MDString::get(Ctx, InstrProfCorrelator::FunctionNameAttributeName),
+          MDString::get(Ctx, getPGOFuncNameVarInitializer(NamePtr)),
+      };
+      Metadata *CFGHashAnnotation[] = {
+          MDString::get(Ctx, InstrProfCorrelator::CFGHashAttributeName),
+          ConstantAsMetadata::get(Inc->getHash()),
+      };
+      Metadata *NumCountersAnnotation[] = {
+          MDString::get(Ctx, InstrProfCorrelator::NumCountersAttributeName),
+          ConstantAsMetadata::get(Inc->getNumCounters()),
+      };
+      auto Annotations = DB.getOrCreateArray({
+          MDNode::get(Ctx, FunctionNameAnnotation),
+          MDNode::get(Ctx, CFGHashAnnotation),
+          MDNode::get(Ctx, NumCountersAnnotation),
+      });
+      auto *DICounter = DB.createGlobalVariableExpression(
+          SP, CounterPtr->getName(), /*LinkageName=*/StringRef(), SP->getFile(),
+          /*LineNo=*/0, DB.createUnspecifiedType("Profile Data Type"),
+          CounterPtr->hasLocalLinkage(), /*IsDefined=*/true, /*Expr=*/nullptr,
+          /*Decl=*/nullptr, /*TemplateParams=*/nullptr, /*AlignInBits=*/0,
+          Annotations);
+      CounterPtr->addDebugInfo(DICounter);
+      DB.finalize();
+    } else {
+      std::string Msg = ("Missing debug info for function " + Fn->getName() +
+                         "; required for profile correlation.")
+                            .str();
+      Ctx.diagnose(
+          DiagnosticInfoPGOProfile(M->getName().data(), Msg, DS_Warning));
+    }
+  }
 
   auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
   // Allocate statically the array of pointers to value profile nodes for
@@ -939,6 +997,9 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
         ConstantExpr::getBitCast(ValuesVar, Type::getInt8PtrTy(Ctx));
   }
 
+  if (DebugInfoCorrelate)
+    return PD.RegionCounters;
+
   // Create data variable.
   auto *IntPtrTy = M->getDataLayout().getIntPtrType(M->getContext());
   auto *Int16Ty = Type::getInt16Ty(Ctx);
diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index 4d15b784f486..446e601cd4d7 100644
--- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -307,6 +307,11 @@ static cl::opt<bool>
                   cl::desc("Enable KernelMemorySanitizer instrumentation"),
                   cl::Hidden, cl::init(false));
 
+static cl::opt<bool>
+    ClDisableChecks("msan-disable-checks",
+                    cl::desc("Apply no_sanitize to the whole file"), cl::Hidden,
+                    cl::init(false));
+
 // This is an experiment to enable handling of cases where shadow is a non-zero
 // compile-time constant. For some unexplainable reason they were silently
 // ignored in the instrumentation.
@@ -1095,7 +1100,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
   MemorySanitizerVisitor(Function &F, MemorySanitizer &MS,
                          const TargetLibraryInfo &TLI)
       : F(F), MS(MS), VAHelper(CreateVarArgHelper(F, MS, *this)), TLI(&TLI) {
-    bool SanitizeFunction = F.hasFnAttribute(Attribute::SanitizeMemory);
+    bool SanitizeFunction =
+        F.hasFnAttribute(Attribute::SanitizeMemory) && !ClDisableChecks;
     InsertChecks = SanitizeFunction;
     PropagateShadow = SanitizeFunction;
     PoisonStack = SanitizeFunction && ClPoisonStack;
@@ -1214,7 +1220,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
       Value *Shadow = SI->isAtomic() ? getCleanShadow(Val) : getShadow(Val);
       Value *ShadowPtr, *OriginPtr;
       Type *ShadowTy = Shadow->getType();
-      const Align Alignment = assumeAligned(SI->getAlignment());
+      const Align Alignment = SI->getAlign();
       const Align OriginAlignment = std::max(kMinOriginAlignment, Alignment);
       std::tie(ShadowPtr, OriginPtr) =
           getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ true);
@@ -3887,8 +3893,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
           &I, IRB, IRB.getInt8Ty(), Align(1), /*isStore*/ true);
 
       Value *PoisonValue = IRB.getInt8(PoisonStack ? ClPoisonStackPattern : 0);
-      IRB.CreateMemSet(ShadowBase, PoisonValue, Len,
-                       MaybeAlign(I.getAlignment()));
+      IRB.CreateMemSet(ShadowBase, PoisonValue, Len, I.getAlign());
     }
 
     if (PoisonStack && MS.TrackOrigins) {
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index af5946325bbb..b6ba1fc2132c 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -273,14 +273,14 @@ static cl::opt<bool> PGOVerifyBFI(
              "internal option -pass-remakrs-analysis=pgo."));
 
 static cl::opt<unsigned> PGOVerifyBFIRatio(
-    "pgo-verify-bfi-ratio", cl::init(5), cl::Hidden,
-    cl::desc("Set the threshold for pgo-verify-big -- only print out "
+    "pgo-verify-bfi-ratio", cl::init(2), cl::Hidden,
+    cl::desc("Set the threshold for pgo-verify-bfi:  only print out "
              "mismatched BFI if the difference percentage is greater than "
              "this value (in percentage)."));
 
 static cl::opt<unsigned> PGOVerifyBFICutoff(
-    "pgo-verify-bfi-cutoff", cl::init(1), cl::Hidden,
-    cl::desc("Set the threshold for pgo-verify-bfi -- skip the counts whose "
+    "pgo-verify-bfi-cutoff", cl::init(5), cl::Hidden,
+    cl::desc("Set the threshold for pgo-verify-bfi: skip the counts whose "
              "profile count value is below."));
 
 namespace llvm {
@@ -291,6 +291,8 @@ extern cl::opt<PGOViewCountsType> PGOViewCounts;
 // Command line option to specify the name of the function for CFG dump
 // Defined in Analysis/BlockFrequencyInfo.cpp:  -view-bfi-func-name=
 extern cl::opt<std::string> ViewBlockFreqFuncName;
+
+extern cl::opt<bool> DebugInfoCorrelate;
 } // namespace llvm
 
 static cl::opt<bool>
@@ -467,8 +469,9 @@ private:
     createProfileFileNameVar(M, InstrProfileOutput);
     // The variable in a comdat may be discarded by LTO. Ensure the
     // declaration will be retained.
-    appendToCompilerUsed(
-        M, createIRLevelProfileFlagVar(M, /*IsCS=*/true, PGOInstrumentEntry));
+    appendToCompilerUsed(M, createIRLevelProfileFlagVar(M, /*IsCS=*/true,
+                                                        PGOInstrumentEntry,
+                                                        DebugInfoCorrelate));
     return false;
   }
   std::string InstrProfileOutput;
@@ -1616,7 +1619,8 @@ static bool InstrumentAllFunctions(
   // For the context-sensitve instrumentation, we should have a separated pass
   // (before LTO/ThinLTO linking) to create these variables.
   if (!IsCS)
-    createIRLevelProfileFlagVar(M, /*IsCS=*/false, PGOInstrumentEntry);
+    createIRLevelProfileFlagVar(M, /*IsCS=*/false, PGOInstrumentEntry,
+                                DebugInfoCorrelate);
   std::unordered_multimap<Comdat *, GlobalValue *> ComdatMembers;
   collectComdatMembers(M, ComdatMembers);
 
@@ -1638,8 +1642,9 @@ PGOInstrumentationGenCreateVar::run(Module &M, ModuleAnalysisManager &AM) {
   createProfileFileNameVar(M, CSInstrName);
   // The variable in a comdat may be discarded by LTO. Ensure the declaration
   // will be retained.
-  appendToCompilerUsed(
-      M, createIRLevelProfileFlagVar(M, /*IsCS=*/true, PGOInstrumentEntry));
+  appendToCompilerUsed(M, createIRLevelProfileFlagVar(M, /*IsCS=*/true,
+                                                      PGOInstrumentEntry,
+                                                      DebugInfoCorrelate));
   return PreservedAnalyses::all();
 }
 
@@ -1774,7 +1779,7 @@ static void verifyFuncBFI(PGOUseFunc &Func, LoopInfo &LI,
       uint64_t Diff = (BFICountValue >= CountValue)
                           ? BFICountValue - CountValue
                           : CountValue - BFICountValue;
-      if (Diff < CountValue / 100 * PGOVerifyBFIRatio)
+      if (Diff <= CountValue / 100 * PGOVerifyBFIRatio)
         continue;
     }
     BBMisMatchNum++;
diff --git a/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp b/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp
index 27f54f8026e1..37a7053d778e 100644
--- a/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp
+++ b/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp
@@ -271,8 +271,7 @@ static void findBestInsertionSet(DominatorTree &DT, BlockFrequencyInfo &BFI,
   // subtree of BB (subtree not including the BB itself).
   DenseMap<BasicBlock *, InsertPtsCostPair> InsertPtsMap;
   InsertPtsMap.reserve(Orders.size() + 1);
-  for (auto RIt = Orders.rbegin(); RIt != Orders.rend(); RIt++) {
-    BasicBlock *Node = *RIt;
+  for (BasicBlock *Node : llvm::reverse(Orders)) {
     bool NodeInBBs = BBs.count(Node);
     auto &InsertPts = InsertPtsMap[Node].first;
     BlockFrequency &InsertPtsFreq = InsertPtsMap[Node].second;
diff --git a/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp b/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp
index 8c4523206070..dda1a2f08076 100644
--- a/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp
+++ b/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp
@@ -588,7 +588,7 @@ struct AllSwitchPaths {
         PrevBB = BB;
       }
 
-      if (TPath.isExitValueSet())
+      if (TPath.isExitValueSet() && isSupported(TPath))
         TPaths.push_back(TPath);
     }
   }
@@ -683,6 +683,62 @@ private:
     return Res;
   }
 
+  /// The determinator BB should precede the switch-defining BB.
+  ///
+  /// Otherwise, it is possible that the state defined in the determinator block
+  /// defines the state for the next iteration of the loop, rather than for the
+  /// current one.
+  ///
+  /// Currently supported paths:
+  /// \code
+  /// < switch bb1 determ def > [ 42, determ ]
+  /// < switch_and_def bb1 determ > [ 42, determ ]
+  /// < switch_and_def_and_determ bb1 > [ 42, switch_and_def_and_determ ]
+  /// \endcode
+  ///
+  /// Unsupported paths:
+  /// \code
+  /// < switch bb1 def determ > [ 43, determ ]
+  /// < switch_and_determ bb1 def > [ 43, switch_and_determ ]
+  /// \endcode
+  bool isSupported(const ThreadingPath &TPath) {
+    Instruction *SwitchCondI = dyn_cast<Instruction>(Switch->getCondition());
+    assert(SwitchCondI);
+    if (!SwitchCondI)
+      return false;
+
+    const BasicBlock *SwitchCondDefBB = SwitchCondI->getParent();
+    const BasicBlock *SwitchCondUseBB = Switch->getParent();
+    const BasicBlock *DeterminatorBB = TPath.getDeterminatorBB();
+
+    assert(
+        SwitchCondUseBB == TPath.getPath().front() &&
+        "The first BB in a threading path should have the switch instruction");
+    if (SwitchCondUseBB != TPath.getPath().front())
+      return false;
+
+    // Make DeterminatorBB the first element in Path.
+    PathType Path = TPath.getPath();
+    auto ItDet = std::find(Path.begin(), Path.end(), DeterminatorBB);
+    std::rotate(Path.begin(), ItDet, Path.end());
+
+    bool IsDetBBSeen = false;
+    bool IsDefBBSeen = false;
+    bool IsUseBBSeen = false;
+    for (BasicBlock *BB : Path) {
+      if (BB == DeterminatorBB)
+        IsDetBBSeen = true;
+      if (BB == SwitchCondDefBB)
+        IsDefBBSeen = true;
+      if (BB == SwitchCondUseBB)
+        IsUseBBSeen = true;
+      if (IsDetBBSeen && IsUseBBSeen && !IsDefBBSeen)
+        return false;
+    }
+
+    return true;
+  }
+
   SwitchInst *Switch;
   BasicBlock *SwitchBlock;
   OptimizationRemarkEmitter *ORE;
diff --git a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
index e0d3a6accadd..eadbb4293539 100644
--- a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
@@ -175,44 +175,6 @@ static cl::opt<bool>
 using OverlapIntervalsTy = std::map<int64_t, int64_t>;
 using InstOverlapIntervalsTy = DenseMap<Instruction *, OverlapIntervalsTy>;
 
-/// If the value of this instruction and the memory it writes to is unused, may
-/// we delete this instruction?
-static bool isRemovable(Instruction *I) {
-  // Don't remove volatile/atomic stores.
-  if (StoreInst *SI = dyn_cast<StoreInst>(I))
-    return SI->isUnordered();
-
-  if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
-    switch (II->getIntrinsicID()) {
-    default: llvm_unreachable("Does not have LocForWrite");
-    case Intrinsic::lifetime_end:
-      // Never remove dead lifetime_end's, e.g. because it is followed by a
-      // free.
-      return false;
-    case Intrinsic::init_trampoline:
-      // Always safe to remove init_trampoline.
-      return true;
-    case Intrinsic::memset:
-    case Intrinsic::memmove:
-    case Intrinsic::memcpy:
-    case Intrinsic::memcpy_inline:
-      // Don't remove volatile memory intrinsics.
-      return !cast<MemIntrinsic>(II)->isVolatile();
-    case Intrinsic::memcpy_element_unordered_atomic:
-    case Intrinsic::memmove_element_unordered_atomic:
-    case Intrinsic::memset_element_unordered_atomic:
-    case Intrinsic::masked_store:
-      return true;
-    }
-  }
-
-  // note: only get here for calls with analyzable writes - i.e. libcalls
-  if (auto *CB = dyn_cast<CallBase>(I))
-    return CB->use_empty();
-
-  return false;
-}
-
 /// Returns true if the end of this instruction can be safely shortened in
 /// length.
 static bool isShortenableAtTheEnd(Instruction *I) {
@@ -835,7 +797,7 @@ struct DSEState {
 
         auto *MD = dyn_cast_or_null<MemoryDef>(MA);
         if (MD && MemDefs.size() < MemorySSADefsPerBlockLimit &&
-            (getLocForWriteEx(&I) || isMemTerminatorInst(&I)))
+            (getLocForWrite(&I) || isMemTerminatorInst(&I)))
           MemDefs.push_back(MD);
       }
     }
@@ -1022,48 +984,39 @@ struct DSEState {
     return I.first->second;
   }
 
-  Optional<MemoryLocation> getLocForWriteEx(Instruction *I) const {
+  Optional<MemoryLocation> getLocForWrite(Instruction *I) const {
     if (!I->mayWriteToMemory())
       return None;
 
-    if (auto *MTI = dyn_cast<AnyMemIntrinsic>(I))
-      return {MemoryLocation::getForDest(MTI)};
+    if (auto *CB = dyn_cast<CallBase>(I))
+      return MemoryLocation::getForDest(CB, TLI);
+
+    return MemoryLocation::getOrNone(I);
+  }
+
+  /// Assuming this instruction has a dead analyzable write, can we delete
+  /// this instruction?
+  bool isRemovable(Instruction *I) {
+    assert(getLocForWrite(I) && "Must have analyzable write");
+
+    // Don't remove volatile/atomic stores.
+    if (StoreInst *SI = dyn_cast<StoreInst>(I))
+      return SI->isUnordered();
 
     if (auto *CB = dyn_cast<CallBase>(I)) {
-      // If the functions may write to memory we do not know about, bail out.
-      if (!CB->onlyAccessesArgMemory() &&
-          !CB->onlyAccessesInaccessibleMemOrArgMem())
-        return None;
+      // Don't remove volatile memory intrinsics.
+      if (auto *MI = dyn_cast<MemIntrinsic>(CB))
+        return !MI->isVolatile();
 
-      LibFunc LF;
-      if (TLI.getLibFunc(*CB, LF) && TLI.has(LF)) {
-        switch (LF) {
-        case LibFunc_strncpy:
-          if (const auto *Len = dyn_cast<ConstantInt>(CB->getArgOperand(2)))
-            return MemoryLocation(CB->getArgOperand(0),
-                                  LocationSize::precise(Len->getZExtValue()),
-                                  CB->getAAMetadata());
-          LLVM_FALLTHROUGH;
-        case LibFunc_strcpy:
-        case LibFunc_strcat:
-        case LibFunc_strncat:
-          return {MemoryLocation::getAfter(CB->getArgOperand(0))};
-        default:
-          break;
-        }
-      }
-      switch (CB->getIntrinsicID()) {
-      case Intrinsic::init_trampoline:
-        return {MemoryLocation::getAfter(CB->getArgOperand(0))};
-      case Intrinsic::masked_store:
-        return {MemoryLocation::getForArgument(CB, 1, TLI)};
-      default:
-        break;
-      }
-      return None;
+      // Never remove dead lifetime intrinsics, e.g. because they are followed
+      // by a free.
+      if (CB->isLifetimeStartOrEnd())
+        return false;
+
+      return CB->use_empty() && CB->willReturn() && CB->doesNotThrow();
     }
 
-    return MemoryLocation::getOrNone(I);
+    return false;
   }
 
   /// Returns true if \p UseInst completely overwrites \p DefLoc
@@ -1081,7 +1034,7 @@ struct DSEState {
         return false;
 
     int64_t InstWriteOffset, DepWriteOffset;
-    if (auto CC = getLocForWriteEx(UseInst))
+    if (auto CC = getLocForWrite(UseInst))
       return isOverwrite(UseInst, DefInst, *CC, DefLoc, InstWriteOffset,
                          DepWriteOffset) == OW_Complete;
     return false;
@@ -1093,7 +1046,7 @@ struct DSEState {
                       << *Def->getMemoryInst()
                       << ") is at the end the function \n");
 
-    auto MaybeLoc = getLocForWriteEx(Def->getMemoryInst());
+    auto MaybeLoc = getLocForWrite(Def->getMemoryInst());
     if (!MaybeLoc) {
       LLVM_DEBUG(dbgs() << "  ... could not get location for write.\n");
       return false;
@@ -1237,30 +1190,14 @@ struct DSEState {
   /// loop. In particular, this guarantees that it only references a single
   /// MemoryLocation during execution of the containing function.
   bool isGuaranteedLoopInvariant(const Value *Ptr) {
-    auto IsGuaranteedLoopInvariantBase = [this](const Value *Ptr) {
-      Ptr = Ptr->stripPointerCasts();
-      if (auto *I = dyn_cast<Instruction>(Ptr)) {
-        if (isa<AllocaInst>(Ptr))
-          return true;
-
-        if (isAllocLikeFn(I, &TLI))
-          return true;
-
-        return false;
-      }
-      return true;
-    };
-
     Ptr = Ptr->stripPointerCasts();
-    if (auto *I = dyn_cast<Instruction>(Ptr)) {
-      if (I->getParent()->isEntryBlock())
-        return true;
-    }
-    if (auto *GEP = dyn_cast<GEPOperator>(Ptr)) {
-      return IsGuaranteedLoopInvariantBase(GEP->getPointerOperand()) &&
-             GEP->hasAllConstantIndices();
-    }
-    return IsGuaranteedLoopInvariantBase(Ptr);
+    if (auto *GEP = dyn_cast<GEPOperator>(Ptr))
+      if (GEP->hasAllConstantIndices())
+        Ptr = GEP->getPointerOperand()->stripPointerCasts();
+
+    if (auto *I = dyn_cast<Instruction>(Ptr))
+      return I->getParent()->isEntryBlock();
+    return true;
   }
 
   // Find a MemoryDef writing to \p KillingLoc and dominating \p StartAccess,
@@ -1372,7 +1309,7 @@ struct DSEState {
 
       // If Current does not have an analyzable write location or is not
       // removable, skip it.
-      CurrentLoc = getLocForWriteEx(CurrentI);
+      CurrentLoc = getLocForWrite(CurrentI);
       if (!CurrentLoc || !isRemovable(CurrentI)) {
         CanOptimize = false;
         continue;
@@ -1729,14 +1666,13 @@ struct DSEState {
     LLVM_DEBUG(
         dbgs()
         << "Trying to eliminate MemoryDefs at the end of the function\n");
-    for (int I = MemDefs.size() - 1; I >= 0; I--) {
-      MemoryDef *Def = MemDefs[I];
-      if (SkipStores.contains(Def) || !isRemovable(Def->getMemoryInst()))
+    for (MemoryDef *Def : llvm::reverse(MemDefs)) {
+      if (SkipStores.contains(Def))
         continue;
 
       Instruction *DefI = Def->getMemoryInst();
-      auto DefLoc = getLocForWriteEx(DefI);
-      if (!DefLoc)
+      auto DefLoc = getLocForWrite(DefI);
+      if (!DefLoc || !isRemovable(DefI))
         continue;
 
       // NOTE: Currently eliminating writes at the end of a function is limited
@@ -1763,13 +1699,19 @@ struct DSEState {
   /// \returns true if \p Def is a no-op store, either because it
   /// directly stores back a loaded value or stores zero to a calloced object.
   bool storeIsNoop(MemoryDef *Def, const Value *DefUO) {
-    StoreInst *Store = dyn_cast<StoreInst>(Def->getMemoryInst());
-    MemSetInst *MemSet = dyn_cast<MemSetInst>(Def->getMemoryInst());
+    Instruction *DefI = Def->getMemoryInst();
+    StoreInst *Store = dyn_cast<StoreInst>(DefI);
+    MemSetInst *MemSet = dyn_cast<MemSetInst>(DefI);
     Constant *StoredConstant = nullptr;
     if (Store)
       StoredConstant = dyn_cast<Constant>(Store->getOperand(0));
-    if (MemSet)
+    else if (MemSet)
       StoredConstant = dyn_cast<Constant>(MemSet->getValue());
+    else
+      return false;
+
+    if (!isRemovable(DefI))
+      return false;
 
     if (StoredConstant && StoredConstant->isNullValue()) {
       auto *DefUOInst = dyn_cast<Instruction>(DefUO);
@@ -1902,7 +1844,7 @@ struct DSEState {
     bool Changed = false;
     for (auto OI : IOL) {
       Instruction *DeadI = OI.first;
-      MemoryLocation Loc = *getLocForWriteEx(DeadI);
+      MemoryLocation Loc = *getLocForWrite(DeadI);
       assert(isRemovable(DeadI) && "Expect only removable instruction");
 
       const Value *Ptr = Loc.Ptr->stripPointerCasts();
@@ -1925,9 +1867,14 @@ struct DSEState {
     LLVM_DEBUG(dbgs() << "Trying to eliminate MemoryDefs that write the "
                          "already existing value\n");
     for (auto *Def : MemDefs) {
-      if (SkipStores.contains(Def) || MSSA.isLiveOnEntryDef(Def) ||
-          !isRemovable(Def->getMemoryInst()))
+      if (SkipStores.contains(Def) || MSSA.isLiveOnEntryDef(Def))
         continue;
+
+      Instruction *DefInst = Def->getMemoryInst();
+      auto MaybeDefLoc = getLocForWrite(DefInst);
+      if (!MaybeDefLoc || !isRemovable(DefInst))
+        continue;
+
       MemoryDef *UpperDef;
       // To conserve compile-time, we avoid walking to the next clobbering def.
       // Instead, we just try to get the optimized access, if it exists. DSE
@@ -1939,17 +1886,14 @@ struct DSEState {
       if (!UpperDef || MSSA.isLiveOnEntryDef(UpperDef))
         continue;
 
-      Instruction *DefInst = Def->getMemoryInst();
       Instruction *UpperInst = UpperDef->getMemoryInst();
-      auto IsRedundantStore = [this, DefInst,
-                               UpperInst](MemoryLocation UpperLoc) {
+      auto IsRedundantStore = [&]() {
         if (DefInst->isIdenticalTo(UpperInst))
           return true;
         if (auto *MemSetI = dyn_cast<MemSetInst>(UpperInst)) {
           if (auto *SI = dyn_cast<StoreInst>(DefInst)) {
-            auto MaybeDefLoc = getLocForWriteEx(DefInst);
-            if (!MaybeDefLoc)
-              return false;
+            // MemSetInst must have a write location.
+            MemoryLocation UpperLoc = *getLocForWrite(UpperInst);
             int64_t InstWriteOffset = 0;
             int64_t DepWriteOffset = 0;
             auto OR = isOverwrite(UpperInst, DefInst, UpperLoc, *MaybeDefLoc,
@@ -1962,9 +1906,7 @@ struct DSEState {
         return false;
       };
 
-      auto MaybeUpperLoc = getLocForWriteEx(UpperInst);
-      if (!MaybeUpperLoc || !IsRedundantStore(*MaybeUpperLoc) ||
-          isReadClobber(*MaybeUpperLoc, DefInst))
+      if (!IsRedundantStore() || isReadClobber(*MaybeDefLoc, DefInst))
         continue;
       LLVM_DEBUG(dbgs() << "DSE: Remove No-Op Store:\n  DEAD: " << *DefInst
                         << '\n');
@@ -1995,7 +1937,7 @@ static bool eliminateDeadStores(Function &F, AliasAnalysis &AA, MemorySSA &MSSA,
       MaybeKillingLoc = State.getLocForTerminator(KillingI).map(
           [](const std::pair<MemoryLocation, bool> &P) { return P.first; });
     else
-      MaybeKillingLoc = State.getLocForWriteEx(KillingI);
+      MaybeKillingLoc = State.getLocForWrite(KillingI);
 
     if (!MaybeKillingLoc) {
       LLVM_DEBUG(dbgs() << "Failed to find analyzable write location for "
@@ -2059,7 +2001,7 @@ static bool eliminateDeadStores(Function &F, AliasAnalysis &AA, MemorySSA &MSSA,
       if (!DebugCounter::shouldExecute(MemorySSACounter))
         continue;
 
-      MemoryLocation DeadLoc = *State.getLocForWriteEx(DeadI);
+      MemoryLocation DeadLoc = *State.getLocForWrite(DeadI);
 
       if (IsMemTerm) {
         const Value *DeadUndObj = getUnderlyingObject(DeadLoc.Ptr);
@@ -2124,8 +2066,7 @@ static bool eliminateDeadStores(Function &F, AliasAnalysis &AA, MemorySSA &MSSA,
     }
 
     // Check if the store is a no-op.
-    if (!Shortend && isRemovable(KillingI) &&
-        State.storeIsNoop(KillingDef, KillingUndObj)) {
+    if (!Shortend && State.storeIsNoop(KillingDef, KillingUndObj)) {
       LLVM_DEBUG(dbgs() << "DSE: Remove No-Op Store:\n  DEAD: " << *KillingI
                         << '\n');
       State.deleteDeadInstruction(KillingI);
diff --git a/llvm/lib/Transforms/Scalar/EarlyCSE.cpp b/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
index 90f71f7729a7..a24997dd3fd4 100644
--- a/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
+++ b/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
@@ -1366,8 +1366,16 @@ bool EarlyCSE::processNode(DomTreeNode *Node) {
           LLVM_DEBUG(dbgs() << "Skipping due to debug counter\n");
           continue;
         }
-        if (auto *I = dyn_cast<Instruction>(V))
-          I->andIRFlags(&Inst);
+        if (auto *I = dyn_cast<Instruction>(V)) {
+          // If I being poison triggers UB, there is no need to drop those
+          // flags. Otherwise, only retain flags present on both I and Inst.
+          // TODO: Currently some fast-math flags are not treated as
+          // poison-generating even though they should. Until this is fixed,
+          // always retain flags present on both I and Inst for floating point
+          // instructions.
+          if (isa<FPMathOperator>(I) || (I->hasPoisonGeneratingFlags() && !programUndefinedIfPoison(I)))
+            I->andIRFlags(&Inst);
+        }
         Inst.replaceAllUsesWith(V);
         salvageKnowledge(&Inst, &AC);
         removeMSSA(Inst);
diff --git a/llvm/lib/Transforms/Scalar/FlattenCFGPass.cpp b/llvm/lib/Transforms/Scalar/FlattenCFGPass.cpp
index e54a270fb276..44017b555769 100644
--- a/llvm/lib/Transforms/Scalar/FlattenCFGPass.cpp
+++ b/llvm/lib/Transforms/Scalar/FlattenCFGPass.cpp
@@ -13,10 +13,12 @@
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/IR/CFG.h"
 #include "llvm/IR/InstrTypes.h"
+#include "llvm/IR/PassManager.h"
 #include "llvm/IR/ValueHandle.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Pass.h"
 #include "llvm/Transforms/Scalar.h"
+#include "llvm/Transforms/Scalar/FlattenCFG.h"
 #include "llvm/Transforms/Utils/Local.h"
 
 using namespace llvm;
@@ -24,11 +26,11 @@ using namespace llvm;
 #define DEBUG_TYPE "flattencfg"
 
 namespace {
-struct FlattenCFGPass : public FunctionPass {
+struct FlattenCFGLegacyPass : public FunctionPass {
   static char ID; // Pass identification, replacement for typeid
 public:
-  FlattenCFGPass() : FunctionPass(ID) {
-    initializeFlattenCFGPassPass(*PassRegistry::getPassRegistry());
+  FlattenCFGLegacyPass() : FunctionPass(ID) {
+    initializeFlattenCFGLegacyPassPass(*PassRegistry::getPassRegistry());
   }
   bool runOnFunction(Function &F) override;
 
@@ -39,21 +41,10 @@ public:
 private:
   AliasAnalysis *AA;
 };
-}
-
-char FlattenCFGPass::ID = 0;
-INITIALIZE_PASS_BEGIN(FlattenCFGPass, "flattencfg", "Flatten the CFG", false,
-                      false)
-INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
-INITIALIZE_PASS_END(FlattenCFGPass, "flattencfg", "Flatten the CFG", false,
-                    false)
-
-// Public interface to the FlattenCFG pass
-FunctionPass *llvm::createFlattenCFGPass() { return new FlattenCFGPass(); }
 
 /// iterativelyFlattenCFG - Call FlattenCFG on all the blocks in the function,
 /// iterating until no more changes are made.
-static bool iterativelyFlattenCFG(Function &F, AliasAnalysis *AA) {
+bool iterativelyFlattenCFG(Function &F, AliasAnalysis *AA) {
   bool Changed = false;
   bool LocalChange = true;
 
@@ -78,8 +69,22 @@ static bool iterativelyFlattenCFG(Function &F, AliasAnalysis *AA) {
   }
   return Changed;
 }
+} // namespace
 
-bool FlattenCFGPass::runOnFunction(Function &F) {
+char FlattenCFGLegacyPass::ID = 0;
+
+INITIALIZE_PASS_BEGIN(FlattenCFGLegacyPass, "flattencfg", "Flatten the CFG",
+                      false, false)
+INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
+INITIALIZE_PASS_END(FlattenCFGLegacyPass, "flattencfg", "Flatten the CFG",
+                    false, false)
+
+// Public interface to the FlattenCFG pass
+FunctionPass *llvm::createFlattenCFGPass() {
+  return new FlattenCFGLegacyPass();
+}
+
+bool FlattenCFGLegacyPass::runOnFunction(Function &F) {
   AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
   bool EverChanged = false;
   // iterativelyFlattenCFG can make some blocks dead.
@@ -89,3 +94,15 @@ bool FlattenCFGPass::runOnFunction(Function &F) {
   }
   return EverChanged;
 }
+
+PreservedAnalyses FlattenCFGPass::run(Function &F,
+                                      FunctionAnalysisManager &AM) {
+  bool EverChanged = false;
+  AliasAnalysis *AA = &AM.getResult<AAManager>(F);
+  // iterativelyFlattenCFG can make some blocks dead.
+  while (iterativelyFlattenCFG(F, AA)) {
+    removeUnreachableBlocks(F);
+    EverChanged = true;
+  }
+  return EverChanged ? PreservedAnalyses::none() : PreservedAnalyses::all();
+}
diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp
index 6f97f3e93123..bc792ca3d8da 100644
--- a/llvm/lib/Transforms/Scalar/LICM.cpp
+++ b/llvm/lib/Transforms/Scalar/LICM.cpp
@@ -107,11 +107,6 @@ static cl::opt<bool> ControlFlowHoisting(
     "licm-control-flow-hoisting", cl::Hidden, cl::init(false),
     cl::desc("Enable control flow (and PHI) hoisting in LICM"));
 
-static cl::opt<unsigned> HoistSinkColdnessThreshold(
-    "licm-coldness-threshold", cl::Hidden, cl::init(4),
-    cl::desc("Relative coldness Threshold of hoisting/sinking destination "
-             "block for LICM to be considered beneficial"));
-
 static cl::opt<uint32_t> MaxNumUsesTraversed(
     "licm-max-num-uses-traversed", cl::Hidden, cl::init(8),
     cl::desc("Max num uses visited for identifying load "
@@ -819,35 +814,6 @@ public:
 };
 } // namespace
 
-// Hoisting/sinking instruction out of a loop isn't always beneficial. It's only
-// only worthwhile if the destination block is actually colder than current
-// block.
-static bool worthSinkOrHoistInst(Instruction &I, BasicBlock *DstBlock,
-                                 OptimizationRemarkEmitter *ORE,
-                                 BlockFrequencyInfo *BFI) {
-  // Check block frequency only when runtime profile is available
-  // to avoid pathological cases. With static profile, lean towards
-  // hosting because it helps canonicalize the loop for vectorizer.
-  if (!DstBlock->getParent()->hasProfileData())
-    return true;
-
-  if (!HoistSinkColdnessThreshold || !BFI)
-    return true;
-
-  BasicBlock *SrcBlock = I.getParent();
-  if (BFI->getBlockFreq(DstBlock).getFrequency() / HoistSinkColdnessThreshold >
-      BFI->getBlockFreq(SrcBlock).getFrequency()) {
-    ORE->emit([&]() {
-      return OptimizationRemarkMissed(DEBUG_TYPE, "SinkHoistInst", &I)
-             << "failed to sink or hoist instruction because containing block "
-                "has lower frequency than destination block";
-    });
-    return false;
-  }
-
-  return true;
-}
-
 /// Walk the specified region of the CFG (defined by all blocks dominated by
 /// the specified block, and that are in the current loop) in depth first
 /// order w.r.t the DominatorTree.  This allows us to visit definitions before
@@ -909,7 +875,6 @@ bool llvm::hoistRegion(DomTreeNode *N, AAResults *AA, LoopInfo *LI,
       if (CurLoop->hasLoopInvariantOperands(&I) &&
           canSinkOrHoistInst(I, AA, DT, CurLoop, /*CurAST*/ nullptr, MSSAU,
                              true, &Flags, ORE) &&
-          worthSinkOrHoistInst(I, CurLoop->getLoopPreheader(), ORE, BFI) &&
           isSafeToExecuteUnconditionally(
               I, DT, TLI, CurLoop, SafetyInfo, ORE,
               CurLoop->getLoopPreheader()->getTerminator())) {
@@ -1741,7 +1706,6 @@ static bool sink(Instruction &I, LoopInfo *LI, DominatorTree *DT,
   // First check if I is worth sinking for all uses. Sink only when it is worth
   // across all uses.
   SmallSetVector<User*, 8> Users(I.user_begin(), I.user_end());
-  SmallVector<PHINode *, 8> ExitPNs;
   for (auto *UI : Users) {
     auto *User = cast<Instruction>(UI);
 
@@ -1751,14 +1715,6 @@ static bool sink(Instruction &I, LoopInfo *LI, DominatorTree *DT,
     PHINode *PN = cast<PHINode>(User);
     assert(ExitBlockSet.count(PN->getParent()) &&
            "The LCSSA PHI is not in an exit block!");
-    if (!worthSinkOrHoistInst(I, PN->getParent(), ORE, BFI)) {
-      return Changed;
-    }
-
-    ExitPNs.push_back(PN);
-  }
-
-  for (auto *PN : ExitPNs) {
 
     // The PHI must be trivially replaceable.
     Instruction *New = sinkThroughTriviallyReplaceablePHI(
diff --git a/llvm/lib/Transforms/Scalar/LoopDataPrefetch.cpp b/llvm/lib/Transforms/Scalar/LoopDataPrefetch.cpp
index 77d76609c926..57e36e5b9b90 100644
--- a/llvm/lib/Transforms/Scalar/LoopDataPrefetch.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopDataPrefetch.cpp
@@ -224,8 +224,8 @@ bool LoopDataPrefetch::run() {
   bool MadeChange = false;
 
   for (Loop *I : *LI)
-    for (auto L = df_begin(I), LE = df_end(I); L != LE; ++L)
-      MadeChange |= runOnLoop(*L);
+    for (Loop *L : depth_first(I))
+      MadeChange |= runOnLoop(L);
 
   return MadeChange;
 }
diff --git a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
index 42da86a9ecf5..5d00fa56e888 100644
--- a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
@@ -786,9 +786,9 @@ bool LoopIdiomRecognize::processLoopStores(SmallVectorImpl<StoreInst *> &SL,
     Type *IntIdxTy = DL->getIndexType(StorePtr->getType());
     const SCEV *StoreSizeSCEV = SE->getConstant(IntIdxTy, StoreSize);
     if (processLoopStridedStore(StorePtr, StoreSizeSCEV,
-                                MaybeAlign(HeadStore->getAlignment()),
-                                StoredVal, HeadStore, AdjacentStores, StoreEv,
-                                BECount, IsNegStride)) {
+                                MaybeAlign(HeadStore->getAlign()), StoredVal,
+                                HeadStore, AdjacentStores, StoreEv, BECount,
+                                IsNegStride)) {
       TransformedStores.insert(AdjacentStores.begin(), AdjacentStores.end());
       Changed = true;
     }
@@ -967,12 +967,22 @@ bool LoopIdiomRecognize::processLoopMemSet(MemSetInst *MSI,
                       << "\n");
 
     if (PositiveStrideSCEV != MemsetSizeSCEV) {
-      // TODO: folding can be done to the SCEVs
-      // The folding is to fold expressions that is covered by the loop guard
-      // at loop entry. After the folding, compare again and proceed
-      // optimization if equal.
-      LLVM_DEBUG(dbgs() << "  SCEV don't match, abort\n");
-      return false;
+      // If an expression is covered by the loop guard, compare again and
+      // proceed with optimization if equal.
+      const SCEV *FoldedPositiveStride =
+          SE->applyLoopGuards(PositiveStrideSCEV, CurLoop);
+      const SCEV *FoldedMemsetSize =
+          SE->applyLoopGuards(MemsetSizeSCEV, CurLoop);
+
+      LLVM_DEBUG(dbgs() << "  Try to fold SCEV based on loop guard\n"
+                        << "    FoldedMemsetSize: " << *FoldedMemsetSize << "\n"
+                        << "    FoldedPositiveStride: " << *FoldedPositiveStride
+                        << "\n");
+
+      if (FoldedPositiveStride != FoldedMemsetSize) {
+        LLVM_DEBUG(dbgs() << "  SCEV don't match, abort\n");
+        return false;
+      }
     }
   }
 
diff --git a/llvm/lib/Transforms/Scalar/LoopRerollPass.cpp b/llvm/lib/Transforms/Scalar/LoopRerollPass.cpp
index 56d66b93dd69..9d22eceb987f 100644
--- a/llvm/lib/Transforms/Scalar/LoopRerollPass.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopRerollPass.cpp
@@ -1456,16 +1456,12 @@ void LoopReroll::DAGRootTracker::replace(const SCEV *BackedgeTakenCount) {
   }
 
   // Remove instructions associated with non-base iterations.
-  for (BasicBlock::reverse_iterator J = Header->rbegin(), JE = Header->rend();
-       J != JE;) {
-    unsigned I = Uses[&*J].find_first();
+  for (Instruction &Inst : llvm::make_early_inc_range(llvm::reverse(*Header))) {
+    unsigned I = Uses[&Inst].find_first();
     if (I > 0 && I < IL_All) {
-      LLVM_DEBUG(dbgs() << "LRR: removing: " << *J << "\n");
-      J++->eraseFromParent();
-      continue;
+      LLVM_DEBUG(dbgs() << "LRR: removing: " << Inst << "\n");
+      Inst.eraseFromParent();
     }
-
-    ++J;
   }
 
   // Rewrite each BaseInst using SCEV.
diff --git a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
index a9a2266e1196..798af48c2337 100644
--- a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
@@ -6011,7 +6011,7 @@ struct SCEVDbgValueBuilder {
     // See setFinalExpression: prepend our opcodes on the start of any old
     // expression opcodes.
     assert(!DI.hasArgList());
-    llvm::SmallVector<uint64_t, 6> FinalExpr(Expr.begin() + 2, Expr.end());
+    llvm::SmallVector<uint64_t, 6> FinalExpr(llvm::drop_begin(Expr, 2));
     auto *NewExpr =
         DIExpression::prependOpcodes(OldExpr, FinalExpr, /*StackValue*/ true);
     DI.setExpression(NewExpr);
diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
index 39c8b65968aa..893928fb0560 100644
--- a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
@@ -1136,6 +1136,31 @@ static LoopUnrollResult tryToUnrollLoop(
   TransformationMode TM = hasUnrollTransformation(L);
   if (TM & TM_Disable)
     return LoopUnrollResult::Unmodified;
+
+  // If this loop isn't forced to be unrolled, avoid unrolling it when the
+  // parent loop has an explicit unroll-and-jam pragma. This is to prevent
+  // automatic unrolling from interfering with the user requested
+  // transformation.
+  Loop *ParentL = L->getParentLoop();
+  if (ParentL != NULL &&
+      hasUnrollAndJamTransformation(ParentL) == TM_ForcedByUser &&
+      hasUnrollTransformation(L) != TM_ForcedByUser) {
+    LLVM_DEBUG(dbgs() << "Not unrolling loop since parent loop has"
+                      << " llvm.loop.unroll_and_jam.\n");
+    return LoopUnrollResult::Unmodified;
+  }
+
+  // If this loop isn't forced to be unrolled, avoid unrolling it when the
+  // loop has an explicit unroll-and-jam pragma. This is to prevent automatic
+  // unrolling from interfering with the user requested transformation.
+  if (hasUnrollAndJamTransformation(L) == TM_ForcedByUser &&
+      hasUnrollTransformation(L) != TM_ForcedByUser) {
+    LLVM_DEBUG(
+        dbgs()
+        << "  Not unrolling loop since it has llvm.loop.unroll_and_jam.\n");
+    return LoopUnrollResult::Unmodified;
+  }
+
   if (!L->isLoopSimplifyForm()) {
     LLVM_DEBUG(
         dbgs() << "  Not unrolling loop which is not in loop-simplify form.\n");
diff --git a/llvm/lib/Transforms/Scalar/NewGVN.cpp b/llvm/lib/Transforms/Scalar/NewGVN.cpp
index 91215cd19e2b..10a8742940b1 100644
--- a/llvm/lib/Transforms/Scalar/NewGVN.cpp
+++ b/llvm/lib/Transforms/Scalar/NewGVN.cpp
@@ -638,6 +638,7 @@ class NewGVN {
   BitVector TouchedInstructions;
 
   DenseMap<const BasicBlock *, std::pair<unsigned, unsigned>> BlockInstRange;
+  mutable DenseMap<const IntrinsicInst *, const Value *> IntrinsicInstPred;
 
 #ifndef NDEBUG
   // Debugging for how many times each block and instruction got processed.
@@ -794,7 +795,7 @@ private:
                                                  BasicBlock *PHIBlock) const;
   const Expression *performSymbolicAggrValueEvaluation(Instruction *) const;
   ExprResult performSymbolicCmpEvaluation(Instruction *) const;
-  ExprResult performSymbolicPredicateInfoEvaluation(Instruction *) const;
+  ExprResult performSymbolicPredicateInfoEvaluation(IntrinsicInst *) const;
 
   // Congruence finding.
   bool someEquivalentDominates(const Instruction *, const Instruction *) const;
@@ -815,6 +816,8 @@ private:
   // Ranking
   unsigned int getRank(const Value *) const;
   bool shouldSwapOperands(const Value *, const Value *) const;
+  bool shouldSwapOperandsForIntrinsic(const Value *, const Value *,
+                                      const IntrinsicInst *I) const;
 
   // Reachability handling.
   void updateReachableEdge(BasicBlock *, BasicBlock *);
@@ -1552,7 +1555,7 @@ const Expression *NewGVN::performSymbolicLoadEvaluation(Instruction *I) const {
 }
 
 NewGVN::ExprResult
-NewGVN::performSymbolicPredicateInfoEvaluation(Instruction *I) const {
+NewGVN::performSymbolicPredicateInfoEvaluation(IntrinsicInst *I) const {
   auto *PI = PredInfo->getPredicateInfoFor(I);
   if (!PI)
     return ExprResult::none();
@@ -1572,7 +1575,7 @@ NewGVN::performSymbolicPredicateInfoEvaluation(Instruction *I) const {
   Value *AdditionallyUsedValue = CmpOp0;
 
   // Sort the ops.
-  if (shouldSwapOperands(FirstOp, SecondOp)) {
+  if (shouldSwapOperandsForIntrinsic(FirstOp, SecondOp, I)) {
     std::swap(FirstOp, SecondOp);
     Predicate = CmpInst::getSwappedPredicate(Predicate);
     AdditionallyUsedValue = CmpOp1;
@@ -1598,7 +1601,7 @@ NewGVN::ExprResult NewGVN::performSymbolicCallEvaluation(Instruction *I) const {
     // Intrinsics with the returned attribute are copies of arguments.
     if (auto *ReturnedValue = II->getReturnedArgOperand()) {
       if (II->getIntrinsicID() == Intrinsic::ssa_copy)
-        if (auto Res = performSymbolicPredicateInfoEvaluation(I))
+        if (auto Res = performSymbolicPredicateInfoEvaluation(II))
           return Res;
       return ExprResult::some(createVariableOrConstant(ReturnedValue));
     }
@@ -2951,6 +2954,7 @@ void NewGVN::cleanupTables() {
   PredicateToUsers.clear();
   MemoryToUsers.clear();
   RevisitOnReachabilityChange.clear();
+  IntrinsicInstPred.clear();
 }
 
 // Assign local DFS number mapping to instructions, and leave space for Value
@@ -4152,6 +4156,29 @@ bool NewGVN::shouldSwapOperands(const Value *A, const Value *B) const {
   return std::make_pair(getRank(A), A) > std::make_pair(getRank(B), B);
 }
 
+bool NewGVN::shouldSwapOperandsForIntrinsic(const Value *A, const Value *B,
+                                            const IntrinsicInst *I) const {
+  auto LookupResult = IntrinsicInstPred.find(I);
+  if (shouldSwapOperands(A, B)) {
+    if (LookupResult == IntrinsicInstPred.end())
+      IntrinsicInstPred.insert({I, B});
+    else
+      LookupResult->second = B;
+    return true;
+  }
+
+  if (LookupResult != IntrinsicInstPred.end()) {
+    auto *SeenPredicate = LookupResult->second;
+    if (SeenPredicate) {
+      if (SeenPredicate == B)
+        return true;
+      else
+        LookupResult->second = nullptr;
+    }
+  }
+  return false;
+}
+
 namespace {
 
 class NewGVNLegacyPass : public FunctionPass {
diff --git a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
index 2d3490b2d29e..e12eca0ed287 100644
--- a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
+++ b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
@@ -1359,16 +1359,6 @@ static constexpr Attribute::AttrKind FnAttrsToStrip[] =
    Attribute::InaccessibleMemOrArgMemOnly,
    Attribute::NoSync, Attribute::NoFree};
 
-// List of all parameter and return attributes which must be stripped when
-// lowering from the abstract machine model.  Note that we list attributes
-// here which aren't valid as return attributes, that is okay.  There are
-// also some additional attributes with arguments which are handled
-// explicitly and are not in this list.
-static constexpr Attribute::AttrKind ParamAttrsToStrip[] =
-  {Attribute::ReadNone, Attribute::ReadOnly, Attribute::WriteOnly,
-   Attribute::NoAlias, Attribute::NoFree};
-
-
 // Create new attribute set containing only attributes which can be transferred
 // from original call to the safepoint.
 static AttributeList legalizeCallAttributes(LLVMContext &Ctx,
@@ -2650,24 +2640,19 @@ static bool insertParsePoints(Function &F, DominatorTree &DT,
   return !Records.empty();
 }
 
-// Handles both return values and arguments for Functions and calls.
-template <typename AttrHolder>
-static void RemoveNonValidAttrAtIndex(LLVMContext &Ctx, AttrHolder &AH,
-                                      unsigned Index) {
+// List of all parameter and return attributes which must be stripped when
+// lowering from the abstract machine model.  Note that we list attributes
+// here which aren't valid as return attributes, that is okay.
+static AttrBuilder getParamAndReturnAttributesToRemove() {
   AttrBuilder R;
-  AttributeSet AS = AH.getAttributes().getAttributes(Index);
-  if (AS.getDereferenceableBytes())
-    R.addAttribute(Attribute::get(Ctx, Attribute::Dereferenceable,
-                                  AS.getDereferenceableBytes()));
-  if (AS.getDereferenceableOrNullBytes())
-    R.addAttribute(Attribute::get(Ctx, Attribute::DereferenceableOrNull,
-                                  AS.getDereferenceableOrNullBytes()));
-  for (auto Attr : ParamAttrsToStrip)
-    if (AS.hasAttribute(Attr))
-      R.addAttribute(Attr);
-
-  if (!R.empty())
-    AH.setAttributes(AH.getAttributes().removeAttributesAtIndex(Ctx, Index, R));
+  R.addDereferenceableAttr(1);
+  R.addDereferenceableOrNullAttr(1);
+  R.addAttribute(Attribute::ReadNone);
+  R.addAttribute(Attribute::ReadOnly);
+  R.addAttribute(Attribute::WriteOnly);
+  R.addAttribute(Attribute::NoAlias);
+  R.addAttribute(Attribute::NoFree);
+  return R;
 }
 
 static void stripNonValidAttributesFromPrototype(Function &F) {
@@ -2683,13 +2668,13 @@ static void stripNonValidAttributesFromPrototype(Function &F) {
     return;
   }
 
+  AttrBuilder R = getParamAndReturnAttributesToRemove();
   for (Argument &A : F.args())
     if (isa<PointerType>(A.getType()))
-      RemoveNonValidAttrAtIndex(Ctx, F,
-                                A.getArgNo() + AttributeList::FirstArgIndex);
+      F.removeParamAttrs(A.getArgNo(), R);
 
   if (isa<PointerType>(F.getReturnType()))
-    RemoveNonValidAttrAtIndex(Ctx, F, AttributeList::ReturnIndex);
+    F.removeRetAttrs(R);
 
   for (auto Attr : FnAttrsToStrip)
     F.removeFnAttr(Attr);
@@ -2757,13 +2742,13 @@ static void stripNonValidDataFromBody(Function &F) {
 
     stripInvalidMetadataFromInstruction(I);
 
+    AttrBuilder R = getParamAndReturnAttributesToRemove();
     if (auto *Call = dyn_cast<CallBase>(&I)) {
       for (int i = 0, e = Call->arg_size(); i != e; i++)
         if (isa<PointerType>(Call->getArgOperand(i)->getType()))
-          RemoveNonValidAttrAtIndex(Ctx, *Call,
-                                    i + AttributeList::FirstArgIndex);
+          Call->removeParamAttrs(i, R);
       if (isa<PointerType>(Call->getType()))
-        RemoveNonValidAttrAtIndex(Ctx, *Call, AttributeList::ReturnIndex);
+        Call->removeRetAttrs(R);
     }
   }
 
diff --git a/llvm/lib/Transforms/Scalar/SCCP.cpp b/llvm/lib/Transforms/Scalar/SCCP.cpp
index 28e00c873361..ff2f8a25f379 100644
--- a/llvm/lib/Transforms/Scalar/SCCP.cpp
+++ b/llvm/lib/Transforms/Scalar/SCCP.cpp
@@ -101,8 +101,7 @@ static bool tryToReplaceWithConstant(SCCPSolver &Solver, Value *V) {
   Constant *Const = nullptr;
   if (V->getType()->isStructTy()) {
     std::vector<ValueLatticeElement> IVs = Solver.getStructLatticeValueFor(V);
-    if (any_of(IVs,
-               [](const ValueLatticeElement &LV) { return isOverdefined(LV); }))
+    if (llvm::any_of(IVs, isOverdefined))
       return false;
     std::vector<Constant *> ConstVals;
     auto *ST = cast<StructType>(V->getType());
diff --git a/llvm/lib/Transforms/Scalar/Scalar.cpp b/llvm/lib/Transforms/Scalar/Scalar.cpp
index a041af0d70d0..f9650efc051f 100644
--- a/llvm/lib/Transforms/Scalar/Scalar.cpp
+++ b/llvm/lib/Transforms/Scalar/Scalar.cpp
@@ -54,7 +54,7 @@ void llvm::initializeScalarOpts(PassRegistry &Registry) {
   initializeMakeGuardsExplicitLegacyPassPass(Registry);
   initializeGVNHoistLegacyPassPass(Registry);
   initializeGVNSinkLegacyPassPass(Registry);
-  initializeFlattenCFGPassPass(Registry);
+  initializeFlattenCFGLegacyPassPass(Registry);
   initializeIRCELegacyPassPass(Registry);
   initializeIndVarSimplifyLegacyPassPass(Registry);
   initializeInferAddressSpacesPass(Registry);
diff --git a/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp b/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
index ffa2f9adb978..d23925042b0a 100644
--- a/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
+++ b/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
@@ -648,13 +648,13 @@ Value *ConstantOffsetExtractor::applyExts(Value *V) {
   Value *Current = V;
   // ExtInsts is built in the use-def order. Therefore, we apply them to V
   // in the reversed order.
-  for (auto I = ExtInsts.rbegin(), E = ExtInsts.rend(); I != E; ++I) {
+  for (CastInst *I : llvm::reverse(ExtInsts)) {
     if (Constant *C = dyn_cast<Constant>(Current)) {
       // If Current is a constant, apply s/zext using ConstantExpr::getCast.
       // ConstantExpr::getCast emits a ConstantInt if C is a ConstantInt.
-      Current = ConstantExpr::getCast((*I)->getOpcode(), C, (*I)->getType());
+      Current = ConstantExpr::getCast(I->getOpcode(), C, I->getType());
     } else {
-      Instruction *Ext = (*I)->clone();
+      Instruction *Ext = I->clone();
       Ext->setOperand(0, Current);
       Ext->insertBefore(IP);
       Current = Ext;
diff --git a/llvm/lib/Transforms/Utils/CodeLayout.cpp b/llvm/lib/Transforms/Utils/CodeLayout.cpp
new file mode 100644
index 000000000000..dfb9f608eab2
--- /dev/null
+++ b/llvm/lib/Transforms/Utils/CodeLayout.cpp
@@ -0,0 +1,942 @@
+//===- CodeLayout.cpp - Implementation of code layout algorithms ----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// ExtTSP - layout of basic blocks with i-cache optimization.
+//
+// The algorithm tries to find a layout of nodes (basic blocks) of a given CFG
+// optimizing jump locality and thus processor I-cache utilization. This is
+// achieved via increasing the number of fall-through jumps and co-locating
+// frequently executed nodes together. The name follows the underlying
+// optimization problem, Extended-TSP, which is a generalization of classical
+// (maximum) Traveling Salesmen Problem.
+//
+// The algorithm is a greedy heuristic that works with chains (ordered lists)
+// of basic blocks. Initially all chains are isolated basic blocks. On every
+// iteration, we pick a pair of chains whose merging yields the biggest increase
+// in the ExtTSP score, which models how i-cache "friendly" a specific chain is.
+// A pair of chains giving the maximum gain is merged into a new chain. The
+// procedure stops when there is only one chain left, or when merging does not
+// increase ExtTSP. In the latter case, the remaining chains are sorted by
+// density in the decreasing order.
+//
+// An important aspect is the way two chains are merged. Unlike earlier
+// algorithms (e.g., based on the approach of Pettis-Hansen), two
+// chains, X and Y, are first split into three, X1, X2, and Y. Then we
+// consider all possible ways of gluing the three chains (e.g., X1YX2, X1X2Y,
+// X2X1Y, X2YX1, YX1X2, YX2X1) and choose the one producing the largest score.
+// This improves the quality of the final result (the search space is larger)
+// while keeping the implementation sufficiently fast.
+//
+// Reference:
+//   * A. Newell and S. Pupyrev, Improved Basic Block Reordering,
+//     IEEE Transactions on Computers, 2020
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Utils/CodeLayout.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+
+using namespace llvm;
+#define DEBUG_TYPE "code-layout"
+
+// Algorithm-specific constants. The values are tuned for the best performance
+// of large-scale front-end bound binaries.
+static cl::opt<double>
+    ForwardWeight("ext-tsp-forward-weight", cl::Hidden, cl::init(0.1),
+                  cl::desc("The weight of forward jumps for ExtTSP value"));
+
+static cl::opt<double>
+    BackwardWeight("ext-tsp-backward-weight", cl::Hidden, cl::init(0.1),
+                   cl::desc("The weight of backward jumps for ExtTSP value"));
+
+static cl::opt<unsigned> ForwardDistance(
+    "ext-tsp-forward-distance", cl::Hidden, cl::init(1024),
+    cl::desc("The maximum distance (in bytes) of a forward jump for ExtTSP"));
+
+static cl::opt<unsigned> BackwardDistance(
+    "ext-tsp-backward-distance", cl::Hidden, cl::init(640),
+    cl::desc("The maximum distance (in bytes) of a backward jump for ExtTSP"));
+
+// The maximum size of a chain for splitting. Larger values of the threshold
+// may yield better quality at the cost of worsen run-time.
+static cl::opt<unsigned> ChainSplitThreshold(
+    "ext-tsp-chain-split-threshold", cl::Hidden, cl::init(128),
+    cl::desc("The maximum size of a chain to apply splitting"));
+
+// The option enables splitting (large) chains along in-coming and out-going
+// jumps. This typically results in a better quality.
+static cl::opt<bool> EnableChainSplitAlongJumps(
+    "ext-tsp-enable-chain-split-along-jumps", cl::Hidden, cl::init(true),
+    cl::desc("The maximum size of a chain to apply splitting"));
+
+namespace {
+
+// Epsilon for comparison of doubles.
+constexpr double EPS = 1e-8;
+
+// Compute the Ext-TSP score for a jump between a given pair of blocks,
+// using their sizes, (estimated) addresses and the jump execution count.
+double extTSPScore(uint64_t SrcAddr, uint64_t SrcSize, uint64_t DstAddr,
+                   uint64_t Count) {
+  // Fallthrough
+  if (SrcAddr + SrcSize == DstAddr) {
+    // Assume that FallthroughWeight = 1.0 after normalization
+    return static_cast<double>(Count);
+  }
+  // Forward
+  if (SrcAddr + SrcSize < DstAddr) {
+    const auto Dist = DstAddr - (SrcAddr + SrcSize);
+    if (Dist <= ForwardDistance) {
+      double Prob = 1.0 - static_cast<double>(Dist) / ForwardDistance;
+      return ForwardWeight * Prob * Count;
+    }
+    return 0;
+  }
+  // Backward
+  const auto Dist = SrcAddr + SrcSize - DstAddr;
+  if (Dist <= BackwardDistance) {
+    double Prob = 1.0 - static_cast<double>(Dist) / BackwardDistance;
+    return BackwardWeight * Prob * Count;
+  }
+  return 0;
+}
+
+/// A type of merging two chains, X and Y. The former chain is split into
+/// X1 and X2 and then concatenated with Y in the order specified by the type.
+enum class MergeTypeTy : int { X_Y, X1_Y_X2, Y_X2_X1, X2_X1_Y };
+
+/// The gain of merging two chains, that is, the Ext-TSP score of the merge
+/// together with the corresponfiding merge 'type' and 'offset'.
+class MergeGainTy {
+public:
+  explicit MergeGainTy() {}
+  explicit MergeGainTy(double Score, size_t MergeOffset, MergeTypeTy MergeType)
+      : Score(Score), MergeOffset(MergeOffset), MergeType(MergeType) {}
+
+  double score() const { return Score; }
+
+  size_t mergeOffset() const { return MergeOffset; }
+
+  MergeTypeTy mergeType() const { return MergeType; }
+
+  // Returns 'true' iff Other is preferred over this.
+  bool operator<(const MergeGainTy &Other) const {
+    return (Other.Score > EPS && Other.Score > Score + EPS);
+  }
+
+  // Update the current gain if Other is preferred over this.
+  void updateIfLessThan(const MergeGainTy &Other) {
+    if (*this < Other)
+      *this = Other;
+  }
+
+private:
+  double Score{-1.0};
+  size_t MergeOffset{0};
+  MergeTypeTy MergeType{MergeTypeTy::X_Y};
+};
+
+class Block;
+class Jump;
+class Chain;
+class ChainEdge;
+
+/// A node in the graph, typically corresponding to a basic block in CFG.
+class Block {
+public:
+  Block(const Block &) = delete;
+  Block(Block &&) = default;
+  Block &operator=(const Block &) = delete;
+  Block &operator=(Block &&) = default;
+
+  // The original index of the block in CFG.
+  size_t Index{0};
+  // The index of the block in the current chain.
+  size_t CurIndex{0};
+  // Size of the block in the binary.
+  uint64_t Size{0};
+  // Execution count of the block in the profile data.
+  uint64_t ExecutionCount{0};
+  // Current chain of the node.
+  Chain *CurChain{nullptr};
+  // An offset of the block in the current chain.
+  mutable uint64_t EstimatedAddr{0};
+  // Forced successor of the block in CFG.
+  Block *ForcedSucc{nullptr};
+  // Forced predecessor of the block in CFG.
+  Block *ForcedPred{nullptr};
+  // Outgoing jumps from the block.
+  std::vector<Jump *> OutJumps;
+  // Incoming jumps to the block.
+  std::vector<Jump *> InJumps;
+
+public:
+  explicit Block(size_t Index, uint64_t Size_, uint64_t EC)
+      : Index(Index), Size(Size_), ExecutionCount(EC) {}
+  bool isEntry() const { return Index == 0; }
+};
+
+/// An arc in the graph, typically corresponding to a jump between two blocks.
+class Jump {
+public:
+  Jump(const Jump &) = delete;
+  Jump(Jump &&) = default;
+  Jump &operator=(const Jump &) = delete;
+  Jump &operator=(Jump &&) = default;
+
+  // Source block of the jump.
+  Block *Source;
+  // Target block of the jump.
+  Block *Target;
+  // Execution count of the arc in the profile data.
+  uint64_t ExecutionCount{0};
+
+public:
+  explicit Jump(Block *Source, Block *Target, uint64_t ExecutionCount)
+      : Source(Source), Target(Target), ExecutionCount(ExecutionCount) {}
+};
+
+/// A chain (ordered sequence) of blocks.
+class Chain {
+public:
+  Chain(const Chain &) = delete;
+  Chain(Chain &&) = default;
+  Chain &operator=(const Chain &) = delete;
+  Chain &operator=(Chain &&) = default;
+
+  explicit Chain(uint64_t Id, Block *Block)
+      : Id(Id), Score(0), Blocks(1, Block) {}
+
+  uint64_t id() const { return Id; }
+
+  bool isEntry() const { return Blocks[0]->Index == 0; }
+
+  double score() const { return Score; }
+
+  void setScore(double NewScore) { Score = NewScore; }
+
+  const std::vector<Block *> &blocks() const { return Blocks; }
+
+  const std::vector<std::pair<Chain *, ChainEdge *>> &edges() const {
+    return Edges;
+  }
+
+  ChainEdge *getEdge(Chain *Other) const {
+    for (auto It : Edges) {
+      if (It.first == Other)
+        return It.second;
+    }
+    return nullptr;
+  }
+
+  void removeEdge(Chain *Other) {
+    auto It = Edges.begin();
+    while (It != Edges.end()) {
+      if (It->first == Other) {
+        Edges.erase(It);
+        return;
+      }
+      It++;
+    }
+  }
+
+  void addEdge(Chain *Other, ChainEdge *Edge) {
+    Edges.push_back(std::make_pair(Other, Edge));
+  }
+
+  void merge(Chain *Other, const std::vector<Block *> &MergedBlocks) {
+    Blocks = MergedBlocks;
+    // Update the block's chains
+    for (size_t Idx = 0; Idx < Blocks.size(); Idx++) {
+      Blocks[Idx]->CurChain = this;
+      Blocks[Idx]->CurIndex = Idx;
+    }
+  }
+
+  void mergeEdges(Chain *Other);
+
+  void clear() {
+    Blocks.clear();
+    Blocks.shrink_to_fit();
+    Edges.clear();
+    Edges.shrink_to_fit();
+  }
+
+private:
+  // Unique chain identifier.
+  uint64_t Id;
+  // Cached ext-tsp score for the chain.
+  double Score;
+  // Blocks of the chain.
+  std::vector<Block *> Blocks;
+  // Adjacent chains and corresponding edges (lists of jumps).
+  std::vector<std::pair<Chain *, ChainEdge *>> Edges;
+};
+
+/// An edge in CFG representing jumps between two chains.
+/// When blocks are merged into chains, the edges are combined too so that
+/// there is always at most one edge between a pair of chains
+class ChainEdge {
+public:
+  ChainEdge(const ChainEdge &) = delete;
+  ChainEdge(ChainEdge &&) = default;
+  ChainEdge &operator=(const ChainEdge &) = delete;
+  ChainEdge &operator=(ChainEdge &&) = default;
+
+  explicit ChainEdge(Jump *Jump)
+      : SrcChain(Jump->Source->CurChain), DstChain(Jump->Target->CurChain),
+        Jumps(1, Jump) {}
+
+  const std::vector<Jump *> &jumps() const { return Jumps; }
+
+  void changeEndpoint(Chain *From, Chain *To) {
+    if (From == SrcChain)
+      SrcChain = To;
+    if (From == DstChain)
+      DstChain = To;
+  }
+
+  void appendJump(Jump *Jump) { Jumps.push_back(Jump); }
+
+  void moveJumps(ChainEdge *Other) {
+    Jumps.insert(Jumps.end(), Other->Jumps.begin(), Other->Jumps.end());
+    Other->Jumps.clear();
+    Other->Jumps.shrink_to_fit();
+  }
+
+  bool hasCachedMergeGain(Chain *Src, Chain *Dst) const {
+    return Src == SrcChain ? CacheValidForward : CacheValidBackward;
+  }
+
+  MergeGainTy getCachedMergeGain(Chain *Src, Chain *Dst) const {
+    return Src == SrcChain ? CachedGainForward : CachedGainBackward;
+  }
+
+  void setCachedMergeGain(Chain *Src, Chain *Dst, MergeGainTy MergeGain) {
+    if (Src == SrcChain) {
+      CachedGainForward = MergeGain;
+      CacheValidForward = true;
+    } else {
+      CachedGainBackward = MergeGain;
+      CacheValidBackward = true;
+    }
+  }
+
+  void invalidateCache() {
+    CacheValidForward = false;
+    CacheValidBackward = false;
+  }
+
+private:
+  // Source chain.
+  Chain *SrcChain{nullptr};
+  // Destination chain.
+  Chain *DstChain{nullptr};
+  // Original jumps in the binary with correspinding execution counts.
+  std::vector<Jump *> Jumps;
+  // Cached ext-tsp value for merging the pair of chains.
+  // Since the gain of merging (Src, Dst) and (Dst, Src) might be different,
+  // we store both values here.
+  MergeGainTy CachedGainForward;
+  MergeGainTy CachedGainBackward;
+  // Whether the cached value must be recomputed.
+  bool CacheValidForward{false};
+  bool CacheValidBackward{false};
+};
+
+void Chain::mergeEdges(Chain *Other) {
+  assert(this != Other && "cannot merge a chain with itself");
+
+  // Update edges adjacent to chain Other
+  for (auto EdgeIt : Other->Edges) {
+    const auto DstChain = EdgeIt.first;
+    const auto DstEdge = EdgeIt.second;
+    const auto TargetChain = DstChain == Other ? this : DstChain;
+    auto CurEdge = getEdge(TargetChain);
+    if (CurEdge == nullptr) {
+      DstEdge->changeEndpoint(Other, this);
+      this->addEdge(TargetChain, DstEdge);
+      if (DstChain != this && DstChain != Other) {
+        DstChain->addEdge(this, DstEdge);
+      }
+    } else {
+      CurEdge->moveJumps(DstEdge);
+    }
+    // Cleanup leftover edge
+    if (DstChain != Other) {
+      DstChain->removeEdge(Other);
+    }
+  }
+}
+
+using BlockIter = std::vector<Block *>::const_iterator;
+
+/// A wrapper around three chains of blocks; it is used to avoid extra
+/// instantiation of the vectors.
+class MergedChain {
+public:
+  MergedChain(BlockIter Begin1, BlockIter End1, BlockIter Begin2 = BlockIter(),
+              BlockIter End2 = BlockIter(), BlockIter Begin3 = BlockIter(),
+              BlockIter End3 = BlockIter())
+      : Begin1(Begin1), End1(End1), Begin2(Begin2), End2(End2), Begin3(Begin3),
+        End3(End3) {}
+
+  template <typename F> void forEach(const F &Func) const {
+    for (auto It = Begin1; It != End1; It++)
+      Func(*It);
+    for (auto It = Begin2; It != End2; It++)
+      Func(*It);
+    for (auto It = Begin3; It != End3; It++)
+      Func(*It);
+  }
+
+  std::vector<Block *> getBlocks() const {
+    std::vector<Block *> Result;
+    Result.reserve(std::distance(Begin1, End1) + std::distance(Begin2, End2) +
+                   std::distance(Begin3, End3));
+    Result.insert(Result.end(), Begin1, End1);
+    Result.insert(Result.end(), Begin2, End2);
+    Result.insert(Result.end(), Begin3, End3);
+    return Result;
+  }
+
+  const Block *getFirstBlock() const { return *Begin1; }
+
+private:
+  BlockIter Begin1;
+  BlockIter End1;
+  BlockIter Begin2;
+  BlockIter End2;
+  BlockIter Begin3;
+  BlockIter End3;
+};
+
+/// The implementation of the ExtTSP algorithm.
+class ExtTSPImpl {
+  using EdgeT = std::pair<uint64_t, uint64_t>;
+  using EdgeCountMap = DenseMap<EdgeT, uint64_t>;
+
+public:
+  ExtTSPImpl(size_t NumNodes, const std::vector<uint64_t> &NodeSizes,
+             const std::vector<uint64_t> &NodeCounts,
+             const EdgeCountMap &EdgeCounts)
+      : NumNodes(NumNodes) {
+    initialize(NodeSizes, NodeCounts, EdgeCounts);
+  }
+
+  /// Run the algorithm and return an optimized ordering of blocks.
+  void run(std::vector<uint64_t> &Result) {
+    // Pass 1: Merge blocks with their mutually forced successors
+    mergeForcedPairs();
+
+    // Pass 2: Merge pairs of chains while improving the ExtTSP objective
+    mergeChainPairs();
+
+    // Pass 3: Merge cold blocks to reduce code size
+    mergeColdChains();
+
+    // Collect blocks from all chains
+    concatChains(Result);
+  }
+
+private:
+  /// Initialize the algorithm's data structures.
+  void initialize(const std::vector<uint64_t> &NodeSizes,
+                  const std::vector<uint64_t> &NodeCounts,
+                  const EdgeCountMap &EdgeCounts) {
+    // Initialize blocks
+    AllBlocks.reserve(NumNodes);
+    for (uint64_t Node = 0; Node < NumNodes; Node++) {
+      uint64_t Size = std::max<uint64_t>(NodeSizes[Node], 1ULL);
+      uint64_t ExecutionCount = NodeCounts[Node];
+      // The execution count of the entry block is set to at least 1
+      if (Node == 0 && ExecutionCount == 0)
+        ExecutionCount = 1;
+      AllBlocks.emplace_back(Node, Size, ExecutionCount);
+    }
+
+    // Initialize jumps between blocks
+    SuccNodes = std::vector<std::vector<uint64_t>>(NumNodes);
+    PredNodes = std::vector<std::vector<uint64_t>>(NumNodes);
+    AllJumps.reserve(EdgeCounts.size());
+    for (auto It : EdgeCounts) {
+      auto Pred = It.first.first;
+      auto Succ = It.first.second;
+      // Ignore self-edges
+      if (Pred == Succ)
+        continue;
+
+      SuccNodes[Pred].push_back(Succ);
+      PredNodes[Succ].push_back(Pred);
+      auto ExecutionCount = It.second;
+      if (ExecutionCount > 0) {
+        auto &Block = AllBlocks[Pred];
+        auto &SuccBlock = AllBlocks[Succ];
+        AllJumps.emplace_back(&Block, &SuccBlock, ExecutionCount);
+        SuccBlock.InJumps.push_back(&AllJumps.back());
+        Block.OutJumps.push_back(&AllJumps.back());
+      }
+    }
+
+    // Initialize chains
+    AllChains.reserve(NumNodes);
+    HotChains.reserve(NumNodes);
+    for (auto &Block : AllBlocks) {
+      AllChains.emplace_back(Block.Index, &Block);
+      Block.CurChain = &AllChains.back();
+      if (Block.ExecutionCount > 0) {
+        HotChains.push_back(&AllChains.back());
+      }
+    }
+
+    // Initialize chain edges
+    AllEdges.reserve(AllJumps.size());
+    for (auto &Block : AllBlocks) {
+      for (auto &Jump : Block.OutJumps) {
+        const auto SuccBlock = Jump->Target;
+        auto CurEdge = Block.CurChain->getEdge(SuccBlock->CurChain);
+        // this edge is already present in the graph
+        if (CurEdge != nullptr) {
+          assert(SuccBlock->CurChain->getEdge(Block.CurChain) != nullptr);
+          CurEdge->appendJump(Jump);
+          continue;
+        }
+        // this is a new edge
+        AllEdges.emplace_back(Jump);
+        Block.CurChain->addEdge(SuccBlock->CurChain, &AllEdges.back());
+        SuccBlock->CurChain->addEdge(Block.CurChain, &AllEdges.back());
+      }
+    }
+  }
+
+  /// For a pair of blocks, A and B, block B is the forced successor of A,
+  /// if (i) all jumps (based on profile) from A goes to B and (ii) all jumps
+  /// to B are from A. Such blocks should be adjacent in the optimal ordering;
+  /// the method finds and merges such pairs of blocks.
+  void mergeForcedPairs() {
+    // Find fallthroughs based on edge weights
+    for (auto &Block : AllBlocks) {
+      if (SuccNodes[Block.Index].size() == 1 &&
+          PredNodes[SuccNodes[Block.Index][0]].size() == 1 &&
+          SuccNodes[Block.Index][0] != 0) {
+        size_t SuccIndex = SuccNodes[Block.Index][0];
+        Block.ForcedSucc = &AllBlocks[SuccIndex];
+        AllBlocks[SuccIndex].ForcedPred = &Block;
+      }
+    }
+
+    // There might be 'cycles' in the forced dependencies, since profile
+    // data isn't 100% accurate. Typically this is observed in loops, when the
+    // loop edges are the hottest successors for the basic blocks of the loop.
+    // Break the cycles by choosing the block with the smallest index as the
+    // head. This helps to keep the original order of the loops, which likely
+    // have already been rotated in the optimized manner.
+    for (auto &Block : AllBlocks) {
+      if (Block.ForcedSucc == nullptr || Block.ForcedPred == nullptr)
+        continue;
+
+      auto SuccBlock = Block.ForcedSucc;
+      while (SuccBlock != nullptr && SuccBlock != &Block) {
+        SuccBlock = SuccBlock->ForcedSucc;
+      }
+      if (SuccBlock == nullptr)
+        continue;
+      // Break the cycle
+      AllBlocks[Block.ForcedPred->Index].ForcedSucc = nullptr;
+      Block.ForcedPred = nullptr;
+    }
+
+    // Merge blocks with their fallthrough successors
+    for (auto &Block : AllBlocks) {
+      if (Block.ForcedPred == nullptr && Block.ForcedSucc != nullptr) {
+        auto CurBlock = &Block;
+        while (CurBlock->ForcedSucc != nullptr) {
+          const auto NextBlock = CurBlock->ForcedSucc;
+          mergeChains(Block.CurChain, NextBlock->CurChain, 0, MergeTypeTy::X_Y);
+          CurBlock = NextBlock;
+        }
+      }
+    }
+  }
+
+  /// Merge pairs of chains while improving the ExtTSP objective.
+  void mergeChainPairs() {
+    /// Deterministically compare pairs of chains
+    auto compareChainPairs = [](const Chain *A1, const Chain *B1,
+                                const Chain *A2, const Chain *B2) {
+      if (A1 != A2)
+        return A1->id() < A2->id();
+      return B1->id() < B2->id();
+    };
+
+    while (HotChains.size() > 1) {
+      Chain *BestChainPred = nullptr;
+      Chain *BestChainSucc = nullptr;
+      auto BestGain = MergeGainTy();
+      // Iterate over all pairs of chains
+      for (auto ChainPred : HotChains) {
+        // Get candidates for merging with the current chain
+        for (auto EdgeIter : ChainPred->edges()) {
+          auto ChainSucc = EdgeIter.first;
+          auto ChainEdge = EdgeIter.second;
+          // Ignore loop edges
+          if (ChainPred == ChainSucc)
+            continue;
+
+          // Compute the gain of merging the two chains
+          auto CurGain = getBestMergeGain(ChainPred, ChainSucc, ChainEdge);
+          if (CurGain.score() <= EPS)
+            continue;
+
+          if (BestGain < CurGain ||
+              (std::abs(CurGain.score() - BestGain.score()) < EPS &&
+               compareChainPairs(ChainPred, ChainSucc, BestChainPred,
+                                 BestChainSucc))) {
+            BestGain = CurGain;
+            BestChainPred = ChainPred;
+            BestChainSucc = ChainSucc;
+          }
+        }
+      }
+
+      // Stop merging when there is no improvement
+      if (BestGain.score() <= EPS)
+        break;
+
+      // Merge the best pair of chains
+      mergeChains(BestChainPred, BestChainSucc, BestGain.mergeOffset(),
+                  BestGain.mergeType());
+    }
+  }
+
+  /// Merge cold blocks to reduce code size.
+  void mergeColdChains() {
+    for (size_t SrcBB = 0; SrcBB < NumNodes; SrcBB++) {
+      // Iterating over neighbors in the reverse order to make sure original
+      // fallthrough jumps are merged first
+      size_t NumSuccs = SuccNodes[SrcBB].size();
+      for (size_t Idx = 0; Idx < NumSuccs; Idx++) {
+        auto DstBB = SuccNodes[SrcBB][NumSuccs - Idx - 1];
+        auto SrcChain = AllBlocks[SrcBB].CurChain;
+        auto DstChain = AllBlocks[DstBB].CurChain;
+        if (SrcChain != DstChain && !DstChain->isEntry() &&
+            SrcChain->blocks().back()->Index == SrcBB &&
+            DstChain->blocks().front()->Index == DstBB) {
+          mergeChains(SrcChain, DstChain, 0, MergeTypeTy::X_Y);
+        }
+      }
+    }
+  }
+
+  /// Compute the Ext-TSP score for a given block order and a list of jumps.
+  double extTSPScore(const MergedChain &MergedBlocks,
+                     const std::vector<Jump *> &Jumps) const {
+    if (Jumps.empty())
+      return 0.0;
+    uint64_t CurAddr = 0;
+    MergedBlocks.forEach([&](const Block *BB) {
+      BB->EstimatedAddr = CurAddr;
+      CurAddr += BB->Size;
+    });
+
+    double Score = 0;
+    for (auto &Jump : Jumps) {
+      const auto SrcBlock = Jump->Source;
+      const auto DstBlock = Jump->Target;
+      Score += ::extTSPScore(SrcBlock->EstimatedAddr, SrcBlock->Size,
+                             DstBlock->EstimatedAddr, Jump->ExecutionCount);
+    }
+    return Score;
+  }
+
+  /// Compute the gain of merging two chains.
+  ///
+  /// The function considers all possible ways of merging two chains and
+  /// computes the one having the largest increase in ExtTSP objective. The
+  /// result is a pair with the first element being the gain and the second
+  /// element being the corresponding merging type.
+  MergeGainTy getBestMergeGain(Chain *ChainPred, Chain *ChainSucc,
+                               ChainEdge *Edge) const {
+    if (Edge->hasCachedMergeGain(ChainPred, ChainSucc)) {
+      return Edge->getCachedMergeGain(ChainPred, ChainSucc);
+    }
+
+    // Precompute jumps between ChainPred and ChainSucc
+    auto Jumps = Edge->jumps();
+    auto EdgePP = ChainPred->getEdge(ChainPred);
+    if (EdgePP != nullptr) {
+      Jumps.insert(Jumps.end(), EdgePP->jumps().begin(), EdgePP->jumps().end());
+    }
+    assert(!Jumps.empty() && "trying to merge chains w/o jumps");
+
+    // The object holds the best currently chosen gain of merging the two chains
+    MergeGainTy Gain = MergeGainTy();
+
+    /// Given a merge offset and a list of merge types, try to merge two chains
+    /// and update Gain with a better alternative
+    auto tryChainMerging = [&](size_t Offset,
+                               const std::vector<MergeTypeTy> &MergeTypes) {
+      // Skip merging corresponding to concatenation w/o splitting
+      if (Offset == 0 || Offset == ChainPred->blocks().size())
+        return;
+      // Skip merging if it breaks Forced successors
+      auto BB = ChainPred->blocks()[Offset - 1];
+      if (BB->ForcedSucc != nullptr)
+        return;
+      // Apply the merge, compute the corresponding gain, and update the best
+      // value, if the merge is beneficial
+      for (auto &MergeType : MergeTypes) {
+        Gain.updateIfLessThan(
+            computeMergeGain(ChainPred, ChainSucc, Jumps, Offset, MergeType));
+      }
+    };
+
+    // Try to concatenate two chains w/o splitting
+    Gain.updateIfLessThan(
+        computeMergeGain(ChainPred, ChainSucc, Jumps, 0, MergeTypeTy::X_Y));
+
+    if (EnableChainSplitAlongJumps) {
+      // Attach (a part of) ChainPred before the first block of ChainSucc
+      for (auto &Jump : ChainSucc->blocks().front()->InJumps) {
+        const auto SrcBlock = Jump->Source;
+        if (SrcBlock->CurChain != ChainPred)
+          continue;
+        size_t Offset = SrcBlock->CurIndex + 1;
+        tryChainMerging(Offset, {MergeTypeTy::X1_Y_X2, MergeTypeTy::X2_X1_Y});
+      }
+
+      // Attach (a part of) ChainPred after the last block of ChainSucc
+      for (auto &Jump : ChainSucc->blocks().back()->OutJumps) {
+        const auto DstBlock = Jump->Source;
+        if (DstBlock->CurChain != ChainPred)
+          continue;
+        size_t Offset = DstBlock->CurIndex;
+        tryChainMerging(Offset, {MergeTypeTy::X1_Y_X2, MergeTypeTy::Y_X2_X1});
+      }
+    }
+
+    // Try to break ChainPred in various ways and concatenate with ChainSucc
+    if (ChainPred->blocks().size() <= ChainSplitThreshold) {
+      for (size_t Offset = 1; Offset < ChainPred->blocks().size(); Offset++) {
+        // Try to split the chain in different ways. In practice, applying
+        // X2_Y_X1 merging is almost never provides benefits; thus, we exclude
+        // it from consideration to reduce the search space
+        tryChainMerging(Offset, {MergeTypeTy::X1_Y_X2, MergeTypeTy::Y_X2_X1,
+                                 MergeTypeTy::X2_X1_Y});
+      }
+    }
+    Edge->setCachedMergeGain(ChainPred, ChainSucc, Gain);
+    return Gain;
+  }
+
+  /// Compute the score gain of merging two chains, respecting a given
+  /// merge 'type' and 'offset'.
+  ///
+  /// The two chains are not modified in the method.
+  MergeGainTy computeMergeGain(const Chain *ChainPred, const Chain *ChainSucc,
+                               const std::vector<Jump *> &Jumps,
+                               size_t MergeOffset,
+                               MergeTypeTy MergeType) const {
+    auto MergedBlocks = mergeBlocks(ChainPred->blocks(), ChainSucc->blocks(),
+                                    MergeOffset, MergeType);
+
+    // Do not allow a merge that does not preserve the original entry block
+    if ((ChainPred->isEntry() || ChainSucc->isEntry()) &&
+        !MergedBlocks.getFirstBlock()->isEntry())
+      return MergeGainTy();
+
+    // The gain for the new chain
+    auto NewGainScore = extTSPScore(MergedBlocks, Jumps) - ChainPred->score();
+    return MergeGainTy(NewGainScore, MergeOffset, MergeType);
+  }
+
+  /// Merge two chains of blocks respecting a given merge 'type' and 'offset'.
+  ///
+  /// If MergeType == 0, then the result is a concatentation of two chains.
+  /// Otherwise, the first chain is cut into two sub-chains at the offset,
+  /// and merged using all possible ways of concatenating three chains.
+  MergedChain mergeBlocks(const std::vector<Block *> &X,
+                          const std::vector<Block *> &Y, size_t MergeOffset,
+                          MergeTypeTy MergeType) const {
+    // Split the first chain, X, into X1 and X2
+    BlockIter BeginX1 = X.begin();
+    BlockIter EndX1 = X.begin() + MergeOffset;
+    BlockIter BeginX2 = X.begin() + MergeOffset;
+    BlockIter EndX2 = X.end();
+    BlockIter BeginY = Y.begin();
+    BlockIter EndY = Y.end();
+
+    // Construct a new chain from the three existing ones
+    switch (MergeType) {
+    case MergeTypeTy::X_Y:
+      return MergedChain(BeginX1, EndX2, BeginY, EndY);
+    case MergeTypeTy::X1_Y_X2:
+      return MergedChain(BeginX1, EndX1, BeginY, EndY, BeginX2, EndX2);
+    case MergeTypeTy::Y_X2_X1:
+      return MergedChain(BeginY, EndY, BeginX2, EndX2, BeginX1, EndX1);
+    case MergeTypeTy::X2_X1_Y:
+      return MergedChain(BeginX2, EndX2, BeginX1, EndX1, BeginY, EndY);
+    }
+    llvm_unreachable("unexpected chain merge type");
+  }
+
+  /// Merge chain From into chain Into, update the list of active chains,
+  /// adjacency information, and the corresponding cached values.
+  void mergeChains(Chain *Into, Chain *From, size_t MergeOffset,
+                   MergeTypeTy MergeType) {
+    assert(Into != From && "a chain cannot be merged with itself");
+
+    // Merge the blocks
+    auto MergedBlocks =
+        mergeBlocks(Into->blocks(), From->blocks(), MergeOffset, MergeType);
+    Into->merge(From, MergedBlocks.getBlocks());
+    Into->mergeEdges(From);
+    From->clear();
+
+    // Update cached ext-tsp score for the new chain
+    auto SelfEdge = Into->getEdge(Into);
+    if (SelfEdge != nullptr) {
+      MergedBlocks = MergedChain(Into->blocks().begin(), Into->blocks().end());
+      Into->setScore(extTSPScore(MergedBlocks, SelfEdge->jumps()));
+    }
+
+    // Remove chain From from the list of active chains
+    auto Iter = std::remove(HotChains.begin(), HotChains.end(), From);
+    HotChains.erase(Iter, HotChains.end());
+
+    // Invalidate caches
+    for (auto EdgeIter : Into->edges()) {
+      EdgeIter.second->invalidateCache();
+    }
+  }
+
+  /// Concatenate all chains into a final order of blocks.
+  void concatChains(std::vector<uint64_t> &Order) {
+    // Collect chains and calculate some stats for their sorting
+    std::vector<Chain *> SortedChains;
+    DenseMap<const Chain *, double> ChainDensity;
+    for (auto &Chain : AllChains) {
+      if (!Chain.blocks().empty()) {
+        SortedChains.push_back(&Chain);
+        // Using doubles to avoid overflow of ExecutionCount
+        double Size = 0;
+        double ExecutionCount = 0;
+        for (auto Block : Chain.blocks()) {
+          Size += static_cast<double>(Block->Size);
+          ExecutionCount += static_cast<double>(Block->ExecutionCount);
+        }
+        assert(Size > 0 && "a chain of zero size");
+        ChainDensity[&Chain] = ExecutionCount / Size;
+      }
+    }
+
+    // Sorting chains by density in the decreasing order
+    std::stable_sort(SortedChains.begin(), SortedChains.end(),
+                     [&](const Chain *C1, const Chain *C2) {
+                       // Makre sure the original entry block is at the
+                       // beginning of the order
+                       if (C1->isEntry() != C2->isEntry()) {
+                         return C1->isEntry();
+                       }
+
+                       const double D1 = ChainDensity[C1];
+                       const double D2 = ChainDensity[C2];
+                       // Compare by density and break ties by chain identifiers
+                       return (D1 != D2) ? (D1 > D2) : (C1->id() < C2->id());
+                     });
+
+    // Collect the blocks in the order specified by their chains
+    Order.reserve(NumNodes);
+    for (auto Chain : SortedChains) {
+      for (auto Block : Chain->blocks()) {
+        Order.push_back(Block->Index);
+      }
+    }
+  }
+
+private:
+  /// The number of nodes in the graph.
+  const size_t NumNodes;
+
+  /// Successors of each node.
+  std::vector<std::vector<uint64_t>> SuccNodes;
+
+  /// Predecessors of each node.
+  std::vector<std::vector<uint64_t>> PredNodes;
+
+  /// All basic blocks.
+  std::vector<Block> AllBlocks;
+
+  /// All jumps between blocks.
+  std::vector<Jump> AllJumps;
+
+  /// All chains of basic blocks.
+  std::vector<Chain> AllChains;
+
+  /// All edges between chains.
+  std::vector<ChainEdge> AllEdges;
+
+  /// Active chains. The vector gets updated at runtime when chains are merged.
+  std::vector<Chain *> HotChains;
+};
+
+} // end of anonymous namespace
+
+std::vector<uint64_t> llvm::applyExtTspLayout(
+    const std::vector<uint64_t> &NodeSizes,
+    const std::vector<uint64_t> &NodeCounts,
+    const DenseMap<std::pair<uint64_t, uint64_t>, uint64_t> &EdgeCounts) {
+  size_t NumNodes = NodeSizes.size();
+
+  // Verify correctness of the input data.
+  assert(NodeCounts.size() == NodeSizes.size() && "Incorrect input");
+  assert(NumNodes > 2 && "Incorrect input");
+
+  // Apply the reordering algorithm.
+  auto Alg = ExtTSPImpl(NumNodes, NodeSizes, NodeCounts, EdgeCounts);
+  std::vector<uint64_t> Result;
+  Alg.run(Result);
+
+  // Verify correctness of the output.
+  assert(Result.front() == 0 && "Original entry point is not preserved");
+  assert(Result.size() == NumNodes && "Incorrect size of reordered layout");
+  return Result;
+}
+
+double llvm::calcExtTspScore(
+    const std::vector<uint64_t> &Order, const std::vector<uint64_t> &NodeSizes,
+    const std::vector<uint64_t> &NodeCounts,
+    const DenseMap<std::pair<uint64_t, uint64_t>, uint64_t> &EdgeCounts) {
+  // Estimate addresses of the blocks in memory
+  auto Addr = std::vector<uint64_t>(NodeSizes.size(), 0);
+  for (size_t Idx = 1; Idx < Order.size(); Idx++) {
+    Addr[Order[Idx]] = Addr[Order[Idx - 1]] + NodeSizes[Order[Idx - 1]];
+  }
+
+  // Increase the score for each jump
+  double Score = 0;
+  for (auto It : EdgeCounts) {
+    auto Pred = It.first.first;
+    auto Succ = It.first.second;
+    uint64_t Count = It.second;
+    Score += extTSPScore(Addr[Pred], NodeSizes[Pred], Addr[Succ], Count);
+  }
+  return Score;
+}
+
+double llvm::calcExtTspScore(
+    const std::vector<uint64_t> &NodeSizes,
+    const std::vector<uint64_t> &NodeCounts,
+    const DenseMap<std::pair<uint64_t, uint64_t>, uint64_t> &EdgeCounts) {
+  auto Order = std::vector<uint64_t>(NodeSizes.size());
+  for (size_t Idx = 0; Idx < NodeSizes.size(); Idx++) {
+    Order[Idx] = Idx;
+  }
+  return calcExtTspScore(Order, NodeSizes, NodeCounts, EdgeCounts);
+}
diff --git a/llvm/lib/Transforms/Utils/Debugify.cpp b/llvm/lib/Transforms/Utils/Debugify.cpp
index fc7083b0c30d..589622d69578 100644
--- a/llvm/lib/Transforms/Utils/Debugify.cpp
+++ b/llvm/lib/Transforms/Utils/Debugify.cpp
@@ -596,7 +596,7 @@ bool llvm::checkDebugInfoMetadata(Module &M,
   auto DILocsBefore = DIPreservationMap[NameOfWrappedPass].DILocations;
   auto DILocsAfter = DIPreservationAfter[NameOfWrappedPass].DILocations;
 
-  auto InstToDelete = DIPreservationAfter[NameOfWrappedPass].InstToDelete;
+  auto InstToDelete = DIPreservationMap[NameOfWrappedPass].InstToDelete;
 
   auto DIVarsBefore = DIPreservationMap[NameOfWrappedPass].DIVariables;
   auto DIVarsAfter = DIPreservationAfter[NameOfWrappedPass].DIVariables;
diff --git a/llvm/lib/Transforms/Utils/FunctionComparator.cpp b/llvm/lib/Transforms/Utils/FunctionComparator.cpp
index 326864803d7c..06596f7b04e1 100644
--- a/llvm/lib/Transforms/Utils/FunctionComparator.cpp
+++ b/llvm/lib/Transforms/Utils/FunctionComparator.cpp
@@ -58,6 +58,14 @@ int FunctionComparator::cmpNumbers(uint64_t L, uint64_t R) const {
   return 0;
 }
 
+int FunctionComparator::cmpAligns(Align L, Align R) const {
+  if (L.value() < R.value())
+    return -1;
+  if (L.value() > R.value())
+    return 1;
+  return 0;
+}
+
 int FunctionComparator::cmpOrderings(AtomicOrdering L, AtomicOrdering R) const {
   if ((int)L < (int)R)
     return -1;
@@ -556,13 +564,12 @@ int FunctionComparator::cmpOperations(const Instruction *L,
     if (int Res = cmpTypes(AI->getAllocatedType(),
                            cast<AllocaInst>(R)->getAllocatedType()))
       return Res;
-    return cmpNumbers(AI->getAlignment(), cast<AllocaInst>(R)->getAlignment());
+    return cmpAligns(AI->getAlign(), cast<AllocaInst>(R)->getAlign());
   }
   if (const LoadInst *LI = dyn_cast<LoadInst>(L)) {
     if (int Res = cmpNumbers(LI->isVolatile(), cast<LoadInst>(R)->isVolatile()))
       return Res;
-    if (int Res =
-            cmpNumbers(LI->getAlignment(), cast<LoadInst>(R)->getAlignment()))
+    if (int Res = cmpAligns(LI->getAlign(), cast<LoadInst>(R)->getAlign()))
       return Res;
     if (int Res =
             cmpOrderings(LI->getOrdering(), cast<LoadInst>(R)->getOrdering()))
@@ -578,8 +585,7 @@ int FunctionComparator::cmpOperations(const Instruction *L,
     if (int Res =
             cmpNumbers(SI->isVolatile(), cast<StoreInst>(R)->isVolatile()))
       return Res;
-    if (int Res =
-            cmpNumbers(SI->getAlignment(), cast<StoreInst>(R)->getAlignment()))
+    if (int Res = cmpAligns(SI->getAlign(), cast<StoreInst>(R)->getAlign()))
       return Res;
     if (int Res =
             cmpOrderings(SI->getOrdering(), cast<StoreInst>(R)->getOrdering()))
diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp
index ec926b1f5a94..ecad79b68185 100644
--- a/llvm/lib/Transforms/Utils/Local.cpp
+++ b/llvm/lib/Transforms/Utils/Local.cpp
@@ -402,6 +402,18 @@ bool llvm::isInstructionTriviallyDead(Instruction *I,
   return wouldInstructionBeTriviallyDead(I, TLI);
 }
 
+bool llvm::wouldInstructionBeTriviallyDeadOnUnusedPaths(
+    Instruction *I, const TargetLibraryInfo *TLI) {
+  // Instructions that are "markers" and have implied meaning on code around
+  // them (without explicit uses), are not dead on unused paths.
+  if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I))
+    if (II->getIntrinsicID() == Intrinsic::stacksave ||
+        II->getIntrinsicID() == Intrinsic::launder_invariant_group ||
+        II->isLifetimeStartOrEnd())
+      return false;
+  return wouldInstructionBeTriviallyDead(I, TLI);
+}
+
 bool llvm::wouldInstructionBeTriviallyDead(Instruction *I,
                                            const TargetLibraryInfo *TLI) {
   if (I->isTerminator())
diff --git a/llvm/lib/Transforms/Utils/LoopPeel.cpp b/llvm/lib/Transforms/Utils/LoopPeel.cpp
index f3cf42be8ba1..69fd110dc3c2 100644
--- a/llvm/lib/Transforms/Utils/LoopPeel.cpp
+++ b/llvm/lib/Transforms/Utils/LoopPeel.cpp
@@ -104,9 +104,7 @@ bool llvm::canPeel(Loop *L) {
   // note that LoopPeeling currently can only update the branch weights of latch
   // blocks and branch weights to blocks with deopt or unreachable do not need
   // updating.
-  return all_of(Exits, [](const BasicBlock *BB) {
-    return IsBlockFollowedByDeoptOrUnreachable(BB);
-  });
+  return llvm::all_of(Exits, IsBlockFollowedByDeoptOrUnreachable);
 }
 
 // This function calculates the number of iterations after which the given Phi
@@ -333,6 +331,31 @@ static unsigned countToEliminateCompares(Loop &L, unsigned MaxPeelCount,
   return DesiredPeelCount;
 }
 
+/// This "heuristic" exactly matches implicit behavior which used to exist
+/// inside getLoopEstimatedTripCount.  It was added here to keep an
+/// improvement inside that API from causing peeling to become more agressive.
+/// This should probably be removed.
+static bool violatesLegacyMultiExitLoopCheck(Loop *L) {
+  BasicBlock *Latch = L->getLoopLatch();
+  if (!Latch)
+    return true;
+
+  BranchInst *LatchBR = dyn_cast<BranchInst>(Latch->getTerminator());
+  if (!LatchBR || LatchBR->getNumSuccessors() != 2 || !L->isLoopExiting(Latch))
+    return true;
+
+  assert((LatchBR->getSuccessor(0) == L->getHeader() ||
+          LatchBR->getSuccessor(1) == L->getHeader()) &&
+         "At least one edge out of the latch must go to the header");
+
+  SmallVector<BasicBlock *, 4> ExitBlocks;
+  L->getUniqueNonLatchExitBlocks(ExitBlocks);
+  return any_of(ExitBlocks, [](const BasicBlock *EB) {
+      return !EB->getTerminatingDeoptimizeCall();
+    });
+}
+
+
 // Return the number of iterations we want to peel off.
 void llvm::computePeelCount(Loop *L, unsigned LoopSize,
                             TargetTransformInfo::PeelingPreferences &PP,
@@ -436,6 +459,8 @@ void llvm::computePeelCount(Loop *L, unsigned LoopSize,
   // We only do this in the presence of profile information, since otherwise
   // our estimates of the trip count are not reliable enough.
   if (L->getHeader()->getParent()->hasProfileData()) {
+    if (violatesLegacyMultiExitLoopCheck(L))
+      return;
     Optional<unsigned> PeelCount = getLoopEstimatedTripCount(L);
     if (!PeelCount)
       return;
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index c8e42acdffb3..93157bd87c34 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -773,8 +773,8 @@ void llvm::breakLoopBackedge(Loop *L, DominatorTree &DT, ScalarEvolution &SE,
 }
 
 
-/// Checks if \p L has single exit through latch block except possibly
-/// "deoptimizing" exits. Returns branch instruction terminating the loop
+/// Checks if \p L has an exiting latch branch.  There may also be other
+/// exiting blocks.  Returns branch instruction terminating the loop
 /// latch if above check is successful, nullptr otherwise.
 static BranchInst *getExpectedExitLoopLatchBranch(Loop *L) {
   BasicBlock *Latch = L->getLoopLatch();
@@ -789,53 +789,61 @@ static BranchInst *getExpectedExitLoopLatchBranch(Loop *L) {
           LatchBR->getSuccessor(1) == L->getHeader()) &&
          "At least one edge out of the latch must go to the header");
 
-  SmallVector<BasicBlock *, 4> ExitBlocks;
-  L->getUniqueNonLatchExitBlocks(ExitBlocks);
-  if (any_of(ExitBlocks, [](const BasicBlock *EB) {
-        return !EB->getTerminatingDeoptimizeCall();
-      }))
-    return nullptr;
-
   return LatchBR;
 }
 
-Optional<unsigned>
-llvm::getLoopEstimatedTripCount(Loop *L,
-                                unsigned *EstimatedLoopInvocationWeight) {
-  // Support loops with an exiting latch and other existing exists only
-  // deoptimize.
-  BranchInst *LatchBranch = getExpectedExitLoopLatchBranch(L);
-  if (!LatchBranch)
-    return None;
-
+/// Return the estimated trip count for any exiting branch which dominates
+/// the loop latch.
+static Optional<uint64_t>
+getEstimatedTripCount(BranchInst *ExitingBranch, Loop *L,
+                      uint64_t &OrigExitWeight) {
   // To estimate the number of times the loop body was executed, we want to
   // know the number of times the backedge was taken, vs. the number of times
   // we exited the loop.
-  uint64_t BackedgeTakenWeight, LatchExitWeight;
-  if (!LatchBranch->extractProfMetadata(BackedgeTakenWeight, LatchExitWeight))
+  uint64_t LoopWeight, ExitWeight;
+  if (!ExitingBranch->extractProfMetadata(LoopWeight, ExitWeight))
     return None;
 
-  if (LatchBranch->getSuccessor(0) != L->getHeader())
-    std::swap(BackedgeTakenWeight, LatchExitWeight);
+  if (L->contains(ExitingBranch->getSuccessor(1)))
+    std::swap(LoopWeight, ExitWeight);
 
-  if (!LatchExitWeight)
+  if (!ExitWeight)
+    // Don't have a way to return predicated infinite
     return None;
 
-  if (EstimatedLoopInvocationWeight)
-    *EstimatedLoopInvocationWeight = LatchExitWeight;
+  OrigExitWeight = ExitWeight;
 
-  // Estimated backedge taken count is a ratio of the backedge taken weight by
-  // the weight of the edge exiting the loop, rounded to nearest.
-  uint64_t BackedgeTakenCount =
-      llvm::divideNearest(BackedgeTakenWeight, LatchExitWeight);
-  // Estimated trip count is one plus estimated backedge taken count.
-  return BackedgeTakenCount + 1;
+  // Estimated exit count is a ratio of the loop weight by the weight of the
+  // edge exiting the loop, rounded to nearest.
+  uint64_t ExitCount = llvm::divideNearest(LoopWeight, ExitWeight);
+  // Estimated trip count is one plus estimated exit count.
+  return ExitCount + 1;
+}
+
+Optional<unsigned>
+llvm::getLoopEstimatedTripCount(Loop *L,
+                                unsigned *EstimatedLoopInvocationWeight) {
+  // Currently we take the estimate exit count only from the loop latch,
+  // ignoring other exiting blocks.  This can overestimate the trip count
+  // if we exit through another exit, but can never underestimate it.
+  // TODO: incorporate information from other exits
+  if (BranchInst *LatchBranch = getExpectedExitLoopLatchBranch(L)) {
+    uint64_t ExitWeight;
+    if (Optional<uint64_t> EstTripCount =
+        getEstimatedTripCount(LatchBranch, L, ExitWeight)) {
+      if (EstimatedLoopInvocationWeight)
+        *EstimatedLoopInvocationWeight = ExitWeight;
+      return *EstTripCount;
+    }
+  }
+  return None;
 }
 
 bool llvm::setLoopEstimatedTripCount(Loop *L, unsigned EstimatedTripCount,
                                      unsigned EstimatedloopInvocationWeight) {
-  // Support loops with an exiting latch and other existing exists only
-  // deoptimize.
+  // At the moment, we currently support changing the estimate trip count of
+  // the latch branch only.  We could extend this API to manipulate estimated
+  // trip counts for any exit.
   BranchInst *LatchBranch = getExpectedExitLoopLatchBranch(L);
   if (!LatchBranch)
     return false;
@@ -923,8 +931,7 @@ Value *llvm::createMinMaxOp(IRBuilderBase &Builder, RecurKind RK, Value *Left,
 
 // Helper to generate an ordered reduction.
 Value *llvm::getOrderedReduction(IRBuilderBase &Builder, Value *Acc, Value *Src,
-                                 unsigned Op, RecurKind RdxKind,
-                                 ArrayRef<Value *> RedOps) {
+                                 unsigned Op, RecurKind RdxKind) {
   unsigned VF = cast<FixedVectorType>(Src->getType())->getNumElements();
 
   // Extract and apply reduction ops in ascending order:
@@ -942,9 +949,6 @@ Value *llvm::getOrderedReduction(IRBuilderBase &Builder, Value *Acc, Value *Src,
              "Invalid min/max");
       Result = createMinMaxOp(Builder, RdxKind, Result, Ext);
     }
-
-    if (!RedOps.empty())
-      propagateIRFlags(Result, RedOps);
   }
 
   return Result;
@@ -952,14 +956,20 @@ Value *llvm::getOrderedReduction(IRBuilderBase &Builder, Value *Acc, Value *Src,
 
 // Helper to generate a log2 shuffle reduction.
 Value *llvm::getShuffleReduction(IRBuilderBase &Builder, Value *Src,
-                                 unsigned Op, RecurKind RdxKind,
-                                 ArrayRef<Value *> RedOps) {
+                                 unsigned Op, RecurKind RdxKind) {
   unsigned VF = cast<FixedVectorType>(Src->getType())->getNumElements();
   // VF is a power of 2 so we can emit the reduction using log2(VF) shuffles
   // and vector ops, reducing the set of values being computed by half each
   // round.
   assert(isPowerOf2_32(VF) &&
          "Reduction emission only supported for pow2 vectors!");
+  // Note: fast-math-flags flags are controlled by the builder configuration
+  // and are assumed to apply to all generated arithmetic instructions.  Other
+  // poison generating flags (nsw/nuw/inbounds/inrange/exact) are not part
+  // of the builder configuration, and since they're not passed explicitly,
+  // will never be relevant here.  Note that it would be generally unsound to
+  // propagate these from an intrinsic call to the expansion anyways as we/
+  // change the order of operations.
   Value *TmpVec = Src;
   SmallVector<int, 32> ShuffleMask(VF);
   for (unsigned i = VF; i != 1; i >>= 1) {
@@ -973,7 +983,6 @@ Value *llvm::getShuffleReduction(IRBuilderBase &Builder, Value *Src,
     Value *Shuf = Builder.CreateShuffleVector(TmpVec, ShuffleMask, "rdx.shuf");
 
     if (Op != Instruction::ICmp && Op != Instruction::FCmp) {
-      // The builder propagates its fast-math-flags setting.
       TmpVec = Builder.CreateBinOp((Instruction::BinaryOps)Op, TmpVec, Shuf,
                                    "bin.rdx");
     } else {
@@ -981,13 +990,6 @@ Value *llvm::getShuffleReduction(IRBuilderBase &Builder, Value *Src,
              "Invalid min/max");
       TmpVec = createMinMaxOp(Builder, RdxKind, TmpVec, Shuf);
     }
-    if (!RedOps.empty())
-      propagateIRFlags(TmpVec, RedOps);
-
-    // We may compute the reassociated scalar ops in a way that does not
-    // preserve nsw/nuw etc. Conservatively, drop those flags.
-    if (auto *ReductionInst = dyn_cast<Instruction>(TmpVec))
-      ReductionInst->dropPoisonGeneratingFlags();
   }
   // The result is in the first element of the vector.
   return Builder.CreateExtractElement(TmpVec, Builder.getInt32(0));
@@ -1035,8 +1037,7 @@ Value *llvm::createSelectCmpTargetReduction(IRBuilderBase &Builder,
 
 Value *llvm::createSimpleTargetReduction(IRBuilderBase &Builder,
                                          const TargetTransformInfo *TTI,
-                                         Value *Src, RecurKind RdxKind,
-                                         ArrayRef<Value *> RedOps) {
+                                         Value *Src, RecurKind RdxKind) {
   auto *SrcVecEltTy = cast<VectorType>(Src->getType())->getElementType();
   switch (RdxKind) {
   case RecurKind::Add:
diff --git a/llvm/lib/Transforms/Utils/MetaRenamer.cpp b/llvm/lib/Transforms/Utils/MetaRenamer.cpp
index 3ce10535d45f..9fba2f3f86b5 100644
--- a/llvm/lib/Transforms/Utils/MetaRenamer.cpp
+++ b/llvm/lib/Transforms/Utils/MetaRenamer.cpp
@@ -15,6 +15,7 @@
 #include "llvm/Transforms/Utils/MetaRenamer.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
@@ -31,10 +32,36 @@
 #include "llvm/IR/TypeFinder.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Pass.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Transforms/Utils.h"
 
 using namespace llvm;
 
+static cl::opt<std::string> RenameExcludeFunctionPrefixes(
+    "rename-exclude-function-prefixes",
+    cl::desc("Prefixes for functions that don't need to be renamed, separated "
+             "by a comma"),
+    cl::Hidden);
+
+static cl::opt<std::string> RenameExcludeAliasPrefixes(
+    "rename-exclude-alias-prefixes",
+    cl::desc("Prefixes for aliases that don't need to be renamed, separated "
+             "by a comma"),
+    cl::Hidden);
+
+static cl::opt<std::string> RenameExcludeGlobalPrefixes(
+    "rename-exclude-global-prefixes",
+    cl::desc(
+        "Prefixes for global values that don't need to be renamed, separated "
+        "by a comma"),
+    cl::Hidden);
+
+static cl::opt<std::string> RenameExcludeStructPrefixes(
+    "rename-exclude-struct-prefixes",
+    cl::desc("Prefixes for structs that don't need to be renamed, separated "
+             "by a comma"),
+    cl::Hidden);
+
 static const char *const metaNames[] = {
   // See http://en.wikipedia.org/wiki/Metasyntactic_variable
   "foo", "bar", "baz", "quux", "barney", "snork", "zot", "blam", "hoge",
@@ -66,6 +93,18 @@ struct Renamer {
   PRNG prng;
 };
 
+static void
+parseExcludedPrefixes(StringRef PrefixesStr,
+                      SmallVectorImpl<StringRef> &ExcludedPrefixes) {
+  for (;;) {
+    auto PrefixesSplit = PrefixesStr.split(',');
+    if (PrefixesSplit.first.empty())
+      break;
+    ExcludedPrefixes.push_back(PrefixesSplit.first);
+    PrefixesStr = PrefixesSplit.second;
+  }
+}
+
 void MetaRename(Function &F) {
   for (Argument &Arg : F.args())
     if (!Arg.getType()->isVoidTy())
@@ -91,10 +130,26 @@ void MetaRename(Module &M,
 
   Renamer renamer(randSeed);
 
+  SmallVector<StringRef, 8> ExcludedAliasesPrefixes;
+  SmallVector<StringRef, 8> ExcludedGlobalsPrefixes;
+  SmallVector<StringRef, 8> ExcludedStructsPrefixes;
+  SmallVector<StringRef, 8> ExcludedFuncPrefixes;
+  parseExcludedPrefixes(RenameExcludeAliasPrefixes, ExcludedAliasesPrefixes);
+  parseExcludedPrefixes(RenameExcludeGlobalPrefixes, ExcludedGlobalsPrefixes);
+  parseExcludedPrefixes(RenameExcludeStructPrefixes, ExcludedStructsPrefixes);
+  parseExcludedPrefixes(RenameExcludeFunctionPrefixes, ExcludedFuncPrefixes);
+
+  auto IsNameExcluded = [](StringRef &Name,
+                           SmallVectorImpl<StringRef> &ExcludedPrefixes) {
+    return any_of(ExcludedPrefixes,
+                  [&Name](auto &Prefix) { return Name.startswith(Prefix); });
+  };
+
   // Rename all aliases
   for (GlobalAlias &GA : M.aliases()) {
     StringRef Name = GA.getName();
-    if (Name.startswith("llvm.") || (!Name.empty() && Name[0] == 1))
+    if (Name.startswith("llvm.") || (!Name.empty() && Name[0] == 1) ||
+        IsNameExcluded(Name, ExcludedAliasesPrefixes))
       continue;
 
     GA.setName("alias");
@@ -103,7 +158,8 @@ void MetaRename(Module &M,
   // Rename all global variables
   for (GlobalVariable &GV : M.globals()) {
     StringRef Name = GV.getName();
-    if (Name.startswith("llvm.") || (!Name.empty() && Name[0] == 1))
+    if (Name.startswith("llvm.") || (!Name.empty() && Name[0] == 1) ||
+        IsNameExcluded(Name, ExcludedGlobalsPrefixes))
       continue;
 
     GV.setName("global");
@@ -113,7 +169,9 @@ void MetaRename(Module &M,
   TypeFinder StructTypes;
   StructTypes.run(M, true);
   for (StructType *STy : StructTypes) {
-    if (STy->isLiteral() || STy->getName().empty())
+    StringRef Name = STy->getName();
+    if (STy->isLiteral() || Name.empty() ||
+        IsNameExcluded(Name, ExcludedStructsPrefixes))
       continue;
 
     SmallString<128> NameStorage;
@@ -128,7 +186,8 @@ void MetaRename(Module &M,
     // Leave library functions alone because their presence or absence could
     // affect the behavior of other passes.
     if (Name.startswith("llvm.") || (!Name.empty() && Name[0] == 1) ||
-        GetTLI(F).getLibFunc(F, Tmp))
+        GetTLI(F).getLibFunc(F, Tmp) ||
+        IsNameExcluded(Name, ExcludedFuncPrefixes))
       continue;
 
     // Leave @main alone. The output of -metarenamer might be passed to
diff --git a/llvm/lib/Transforms/Utils/RelLookupTableConverter.cpp b/llvm/lib/Transforms/Utils/RelLookupTableConverter.cpp
index 3ebc89158173..65207056a3f4 100644
--- a/llvm/lib/Transforms/Utils/RelLookupTableConverter.cpp
+++ b/llvm/lib/Transforms/Utils/RelLookupTableConverter.cpp
@@ -144,6 +144,10 @@ static void convertToRelLookupTable(GlobalVariable &LookupTable) {
   Value *Offset =
       Builder.CreateShl(Index, ConstantInt::get(IntTy, 2), "reltable.shift");
 
+  // Insert the call to load.relative instrinsic before LOAD.
+  // GEP might not be immediately followed by a LOAD, like it can be hoisted
+  // outside the loop or another instruction might be inserted them in between.
+  Builder.SetInsertPoint(Load);
   Function *LoadRelIntrinsic = llvm::Intrinsic::getDeclaration(
       &M, Intrinsic::load_relative, {Index->getType()});
   Value *Base = Builder.CreateBitCast(RelLookupTable, Builder.getInt8PtrTy());
diff --git a/llvm/lib/Transforms/Utils/SampleProfileInference.cpp b/llvm/lib/Transforms/Utils/SampleProfileInference.cpp
index 9495e442e0bf..2f2dff6b5f0b 100644
--- a/llvm/lib/Transforms/Utils/SampleProfileInference.cpp
+++ b/llvm/lib/Transforms/Utils/SampleProfileInference.cpp
@@ -220,7 +220,7 @@ private:
       Now = Pred;
     }
 
-    assert(PathCapacity > 0 && "found incorrect augmenting path");
+    assert(PathCapacity > 0 && "found an incorrect augmenting path");
 
     // Update the flow along the path
     Now = Target;
@@ -271,6 +271,352 @@ private:
   uint64_t Target;
 };
 
+/// A post-processing adjustment of control flow. It applies two steps by
+/// rerouting some flow and making it more realistic:
+///
+/// - First, it removes all isolated components ("islands") with a positive flow
+///   that are unreachable from the entry block. For every such component, we
+///   find the shortest from the entry to an exit passing through the component,
+///   and increase the flow by one unit along the path.
+///
+/// - Second, it identifies all "unknown subgraphs" consisting of basic blocks
+///   with no sampled counts. Then it rebalnces the flow that goes through such
+///   a subgraph so that each branch is taken with probability 50%.
+///   An unknown subgraph is such that for every two nodes u and v:
+///     - u dominates v and u is not unknown;
+///     - v post-dominates u; and
+///     - all inner-nodes of all (u,v)-paths are unknown.
+///
+class FlowAdjuster {
+public:
+  FlowAdjuster(FlowFunction &Func) : Func(Func) {
+    assert(Func.Blocks[Func.Entry].isEntry() &&
+           "incorrect index of the entry block");
+  }
+
+  // Run the post-processing
+  void run() {
+    /// Adjust the flow to get rid of isolated components.
+    joinIsolatedComponents();
+
+    /// Rebalance the flow inside unknown subgraphs.
+    rebalanceUnknownSubgraphs();
+  }
+
+  /// The probability for the first successor of a unknown subgraph
+  static constexpr double UnknownFirstSuccProbability = 0.5;
+
+private:
+  void joinIsolatedComponents() {
+    // Find blocks that are reachable from the source
+    auto Visited = std::vector<bool>(NumBlocks(), false);
+    findReachable(Func.Entry, Visited);
+
+    // Iterate over all non-reachable blocks and adjust their weights
+    for (uint64_t I = 0; I < NumBlocks(); I++) {
+      auto &Block = Func.Blocks[I];
+      if (Block.Flow > 0 && !Visited[I]) {
+        // Find a path from the entry to an exit passing through the block I
+        auto Path = findShortestPath(I);
+        // Increase the flow along the path
+        assert(Path.size() > 0 && Path[0]->Source == Func.Entry &&
+               "incorrectly computed path adjusting control flow");
+        Func.Blocks[Func.Entry].Flow += 1;
+        for (auto &Jump : Path) {
+          Jump->Flow += 1;
+          Func.Blocks[Jump->Target].Flow += 1;
+          // Update reachability
+          findReachable(Jump->Target, Visited);
+        }
+      }
+    }
+  }
+
+  /// Run BFS from a given block along the jumps with a positive flow and mark
+  /// all reachable blocks.
+  void findReachable(uint64_t Src, std::vector<bool> &Visited) {
+    if (Visited[Src])
+      return;
+    std::queue<uint64_t> Queue;
+    Queue.push(Src);
+    Visited[Src] = true;
+    while (!Queue.empty()) {
+      Src = Queue.front();
+      Queue.pop();
+      for (auto Jump : Func.Blocks[Src].SuccJumps) {
+        uint64_t Dst = Jump->Target;
+        if (Jump->Flow > 0 && !Visited[Dst]) {
+          Queue.push(Dst);
+          Visited[Dst] = true;
+        }
+      }
+    }
+  }
+
+  /// Find the shortest path from the entry block to an exit block passing
+  /// through a given block.
+  std::vector<FlowJump *> findShortestPath(uint64_t BlockIdx) {
+    // A path from the entry block to BlockIdx
+    auto ForwardPath = findShortestPath(Func.Entry, BlockIdx);
+    // A path from BlockIdx to an exit block
+    auto BackwardPath = findShortestPath(BlockIdx, AnyExitBlock);
+
+    // Concatenate the two paths
+    std::vector<FlowJump *> Result;
+    Result.insert(Result.end(), ForwardPath.begin(), ForwardPath.end());
+    Result.insert(Result.end(), BackwardPath.begin(), BackwardPath.end());
+    return Result;
+  }
+
+  /// Apply the Dijkstra algorithm to find the shortest path from a given
+  /// Source to a given Target block.
+  /// If Target == -1, then the path ends at an exit block.
+  std::vector<FlowJump *> findShortestPath(uint64_t Source, uint64_t Target) {
+    // Quit early, if possible
+    if (Source == Target)
+      return std::vector<FlowJump *>();
+    if (Func.Blocks[Source].isExit() && Target == AnyExitBlock)
+      return std::vector<FlowJump *>();
+
+    // Initialize data structures
+    auto Distance = std::vector<int64_t>(NumBlocks(), INF);
+    auto Parent = std::vector<FlowJump *>(NumBlocks(), nullptr);
+    Distance[Source] = 0;
+    std::set<std::pair<uint64_t, uint64_t>> Queue;
+    Queue.insert(std::make_pair(Distance[Source], Source));
+
+    // Run the Dijkstra algorithm
+    while (!Queue.empty()) {
+      uint64_t Src = Queue.begin()->second;
+      Queue.erase(Queue.begin());
+      // If we found a solution, quit early
+      if (Src == Target ||
+          (Func.Blocks[Src].isExit() && Target == AnyExitBlock))
+        break;
+
+      for (auto Jump : Func.Blocks[Src].SuccJumps) {
+        uint64_t Dst = Jump->Target;
+        int64_t JumpDist = jumpDistance(Jump);
+        if (Distance[Dst] > Distance[Src] + JumpDist) {
+          Queue.erase(std::make_pair(Distance[Dst], Dst));
+
+          Distance[Dst] = Distance[Src] + JumpDist;
+          Parent[Dst] = Jump;
+
+          Queue.insert(std::make_pair(Distance[Dst], Dst));
+        }
+      }
+    }
+    // If Target is not provided, find the closest exit block
+    if (Target == AnyExitBlock) {
+      for (uint64_t I = 0; I < NumBlocks(); I++) {
+        if (Func.Blocks[I].isExit() && Parent[I] != nullptr) {
+          if (Target == AnyExitBlock || Distance[Target] > Distance[I]) {
+            Target = I;
+          }
+        }
+      }
+    }
+    assert(Parent[Target] != nullptr && "a path does not exist");
+
+    // Extract the constructed path
+    std::vector<FlowJump *> Result;
+    uint64_t Now = Target;
+    while (Now != Source) {
+      assert(Now == Parent[Now]->Target && "incorrect parent jump");
+      Result.push_back(Parent[Now]);
+      Now = Parent[Now]->Source;
+    }
+    // Reverse the path, since it is extracted from Target to Source
+    std::reverse(Result.begin(), Result.end());
+    return Result;
+  }
+
+  /// A distance of a path for a given jump.
+  /// In order to incite the path to use blocks/jumps with large positive flow,
+  /// and avoid changing branch probability of outgoing edges drastically,
+  /// set the distance as follows:
+  ///   if Jump.Flow > 0, then distance = max(100 - Jump->Flow, 0)
+  ///   if Block.Weight > 0, then distance = 1
+  ///   otherwise distance >> 1
+  int64_t jumpDistance(FlowJump *Jump) const {
+    int64_t BaseDistance = 100;
+    if (Jump->IsUnlikely)
+      return MinCostMaxFlow::AuxCostUnlikely;
+    if (Jump->Flow > 0)
+      return std::max(BaseDistance - (int64_t)Jump->Flow, (int64_t)0);
+    if (Func.Blocks[Jump->Target].Weight > 0)
+      return BaseDistance;
+    return BaseDistance * (NumBlocks() + 1);
+  };
+
+  uint64_t NumBlocks() const { return Func.Blocks.size(); }
+
+  /// Rebalance unknown subgraphs so as each branch splits with probabilities
+  /// UnknownFirstSuccProbability and 1 - UnknownFirstSuccProbability
+  void rebalanceUnknownSubgraphs() {
+    assert(UnknownFirstSuccProbability >= 0.0 &&
+           UnknownFirstSuccProbability <= 1.0 &&
+           "the share of the unknown successor should be between 0 and 1");
+    // Try to find unknown subgraphs from each non-unknown block
+    for (uint64_t I = 0; I < Func.Blocks.size(); I++) {
+      auto SrcBlock = &Func.Blocks[I];
+      // Do not attempt to find unknown successors from a unknown or a
+      // zero-flow block
+      if (SrcBlock->UnknownWeight || SrcBlock->Flow == 0)
+        continue;
+
+      std::vector<FlowBlock *> UnknownSuccs;
+      FlowBlock *DstBlock = nullptr;
+      // Find a unknown subgraphs starting at block SrcBlock
+      if (!findUnknownSubgraph(SrcBlock, DstBlock, UnknownSuccs))
+        continue;
+      // At the moment, we do not rebalance subgraphs containing cycles among
+      // unknown blocks
+      if (!isAcyclicSubgraph(SrcBlock, DstBlock, UnknownSuccs))
+        continue;
+
+      // Rebalance the flow
+      rebalanceUnknownSubgraph(SrcBlock, DstBlock, UnknownSuccs);
+    }
+  }
+
+  /// Find a unknown subgraph starting at block SrcBlock.
+  /// If the search is successful, the method sets DstBlock and UnknownSuccs.
+  bool findUnknownSubgraph(FlowBlock *SrcBlock, FlowBlock *&DstBlock,
+                           std::vector<FlowBlock *> &UnknownSuccs) {
+    // Run BFS from SrcBlock and make sure all paths are going through unknown
+    // blocks and end at a non-unknown DstBlock
+    auto Visited = std::vector<bool>(NumBlocks(), false);
+    std::queue<uint64_t> Queue;
+    DstBlock = nullptr;
+
+    Queue.push(SrcBlock->Index);
+    Visited[SrcBlock->Index] = true;
+    while (!Queue.empty()) {
+      auto &Block = Func.Blocks[Queue.front()];
+      Queue.pop();
+      // Process blocks reachable from Block
+      for (auto Jump : Block.SuccJumps) {
+        uint64_t Dst = Jump->Target;
+        if (Visited[Dst])
+          continue;
+        Visited[Dst] = true;
+        if (!Func.Blocks[Dst].UnknownWeight) {
+          // If we see non-unique non-unknown block reachable from SrcBlock,
+          // stop processing and skip rebalancing
+          FlowBlock *CandidateDstBlock = &Func.Blocks[Dst];
+          if (DstBlock != nullptr && DstBlock != CandidateDstBlock)
+            return false;
+          DstBlock = CandidateDstBlock;
+        } else {
+          Queue.push(Dst);
+          UnknownSuccs.push_back(&Func.Blocks[Dst]);
+        }
+      }
+    }
+
+    // If the list of unknown blocks is empty, we don't need rebalancing
+    if (UnknownSuccs.empty())
+      return false;
+    // If all reachable nodes from SrcBlock are unknown, skip rebalancing
+    if (DstBlock == nullptr)
+      return false;
+    // If any of the unknown blocks is an exit block, skip rebalancing
+    for (auto Block : UnknownSuccs) {
+      if (Block->isExit())
+        return false;
+    }
+
+    return true;
+  }
+
+  /// Verify if the given unknown subgraph is acyclic, and if yes, reorder
+  /// UnknownSuccs in the topological order (so that all jumps are "forward").
+  bool isAcyclicSubgraph(FlowBlock *SrcBlock, FlowBlock *DstBlock,
+                         std::vector<FlowBlock *> &UnknownSuccs) {
+    // Extract local in-degrees in the considered subgraph
+    auto LocalInDegree = std::vector<uint64_t>(NumBlocks(), 0);
+    for (auto Jump : SrcBlock->SuccJumps) {
+      LocalInDegree[Jump->Target]++;
+    }
+    for (uint64_t I = 0; I < UnknownSuccs.size(); I++) {
+      for (auto Jump : UnknownSuccs[I]->SuccJumps) {
+        LocalInDegree[Jump->Target]++;
+      }
+    }
+    // A loop containing SrcBlock
+    if (LocalInDegree[SrcBlock->Index] > 0)
+      return false;
+
+    std::vector<FlowBlock *> AcyclicOrder;
+    std::queue<uint64_t> Queue;
+    Queue.push(SrcBlock->Index);
+    while (!Queue.empty()) {
+      auto &Block = Func.Blocks[Queue.front()];
+      Queue.pop();
+      // Stop propagation once we reach DstBlock
+      if (Block.Index == DstBlock->Index)
+        break;
+
+      AcyclicOrder.push_back(&Block);
+      // Add to the queue all successors with zero local in-degree
+      for (auto Jump : Block.SuccJumps) {
+        uint64_t Dst = Jump->Target;
+        LocalInDegree[Dst]--;
+        if (LocalInDegree[Dst] == 0) {
+          Queue.push(Dst);
+        }
+      }
+    }
+
+    // If there is a cycle in the subgraph, AcyclicOrder contains only a subset
+    // of all blocks
+    if (UnknownSuccs.size() + 1 != AcyclicOrder.size())
+      return false;
+    UnknownSuccs = AcyclicOrder;
+    return true;
+  }
+
+  /// Rebalance a given subgraph.
+  void rebalanceUnknownSubgraph(FlowBlock *SrcBlock, FlowBlock *DstBlock,
+                                std::vector<FlowBlock *> &UnknownSuccs) {
+    assert(SrcBlock->Flow > 0 && "zero-flow block in unknown subgraph");
+    assert(UnknownSuccs.front() == SrcBlock && "incorrect order of unknowns");
+
+    for (auto Block : UnknownSuccs) {
+      // Block's flow is the sum of incoming flows
+      uint64_t TotalFlow = 0;
+      if (Block == SrcBlock) {
+        TotalFlow = Block->Flow;
+      } else {
+        for (auto Jump : Block->PredJumps) {
+          TotalFlow += Jump->Flow;
+        }
+        Block->Flow = TotalFlow;
+      }
+
+      // Process all successor jumps and update corresponding flow values
+      for (uint64_t I = 0; I < Block->SuccJumps.size(); I++) {
+        auto Jump = Block->SuccJumps[I];
+        if (I + 1 == Block->SuccJumps.size()) {
+          Jump->Flow = TotalFlow;
+          continue;
+        }
+        uint64_t Flow = uint64_t(TotalFlow * UnknownFirstSuccProbability);
+        Jump->Flow = Flow;
+        TotalFlow -= Flow;
+      }
+    }
+  }
+
+  /// A constant indicating an arbitrary exit block of a function.
+  static constexpr uint64_t AnyExitBlock = uint64_t(-1);
+
+  /// The function.
+  FlowFunction &Func;
+};
+
 /// Initializing flow network for a given function.
 ///
 /// Every block is split into three nodes that are responsible for (i) an
@@ -440,6 +786,39 @@ void verifyWeights(const FlowFunction &Func) {
     }
   }
   assert(TotalInFlow == TotalOutFlow && "incorrectly computed control flow");
+
+  // Verify that there are no isolated flow components
+  // One could modify FlowFunction to hold edges indexed by the sources, which
+  // will avoid a creation of the object
+  auto PositiveFlowEdges = std::vector<std::vector<uint64_t>>(NumBlocks);
+  for (auto &Jump : Func.Jumps) {
+    if (Jump.Flow > 0) {
+      PositiveFlowEdges[Jump.Source].push_back(Jump.Target);
+    }
+  }
+
+  // Run BFS from the source along edges with positive flow
+  std::queue<uint64_t> Queue;
+  auto Visited = std::vector<bool>(NumBlocks, false);
+  Queue.push(Func.Entry);
+  Visited[Func.Entry] = true;
+  while (!Queue.empty()) {
+    uint64_t Src = Queue.front();
+    Queue.pop();
+    for (uint64_t Dst : PositiveFlowEdges[Src]) {
+      if (!Visited[Dst]) {
+        Queue.push(Dst);
+        Visited[Dst] = true;
+      }
+    }
+  }
+
+  // Verify that every block that has a positive flow is reached from the source
+  // along edges with a positive flow
+  for (uint64_t I = 0; I < NumBlocks; I++) {
+    auto &Block = Func.Blocks[I];
+    assert((Visited[I] || Block.Flow == 0) && "an isolated flow component");
+  }
 }
 #endif
 
@@ -455,6 +834,10 @@ void llvm::applyFlowInference(FlowFunction &Func) {
   // Extract flow values for every block and every edge
   extractWeights(InferenceNetwork, Func);
 
+  // Post-processing adjustments to the flow
+  auto Adjuster = FlowAdjuster(Func);
+  Adjuster.run();
+
 #ifndef NDEBUG
   // Verify the result
   verifyWeights(Func);
diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
index 71c15d5c51fc..c840ee85795f 100644
--- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
+++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
@@ -1047,9 +1047,9 @@ bool SCEVExpander::hoistIVInc(Instruction *IncV, Instruction *InsertPos) {
     if (SE.DT.dominates(IncV, InsertPos))
       break;
   }
-  for (auto I = IVIncs.rbegin(), E = IVIncs.rend(); I != E; ++I) {
-    fixupInsertPoints(*I);
-    (*I)->moveBefore(InsertPos);
+  for (Instruction *I : llvm::reverse(IVIncs)) {
+    fixupInsertPoints(I);
+    I->moveBefore(InsertPos);
   }
   return true;
 }
diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index afa3ecde77f9..1046998c26de 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -3629,7 +3629,7 @@ static bool tryWidenCondBranchToCondBranch(BranchInst *PBI, BranchInst *BI,
     return false; // TODO
   // Use lambda to lazily compute expensive condition after cheap ones.
   auto NoSideEffects = [](BasicBlock &BB) {
-    return !llvm::any_of(BB, [](const Instruction &I) {
+    return llvm::none_of(BB, [](const Instruction &I) {
         return I.mayWriteToMemory() || I.mayHaveSideEffects();
       });
   };
diff --git a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
index e190a1294eb3..02727a3dbf9c 100644
--- a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
@@ -193,6 +193,19 @@ static void annotateNonNullAndDereferenceable(CallInst *CI, ArrayRef<unsigned> A
   }
 }
 
+// Copy CallInst "flags" like musttail, notail, and tail. Return New param for
+// easier chaining. Calls to emit* and B.createCall should probably be wrapped
+// in this function when New is created to replace Old. Callers should take
+// care to check Old.isMustTailCall() if they aren't replacing Old directly
+// with New.
+static Value *copyFlags(const CallInst &Old, Value *New) {
+  assert(!Old.isMustTailCall() && "do not copy musttail call flags");
+  assert(!Old.isNoTailCall() && "do not copy notail call flags");
+  if (auto *NewCI = dyn_cast_or_null<CallInst>(New))
+    NewCI->setTailCallKind(Old.getTailCallKind());
+  return New;
+}
+
 //===----------------------------------------------------------------------===//
 // String and Memory Library Call Optimizations
 //===----------------------------------------------------------------------===//
@@ -215,7 +228,7 @@ Value *LibCallSimplifier::optimizeStrCat(CallInst *CI, IRBuilderBase &B) {
   if (Len == 0)
     return Dst;
 
-  return emitStrLenMemCpy(Src, Dst, Len, B);
+  return copyFlags(*CI, emitStrLenMemCpy(Src, Dst, Len, B));
 }
 
 Value *LibCallSimplifier::emitStrLenMemCpy(Value *Src, Value *Dst, uint64_t Len,
@@ -279,7 +292,7 @@ Value *LibCallSimplifier::optimizeStrNCat(CallInst *CI, IRBuilderBase &B) {
 
   // strncat(x, s, c) -> strcat(x, s)
   // s is constant so the strcat can be optimized further.
-  return emitStrLenMemCpy(Src, Dst, SrcLen, B);
+  return copyFlags(*CI, emitStrLenMemCpy(Src, Dst, SrcLen, B));
 }
 
 Value *LibCallSimplifier::optimizeStrChr(CallInst *CI, IRBuilderBase &B) {
@@ -300,9 +313,11 @@ Value *LibCallSimplifier::optimizeStrChr(CallInst *CI, IRBuilderBase &B) {
     if (!FT->getParamType(1)->isIntegerTy(32)) // memchr needs i32.
       return nullptr;
 
-    return emitMemChr(SrcStr, CI->getArgOperand(1), // include nul.
-                      ConstantInt::get(DL.getIntPtrType(CI->getContext()), Len),
-                      B, DL, TLI);
+    return copyFlags(
+        *CI,
+        emitMemChr(SrcStr, CI->getArgOperand(1), // include nul.
+                   ConstantInt::get(DL.getIntPtrType(CI->getContext()), Len), B,
+                   DL, TLI));
   }
 
   // Otherwise, the character is a constant, see if the first argument is
@@ -340,7 +355,7 @@ Value *LibCallSimplifier::optimizeStrRChr(CallInst *CI, IRBuilderBase &B) {
   if (!getConstantStringInfo(SrcStr, Str)) {
     // strrchr(s, 0) -> strchr(s, 0)
     if (CharC->isZero())
-      return emitStrChr(SrcStr, '\0', B, TLI);
+      return copyFlags(*CI, emitStrChr(SrcStr, '\0', B, TLI));
     return nullptr;
   }
 
@@ -385,25 +400,28 @@ Value *LibCallSimplifier::optimizeStrCmp(CallInst *CI, IRBuilderBase &B) {
     annotateDereferenceableBytes(CI, 1, Len2);
 
   if (Len1 && Len2) {
-    return emitMemCmp(Str1P, Str2P,
-                      ConstantInt::get(DL.getIntPtrType(CI->getContext()),
-                                       std::min(Len1, Len2)),
-                      B, DL, TLI);
+    return copyFlags(
+        *CI, emitMemCmp(Str1P, Str2P,
+                        ConstantInt::get(DL.getIntPtrType(CI->getContext()),
+                                         std::min(Len1, Len2)),
+                        B, DL, TLI));
   }
 
   // strcmp to memcmp
   if (!HasStr1 && HasStr2) {
     if (canTransformToMemCmp(CI, Str1P, Len2, DL))
-      return emitMemCmp(
-          Str1P, Str2P,
-          ConstantInt::get(DL.getIntPtrType(CI->getContext()), Len2), B, DL,
-          TLI);
+      return copyFlags(
+          *CI,
+          emitMemCmp(Str1P, Str2P,
+                     ConstantInt::get(DL.getIntPtrType(CI->getContext()), Len2),
+                     B, DL, TLI));
   } else if (HasStr1 && !HasStr2) {
     if (canTransformToMemCmp(CI, Str2P, Len1, DL))
-      return emitMemCmp(
-          Str1P, Str2P,
-          ConstantInt::get(DL.getIntPtrType(CI->getContext()), Len1), B, DL,
-          TLI);
+      return copyFlags(
+          *CI,
+          emitMemCmp(Str1P, Str2P,
+                     ConstantInt::get(DL.getIntPtrType(CI->getContext()), Len1),
+                     B, DL, TLI));
   }
 
   annotateNonNullNoUndefBasedOnAccess(CI, {0, 1});
@@ -430,7 +448,7 @@ Value *LibCallSimplifier::optimizeStrNCmp(CallInst *CI, IRBuilderBase &B) {
     return ConstantInt::get(CI->getType(), 0);
 
   if (Length == 1) // strncmp(x,y,1) -> memcmp(x,y,1)
-    return emitMemCmp(Str1P, Str2P, Size, B, DL, TLI);
+    return copyFlags(*CI, emitMemCmp(Str1P, Str2P, Size, B, DL, TLI));
 
   StringRef Str1, Str2;
   bool HasStr1 = getConstantStringInfo(Str1P, Str1);
@@ -462,17 +480,19 @@ Value *LibCallSimplifier::optimizeStrNCmp(CallInst *CI, IRBuilderBase &B) {
   if (!HasStr1 && HasStr2) {
     Len2 = std::min(Len2, Length);
     if (canTransformToMemCmp(CI, Str1P, Len2, DL))
-      return emitMemCmp(
-          Str1P, Str2P,
-          ConstantInt::get(DL.getIntPtrType(CI->getContext()), Len2), B, DL,
-          TLI);
+      return copyFlags(
+          *CI,
+          emitMemCmp(Str1P, Str2P,
+                     ConstantInt::get(DL.getIntPtrType(CI->getContext()), Len2),
+                     B, DL, TLI));
   } else if (HasStr1 && !HasStr2) {
     Len1 = std::min(Len1, Length);
     if (canTransformToMemCmp(CI, Str2P, Len1, DL))
-      return emitMemCmp(
-          Str1P, Str2P,
-          ConstantInt::get(DL.getIntPtrType(CI->getContext()), Len1), B, DL,
-          TLI);
+      return copyFlags(
+          *CI,
+          emitMemCmp(Str1P, Str2P,
+                     ConstantInt::get(DL.getIntPtrType(CI->getContext()), Len1),
+                     B, DL, TLI));
   }
 
   return nullptr;
@@ -485,7 +505,7 @@ Value *LibCallSimplifier::optimizeStrNDup(CallInst *CI, IRBuilderBase &B) {
   if (SrcLen && Size) {
     annotateDereferenceableBytes(CI, 0, SrcLen);
     if (SrcLen <= Size->getZExtValue() + 1)
-      return emitStrDup(Src, B, TLI);
+      return copyFlags(*CI, emitStrDup(Src, B, TLI));
   }
 
   return nullptr;
@@ -495,7 +515,7 @@ Value *LibCallSimplifier::optimizeStrCpy(CallInst *CI, IRBuilderBase &B) {
   Value *Dst = CI->getArgOperand(0), *Src = CI->getArgOperand(1);
   if (Dst == Src) // strcpy(x,x)  -> x
     return Src;
-  
+
   annotateNonNullNoUndefBasedOnAccess(CI, {0, 1});
   // See if we can get the length of the input string.
   uint64_t Len = GetStringLength(Src);
@@ -511,6 +531,7 @@ Value *LibCallSimplifier::optimizeStrCpy(CallInst *CI, IRBuilderBase &B) {
                      ConstantInt::get(DL.getIntPtrType(CI->getContext()), Len));
   NewCI->setAttributes(CI->getAttributes());
   NewCI->removeRetAttrs(AttributeFuncs::typeIncompatible(NewCI->getType()));
+  copyFlags(*CI, NewCI);
   return Dst;
 }
 
@@ -520,7 +541,7 @@ Value *LibCallSimplifier::optimizeStpCpy(CallInst *CI, IRBuilderBase &B) {
 
   // stpcpy(d,s) -> strcpy(d,s) if the result is not used.
   if (CI->use_empty())
-    return emitStrCpy(Dst, Src, B, TLI);
+    return copyFlags(*CI, emitStrCpy(Dst, Src, B, TLI));
 
   if (Dst == Src) { // stpcpy(x,x)  -> x+strlen(x)
     Value *StrLen = emitStrLen(Src, B, DL, TLI);
@@ -544,6 +565,7 @@ Value *LibCallSimplifier::optimizeStpCpy(CallInst *CI, IRBuilderBase &B) {
   CallInst *NewCI = B.CreateMemCpy(Dst, Align(1), Src, Align(1), LenV);
   NewCI->setAttributes(CI->getAttributes());
   NewCI->removeRetAttrs(AttributeFuncs::typeIncompatible(NewCI->getType()));
+  copyFlags(*CI, NewCI);
   return DstEnd;
 }
 
@@ -583,6 +605,7 @@ Value *LibCallSimplifier::optimizeStrNCpy(CallInst *CI, IRBuilderBase &B) {
     AttrBuilder ArgAttrs(CI->getAttributes().getParamAttrs(0));
     NewCI->setAttributes(NewCI->getAttributes().addParamAttributes(
         CI->getContext(), 0, ArgAttrs));
+    copyFlags(*CI, NewCI);
     return Dst;
   }
 
@@ -606,6 +629,7 @@ Value *LibCallSimplifier::optimizeStrNCpy(CallInst *CI, IRBuilderBase &B) {
                                    ConstantInt::get(DL.getIntPtrType(PT), Len));
   NewCI->setAttributes(CI->getAttributes());
   NewCI->removeRetAttrs(AttributeFuncs::typeIncompatible(NewCI->getType()));
+  copyFlags(*CI, NewCI);
   return Dst;
 }
 
@@ -737,7 +761,7 @@ Value *LibCallSimplifier::optimizeStrPBrk(CallInst *CI, IRBuilderBase &B) {
 
   // strpbrk(s, "a") -> strchr(s, 'a')
   if (HasS2 && S2.size() == 1)
-    return emitStrChr(CI->getArgOperand(0), S2[0], B, TLI);
+    return copyFlags(*CI, emitStrChr(CI->getArgOperand(0), S2[0], B, TLI));
 
   return nullptr;
 }
@@ -793,7 +817,7 @@ Value *LibCallSimplifier::optimizeStrCSpn(CallInst *CI, IRBuilderBase &B) {
 
   // strcspn(s, "") -> strlen(s)
   if (HasS2 && S2.empty())
-    return emitStrLen(CI->getArgOperand(0), B, DL, TLI);
+    return copyFlags(*CI, emitStrLen(CI->getArgOperand(0), B, DL, TLI));
 
   return nullptr;
 }
@@ -1062,7 +1086,7 @@ Value *LibCallSimplifier::optimizeMemCmp(CallInst *CI, IRBuilderBase &B) {
     Value *LHS = CI->getArgOperand(0);
     Value *RHS = CI->getArgOperand(1);
     Value *Size = CI->getArgOperand(2);
-    return emitBCmp(LHS, RHS, Size, B, DL, TLI);
+    return copyFlags(*CI, emitBCmp(LHS, RHS, Size, B, DL, TLI));
   }
 
   return nullptr;
@@ -1083,6 +1107,7 @@ Value *LibCallSimplifier::optimizeMemCpy(CallInst *CI, IRBuilderBase &B) {
                                    CI->getArgOperand(1), Align(1), Size);
   NewCI->setAttributes(CI->getAttributes());
   NewCI->removeRetAttrs(AttributeFuncs::typeIncompatible(NewCI->getType()));
+  copyFlags(*CI, NewCI);
   return CI->getArgOperand(0);
 }
 
@@ -1110,7 +1135,8 @@ Value *LibCallSimplifier::optimizeMemCCpy(CallInst *CI, IRBuilderBase &B) {
   size_t Pos = SrcStr.find(StopChar->getSExtValue() & 0xFF);
   if (Pos == StringRef::npos) {
     if (N->getZExtValue() <= SrcStr.size()) {
-      B.CreateMemCpy(Dst, Align(1), Src, Align(1), CI->getArgOperand(3));
+      copyFlags(*CI, B.CreateMemCpy(Dst, Align(1), Src, Align(1),
+                                    CI->getArgOperand(3)));
       return Constant::getNullValue(CI->getType());
     }
     return nullptr;
@@ -1119,7 +1145,7 @@ Value *LibCallSimplifier::optimizeMemCCpy(CallInst *CI, IRBuilderBase &B) {
   Value *NewN =
       ConstantInt::get(N->getType(), std::min(uint64_t(Pos + 1), N->getZExtValue()));
   // memccpy -> llvm.memcpy
-  B.CreateMemCpy(Dst, Align(1), Src, Align(1), NewN);
+  copyFlags(*CI, B.CreateMemCpy(Dst, Align(1), Src, Align(1), NewN));
   return Pos + 1 <= N->getZExtValue()
              ? B.CreateInBoundsGEP(B.getInt8Ty(), Dst, NewN)
              : Constant::getNullValue(CI->getType());
@@ -1136,6 +1162,7 @@ Value *LibCallSimplifier::optimizeMemPCpy(CallInst *CI, IRBuilderBase &B) {
   // TODO: Attach return value attributes to the 1st operand to preserve them?
   NewCI->setAttributes(CI->getAttributes());
   NewCI->removeRetAttrs(AttributeFuncs::typeIncompatible(NewCI->getType()));
+  copyFlags(*CI, NewCI);
   return B.CreateInBoundsGEP(B.getInt8Ty(), Dst, N);
 }
 
@@ -1150,6 +1177,7 @@ Value *LibCallSimplifier::optimizeMemMove(CallInst *CI, IRBuilderBase &B) {
                                     CI->getArgOperand(1), Align(1), Size);
   NewCI->setAttributes(CI->getAttributes());
   NewCI->removeRetAttrs(AttributeFuncs::typeIncompatible(NewCI->getType()));
+  copyFlags(*CI, NewCI);
   return CI->getArgOperand(0);
 }
 
@@ -1164,12 +1192,13 @@ Value *LibCallSimplifier::optimizeMemSet(CallInst *CI, IRBuilderBase &B) {
   CallInst *NewCI = B.CreateMemSet(CI->getArgOperand(0), Val, Size, Align(1));
   NewCI->setAttributes(CI->getAttributes());
   NewCI->removeRetAttrs(AttributeFuncs::typeIncompatible(NewCI->getType()));
+  copyFlags(*CI, NewCI);
   return CI->getArgOperand(0);
 }
 
 Value *LibCallSimplifier::optimizeRealloc(CallInst *CI, IRBuilderBase &B) {
   if (isa<ConstantPointerNull>(CI->getArgOperand(0)))
-    return emitMalloc(CI->getArgOperand(1), B, DL, TLI);
+    return copyFlags(*CI, emitMalloc(CI->getArgOperand(1), B, DL, TLI));
 
   return nullptr;
 }
@@ -1190,7 +1219,7 @@ static Value *replaceUnaryCall(CallInst *CI, IRBuilderBase &B,
   Function *F = Intrinsic::getDeclaration(M, IID, CI->getType());
   CallInst *NewCall = B.CreateCall(F, V);
   NewCall->takeName(CI);
-  return NewCall;
+  return copyFlags(*CI, NewCall);
 }
 
 /// Return a variant of Val with float type.
@@ -1311,7 +1340,8 @@ Value *LibCallSimplifier::optimizeCAbs(CallInst *CI, IRBuilderBase &B) {
 
   Function *FSqrt = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::sqrt,
                                               CI->getType());
-  return B.CreateCall(FSqrt, B.CreateFAdd(RealReal, ImagImag), "cabs");
+  return copyFlags(
+      *CI, B.CreateCall(FSqrt, B.CreateFAdd(RealReal, ImagImag), "cabs"));
 }
 
 static Value *optimizeTrigReflections(CallInst *Call, LibFunc Func,
@@ -1334,14 +1364,16 @@ static Value *optimizeTrigReflections(CallInst *Call, LibFunc Func,
     // sin(-X) --> -sin(X)
     // tan(-X) --> -tan(X)
     if (match(Call->getArgOperand(0), m_OneUse(m_FNeg(m_Value(X)))))
-      return B.CreateFNeg(B.CreateCall(Call->getCalledFunction(), X));
+      return B.CreateFNeg(
+          copyFlags(*Call, B.CreateCall(Call->getCalledFunction(), X)));
     break;
   case LibFunc_cos:
   case LibFunc_cosf:
   case LibFunc_cosl:
     // cos(-X) --> cos(X)
     if (match(Call->getArgOperand(0), m_FNeg(m_Value(X))))
-      return B.CreateCall(Call->getCalledFunction(), X, "cos");
+      return copyFlags(*Call,
+                       B.CreateCall(Call->getCalledFunction(), X, "cos"));
     break;
   default:
     break;
@@ -1476,9 +1508,10 @@ Value *LibCallSimplifier::replacePowWithExp(CallInst *Pow, IRBuilderBase &B) {
       (isa<SIToFPInst>(Expo) || isa<UIToFPInst>(Expo)) &&
       hasFloatFn(TLI, Ty, LibFunc_ldexp, LibFunc_ldexpf, LibFunc_ldexpl)) {
     if (Value *ExpoI = getIntToFPVal(Expo, B, TLI->getIntSize()))
-      return emitBinaryFloatFnCall(ConstantFP::get(Ty, 1.0), ExpoI, TLI,
-                                   LibFunc_ldexp, LibFunc_ldexpf, LibFunc_ldexpl,
-                                   B, Attrs);
+      return copyFlags(*Pow,
+                       emitBinaryFloatFnCall(ConstantFP::get(Ty, 1.0), ExpoI,
+                                             TLI, LibFunc_ldexp, LibFunc_ldexpf,
+                                             LibFunc_ldexpl, B, Attrs));
   }
 
   // pow(2.0 ** n, x) -> exp2(n * x)
@@ -1496,11 +1529,13 @@ Value *LibCallSimplifier::replacePowWithExp(CallInst *Pow, IRBuilderBase &B) {
       double N = NI.logBase2() * (IsReciprocal ? -1.0 : 1.0);
       Value *FMul = B.CreateFMul(Expo, ConstantFP::get(Ty, N), "mul");
       if (Pow->doesNotAccessMemory())
-        return B.CreateCall(Intrinsic::getDeclaration(Mod, Intrinsic::exp2, Ty),
-                            FMul, "exp2");
+        return copyFlags(*Pow, B.CreateCall(Intrinsic::getDeclaration(
+                                                Mod, Intrinsic::exp2, Ty),
+                                            FMul, "exp2"));
       else
-        return emitUnaryFloatFnCall(FMul, TLI, LibFunc_exp2, LibFunc_exp2f,
-                                    LibFunc_exp2l, B, Attrs);
+        return copyFlags(*Pow, emitUnaryFloatFnCall(FMul, TLI, LibFunc_exp2,
+                                                    LibFunc_exp2f,
+                                                    LibFunc_exp2l, B, Attrs));
     }
   }
 
@@ -1508,8 +1543,9 @@ Value *LibCallSimplifier::replacePowWithExp(CallInst *Pow, IRBuilderBase &B) {
   // TODO: There is no exp10() intrinsic yet, but some day there shall be one.
   if (match(Base, m_SpecificFP(10.0)) &&
       hasFloatFn(TLI, Ty, LibFunc_exp10, LibFunc_exp10f, LibFunc_exp10l))
-    return emitUnaryFloatFnCall(Expo, TLI, LibFunc_exp10, LibFunc_exp10f,
-                                LibFunc_exp10l, B, Attrs);
+    return copyFlags(*Pow, emitUnaryFloatFnCall(Expo, TLI, LibFunc_exp10,
+                                                LibFunc_exp10f, LibFunc_exp10l,
+                                                B, Attrs));
 
   // pow(x, y) -> exp2(log2(x) * y)
   if (Pow->hasApproxFunc() && Pow->hasNoNaNs() && BaseF->isFiniteNonZero() &&
@@ -1528,11 +1564,13 @@ Value *LibCallSimplifier::replacePowWithExp(CallInst *Pow, IRBuilderBase &B) {
     if (Log) {
       Value *FMul = B.CreateFMul(Log, Expo, "mul");
       if (Pow->doesNotAccessMemory())
-        return B.CreateCall(Intrinsic::getDeclaration(Mod, Intrinsic::exp2, Ty),
-                            FMul, "exp2");
+        return copyFlags(*Pow, B.CreateCall(Intrinsic::getDeclaration(
+                                                Mod, Intrinsic::exp2, Ty),
+                                            FMul, "exp2"));
       else if (hasFloatFn(TLI, Ty, LibFunc_exp2, LibFunc_exp2f, LibFunc_exp2l))
-        return emitUnaryFloatFnCall(FMul, TLI, LibFunc_exp2, LibFunc_exp2f,
-                                    LibFunc_exp2l, B, Attrs);
+        return copyFlags(*Pow, emitUnaryFloatFnCall(FMul, TLI, LibFunc_exp2,
+                                                    LibFunc_exp2f,
+                                                    LibFunc_exp2l, B, Attrs));
     }
   }
 
@@ -1595,6 +1633,8 @@ Value *LibCallSimplifier::replacePowWithSqrt(CallInst *Pow, IRBuilderBase &B) {
     Sqrt = B.CreateCall(FAbsFn, Sqrt, "abs");
   }
 
+  Sqrt = copyFlags(*Pow, Sqrt);
+
   // Handle non finite base by expanding to
   // (x == -infinity ? +infinity : sqrt(x)).
   if (!Pow->hasNoInfs()) {
@@ -1721,15 +1761,18 @@ Value *LibCallSimplifier::optimizePow(CallInst *Pow, IRBuilderBase &B) {
     if (ExpoF->isInteger() &&
         ExpoF->convertToInteger(IntExpo, APFloat::rmTowardZero, &Ignored) ==
             APFloat::opOK) {
-      return createPowWithIntegerExponent(
-          Base, ConstantInt::get(B.getIntNTy(TLI->getIntSize()), IntExpo), M, B);
+      return copyFlags(
+          *Pow,
+          createPowWithIntegerExponent(
+              Base, ConstantInt::get(B.getIntNTy(TLI->getIntSize()), IntExpo),
+              M, B));
     }
   }
 
   // powf(x, itofp(y)) -> powi(x, y)
   if (AllowApprox && (isa<SIToFPInst>(Expo) || isa<UIToFPInst>(Expo))) {
     if (Value *ExpoI = getIntToFPVal(Expo, B, TLI->getIntSize()))
-      return createPowWithIntegerExponent(Base, ExpoI, M, B);
+      return copyFlags(*Pow, createPowWithIntegerExponent(Base, ExpoI, M, B));
   }
 
   // Shrink pow() to powf() if the arguments are single precision,
@@ -1792,7 +1835,8 @@ Value *LibCallSimplifier::optimizeFMinFMax(CallInst *CI, IRBuilderBase &B) {
   Intrinsic::ID IID = Callee->getName().startswith("fmin") ? Intrinsic::minnum
                                                            : Intrinsic::maxnum;
   Function *F = Intrinsic::getDeclaration(CI->getModule(), IID, CI->getType());
-  return B.CreateCall(F, { CI->getArgOperand(0), CI->getArgOperand(1) });
+  return copyFlags(
+      *CI, B.CreateCall(F, {CI->getArgOperand(0), CI->getArgOperand(1)}));
 }
 
 Value *LibCallSimplifier::optimizeLog(CallInst *Log, IRBuilderBase &B) {
@@ -2010,9 +2054,9 @@ Value *LibCallSimplifier::optimizeSqrt(CallInst *CI, IRBuilderBase &B) {
     // of the square root calculation.
     Function *Sqrt = Intrinsic::getDeclaration(M, Intrinsic::sqrt, ArgType);
     Value *SqrtCall = B.CreateCall(Sqrt, OtherOp, "sqrt");
-    return B.CreateFMul(FabsCall, SqrtCall);
+    return copyFlags(*CI, B.CreateFMul(FabsCall, SqrtCall));
   }
-  return FabsCall;
+  return copyFlags(*CI, FabsCall);
 }
 
 // TODO: Generalize to handle any trig function and its inverse.
@@ -2327,7 +2371,7 @@ Value *LibCallSimplifier::optimizePrintFString(CallInst *CI, IRBuilderBase &B) {
 
   // printf("x") -> putchar('x'), even for "%" and "%%".
   if (FormatStr.size() == 1 || FormatStr == "%%")
-    return emitPutChar(B.getInt32(FormatStr[0]), B, TLI);
+    return copyFlags(*CI, emitPutChar(B.getInt32(FormatStr[0]), B, TLI));
 
   // Try to remove call or emit putchar/puts.
   if (FormatStr == "%s" && CI->arg_size() > 1) {
@@ -2339,12 +2383,12 @@ Value *LibCallSimplifier::optimizePrintFString(CallInst *CI, IRBuilderBase &B) {
       return (Value *)CI;
     // printf("%s", "a") --> putchar('a')
     if (OperandStr.size() == 1)
-      return emitPutChar(B.getInt32(OperandStr[0]), B, TLI);
+      return copyFlags(*CI, emitPutChar(B.getInt32(OperandStr[0]), B, TLI));
     // printf("%s", str"\n") --> puts(str)
     if (OperandStr.back() == '\n') {
       OperandStr = OperandStr.drop_back();
       Value *GV = B.CreateGlobalString(OperandStr, "str");
-      return emitPutS(GV, B, TLI);
+      return copyFlags(*CI, emitPutS(GV, B, TLI));
     }
     return nullptr;
   }
@@ -2356,19 +2400,19 @@ Value *LibCallSimplifier::optimizePrintFString(CallInst *CI, IRBuilderBase &B) {
     // pass to be run after this pass, to merge duplicate strings.
     FormatStr = FormatStr.drop_back();
     Value *GV = B.CreateGlobalString(FormatStr, "str");
-    return emitPutS(GV, B, TLI);
+    return copyFlags(*CI, emitPutS(GV, B, TLI));
   }
 
   // Optimize specific format strings.
   // printf("%c", chr) --> putchar(chr)
   if (FormatStr == "%c" && CI->arg_size() > 1 &&
       CI->getArgOperand(1)->getType()->isIntegerTy())
-    return emitPutChar(CI->getArgOperand(1), B, TLI);
+    return copyFlags(*CI, emitPutChar(CI->getArgOperand(1), B, TLI));
 
   // printf("%s\n", str) --> puts(str)
   if (FormatStr == "%s\n" && CI->arg_size() > 1 &&
       CI->getArgOperand(1)->getType()->isPointerTy())
-    return emitPutS(CI->getArgOperand(1), B, TLI);
+    return copyFlags(*CI, emitPutS(CI->getArgOperand(1), B, TLI));
   return nullptr;
 }
 
@@ -2459,7 +2503,7 @@ Value *LibCallSimplifier::optimizeSPrintFString(CallInst *CI,
 
     if (CI->use_empty())
       // sprintf(dest, "%s", str) -> strcpy(dest, str)
-      return emitStrCpy(Dest, CI->getArgOperand(2), B, TLI);
+      return copyFlags(*CI, emitStrCpy(Dest, CI->getArgOperand(2), B, TLI));
 
     uint64_t SrcLen = GetStringLength(CI->getArgOperand(2));
     if (SrcLen) {
@@ -2558,10 +2602,12 @@ Value *LibCallSimplifier::optimizeSnPrintFString(CallInst *CI,
 
     // snprintf(dst, size, fmt) -> llvm.memcpy(align 1 dst, align 1 fmt,
     // strlen(fmt)+1)
-    B.CreateMemCpy(
-        CI->getArgOperand(0), Align(1), CI->getArgOperand(2), Align(1),
-        ConstantInt::get(DL.getIntPtrType(CI->getContext()),
-                         FormatStr.size() + 1)); // Copy the null byte.
+    copyFlags(
+        *CI,
+        B.CreateMemCpy(
+            CI->getArgOperand(0), Align(1), CI->getArgOperand(2), Align(1),
+            ConstantInt::get(DL.getIntPtrType(CI->getContext()),
+                             FormatStr.size() + 1))); // Copy the null byte.
     return ConstantInt::get(CI->getType(), FormatStr.size());
   }
 
@@ -2599,8 +2645,10 @@ Value *LibCallSimplifier::optimizeSnPrintFString(CallInst *CI,
       else if (N < Str.size() + 1)
         return nullptr;
 
-      B.CreateMemCpy(CI->getArgOperand(0), Align(1), CI->getArgOperand(3),
-                     Align(1), ConstantInt::get(CI->getType(), Str.size() + 1));
+      copyFlags(
+          *CI, B.CreateMemCpy(CI->getArgOperand(0), Align(1),
+                              CI->getArgOperand(3), Align(1),
+                              ConstantInt::get(CI->getType(), Str.size() + 1)));
 
       // The snprintf result is the unincremented number of bytes in the string.
       return ConstantInt::get(CI->getType(), Str.size());
@@ -2640,10 +2688,11 @@ Value *LibCallSimplifier::optimizeFPrintFString(CallInst *CI,
     if (FormatStr.contains('%'))
       return nullptr; // We found a format specifier.
 
-    return emitFWrite(
-        CI->getArgOperand(1),
-        ConstantInt::get(DL.getIntPtrType(CI->getContext()), FormatStr.size()),
-        CI->getArgOperand(0), B, DL, TLI);
+    return copyFlags(
+        *CI, emitFWrite(CI->getArgOperand(1),
+                        ConstantInt::get(DL.getIntPtrType(CI->getContext()),
+                                         FormatStr.size()),
+                        CI->getArgOperand(0), B, DL, TLI));
   }
 
   // The remaining optimizations require the format string to be "%s" or "%c"
@@ -2656,14 +2705,16 @@ Value *LibCallSimplifier::optimizeFPrintFString(CallInst *CI,
     // fprintf(F, "%c", chr) --> fputc(chr, F)
     if (!CI->getArgOperand(2)->getType()->isIntegerTy())
       return nullptr;
-    return emitFPutC(CI->getArgOperand(2), CI->getArgOperand(0), B, TLI);
+    return copyFlags(
+        *CI, emitFPutC(CI->getArgOperand(2), CI->getArgOperand(0), B, TLI));
   }
 
   if (FormatStr[1] == 's') {
     // fprintf(F, "%s", str) --> fputs(str, F)
     if (!CI->getArgOperand(2)->getType()->isPointerTy())
       return nullptr;
-    return emitFPutS(CI->getArgOperand(2), CI->getArgOperand(0), B, TLI);
+    return copyFlags(
+        *CI, emitFPutS(CI->getArgOperand(2), CI->getArgOperand(0), B, TLI));
   }
   return nullptr;
 }
@@ -2750,10 +2801,11 @@ Value *LibCallSimplifier::optimizeFPuts(CallInst *CI, IRBuilderBase &B) {
     return nullptr;
 
   // Known to have no uses (see above).
-  return emitFWrite(
-      CI->getArgOperand(0),
-      ConstantInt::get(DL.getIntPtrType(CI->getContext()), Len - 1),
-      CI->getArgOperand(1), B, DL, TLI);
+  return copyFlags(
+      *CI,
+      emitFWrite(CI->getArgOperand(0),
+                 ConstantInt::get(DL.getIntPtrType(CI->getContext()), Len - 1),
+                 CI->getArgOperand(1), B, DL, TLI));
 }
 
 Value *LibCallSimplifier::optimizePuts(CallInst *CI, IRBuilderBase &B) {
@@ -2765,15 +2817,16 @@ Value *LibCallSimplifier::optimizePuts(CallInst *CI, IRBuilderBase &B) {
   // puts("") -> putchar('\n')
   StringRef Str;
   if (getConstantStringInfo(CI->getArgOperand(0), Str) && Str.empty())
-    return emitPutChar(B.getInt32('\n'), B, TLI);
+    return copyFlags(*CI, emitPutChar(B.getInt32('\n'), B, TLI));
 
   return nullptr;
 }
 
 Value *LibCallSimplifier::optimizeBCopy(CallInst *CI, IRBuilderBase &B) {
   // bcopy(src, dst, n) -> llvm.memmove(dst, src, n)
-  return B.CreateMemMove(CI->getArgOperand(1), Align(1), CI->getArgOperand(0),
-                         Align(1), CI->getArgOperand(2));
+  return copyFlags(*CI, B.CreateMemMove(CI->getArgOperand(1), Align(1),
+                                        CI->getArgOperand(0), Align(1),
+                                        CI->getArgOperand(2)));
 }
 
 bool LibCallSimplifier::hasFloatVersion(StringRef FuncName) {
@@ -2971,6 +3024,8 @@ Value *LibCallSimplifier::optimizeFloatingPointLibCall(CallInst *CI,
 }
 
 Value *LibCallSimplifier::optimizeCall(CallInst *CI, IRBuilderBase &Builder) {
+  assert(!CI->isMustTailCall() && "These transforms aren't musttail safe.");
+
   // TODO: Split out the code below that operates on FP calls so that
   //       we can all non-FP calls with the StrictFP attribute to be
   //       optimized.
@@ -3212,6 +3267,7 @@ Value *FortifiedLibCallSimplifier::optimizeMemCpyChk(CallInst *CI,
                        Align(1), CI->getArgOperand(2));
     NewCI->setAttributes(CI->getAttributes());
     NewCI->removeRetAttrs(AttributeFuncs::typeIncompatible(NewCI->getType()));
+    copyFlags(*CI, NewCI);
     return CI->getArgOperand(0);
   }
   return nullptr;
@@ -3225,6 +3281,7 @@ Value *FortifiedLibCallSimplifier::optimizeMemMoveChk(CallInst *CI,
                         Align(1), CI->getArgOperand(2));
     NewCI->setAttributes(CI->getAttributes());
     NewCI->removeRetAttrs(AttributeFuncs::typeIncompatible(NewCI->getType()));
+    copyFlags(*CI, NewCI);
     return CI->getArgOperand(0);
   }
   return nullptr;
@@ -3238,6 +3295,7 @@ Value *FortifiedLibCallSimplifier::optimizeMemSetChk(CallInst *CI,
                                      CI->getArgOperand(2), Align(1));
     NewCI->setAttributes(CI->getAttributes());
     NewCI->removeRetAttrs(AttributeFuncs::typeIncompatible(NewCI->getType()));
+    copyFlags(*CI, NewCI);
     return CI->getArgOperand(0);
   }
   return nullptr;
@@ -3252,7 +3310,7 @@ Value *FortifiedLibCallSimplifier::optimizeMemPCpyChk(CallInst *CI,
       CallInst *NewCI = cast<CallInst>(Call);
       NewCI->setAttributes(CI->getAttributes());
       NewCI->removeRetAttrs(AttributeFuncs::typeIncompatible(NewCI->getType()));
-      return NewCI;
+      return copyFlags(*CI, NewCI);
     }
   return nullptr;
 }
@@ -3277,9 +3335,9 @@ Value *FortifiedLibCallSimplifier::optimizeStrpCpyChk(CallInst *CI,
   // string lengths for varying.
   if (isFortifiedCallFoldable(CI, 2, None, 1)) {
     if (Func == LibFunc_strcpy_chk)
-      return emitStrCpy(Dst, Src, B, TLI);
+      return copyFlags(*CI, emitStrCpy(Dst, Src, B, TLI));
     else
-      return emitStpCpy(Dst, Src, B, TLI);
+      return copyFlags(*CI, emitStpCpy(Dst, Src, B, TLI));
   }
 
   if (OnlyLowerUnknownSize)
@@ -3303,14 +3361,14 @@ Value *FortifiedLibCallSimplifier::optimizeStrpCpyChk(CallInst *CI,
   // a __memcpy_chk, we still need to return the correct end pointer.
   if (Ret && Func == LibFunc_stpcpy_chk)
     return B.CreateGEP(B.getInt8Ty(), Dst, ConstantInt::get(SizeTTy, Len - 1));
-  return Ret;
+  return copyFlags(*CI, cast<CallInst>(Ret));
 }
 
 Value *FortifiedLibCallSimplifier::optimizeStrLenChk(CallInst *CI,
                                                      IRBuilderBase &B) {
   if (isFortifiedCallFoldable(CI, 1, None, 0))
-    return emitStrLen(CI->getArgOperand(0), B, CI->getModule()->getDataLayout(),
-                      TLI);
+    return copyFlags(*CI, emitStrLen(CI->getArgOperand(0), B,
+                                     CI->getModule()->getDataLayout(), TLI));
   return nullptr;
 }
 
@@ -3319,11 +3377,13 @@ Value *FortifiedLibCallSimplifier::optimizeStrpNCpyChk(CallInst *CI,
                                                        LibFunc Func) {
   if (isFortifiedCallFoldable(CI, 3, 2)) {
     if (Func == LibFunc_strncpy_chk)
-      return emitStrNCpy(CI->getArgOperand(0), CI->getArgOperand(1),
-                               CI->getArgOperand(2), B, TLI);
+      return copyFlags(*CI,
+                       emitStrNCpy(CI->getArgOperand(0), CI->getArgOperand(1),
+                                   CI->getArgOperand(2), B, TLI));
     else
-      return emitStpNCpy(CI->getArgOperand(0), CI->getArgOperand(1),
-                         CI->getArgOperand(2), B, TLI);
+      return copyFlags(*CI,
+                       emitStpNCpy(CI->getArgOperand(0), CI->getArgOperand(1),
+                                   CI->getArgOperand(2), B, TLI));
   }
 
   return nullptr;
@@ -3332,8 +3392,9 @@ Value *FortifiedLibCallSimplifier::optimizeStrpNCpyChk(CallInst *CI,
 Value *FortifiedLibCallSimplifier::optimizeMemCCpyChk(CallInst *CI,
                                                       IRBuilderBase &B) {
   if (isFortifiedCallFoldable(CI, 4, 3))
-    return emitMemCCpy(CI->getArgOperand(0), CI->getArgOperand(1),
-                       CI->getArgOperand(2), CI->getArgOperand(3), B, TLI);
+    return copyFlags(
+        *CI, emitMemCCpy(CI->getArgOperand(0), CI->getArgOperand(1),
+                         CI->getArgOperand(2), CI->getArgOperand(3), B, TLI));
 
   return nullptr;
 }
@@ -3342,8 +3403,9 @@ Value *FortifiedLibCallSimplifier::optimizeSNPrintfChk(CallInst *CI,
                                                        IRBuilderBase &B) {
   if (isFortifiedCallFoldable(CI, 3, 1, None, 2)) {
     SmallVector<Value *, 8> VariadicArgs(drop_begin(CI->args(), 5));
-    return emitSNPrintf(CI->getArgOperand(0), CI->getArgOperand(1),
-                        CI->getArgOperand(4), VariadicArgs, B, TLI);
+    return copyFlags(*CI,
+                     emitSNPrintf(CI->getArgOperand(0), CI->getArgOperand(1),
+                                  CI->getArgOperand(4), VariadicArgs, B, TLI));
   }
 
   return nullptr;
@@ -3353,8 +3415,9 @@ Value *FortifiedLibCallSimplifier::optimizeSPrintfChk(CallInst *CI,
                                                       IRBuilderBase &B) {
   if (isFortifiedCallFoldable(CI, 2, None, None, 1)) {
     SmallVector<Value *, 8> VariadicArgs(drop_begin(CI->args(), 4));
-    return emitSPrintf(CI->getArgOperand(0), CI->getArgOperand(3), VariadicArgs,
-                       B, TLI);
+    return copyFlags(*CI,
+                     emitSPrintf(CI->getArgOperand(0), CI->getArgOperand(3),
+                                 VariadicArgs, B, TLI));
   }
 
   return nullptr;
@@ -3363,7 +3426,8 @@ Value *FortifiedLibCallSimplifier::optimizeSPrintfChk(CallInst *CI,
 Value *FortifiedLibCallSimplifier::optimizeStrCatChk(CallInst *CI,
                                                      IRBuilderBase &B) {
   if (isFortifiedCallFoldable(CI, 2))
-    return emitStrCat(CI->getArgOperand(0), CI->getArgOperand(1), B, TLI);
+    return copyFlags(
+        *CI, emitStrCat(CI->getArgOperand(0), CI->getArgOperand(1), B, TLI));
 
   return nullptr;
 }
@@ -3371,8 +3435,9 @@ Value *FortifiedLibCallSimplifier::optimizeStrCatChk(CallInst *CI,
 Value *FortifiedLibCallSimplifier::optimizeStrLCat(CallInst *CI,
                                                    IRBuilderBase &B) {
   if (isFortifiedCallFoldable(CI, 3))
-    return emitStrLCat(CI->getArgOperand(0), CI->getArgOperand(1),
-                       CI->getArgOperand(2), B, TLI);
+    return copyFlags(*CI,
+                     emitStrLCat(CI->getArgOperand(0), CI->getArgOperand(1),
+                                 CI->getArgOperand(2), B, TLI));
 
   return nullptr;
 }
@@ -3380,8 +3445,9 @@ Value *FortifiedLibCallSimplifier::optimizeStrLCat(CallInst *CI,
 Value *FortifiedLibCallSimplifier::optimizeStrNCatChk(CallInst *CI,
                                                       IRBuilderBase &B) {
   if (isFortifiedCallFoldable(CI, 3))
-    return emitStrNCat(CI->getArgOperand(0), CI->getArgOperand(1),
-                       CI->getArgOperand(2), B, TLI);
+    return copyFlags(*CI,
+                     emitStrNCat(CI->getArgOperand(0), CI->getArgOperand(1),
+                                 CI->getArgOperand(2), B, TLI));
 
   return nullptr;
 }
@@ -3389,8 +3455,9 @@ Value *FortifiedLibCallSimplifier::optimizeStrNCatChk(CallInst *CI,
 Value *FortifiedLibCallSimplifier::optimizeStrLCpyChk(CallInst *CI,
                                                       IRBuilderBase &B) {
   if (isFortifiedCallFoldable(CI, 3))
-    return emitStrLCpy(CI->getArgOperand(0), CI->getArgOperand(1),
-                       CI->getArgOperand(2), B, TLI);
+    return copyFlags(*CI,
+                     emitStrLCpy(CI->getArgOperand(0), CI->getArgOperand(1),
+                                 CI->getArgOperand(2), B, TLI));
 
   return nullptr;
 }
@@ -3398,8 +3465,9 @@ Value *FortifiedLibCallSimplifier::optimizeStrLCpyChk(CallInst *CI,
 Value *FortifiedLibCallSimplifier::optimizeVSNPrintfChk(CallInst *CI,
                                                         IRBuilderBase &B) {
   if (isFortifiedCallFoldable(CI, 3, 1, None, 2))
-    return emitVSNPrintf(CI->getArgOperand(0), CI->getArgOperand(1),
-                         CI->getArgOperand(4), CI->getArgOperand(5), B, TLI);
+    return copyFlags(
+        *CI, emitVSNPrintf(CI->getArgOperand(0), CI->getArgOperand(1),
+                           CI->getArgOperand(4), CI->getArgOperand(5), B, TLI));
 
   return nullptr;
 }
@@ -3407,8 +3475,9 @@ Value *FortifiedLibCallSimplifier::optimizeVSNPrintfChk(CallInst *CI,
 Value *FortifiedLibCallSimplifier::optimizeVSPrintfChk(CallInst *CI,
                                                        IRBuilderBase &B) {
   if (isFortifiedCallFoldable(CI, 2, None, None, 1))
-    return emitVSPrintf(CI->getArgOperand(0), CI->getArgOperand(3),
-                        CI->getArgOperand(4), B, TLI);
+    return copyFlags(*CI,
+                     emitVSPrintf(CI->getArgOperand(0), CI->getArgOperand(3),
+                                  CI->getArgOperand(4), B, TLI));
 
   return nullptr;
 }
diff --git a/llvm/lib/Transforms/Utils/ValueMapper.cpp b/llvm/lib/Transforms/Utils/ValueMapper.cpp
index c3eafd6b2492..b822db938af8 100644
--- a/llvm/lib/Transforms/Utils/ValueMapper.cpp
+++ b/llvm/lib/Transforms/Utils/ValueMapper.cpp
@@ -450,6 +450,12 @@ Value *Mapper::mapValue(const Value *V) {
                DSOLocalEquivalent::get(Func), NewTy);
   }
 
+  if (const auto *NC = dyn_cast<NoCFIValue>(C)) {
+    auto *Val = mapValue(NC->getGlobalValue());
+    GlobalValue *GV = cast<GlobalValue>(Val);
+    return getVM()[NC] = NoCFIValue::get(GV);
+  }
+
   auto mapValueOrNull = [this](Value *V) {
     auto Mapped = mapValue(V);
     assert((Mapped || (Flags & RF_NullMapMissingGlobalValues)) &&
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 805011191da0..81e5aa223c07 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -55,22 +55,23 @@ static cl::opt<unsigned> PragmaVectorizeSCEVCheckThreshold(
     cl::desc("The maximum number of SCEV checks allowed with a "
              "vectorize(enable) pragma"));
 
-// FIXME: When scalable vectorization is stable enough, change the default
-// to SK_PreferFixedWidth.
-static cl::opt<LoopVectorizeHints::ScalableForceKind> ScalableVectorization(
-    "scalable-vectorization", cl::init(LoopVectorizeHints::SK_FixedWidthOnly),
-    cl::Hidden,
-    cl::desc("Control whether the compiler can use scalable vectors to "
-             "vectorize a loop"),
-    cl::values(
-        clEnumValN(LoopVectorizeHints::SK_FixedWidthOnly, "off",
-                   "Scalable vectorization is disabled."),
-        clEnumValN(LoopVectorizeHints::SK_PreferFixedWidth, "on",
-                   "Scalable vectorization is available, but favor fixed-width "
-                   "vectorization when the cost is inconclusive."),
-        clEnumValN(LoopVectorizeHints::SK_PreferScalable, "preferred",
-                   "Scalable vectorization is available and favored when the "
-                   "cost is inconclusive.")));
+static cl::opt<LoopVectorizeHints::ScalableForceKind>
+    ForceScalableVectorization(
+        "scalable-vectorization", cl::init(LoopVectorizeHints::SK_Unspecified),
+        cl::Hidden,
+        cl::desc("Control whether the compiler can use scalable vectors to "
+                 "vectorize a loop"),
+        cl::values(
+            clEnumValN(LoopVectorizeHints::SK_FixedWidthOnly, "off",
+                       "Scalable vectorization is disabled."),
+            clEnumValN(
+                LoopVectorizeHints::SK_PreferScalable, "preferred",
+                "Scalable vectorization is available and favored when the "
+                "cost is inconclusive."),
+            clEnumValN(
+                LoopVectorizeHints::SK_PreferScalable, "on",
+                "Scalable vectorization is available and favored when the "
+                "cost is inconclusive.")));
 
 /// Maximum vectorization interleave count.
 static const unsigned MaxInterleaveFactor = 16;
@@ -95,7 +96,8 @@ bool LoopVectorizeHints::Hint::validate(unsigned Val) {
 
 LoopVectorizeHints::LoopVectorizeHints(const Loop *L,
                                        bool InterleaveOnlyWhenForced,
-                                       OptimizationRemarkEmitter &ORE)
+                                       OptimizationRemarkEmitter &ORE,
+                                       const TargetTransformInfo *TTI)
     : Width("vectorize.width", VectorizerParams::VectorizationFactor, HK_WIDTH),
       Interleave("interleave.count", InterleaveOnlyWhenForced, HK_INTERLEAVE),
       Force("vectorize.enable", FK_Undefined, HK_FORCE),
@@ -110,14 +112,32 @@ LoopVectorizeHints::LoopVectorizeHints(const Loop *L,
   if (VectorizerParams::isInterleaveForced())
     Interleave.Value = VectorizerParams::VectorizationInterleave;
 
+  // If the metadata doesn't explicitly specify whether to enable scalable
+  // vectorization, then decide based on the following criteria (increasing
+  // level of priority):
+  //  - Target default
+  //  - Metadata width
+  //  - Force option (always overrides)
+  if ((LoopVectorizeHints::ScalableForceKind)Scalable.Value == SK_Unspecified) {
+    if (TTI)
+      Scalable.Value = TTI->enableScalableVectorization() ? SK_PreferScalable
+                                                          : SK_FixedWidthOnly;
+
+    if (Width.Value)
+      // If the width is set, but the metadata says nothing about the scalable
+      // property, then assume it concerns only a fixed-width UserVF.
+      // If width is not set, the flag takes precedence.
+      Scalable.Value = SK_FixedWidthOnly;
+  }
+
+  // If the flag is set to force any use of scalable vectors, override the loop
+  // hints.
+  if (ForceScalableVectorization.getValue() !=
+      LoopVectorizeHints::SK_Unspecified)
+    Scalable.Value = ForceScalableVectorization.getValue();
+
+  // Scalable vectorization is disabled if no preference is specified.
   if ((LoopVectorizeHints::ScalableForceKind)Scalable.Value == SK_Unspecified)
-    // If the width is set, but the metadata says nothing about the scalable
-    // property, then assume it concerns only a fixed-width UserVF.
-    // If width is not set, the flag takes precedence.
-    Scalable.Value = Width.Value ? SK_FixedWidthOnly : ScalableVectorization;
-  else if (ScalableVectorization == SK_FixedWidthOnly)
-    // If the flag is set to disable any use of scalable vectors, override the
-    // loop hint.
     Scalable.Value = SK_FixedWidthOnly;
 
   if (IsVectorized.Value != 1)
@@ -929,7 +949,7 @@ bool LoopVectorizationLegality::canVectorizeFPMath(
   }));
 }
 
-bool LoopVectorizationLegality::isInductionPhi(const Value *V) {
+bool LoopVectorizationLegality::isInductionPhi(const Value *V) const {
   Value *In0 = const_cast<Value *>(V);
   PHINode *PN = dyn_cast_or_null<PHINode>(In0);
   if (!PN)
@@ -938,16 +958,29 @@ bool LoopVectorizationLegality::isInductionPhi(const Value *V) {
   return Inductions.count(PN);
 }
 
-bool LoopVectorizationLegality::isCastedInductionVariable(const Value *V) {
+const InductionDescriptor *
+LoopVectorizationLegality::getIntOrFpInductionDescriptor(PHINode *Phi) const {
+  if (!isInductionPhi(Phi))
+    return nullptr;
+  auto &ID = getInductionVars().find(Phi)->second;
+  if (ID.getKind() == InductionDescriptor::IK_IntInduction ||
+      ID.getKind() == InductionDescriptor::IK_FpInduction)
+    return &ID;
+  return nullptr;
+}
+
+bool LoopVectorizationLegality::isCastedInductionVariable(
+    const Value *V) const {
   auto *Inst = dyn_cast<Instruction>(V);
   return (Inst && InductionCastsToIgnore.count(Inst));
 }
 
-bool LoopVectorizationLegality::isInductionVariable(const Value *V) {
+bool LoopVectorizationLegality::isInductionVariable(const Value *V) const {
   return isInductionPhi(V) || isCastedInductionVariable(V);
 }
 
-bool LoopVectorizationLegality::isFirstOrderRecurrence(const PHINode *Phi) {
+bool LoopVectorizationLegality::isFirstOrderRecurrence(
+    const PHINode *Phi) const {
   return FirstOrderRecurrences.count(Phi);
 }
 
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index a7d6609f8c56..71eb39a18d2f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -45,16 +45,17 @@ class VPBuilder {
   VPBasicBlock::iterator InsertPt = VPBasicBlock::iterator();
 
   VPInstruction *createInstruction(unsigned Opcode,
-                                   ArrayRef<VPValue *> Operands) {
-    VPInstruction *Instr = new VPInstruction(Opcode, Operands);
+                                   ArrayRef<VPValue *> Operands, DebugLoc DL) {
+    VPInstruction *Instr = new VPInstruction(Opcode, Operands, DL);
     if (BB)
       BB->insert(Instr, InsertPt);
     return Instr;
   }
 
   VPInstruction *createInstruction(unsigned Opcode,
-                                   std::initializer_list<VPValue *> Operands) {
-    return createInstruction(Opcode, ArrayRef<VPValue *>(Operands));
+                                   std::initializer_list<VPValue *> Operands,
+                                   DebugLoc DL) {
+    return createInstruction(Opcode, ArrayRef<VPValue *>(Operands), DL);
   }
 
 public:
@@ -123,30 +124,33 @@ public:
   /// its underlying Instruction.
   VPValue *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,
                         Instruction *Inst = nullptr) {
-    VPInstruction *NewVPInst = createInstruction(Opcode, Operands);
+    DebugLoc DL;
+    if (Inst)
+      DL = Inst->getDebugLoc();
+    VPInstruction *NewVPInst = createInstruction(Opcode, Operands, DL);
     NewVPInst->setUnderlyingValue(Inst);
     return NewVPInst;
   }
-  VPValue *createNaryOp(unsigned Opcode,
-                        std::initializer_list<VPValue *> Operands,
-                        Instruction *Inst = nullptr) {
-    return createNaryOp(Opcode, ArrayRef<VPValue *>(Operands), Inst);
+  VPValue *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,
+                        DebugLoc DL) {
+    return createInstruction(Opcode, Operands, DL);
   }
 
-  VPValue *createNot(VPValue *Operand) {
-    return createInstruction(VPInstruction::Not, {Operand});
+  VPValue *createNot(VPValue *Operand, DebugLoc DL) {
+    return createInstruction(VPInstruction::Not, {Operand}, DL);
   }
 
-  VPValue *createAnd(VPValue *LHS, VPValue *RHS) {
-    return createInstruction(Instruction::BinaryOps::And, {LHS, RHS});
+  VPValue *createAnd(VPValue *LHS, VPValue *RHS, DebugLoc DL) {
+    return createInstruction(Instruction::BinaryOps::And, {LHS, RHS}, DL);
   }
 
-  VPValue *createOr(VPValue *LHS, VPValue *RHS) {
-    return createInstruction(Instruction::BinaryOps::Or, {LHS, RHS});
+  VPValue *createOr(VPValue *LHS, VPValue *RHS, DebugLoc DL) {
+    return createInstruction(Instruction::BinaryOps::Or, {LHS, RHS}, DL);
   }
 
-  VPValue *createSelect(VPValue *Cond, VPValue *TrueVal, VPValue *FalseVal) {
-    return createNaryOp(Instruction::Select, {Cond, TrueVal, FalseVal});
+  VPValue *createSelect(VPValue *Cond, VPValue *TrueVal, VPValue *FalseVal,
+                        DebugLoc DL) {
+    return createNaryOp(Instruction::Select, {Cond, TrueVal, FalseVal}, DL);
   }
 
   //===--------------------------------------------------------------------===//
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 5ca0adb4242c..4747f34fcc62 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -428,6 +428,8 @@ class GeneratedRTChecks;
 
 namespace llvm {
 
+AnalysisKey ShouldRunExtraVectorPasses::Key;
+
 /// InnerLoopVectorizer vectorizes loops which contain only one basic
 /// block to a specified vectorization factor (VF).
 /// This class performs the widening of scalars into vectors, or multiple
@@ -506,8 +508,8 @@ public:
   /// Widen an integer or floating-point induction variable \p IV. If \p Trunc
   /// is provided, the integer induction variable will first be truncated to
   /// the corresponding type.
-  void widenIntOrFpInduction(PHINode *IV, Value *Start, TruncInst *Trunc,
-                             VPValue *Def, VPValue *CastDef,
+  void widenIntOrFpInduction(PHINode *IV, const InductionDescriptor &ID,
+                             Value *Start, TruncInst *Trunc, VPValue *Def,
                              VPTransformState &State);
 
   /// Construct the vector value of a scalarized value \p V one lane at a time.
@@ -534,7 +536,7 @@ public:
 
   /// Returns true if the reordering of FP operations is not allowed, but we are
   /// able to vectorize with strict in-order reductions for the given RdxDesc.
-  bool useOrderedReductions(RecurrenceDescriptor &RdxDesc);
+  bool useOrderedReductions(const RecurrenceDescriptor &RdxDesc);
 
   /// Create a broadcast instruction. This method generates a broadcast
   /// instruction (shuffle) for loop invariant values and for the induction
@@ -619,7 +621,7 @@ protected:
   /// can also be a truncate instruction.
   void buildScalarSteps(Value *ScalarIV, Value *Step, Instruction *EntryVal,
                         const InductionDescriptor &ID, VPValue *Def,
-                        VPValue *CastDef, VPTransformState &State);
+                        VPTransformState &State);
 
   /// Create a vector induction phi node based on an existing scalar one. \p
   /// EntryVal is the value from the original loop that maps to the vector phi
@@ -629,7 +631,6 @@ protected:
   void createVectorIntOrFpInductionPHI(const InductionDescriptor &II,
                                        Value *Step, Value *Start,
                                        Instruction *EntryVal, VPValue *Def,
-                                       VPValue *CastDef,
                                        VPTransformState &State);
 
   /// Returns true if an instruction \p I should be scalarized instead of
@@ -639,29 +640,6 @@ protected:
   /// Returns true if we should generate a scalar version of \p IV.
   bool needsScalarInduction(Instruction *IV) const;
 
-  /// If there is a cast involved in the induction variable \p ID, which should
-  /// be ignored in the vectorized loop body, this function records the
-  /// VectorLoopValue of the respective Phi also as the VectorLoopValue of the
-  /// cast. We had already proved that the casted Phi is equal to the uncasted
-  /// Phi in the vectorized loop (under a runtime guard), and therefore
-  /// there is no need to vectorize the cast - the same value can be used in the
-  /// vector loop for both the Phi and the cast.
-  /// If \p VectorLoopValue is a scalarized value, \p Lane is also specified,
-  /// Otherwise, \p VectorLoopValue is a widened/vectorized value.
-  ///
-  /// \p EntryVal is the value from the original loop that maps to the vector
-  /// phi node and is used to distinguish what is the IV currently being
-  /// processed - original one (if \p EntryVal is a phi corresponding to the
-  /// original IV) or the "newly-created" one based on the proof mentioned above
-  /// (see also buildScalarSteps() and createVectorIntOrFPInductionPHI()). In the
-  /// latter case \p EntryVal is a TruncInst and we must not record anything for
-  /// that IV, but it's error-prone to expect callers of this routine to care
-  /// about that, hence this explicit parameter.
-  void recordVectorLoopValueForInductionCast(
-      const InductionDescriptor &ID, const Instruction *EntryVal,
-      Value *VectorLoopValue, VPValue *CastDef, VPTransformState &State,
-      unsigned Part, unsigned Lane = UINT_MAX);
-
   /// Generate a shuffle sequence that will reverse the vector Vec.
   virtual Value *reverseVector(Value *Vec);
 
@@ -698,7 +676,8 @@ protected:
   /// flags, which can be found from the original scalar operations.
   Value *emitTransformedIndex(IRBuilder<> &B, Value *Index, ScalarEvolution *SE,
                               const DataLayout &DL,
-                              const InductionDescriptor &ID) const;
+                              const InductionDescriptor &ID,
+                              BasicBlock *VectorHeader) const;
 
   /// Emit basic blocks (prefixed with \p Prefix) for the iteration check,
   /// vector loop preheader, middle block and scalar preheader. Also
@@ -1728,7 +1707,8 @@ private:
   /// disabled or unsupported, then the scalable part will be equal to
   /// ElementCount::getScalable(0).
   FixedScalableVFPair computeFeasibleMaxVF(unsigned ConstTripCount,
-                                           ElementCount UserVF);
+                                           ElementCount UserVF,
+                                           bool FoldTailByMasking);
 
   /// \return the maximized element count based on the targets vector
   /// registers and the loop trip-count, but limited to a maximum safe VF.
@@ -1741,7 +1721,8 @@ private:
   ElementCount getMaximizedVFForTarget(unsigned ConstTripCount,
                                        unsigned SmallestType,
                                        unsigned WidestType,
-                                       const ElementCount &MaxSafeVF);
+                                       const ElementCount &MaxSafeVF,
+                                       bool FoldTailByMasking);
 
   /// \return the maximum legal scalable VF, based on the safe max number
   /// of elements.
@@ -2356,8 +2337,8 @@ Value *InnerLoopVectorizer::getBroadcastInstrs(Value *V) {
 
 void InnerLoopVectorizer::createVectorIntOrFpInductionPHI(
     const InductionDescriptor &II, Value *Step, Value *Start,
-    Instruction *EntryVal, VPValue *Def, VPValue *CastDef,
-    VPTransformState &State) {
+    Instruction *EntryVal, VPValue *Def, VPTransformState &State) {
+  IRBuilder<> &Builder = State.Builder;
   assert((isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal)) &&
          "Expected either an induction phi-node or a truncate of it!");
 
@@ -2373,7 +2354,7 @@ void InnerLoopVectorizer::createVectorIntOrFpInductionPHI(
   }
 
   Value *Zero = getSignedIntOrFpConstant(Start->getType(), 0);
-  Value *SplatStart = Builder.CreateVectorSplat(VF, Start);
+  Value *SplatStart = Builder.CreateVectorSplat(State.VF, Start);
   Value *SteppedStart =
       getStepVector(SplatStart, Zero, Step, II.getInductionOpcode());
 
@@ -2394,9 +2375,9 @@ void InnerLoopVectorizer::createVectorIntOrFpInductionPHI(
   Type *StepType = Step->getType();
   Value *RuntimeVF;
   if (Step->getType()->isFloatingPointTy())
-    RuntimeVF = getRuntimeVFAsFloat(Builder, StepType, VF);
+    RuntimeVF = getRuntimeVFAsFloat(Builder, StepType, State.VF);
   else
-    RuntimeVF = getRuntimeVF(Builder, StepType, VF);
+    RuntimeVF = getRuntimeVF(Builder, StepType, State.VF);
   Value *Mul = Builder.CreateBinOp(MulOp, Step, RuntimeVF);
 
   // Create a vector splat to use in the induction update.
@@ -2405,8 +2386,8 @@ void InnerLoopVectorizer::createVectorIntOrFpInductionPHI(
   //        IRBuilder. IRBuilder can constant-fold the multiply, but it doesn't
   //        handle a constant vector splat.
   Value *SplatVF = isa<Constant>(Mul)
-                       ? ConstantVector::getSplat(VF, cast<Constant>(Mul))
-                       : Builder.CreateVectorSplat(VF, Mul);
+                       ? ConstantVector::getSplat(State.VF, cast<Constant>(Mul))
+                       : Builder.CreateVectorSplat(State.VF, Mul);
   Builder.restoreIP(CurrIP);
 
   // We may need to add the step a number of times, depending on the unroll
@@ -2420,8 +2401,6 @@ void InnerLoopVectorizer::createVectorIntOrFpInductionPHI(
 
     if (isa<TruncInst>(EntryVal))
       addMetadata(LastInduction, EntryVal);
-    recordVectorLoopValueForInductionCast(II, EntryVal, LastInduction, CastDef,
-                                          State, Part);
 
     LastInduction = cast<Instruction>(
         Builder.CreateBinOp(AddOp, LastInduction, SplatVF, "step.add"));
@@ -2455,56 +2434,21 @@ bool InnerLoopVectorizer::needsScalarInduction(Instruction *IV) const {
   return llvm::any_of(IV->users(), isScalarInst);
 }
 
-void InnerLoopVectorizer::recordVectorLoopValueForInductionCast(
-    const InductionDescriptor &ID, const Instruction *EntryVal,
-    Value *VectorLoopVal, VPValue *CastDef, VPTransformState &State,
-    unsigned Part, unsigned Lane) {
-  assert((isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal)) &&
-         "Expected either an induction phi-node or a truncate of it!");
-
-  // This induction variable is not the phi from the original loop but the
-  // newly-created IV based on the proof that casted Phi is equal to the
-  // uncasted Phi in the vectorized loop (under a runtime guard possibly). It
-  // re-uses the same InductionDescriptor that original IV uses but we don't
-  // have to do any recording in this case - that is done when original IV is
-  // processed.
-  if (isa<TruncInst>(EntryVal))
-    return;
-
-  if (!CastDef) {
-    assert(ID.getCastInsts().empty() &&
-           "there are casts for ID, but no CastDef");
-    return;
-  }
-  assert(!ID.getCastInsts().empty() &&
-         "there is a CastDef, but no casts for ID");
-  // Only the first Cast instruction in the Casts vector is of interest.
-  // The rest of the Casts (if exist) have no uses outside the
-  // induction update chain itself.
-  if (Lane < UINT_MAX)
-    State.set(CastDef, VectorLoopVal, VPIteration(Part, Lane));
-  else
-    State.set(CastDef, VectorLoopVal, Part);
-}
-
-void InnerLoopVectorizer::widenIntOrFpInduction(PHINode *IV, Value *Start,
-                                                TruncInst *Trunc, VPValue *Def,
-                                                VPValue *CastDef,
+void InnerLoopVectorizer::widenIntOrFpInduction(PHINode *IV,
+                                                const InductionDescriptor &ID,
+                                                Value *Start, TruncInst *Trunc,
+                                                VPValue *Def,
                                                 VPTransformState &State) {
+  IRBuilder<> &Builder = State.Builder;
   assert((IV->getType()->isIntegerTy() || IV != OldInduction) &&
          "Primary induction variable must have an integer type");
-
-  auto II = Legal->getInductionVars().find(IV);
-  assert(II != Legal->getInductionVars().end() && "IV is not an induction");
-
-  auto ID = II->second;
   assert(IV->getType() == ID.getStartValue()->getType() && "Types must match");
 
   // The value from the original loop to which we are mapping the new induction
   // variable.
   Instruction *EntryVal = Trunc ? cast<Instruction>(Trunc) : IV;
 
-  auto &DL = OrigLoop->getHeader()->getModule()->getDataLayout();
+  auto &DL = EntryVal->getModule()->getDataLayout();
 
   // Generate code for the induction step. Note that induction steps are
   // required to be loop-invariant
@@ -2514,7 +2458,7 @@ void InnerLoopVectorizer::widenIntOrFpInduction(PHINode *IV, Value *Start,
     if (PSE.getSE()->isSCEVable(IV->getType())) {
       SCEVExpander Exp(*PSE.getSE(), DL, "induction");
       return Exp.expandCodeFor(Step, Step->getType(),
-                               LoopVectorPreHeader->getTerminator());
+                               State.CFG.VectorPreHeader->getTerminator());
     }
     return cast<SCEVUnknown>(Step)->getValue();
   };
@@ -2530,7 +2474,8 @@ void InnerLoopVectorizer::widenIntOrFpInduction(PHINode *IV, Value *Start,
                      ? Builder.CreateSExtOrTrunc(Induction, IV->getType())
                      : Builder.CreateCast(Instruction::SIToFP, Induction,
                                           IV->getType());
-      ScalarIV = emitTransformedIndex(Builder, ScalarIV, PSE.getSE(), DL, ID);
+      ScalarIV = emitTransformedIndex(Builder, ScalarIV, PSE.getSE(), DL, ID,
+                                      State.CFG.PrevBB);
       ScalarIV->setName("offset.idx");
     }
     if (Trunc) {
@@ -2548,20 +2493,19 @@ void InnerLoopVectorizer::widenIntOrFpInduction(PHINode *IV, Value *Start,
   auto CreateSplatIV = [&](Value *ScalarIV, Value *Step) {
     Value *Broadcasted = getBroadcastInstrs(ScalarIV);
     for (unsigned Part = 0; Part < UF; ++Part) {
-      assert(!VF.isScalable() && "scalable vectors not yet supported.");
+      assert(!State.VF.isScalable() && "scalable vectors not yet supported.");
       Value *StartIdx;
       if (Step->getType()->isFloatingPointTy())
-        StartIdx = getRuntimeVFAsFloat(Builder, Step->getType(), VF * Part);
+        StartIdx =
+            getRuntimeVFAsFloat(Builder, Step->getType(), State.VF * Part);
       else
-        StartIdx = getRuntimeVF(Builder, Step->getType(), VF * Part);
+        StartIdx = getRuntimeVF(Builder, Step->getType(), State.VF * Part);
 
       Value *EntryPart =
           getStepVector(Broadcasted, StartIdx, Step, ID.getInductionOpcode());
       State.set(Def, EntryPart, Part);
       if (Trunc)
         addMetadata(EntryPart, Trunc);
-      recordVectorLoopValueForInductionCast(ID, EntryVal, EntryPart, CastDef,
-                                            State, Part);
     }
   };
 
@@ -2572,7 +2516,7 @@ void InnerLoopVectorizer::widenIntOrFpInduction(PHINode *IV, Value *Start,
 
   // Now do the actual transformations, and start with creating the step value.
   Value *Step = CreateStepValue(ID.getStep());
-  if (VF.isZero() || VF.isScalar()) {
+  if (State.VF.isZero() || State.VF.isScalar()) {
     Value *ScalarIV = CreateScalarIV(Step);
     CreateSplatIV(ScalarIV, Step);
     return;
@@ -2583,8 +2527,7 @@ void InnerLoopVectorizer::widenIntOrFpInduction(PHINode *IV, Value *Start,
   // least one user in the loop that is not widened.
   auto NeedsScalarIV = needsScalarInduction(EntryVal);
   if (!NeedsScalarIV) {
-    createVectorIntOrFpInductionPHI(ID, Step, Start, EntryVal, Def, CastDef,
-                                    State);
+    createVectorIntOrFpInductionPHI(ID, Step, Start, EntryVal, Def, State);
     return;
   }
 
@@ -2592,14 +2535,13 @@ void InnerLoopVectorizer::widenIntOrFpInduction(PHINode *IV, Value *Start,
   // create the phi node, we will splat the scalar induction variable in each
   // loop iteration.
   if (!shouldScalarizeInstruction(EntryVal)) {
-    createVectorIntOrFpInductionPHI(ID, Step, Start, EntryVal, Def, CastDef,
-                                    State);
+    createVectorIntOrFpInductionPHI(ID, Step, Start, EntryVal, Def, State);
     Value *ScalarIV = CreateScalarIV(Step);
     // Create scalar steps that can be used by instructions we will later
     // scalarize. Note that the addition of the scalar steps will not increase
     // the number of instructions in the loop in the common case prior to
     // InstCombine. We will be trading one vector extract for each scalar step.
-    buildScalarSteps(ScalarIV, Step, EntryVal, ID, Def, CastDef, State);
+    buildScalarSteps(ScalarIV, Step, EntryVal, ID, Def, State);
     return;
   }
 
@@ -2609,7 +2551,7 @@ void InnerLoopVectorizer::widenIntOrFpInduction(PHINode *IV, Value *Start,
   Value *ScalarIV = CreateScalarIV(Step);
   if (!Cost->isScalarEpilogueAllowed())
     CreateSplatIV(ScalarIV, Step);
-  buildScalarSteps(ScalarIV, Step, EntryVal, ID, Def, CastDef, State);
+  buildScalarSteps(ScalarIV, Step, EntryVal, ID, Def, State);
 }
 
 Value *InnerLoopVectorizer::getStepVector(Value *Val, Value *StartIdx,
@@ -2663,10 +2605,11 @@ Value *InnerLoopVectorizer::getStepVector(Value *Val, Value *StartIdx,
 void InnerLoopVectorizer::buildScalarSteps(Value *ScalarIV, Value *Step,
                                            Instruction *EntryVal,
                                            const InductionDescriptor &ID,
-                                           VPValue *Def, VPValue *CastDef,
+                                           VPValue *Def,
                                            VPTransformState &State) {
+  IRBuilder<> &Builder = State.Builder;
   // We shouldn't have to build scalar steps if we aren't vectorizing.
-  assert(VF.isVector() && "VF should be greater than one");
+  assert(State.VF.isVector() && "VF should be greater than one");
   // Get the value type and ensure it and the step have the same integer type.
   Type *ScalarIVTy = ScalarIV->getType()->getScalarType();
   assert(ScalarIVTy == Step->getType() &&
@@ -2688,33 +2631,32 @@ void InnerLoopVectorizer::buildScalarSteps(Value *ScalarIV, Value *Step,
   // iteration. If EntryVal is uniform, we only need to generate the first
   // lane. Otherwise, we generate all VF values.
   bool IsUniform =
-      Cost->isUniformAfterVectorization(cast<Instruction>(EntryVal), VF);
-  unsigned Lanes = IsUniform ? 1 : VF.getKnownMinValue();
+      Cost->isUniformAfterVectorization(cast<Instruction>(EntryVal), State.VF);
+  unsigned Lanes = IsUniform ? 1 : State.VF.getKnownMinValue();
   // Compute the scalar steps and save the results in State.
   Type *IntStepTy = IntegerType::get(ScalarIVTy->getContext(),
                                      ScalarIVTy->getScalarSizeInBits());
   Type *VecIVTy = nullptr;
   Value *UnitStepVec = nullptr, *SplatStep = nullptr, *SplatIV = nullptr;
-  if (!IsUniform && VF.isScalable()) {
-    VecIVTy = VectorType::get(ScalarIVTy, VF);
-    UnitStepVec = Builder.CreateStepVector(VectorType::get(IntStepTy, VF));
-    SplatStep = Builder.CreateVectorSplat(VF, Step);
-    SplatIV = Builder.CreateVectorSplat(VF, ScalarIV);
+  if (!IsUniform && State.VF.isScalable()) {
+    VecIVTy = VectorType::get(ScalarIVTy, State.VF);
+    UnitStepVec =
+        Builder.CreateStepVector(VectorType::get(IntStepTy, State.VF));
+    SplatStep = Builder.CreateVectorSplat(State.VF, Step);
+    SplatIV = Builder.CreateVectorSplat(State.VF, ScalarIV);
   }
 
-  for (unsigned Part = 0; Part < UF; ++Part) {
-    Value *StartIdx0 = createStepForVF(Builder, IntStepTy, VF, Part);
+  for (unsigned Part = 0; Part < State.UF; ++Part) {
+    Value *StartIdx0 = createStepForVF(Builder, IntStepTy, State.VF, Part);
 
-    if (!IsUniform && VF.isScalable()) {
-      auto *SplatStartIdx = Builder.CreateVectorSplat(VF, StartIdx0);
+    if (!IsUniform && State.VF.isScalable()) {
+      auto *SplatStartIdx = Builder.CreateVectorSplat(State.VF, StartIdx0);
       auto *InitVec = Builder.CreateAdd(SplatStartIdx, UnitStepVec);
       if (ScalarIVTy->isFloatingPointTy())
         InitVec = Builder.CreateSIToFP(InitVec, VecIVTy);
       auto *Mul = Builder.CreateBinOp(MulOp, InitVec, SplatStep);
       auto *Add = Builder.CreateBinOp(AddOp, SplatIV, Mul);
       State.set(Def, Add, Part);
-      recordVectorLoopValueForInductionCast(ID, EntryVal, Add, CastDef, State,
-                                            Part);
       // It's useful to record the lane values too for the known minimum number
       // of elements so we do those below. This improves the code quality when
       // trying to extract the first element, for example.
@@ -2728,14 +2670,12 @@ void InnerLoopVectorizer::buildScalarSteps(Value *ScalarIV, Value *Step,
           AddOp, StartIdx0, getSignedIntOrFpConstant(ScalarIVTy, Lane));
       // The step returned by `createStepForVF` is a runtime-evaluated value
       // when VF is scalable. Otherwise, it should be folded into a Constant.
-      assert((VF.isScalable() || isa<Constant>(StartIdx)) &&
+      assert((State.VF.isScalable() || isa<Constant>(StartIdx)) &&
              "Expected StartIdx to be folded to a constant when VF is not "
              "scalable");
       auto *Mul = Builder.CreateBinOp(MulOp, StartIdx, Step);
       auto *Add = Builder.CreateBinOp(AddOp, ScalarIV, Mul);
       State.set(Def, Add, VPIteration(Part, Lane));
-      recordVectorLoopValueForInductionCast(ID, EntryVal, Add, CastDef, State,
-                                            Part, Lane);
     }
   }
 }
@@ -3023,21 +2963,19 @@ void InnerLoopVectorizer::scalarizeInstruction(Instruction *Instr,
   // poison-generating flags (nuw/nsw, exact, inbounds, etc.). The scalarized
   // instruction could feed a poison value to the base address of the widen
   // load/store.
-  if (State.MayGeneratePoisonRecipes.count(RepRecipe) > 0)
+  if (State.MayGeneratePoisonRecipes.contains(RepRecipe))
     Cloned->dropPoisonGeneratingFlags();
 
   State.Builder.SetInsertPoint(Builder.GetInsertBlock(),
                                Builder.GetInsertPoint());
   // Replace the operands of the cloned instructions with their scalar
   // equivalents in the new loop.
-  for (unsigned op = 0, e = RepRecipe->getNumOperands(); op != e; ++op) {
-    auto *Operand = dyn_cast<Instruction>(Instr->getOperand(op));
+  for (auto &I : enumerate(RepRecipe->operands())) {
     auto InputInstance = Instance;
-    if (!Operand || !OrigLoop->contains(Operand) ||
-        (Cost->isUniformAfterVectorization(Operand, State.VF)))
+    VPValue *Operand = I.value();
+    if (State.Plan->isUniformAfterVectorization(Operand))
       InputInstance.Lane = VPLane::getFirstLane();
-    auto *NewOp = State.get(RepRecipe->getOperand(op), InputInstance);
-    Cloned->setOperand(op, NewOp);
+    Cloned->setOperand(I.index(), State.get(Operand, InputInstance));
   }
   addNewMetadata(Cloned, Instr);
 
@@ -3339,7 +3277,7 @@ BasicBlock *InnerLoopVectorizer::emitMemRuntimeChecks(Loop *L,
 
 Value *InnerLoopVectorizer::emitTransformedIndex(
     IRBuilder<> &B, Value *Index, ScalarEvolution *SE, const DataLayout &DL,
-    const InductionDescriptor &ID) const {
+    const InductionDescriptor &ID, BasicBlock *VectorHeader) const {
 
   SCEVExpander Exp(*SE, DL, "induction");
   auto Step = ID.getStep();
@@ -3382,15 +3320,15 @@ Value *InnerLoopVectorizer::emitTransformedIndex(
   };
 
   // Get a suitable insert point for SCEV expansion. For blocks in the vector
-  // loop, choose the end of the vector loop header (=LoopVectorBody), because
+  // loop, choose the end of the vector loop header (=VectorHeader), because
   // the DomTree is not kept up-to-date for additional blocks generated in the
   // vector loop. By using the header as insertion point, we guarantee that the
   // expanded instructions dominate all their uses.
-  auto GetInsertPoint = [this, &B]() {
+  auto GetInsertPoint = [this, &B, VectorHeader]() {
     BasicBlock *InsertBB = B.GetInsertPoint()->getParent();
     if (InsertBB != LoopVectorBody &&
-        LI->getLoopFor(LoopVectorBody) == LI->getLoopFor(InsertBB))
-      return LoopVectorBody->getTerminator();
+        LI->getLoopFor(VectorHeader) == LI->getLoopFor(InsertBB))
+      return VectorHeader->getTerminator();
     return &*B.GetInsertPoint();
   };
 
@@ -3538,7 +3476,8 @@ void InnerLoopVectorizer::createInductionResumeValues(
           CastInst::getCastOpcode(VectorTripCount, true, StepType, true);
       Value *CRD = B.CreateCast(CastOp, VectorTripCount, StepType, "cast.crd");
       const DataLayout &DL = LoopScalarBody->getModule()->getDataLayout();
-      EndValue = emitTransformedIndex(B, CRD, PSE.getSE(), DL, II);
+      EndValue =
+          emitTransformedIndex(B, CRD, PSE.getSE(), DL, II, LoopVectorBody);
       EndValue->setName("ind.end");
 
       // Compute the end value for the additional bypass (if applicable).
@@ -3549,7 +3488,7 @@ void InnerLoopVectorizer::createInductionResumeValues(
         CRD =
             B.CreateCast(CastOp, AdditionalBypass.second, StepType, "cast.crd");
         EndValueFromAdditionalBypass =
-            emitTransformedIndex(B, CRD, PSE.getSE(), DL, II);
+            emitTransformedIndex(B, CRD, PSE.getSE(), DL, II, LoopVectorBody);
         EndValueFromAdditionalBypass->setName("ind.end");
       }
     }
@@ -3623,7 +3562,7 @@ BasicBlock *InnerLoopVectorizer::completeLoopSkeleton(Loop *L,
   if (MDNode *LID = OrigLoop->getLoopID())
     L->setLoopID(LID);
 
-  LoopVectorizeHints Hints(L, true, *ORE);
+  LoopVectorizeHints Hints(L, true, *ORE, TTI);
   Hints.setAlreadyVectorized();
 
 #ifdef EXPENSIVE_CHECKS
@@ -3780,7 +3719,8 @@ void InnerLoopVectorizer::fixupIVUsers(PHINode *OrigPhi,
                              II.getStep()->getType())
               : B.CreateSExtOrTrunc(CountMinusOne, II.getStep()->getType());
       CMO->setName("cast.cmo");
-      Value *Escape = emitTransformedIndex(B, CMO, PSE.getSE(), DL, II);
+      Value *Escape =
+          emitTransformedIndex(B, CMO, PSE.getSE(), DL, II, LoopVectorBody);
       Escape->setName("ind.escape");
       MissingVals[UI] = Escape;
     }
@@ -4573,7 +4513,8 @@ void InnerLoopVectorizer::fixNonInductionPHIs(VPTransformState &State) {
   }
 }
 
-bool InnerLoopVectorizer::useOrderedReductions(RecurrenceDescriptor &RdxDesc) {
+bool InnerLoopVectorizer::useOrderedReductions(
+    const RecurrenceDescriptor &RdxDesc) {
   return Cost->useOrderedReductions(RdxDesc);
 }
 
@@ -4648,8 +4589,8 @@ void InnerLoopVectorizer::widenPHIInstruction(Instruction *PN,
           Value *Idx = Builder.CreateAdd(
               PartStart, ConstantInt::get(PtrInd->getType(), Lane));
           Value *GlobalIdx = Builder.CreateAdd(PtrInd, Idx);
-          Value *SclrGep =
-              emitTransformedIndex(Builder, GlobalIdx, PSE.getSE(), DL, II);
+          Value *SclrGep = emitTransformedIndex(Builder, GlobalIdx, PSE.getSE(),
+                                                DL, II, State.CFG.PrevBB);
           SclrGep->setName("next.gep");
           State.set(PhiR, SclrGep, VPIteration(Part, Lane));
         }
@@ -5368,13 +5309,9 @@ LoopVectorizationCostModel::getMaxLegalScalableVF(unsigned MaxSafeElements) {
 
   // Limit MaxScalableVF by the maximum safe dependence distance.
   Optional<unsigned> MaxVScale = TTI.getMaxVScale();
-  if (!MaxVScale && TheFunction->hasFnAttribute(Attribute::VScaleRange)) {
-    unsigned VScaleMax = TheFunction->getFnAttribute(Attribute::VScaleRange)
-                             .getVScaleRangeArgs()
-                             .second;
-    if (VScaleMax > 0)
-      MaxVScale = VScaleMax;
-  }
+  if (!MaxVScale && TheFunction->hasFnAttribute(Attribute::VScaleRange))
+    MaxVScale =
+        TheFunction->getFnAttribute(Attribute::VScaleRange).getVScaleRangeMax();
   MaxScalableVF = ElementCount::getScalable(
       MaxVScale ? (MaxSafeElements / MaxVScale.getValue()) : 0);
   if (!MaxScalableVF)
@@ -5386,9 +5323,8 @@ LoopVectorizationCostModel::getMaxLegalScalableVF(unsigned MaxSafeElements) {
   return MaxScalableVF;
 }
 
-FixedScalableVFPair
-LoopVectorizationCostModel::computeFeasibleMaxVF(unsigned ConstTripCount,
-                                                 ElementCount UserVF) {
+FixedScalableVFPair LoopVectorizationCostModel::computeFeasibleMaxVF(
+    unsigned ConstTripCount, ElementCount UserVF, bool FoldTailByMasking) {
   MinBWs = computeMinimumValueSizes(TheLoop->getBlocks(), *DB, &TTI);
   unsigned SmallestType, WidestType;
   std::tie(SmallestType, WidestType) = getSmallestAndWidestTypes();
@@ -5475,12 +5411,14 @@ LoopVectorizationCostModel::computeFeasibleMaxVF(unsigned ConstTripCount,
 
   FixedScalableVFPair Result(ElementCount::getFixed(1),
                              ElementCount::getScalable(0));
-  if (auto MaxVF = getMaximizedVFForTarget(ConstTripCount, SmallestType,
-                                           WidestType, MaxSafeFixedVF))
+  if (auto MaxVF =
+          getMaximizedVFForTarget(ConstTripCount, SmallestType, WidestType,
+                                  MaxSafeFixedVF, FoldTailByMasking))
     Result.FixedVF = MaxVF;
 
-  if (auto MaxVF = getMaximizedVFForTarget(ConstTripCount, SmallestType,
-                                           WidestType, MaxSafeScalableVF))
+  if (auto MaxVF =
+          getMaximizedVFForTarget(ConstTripCount, SmallestType, WidestType,
+                                  MaxSafeScalableVF, FoldTailByMasking))
     if (MaxVF.isScalable()) {
       Result.ScalableVF = MaxVF;
       LLVM_DEBUG(dbgs() << "LV: Found feasible scalable VF = " << MaxVF
@@ -5513,7 +5451,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
 
   switch (ScalarEpilogueStatus) {
   case CM_ScalarEpilogueAllowed:
-    return computeFeasibleMaxVF(TC, UserVF);
+    return computeFeasibleMaxVF(TC, UserVF, false);
   case CM_ScalarEpilogueNotAllowedUsePredicate:
     LLVM_FALLTHROUGH;
   case CM_ScalarEpilogueNotNeededUsePredicate:
@@ -5551,7 +5489,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
       LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking: vectorize with a "
                            "scalar epilogue instead.\n");
       ScalarEpilogueStatus = CM_ScalarEpilogueAllowed;
-      return computeFeasibleMaxVF(TC, UserVF);
+      return computeFeasibleMaxVF(TC, UserVF, false);
     }
     return FixedScalableVFPair::getNone();
   }
@@ -5568,7 +5506,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
     InterleaveInfo.invalidateGroupsRequiringScalarEpilogue();
   }
 
-  FixedScalableVFPair MaxFactors = computeFeasibleMaxVF(TC, UserVF);
+  FixedScalableVFPair MaxFactors = computeFeasibleMaxVF(TC, UserVF, true);
   // Avoid tail folding if the trip count is known to be a multiple of any VF
   // we chose.
   // FIXME: The condition below pessimises the case for fixed-width vectors,
@@ -5641,7 +5579,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
 
 ElementCount LoopVectorizationCostModel::getMaximizedVFForTarget(
     unsigned ConstTripCount, unsigned SmallestType, unsigned WidestType,
-    const ElementCount &MaxSafeVF) {
+    const ElementCount &MaxSafeVF, bool FoldTailByMasking) {
   bool ComputeScalableMaxVF = MaxSafeVF.isScalable();
   TypeSize WidestRegister = TTI.getRegisterBitWidth(
       ComputeScalableMaxVF ? TargetTransformInfo::RGK_ScalableVector
@@ -5673,14 +5611,17 @@ ElementCount LoopVectorizationCostModel::getMaximizedVFForTarget(
   const auto TripCountEC = ElementCount::getFixed(ConstTripCount);
   if (ConstTripCount &&
       ElementCount::isKnownLE(TripCountEC, MaxVectorElementCount) &&
-      isPowerOf2_32(ConstTripCount)) {
-    // We need to clamp the VF to be the ConstTripCount. There is no point in
-    // choosing a higher viable VF as done in the loop below. If
-    // MaxVectorElementCount is scalable, we only fall back on a fixed VF when
-    // the TC is less than or equal to the known number of lanes.
-    LLVM_DEBUG(dbgs() << "LV: Clamping the MaxVF to the constant trip count: "
-                      << ConstTripCount << "\n");
-    return TripCountEC;
+      (!FoldTailByMasking || isPowerOf2_32(ConstTripCount))) {
+    // If loop trip count (TC) is known at compile time there is no point in
+    // choosing VF greater than TC (as done in the loop below). Select maximum
+    // power of two which doesn't exceed TC.
+    // If MaxVectorElementCount is scalable, we only fall back on a fixed VF
+    // when the TC is less than or equal to the known number of lanes.
+    auto ClampedConstTripCount = PowerOf2Floor(ConstTripCount);
+    LLVM_DEBUG(dbgs() << "LV: Clamping the MaxVF to maximum power of two not "
+                         "exceeding the constant trip count: "
+                      << ClampedConstTripCount << "\n");
+    return ElementCount::getFixed(ClampedConstTripCount);
   }
 
   ElementCount MaxVF = MaxVectorElementCount;
@@ -5758,12 +5699,11 @@ bool LoopVectorizationCostModel::isMoreProfitable(
       EstimatedWidthB *= VScale.getValue();
   }
 
-  // When set to preferred, for now assume vscale may be larger than 1 (or the
-  // one being tuned for), so that scalable vectorization is slightly favorable
-  // over fixed-width vectorization.
-  if (Hints->isScalableVectorizationPreferred())
-    if (A.Width.isScalable() && !B.Width.isScalable())
-      return (CostA * B.Width.getFixedValue()) <= (CostB * EstimatedWidthA);
+  // Assume vscale may be larger than 1 (or the value being tuned for),
+  // so that scalable vectorization is slightly favorable over fixed-width
+  // vectorization.
+  if (A.Width.isScalable() && !B.Width.isScalable())
+    return (CostA * B.Width.getFixedValue()) <= (CostB * EstimatedWidthA);
 
   // To avoid the need for FP division:
   //      (CostA / A.Width) < (CostB / B.Width)
@@ -6068,7 +6008,8 @@ void LoopVectorizationCostModel::collectElementTypesForWidening() {
       if (auto *PN = dyn_cast<PHINode>(&I)) {
         if (!Legal->isReductionVariable(PN))
           continue;
-        const RecurrenceDescriptor &RdxDesc = Legal->getReductionVars()[PN];
+        const RecurrenceDescriptor &RdxDesc =
+            Legal->getReductionVars().find(PN)->second;
         if (PreferInLoopReductions || useOrderedReductions(RdxDesc) ||
             TTI.preferInLoopReduction(RdxDesc.getOpcode(),
                                       RdxDesc.getRecurrenceType(),
@@ -7002,7 +6943,7 @@ Optional<InstructionCost> LoopVectorizationCostModel::getReductionPatternCost(
     ReductionPhi = InLoopReductionImmediateChains[ReductionPhi];
 
   const RecurrenceDescriptor &RdxDesc =
-      Legal->getReductionVars()[cast<PHINode>(ReductionPhi)];
+      Legal->getReductionVars().find(cast<PHINode>(ReductionPhi))->second;
 
   InstructionCost BaseCost = TTI.getArithmeticReductionCost(
       RdxDesc.getOpcode(), VectorTy, RdxDesc.getFastMathFlags(), CostKind);
@@ -7079,22 +7020,41 @@ Optional<InstructionCost> LoopVectorizationCostModel::getReductionPatternCost(
              match(RedOp, m_Mul(m_Instruction(Op0), m_Instruction(Op1)))) {
     if (match(Op0, m_ZExtOrSExt(m_Value())) &&
         Op0->getOpcode() == Op1->getOpcode() &&
-        Op0->getOperand(0)->getType() == Op1->getOperand(0)->getType() &&
         !TheLoop->isLoopInvariant(Op0) && !TheLoop->isLoopInvariant(Op1)) {
       bool IsUnsigned = isa<ZExtInst>(Op0);
-      auto *ExtType = VectorType::get(Op0->getOperand(0)->getType(), VectorTy);
-      // Matched reduce(mul(ext, ext))
-      InstructionCost ExtCost =
-          TTI.getCastInstrCost(Op0->getOpcode(), VectorTy, ExtType,
-                               TTI::CastContextHint::None, CostKind, Op0);
+      Type *Op0Ty = Op0->getOperand(0)->getType();
+      Type *Op1Ty = Op1->getOperand(0)->getType();
+      Type *LargestOpTy =
+          Op0Ty->getIntegerBitWidth() < Op1Ty->getIntegerBitWidth() ? Op1Ty
+                                                                    : Op0Ty;
+      auto *ExtType = VectorType::get(LargestOpTy, VectorTy);
+
+      // Matched reduce(mul(ext(A), ext(B))), where the two ext may be of
+      // different sizes. We take the largest type as the ext to reduce, and add
+      // the remaining cost as, for example reduce(mul(ext(ext(A)), ext(B))).
+      InstructionCost ExtCost0 = TTI.getCastInstrCost(
+          Op0->getOpcode(), VectorTy, VectorType::get(Op0Ty, VectorTy),
+          TTI::CastContextHint::None, CostKind, Op0);
+      InstructionCost ExtCost1 = TTI.getCastInstrCost(
+          Op1->getOpcode(), VectorTy, VectorType::get(Op1Ty, VectorTy),
+          TTI::CastContextHint::None, CostKind, Op1);
       InstructionCost MulCost =
           TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy, CostKind);
 
       InstructionCost RedCost = TTI.getExtendedAddReductionCost(
           /*IsMLA=*/true, IsUnsigned, RdxDesc.getRecurrenceType(), ExtType,
           CostKind);
+      InstructionCost ExtraExtCost = 0;
+      if (Op0Ty != LargestOpTy || Op1Ty != LargestOpTy) {
+        Instruction *ExtraExtOp = (Op0Ty != LargestOpTy) ? Op0 : Op1;
+        ExtraExtCost = TTI.getCastInstrCost(
+            ExtraExtOp->getOpcode(), ExtType,
+            VectorType::get(ExtraExtOp->getOperand(0)->getType(), VectorTy),
+            TTI::CastContextHint::None, CostKind, ExtraExtOp);
+      }
 
-      if (RedCost.isValid() && RedCost < ExtCost * 2 + MulCost + BaseCost)
+      if (RedCost.isValid() &&
+          (RedCost + ExtraExtCost) < (ExtCost0 + ExtCost1 + MulCost + BaseCost))
         return I == RetI ? RedCost : 0;
     } else if (!match(I, m_ZExtOrSExt(m_Value()))) {
       // Matched reduce(mul())
@@ -7570,8 +7530,12 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I, ElementCount VF,
     Type *CondTy = SI->getCondition()->getType();
     if (!ScalarCond)
       CondTy = VectorType::get(CondTy, VF);
-    return TTI.getCmpSelInstrCost(I->getOpcode(), VectorTy, CondTy,
-                                  CmpInst::BAD_ICMP_PREDICATE, CostKind, I);
+
+    CmpInst::Predicate Pred = CmpInst::BAD_ICMP_PREDICATE;
+    if (auto *Cmp = dyn_cast<CmpInst>(SI->getCondition()))
+      Pred = Cmp->getPredicate();
+    return TTI.getCmpSelInstrCost(I->getOpcode(), VectorTy, CondTy, Pred,
+                                  CostKind, I);
   }
   case Instruction::ICmp:
   case Instruction::FCmp: {
@@ -7581,7 +7545,8 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I, ElementCount VF,
       ValTy = IntegerType::get(ValTy->getContext(), MinBWs[Op0AsInstruction]);
     VectorTy = ToVectorTy(ValTy, VF);
     return TTI.getCmpSelInstrCost(I->getOpcode(), VectorTy, nullptr,
-                                  CmpInst::BAD_ICMP_PREDICATE, CostKind, I);
+                                  cast<CmpInst>(I)->getPredicate(), CostKind,
+                                  I);
   }
   case Instruction::Store:
   case Instruction::Load: {
@@ -7762,14 +7727,14 @@ void LoopVectorizationCostModel::collectValuesToIgnore() {
   // Ignore type-promoting instructions we identified during reduction
   // detection.
   for (auto &Reduction : Legal->getReductionVars()) {
-    RecurrenceDescriptor &RedDes = Reduction.second;
+    const RecurrenceDescriptor &RedDes = Reduction.second;
     const SmallPtrSetImpl<Instruction *> &Casts = RedDes.getCastInsts();
     VecValuesToIgnore.insert(Casts.begin(), Casts.end());
   }
   // Ignore type-casting instructions we identified during induction
   // detection.
   for (auto &Induction : Legal->getInductionVars()) {
-    InductionDescriptor &IndDes = Induction.second;
+    const InductionDescriptor &IndDes = Induction.second;
     const SmallVectorImpl<Instruction *> &Casts = IndDes.getCastInsts();
     VecValuesToIgnore.insert(Casts.begin(), Casts.end());
   }
@@ -7778,7 +7743,7 @@ void LoopVectorizationCostModel::collectValuesToIgnore() {
 void LoopVectorizationCostModel::collectInLoopReductions() {
   for (auto &Reduction : Legal->getReductionVars()) {
     PHINode *Phi = Reduction.first;
-    RecurrenceDescriptor &RdxDesc = Reduction.second;
+    const RecurrenceDescriptor &RdxDesc = Reduction.second;
 
     // We don't collect reductions that are type promoted (yet).
     if (RdxDesc.getRecurrenceType() != Phi->getType())
@@ -8064,18 +8029,6 @@ void LoopVectorizationPlanner::collectTriviallyDeadInstructions(
           return U == Ind || DeadInstructions.count(cast<Instruction>(U));
         }))
       DeadInstructions.insert(IndUpdate);
-
-    // We record as "Dead" also the type-casting instructions we had identified
-    // during induction analysis. We don't need any handling for them in the
-    // vectorized loop because we have proven that, under a proper runtime
-    // test guarding the vectorized loop, the value of the phi, and the casted
-    // value of the phi, are the same. The last instruction in this casting chain
-    // will get its scalar/vector/widened def from the scalar/vector/widened def
-    // of the respective phi node. Any other casts in the induction def-use chain
-    // have no other uses outside the phi update chain, and will be ignored.
-    InductionDescriptor &IndDes = Induction.second;
-    const SmallVectorImpl<Instruction *> &Casts = IndDes.getCastInsts();
-    DeadInstructions.insert(Casts.begin(), Casts.end());
   }
 }
 
@@ -8461,7 +8414,7 @@ VPValue *VPRecipeBuilder::createEdgeMask(BasicBlock *Src, BasicBlock *Dst,
   assert(EdgeMask && "No Edge Mask found for condition");
 
   if (BI->getSuccessor(0) != Dst)
-    EdgeMask = Builder.createNot(EdgeMask);
+    EdgeMask = Builder.createNot(EdgeMask, BI->getDebugLoc());
 
   if (SrcMask) { // Otherwise block in-mask is all-one, no need to AND.
     // The condition is 'SrcMask && EdgeMask', which is equivalent to
@@ -8470,7 +8423,8 @@ VPValue *VPRecipeBuilder::createEdgeMask(BasicBlock *Src, BasicBlock *Dst,
     // EdgeMask is poison. Using 'and' here introduces undefined behavior.
     VPValue *False = Plan->getOrAddVPValue(
         ConstantInt::getFalse(BI->getCondition()->getType()));
-    EdgeMask = Builder.createSelect(SrcMask, EdgeMask, False);
+    EdgeMask =
+        Builder.createSelect(SrcMask, EdgeMask, False, BI->getDebugLoc());
   }
 
   return EdgeMaskCache[Edge] = EdgeMask;
@@ -8492,22 +8446,24 @@ VPValue *VPRecipeBuilder::createBlockInMask(BasicBlock *BB, VPlanPtr &Plan) {
     if (!CM.blockNeedsPredicationForAnyReason(BB))
       return BlockMaskCache[BB] = BlockMask; // Loop incoming mask is all-one.
 
-    // Create the block in mask as the first non-phi instruction in the block.
-    VPBuilder::InsertPointGuard Guard(Builder);
-    auto NewInsertionPoint = Builder.getInsertBlock()->getFirstNonPhi();
-    Builder.setInsertPoint(Builder.getInsertBlock(), NewInsertionPoint);
-
     // Introduce the early-exit compare IV <= BTC to form header block mask.
     // This is used instead of IV < TC because TC may wrap, unlike BTC.
-    // Start by constructing the desired canonical IV.
+    // Start by constructing the desired canonical IV in the header block.
     VPValue *IV = nullptr;
     if (Legal->getPrimaryInduction())
       IV = Plan->getOrAddVPValue(Legal->getPrimaryInduction());
     else {
+      VPBasicBlock *HeaderVPBB = Plan->getEntry()->getEntryBasicBlock();
       auto *IVRecipe = new VPWidenCanonicalIVRecipe();
-      Builder.getInsertBlock()->insert(IVRecipe, NewInsertionPoint);
+      HeaderVPBB->insert(IVRecipe, HeaderVPBB->getFirstNonPhi());
       IV = IVRecipe;
     }
+
+    // Create the block in mask as the first non-phi instruction in the block.
+    VPBuilder::InsertPointGuard Guard(Builder);
+    auto NewInsertionPoint = Builder.getInsertBlock()->getFirstNonPhi();
+    Builder.setInsertPoint(Builder.getInsertBlock(), NewInsertionPoint);
+
     VPValue *BTC = Plan->getOrCreateBackedgeTakenCount();
     bool TailFolded = !CM.isScalarEpilogueAllowed();
 
@@ -8534,7 +8490,7 @@ VPValue *VPRecipeBuilder::createBlockInMask(BasicBlock *BB, VPlanPtr &Plan) {
       continue;
     }
 
-    BlockMask = Builder.createOr(BlockMask, EdgeMask);
+    BlockMask = Builder.createOr(BlockMask, EdgeMask, {});
   }
 
   return BlockMaskCache[BB] = BlockMask;
@@ -8591,14 +8547,10 @@ VPRecipeBuilder::tryToOptimizeInductionPHI(PHINode *Phi,
                                            ArrayRef<VPValue *> Operands) const {
   // Check if this is an integer or fp induction. If so, build the recipe that
   // produces its scalar and vector values.
-  InductionDescriptor II = Legal->getInductionVars().lookup(Phi);
-  if (II.getKind() == InductionDescriptor::IK_IntInduction ||
-      II.getKind() == InductionDescriptor::IK_FpInduction) {
-    assert(II.getStartValue() ==
+  if (auto *II = Legal->getIntOrFpInductionDescriptor(Phi)) {
+    assert(II->getStartValue() ==
            Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader()));
-    const SmallVectorImpl<Instruction *> &Casts = II.getCastInsts();
-    return new VPWidenIntOrFpInductionRecipe(
-        Phi, Operands[0], Casts.empty() ? nullptr : Casts.front());
+    return new VPWidenIntOrFpInductionRecipe(Phi, Operands[0], *II);
   }
 
   return nullptr;
@@ -8624,11 +8576,10 @@ VPWidenIntOrFpInductionRecipe *VPRecipeBuilder::tryToOptimizeInductionTruncate(
   if (LoopVectorizationPlanner::getDecisionAndClampRange(
           isOptimizableIVTruncate(I), Range)) {
 
-    InductionDescriptor II =
-        Legal->getInductionVars().lookup(cast<PHINode>(I->getOperand(0)));
+    auto *Phi = cast<PHINode>(I->getOperand(0));
+    const InductionDescriptor &II = *Legal->getIntOrFpInductionDescriptor(Phi);
     VPValue *Start = Plan.getOrAddVPValue(II.getStartValue());
-    return new VPWidenIntOrFpInductionRecipe(cast<PHINode>(I->getOperand(0)),
-                                             Start, nullptr, I);
+    return new VPWidenIntOrFpInductionRecipe(Phi, Start, II, I);
   }
   return nullptr;
 }
@@ -8844,13 +8795,17 @@ VPBasicBlock *VPRecipeBuilder::handleReplication(
     return VPBB;
   }
   LLVM_DEBUG(dbgs() << "LV: Scalarizing and predicating:" << *I << "\n");
-  assert(VPBB->getSuccessors().empty() &&
-         "VPBB has successors when handling predicated replication.");
+
+  VPBlockBase *SingleSucc = VPBB->getSingleSuccessor();
+  assert(SingleSucc && "VPBB must have a single successor when handling "
+                       "predicated replication.");
+  VPBlockUtils::disconnectBlocks(VPBB, SingleSucc);
   // Record predicated instructions for above packing optimizations.
   VPBlockBase *Region = createReplicateRegion(I, Recipe, Plan);
   VPBlockUtils::insertBlockAfter(Region, VPBB);
   auto *RegSucc = new VPBasicBlock();
   VPBlockUtils::insertBlockAfter(RegSucc, Region);
+  VPBlockUtils::connectBlocks(RegSucc, SingleSucc);
   return RegSucc;
 }
 
@@ -8910,7 +8865,8 @@ VPRecipeBuilder::tryToCreateWidenRecipe(Instruction *Instr,
     if (Legal->isReductionVariable(Phi) || Legal->isFirstOrderRecurrence(Phi)) {
       VPValue *StartV = Operands[0];
       if (Legal->isReductionVariable(Phi)) {
-        RecurrenceDescriptor &RdxDesc = Legal->getReductionVars()[Phi];
+        const RecurrenceDescriptor &RdxDesc =
+            Legal->getReductionVars().find(Phi)->second;
         assert(RdxDesc.getRecurrenceStartValue() ==
                Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader()));
         PhiRecipe = new VPReductionPHIRecipe(Phi, RdxDesc, *StartV,
@@ -9031,7 +8987,8 @@ VPlanPtr LoopVectorizationPlanner::buildVPlanWithVPRecipes(
   }
   for (auto &Reduction : CM.getInLoopReductionChains()) {
     PHINode *Phi = Reduction.first;
-    RecurKind Kind = Legal->getReductionVars()[Phi].getRecurrenceKind();
+    RecurKind Kind =
+        Legal->getReductionVars().find(Phi)->second.getRecurrenceKind();
     const SmallVector<Instruction *, 4> &ReductionOperations = Reduction.second;
 
     RecipeBuilder.recordRecipeOf(Phi);
@@ -9069,30 +9026,25 @@ VPlanPtr LoopVectorizationPlanner::buildVPlanWithVPRecipes(
   // visit each basic block after having visited its predecessor basic blocks.
   // ---------------------------------------------------------------------------
 
-  auto Plan = std::make_unique<VPlan>();
+  // Create initial VPlan skeleton, with separate header and latch blocks.
+  VPBasicBlock *HeaderVPBB = new VPBasicBlock();
+  VPBasicBlock *LatchVPBB = new VPBasicBlock("vector.latch");
+  VPBlockUtils::insertBlockAfter(LatchVPBB, HeaderVPBB);
+  auto *TopRegion = new VPRegionBlock(HeaderVPBB, LatchVPBB, "vector loop");
+  auto Plan = std::make_unique<VPlan>(TopRegion);
 
   // Scan the body of the loop in a topological order to visit each basic block
   // after having visited its predecessor basic blocks.
   LoopBlocksDFS DFS(OrigLoop);
   DFS.perform(LI);
 
-  VPBasicBlock *VPBB = nullptr;
-  VPBasicBlock *HeaderVPBB = nullptr;
+  VPBasicBlock *VPBB = HeaderVPBB;
   SmallVector<VPWidenIntOrFpInductionRecipe *> InductionsToMove;
   for (BasicBlock *BB : make_range(DFS.beginRPO(), DFS.endRPO())) {
     // Relevant instructions from basic block BB will be grouped into VPRecipe
     // ingredients and fill a new VPBasicBlock.
     unsigned VPBBsForBB = 0;
-    auto *FirstVPBBForBB = new VPBasicBlock(BB->getName());
-    if (VPBB)
-      VPBlockUtils::insertBlockAfter(FirstVPBBForBB, VPBB);
-    else {
-      auto *TopRegion = new VPRegionBlock("vector loop");
-      TopRegion->setEntry(FirstVPBBForBB);
-      Plan->setEntry(TopRegion);
-      HeaderVPBB = FirstVPBBForBB;
-    }
-    VPBB = FirstVPBBForBB;
+    VPBB->setName(BB->getName());
     Builder.setInsertPoint(VPBB);
 
     // Introduce each ingredient into VPlan.
@@ -9159,13 +9111,21 @@ VPlanPtr LoopVectorizationPlanner::buildVPlanWithVPRecipes(
                                     : "");
       }
     }
+
+    VPBlockUtils::insertBlockAfter(new VPBasicBlock(), VPBB);
+    VPBB = cast<VPBasicBlock>(VPBB->getSingleSuccessor());
   }
 
+  // Fold the last, empty block into its predecessor.
+  VPBB = VPBlockUtils::tryToMergeBlockIntoPredecessor(VPBB);
+  assert(VPBB && "expected to fold last (empty) block");
+  // After here, VPBB should not be used.
+  VPBB = nullptr;
+
   assert(isa<VPRegionBlock>(Plan->getEntry()) &&
          !Plan->getEntry()->getEntryBasicBlock()->empty() &&
          "entry block must be set to a VPRegionBlock having a non-empty entry "
          "VPBasicBlock");
-  cast<VPRegionBlock>(Plan->getEntry())->setExit(VPBB);
   RecipeBuilder.fixHeaderPhis();
 
   // ---------------------------------------------------------------------------
@@ -9231,18 +9191,19 @@ VPlanPtr LoopVectorizationPlanner::buildVPlanWithVPRecipes(
       VPBlockUtils::disconnectBlocks(SplitPred, SplitBlock);
       VPBlockUtils::connectBlocks(SplitPred, SinkRegion);
       VPBlockUtils::connectBlocks(SinkRegion, SplitBlock);
-      if (VPBB == SplitPred)
-        VPBB = SplitBlock;
     }
   }
 
+  VPlanTransforms::removeRedundantInductionCasts(*Plan);
+
   // Now that sink-after is done, move induction recipes for optimized truncates
   // to the phi section of the header block.
   for (VPWidenIntOrFpInductionRecipe *Ind : InductionsToMove)
     Ind->moveBefore(*HeaderVPBB, HeaderVPBB->getFirstNonPhi());
 
   // Adjust the recipes for any inloop reductions.
-  adjustRecipesForReductions(VPBB, Plan, RecipeBuilder, Range.Start);
+  adjustRecipesForReductions(cast<VPBasicBlock>(TopRegion->getExit()), Plan,
+                             RecipeBuilder, Range.Start);
 
   // Introduce a recipe to combine the incoming and previous values of a
   // first-order recurrence.
@@ -9322,6 +9283,11 @@ VPlanPtr LoopVectorizationPlanner::buildVPlanWithVPRecipes(
   RSO.flush();
   Plan->setName(PlanName);
 
+  // Fold Exit block into its predecessor if possible.
+  // TODO: Fold block earlier once all VPlan transforms properly maintain a
+  // VPBasicBlock as exit.
+  VPBlockUtils::tryToMergeBlockIntoPredecessor(TopRegion->getExit());
+
   assert(VPlanVerifier::verifyPlanIsValid(*Plan) && "VPlan is invalid");
   return Plan;
 }
@@ -9355,9 +9321,10 @@ VPlanPtr LoopVectorizationPlanner::buildVPlan(VFRange &Range) {
   }
 
   SmallPtrSet<Instruction *, 1> DeadInstructions;
-  VPlanTransforms::VPInstructionsToVPRecipes(OrigLoop, Plan,
-                                             Legal->getInductionVars(),
-                                             DeadInstructions, *PSE.getSE());
+  VPlanTransforms::VPInstructionsToVPRecipes(
+      OrigLoop, Plan,
+      [this](PHINode *P) { return Legal->getIntOrFpInductionDescriptor(P); },
+      DeadInstructions, *PSE.getSE());
   return Plan;
 }
 
@@ -9371,7 +9338,8 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
     ElementCount MinVF) {
   for (auto &Reduction : CM.getInLoopReductionChains()) {
     PHINode *Phi = Reduction.first;
-    RecurrenceDescriptor &RdxDesc = Legal->getReductionVars()[Phi];
+    const RecurrenceDescriptor &RdxDesc =
+        Legal->getReductionVars().find(Phi)->second;
     const SmallVector<Instruction *, 4> &ReductionOperations = Reduction.second;
 
     if (MinVF.isScalar() && !CM.useOrderedReductions(RdxDesc))
@@ -9565,7 +9533,7 @@ void VPWidenRecipe::execute(VPTransformState &State) {
         // exact, etc.). The control flow has been linearized and the
         // instruction is no longer guarded by the predicate, which could make
         // the flag properties to no longer hold.
-        if (State.MayGeneratePoisonRecipes.count(this) > 0)
+        if (State.MayGeneratePoisonRecipes.contains(this))
           VecOp->dropPoisonGeneratingFlags();
       }
 
@@ -9714,9 +9682,9 @@ void VPWidenGEPRecipe::execute(VPTransformState &State) {
 
 void VPWidenIntOrFpInductionRecipe::execute(VPTransformState &State) {
   assert(!State.Instance && "Int or FP induction being replicated.");
-  State.ILV->widenIntOrFpInduction(IV, getStartValue()->getLiveInIRValue(),
-                                   getTruncInst(), getVPValue(0),
-                                   getCastValue(), State);
+  State.ILV->widenIntOrFpInduction(IV, getInductionDescriptor(),
+                                   getStartValue()->getLiveInIRValue(),
+                                   getTruncInst(), getVPValue(0), State);
 }
 
 void VPWidenPHIRecipe::execute(VPTransformState &State) {
@@ -10293,7 +10261,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
                     << L->getHeader()->getParent()->getName() << "\" from "
                     << DebugLocStr << "\n");
 
-  LoopVectorizeHints Hints(L, InterleaveOnlyWhenForced, *ORE);
+  LoopVectorizeHints Hints(L, InterleaveOnlyWhenForced, *ORE, TTI);
 
   LLVM_DEBUG(
       dbgs() << "LV: Loop hints:"
@@ -10747,8 +10715,17 @@ PreservedAnalyses LoopVectorizePass::run(Function &F,
       PA.preserve<LoopAnalysis>();
       PA.preserve<DominatorTreeAnalysis>();
     }
-    if (!Result.MadeCFGChange)
+
+    if (Result.MadeCFGChange) {
+      // Making CFG changes likely means a loop got vectorized. Indicate that
+      // extra simplification passes should be run.
+      // TODO: MadeCFGChanges is not a prefect proxy. Extra passes should only
+      // be run if runtime checks have been added.
+      AM.getResult<ShouldRunExtraVectorPasses>(F);
+      PA.preserve<ShouldRunExtraVectorPasses>();
+    } else {
       PA.preserveSet<CFGAnalyses>();
+    }
     return PA;
 }
 
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 95061e9053fa..37ae13666f7a 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -631,27 +631,26 @@ static void addMask(SmallVectorImpl<int> &Mask, ArrayRef<int> SubMask) {
 /// after:   6 3 5 4 7 2 1 0
 static void fixupOrderingIndices(SmallVectorImpl<unsigned> &Order) {
   const unsigned Sz = Order.size();
-  SmallBitVector UsedIndices(Sz);
-  SmallVector<int> MaskedIndices;
+  SmallBitVector UnusedIndices(Sz, /*t=*/true);
+  SmallBitVector MaskedIndices(Sz);
   for (unsigned I = 0; I < Sz; ++I) {
     if (Order[I] < Sz)
-      UsedIndices.set(Order[I]);
+      UnusedIndices.reset(Order[I]);
     else
-      MaskedIndices.push_back(I);
+      MaskedIndices.set(I);
   }
-  if (MaskedIndices.empty())
+  if (MaskedIndices.none())
     return;
-  SmallVector<int> AvailableIndices(MaskedIndices.size());
-  unsigned Cnt = 0;
-  int Idx = UsedIndices.find_first();
-  do {
-    AvailableIndices[Cnt] = Idx;
-    Idx = UsedIndices.find_next(Idx);
-    ++Cnt;
-  } while (Idx > 0);
-  assert(Cnt == MaskedIndices.size() && "Non-synced masked/available indices.");
-  for (int I = 0, E = MaskedIndices.size(); I < E; ++I)
-    Order[MaskedIndices[I]] = AvailableIndices[I];
+  assert(UnusedIndices.count() == MaskedIndices.count() &&
+         "Non-synced masked/available indices.");
+  int Idx = UnusedIndices.find_first();
+  int MIdx = MaskedIndices.find_first();
+  while (MIdx >= 0) {
+    assert(Idx >= 0 && "Indices must be synced.");
+    Order[MIdx] = Idx;
+    Idx = UnusedIndices.find_next(Idx);
+    MIdx = MaskedIndices.find_next(MIdx);
+  }
 }
 
 namespace llvm {
@@ -812,6 +811,13 @@ public:
   /// ExtractElement, ExtractValue), which can be part of the graph.
   Optional<OrdersType> findReusedOrderedScalars(const TreeEntry &TE);
 
+  /// Gets reordering data for the given tree entry. If the entry is vectorized
+  /// - just return ReorderIndices, otherwise check if the scalars can be
+  /// reordered and return the most optimal order.
+  /// \param TopToBottom If true, include the order of vectorized stores and
+  /// insertelement nodes, otherwise skip them.
+  Optional<OrdersType> getReorderingData(const TreeEntry &TE, bool TopToBottom);
+
   /// Reorders the current graph to the most profitable order starting from the
   /// root node to the leaf nodes. The best order is chosen only from the nodes
   /// of the same size (vectorization factor). Smaller nodes are considered
@@ -1010,18 +1016,25 @@ public:
       std::swap(OpsVec[OpIdx1][Lane], OpsVec[OpIdx2][Lane]);
     }
 
-    // The hard-coded scores listed here are not very important. When computing
-    // the scores of matching one sub-tree with another, we are basically
-    // counting the number of values that are matching. So even if all scores
-    // are set to 1, we would still get a decent matching result.
+    // The hard-coded scores listed here are not very important, though it shall
+    // be higher for better matches to improve the resulting cost. When
+    // computing the scores of matching one sub-tree with another, we are
+    // basically counting the number of values that are matching. So even if all
+    // scores are set to 1, we would still get a decent matching result.
     // However, sometimes we have to break ties. For example we may have to
     // choose between matching loads vs matching opcodes. This is what these
-    // scores are helping us with: they provide the order of preference.
+    // scores are helping us with: they provide the order of preference. Also,
+    // this is important if the scalar is externally used or used in another
+    // tree entry node in the different lane.
 
     /// Loads from consecutive memory addresses, e.g. load(A[i]), load(A[i+1]).
-    static const int ScoreConsecutiveLoads = 3;
+    static const int ScoreConsecutiveLoads = 4;
+    /// Loads from reversed memory addresses, e.g. load(A[i+1]), load(A[i]).
+    static const int ScoreReversedLoads = 3;
     /// ExtractElementInst from same vector and consecutive indexes.
-    static const int ScoreConsecutiveExtracts = 3;
+    static const int ScoreConsecutiveExtracts = 4;
+    /// ExtractElementInst from same vector and reversed indices.
+    static const int ScoreReversedExtracts = 3;
     /// Constants.
     static const int ScoreConstants = 2;
     /// Instructions with the same opcode.
@@ -1041,7 +1054,10 @@ public:
 
     /// \returns the score of placing \p V1 and \p V2 in consecutive lanes.
     static int getShallowScore(Value *V1, Value *V2, const DataLayout &DL,
-                               ScalarEvolution &SE) {
+                               ScalarEvolution &SE, int NumLanes) {
+      if (V1 == V2)
+        return VLOperands::ScoreSplat;
+
       auto *LI1 = dyn_cast<LoadInst>(V1);
       auto *LI2 = dyn_cast<LoadInst>(V2);
       if (LI1 && LI2) {
@@ -1051,8 +1067,17 @@ public:
         Optional<int> Dist = getPointersDiff(
             LI1->getType(), LI1->getPointerOperand(), LI2->getType(),
             LI2->getPointerOperand(), DL, SE, /*StrictCheck=*/true);
-        return (Dist && *Dist == 1) ? VLOperands::ScoreConsecutiveLoads
-                                    : VLOperands::ScoreFail;
+        if (!Dist)
+          return VLOperands::ScoreFail;
+        // The distance is too large - still may be profitable to use masked
+        // loads/gathers.
+        if (std::abs(*Dist) > NumLanes / 2)
+          return VLOperands::ScoreAltOpcodes;
+        // This still will detect consecutive loads, but we might have "holes"
+        // in some cases. It is ok for non-power-2 vectorization and may produce
+        // better results. It should not affect current vectorization.
+        return (*Dist > 0) ? VLOperands::ScoreConsecutiveLoads
+                           : VLOperands::ScoreReversedLoads;
       }
 
       auto *C1 = dyn_cast<Constant>(V1);
@@ -1062,18 +1087,41 @@ public:
 
       // Extracts from consecutive indexes of the same vector better score as
       // the extracts could be optimized away.
-      Value *EV;
-      ConstantInt *Ex1Idx, *Ex2Idx;
-      if (match(V1, m_ExtractElt(m_Value(EV), m_ConstantInt(Ex1Idx))) &&
-          match(V2, m_ExtractElt(m_Deferred(EV), m_ConstantInt(Ex2Idx))) &&
-          Ex1Idx->getZExtValue() + 1 == Ex2Idx->getZExtValue())
-        return VLOperands::ScoreConsecutiveExtracts;
+      Value *EV1;
+      ConstantInt *Ex1Idx;
+      if (match(V1, m_ExtractElt(m_Value(EV1), m_ConstantInt(Ex1Idx)))) {
+        // Undefs are always profitable for extractelements.
+        if (isa<UndefValue>(V2))
+          return VLOperands::ScoreConsecutiveExtracts;
+        Value *EV2 = nullptr;
+        ConstantInt *Ex2Idx = nullptr;
+        if (match(V2,
+                  m_ExtractElt(m_Value(EV2), m_CombineOr(m_ConstantInt(Ex2Idx),
+                                                         m_Undef())))) {
+          // Undefs are always profitable for extractelements.
+          if (!Ex2Idx)
+            return VLOperands::ScoreConsecutiveExtracts;
+          if (isUndefVector(EV2) && EV2->getType() == EV1->getType())
+            return VLOperands::ScoreConsecutiveExtracts;
+          if (EV2 == EV1) {
+            int Idx1 = Ex1Idx->getZExtValue();
+            int Idx2 = Ex2Idx->getZExtValue();
+            int Dist = Idx2 - Idx1;
+            // The distance is too large - still may be profitable to use
+            // shuffles.
+            if (std::abs(Dist) > NumLanes / 2)
+              return VLOperands::ScoreAltOpcodes;
+            return (Dist > 0) ? VLOperands::ScoreConsecutiveExtracts
+                              : VLOperands::ScoreReversedExtracts;
+          }
+        }
+      }
 
       auto *I1 = dyn_cast<Instruction>(V1);
       auto *I2 = dyn_cast<Instruction>(V2);
       if (I1 && I2) {
-        if (I1 == I2)
-          return VLOperands::ScoreSplat;
+        if (I1->getParent() != I2->getParent())
+          return VLOperands::ScoreFail;
         InstructionsState S = getSameOpcode({I1, I2});
         // Note: Only consider instructions with <= 2 operands to avoid
         // complexity explosion.
@@ -1088,11 +1136,13 @@ public:
       return VLOperands::ScoreFail;
     }
 
-    /// Holds the values and their lane that are taking part in the look-ahead
+    /// Holds the values and their lanes that are taking part in the look-ahead
     /// score calculation. This is used in the external uses cost calculation.
-    SmallDenseMap<Value *, int> InLookAheadValues;
+    /// Need to hold all the lanes in case of splat/broadcast at least to
+    /// correctly check for the use in the different lane.
+    SmallDenseMap<Value *, SmallSet<int, 4>> InLookAheadValues;
 
-    /// \Returns the additinal cost due to uses of \p LHS and \p RHS that are
+    /// \returns the additional cost due to uses of \p LHS and \p RHS that are
     /// either external to the vectorized code, or require shuffling.
     int getExternalUsesCost(const std::pair<Value *, int> &LHS,
                             const std::pair<Value *, int> &RHS) {
@@ -1116,22 +1166,30 @@ public:
         for (User *U : V->users()) {
           if (const TreeEntry *UserTE = R.getTreeEntry(U)) {
             // The user is in the VectorizableTree. Check if we need to insert.
-            auto It = llvm::find(UserTE->Scalars, U);
-            assert(It != UserTE->Scalars.end() && "U is in UserTE");
-            int UserLn = std::distance(UserTE->Scalars.begin(), It);
+            int UserLn = UserTE->findLaneForValue(U);
             assert(UserLn >= 0 && "Bad lane");
-            if (UserLn != Ln)
+            // If the values are different, check just the line of the current
+            // value. If the values are the same, need to add UserInDiffLaneCost
+            // only if UserLn does not match both line numbers.
+            if ((LHS.first != RHS.first && UserLn != Ln) ||
+                (LHS.first == RHS.first && UserLn != LHS.second &&
+                 UserLn != RHS.second)) {
               Cost += UserInDiffLaneCost;
+              break;
+            }
           } else {
             // Check if the user is in the look-ahead code.
             auto It2 = InLookAheadValues.find(U);
             if (It2 != InLookAheadValues.end()) {
               // The user is in the look-ahead code. Check the lane.
-              if (It2->second != Ln)
+              if (!It2->getSecond().contains(Ln)) {
                 Cost += UserInDiffLaneCost;
+                break;
+              }
             } else {
               // The user is neither in SLP tree nor in the look-ahead code.
               Cost += ExternalUseCost;
+              break;
             }
           }
           // Limit the number of visited uses to cap compilation time.
@@ -1170,32 +1228,36 @@ public:
       Value *V1 = LHS.first;
       Value *V2 = RHS.first;
       // Get the shallow score of V1 and V2.
-      int ShallowScoreAtThisLevel =
-          std::max((int)ScoreFail, getShallowScore(V1, V2, DL, SE) -
-                                       getExternalUsesCost(LHS, RHS));
+      int ShallowScoreAtThisLevel = std::max(
+          (int)ScoreFail, getShallowScore(V1, V2, DL, SE, getNumLanes()) -
+                              getExternalUsesCost(LHS, RHS));
       int Lane1 = LHS.second;
       int Lane2 = RHS.second;
 
       // If reached MaxLevel,
       //  or if V1 and V2 are not instructions,
       //  or if they are SPLAT,
-      //  or if they are not consecutive, early return the current cost.
+      //  or if they are not consecutive,
+      //  or if profitable to vectorize loads or extractelements, early return
+      //  the current cost.
       auto *I1 = dyn_cast<Instruction>(V1);
       auto *I2 = dyn_cast<Instruction>(V2);
       if (CurrLevel == MaxLevel || !(I1 && I2) || I1 == I2 ||
           ShallowScoreAtThisLevel == VLOperands::ScoreFail ||
-          (isa<LoadInst>(I1) && isa<LoadInst>(I2) && ShallowScoreAtThisLevel))
+          (((isa<LoadInst>(I1) && isa<LoadInst>(I2)) ||
+            (isa<ExtractElementInst>(I1) && isa<ExtractElementInst>(I2))) &&
+           ShallowScoreAtThisLevel))
         return ShallowScoreAtThisLevel;
       assert(I1 && I2 && "Should have early exited.");
 
       // Keep track of in-tree values for determining the external-use cost.
-      InLookAheadValues[V1] = Lane1;
-      InLookAheadValues[V2] = Lane2;
+      InLookAheadValues[V1].insert(Lane1);
+      InLookAheadValues[V2].insert(Lane2);
 
       // Contains the I2 operand indexes that got matched with I1 operands.
       SmallSet<unsigned, 4> Op2Used;
 
-      // Recursion towards the operands of I1 and I2. We are trying all possbile
+      // Recursion towards the operands of I1 and I2. We are trying all possible
       // operand pairs, and keeping track of the best score.
       for (unsigned OpIdx1 = 0, NumOperands1 = I1->getNumOperands();
            OpIdx1 != NumOperands1; ++OpIdx1) {
@@ -1319,27 +1381,79 @@ public:
       return None;
     }
 
-    /// Helper for reorderOperandVecs. \Returns the lane that we should start
-    /// reordering from. This is the one which has the least number of operands
-    /// that can freely move about.
+    /// Helper for reorderOperandVecs.
+    /// \returns the lane that we should start reordering from. This is the one
+    /// which has the least number of operands that can freely move about or
+    /// less profitable because it already has the most optimal set of operands.
     unsigned getBestLaneToStartReordering() const {
-      unsigned BestLane = 0;
       unsigned Min = UINT_MAX;
-      for (unsigned Lane = 0, NumLanes = getNumLanes(); Lane != NumLanes;
-           ++Lane) {
-        unsigned NumFreeOps = getMaxNumOperandsThatCanBeReordered(Lane);
-        if (NumFreeOps < Min) {
-          Min = NumFreeOps;
-          BestLane = Lane;
+      unsigned SameOpNumber = 0;
+      // std::pair<unsigned, unsigned> is used to implement a simple voting
+      // algorithm and choose the lane with the least number of operands that
+      // can freely move about or less profitable because it already has the
+      // most optimal set of operands. The first unsigned is a counter for
+      // voting, the second unsigned is the counter of lanes with instructions
+      // with same/alternate opcodes and same parent basic block.
+      MapVector<unsigned, std::pair<unsigned, unsigned>> HashMap;
+      // Try to be closer to the original results, if we have multiple lanes
+      // with same cost. If 2 lanes have the same cost, use the one with the
+      // lowest index.
+      for (int I = getNumLanes(); I > 0; --I) {
+        unsigned Lane = I - 1;
+        OperandsOrderData NumFreeOpsHash =
+            getMaxNumOperandsThatCanBeReordered(Lane);
+        // Compare the number of operands that can move and choose the one with
+        // the least number.
+        if (NumFreeOpsHash.NumOfAPOs < Min) {
+          Min = NumFreeOpsHash.NumOfAPOs;
+          SameOpNumber = NumFreeOpsHash.NumOpsWithSameOpcodeParent;
+          HashMap.clear();
+          HashMap[NumFreeOpsHash.Hash] = std::make_pair(1, Lane);
+        } else if (NumFreeOpsHash.NumOfAPOs == Min &&
+                   NumFreeOpsHash.NumOpsWithSameOpcodeParent < SameOpNumber) {
+          // Select the most optimal lane in terms of number of operands that
+          // should be moved around.
+          SameOpNumber = NumFreeOpsHash.NumOpsWithSameOpcodeParent;
+          HashMap[NumFreeOpsHash.Hash] = std::make_pair(1, Lane);
+        } else if (NumFreeOpsHash.NumOfAPOs == Min &&
+                   NumFreeOpsHash.NumOpsWithSameOpcodeParent == SameOpNumber) {
+          ++HashMap[NumFreeOpsHash.Hash].first;
+        }
+      }
+      // Select the lane with the minimum counter.
+      unsigned BestLane = 0;
+      unsigned CntMin = UINT_MAX;
+      for (const auto &Data : reverse(HashMap)) {
+        if (Data.second.first < CntMin) {
+          CntMin = Data.second.first;
+          BestLane = Data.second.second;
         }
       }
       return BestLane;
     }
 
-    /// \Returns the maximum number of operands that are allowed to be reordered
-    /// for \p Lane. This is used as a heuristic for selecting the first lane to
-    /// start operand reordering.
-    unsigned getMaxNumOperandsThatCanBeReordered(unsigned Lane) const {
+    /// Data structure that helps to reorder operands.
+    struct OperandsOrderData {
+      /// The best number of operands with the same APOs, which can be
+      /// reordered.
+      unsigned NumOfAPOs = UINT_MAX;
+      /// Number of operands with the same/alternate instruction opcode and
+      /// parent.
+      unsigned NumOpsWithSameOpcodeParent = 0;
+      /// Hash for the actual operands ordering.
+      /// Used to count operands, actually their position id and opcode
+      /// value. It is used in the voting mechanism to find the lane with the
+      /// least number of operands that can freely move about or less profitable
+      /// because it already has the most optimal set of operands. Can be
+      /// replaced with SmallVector<unsigned> instead but hash code is faster
+      /// and requires less memory.
+      unsigned Hash = 0;
+    };
+    /// \returns the maximum number of operands that are allowed to be reordered
+    /// for \p Lane and the number of compatible instructions(with the same
+    /// parent/opcode). This is used as a heuristic for selecting the first lane
+    /// to start operand reordering.
+    OperandsOrderData getMaxNumOperandsThatCanBeReordered(unsigned Lane) const {
       unsigned CntTrue = 0;
       unsigned NumOperands = getNumOperands();
       // Operands with the same APO can be reordered. We therefore need to count
@@ -1348,11 +1462,45 @@ public:
       // a map. Instead we can simply count the number of operands that
       // correspond to one of them (in this case the 'true' APO), and calculate
       // the other by subtracting it from the total number of operands.
-      for (unsigned OpIdx = 0; OpIdx != NumOperands; ++OpIdx)
-        if (getData(OpIdx, Lane).APO)
+      // Operands with the same instruction opcode and parent are more
+      // profitable since we don't need to move them in many cases, with a high
+      // probability such lane already can be vectorized effectively.
+      bool AllUndefs = true;
+      unsigned NumOpsWithSameOpcodeParent = 0;
+      Instruction *OpcodeI = nullptr;
+      BasicBlock *Parent = nullptr;
+      unsigned Hash = 0;
+      for (unsigned OpIdx = 0; OpIdx != NumOperands; ++OpIdx) {
+        const OperandData &OpData = getData(OpIdx, Lane);
+        if (OpData.APO)
           ++CntTrue;
-      unsigned CntFalse = NumOperands - CntTrue;
-      return std::max(CntTrue, CntFalse);
+        // Use Boyer-Moore majority voting for finding the majority opcode and
+        // the number of times it occurs.
+        if (auto *I = dyn_cast<Instruction>(OpData.V)) {
+          if (!OpcodeI || !getSameOpcode({OpcodeI, I}).getOpcode() ||
+              I->getParent() != Parent) {
+            if (NumOpsWithSameOpcodeParent == 0) {
+              NumOpsWithSameOpcodeParent = 1;
+              OpcodeI = I;
+              Parent = I->getParent();
+            } else {
+              --NumOpsWithSameOpcodeParent;
+            }
+          } else {
+            ++NumOpsWithSameOpcodeParent;
+          }
+        }
+        Hash = hash_combine(
+            Hash, hash_value((OpIdx + 1) * (OpData.V->getValueID() + 1)));
+        AllUndefs = AllUndefs && isa<UndefValue>(OpData.V);
+      }
+      if (AllUndefs)
+        return {};
+      OperandsOrderData Data;
+      Data.NumOfAPOs = std::max(CntTrue, NumOperands - CntTrue);
+      Data.NumOpsWithSameOpcodeParent = NumOpsWithSameOpcodeParent;
+      Data.Hash = Hash;
+      return Data;
     }
 
     /// Go through the instructions in VL and append their operands.
@@ -1500,11 +1648,37 @@ public:
           ReorderingModes[OpIdx] = ReorderingMode::Failed;
       }
 
+      // Check that we don't have same operands. No need to reorder if operands
+      // are just perfect diamond or shuffled diamond match. Do not do it only
+      // for possible broadcasts or non-power of 2 number of scalars (just for
+      // now).
+      auto &&SkipReordering = [this]() {
+        SmallPtrSet<Value *, 4> UniqueValues;
+        ArrayRef<OperandData> Op0 = OpsVec.front();
+        for (const OperandData &Data : Op0)
+          UniqueValues.insert(Data.V);
+        for (ArrayRef<OperandData> Op : drop_begin(OpsVec, 1)) {
+          if (any_of(Op, [&UniqueValues](const OperandData &Data) {
+                return !UniqueValues.contains(Data.V);
+              }))
+            return false;
+        }
+        // TODO: Check if we can remove a check for non-power-2 number of
+        // scalars after full support of non-power-2 vectorization.
+        return UniqueValues.size() != 2 && isPowerOf2_32(UniqueValues.size());
+      };
+
       // If the initial strategy fails for any of the operand indexes, then we
       // perform reordering again in a second pass. This helps avoid assigning
       // high priority to the failed strategy, and should improve reordering for
       // the non-failed operand indexes.
       for (int Pass = 0; Pass != 2; ++Pass) {
+        // Check if no need to reorder operands since they're are perfect or
+        // shuffled diamond match.
+        // Need to to do it to avoid extra external use cost counting for
+        // shuffled matches, which may cause regressions.
+        if (SkipReordering())
+          break;
         // Skip the second pass if the first pass did not fail.
         bool StrategyFailed = false;
         // Mark all operand data as free to use.
@@ -1792,9 +1966,10 @@ private:
       if (Operands.size() < OpIdx + 1)
         Operands.resize(OpIdx + 1);
       assert(Operands[OpIdx].empty() && "Already resized?");
-      Operands[OpIdx].resize(Scalars.size());
-      for (unsigned Lane = 0, E = Scalars.size(); Lane != E; ++Lane)
-        Operands[OpIdx][Lane] = OpVL[Lane];
+      assert(OpVL.size() <= Scalars.size() &&
+             "Number of operands is greater than the number of scalars.");
+      Operands[OpIdx].resize(OpVL.size());
+      copy(OpVL, Operands[OpIdx].begin());
     }
 
     /// Set the operands of this bundle in their original order.
@@ -1944,7 +2119,7 @@ private:
       if (ReuseShuffleIndices.empty())
         dbgs() << "Empty";
       else
-        for (unsigned ReuseIdx : ReuseShuffleIndices)
+        for (int ReuseIdx : ReuseShuffleIndices)
           dbgs() << ReuseIdx << ", ";
       dbgs() << "\n";
       dbgs() << "ReorderIndices: ";
@@ -2819,6 +2994,50 @@ BoUpSLP::findReusedOrderedScalars(const BoUpSLP::TreeEntry &TE) {
   return None;
 }
 
+Optional<BoUpSLP::OrdersType> BoUpSLP::getReorderingData(const TreeEntry &TE,
+                                                         bool TopToBottom) {
+  // No need to reorder if need to shuffle reuses, still need to shuffle the
+  // node.
+  if (!TE.ReuseShuffleIndices.empty())
+    return None;
+  if (TE.State == TreeEntry::Vectorize &&
+      (isa<LoadInst, ExtractElementInst, ExtractValueInst>(TE.getMainOp()) ||
+       (TopToBottom && isa<StoreInst, InsertElementInst>(TE.getMainOp()))) &&
+      !TE.isAltShuffle())
+    return TE.ReorderIndices;
+  if (TE.State == TreeEntry::NeedToGather) {
+    // TODO: add analysis of other gather nodes with extractelement
+    // instructions and other values/instructions, not only undefs.
+    if (((TE.getOpcode() == Instruction::ExtractElement &&
+          !TE.isAltShuffle()) ||
+         (all_of(TE.Scalars,
+                 [](Value *V) {
+                   return isa<UndefValue, ExtractElementInst>(V);
+                 }) &&
+          any_of(TE.Scalars,
+                 [](Value *V) { return isa<ExtractElementInst>(V); }))) &&
+        all_of(TE.Scalars,
+               [](Value *V) {
+                 auto *EE = dyn_cast<ExtractElementInst>(V);
+                 return !EE || isa<FixedVectorType>(EE->getVectorOperandType());
+               }) &&
+        allSameType(TE.Scalars)) {
+      // Check that gather of extractelements can be represented as
+      // just a shuffle of a single vector.
+      OrdersType CurrentOrder;
+      bool Reuse = canReuseExtract(TE.Scalars, TE.getMainOp(), CurrentOrder);
+      if (Reuse || !CurrentOrder.empty()) {
+        if (!CurrentOrder.empty())
+          fixupOrderingIndices(CurrentOrder);
+        return CurrentOrder;
+      }
+    }
+    if (Optional<OrdersType> CurrentOrder = findReusedOrderedScalars(TE))
+      return CurrentOrder;
+  }
+  return None;
+}
+
 void BoUpSLP::reorderTopToBottom() {
   // Maps VF to the graph nodes.
   DenseMap<unsigned, SmallPtrSet<TreeEntry *, 4>> VFToOrderedEntries;
@@ -2826,42 +3045,15 @@ void BoUpSLP::reorderTopToBottom() {
   // their ordering.
   DenseMap<const TreeEntry *, OrdersType> GathersToOrders;
   // Find all reorderable nodes with the given VF.
-  // Currently the are vectorized loads,extracts + some gathering of extracts.
+  // Currently the are vectorized stores,loads,extracts + some gathering of
+  // extracts.
   for_each(VectorizableTree, [this, &VFToOrderedEntries, &GathersToOrders](
                                  const std::unique_ptr<TreeEntry> &TE) {
-    // No need to reorder if need to shuffle reuses, still need to shuffle the
-    // node.
-    if (!TE->ReuseShuffleIndices.empty())
-      return;
-    if (TE->State == TreeEntry::Vectorize &&
-        isa<LoadInst, ExtractElementInst, ExtractValueInst, StoreInst,
-            InsertElementInst>(TE->getMainOp()) &&
-        !TE->isAltShuffle()) {
+    if (Optional<OrdersType> CurrentOrder =
+            getReorderingData(*TE.get(), /*TopToBottom=*/true)) {
       VFToOrderedEntries[TE->Scalars.size()].insert(TE.get());
-      return;
-    }
-    if (TE->State == TreeEntry::NeedToGather) {
-      if (TE->getOpcode() == Instruction::ExtractElement &&
-          !TE->isAltShuffle() &&
-          isa<FixedVectorType>(cast<ExtractElementInst>(TE->getMainOp())
-                                   ->getVectorOperandType()) &&
-          allSameType(TE->Scalars) && allSameBlock(TE->Scalars)) {
-        // Check that gather of extractelements can be represented as
-        // just a shuffle of a single vector.
-        OrdersType CurrentOrder;
-        bool Reuse =
-            canReuseExtract(TE->Scalars, TE->getMainOp(), CurrentOrder);
-        if (Reuse || !CurrentOrder.empty()) {
-          VFToOrderedEntries[TE->Scalars.size()].insert(TE.get());
-          GathersToOrders.try_emplace(TE.get(), CurrentOrder);
-          return;
-        }
-      }
-      if (Optional<OrdersType> CurrentOrder =
-              findReusedOrderedScalars(*TE.get())) {
-        VFToOrderedEntries[TE->Scalars.size()].insert(TE.get());
+      if (TE->State != TreeEntry::Vectorize)
         GathersToOrders.try_emplace(TE.get(), *CurrentOrder);
-      }
     }
   });
 
@@ -2993,44 +3185,11 @@ void BoUpSLP::reorderBottomToTop(bool IgnoreReorder) {
                                  const std::unique_ptr<TreeEntry> &TE) {
     if (TE->State != TreeEntry::Vectorize)
       NonVectorized.push_back(TE.get());
-    // No need to reorder if need to shuffle reuses, still need to shuffle the
-    // node.
-    if (!TE->ReuseShuffleIndices.empty())
-      return;
-    if (TE->State == TreeEntry::Vectorize &&
-        isa<LoadInst, ExtractElementInst, ExtractValueInst>(TE->getMainOp()) &&
-        !TE->isAltShuffle()) {
+    if (Optional<OrdersType> CurrentOrder =
+            getReorderingData(*TE.get(), /*TopToBottom=*/false)) {
       OrderedEntries.insert(TE.get());
-      return;
-    }
-    if (TE->State == TreeEntry::NeedToGather) {
-      if (TE->getOpcode() == Instruction::ExtractElement &&
-          !TE->isAltShuffle() &&
-          isa<FixedVectorType>(cast<ExtractElementInst>(TE->getMainOp())
-                                   ->getVectorOperandType()) &&
-          allSameType(TE->Scalars) && allSameBlock(TE->Scalars)) {
-        // Check that gather of extractelements can be represented as
-        // just a shuffle of a single vector with a single user only.
-        OrdersType CurrentOrder;
-        bool Reuse =
-            canReuseExtract(TE->Scalars, TE->getMainOp(), CurrentOrder);
-        if ((Reuse || !CurrentOrder.empty()) &&
-            !any_of(VectorizableTree,
-                    [&TE](const std::unique_ptr<TreeEntry> &Entry) {
-                      return Entry->State == TreeEntry::NeedToGather &&
-                             Entry.get() != TE.get() &&
-                             Entry->isSame(TE->Scalars);
-                    })) {
-          OrderedEntries.insert(TE.get());
-          GathersToOrders.try_emplace(TE.get(), CurrentOrder);
-          return;
-        }
-      }
-      if (Optional<OrdersType> CurrentOrder =
-              findReusedOrderedScalars(*TE.get())) {
-        OrderedEntries.insert(TE.get());
+      if (TE->State != TreeEntry::Vectorize)
         GathersToOrders.try_emplace(TE.get(), *CurrentOrder);
-      }
     }
   });
 
@@ -3392,9 +3551,14 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
     // Check that every instruction appears once in this bundle.
     DenseMap<Value *, unsigned> UniquePositions;
     for (Value *V : VL) {
+      if (isConstant(V)) {
+        ReuseShuffleIndicies.emplace_back(
+            isa<UndefValue>(V) ? UndefMaskElem : UniqueValues.size());
+        UniqueValues.emplace_back(V);
+        continue;
+      }
       auto Res = UniquePositions.try_emplace(V, UniqueValues.size());
-      ReuseShuffleIndicies.emplace_back(isa<UndefValue>(V) ? -1
-                                                           : Res.first->second);
+      ReuseShuffleIndicies.emplace_back(Res.first->second);
       if (Res.second)
         UniqueValues.emplace_back(V);
     }
@@ -3404,6 +3568,11 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
     } else {
       LLVM_DEBUG(dbgs() << "SLP: Shuffle for reused scalars.\n");
       if (NumUniqueScalarValues <= 1 ||
+          (UniquePositions.size() == 1 && all_of(UniqueValues,
+                                                 [](Value *V) {
+                                                   return isa<UndefValue>(V) ||
+                                                          !isConstant(V);
+                                                 })) ||
           !llvm::isPowerOf2_32(NumUniqueScalarValues)) {
         LLVM_DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");
         newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
@@ -3508,11 +3677,9 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
     }
   }
 
-  // If any of the scalars is marked as a value that needs to stay scalar, then
-  // we need to gather the scalars.
   // The reduction nodes (stored in UserIgnoreList) also should stay scalar.
   for (Value *V : VL) {
-    if (MustGather.count(V) || is_contained(UserIgnoreList, V)) {
+    if (is_contained(UserIgnoreList, V)) {
       LLVM_DEBUG(dbgs() << "SLP: Gathering due to gathered scalar.\n");
       if (TryToFindDuplicates(S))
         newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
@@ -4219,10 +4386,17 @@ unsigned BoUpSLP::canMapToVector(Type *T, const DataLayout &DL) const {
 
 bool BoUpSLP::canReuseExtract(ArrayRef<Value *> VL, Value *OpValue,
                               SmallVectorImpl<unsigned> &CurrentOrder) const {
-  Instruction *E0 = cast<Instruction>(OpValue);
-  assert(E0->getOpcode() == Instruction::ExtractElement ||
-         E0->getOpcode() == Instruction::ExtractValue);
-  assert(E0->getOpcode() == getSameOpcode(VL).getOpcode() && "Invalid opcode");
+  const auto *It = find_if(VL, [](Value *V) {
+    return isa<ExtractElementInst, ExtractValueInst>(V);
+  });
+  assert(It != VL.end() && "Expected at least one extract instruction.");
+  auto *E0 = cast<Instruction>(*It);
+  assert(all_of(VL,
+                [](Value *V) {
+                  return isa<UndefValue, ExtractElementInst, ExtractValueInst>(
+                      V);
+                }) &&
+         "Invalid opcode");
   // Check if all of the extracts come from the same vector and from the
   // correct offset.
   Value *Vec = E0->getOperand(0);
@@ -4255,23 +4429,28 @@ bool BoUpSLP::canReuseExtract(ArrayRef<Value *> VL, Value *OpValue,
   // Also, later we can check that all the indices are used and we have a
   // consecutive access in the extract instructions, by checking that no
   // element of CurrentOrder still has value E + 1.
-  CurrentOrder.assign(E, E + 1);
+  CurrentOrder.assign(E, E);
   unsigned I = 0;
   for (; I < E; ++I) {
-    auto *Inst = cast<Instruction>(VL[I]);
+    auto *Inst = dyn_cast<Instruction>(VL[I]);
+    if (!Inst)
+      continue;
     if (Inst->getOperand(0) != Vec)
       break;
+    if (auto *EE = dyn_cast<ExtractElementInst>(Inst))
+      if (isa<UndefValue>(EE->getIndexOperand()))
+        continue;
     Optional<unsigned> Idx = getExtractIndex(Inst);
     if (!Idx)
       break;
     const unsigned ExtIdx = *Idx;
     if (ExtIdx != I) {
-      if (ExtIdx >= E || CurrentOrder[ExtIdx] != E + 1)
+      if (ExtIdx >= E || CurrentOrder[ExtIdx] != E)
         break;
       ShouldKeepOrder = false;
       CurrentOrder[ExtIdx] = I;
     } else {
-      if (CurrentOrder[I] != E + 1)
+      if (CurrentOrder[I] != E)
         break;
       CurrentOrder[I] = I;
     }
@@ -4287,8 +4466,8 @@ bool BoUpSLP::canReuseExtract(ArrayRef<Value *> VL, Value *OpValue,
 bool BoUpSLP::areAllUsersVectorized(Instruction *I,
                                     ArrayRef<Value *> VectorizedVals) const {
   return (I->hasOneUse() && is_contained(VectorizedVals, I)) ||
-         llvm::all_of(I->users(), [this](User *U) {
-           return ScalarToTreeEntry.count(U) > 0;
+         all_of(I->users(), [this](User *U) {
+           return ScalarToTreeEntry.count(U) > 0 || MustGather.contains(U);
          });
 }
 
@@ -4348,6 +4527,10 @@ computeExtractCost(ArrayRef<Value *> VL, FixedVectorType *VecTy,
   for (auto *V : VL) {
     ++Idx;
 
+    // Need to exclude undefs from analysis.
+    if (isa<UndefValue>(V) || Mask[Idx] == UndefMaskElem)
+      continue;
+
     // Reached the start of a new vector registers.
     if (Idx % EltsPerVector == 0) {
       AllConsecutive = true;
@@ -4357,9 +4540,11 @@ computeExtractCost(ArrayRef<Value *> VL, FixedVectorType *VecTy,
     // Check all extracts for a vector register on the target directly
     // extract values in order.
     unsigned CurrentIdx = *getExtractIndex(cast<Instruction>(V));
-    unsigned PrevIdx = *getExtractIndex(cast<Instruction>(VL[Idx - 1]));
-    AllConsecutive &= PrevIdx + 1 == CurrentIdx &&
-                      CurrentIdx % EltsPerVector == Idx % EltsPerVector;
+    if (!isa<UndefValue>(VL[Idx - 1]) && Mask[Idx - 1] != UndefMaskElem) {
+      unsigned PrevIdx = *getExtractIndex(cast<Instruction>(VL[Idx - 1]));
+      AllConsecutive &= PrevIdx + 1 == CurrentIdx &&
+                        CurrentIdx % EltsPerVector == Idx % EltsPerVector;
+    }
 
     if (AllConsecutive)
       continue;
@@ -4442,9 +4627,9 @@ InstructionCost BoUpSLP::getEntryCost(const TreeEntry *E,
   // FIXME: it tries to fix a problem with MSVC buildbots.
   TargetTransformInfo &TTIRef = *TTI;
   auto &&AdjustExtractsCost = [this, &TTIRef, CostKind, VL, VecTy,
-                               VectorizedVals](InstructionCost &Cost,
-                                               bool IsGather) {
+                               VectorizedVals, E](InstructionCost &Cost) {
     DenseMap<Value *, int> ExtractVectorsTys;
+    SmallPtrSet<Value *, 4> CheckedExtracts;
     for (auto *V : VL) {
       if (isa<UndefValue>(V))
         continue;
@@ -4452,7 +4637,12 @@ InstructionCost BoUpSLP::getEntryCost(const TreeEntry *E,
       // instruction itself is not going to be vectorized, consider this
       // instruction as dead and remove its cost from the final cost of the
       // vectorized tree.
-      if (!areAllUsersVectorized(cast<Instruction>(V), VectorizedVals))
+      // Also, avoid adjusting the cost for extractelements with multiple uses
+      // in different graph entries.
+      const TreeEntry *VE = getTreeEntry(V);
+      if (!CheckedExtracts.insert(V).second ||
+          !areAllUsersVectorized(cast<Instruction>(V), VectorizedVals) ||
+          (VE && VE != E))
         continue;
       auto *EE = cast<ExtractElementInst>(V);
       Optional<unsigned> EEIdx = getExtractIndex(EE);
@@ -4549,11 +4739,6 @@ InstructionCost BoUpSLP::getEntryCost(const TreeEntry *E,
       }
       return GatherCost;
     }
-    if (isSplat(VL)) {
-      // Found the broadcasting of the single scalar, calculate the cost as the
-      // broadcast.
-      return TTI->getShuffleCost(TargetTransformInfo::SK_Broadcast, VecTy);
-    }
     if ((E->getOpcode() == Instruction::ExtractElement ||
          all_of(E->Scalars,
                 [](Value *V) {
@@ -4571,13 +4756,20 @@ InstructionCost BoUpSLP::getEntryCost(const TreeEntry *E,
         // single input vector or of 2 input vectors.
         InstructionCost Cost =
             computeExtractCost(VL, VecTy, *ShuffleKind, Mask, *TTI);
-        AdjustExtractsCost(Cost, /*IsGather=*/true);
+        AdjustExtractsCost(Cost);
         if (NeedToShuffleReuses)
           Cost += TTI->getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc,
                                       FinalVecTy, E->ReuseShuffleIndices);
         return Cost;
       }
     }
+    if (isSplat(VL)) {
+      // Found the broadcasting of the single scalar, calculate the cost as the
+      // broadcast.
+      assert(VecTy == FinalVecTy &&
+             "No reused scalars expected for broadcast.");
+      return TTI->getShuffleCost(TargetTransformInfo::SK_Broadcast, VecTy);
+    }
     InstructionCost ReuseShuffleCost = 0;
     if (NeedToShuffleReuses)
       ReuseShuffleCost = TTI->getShuffleCost(
@@ -4755,7 +4947,7 @@ InstructionCost BoUpSLP::getEntryCost(const TreeEntry *E,
               TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, I);
         }
       } else {
-        AdjustExtractsCost(CommonCost, /*IsGather=*/false);
+        AdjustExtractsCost(CommonCost);
       }
       return CommonCost;
     }
@@ -5211,15 +5403,15 @@ static bool isLoadCombineCandidateImpl(Value *Root, unsigned NumElts,
       FoundOr = true;
   }
   // Check if the input is an extended load of the required or/shift expression.
-  Value *LoadPtr;
+  Value *Load;
   if ((MustMatchOrInst && !FoundOr) || ZextLoad == Root ||
-      !match(ZextLoad, m_ZExt(m_Load(m_Value(LoadPtr)))))
+      !match(ZextLoad, m_ZExt(m_Value(Load))) || !isa<LoadInst>(Load))
     return false;
 
   // Require that the total load bit width is a legal integer type.
   // For example, <8 x i8> --> i64 is a legal integer on a 64-bit target.
   // But <16 x i8> --> i128 is not, so the backend probably can't reduce it.
-  Type *SrcTy = LoadPtr->getType()->getPointerElementType();
+  Type *SrcTy = Load->getType();
   unsigned LoadBitWidth = SrcTy->getIntegerBitWidth() * NumElts;
   if (!TTI->isTypeLegal(IntegerType::get(Root->getContext(), LoadBitWidth)))
     return false;
@@ -9061,8 +9253,7 @@ private:
            "A call to the llvm.fmuladd intrinsic is not handled yet");
 
     ++NumVectorInstructions;
-    return createSimpleTargetReduction(Builder, TTI, VectorizedValue, RdxKind,
-                                       ReductionOps.back());
+    return createSimpleTargetReduction(Builder, TTI, VectorizedValue, RdxKind);
   }
 };
 
@@ -9473,6 +9664,59 @@ tryToVectorizeSequence(SmallVectorImpl<T *> &Incoming,
   return Changed;
 }
 
+/// Compare two cmp instructions. If IsCompatibility is true, function returns
+/// true if 2 cmps have same/swapped predicates and mos compatible corresponding
+/// operands. If IsCompatibility is false, function implements strict weak
+/// ordering relation between two cmp instructions, returning true if the first
+/// instruction is "less" than the second, i.e. its predicate is less than the
+/// predicate of the second or the operands IDs are less than the operands IDs
+/// of the second cmp instruction.
+template <bool IsCompatibility>
+static bool compareCmp(Value *V, Value *V2,
+                       function_ref<bool(Instruction *)> IsDeleted) {
+  auto *CI1 = cast<CmpInst>(V);
+  auto *CI2 = cast<CmpInst>(V2);
+  if (IsDeleted(CI2) || !isValidElementType(CI2->getType()))
+    return false;
+  if (CI1->getOperand(0)->getType()->getTypeID() <
+      CI2->getOperand(0)->getType()->getTypeID())
+    return !IsCompatibility;
+  if (CI1->getOperand(0)->getType()->getTypeID() >
+      CI2->getOperand(0)->getType()->getTypeID())
+    return false;
+  CmpInst::Predicate Pred1 = CI1->getPredicate();
+  CmpInst::Predicate Pred2 = CI2->getPredicate();
+  CmpInst::Predicate SwapPred1 = CmpInst::getSwappedPredicate(Pred1);
+  CmpInst::Predicate SwapPred2 = CmpInst::getSwappedPredicate(Pred2);
+  CmpInst::Predicate BasePred1 = std::min(Pred1, SwapPred1);
+  CmpInst::Predicate BasePred2 = std::min(Pred2, SwapPred2);
+  if (BasePred1 < BasePred2)
+    return !IsCompatibility;
+  if (BasePred1 > BasePred2)
+    return false;
+  // Compare operands.
+  bool LEPreds = Pred1 <= Pred2;
+  bool GEPreds = Pred1 >= Pred2;
+  for (int I = 0, E = CI1->getNumOperands(); I < E; ++I) {
+    auto *Op1 = CI1->getOperand(LEPreds ? I : E - I - 1);
+    auto *Op2 = CI2->getOperand(GEPreds ? I : E - I - 1);
+    if (Op1->getValueID() < Op2->getValueID())
+      return !IsCompatibility;
+    if (Op1->getValueID() > Op2->getValueID())
+      return false;
+    if (auto *I1 = dyn_cast<Instruction>(Op1))
+      if (auto *I2 = dyn_cast<Instruction>(Op2)) {
+        if (I1->getParent() != I2->getParent())
+          return false;
+        InstructionsState S = getSameOpcode({I1, I2});
+        if (S.getOpcode())
+          continue;
+        return false;
+      }
+  }
+  return IsCompatibility;
+}
+
 bool SLPVectorizerPass::vectorizeSimpleInstructions(
     SmallVectorImpl<Instruction *> &Instructions, BasicBlock *BB, BoUpSLP &R,
     bool AtTerminator) {
@@ -9504,37 +9748,16 @@ bool SLPVectorizerPass::vectorizeSimpleInstructions(
     }
     // Try to vectorize list of compares.
     // Sort by type, compare predicate, etc.
-    // TODO: Add analysis on the operand opcodes (profitable to vectorize
-    // instructions with same/alternate opcodes/const values).
     auto &&CompareSorter = [&R](Value *V, Value *V2) {
-      auto *CI1 = cast<CmpInst>(V);
-      auto *CI2 = cast<CmpInst>(V2);
-      if (R.isDeleted(CI2) || !isValidElementType(CI2->getType()))
-        return false;
-      if (CI1->getOperand(0)->getType()->getTypeID() <
-          CI2->getOperand(0)->getType()->getTypeID())
-        return true;
-      if (CI1->getOperand(0)->getType()->getTypeID() >
-          CI2->getOperand(0)->getType()->getTypeID())
-        return false;
-      return CI1->getPredicate() < CI2->getPredicate() ||
-             (CI1->getPredicate() > CI2->getPredicate() &&
-              CI1->getPredicate() <
-                  CmpInst::getSwappedPredicate(CI2->getPredicate()));
+      return compareCmp<false>(V, V2,
+                               [&R](Instruction *I) { return R.isDeleted(I); });
     };
 
     auto &&AreCompatibleCompares = [&R](Value *V1, Value *V2) {
       if (V1 == V2)
         return true;
-      auto *CI1 = cast<CmpInst>(V1);
-      auto *CI2 = cast<CmpInst>(V2);
-      if (R.isDeleted(CI2) || !isValidElementType(CI2->getType()))
-        return false;
-      if (CI1->getOperand(0)->getType() != CI2->getOperand(0)->getType())
-        return false;
-      return CI1->getPredicate() == CI2->getPredicate() ||
-             CI1->getPredicate() ==
-                 CmpInst::getSwappedPredicate(CI2->getPredicate());
+      return compareCmp<true>(V1, V2,
+                              [&R](Instruction *I) { return R.isDeleted(I); });
     };
     auto Limit = [&R](Value *V) {
       unsigned EltSize = R.getVectorElementSize(V);
@@ -9592,10 +9815,15 @@ bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
       return true;
     if (Opcodes1.size() > Opcodes2.size())
       return false;
+    Optional<bool> ConstOrder;
     for (int I = 0, E = Opcodes1.size(); I < E; ++I) {
       // Undefs are compatible with any other value.
-      if (isa<UndefValue>(Opcodes1[I]) || isa<UndefValue>(Opcodes2[I]))
+      if (isa<UndefValue>(Opcodes1[I]) || isa<UndefValue>(Opcodes2[I])) {
+        if (!ConstOrder)
+          ConstOrder =
+              !isa<UndefValue>(Opcodes1[I]) && isa<UndefValue>(Opcodes2[I]);
         continue;
+      }
       if (auto *I1 = dyn_cast<Instruction>(Opcodes1[I]))
         if (auto *I2 = dyn_cast<Instruction>(Opcodes2[I])) {
           DomTreeNodeBase<BasicBlock> *NodeI1 = DT->getNode(I1->getParent());
@@ -9614,14 +9842,17 @@ bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
             continue;
           return I1->getOpcode() < I2->getOpcode();
         }
-      if (isa<Constant>(Opcodes1[I]) && isa<Constant>(Opcodes2[I]))
+      if (isa<Constant>(Opcodes1[I]) && isa<Constant>(Opcodes2[I])) {
+        if (!ConstOrder)
+          ConstOrder = Opcodes1[I]->getValueID() < Opcodes2[I]->getValueID();
         continue;
+      }
       if (Opcodes1[I]->getValueID() < Opcodes2[I]->getValueID())
         return true;
       if (Opcodes1[I]->getValueID() > Opcodes2[I]->getValueID())
         return false;
     }
-    return false;
+    return ConstOrder && *ConstOrder;
   };
   auto AreCompatiblePHIs = [&PHIToOpcodes](Value *V1, Value *V2) {
     if (V1 == V2)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 44b5e1df0839..1d9e71663cd2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -374,8 +374,7 @@ VPBasicBlock *VPBasicBlock::splitAt(iterator SplitAt) {
   assert((SplitAt == end() || SplitAt->getParent() == this) &&
          "can only split at a position in the same block");
 
-  SmallVector<VPBlockBase *, 2> Succs(getSuccessors().begin(),
-                                      getSuccessors().end());
+  SmallVector<VPBlockBase *, 2> Succs(successors());
   // First, disconnect the current block from its successors.
   for (VPBlockBase *Succ : Succs)
     VPBlockUtils::disconnectBlocks(this, Succ);
@@ -642,6 +641,7 @@ void VPRecipeBase::moveBefore(VPBasicBlock &BB,
 void VPInstruction::generateInstruction(VPTransformState &State,
                                         unsigned Part) {
   IRBuilder<> &Builder = State.Builder;
+  Builder.SetCurrentDebugLocation(DL);
 
   if (Instruction::isBinaryOp(getOpcode())) {
     Value *A = State.get(getOperand(0), Part);
@@ -768,6 +768,11 @@ void VPInstruction::print(raw_ostream &O, const Twine &Indent,
     O << " ";
     Operand->printAsOperand(O, SlotTracker);
   }
+
+  if (DL) {
+    O << ", !dbg ";
+    DL.print(O);
+  }
 }
 #endif
 
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 810dd5030f95..f4a1883e35d5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -39,6 +39,7 @@
 #include "llvm/ADT/ilist.h"
 #include "llvm/ADT/ilist_node.h"
 #include "llvm/Analysis/VectorUtils.h"
+#include "llvm/IR/DebugLoc.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/Support/InstructionCost.h"
 #include <algorithm>
@@ -51,6 +52,7 @@ namespace llvm {
 
 class BasicBlock;
 class DominatorTree;
+class InductionDescriptor;
 class InnerLoopVectorizer;
 class LoopInfo;
 class raw_ostream;
@@ -500,6 +502,8 @@ public:
   const VPBlocksTy &getSuccessors() const { return Successors; }
   VPBlocksTy &getSuccessors() { return Successors; }
 
+  iterator_range<VPBlockBase **> successors() { return Successors; }
+
   const VPBlocksTy &getPredecessors() const { return Predecessors; }
   VPBlocksTy &getPredecessors() { return Predecessors; }
 
@@ -795,6 +799,7 @@ private:
   typedef unsigned char OpcodeTy;
   OpcodeTy Opcode;
   FastMathFlags FMF;
+  DebugLoc DL;
 
   /// Utility method serving execute(): generates a single instance of the
   /// modeled instruction.
@@ -804,12 +809,14 @@ protected:
   void setUnderlyingInstr(Instruction *I) { setUnderlyingValue(I); }
 
 public:
-  VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands)
+  VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands, DebugLoc DL)
       : VPRecipeBase(VPRecipeBase::VPInstructionSC, Operands),
-        VPValue(VPValue::VPVInstructionSC, nullptr, this), Opcode(Opcode) {}
+        VPValue(VPValue::VPVInstructionSC, nullptr, this), Opcode(Opcode),
+        DL(DL) {}
 
-  VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands)
-      : VPInstruction(Opcode, ArrayRef<VPValue *>(Operands)) {}
+  VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands,
+                DebugLoc DL = {})
+      : VPInstruction(Opcode, ArrayRef<VPValue *>(Operands), DL) {}
 
   /// Method to support type inquiry through isa, cast, and dyn_cast.
   static inline bool classof(const VPValue *V) {
@@ -818,7 +825,7 @@ public:
 
   VPInstruction *clone() const {
     SmallVector<VPValue *, 2> Operands(operands());
-    return new VPInstruction(Opcode, Operands);
+    return new VPInstruction(Opcode, Operands, DL);
   }
 
   /// Method to support type inquiry through isa, cast, and dyn_cast.
@@ -1003,21 +1010,22 @@ public:
 
 /// A recipe for handling phi nodes of integer and floating-point inductions,
 /// producing their vector and scalar values.
-class VPWidenIntOrFpInductionRecipe : public VPRecipeBase {
+class VPWidenIntOrFpInductionRecipe : public VPRecipeBase, public VPValue {
   PHINode *IV;
+  const InductionDescriptor &IndDesc;
 
 public:
-  VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, Instruction *Cast,
-                                TruncInst *Trunc = nullptr)
-      : VPRecipeBase(VPWidenIntOrFpInductionSC, {Start}), IV(IV) {
-    if (Trunc)
-      new VPValue(Trunc, this);
-    else
-      new VPValue(IV, this);
+  VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start,
+                                const InductionDescriptor &IndDesc)
+      : VPRecipeBase(VPWidenIntOrFpInductionSC, {Start}), VPValue(IV, this),
+        IV(IV), IndDesc(IndDesc) {}
+
+  VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start,
+                                const InductionDescriptor &IndDesc,
+                                TruncInst *Trunc)
+      : VPRecipeBase(VPWidenIntOrFpInductionSC, {Start}), VPValue(Trunc, this),
+        IV(IV), IndDesc(IndDesc) {}
 
-    if (Cast)
-      new VPValue(Cast, this);
-  }
   ~VPWidenIntOrFpInductionRecipe() override = default;
 
   /// Method to support type inquiry through isa, cast, and dyn_cast.
@@ -1038,13 +1046,6 @@ public:
   /// Returns the start value of the induction.
   VPValue *getStartValue() { return getOperand(0); }
 
-  /// Returns the cast VPValue, if one is attached, or nullptr otherwise.
-  VPValue *getCastValue() {
-    if (getNumDefinedValues() != 2)
-      return nullptr;
-    return getVPValue(1);
-  }
-
   /// Returns the first defined value as TruncInst, if it is one or nullptr
   /// otherwise.
   TruncInst *getTruncInst() {
@@ -1053,6 +1054,9 @@ public:
   const TruncInst *getTruncInst() const {
     return dyn_cast_or_null<TruncInst>(getVPValue(0)->getUnderlyingValue());
   }
+
+  /// Returns the induction descriptor for the recipe.
+  const InductionDescriptor &getInductionDescriptor() const { return IndDesc; }
 };
 
 /// A recipe for handling first order recurrences and pointer inductions. For
@@ -1169,7 +1173,7 @@ struct VPFirstOrderRecurrencePHIRecipe : public VPWidenPHIRecipe {
 /// operand.
 class VPReductionPHIRecipe : public VPWidenPHIRecipe {
   /// Descriptor for the reduction.
-  RecurrenceDescriptor &RdxDesc;
+  const RecurrenceDescriptor &RdxDesc;
 
   /// The phi is part of an in-loop reduction.
   bool IsInLoop;
@@ -1180,7 +1184,7 @@ class VPReductionPHIRecipe : public VPWidenPHIRecipe {
 public:
   /// Create a new VPReductionPHIRecipe for the reduction \p Phi described by \p
   /// RdxDesc.
-  VPReductionPHIRecipe(PHINode *Phi, RecurrenceDescriptor &RdxDesc,
+  VPReductionPHIRecipe(PHINode *Phi, const RecurrenceDescriptor &RdxDesc,
                        VPValue &Start, bool IsInLoop = false,
                        bool IsOrdered = false)
       : VPWidenPHIRecipe(VPVReductionPHISC, VPReductionPHISC, Phi, &Start),
@@ -1210,7 +1214,9 @@ public:
              VPSlotTracker &SlotTracker) const override;
 #endif
 
-  RecurrenceDescriptor &getRecurrenceDescriptor() { return RdxDesc; }
+  const RecurrenceDescriptor &getRecurrenceDescriptor() const {
+    return RdxDesc;
+  }
 
   /// Returns true, if the phi is part of an ordered reduction.
   bool isOrdered() const { return IsOrdered; }
@@ -1340,13 +1346,13 @@ public:
 /// The Operands are {ChainOp, VecOp, [Condition]}.
 class VPReductionRecipe : public VPRecipeBase, public VPValue {
   /// The recurrence decriptor for the reduction in question.
-  RecurrenceDescriptor *RdxDesc;
+  const RecurrenceDescriptor *RdxDesc;
   /// Pointer to the TTI, needed to create the target reduction
   const TargetTransformInfo *TTI;
 
 public:
-  VPReductionRecipe(RecurrenceDescriptor *R, Instruction *I, VPValue *ChainOp,
-                    VPValue *VecOp, VPValue *CondOp,
+  VPReductionRecipe(const RecurrenceDescriptor *R, Instruction *I,
+                    VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp,
                     const TargetTransformInfo *TTI)
       : VPRecipeBase(VPRecipeBase::VPReductionSC, {ChainOp, VecOp}),
         VPValue(VPValue::VPVReductionSC, I, this), RdxDesc(R), TTI(TTI) {
@@ -2252,6 +2258,12 @@ public:
     return map_range(Operands, Fn);
   }
 
+  /// Returns true if \p VPV is uniform after vectorization.
+  bool isUniformAfterVectorization(VPValue *VPV) const {
+    auto RepR = dyn_cast_or_null<VPReplicateRecipe>(VPV->getDef());
+    return !VPV->getDef() || (RepR && RepR->isUniform());
+  }
+
 private:
   /// Add to the given dominator tree the header block and every new basic block
   /// that was created between it and the latch block, inclusive.
@@ -2340,18 +2352,23 @@ public:
 
   /// Insert disconnected VPBlockBase \p NewBlock after \p BlockPtr. Add \p
   /// NewBlock as successor of \p BlockPtr and \p BlockPtr as predecessor of \p
-  /// NewBlock, and propagate \p BlockPtr parent to \p NewBlock. If \p BlockPtr
-  /// has more than one successor, its conditional bit is propagated to \p
-  /// NewBlock. \p NewBlock must have neither successors nor predecessors.
+  /// NewBlock, and propagate \p BlockPtr parent to \p NewBlock. \p BlockPtr's
+  /// successors are moved from \p BlockPtr to \p NewBlock and \p BlockPtr's
+  /// conditional bit is propagated to \p NewBlock. \p NewBlock must have
+  /// neither successors nor predecessors.
   static void insertBlockAfter(VPBlockBase *NewBlock, VPBlockBase *BlockPtr) {
     assert(NewBlock->getSuccessors().empty() &&
-           "Can't insert new block with successors.");
-    // TODO: move successors from BlockPtr to NewBlock when this functionality
-    // is necessary. For now, setBlockSingleSuccessor will assert if BlockPtr
-    // already has successors.
-    BlockPtr->setOneSuccessor(NewBlock);
-    NewBlock->setPredecessors({BlockPtr});
+           NewBlock->getPredecessors().empty() &&
+           "Can't insert new block with predecessors or successors.");
     NewBlock->setParent(BlockPtr->getParent());
+    SmallVector<VPBlockBase *> Succs(BlockPtr->successors());
+    for (VPBlockBase *Succ : Succs) {
+      disconnectBlocks(BlockPtr, Succ);
+      connectBlocks(NewBlock, Succ);
+    }
+    NewBlock->setCondBit(BlockPtr->getCondBit());
+    BlockPtr->setCondBit(nullptr);
+    connectBlocks(BlockPtr, NewBlock);
   }
 
   /// Insert disconnected VPBlockBases \p IfTrue and \p IfFalse after \p
@@ -2394,6 +2411,31 @@ public:
     To->removePredecessor(From);
   }
 
+  /// Try to merge \p Block into its single predecessor, if \p Block is a
+  /// VPBasicBlock and its predecessor has a single successor. Returns a pointer
+  /// to the predecessor \p Block was merged into or nullptr otherwise.
+  static VPBasicBlock *tryToMergeBlockIntoPredecessor(VPBlockBase *Block) {
+    auto *VPBB = dyn_cast<VPBasicBlock>(Block);
+    auto *PredVPBB =
+        dyn_cast_or_null<VPBasicBlock>(Block->getSinglePredecessor());
+    if (!VPBB || !PredVPBB || PredVPBB->getNumSuccessors() != 1)
+      return nullptr;
+
+    for (VPRecipeBase &R : make_early_inc_range(*VPBB))
+      R.moveBefore(*PredVPBB, PredVPBB->end());
+    VPBlockUtils::disconnectBlocks(PredVPBB, VPBB);
+    auto *ParentRegion = cast<VPRegionBlock>(Block->getParent());
+    if (ParentRegion->getExit() == Block)
+      ParentRegion->setExit(PredVPBB);
+    SmallVector<VPBlockBase *> Successors(Block->successors());
+    for (auto *Succ : Successors) {
+      VPBlockUtils::disconnectBlocks(Block, Succ);
+      VPBlockUtils::connectBlocks(PredVPBB, Succ);
+    }
+    delete Block;
+    return PredVPBB;
+  }
+
   /// Returns true if the edge \p FromBlock -> \p ToBlock is a back-edge.
   static bool isBackEdge(const VPBlockBase *FromBlock,
                          const VPBlockBase *ToBlock, const VPLoopInfo *VPLI) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index ac3b3505dc34..86ecd6817873 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -50,14 +50,14 @@ VPValue *VPlanPredicator::getOrCreateNotPredicate(VPBasicBlock *PredBB,
 
   case EdgeType::FALSE_EDGE:
     // CurrBB is the False successor of PredBB - compute not of CBV.
-    IntermediateVal = Builder.createNot(CBV);
+    IntermediateVal = Builder.createNot(CBV, {});
     break;
   }
 
   // Now AND intermediate value with PredBB's block predicate if it has one.
   VPValue *BP = PredBB->getPredicate();
   if (BP)
-    return Builder.createAnd(BP, IntermediateVal);
+    return Builder.createAnd(BP, IntermediateVal, {});
   else
     return IntermediateVal;
 }
@@ -96,7 +96,7 @@ VPValue *VPlanPredicator::genPredicateTree(std::list<VPValue *> &Worklist) {
     Worklist.pop_front();
 
     // Create an OR of these values.
-    VPValue *Or = Builder.createOr(LHS, RHS);
+    VPValue *Or = Builder.createOr(LHS, RHS, {});
 
     // Push OR to the back of the worklist.
     Worklist.push_back(Or);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp b/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp
index c52c8a2229e8..9e19e172dea5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp
@@ -467,8 +467,9 @@ VPInstruction *VPlanSlp::buildGraph(ArrayRef<VPValue *> Values) {
     return markFailed();
 
   assert(CombinedOperands.size() > 0 && "Need more some operands");
-  auto *VPI = new VPInstruction(Opcode, CombinedOperands);
-  VPI->setUnderlyingInstr(cast<VPInstruction>(Values[0])->getUnderlyingInstr());
+  auto *Inst = cast<VPInstruction>(Values[0])->getUnderlyingInstr();
+  auto *VPI = new VPInstruction(Opcode, CombinedOperands, Inst->getDebugLoc());
+  VPI->setUnderlyingInstr(Inst);
 
   LLVM_DEBUG(dbgs() << "Create VPInstruction " << *VPI << " "
                     << *cast<VPInstruction>(Values[0]) << "\n");
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index ded5bc04beb5..d2daf558c2c5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -18,7 +18,8 @@ using namespace llvm;
 
 void VPlanTransforms::VPInstructionsToVPRecipes(
     Loop *OrigLoop, VPlanPtr &Plan,
-    LoopVectorizationLegality::InductionList &Inductions,
+    function_ref<const InductionDescriptor *(PHINode *)>
+        GetIntOrFpInductionDescriptor,
     SmallPtrSetImpl<Instruction *> &DeadInstructions, ScalarEvolution &SE) {
 
   auto *TopRegion = cast<VPRegionBlock>(Plan->getEntry());
@@ -44,11 +45,9 @@ void VPlanTransforms::VPInstructionsToVPRecipes(
       VPRecipeBase *NewRecipe = nullptr;
       if (auto *VPPhi = dyn_cast<VPWidenPHIRecipe>(&Ingredient)) {
         auto *Phi = cast<PHINode>(VPPhi->getUnderlyingValue());
-        InductionDescriptor II = Inductions.lookup(Phi);
-        if (II.getKind() == InductionDescriptor::IK_IntInduction ||
-            II.getKind() == InductionDescriptor::IK_FpInduction) {
-          VPValue *Start = Plan->getOrAddVPValue(II.getStartValue());
-          NewRecipe = new VPWidenIntOrFpInductionRecipe(Phi, Start, nullptr);
+        if (const auto *II = GetIntOrFpInductionDescriptor(Phi)) {
+          VPValue *Start = Plan->getOrAddVPValue(II->getStartValue());
+          NewRecipe = new VPWidenIntOrFpInductionRecipe(Phi, Start, *II);
         } else {
           Plan->addVPValue(Phi, VPPhi);
           continue;
@@ -158,8 +157,7 @@ bool VPlanTransforms::sinkScalarOperands(VPlan &Plan) {
       // TODO: add ".cloned" suffix to name of Clone's VPValue.
 
       Clone->insertBefore(SinkCandidate);
-      SmallVector<VPUser *, 4> Users(SinkCandidate->user_begin(),
-                                     SinkCandidate->user_end());
+      SmallVector<VPUser *, 4> Users(SinkCandidate->users());
       for (auto *U : Users) {
         auto *UI = cast<VPRecipeBase>(U);
         if (UI->getParent() == SinkTo)
@@ -266,8 +264,7 @@ bool VPlanTransforms::mergeReplicateRegions(VPlan &Plan) {
       VPValue *PredInst1 =
           cast<VPPredInstPHIRecipe>(&Phi1ToMove)->getOperand(0);
       VPValue *Phi1ToMoveV = Phi1ToMove.getVPSingleValue();
-      SmallVector<VPUser *> Users(Phi1ToMoveV->user_begin(),
-                                  Phi1ToMoveV->user_end());
+      SmallVector<VPUser *> Users(Phi1ToMoveV->users());
       for (VPUser *U : Users) {
         auto *UI = dyn_cast<VPRecipeBase>(U);
         if (!UI || UI->getParent() != Then2)
@@ -295,3 +292,35 @@ bool VPlanTransforms::mergeReplicateRegions(VPlan &Plan) {
     delete ToDelete;
   return Changed;
 }
+
+void VPlanTransforms::removeRedundantInductionCasts(VPlan &Plan) {
+  SmallVector<std::pair<VPRecipeBase *, VPValue *>> CastsToRemove;
+  for (auto &Phi : Plan.getEntry()->getEntryBasicBlock()->phis()) {
+    auto *IV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
+    if (!IV || IV->getTruncInst())
+      continue;
+
+    // Visit all casts connected to IV and in Casts. Collect them.
+    // remember them for removal.
+    auto &Casts = IV->getInductionDescriptor().getCastInsts();
+    VPValue *FindMyCast = IV;
+    for (Instruction *IRCast : reverse(Casts)) {
+      VPRecipeBase *FoundUserCast = nullptr;
+      for (auto *U : FindMyCast->users()) {
+        auto *UserCast = cast<VPRecipeBase>(U);
+        if (UserCast->getNumDefinedValues() == 1 &&
+            UserCast->getVPSingleValue()->getUnderlyingValue() == IRCast) {
+          FoundUserCast = UserCast;
+          break;
+        }
+      }
+      assert(FoundUserCast && "Missing a cast to remove");
+      CastsToRemove.emplace_back(FoundUserCast, IV);
+      FindMyCast = FoundUserCast->getVPSingleValue();
+    }
+  }
+  for (auto &E : CastsToRemove) {
+    E.first->getVPSingleValue()->replaceAllUsesWith(E.second);
+    E.first->eraseFromParent();
+  }
+}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index c740f2c022da..a82a562d5e35 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -14,24 +14,37 @@
 #define LLVM_TRANSFORMS_VECTORIZE_VPLANTRANSFORMS_H
 
 #include "VPlan.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
 
 namespace llvm {
 
+class InductionDescriptor;
 class Instruction;
+class PHINode;
 class ScalarEvolution;
 
 struct VPlanTransforms {
   /// Replaces the VPInstructions in \p Plan with corresponding
   /// widen recipes.
-  static void VPInstructionsToVPRecipes(
-      Loop *OrigLoop, VPlanPtr &Plan,
-      LoopVectorizationLegality::InductionList &Inductions,
-      SmallPtrSetImpl<Instruction *> &DeadInstructions, ScalarEvolution &SE);
+  static void
+  VPInstructionsToVPRecipes(Loop *OrigLoop, VPlanPtr &Plan,
+                            function_ref<const InductionDescriptor *(PHINode *)>
+                                GetIntOrFpInductionDescriptor,
+                            SmallPtrSetImpl<Instruction *> &DeadInstructions,
+                            ScalarEvolution &SE);
 
   static bool sinkScalarOperands(VPlan &Plan);
 
   static bool mergeReplicateRegions(VPlan &Plan);
+
+  /// Remove redundant casts of inductions.
+  ///
+  /// Such redundant casts are casts of induction variables that can be ignored,
+  /// because we already proved that the casted phi is equal to the uncasted phi
+  /// in the vectorized loop. There is no need to vectorize the cast - the same
+  /// value can be used for both the phi and casts in the vector loop.
+  static void removeRedundantInductionCasts(VPlan &Plan);
 };
 
 } // namespace llvm
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index 6d6ea4eb30f1..7732d9367985 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -156,5 +156,31 @@ bool VPlanVerifier::verifyPlanIsValid(const VPlan &Plan) {
       RecipeI++;
     }
   }
+
+  const VPRegionBlock *TopRegion = cast<VPRegionBlock>(Plan.getEntry());
+  const VPBasicBlock *Entry = dyn_cast<VPBasicBlock>(TopRegion->getEntry());
+  if (!Entry) {
+    errs() << "VPlan entry block is not a VPBasicBlock\n";
+    return false;
+  }
+  const VPBasicBlock *Exit = dyn_cast<VPBasicBlock>(TopRegion->getExit());
+  if (!Exit) {
+    errs() << "VPlan exit block is not a VPBasicBlock\n";
+    return false;
+  }
+
+  for (const VPRegionBlock *Region :
+       VPBlockUtils::blocksOnly<const VPRegionBlock>(
+           depth_first(VPBlockRecursiveTraversalWrapper<const VPBlockBase *>(
+               Plan.getEntry())))) {
+    if (Region->getEntry()->getNumPredecessors() != 0) {
+      errs() << "region entry block has predecessors\n";
+      return false;
+    }
+    if (Region->getExit()->getNumSuccessors() != 0) {
+      errs() << "region exit block has successors\n";
+      return false;
+    }
+  }
   return true;
 }
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index 57b11e9414ba..c0aedab2fed0 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -989,9 +989,9 @@ bool VectorCombine::scalarizeLoadExtract(Instruction &I) {
   if (!FixedVT)
     return false;
 
-  InstructionCost OriginalCost = TTI.getMemoryOpCost(
-      Instruction::Load, LI->getType(), Align(LI->getAlignment()),
-      LI->getPointerAddressSpace());
+  InstructionCost OriginalCost =
+      TTI.getMemoryOpCost(Instruction::Load, LI->getType(), LI->getAlign(),
+                          LI->getPointerAddressSpace());
   InstructionCost ScalarizedCost = 0;
 
   Instruction *LastCheckedInst = LI;
diff --git a/llvm/tools/llc/llc.cpp b/llvm/tools/llc/llc.cpp
index 9d80f062c8f9..c07f4e66486c 100644
--- a/llvm/tools/llc/llc.cpp
+++ b/llvm/tools/llc/llc.cpp
@@ -605,6 +605,9 @@ static int compileModule(char **argv, LLVMContext &Context) {
       GetOutputStream(TheTarget->getName(), TheTriple.getOS(), argv[0]);
   if (!Out) return 1;
 
+  // Ensure the filename is passed down to CodeViewDebug.
+  Target->Options.ObjectFilenameForDebug = Out->outputFilename();
+
   std::unique_ptr<ToolOutputFile> DwoOut;
   if (!SplitDwarfOutputFile.empty()) {
     std::error_code EC;
diff --git a/llvm/tools/llvm-dwarfdump/llvm-dwarfdump.cpp b/llvm/tools/llvm-dwarfdump/llvm-dwarfdump.cpp
index 9eeaddf14928..9c2ddc3867a5 100644
--- a/llvm/tools/llvm-dwarfdump/llvm-dwarfdump.cpp
+++ b/llvm/tools/llvm-dwarfdump/llvm-dwarfdump.cpp
@@ -24,7 +24,6 @@
 #include "llvm/Support/Format.h"
 #include "llvm/Support/InitLLVM.h"
 #include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/Path.h"
 #include "llvm/Support/Regex.h"
 #include "llvm/Support/TargetSelect.h"
 #include "llvm/Support/ToolOutputFile.h"
@@ -265,6 +264,13 @@ static cl::extrahelp
 /// @}
 //===----------------------------------------------------------------------===//
 
+static void error(Error Err) {
+  if (!Err)
+    return;
+  WithColor::error() << toString(std::move(Err)) << "\n";
+  exit(1);
+}
+
 static void error(StringRef Prefix, Error Err) {
   if (!Err)
     return;
@@ -289,8 +295,10 @@ static DIDumpOptions getDumpOpts(DWARFContext &C) {
   DumpOpts.Verbose = Verbose;
   DumpOpts.RecoverableErrorHandler = C.getRecoverableErrorHandler();
   // In -verify mode, print DIEs without children in error messages.
-  if (Verify)
+  if (Verify) {
+    DumpOpts.Verbose = true;
     return DumpOpts.noImplicitRecursion();
+  }
   return DumpOpts;
 }
 
@@ -581,41 +589,6 @@ static bool handleFile(StringRef Filename, HandlerFn HandleObj,
   return handleBuffer(Filename, *Buffer, HandleObj, OS);
 }
 
-/// If the input path is a .dSYM bundle (as created by the dsymutil tool),
-/// replace it with individual entries for each of the object files inside the
-/// bundle otherwise return the input path.
-static std::vector<std::string> expandBundle(const std::string &InputPath) {
-  std::vector<std::string> BundlePaths;
-  SmallString<256> BundlePath(InputPath);
-  // Normalize input path. This is necessary to accept `bundle.dSYM/`.
-  sys::path::remove_dots(BundlePath);
-  // Manually open up the bundle to avoid introducing additional dependencies.
-  if (sys::fs::is_directory(BundlePath) &&
-      sys::path::extension(BundlePath) == ".dSYM") {
-    std::error_code EC;
-    sys::path::append(BundlePath, "Contents", "Resources", "DWARF");
-    for (sys::fs::directory_iterator Dir(BundlePath, EC), DirEnd;
-         Dir != DirEnd && !EC; Dir.increment(EC)) {
-      const std::string &Path = Dir->path();
-      sys::fs::file_status Status;
-      EC = sys::fs::status(Path, Status);
-      error(Path, EC);
-      switch (Status.type()) {
-      case sys::fs::file_type::regular_file:
-      case sys::fs::file_type::symlink_file:
-      case sys::fs::file_type::type_unknown:
-        BundlePaths.push_back(Path);
-        break;
-      default: /*ignore*/;
-      }
-    }
-    error(BundlePath, EC);
-  }
-  if (!BundlePaths.size())
-    BundlePaths.push_back(InputPath);
-  return BundlePaths;
-}
-
 int main(int argc, char **argv) {
   InitLLVM X(argc, argv);
 
@@ -684,8 +657,14 @@ int main(int argc, char **argv) {
   // Expand any .dSYM bundles to the individual object files contained therein.
   std::vector<std::string> Objects;
   for (const auto &F : InputFilenames) {
-    auto Objs = expandBundle(F);
-    llvm::append_range(Objects, Objs);
+    if (auto DsymObjectsOrErr = MachOObjectFile::findDsymObjectMembers(F)) {
+      if (DsymObjectsOrErr->empty())
+        Objects.push_back(F);
+      else
+        llvm::append_range(Objects, *DsymObjectsOrErr);
+    } else {
+      error(DsymObjectsOrErr.takeError());
+    }
   }
 
   bool Success = true;
diff --git a/llvm/tools/llvm-lto2/llvm-lto2.cpp b/llvm/tools/llvm-lto2/llvm-lto2.cpp
index 6f6f6c1ed90f..7416e5850944 100644
--- a/llvm/tools/llvm-lto2/llvm-lto2.cpp
+++ b/llvm/tools/llvm-lto2/llvm-lto2.cpp
@@ -29,6 +29,7 @@
 #include "llvm/Support/PluginLoader.h"
 #include "llvm/Support/TargetSelect.h"
 #include "llvm/Support/Threading.h"
+#include <atomic>
 
 using namespace llvm;
 using namespace lto;
@@ -236,13 +237,6 @@ static int run(int argc, char **argv) {
   std::vector<std::unique_ptr<MemoryBuffer>> MBs;
 
   Config Conf;
-  Conf.DiagHandler = [](const DiagnosticInfo &DI) {
-    DiagnosticPrinterRawOStream DP(errs());
-    DI.print(DP);
-    errs() << '\n';
-    if (DI.getSeverity() == DS_Error)
-      exit(1);
-  };
 
   Conf.CPU = codegen::getMCPU();
   Conf.Options = codegen::InitTargetOptionsFromCodeGenFlags(Triple());
@@ -314,9 +308,25 @@ static int run(int argc, char **argv) {
   else
     Backend = createInProcessThinBackend(
         llvm::heavyweight_hardware_concurrency(Threads));
+  // Track whether we hit an error; in particular, in the multi-threaded case,
+  // we can't exit() early because the rest of the threads wouldn't have had a
+  // change to be join-ed, and that would result in a "terminate called without
+  // an active exception". Altogether, this results in nondeterministic
+  // behavior. Instead, we don't exit in the multi-threaded case, but we make
+  // sure to report the error and then at the end (after joining cleanly)
+  // exit(1).
+  std::atomic<bool> HasErrors;
+  std::atomic_init(&HasErrors, false);
+  Conf.DiagHandler = [&](const DiagnosticInfo &DI) {
+    DiagnosticPrinterRawOStream DP(errs());
+    DI.print(DP);
+    errs() << '\n';
+    if (DI.getSeverity() == DS_Error)
+      HasErrors = true;
+  };
+
   LTO Lto(std::move(Conf), std::move(Backend));
 
-  bool HasErrors = false;
   for (std::string F : InputFilenames) {
     std::unique_ptr<MemoryBuffer> MB = check(MemoryBuffer::getFile(F), F);
     std::unique_ptr<InputFile> Input =
@@ -368,7 +378,7 @@ static int run(int argc, char **argv) {
     std::error_code EC;
     auto S = std::make_unique<raw_fd_ostream>(Path, EC, sys::fs::OF_None);
     check(EC, Path);
-    return std::make_unique<CachedFileStream>(std::move(S));
+    return std::make_unique<CachedFileStream>(std::move(S), Path);
   };
 
   auto AddBuffer = [&](size_t Task, std::unique_ptr<MemoryBuffer> MB) {
@@ -381,7 +391,7 @@ static int run(int argc, char **argv) {
                   "failed to create cache");
 
   check(Lto.run(AddStream, Cache), "LTO::run failed");
-  return 0;
+  return static_cast<int>(HasErrors);
 }
 
 static int dumpSymtab(int argc, char **argv) {
diff --git a/llvm/tools/llvm-mca/Views/BottleneckAnalysis.cpp b/llvm/tools/llvm-mca/Views/BottleneckAnalysis.cpp
index 5b110d6602df..dc0a07e75e48 100644
--- a/llvm/tools/llvm-mca/Views/BottleneckAnalysis.cpp
+++ b/llvm/tools/llvm-mca/Views/BottleneckAnalysis.cpp
@@ -275,9 +275,9 @@ void DependencyGraph::getCriticalSequence(
       [](const DGNode &Lhs, const DGNode &Rhs) { return Lhs.Cost < Rhs.Cost; });
   unsigned IID = std::distance(Nodes.begin(), It);
   Seq.resize(Nodes[IID].Depth);
-  for (unsigned I = Seq.size(), E = 0; I > E; --I) {
+  for (const DependencyEdge *&DE : llvm::reverse(Seq)) {
     const DGNode &N = Nodes[IID];
-    Seq[I - 1] = &N.CriticalPredecessor;
+    DE = &N.CriticalPredecessor;
     IID = N.CriticalPredecessor.FromIID;
   }
 }
diff --git a/llvm/tools/llvm-mca/llvm-mca.cpp b/llvm/tools/llvm-mca/llvm-mca.cpp
index 0b58ca377ce1..0501336ab207 100644
--- a/llvm/tools/llvm-mca/llvm-mca.cpp
+++ b/llvm/tools/llvm-mca/llvm-mca.cpp
@@ -347,12 +347,6 @@ int main(int argc, char **argv) {
   if (!STI->isCPUStringValid(MCPU))
     return 1;
 
-  bool IsOutOfOrder = STI->getSchedModel().isOutOfOrder();
-  if (!PrintInstructionTables && !IsOutOfOrder) {
-    WithColor::warning() << "support for in-order CPU '" << MCPU
-                         << "' is experimental.\n";
-  }
-
   if (!STI->getSchedModel().hasInstrSchedModel()) {
     WithColor::error()
         << "unable to find instruction-level scheduling information for"
@@ -367,6 +361,7 @@ int main(int argc, char **argv) {
   }
 
   // Apply overrides to llvm-mca specific options.
+  bool IsOutOfOrder = STI->getSchedModel().isOutOfOrder();
   processViewOptions(IsOutOfOrder);
 
   std::unique_ptr<MCRegisterInfo> MRI(TheTarget->createMCRegInfo(TripleName));
diff --git a/llvm/tools/llvm-objcopy/ELF/Object.cpp b/llvm/tools/llvm-objcopy/ELF/Object.cpp
index 3db5028e85f7..659e12bf0306 100644
--- a/llvm/tools/llvm-objcopy/ELF/Object.cpp
+++ b/llvm/tools/llvm-objcopy/ELF/Object.cpp
@@ -978,12 +978,12 @@ static void setAddend(Elf_Rel_Impl<ELFT, true> &Rela, uint64_t Addend) {
 }
 
 template <class RelRange, class T>
-static void writeRel(const RelRange &Relocations, T *Buf) {
+static void writeRel(const RelRange &Relocations, T *Buf, bool IsMips64EL) {
   for (const auto &Reloc : Relocations) {
     Buf->r_offset = Reloc.Offset;
     setAddend(*Buf, Reloc.Addend);
     Buf->setSymbolAndType(Reloc.RelocSymbol ? Reloc.RelocSymbol->Index : 0,
-                          Reloc.Type, false);
+                          Reloc.Type, IsMips64EL);
     ++Buf;
   }
 }
@@ -992,9 +992,11 @@ template <class ELFT>
 Error ELFSectionWriter<ELFT>::visit(const RelocationSection &Sec) {
   uint8_t *Buf = reinterpret_cast<uint8_t *>(Out.getBufferStart()) + Sec.Offset;
   if (Sec.Type == SHT_REL)
-    writeRel(Sec.Relocations, reinterpret_cast<Elf_Rel *>(Buf));
+    writeRel(Sec.Relocations, reinterpret_cast<Elf_Rel *>(Buf),
+             Sec.getObject().IsMips64EL);
   else
-    writeRel(Sec.Relocations, reinterpret_cast<Elf_Rela *>(Buf));
+    writeRel(Sec.Relocations, reinterpret_cast<Elf_Rela *>(Buf),
+             Sec.getObject().IsMips64EL);
   return Error::success();
 }
 
@@ -1398,6 +1400,14 @@ Expected<std::unique_ptr<Object>> IHexELFBuilder::build() {
   return std::move(Obj);
 }
 
+template <class ELFT>
+ELFBuilder<ELFT>::ELFBuilder(const ELFObjectFile<ELFT> &ElfObj, Object &Obj,
+                             Optional<StringRef> ExtractPartition)
+    : ElfFile(ElfObj.getELFFile()), Obj(Obj),
+      ExtractPartition(ExtractPartition) {
+  Obj.IsMips64EL = ElfFile.isMips64EL();
+}
+
 template <class ELFT> void ELFBuilder<ELFT>::setParentSegment(Segment &Child) {
   for (Segment &Parent : Obj.segments()) {
     // Every segment will overlap with itself but we don't want a segment to
@@ -1639,21 +1649,21 @@ static void getAddend(uint64_t &ToSet, const Elf_Rel_Impl<ELFT, true> &Rela) {
 }
 
 template <class T>
-static Error initRelocations(RelocationSection *Relocs,
-                             SymbolTableSection *SymbolTable, T RelRange) {
+static Error initRelocations(RelocationSection *Relocs, T RelRange) {
   for (const auto &Rel : RelRange) {
     Relocation ToAdd;
     ToAdd.Offset = Rel.r_offset;
     getAddend(ToAdd.Addend, Rel);
-    ToAdd.Type = Rel.getType(false);
+    ToAdd.Type = Rel.getType(Relocs->getObject().IsMips64EL);
 
-    if (uint32_t Sym = Rel.getSymbol(false)) {
-      if (!SymbolTable)
+    if (uint32_t Sym = Rel.getSymbol(Relocs->getObject().IsMips64EL)) {
+      if (!Relocs->getObject().SymbolTable)
         return createStringError(
             errc::invalid_argument,
             "'" + Relocs->Name + "': relocation references symbol with index " +
                 Twine(Sym) + ", but there is no symbol table");
-      Expected<Symbol *> SymByIndex = SymbolTable->getSymbolByIndex(Sym);
+      Expected<Symbol *> SymByIndex =
+          Relocs->getObject().SymbolTable->getSymbolByIndex(Sym);
       if (!SymByIndex)
         return SymByIndex.takeError();
 
@@ -1698,7 +1708,7 @@ Expected<SectionBase &> ELFBuilder<ELFT>::makeSection(const Elf_Shdr &Shdr) {
       else
         return Data.takeError();
     }
-    return Obj.addSection<RelocationSection>();
+    return Obj.addSection<RelocationSection>(Obj);
   case SHT_STRTAB:
     // If a string table is allocated we don't want to mess with it. That would
     // mean altering the memory image. There are no special link types or
@@ -1879,7 +1889,7 @@ template <class ELFT> Error ELFBuilder<ELFT>::readSections(bool EnsureSymtab) {
         if (!Rels)
           return Rels.takeError();
 
-        if (Error Err = initRelocations(RelSec, Obj.SymbolTable, *Rels))
+        if (Error Err = initRelocations(RelSec, *Rels))
           return Err;
       } else {
         Expected<typename ELFFile<ELFT>::Elf_Rela_Range> Relas =
@@ -1887,7 +1897,7 @@ template <class ELFT> Error ELFBuilder<ELFT>::readSections(bool EnsureSymtab) {
         if (!Relas)
           return Relas.takeError();
 
-        if (Error Err = initRelocations(RelSec, Obj.SymbolTable, *Relas))
+        if (Error Err = initRelocations(RelSec, *Relas))
           return Err;
       }
     } else if (auto GroupSec = dyn_cast<GroupSection>(&Sec)) {
diff --git a/llvm/tools/llvm-objcopy/ELF/Object.h b/llvm/tools/llvm-objcopy/ELF/Object.h
index 811af4b51310..439380fc725b 100644
--- a/llvm/tools/llvm-objcopy/ELF/Object.h
+++ b/llvm/tools/llvm-objcopy/ELF/Object.h
@@ -783,8 +783,10 @@ class RelocationSection
   MAKE_SEC_WRITER_FRIEND
 
   std::vector<Relocation> Relocations;
+  const Object &Obj;
 
 public:
+  RelocationSection(const Object &O) : Obj(O) {}
   void addRelocation(Relocation Rel) { Relocations.push_back(Rel); }
   Error accept(SectionVisitor &Visitor) const override;
   Error accept(MutableSectionVisitor &Visitor) override;
@@ -795,6 +797,7 @@ public:
   void markSymbols() override;
   void replaceSectionReferences(
       const DenseMap<SectionBase *, SectionBase *> &FromTo) override;
+  const Object &getObject() const { return Obj; }
 
   static bool classof(const SectionBase *S) {
     if (S->OriginalFlags & ELF::SHF_ALLOC)
@@ -971,9 +974,7 @@ private:
 
 public:
   ELFBuilder(const ELFObjectFile<ELFT> &ElfObj, Object &Obj,
-             Optional<StringRef> ExtractPartition)
-      : ElfFile(ElfObj.getELFFile()), Obj(Obj),
-        ExtractPartition(ExtractPartition) {}
+             Optional<StringRef> ExtractPartition);
 
   Error build(bool EnsureSymtab);
 };
@@ -1063,6 +1064,8 @@ public:
   SymbolTableSection *SymbolTable = nullptr;
   SectionIndexSection *SectionIndexTable = nullptr;
 
+  bool IsMips64EL = false;
+
   SectionTableRef sections() const { return SectionTableRef(Sections); }
   iterator_range<
       filter_iterator<pointee_iterator<std::vector<SecPtr>::const_iterator>,
diff --git a/llvm/tools/llvm-objcopy/MachO/MachOObjcopy.cpp b/llvm/tools/llvm-objcopy/MachO/MachOObjcopy.cpp
index 9e7b91d73057..915394b65b12 100644
--- a/llvm/tools/llvm-objcopy/MachO/MachOObjcopy.cpp
+++ b/llvm/tools/llvm-objcopy/MachO/MachOObjcopy.cpp
@@ -476,9 +476,8 @@ Error objcopy::macho::executeObjcopyOnMachOUniversalBinary(
                                          **ObjOrErr, MemStream))
       return E;
 
-    std::unique_ptr<MemoryBuffer> MB =
-        std::make_unique<SmallVectorMemoryBuffer>(std::move(Buffer),
-                                                  ArchFlagName);
+    auto MB = std::make_unique<SmallVectorMemoryBuffer>(
+        std::move(Buffer), ArchFlagName, /*RequiresNullTerminator=*/false);
     Expected<std::unique_ptr<Binary>> BinaryOrErr = object::createBinary(*MB);
     if (!BinaryOrErr)
       return BinaryOrErr.takeError();
diff --git a/llvm/tools/llvm-objcopy/llvm-objcopy.cpp b/llvm/tools/llvm-objcopy/llvm-objcopy.cpp
index ad166487eb78..a5963985f78a 100644
--- a/llvm/tools/llvm-objcopy/llvm-objcopy.cpp
+++ b/llvm/tools/llvm-objcopy/llvm-objcopy.cpp
@@ -236,7 +236,8 @@ createNewArchiveMembers(const MultiFormatConfig &Config, const Archive &Ar) {
       return createFileError(Ar.getFileName(), Member.takeError());
 
     Member->Buf = std::make_unique<SmallVectorMemoryBuffer>(
-        std::move(Buffer), ChildNameOrErr.get());
+        std::move(Buffer), ChildNameOrErr.get(),
+        /*RequiresNullTerminator=*/false);
     Member->MemberName = Member->Buf->getBufferIdentifier();
     NewArchiveMembers.push_back(std::move(*Member));
   }
diff --git a/llvm/tools/llvm-objdump/llvm-objdump.cpp b/llvm/tools/llvm-objdump/llvm-objdump.cpp
index 6f6f543f2f47..a16f760cd1cb 100644
--- a/llvm/tools/llvm-objdump/llvm-objdump.cpp
+++ b/llvm/tools/llvm-objdump/llvm-objdump.cpp
@@ -978,8 +978,8 @@ collectLocalBranchTargets(ArrayRef<uint8_t> Bytes, const MCInstrAnalysis *MIA,
                           const MCSubtargetInfo *STI, uint64_t SectionAddr,
                           uint64_t Start, uint64_t End,
                           std::unordered_map<uint64_t, std::string> &Labels) {
-  // So far only supports X86.
-  if (!STI->getTargetTriple().isX86())
+  // So far only supports PowerPC and X86.
+  if (!STI->getTargetTriple().isPPC() && !STI->getTargetTriple().isX86())
     return;
 
   Labels.clear();
@@ -999,8 +999,11 @@ collectLocalBranchTargets(ArrayRef<uint8_t> Bytes, const MCInstrAnalysis *MIA,
     if (Disassembled && MIA) {
       uint64_t Target;
       bool TargetKnown = MIA->evaluateBranch(Inst, Index, Size, Target);
+      // On PowerPC, if the address of a branch is the same as the target, it
+      // means that it's a function call. Do not mark the label for this case.
       if (TargetKnown && (Target >= Start && Target < End) &&
-          !Labels.count(Target))
+          !Labels.count(Target) &&
+          !(STI->getTargetTriple().isPPC() && Target == Index))
         Labels[Target] = ("L" + Twine(LabelCount++)).str();
     }
 
diff --git a/llvm/tools/llvm-pdbutil/PdbYaml.h b/llvm/tools/llvm-pdbutil/PdbYaml.h
index ed6346c2c4db..2c2878c16546 100644
--- a/llvm/tools/llvm-pdbutil/PdbYaml.h
+++ b/llvm/tools/llvm-pdbutil/PdbYaml.h
@@ -40,7 +40,7 @@ struct MSFHeaders {
   uint32_t NumDirectoryBlocks = 0;
   std::vector<uint32_t> DirectoryBlocks;
   uint32_t NumStreams = 0;
-  uint32_t FileSize = 0;
+  uint64_t FileSize = 0;
 };
 
 struct StreamBlockList {
diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp
index 7208011c9866..285b41f57147 100644
--- a/llvm/tools/llvm-profdata/llvm-profdata.cpp
+++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp
@@ -13,7 +13,10 @@
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/DebugInfo/DWARF/DWARFContext.h"
 #include "llvm/IR/LLVMContext.h"
+#include "llvm/Object/Binary.h"
+#include "llvm/ProfileData/InstrProfCorrelator.h"
 #include "llvm/ProfileData/InstrProfReader.h"
 #include "llvm/ProfileData/InstrProfWriter.h"
 #include "llvm/ProfileData/ProfileCommon.h"
@@ -233,6 +236,7 @@ static void overlapInput(const std::string &BaseFilename,
 
 /// Load an input into a writer context.
 static void loadInput(const WeightedFile &Input, SymbolRemapper *Remapper,
+                      const InstrProfCorrelator *Correlator,
                       WriterContext *WC) {
   std::unique_lock<std::mutex> CtxGuard{WC->Lock};
 
@@ -241,7 +245,7 @@ static void loadInput(const WeightedFile &Input, SymbolRemapper *Remapper,
   // invalid outside of this packaged task.
   std::string Filename = Input.Filename;
 
-  auto ReaderOrErr = InstrProfReader::create(Input.Filename);
+  auto ReaderOrErr = InstrProfReader::create(Input.Filename, Correlator);
   if (Error E = ReaderOrErr.takeError()) {
     // Skip the empty profiles by returning sliently.
     instrprof_error IPE = InstrProfError::take(std::move(E));
@@ -325,6 +329,7 @@ static void writeInstrProfile(StringRef OutputFilename,
 }
 
 static void mergeInstrProfile(const WeightedFileVector &Inputs,
+                              StringRef DebugInfoFilename,
                               SymbolRemapper *Remapper,
                               StringRef OutputFilename,
                               ProfileFormat OutputFormat, bool OutputSparse,
@@ -333,6 +338,15 @@ static void mergeInstrProfile(const WeightedFileVector &Inputs,
       OutputFormat != PF_Ext_Binary && OutputFormat != PF_Text)
     exitWithError("unknown format is specified");
 
+  std::unique_ptr<InstrProfCorrelator> Correlator;
+  if (!DebugInfoFilename.empty()) {
+    if (auto Err =
+            InstrProfCorrelator::get(DebugInfoFilename).moveInto(Correlator))
+      exitWithError(std::move(Err), DebugInfoFilename);
+    if (auto Err = Correlator->correlateProfileData())
+      exitWithError(std::move(Err), DebugInfoFilename);
+  }
+
   std::mutex ErrorLock;
   SmallSet<instrprof_error, 4> WriterErrorCodes;
 
@@ -352,14 +366,15 @@ static void mergeInstrProfile(const WeightedFileVector &Inputs,
 
   if (NumThreads == 1) {
     for (const auto &Input : Inputs)
-      loadInput(Input, Remapper, Contexts[0].get());
+      loadInput(Input, Remapper, Correlator.get(), Contexts[0].get());
   } else {
     ThreadPool Pool(hardware_concurrency(NumThreads));
 
     // Load the inputs in parallel (N/NumThreads serial steps).
     unsigned Ctx = 0;
     for (const auto &Input : Inputs) {
-      Pool.async(loadInput, Input, Remapper, Contexts[Ctx].get());
+      Pool.async(loadInput, Input, Remapper, Correlator.get(),
+                 Contexts[Ctx].get());
       Ctx = (Ctx + 1) % NumThreads;
     }
     Pool.wait();
@@ -575,7 +590,7 @@ static void supplementInstrProfile(
   SmallSet<instrprof_error, 4> WriterErrorCodes;
   auto WC = std::make_unique<WriterContext>(OutputSparse, ErrorLock,
                                             WriterErrorCodes);
-  loadInput(Inputs[0], nullptr, WC.get());
+  loadInput(Inputs[0], nullptr, nullptr, WC.get());
   if (WC->Errors.size() > 0)
     exitWithError(std::move(WC->Errors[0].first), InstrFilename);
 
@@ -687,7 +702,7 @@ static void
 mergeSampleProfile(const WeightedFileVector &Inputs, SymbolRemapper *Remapper,
                    StringRef OutputFilename, ProfileFormat OutputFormat,
                    StringRef ProfileSymbolListFile, bool CompressAllSections,
-                   bool UseMD5, bool GenPartialProfile,
+                   bool UseMD5, bool GenPartialProfile, bool GenCSNestedProfile,
                    bool SampleMergeColdContext, bool SampleTrimColdContext,
                    bool SampleColdContextFrameDepth, FailureMode FailMode) {
   using namespace sampleprof;
@@ -696,7 +711,7 @@ mergeSampleProfile(const WeightedFileVector &Inputs, SymbolRemapper *Remapper,
   LLVMContext Context;
   sampleprof::ProfileSymbolList WriterList;
   Optional<bool> ProfileIsProbeBased;
-  Optional<bool> ProfileIsCS;
+  Optional<bool> ProfileIsCSFlat;
   for (const auto &Input : Inputs) {
     auto ReaderOrErr = SampleProfileReader::create(Input.Filename, Context,
                                                    FSDiscriminatorPassOption);
@@ -723,9 +738,10 @@ mergeSampleProfile(const WeightedFileVector &Inputs, SymbolRemapper *Remapper,
       exitWithError(
           "cannot merge probe-based profile with non-probe-based profile");
     ProfileIsProbeBased = FunctionSamples::ProfileIsProbeBased;
-    if (ProfileIsCS.hasValue() && ProfileIsCS != FunctionSamples::ProfileIsCS)
+    if (ProfileIsCSFlat.hasValue() &&
+        ProfileIsCSFlat != FunctionSamples::ProfileIsCSFlat)
       exitWithError("cannot merge CS profile with non-CS profile");
-    ProfileIsCS = FunctionSamples::ProfileIsCS;
+    ProfileIsCSFlat = FunctionSamples::ProfileIsCSFlat;
     for (SampleProfileMap::iterator I = Profiles.begin(), E = Profiles.end();
          I != E; ++I) {
       sampleprof_error Result = sampleprof_error::success;
@@ -748,7 +764,7 @@ mergeSampleProfile(const WeightedFileVector &Inputs, SymbolRemapper *Remapper,
       WriterList.merge(*ReaderList);
   }
 
-  if (ProfileIsCS && (SampleMergeColdContext || SampleTrimColdContext)) {
+  if (ProfileIsCSFlat && (SampleMergeColdContext || SampleTrimColdContext)) {
     // Use threshold calculated from profile summary unless specified.
     SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);
     auto Summary = Builder.computeSummaryForProfiles(ProfileMap);
@@ -763,6 +779,12 @@ mergeSampleProfile(const WeightedFileVector &Inputs, SymbolRemapper *Remapper,
             SampleMergeColdContext, SampleColdContextFrameDepth, false);
   }
 
+  if (ProfileIsCSFlat && GenCSNestedProfile) {
+    CSProfileConverter CSConverter(ProfileMap);
+    CSConverter.convertProfiles();
+    ProfileIsCSFlat = FunctionSamples::ProfileIsCSFlat = false;
+  }
+
   auto WriterOrErr =
       SampleProfileWriter::create(OutputFilename, FormatMap[OutputFormat]);
   if (std::error_code EC = WriterOrErr.getError())
@@ -941,7 +963,13 @@ static int merge_main(int argc, const char *argv[]) {
   cl::opt<unsigned> InstrProfColdThreshold(
       "instr-prof-cold-threshold", cl::init(0), cl::Hidden,
       cl::desc("User specified cold threshold for instr profile which will "
-               "override the cold threshold got from profile summary."));
+               "override the cold threshold got from profile summary. "));
+  cl::opt<bool> GenCSNestedProfile(
+      "gen-cs-nested-profile", cl::Hidden, cl::init(false),
+      cl::desc("Generate nested function profiles for CSSPGO"));
+  cl::opt<std::string> DebugInfoFilename(
+      "debug-info", cl::init(""), cl::Hidden,
+      cl::desc("Use the provided debug info to correlate the raw profile."));
 
   cl::ParseCommandLineOptions(argc, argv, "LLVM profile data merger\n");
 
@@ -982,15 +1010,15 @@ static int merge_main(int argc, const char *argv[]) {
   }
 
   if (ProfileKind == instr)
-    mergeInstrProfile(WeightedInputs, Remapper.get(), OutputFilename,
-                      OutputFormat, OutputSparse, NumThreads, FailureMode);
+    mergeInstrProfile(WeightedInputs, DebugInfoFilename, Remapper.get(),
+                      OutputFilename, OutputFormat, OutputSparse, NumThreads,
+                      FailureMode);
   else
     mergeSampleProfile(WeightedInputs, Remapper.get(), OutputFilename,
                        OutputFormat, ProfileSymbolListFile, CompressAllSections,
-                       UseMD5, GenPartialProfile, SampleMergeColdContext,
-                       SampleTrimColdContext, SampleColdContextFrameDepth,
-                       FailureMode);
-
+                       UseMD5, GenPartialProfile, GenCSNestedProfile,
+                       SampleMergeColdContext, SampleTrimColdContext,
+                       SampleColdContextFrameDepth, FailureMode);
   return 0;
 }
 
@@ -1015,7 +1043,7 @@ static void overlapInstrProfile(const std::string &BaseFilename,
     OS << "Sum of edge counts for profile " << TestFilename << " is 0.\n";
     exit(0);
   }
-  loadInput(WeightedInput, nullptr, &Context);
+  loadInput(WeightedInput, nullptr, nullptr, &Context);
   overlapInput(BaseFilename, TestFilename, &Context, Overlap, FuncFilter, OS,
                IsCS);
   Overlap.dump(OS);
@@ -1911,7 +1939,7 @@ std::error_code SampleOverlapAggregator::loadProfiles() {
   if (BaseReader->profileIsProbeBased() != TestReader->profileIsProbeBased())
     exitWithError(
         "cannot compare probe-based profile with non-probe-based profile");
-  if (BaseReader->profileIsCS() != TestReader->profileIsCS())
+  if (BaseReader->profileIsCSFlat() != TestReader->profileIsCSFlat())
     exitWithError("cannot compare CS profile with non-CS profile");
 
   // Load BaseHotThreshold and TestHotThreshold as 99-percentile threshold in
diff --git a/llvm/tools/llvm-readobj/ARMEHABIPrinter.h b/llvm/tools/llvm-readobj/ARMEHABIPrinter.h
index d97cea4b6d6a..d641b172eb91 100644
--- a/llvm/tools/llvm-readobj/ARMEHABIPrinter.h
+++ b/llvm/tools/llvm-readobj/ARMEHABIPrinter.h
@@ -158,9 +158,8 @@ inline void OpcodeDecoder::Decode_10110001_0000iiii(const uint8_t *Opcodes,
   uint8_t Opcode0 = Opcodes[OI++ ^ 3];
   uint8_t Opcode1 = Opcodes[OI++ ^ 3];
 
-  SW.startLine()
-    << format("0x%02X 0x%02X ; %s", Opcode0, Opcode1,
-              ((Opcode1 & 0xf0) || Opcode1 == 0x00) ? "spare" : "pop ");
+  SW.startLine() << format("0x%02X 0x%02X ; %s", Opcode0, Opcode1,
+                           (Opcode1 & 0xf0) ? "spare" : "pop ");
   if (((Opcode1 & 0xf0) == 0x00) && Opcode1)
     PrintGPR((Opcode1 & 0x0f));
   OS << '\n';
@@ -195,7 +194,8 @@ inline void OpcodeDecoder::Decode_10110011_sssscccc(const uint8_t *Opcodes,
 inline void OpcodeDecoder::Decode_101101nn(const uint8_t *Opcodes,
                                            unsigned &OI) {
   uint8_t Opcode = Opcodes[OI++ ^ 3];
-  SW.startLine() << format("0x%02X      ; spare\n", Opcode);
+  SW.startLine() << format("0x%02X      ; %s\n", Opcode,
+                           (Opcode == 0xb4) ? "pop ra_auth_code" : "spare");
 }
 inline void OpcodeDecoder::Decode_10111nnn(const uint8_t *Opcodes,
                                            unsigned &OI) {
@@ -512,7 +512,7 @@ template <typename ET>
 void PrinterContext<ET>::PrintOpcodes(const uint8_t *Entry,
                                       size_t Length, off_t Offset) const {
   ListScope OCC(SW, "Opcodes");
-  OpcodeDecoder(OCC.W).Decode(Entry, Offset, Length);
+  OpcodeDecoder(SW).Decode(Entry, Offset, Length);
 }
 
 template <typename ET>
diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp
index 9dd777dd98e7..9d9c22234726 100644
--- a/llvm/tools/llvm-readobj/ELFDumper.cpp
+++ b/llvm/tools/llvm-readobj/ELFDumper.cpp
@@ -31,6 +31,7 @@
 #include "llvm/BinaryFormat/AMDGPUMetadataVerifier.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/Demangle/Demangle.h"
+#include "llvm/Object/Archive.h"
 #include "llvm/Object/ELF.h"
 #include "llvm/Object/ELFObjectFile.h"
 #include "llvm/Object/ELFTypes.h"
@@ -548,6 +549,9 @@ public:
     assert(&this->W.getOStream() == &llvm::fouts());
   }
 
+  void printFileSummary(StringRef FileStr, ObjectFile &Obj,
+                        ArrayRef<std::string> InputFilenames,
+                        const Archive *A) override;
   void printFileHeaders() override;
   void printGroupSections() override;
   void printRelocations() override;
@@ -588,14 +592,6 @@ private:
   };
 
   template <typename T, typename TEnum>
-  std::string printEnum(T Value, ArrayRef<EnumEntry<TEnum>> EnumValues) const {
-    for (const EnumEntry<TEnum> &EnumItem : EnumValues)
-      if (EnumItem.Value == Value)
-        return std::string(EnumItem.AltName);
-    return to_hexString(Value, false);
-  }
-
-  template <typename T, typename TEnum>
   std::string printFlags(T Value, ArrayRef<EnumEntry<TEnum>> EnumValues,
                          TEnum EnumMask1 = {}, TEnum EnumMask2 = {},
                          TEnum EnumMask3 = {}) const {
@@ -705,9 +701,27 @@ private:
   void printMipsPLT(const MipsGOTParser<ELFT> &Parser) override;
   void printMipsABIFlags() override;
 
+protected:
   ScopedPrinter &W;
 };
 
+// JSONELFDumper shares most of the same implementation as LLVMELFDumper except
+// it uses a JSONScopedPrinter.
+template <typename ELFT> class JSONELFDumper : public LLVMELFDumper<ELFT> {
+public:
+  LLVM_ELF_IMPORT_TYPES_ELFT(ELFT)
+
+  JSONELFDumper(const object::ELFObjectFile<ELFT> &ObjF, ScopedPrinter &Writer)
+      : LLVMELFDumper<ELFT>(ObjF, Writer) {}
+
+  void printFileSummary(StringRef FileStr, ObjectFile &Obj,
+                        ArrayRef<std::string> InputFilenames,
+                        const Archive *A) override;
+
+private:
+  std::unique_ptr<DictScope> FileScope;
+};
+
 } // end anonymous namespace
 
 namespace llvm {
@@ -717,6 +731,8 @@ static std::unique_ptr<ObjDumper>
 createELFDumper(const ELFObjectFile<ELFT> &Obj, ScopedPrinter &Writer) {
   if (opts::Output == opts::GNU)
     return std::make_unique<GNUELFDumper<ELFT>>(Obj, Writer);
+  else if (opts::Output == opts::JSON)
+    return std::make_unique<JSONELFDumper<ELFT>>(Obj, Writer);
   return std::make_unique<LLVMELFDumper<ELFT>>(Obj, Writer);
 }
 
@@ -3233,6 +3249,16 @@ static const EnumEntry<unsigned> *getObjectFileEnumEntry(unsigned Type) {
   return nullptr;
 }
 
+template <class ELFT>
+void GNUELFDumper<ELFT>::printFileSummary(StringRef FileStr, ObjectFile &Obj,
+                                          ArrayRef<std::string> InputFilenames,
+                                          const Archive *A) {
+  if (InputFilenames.size() > 1 || A) {
+    this->W.startLine() << "\n";
+    this->W.printString("File", FileStr);
+  }
+}
+
 template <class ELFT> void GNUELFDumper<ELFT>::printFileHeaders() {
   const Elf_Ehdr &e = this->Obj.getHeader();
   OS << "ELF Header:\n";
@@ -3241,9 +3267,9 @@ template <class ELFT> void GNUELFDumper<ELFT>::printFileHeaders() {
   for (int i = 0; i < ELF::EI_NIDENT; i++)
     OS << format(" %02x", static_cast<int>(e.e_ident[i]));
   OS << "\n";
-  Str = printEnum(e.e_ident[ELF::EI_CLASS], makeArrayRef(ElfClass));
+  Str = enumToString(e.e_ident[ELF::EI_CLASS], makeArrayRef(ElfClass));
   printFields(OS, "Class:", Str);
-  Str = printEnum(e.e_ident[ELF::EI_DATA], makeArrayRef(ElfDataEncoding));
+  Str = enumToString(e.e_ident[ELF::EI_DATA], makeArrayRef(ElfDataEncoding));
   printFields(OS, "Data:", Str);
   OS.PadToColumn(2u);
   OS << "Version:";
@@ -3252,7 +3278,7 @@ template <class ELFT> void GNUELFDumper<ELFT>::printFileHeaders() {
   if (e.e_version == ELF::EV_CURRENT)
     OS << " (current)";
   OS << "\n";
-  Str = printEnum(e.e_ident[ELF::EI_OSABI], makeArrayRef(ElfOSABI));
+  Str = enumToString(e.e_ident[ELF::EI_OSABI], makeArrayRef(ElfOSABI));
   printFields(OS, "OS/ABI:", Str);
   printFields(OS,
               "ABI Version:", std::to_string(e.e_ident[ELF::EI_ABIVERSION]));
@@ -3269,7 +3295,7 @@ template <class ELFT> void GNUELFDumper<ELFT>::printFileHeaders() {
   }
   printFields(OS, "Type:", Str);
 
-  Str = printEnum(e.e_machine, makeArrayRef(ElfMachineType));
+  Str = enumToString(e.e_machine, makeArrayRef(ElfMachineType));
   printFields(OS, "Machine:", Str);
   Str = "0x" + to_hexString(e.e_version);
   printFields(OS, "Version:", Str);
@@ -3759,14 +3785,14 @@ void GNUELFDumper<ELFT>::printSymbol(const Elf_Sym &Symbol, unsigned SymIndex,
   unsigned char SymbolType = Symbol.getType();
   if (this->Obj.getHeader().e_machine == ELF::EM_AMDGPU &&
       SymbolType >= ELF::STT_LOOS && SymbolType < ELF::STT_HIOS)
-    Fields[3].Str = printEnum(SymbolType, makeArrayRef(AMDGPUSymbolTypes));
+    Fields[3].Str = enumToString(SymbolType, makeArrayRef(AMDGPUSymbolTypes));
   else
-    Fields[3].Str = printEnum(SymbolType, makeArrayRef(ElfSymbolTypes));
+    Fields[3].Str = enumToString(SymbolType, makeArrayRef(ElfSymbolTypes));
 
   Fields[4].Str =
-      printEnum(Symbol.getBinding(), makeArrayRef(ElfSymbolBindings));
+      enumToString(Symbol.getBinding(), makeArrayRef(ElfSymbolBindings));
   Fields[5].Str =
-      printEnum(Symbol.getVisibility(), makeArrayRef(ElfSymbolVisibilities));
+      enumToString(Symbol.getVisibility(), makeArrayRef(ElfSymbolVisibilities));
 
   if (Symbol.st_other & ~0x3) {
     if (this->Obj.getHeader().e_machine == ELF::EM_AARCH64) {
@@ -3822,14 +3848,14 @@ void GNUELFDumper<ELFT>::printHashedSymbol(const Elf_Sym *Symbol,
   unsigned char SymbolType = Symbol->getType();
   if (this->Obj.getHeader().e_machine == ELF::EM_AMDGPU &&
       SymbolType >= ELF::STT_LOOS && SymbolType < ELF::STT_HIOS)
-    Fields[4].Str = printEnum(SymbolType, makeArrayRef(AMDGPUSymbolTypes));
+    Fields[4].Str = enumToString(SymbolType, makeArrayRef(AMDGPUSymbolTypes));
   else
-    Fields[4].Str = printEnum(SymbolType, makeArrayRef(ElfSymbolTypes));
+    Fields[4].Str = enumToString(SymbolType, makeArrayRef(ElfSymbolTypes));
 
   Fields[5].Str =
-      printEnum(Symbol->getBinding(), makeArrayRef(ElfSymbolBindings));
-  Fields[6].Str =
-      printEnum(Symbol->getVisibility(), makeArrayRef(ElfSymbolVisibilities));
+      enumToString(Symbol->getBinding(), makeArrayRef(ElfSymbolBindings));
+  Fields[6].Str = enumToString(Symbol->getVisibility(),
+                               makeArrayRef(ElfSymbolVisibilities));
   Fields[7].Str = getSymbolSectionNdx(*Symbol, SymIndex, ShndxTable);
   Fields[8].Str =
       this->getFullSymbolName(*Symbol, SymIndex, ShndxTable, StrTable, true);
@@ -4192,7 +4218,7 @@ template <class ELFT> void GNUELFDumper<ELFT>::printProgramHeaders() {
   Field Fields[8] = {2,         17,        26,        37 + Bias,
                      48 + Bias, 56 + Bias, 64 + Bias, 68 + Bias};
   OS << "\nElf file type is "
-     << printEnum(Header.e_type, makeArrayRef(ElfObjectFileType)) << "\n"
+     << enumToString(Header.e_type, makeArrayRef(ElfObjectFileType)) << "\n"
      << "Entry point " << format_hex(Header.e_entry, 3) << "\n"
      << "There are " << Header.e_phnum << " program headers,"
      << " starting at offset " << Header.e_phoff << "\n\n"
@@ -5466,7 +5492,7 @@ StringRef getNoteTypeName(const typename ELFT::Note &Note, unsigned ELFType) {
       return Result;
     return FindNote(CoreNoteTypes);
   }
-  if (Name.startswith("OpenBSD") && ELFType == ELF::ET_CORE) {
+  if (ELFType == ELF::ET_CORE && Name.startswith("OpenBSD")) {
     // OpenBSD also places the generic core notes in the OpenBSD namespace.
     StringRef Result = FindNote(OpenBSDCoreNoteTypes);
     if (!Result.empty())
@@ -6180,7 +6206,7 @@ void GNUELFDumper<ELFT>::printMipsGOT(const MipsGOTParser<ELFT> &Parser) {
       OS.PadToColumn(31 + 2 * Bias);
       OS << to_string(format_hex_no_prefix(Sym.st_value, 8 + Bias));
       OS.PadToColumn(40 + 3 * Bias);
-      OS << printEnum(Sym.getType(), makeArrayRef(ElfSymbolTypes));
+      OS << enumToString(Sym.getType(), makeArrayRef(ElfSymbolTypes));
       OS.PadToColumn(48 + 3 * Bias);
       OS << getSymbolSectionNdx(Sym, &Sym - this->dynamic_symbols().begin(),
                                 ShndxTable);
@@ -6234,7 +6260,7 @@ void GNUELFDumper<ELFT>::printMipsPLT(const MipsGOTParser<ELFT> &Parser) {
       OS.PadToColumn(20 + 2 * Bias);
       OS << to_string(format_hex_no_prefix(Sym.st_value, 8 + Bias));
       OS.PadToColumn(29 + 3 * Bias);
-      OS << printEnum(Sym.getType(), makeArrayRef(ElfSymbolTypes));
+      OS << enumToString(Sym.getType(), makeArrayRef(ElfSymbolTypes));
       OS.PadToColumn(37 + 3 * Bias);
       OS << getSymbolSectionNdx(Sym, &Sym - this->dynamic_symbols().begin(),
                                 ShndxTable);
@@ -6281,10 +6307,10 @@ template <class ELFT> void GNUELFDumper<ELFT>::printMipsABIFlags() {
   OS << "GPR size: " << getMipsRegisterSize(Flags->gpr_size) << "\n";
   OS << "CPR1 size: " << getMipsRegisterSize(Flags->cpr1_size) << "\n";
   OS << "CPR2 size: " << getMipsRegisterSize(Flags->cpr2_size) << "\n";
-  OS << "FP ABI: " << printEnum(Flags->fp_abi, makeArrayRef(ElfMipsFpABIType))
-     << "\n";
+  OS << "FP ABI: "
+     << enumToString(Flags->fp_abi, makeArrayRef(ElfMipsFpABIType)) << "\n";
   OS << "ISA Extension: "
-     << printEnum(Flags->isa_ext, makeArrayRef(ElfMipsISAExtType)) << "\n";
+     << enumToString(Flags->isa_ext, makeArrayRef(ElfMipsISAExtType)) << "\n";
   if (Flags->ases == 0)
     OS << "ASEs: None\n";
   else
@@ -7314,3 +7340,18 @@ template <class ELFT> void LLVMELFDumper<ELFT>::printMipsABIFlags() {
   W.printFlags("Flags 1", Flags->flags1, makeArrayRef(ElfMipsFlags1));
   W.printHex("Flags 2", Flags->flags2);
 }
+
+template <class ELFT>
+void JSONELFDumper<ELFT>::printFileSummary(StringRef FileStr, ObjectFile &Obj,
+                                           ArrayRef<std::string> InputFilenames,
+                                           const Archive *A) {
+  FileScope = std::make_unique<DictScope>(this->W, FileStr);
+  DictScope D(this->W, "FileSummary");
+  this->W.printString("File", FileStr);
+  this->W.printString("Format", Obj.getFileFormatName());
+  this->W.printString("Arch", Triple::getArchTypeName(Obj.getArch()));
+  this->W.printString(
+      "AddressSize",
+      std::string(formatv("{0}bit", 8 * Obj.getBytesInAddress())));
+  this->printLoadName();
+}
diff --git a/llvm/tools/llvm-readobj/ObjDumper.cpp b/llvm/tools/llvm-readobj/ObjDumper.cpp
index dc4a3031f914..6dde3725b4d6 100644
--- a/llvm/tools/llvm-readobj/ObjDumper.cpp
+++ b/llvm/tools/llvm-readobj/ObjDumper.cpp
@@ -13,6 +13,7 @@
 
 #include "ObjDumper.h"
 #include "llvm-readobj.h"
+#include "llvm/Object/Archive.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/FormatVariadic.h"
@@ -85,6 +86,18 @@ void ObjDumper::printAsStringList(StringRef StringContent,
   }
 }
 
+void ObjDumper::printFileSummary(StringRef FileStr, object::ObjectFile &Obj,
+                                 ArrayRef<std::string> InputFilenames,
+                                 const object::Archive *A) {
+  W.startLine() << "\n";
+  W.printString("File", FileStr);
+  W.printString("Format", Obj.getFileFormatName());
+  W.printString("Arch", Triple::getArchTypeName(Obj.getArch()));
+  W.printString("AddressSize",
+                std::string(formatv("{0}bit", 8 * Obj.getBytesInAddress())));
+  this->printLoadName();
+}
+
 static std::vector<object::SectionRef>
 getSectionRefsByNameOrIndex(const object::ObjectFile &Obj,
                             ArrayRef<std::string> Sections) {
diff --git a/llvm/tools/llvm-readobj/ObjDumper.h b/llvm/tools/llvm-readobj/ObjDumper.h
index b395a95f3cb4..a09a243d381e 100644
--- a/llvm/tools/llvm-readobj/ObjDumper.h
+++ b/llvm/tools/llvm-readobj/ObjDumper.h
@@ -20,6 +20,7 @@
 
 namespace llvm {
 namespace object {
+class Archive;
 class COFFImportFile;
 class ObjectFile;
 class XCOFFObjectFile;
@@ -39,6 +40,9 @@ public:
 
   virtual bool canDumpContent() { return true; }
 
+  virtual void printFileSummary(StringRef FileStr, object::ObjectFile &Obj,
+                                ArrayRef<std::string> InputFilenames,
+                                const object::Archive *A);
   virtual void printFileHeaders() = 0;
   virtual void printSectionHeaders() = 0;
   virtual void printRelocations() = 0;
diff --git a/llvm/tools/llvm-readobj/Opts.td b/llvm/tools/llvm-readobj/Opts.td
index 7723691e8225..d0f273fa60c7 100644
--- a/llvm/tools/llvm-readobj/Opts.td
+++ b/llvm/tools/llvm-readobj/Opts.td
@@ -28,6 +28,7 @@ def expand_relocs : FF<"expand-relocs", "Expand each shown relocation to multipl
 def file_header : FF<"file-header", "Display file header">;
 def headers : FF<"headers", "Equivalent to setting: --file-header, --program-headers, --section-headers">;
 defm hex_dump : Eq<"hex-dump", "Display the specified section(s) as hexadecimal bytes">, MetaVarName<"<name or index>">;
+def pretty_print : FF<"pretty-print", "Pretty print JSON output">;
 def relocs : FF<"relocs", "Display the relocation entries in the file">;
 def section_data : FF<"section-data", "Display section data for each section shown. This option has no effect for GNU style output">;
 def section_details : FF<"section-details", "Display the section details">;
@@ -47,7 +48,7 @@ def unwind : FF<"unwind", "Display unwind information">;
 def grp_elf : OptionGroup<"kind">, HelpText<"OPTIONS (ELF specific)">;
 def dynamic_table : FF<"dynamic-table", "Display the dynamic section table">, Group<grp_elf>;
 def elf_linker_options : FF<"elf-linker-options", "Display the .linker-options section">, Group<grp_elf>;
-defm elf_output_style : Eq<"elf-output-style", "Specify ELF dump style: LLVM or GNU">, Group<grp_elf>;
+defm elf_output_style : Eq<"elf-output-style", "Specify ELF dump style: LLVM, GNU, JSON">, Group<grp_elf>;
 def histogram : FF<"histogram", "Display bucket list histogram for hash sections">, Group<grp_elf>;
 def section_groups : FF<"section-groups", "Display section groups">, Group<grp_elf>;
 def gnu_hash_table : FF<"gnu-hash-table", "Display the GNU hash table for dynamic symbols">, Group<grp_elf>;
diff --git a/llvm/tools/llvm-readobj/llvm-readobj.cpp b/llvm/tools/llvm-readobj/llvm-readobj.cpp
index a598e2c28832..46862bbad7cb 100644
--- a/llvm/tools/llvm-readobj/llvm-readobj.cpp
+++ b/llvm/tools/llvm-readobj/llvm-readobj.cpp
@@ -99,6 +99,7 @@ static bool DynamicSymbols;
 static bool FileHeaders;
 static bool Headers;
 static std::vector<std::string> HexDump;
+static bool PrettyPrint;
 static bool PrintStackMap;
 static bool PrintStackSizes;
 static bool Relocations;
@@ -230,13 +231,17 @@ static void parseOptions(const opt::InputArgList &Args) {
   opts::DynamicTable = Args.hasArg(OPT_dynamic_table);
   opts::ELFLinkerOptions = Args.hasArg(OPT_elf_linker_options);
   if (Arg *A = Args.getLastArg(OPT_elf_output_style_EQ)) {
-    StringRef V(A->getValue());
-    if (V == "LLVM")
-      opts::Output = opts::OutputStyleTy::LLVM;
-    else if (V == "GNU")
-      opts::Output = opts::OutputStyleTy::GNU;
-    else
-      error("--elf-output-style value should be either 'LLVM' or 'GNU'");
+    std::string OutputStyleChoice = A->getValue();
+    opts::Output = StringSwitch<opts::OutputStyleTy>(OutputStyleChoice)
+                       .Case("LLVM", opts::OutputStyleTy::LLVM)
+                       .Case("GNU", opts::OutputStyleTy::GNU)
+                       .Case("JSON", opts::OutputStyleTy::JSON)
+                       .Default(opts::OutputStyleTy::UNKNOWN);
+    if (opts::Output == opts::OutputStyleTy::UNKNOWN) {
+      error("--elf-output-style value should be either 'LLVM', 'GNU', or "
+            "'JSON', but was '" +
+            OutputStyleChoice + "'");
+    }
   }
   opts::GnuHashTable = Args.hasArg(OPT_gnu_hash_table);
   opts::HashSymbols = Args.hasArg(OPT_hash_symbols);
@@ -244,6 +249,7 @@ static void parseOptions(const opt::InputArgList &Args) {
   opts::HashHistogram = Args.hasArg(OPT_histogram);
   opts::NeededLibraries = Args.hasArg(OPT_needed_libs);
   opts::Notes = Args.hasArg(OPT_notes);
+  opts::PrettyPrint = Args.hasArg(OPT_pretty_print);
   opts::ProgramHeaders = Args.hasArg(OPT_program_headers);
   opts::RawRelr = Args.hasArg(OPT_raw_relr);
   opts::SectionGroups = Args.hasArg(OPT_section_groups);
@@ -333,18 +339,7 @@ static void dumpObject(ObjectFile &Obj, ScopedPrinter &Writer,
     reportError(DumperOrErr.takeError(), FileStr);
   Dumper = (*DumperOrErr).get();
 
-  if (opts::Output == opts::LLVM || opts::InputFilenames.size() > 1 || A) {
-    Writer.startLine() << "\n";
-    Writer.printString("File", FileStr);
-  }
-  if (opts::Output == opts::LLVM) {
-    Writer.printString("Format", Obj.getFileFormatName());
-    Writer.printString("Arch", Triple::getArchTypeName(Obj.getArch()));
-    Writer.printString(
-        "AddressSize",
-        std::string(formatv("{0}bit", 8 * Obj.getBytesInAddress())));
-    Dumper->printLoadName();
-  }
+  Dumper->printFileSummary(FileStr, Obj, opts::InputFilenames, A);
 
   if (opts::FileHeaders)
     Dumper->printFileHeaders();
@@ -550,6 +545,13 @@ static void dumpInput(StringRef File, ScopedPrinter &Writer) {
       OwningBinary<Binary>(std::move(Bin), std::move(Buffer)));
 }
 
+std::unique_ptr<ScopedPrinter> createWriter() {
+  if (opts::Output == opts::JSON)
+    return std::make_unique<JSONScopedPrinter>(
+        fouts(), opts::PrettyPrint ? 2 : 0, std::make_unique<ListScope>());
+  return std::make_unique<ScopedPrinter>(fouts());
+}
+
 int main(int argc, char *argv[]) {
   InitLLVM X(argc, argv);
   BumpPtrAllocator A;
@@ -610,16 +612,17 @@ int main(int argc, char *argv[]) {
     opts::SectionHeaders = true;
   }
 
-  ScopedPrinter Writer(fouts());
+  std::unique_ptr<ScopedPrinter> Writer = createWriter();
+
   for (const std::string &I : opts::InputFilenames)
-    dumpInput(I, Writer);
+    dumpInput(I, *Writer.get());
 
   if (opts::CodeViewMergedTypes) {
     if (opts::CodeViewEnableGHash)
-      dumpCodeViewMergedTypes(Writer, CVTypes.GlobalIDTable.records(),
+      dumpCodeViewMergedTypes(*Writer.get(), CVTypes.GlobalIDTable.records(),
                               CVTypes.GlobalTypeTable.records());
     else
-      dumpCodeViewMergedTypes(Writer, CVTypes.IDTable.records(),
+      dumpCodeViewMergedTypes(*Writer.get(), CVTypes.IDTable.records(),
                               CVTypes.TypeTable.records());
   }
 
diff --git a/llvm/tools/llvm-readobj/llvm-readobj.h b/llvm/tools/llvm-readobj/llvm-readobj.h
index 7672da5c0aae..0ea695d1673d 100644
--- a/llvm/tools/llvm-readobj/llvm-readobj.h
+++ b/llvm/tools/llvm-readobj/llvm-readobj.h
@@ -39,7 +39,7 @@ extern bool ExpandRelocs;
 extern bool RawRelr;
 extern bool CodeViewSubsectionBytes;
 extern bool Demangle;
-enum OutputStyleTy { LLVM, GNU };
+enum OutputStyleTy { LLVM, GNU, JSON, UNKNOWN };
 extern OutputStyleTy Output;
 } // namespace opts
 
diff --git a/llvm/tools/llvm-symbolizer/llvm-symbolizer.cpp b/llvm/tools/llvm-symbolizer/llvm-symbolizer.cpp
index 2adbf1f1731d..66a2e703129b 100644
--- a/llvm/tools/llvm-symbolizer/llvm-symbolizer.cpp
+++ b/llvm/tools/llvm-symbolizer/llvm-symbolizer.cpp
@@ -19,6 +19,7 @@
 #include "llvm/Config/config.h"
 #include "llvm/DebugInfo/Symbolize/DIPrinter.h"
 #include "llvm/DebugInfo/Symbolize/Symbolize.h"
+#include "llvm/Debuginfod/HTTPClient.h"
 #include "llvm/Option/Arg.h"
 #include "llvm/Option/ArgList.h"
 #include "llvm/Option/Option.h"
@@ -261,6 +262,8 @@ static FunctionNameKind decideHowToPrintFunctions(const opt::InputArgList &Args,
 
 int main(int argc, char **argv) {
   InitLLVM X(argc, argv);
+  // The HTTPClient must be initialized for use by the debuginfod client.
+  HTTPClient::initialize();
   sys::InitializeCOMRAII COM(sys::COMThreadingMode::MultiThreaded);
 
   bool IsAddr2Line = sys::path::stem(argv[0]).contains("addr2line");
diff --git a/llvm/tools/llvm-tapi-diff/llvm-tapi-diff.cpp b/llvm/tools/llvm-tapi-diff/llvm-tapi-diff.cpp
index 40f1eec162be..772f124c5a59 100644
--- a/llvm/tools/llvm-tapi-diff/llvm-tapi-diff.cpp
+++ b/llvm/tools/llvm-tapi-diff/llvm-tapi-diff.cpp
@@ -1,6 +1,4 @@
-//===-- llvm-tapi-diff.cpp - tbd comparator command-line driver ---*-
-// C++
-//-*-===//
+//===-- llvm-tapi-diff.cpp - tbd comparator command-line driver --*- C++-*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -31,16 +29,8 @@ cl::opt<std::string> InputFileNameLHS(cl::Positional, cl::desc("<first file>"),
                                       cl::cat(NMCat));
 cl::opt<std::string> InputFileNameRHS(cl::Positional, cl::desc("<second file>"),
                                       cl::cat(NMCat));
-
-std::string ToolName;
 } // anonymous namespace
 
-ExitOnError ExitOnErr;
-
-void setErrorBanner(ExitOnError &ExitOnErr, std::string InputFile) {
-  ExitOnErr.setBanner(ToolName + ": error: " + InputFile + ": ");
-}
-
 Expected<std::unique_ptr<Binary>> convertFileToBinary(std::string &Filename) {
   ErrorOr<std::unique_ptr<MemoryBuffer>> BufferOrErr =
       MemoryBuffer::getFileOrSTDIN(Filename);
@@ -52,35 +42,29 @@ Expected<std::unique_ptr<Binary>> convertFileToBinary(std::string &Filename) {
 int main(int Argc, char **Argv) {
   InitLLVM X(Argc, Argv);
   cl::HideUnrelatedOptions(NMCat);
-  cl::ParseCommandLineOptions(
-      Argc, Argv,
-      "This tool will compare two tbd files and return the "
-      "differences in those files.");
+  cl::ParseCommandLineOptions(Argc, Argv, "Text-based Stubs Comparison Tool");
   if (InputFileNameLHS.empty() || InputFileNameRHS.empty()) {
     cl::PrintHelpMessage();
     return EXIT_FAILURE;
   }
 
-  ToolName = Argv[0];
-
-  setErrorBanner(ExitOnErr, InputFileNameLHS);
+  ExitOnError ExitOnErr("error: '" + InputFileNameLHS + "' ",
+                        /*DefaultErrorExitCode=*/2);
   auto BinLHS = ExitOnErr(convertFileToBinary(InputFileNameLHS));
 
   TapiUniversal *FileLHS = dyn_cast<TapiUniversal>(BinLHS.get());
   if (!FileLHS) {
-    ExitOnErr(
-        createStringError(std::errc::executable_format_error,
-                          "Error when parsing file, unsupported file format"));
+    ExitOnErr(createStringError(std::errc::executable_format_error,
+                                "unsupported file format"));
   }
 
-  setErrorBanner(ExitOnErr, InputFileNameRHS);
+  ExitOnErr.setBanner("error: '" + InputFileNameRHS + "' ");
   auto BinRHS = ExitOnErr(convertFileToBinary(InputFileNameRHS));
 
   TapiUniversal *FileRHS = dyn_cast<TapiUniversal>(BinRHS.get());
   if (!FileRHS) {
-    ExitOnErr(
-        createStringError(std::errc::executable_format_error,
-                          "Error when parsing file, unsupported file format"));
+    ExitOnErr(createStringError(std::errc::executable_format_error,
+                                "unsupported file format"));
   }
 
   raw_ostream &OS = outs();
diff --git a/llvm/utils/TableGen/AsmWriterEmitter.cpp b/llvm/utils/TableGen/AsmWriterEmitter.cpp
index bb13c4033db7..9283ceeb31e0 100644
--- a/llvm/utils/TableGen/AsmWriterEmitter.cpp
+++ b/llvm/utils/TableGen/AsmWriterEmitter.cpp
@@ -757,8 +757,6 @@ public:
         ++I;
       }
     }
-
-    OS.flush();
     return OutString;
   }
 
diff --git a/llvm/utils/TableGen/CodeEmitterGen.cpp b/llvm/utils/TableGen/CodeEmitterGen.cpp
index ee77ef5eda5f..fbac0d969917 100644
--- a/llvm/utils/TableGen/CodeEmitterGen.cpp
+++ b/llvm/utils/TableGen/CodeEmitterGen.cpp
@@ -515,7 +515,7 @@ void CodeEmitterGen::run(raw_ostream &o) {
     << "    std::string msg;\n"
     << "    raw_string_ostream Msg(msg);\n"
     << "    Msg << \"Not supported instr: \" << MI;\n"
-    << "    report_fatal_error(Msg.str().c_str());\n"
+    << "    report_fatal_error(msg.c_str());\n"
     << "  }\n";
   if (UseAPInt)
     o << "  Inst = Value;\n";
diff --git a/llvm/utils/TableGen/CodeGenDAGPatterns.cpp b/llvm/utils/TableGen/CodeGenDAGPatterns.cpp
index 4a247050ceeb..4de619df5b5f 100644
--- a/llvm/utils/TableGen/CodeGenDAGPatterns.cpp
+++ b/llvm/utils/TableGen/CodeGenDAGPatterns.cpp
@@ -1583,7 +1583,7 @@ static TreePatternNode *getOperandNum(unsigned OpNo, TreePatternNode *N,
     OS << "Invalid operand number in type constraint "
            << (OpNo+NumResults) << " ";
     N->print(OS);
-    PrintFatalError(OS.str());
+    PrintFatalError(S);
   }
 
   return N->getChild(OpNo);
@@ -2268,7 +2268,9 @@ static TypeSetByHwMode getImplicitType(Record *R, unsigned ResNo,
     assert(ResNo == 0 && "FIXME: ComplexPattern with multiple results?");
     if (NotRegisters)
       return TypeSetByHwMode(); // Unknown.
-    return TypeSetByHwMode(CDP.getComplexPattern(R).getValueType());
+    Record *T = CDP.getComplexPattern(R).getValueType();
+    const CodeGenHwModes &CGH = CDP.getTargetInfo().getHwModes();
+    return TypeSetByHwMode(getValueTypeByHwMode(T, CGH));
   }
   if (R->isSubClassOf("PointerLikeRegClass")) {
     assert(ResNo == 0 && "Regclass can only have one result!");
@@ -2673,6 +2675,22 @@ bool TreePatternNode::ApplyTypeConstraints(TreePattern &TP, bool NotRegisters) {
   if (getOperator()->isSubClassOf("ComplexPattern")) {
     bool MadeChange = false;
 
+    if (!NotRegisters) {
+      assert(Types.size() == 1 && "ComplexPatterns only produce one result!");
+      Record *T = CDP.getComplexPattern(getOperator()).getValueType();
+      const CodeGenHwModes &CGH = CDP.getTargetInfo().getHwModes();
+      const ValueTypeByHwMode VVT = getValueTypeByHwMode(T, CGH);
+      // TODO: AArch64 and AMDGPU use ComplexPattern<untyped, ...> and then
+      // exclusively use those as non-leaf nodes with explicit type casts, so
+      // for backwards compatibility we do no inference in that case. This is
+      // not supported when the ComplexPattern is used as a leaf value,
+      // however; this inconsistency should be resolved, either by adding this
+      // case there or by altering the backends to not do this (e.g. using Any
+      // instead may work).
+      if (!VVT.isSimple() || VVT.getSimple() != MVT::Untyped)
+        MadeChange |= UpdateNodeType(0, VVT, TP);
+    }
+
     for (unsigned i = 0; i < getNumChildren(); ++i)
       MadeChange |= getChild(i)->ApplyTypeConstraints(TP, NotRegisters);
 
diff --git a/llvm/utils/TableGen/CodeGenSchedule.cpp b/llvm/utils/TableGen/CodeGenSchedule.cpp
index ee52b2e7ab9f..7c1c37f7b370 100644
--- a/llvm/utils/TableGen/CodeGenSchedule.cpp
+++ b/llvm/utils/TableGen/CodeGenSchedule.cpp
@@ -1398,7 +1398,7 @@ bool PredTransitions::mutuallyExclusive(Record *PredDef,
       //
       // if (A) return ...;
       // if (B) return ...;
-      if (!count(Preds, PC.Predicate))
+      if (!llvm::is_contained(Preds, PC.Predicate))
         continue;
       return true;
     }
diff --git a/llvm/utils/TableGen/CodeGenTarget.cpp b/llvm/utils/TableGen/CodeGenTarget.cpp
index d3beaf61989e..2c1583f7979d 100644
--- a/llvm/utils/TableGen/CodeGenTarget.cpp
+++ b/llvm/utils/TableGen/CodeGenTarget.cpp
@@ -576,7 +576,7 @@ bool CodeGenTarget::guessInstructionProperties() const {
 // ComplexPattern implementation
 //
 ComplexPattern::ComplexPattern(Record *R) {
-  Ty          = ::getValueType(R->getValueAsDef("Ty"));
+  Ty          = R->getValueAsDef("Ty");
   NumOperands = R->getValueAsInt("NumOperands");
   SelectFunc = std::string(R->getValueAsString("SelectFunc"));
   RootNodes   = R->getValueAsListOfDefs("RootNodes");
diff --git a/llvm/utils/TableGen/CodeGenTarget.h b/llvm/utils/TableGen/CodeGenTarget.h
index 9de9b512f74f..5bd84c873f2f 100644
--- a/llvm/utils/TableGen/CodeGenTarget.h
+++ b/llvm/utils/TableGen/CodeGenTarget.h
@@ -202,7 +202,7 @@ private:
 /// ComplexPattern - ComplexPattern info, corresponding to the ComplexPattern
 /// tablegen class in TargetSelectionDAG.td
 class ComplexPattern {
-  MVT::SimpleValueType Ty;
+  Record *Ty;
   unsigned NumOperands;
   std::string SelectFunc;
   std::vector<Record*> RootNodes;
@@ -211,7 +211,7 @@ class ComplexPattern {
 public:
   ComplexPattern(Record *R);
 
-  MVT::SimpleValueType getValueType() const { return Ty; }
+  Record *getValueType() const { return Ty; }
   unsigned getNumOperands() const { return NumOperands; }
   const std::string &getSelectFunc() const { return SelectFunc; }
   const std::vector<Record*> &getRootNodes() const {
diff --git a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
index a552998e4eee..5b0d16a8f3c8 100644
--- a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
+++ b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
@@ -169,7 +169,6 @@ static std::string GetPatFromTreePatternNode(const TreePatternNode *N) {
   std::string str;
   raw_string_ostream Stream(str);
   Stream << *N;
-  Stream.str();
   return str;
 }
 
@@ -235,7 +234,6 @@ static std::string getIncludePath(const Record *R) {
 
   Stream << SrcMgr.getBufferInfo(CurBuf).Buffer->getBufferIdentifier() << ":"
          << SrcMgr.FindLineNumber(L, CurBuf);
-  Stream.str();
   return str;
 }
 
diff --git a/llvm/utils/TableGen/DAGISelMatcherGen.cpp b/llvm/utils/TableGen/DAGISelMatcherGen.cpp
index 2625595cf9d5..2361ed8a7a95 100644
--- a/llvm/utils/TableGen/DAGISelMatcherGen.cpp
+++ b/llvm/utils/TableGen/DAGISelMatcherGen.cpp
@@ -267,7 +267,7 @@ void MatcherGen::EmitLeafMatchCode(const TreePatternNode *N) {
       std::string S;
       raw_string_ostream OS(S);
       OS << "We expect complex pattern uses to have names: " << *N;
-      PrintFatalError(OS.str());
+      PrintFatalError(S);
     }
 
     // Remember this ComplexPattern so that we can emit it after all the other
diff --git a/llvm/utils/TableGen/FastISelEmitter.cpp b/llvm/utils/TableGen/FastISelEmitter.cpp
index d64262124308..ac9fe6db4328 100644
--- a/llvm/utils/TableGen/FastISelEmitter.cpp
+++ b/llvm/utils/TableGen/FastISelEmitter.cpp
@@ -568,7 +568,6 @@ void FastISelMap::collectPatterns(CodeGenDAGPatterns &CGP) {
     std::string ManglingSuffix;
     raw_string_ostream SuffixOS(ManglingSuffix);
     Operands.PrintManglingSuffix(SuffixOS, ImmediatePredicates, true);
-    SuffixOS.flush();
     if (!StringSwitch<bool>(ManglingSuffix)
         .Cases("", "r", "rr", "ri", "i", "f", true)
         .Default(false))
diff --git a/llvm/utils/TableGen/GICombinerEmitter.cpp b/llvm/utils/TableGen/GICombinerEmitter.cpp
index c03cd371ef9d..63a9ed682d4f 100644
--- a/llvm/utils/TableGen/GICombinerEmitter.cpp
+++ b/llvm/utils/TableGen/GICombinerEmitter.cpp
@@ -633,7 +633,7 @@ void GICombinerEmitter::emitNameMatcher(raw_ostream &OS) const {
     raw_string_ostream SS(Code);
     SS << "return " << EnumeratedRule.getID() << ";\n";
     Cases.push_back(
-        std::make_pair(std::string(EnumeratedRule.getName()), SS.str()));
+        std::make_pair(std::string(EnumeratedRule.getName()), Code));
   }
 
   OS << "static Optional<uint64_t> getRuleIdxForIdentifier(StringRef "
diff --git a/llvm/utils/TableGen/GlobalISelEmitter.cpp b/llvm/utils/TableGen/GlobalISelEmitter.cpp
index d08186b7094b..7b1bd41a951b 100644
--- a/llvm/utils/TableGen/GlobalISelEmitter.cpp
+++ b/llvm/utils/TableGen/GlobalISelEmitter.cpp
@@ -109,7 +109,7 @@ public:
     raw_string_ostream OS(Str);
 
     emitCxxEnumValue(OS);
-    return OS.str();
+    return Str;
   }
 
   void emitCxxEnumValue(raw_ostream &OS) const {
@@ -1673,7 +1673,7 @@ public:
         CommentOS << "Operand " << OpIdx;
       else
         CommentOS << SymbolicName;
-      Table << MatchTable::Comment(CommentOS.str()) << MatchTable::LineBreak;
+      Table << MatchTable::Comment(Comment) << MatchTable::LineBreak;
     }
 
     emitPredicateListOpcodes(Table, Rule);
diff --git a/llvm/utils/TableGen/IntrinsicEmitter.cpp b/llvm/utils/TableGen/IntrinsicEmitter.cpp
index 437b5f002027..f4e5eb59cb80 100644
--- a/llvm/utils/TableGen/IntrinsicEmitter.cpp
+++ b/llvm/utils/TableGen/IntrinsicEmitter.cpp
@@ -250,7 +250,10 @@ enum IIT_Info {
   IIT_STRUCT9 = 49,
   IIT_V256 = 50,
   IIT_AMX  = 51,
-  IIT_PPCF128 = 52
+  IIT_PPCF128 = 52,
+  IIT_V3 = 53,
+  IIT_EXTERNREF = 54,
+  IIT_FUNCREF = 55
 };
 
 static void EncodeFixedValueType(MVT::SimpleValueType VT,
@@ -284,6 +287,10 @@ static void EncodeFixedValueType(MVT::SimpleValueType VT,
   case MVT::Other: return Sig.push_back(IIT_EMPTYSTRUCT);
   // MVT::isVoid is used to represent varargs here.
   case MVT::isVoid: return Sig.push_back(IIT_VARARG);
+  case MVT::externref:
+    return Sig.push_back(IIT_EXTERNREF);
+  case MVT::funcref:
+    return Sig.push_back(IIT_FUNCREF);
   }
 }
 
@@ -384,6 +391,7 @@ static void EncodeFixedType(Record *R, std::vector<unsigned char> &ArgCodes,
     default: PrintFatalError("unhandled vector type width in intrinsic!");
     case 1: Sig.push_back(IIT_V1); break;
     case 2: Sig.push_back(IIT_V2); break;
+    case 3: Sig.push_back(IIT_V3); break;
     case 4: Sig.push_back(IIT_V4); break;
     case 8: Sig.push_back(IIT_V8); break;
     case 16: Sig.push_back(IIT_V16); break;
diff --git a/llvm/utils/TableGen/PredicateExpander.cpp b/llvm/utils/TableGen/PredicateExpander.cpp
index a7256499d566..b129401461b5 100644
--- a/llvm/utils/TableGen/PredicateExpander.cpp
+++ b/llvm/utils/TableGen/PredicateExpander.cpp
@@ -233,7 +233,6 @@ void PredicateExpander::expandReturnStatement(raw_ostream &OS,
   SS << "return ";
   expandPredicate(SS, Rec);
   SS << ";";
-  SS.flush();
   OS << Buffer;
 }
 
@@ -276,7 +275,6 @@ void PredicateExpander::expandOpcodeSwitchStatement(raw_ostream &OS,
 
   SS.indent(getIndentLevel() * 2);
   SS << "} // end of switch-stmt";
-  SS.flush();
   OS << Buffer;
 }
 
diff --git a/llvm/utils/TableGen/SubtargetEmitter.cpp b/llvm/utils/TableGen/SubtargetEmitter.cpp
index c88fe949c25c..78bbb3196e5c 100644
--- a/llvm/utils/TableGen/SubtargetEmitter.cpp
+++ b/llvm/utils/TableGen/SubtargetEmitter.cpp
@@ -1433,7 +1433,6 @@ static void emitPredicateProlog(const RecordKeeper &Records, raw_ostream &OS) {
   for (Record *P : Prologs)
     Stream << P->getValueAsString("Code") << '\n';
 
-  Stream.flush();
   OS << Buffer;
 }
 
@@ -1492,7 +1491,6 @@ static void emitPredicates(const CodeGenSchedTransition &T,
   }
 
   SS << "return " << T.ToClassIdx << "; // " << SC.Name << '\n';
-  SS.flush();
   OS << Buffer;
 }
 
@@ -1526,9 +1524,7 @@ static void collectVariantClasses(const CodeGenSchedModels &SchedModels,
     if (OnlyExpandMCInstPredicates) {
       // Ignore this variant scheduling class no transitions use any meaningful
       // MCSchedPredicate definitions.
-      if (!any_of(SC.Transitions, [](const CodeGenSchedTransition &T) {
-            return hasMCSchedPredicates(T);
-          }))
+      if (llvm::none_of(SC.Transitions, hasMCSchedPredicates))
         continue;
     }
 
@@ -1550,8 +1546,7 @@ static void collectProcessorIndices(const CodeGenSchedClass &SC,
 }
 
 static bool isAlwaysTrue(const CodeGenSchedTransition &T) {
-  return llvm::all_of(T.PredTerm,
-                      [](const Record *R) { return isTruePredicate(R); });
+  return llvm::all_of(T.PredTerm, isTruePredicate);
 }
 
 void SubtargetEmitter::emitSchedModelHelpersImpl(
diff --git a/openmp/runtime/src/i18n/en_US.txt b/openmp/runtime/src/i18n/en_US.txt
index 351da540fadb..6c89ef23e851 100644
--- a/openmp/runtime/src/i18n/en_US.txt
+++ b/openmp/runtime/src/i18n/en_US.txt
@@ -361,6 +361,7 @@ OmpNoAllocator               "Allocator %1$s is not available, will use default
 TopologyGeneric              "%1$s: %2$s (%3$d total cores)"
 AffGranularityBad            "%1$s: granularity setting: %2$s does not exist in topology.  Using granularity=%3$s instead."
 TopologyHybrid               "%1$s: hybrid core type detected: %2$d %3$s cores."
+TopologyHybridCoreEff        "%1$s:   %2$d with core efficiency %3$d."
 
 # --- OpenMP errors detected at runtime ---
 #
@@ -471,6 +472,13 @@ AffHWSubsetOutOfOrder        "KMP_HW_SUBSET ignored: %1$s layer should come afte
 AffEqualTopologyTypes        "%1$s: topology layer \"%2$s\" is equivalent to \"%3$s\"."
 AffGranTooCoarseProcGroup    "%1$s: granularity=%2$s is too coarse, setting granularity=group."
 StgDeprecatedValue           "%1$s: \"%2$s\" value is deprecated. Please use \"%3$s\" instead."
+NumTeamsNotPositive          "num_teams value must be positive, it is %1$d, using %2$d instead."
+AffHWSubsetIncompat          "KMP_HW_SUBSET ignored: %1$s, %2$s: attributes are ambiguous, please only specify one."
+AffHWSubsetAttrRepeat        "KMP_HW_SUBSET ignored: %1$s: attribute specified more than once."
+AffHWSubsetAttrInvalid       "KMP_HW_SUBSET ignored: %1$s: attribute value %2$s is invalid."
+AffHWSubsetAllFiltered       "KMP_HW_SUBSET ignored: all hardware resources would be filtered, please reduce the filter."
+AffHWSubsetAttrsNonHybrid    "KMP_HW_SUBSET ignored: Too many attributes specified. This machine is not a hybrid architecutre."
+AffHWSubsetIgnoringAttr      "KMP_HW_SUBSET: ignoring %1$s attribute. This machine is not a hybrid architecutre."
 
 # --------------------------------------------------------------------------------------------------
 -*- HINTS -*-
@@ -529,6 +537,7 @@ BadExeFormat                 "System error #193 is \"Bad format of EXE or DLL fi
                              "Check whether \"%1$s\" is a file for %2$s architecture."
 SystemLimitOnThreads         "System-related limit on the number of threads."
 SetNewBound                  "Try setting new bounds (preferably less than or equal to %1$d) for num_teams clause."
+ValidValuesRange             "Valid values are from %1$d to %2$d."
 
 
 # --------------------------------------------------------------------------------------------------
diff --git a/openmp/runtime/src/include/omp_lib.h.var b/openmp/runtime/src/include/omp_lib.h.var
index a1d0c1f97770..a8287beee66c 100644
--- a/openmp/runtime/src/include/omp_lib.h.var
+++ b/openmp/runtime/src/include/omp_lib.h.var
@@ -769,7 +769,8 @@
           integer(omp_allocator_handle_kind), value :: allocator
         end function omp_calloc
 
-        function omp_aligned_calloc(alignment, nmemb, size, allocator) bind(c)
+        function omp_aligned_calloc(alignment, nmemb, size,                                                                         &
+     &      allocator) bind(c)
           use omp_lib_kinds
           use, intrinsic :: iso_c_binding, only : c_ptr, c_size_t
           type(c_ptr) omp_aligned_calloc
@@ -777,7 +778,8 @@
           integer(omp_allocator_handle_kind), value :: allocator
         end function omp_aligned_calloc
 
-        function omp_realloc(ptr, size, allocator, free_allocator) bind(c)
+        function omp_realloc(ptr, size, allocator,                                                                                  &
+     &      free_allocator) bind(c)
           use omp_lib_kinds
           use, intrinsic :: iso_c_binding, only : c_ptr, c_size_t
           type(c_ptr) omp_realloc
diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h
index ee068ab32f70..ede6aa992d53 100644
--- a/openmp/runtime/src/kmp.h
+++ b/openmp/runtime/src/kmp.h
@@ -618,6 +618,19 @@ enum kmp_hw_t : int {
   KMP_HW_LAST
 };
 
+typedef enum kmp_hw_core_type_t {
+  KMP_HW_CORE_TYPE_UNKNOWN = 0x0,
+#if KMP_ARCH_X86 || KMP_ARCH_X86_64
+  KMP_HW_CORE_TYPE_ATOM = 0x20,
+  KMP_HW_CORE_TYPE_CORE = 0x40,
+  KMP_HW_MAX_NUM_CORE_TYPES = 3,
+#else
+  KMP_HW_MAX_NUM_CORE_TYPES = 1,
+#endif
+} kmp_hw_core_type_t;
+
+#define KMP_HW_MAX_NUM_CORE_EFFS 8
+
 #define KMP_DEBUG_ASSERT_VALID_HW_TYPE(type)                                   \
   KMP_DEBUG_ASSERT(type >= (kmp_hw_t)0 && type < KMP_HW_LAST)
 #define KMP_ASSERT_VALID_HW_TYPE(type)                                         \
@@ -629,6 +642,7 @@ enum kmp_hw_t : int {
 
 const char *__kmp_hw_get_keyword(kmp_hw_t type, bool plural = false);
 const char *__kmp_hw_get_catalog_string(kmp_hw_t type, bool plural = false);
+const char *__kmp_hw_get_core_type_string(kmp_hw_core_type_t type);
 
 /* Only Linux* OS and Windows* OS support thread affinity. */
 #if KMP_AFFINITY_SUPPORTED
diff --git a/openmp/runtime/src/kmp_affinity.cpp b/openmp/runtime/src/kmp_affinity.cpp
index 84086f30317a..71e8b7fd10eb 100644
--- a/openmp/runtime/src/kmp_affinity.cpp
+++ b/openmp/runtime/src/kmp_affinity.cpp
@@ -189,8 +189,11 @@ void kmp_hw_thread_t::print() const {
   for (int i = 0; i < depth; ++i) {
     printf("%4d ", ids[i]);
   }
-  if (core_type != KMP_HW_CORE_TYPE_UNKNOWN) {
-    printf(" (%s)", __kmp_hw_get_core_type_string(core_type));
+  if (attrs) {
+    if (attrs.is_core_type_valid())
+      printf(" (%s)", __kmp_hw_get_core_type_string(attrs.get_core_type()));
+    if (attrs.is_core_eff_valid())
+      printf(" (eff=%d)", attrs.get_core_eff());
   }
   printf("\n");
 }
@@ -391,12 +394,6 @@ void kmp_topology_t::_gather_enumeration_information() {
     count[i] = 0;
     ratio[i] = 0;
   }
-  if (__kmp_is_hybrid_cpu()) {
-    for (int i = 0; i < KMP_HW_MAX_NUM_CORE_TYPES; ++i) {
-      core_types_count[i] = 0;
-      core_types[i] = KMP_HW_CORE_TYPE_UNKNOWN;
-    }
-  }
   int core_level = get_level(KMP_HW_CORE);
   for (int i = 0; i < num_hw_threads; ++i) {
     kmp_hw_thread_t &hw_thread = hw_threads[i];
@@ -413,9 +410,29 @@ void kmp_topology_t::_gather_enumeration_information() {
             ratio[l] = max[l];
           max[l] = 1;
         }
-        // Figure out the number of each core type for hybrid CPUs
-        if (__kmp_is_hybrid_cpu() && core_level >= 0 && layer <= core_level)
-          _increment_core_type(hw_thread.core_type);
+        // Figure out the number of different core types
+        // and efficiencies for hybrid CPUs
+        if (__kmp_is_hybrid_cpu() && core_level >= 0 && layer <= core_level) {
+          if (hw_thread.attrs.is_core_eff_valid() &&
+              hw_thread.attrs.core_eff >= num_core_efficiencies) {
+            // Because efficiencies can range from 0 to max efficiency - 1,
+            // the number of efficiencies is max efficiency + 1
+            num_core_efficiencies = hw_thread.attrs.core_eff + 1;
+          }
+          if (hw_thread.attrs.is_core_type_valid()) {
+            bool found = false;
+            for (int j = 0; j < num_core_types; ++j) {
+              if (hw_thread.attrs.get_core_type() == core_types[j]) {
+                found = true;
+                break;
+              }
+            }
+            if (!found) {
+              KMP_ASSERT(num_core_types < KMP_HW_MAX_NUM_CORE_TYPES);
+              core_types[num_core_types++] = hw_thread.attrs.get_core_type();
+            }
+          }
+        }
         break;
       }
     }
@@ -429,6 +446,42 @@ void kmp_topology_t::_gather_enumeration_information() {
   }
 }
 
+int kmp_topology_t::_get_ncores_with_attr(const kmp_hw_attr_t &attr,
+                                          int above_level,
+                                          bool find_all) const {
+  int current, current_max;
+  int previous_id[KMP_HW_LAST];
+  for (int i = 0; i < depth; ++i)
+    previous_id[i] = kmp_hw_thread_t::UNKNOWN_ID;
+  int core_level = get_level(KMP_HW_CORE);
+  if (find_all)
+    above_level = -1;
+  KMP_ASSERT(above_level < core_level);
+  current_max = 0;
+  current = 0;
+  for (int i = 0; i < num_hw_threads; ++i) {
+    kmp_hw_thread_t &hw_thread = hw_threads[i];
+    if (!find_all && hw_thread.ids[above_level] != previous_id[above_level]) {
+      if (current > current_max)
+        current_max = current;
+      current = hw_thread.attrs.contains(attr);
+    } else {
+      for (int level = above_level + 1; level <= core_level; ++level) {
+        if (hw_thread.ids[level] != previous_id[level]) {
+          if (hw_thread.attrs.contains(attr))
+            current++;
+          break;
+        }
+      }
+    }
+    for (int level = 0; level < depth; ++level)
+      previous_id[level] = hw_thread.ids[level];
+  }
+  if (current > current_max)
+    current_max = current;
+  return current_max;
+}
+
 // Find out if the topology is uniform
 void kmp_topology_t::_discover_uniformity() {
   int num = 1;
@@ -517,6 +570,10 @@ kmp_topology_t *kmp_topology_t::allocate(int nproc, int ndepth,
   retval->types = (kmp_hw_t *)arr;
   retval->ratio = arr + (size_t)KMP_HW_LAST;
   retval->count = arr + 2 * (size_t)KMP_HW_LAST;
+  retval->num_core_efficiencies = 0;
+  retval->num_core_types = 0;
+  for (int i = 0; i < KMP_HW_MAX_NUM_CORE_TYPES; ++i)
+    retval->core_types[i] = KMP_HW_CORE_TYPE_UNKNOWN;
   KMP_FOREACH_HW_TYPE(type) { retval->equivalent[type] = KMP_HW_UNKNOWN; }
   for (int i = 0; i < ndepth; ++i) {
     retval->types[i] = types[i];
@@ -574,18 +631,12 @@ void kmp_topology_t::dump() const {
   }
   printf("\n");
 
-  printf("* core_types:\n");
-  for (int i = 0; i < KMP_HW_MAX_NUM_CORE_TYPES; ++i) {
-    if (core_types[i] != KMP_HW_CORE_TYPE_UNKNOWN) {
-      printf("    %d %s core%c\n", core_types_count[i],
-             __kmp_hw_get_core_type_string(core_types[i]),
-             ((core_types_count[i] > 1) ? 's' : ' '));
-    } else {
-      if (i == 0)
-        printf("No hybrid information available\n");
-      break;
-    }
-  }
+  printf("* num_core_eff: %d\n", num_core_efficiencies);
+  printf("* num_core_types: %d\n", num_core_types);
+  printf("* core_types: ");
+  for (int i = 0; i < num_core_types; ++i)
+    printf("%3d ", core_types[i]);
+  printf("\n");
 
   printf("* equivalent map:\n");
   KMP_FOREACH_HW_TYPE(i) {
@@ -680,12 +731,26 @@ void kmp_topology_t::print(const char *env_var) const {
   }
   KMP_INFORM(TopologyGeneric, env_var, buf.str, ncores);
 
+  // Hybrid topology information
   if (__kmp_is_hybrid_cpu()) {
-    for (int i = 0; i < KMP_HW_MAX_NUM_CORE_TYPES; ++i) {
-      if (core_types[i] == KMP_HW_CORE_TYPE_UNKNOWN)
-        break;
-      KMP_INFORM(TopologyHybrid, env_var, core_types_count[i],
-                 __kmp_hw_get_core_type_string(core_types[i]));
+    for (int i = 0; i < num_core_types; ++i) {
+      kmp_hw_core_type_t core_type = core_types[i];
+      kmp_hw_attr_t attr;
+      attr.clear();
+      attr.set_core_type(core_type);
+      int ncores = get_ncores_with_attr(attr);
+      if (ncores > 0) {
+        KMP_INFORM(TopologyHybrid, env_var, ncores,
+                   __kmp_hw_get_core_type_string(core_type));
+        KMP_ASSERT(num_core_efficiencies <= KMP_HW_MAX_NUM_CORE_EFFS)
+        for (int eff = 0; eff < num_core_efficiencies; ++eff) {
+          attr.set_core_eff(eff);
+          int ncores_with_eff = get_ncores_with_attr(attr);
+          if (ncores_with_eff > 0) {
+            KMP_INFORM(TopologyHybridCoreEff, env_var, ncores_with_eff, eff);
+          }
+        }
+      }
     }
   }
 
@@ -705,7 +770,8 @@ void kmp_topology_t::print(const char *env_var) const {
     }
     if (__kmp_is_hybrid_cpu())
       __kmp_str_buf_print(
-          &buf, "(%s)", __kmp_hw_get_core_type_string(hw_threads[i].core_type));
+          &buf, "(%s)",
+          __kmp_hw_get_core_type_string(hw_threads[i].attrs.get_core_type()));
     KMP_INFORM(OSProcMapToPack, env_var, hw_threads[i].os_id, buf.str);
   }
 
@@ -816,6 +882,56 @@ void kmp_topology_t::canonicalize(int npackages, int ncores_per_pkg,
   _discover_uniformity();
 }
 
+// Represents running sub IDs for a single core attribute where
+// attribute values have SIZE possibilities.
+template <size_t SIZE, typename IndexFunc> struct kmp_sub_ids_t {
+  int last_level; // last level in topology to consider for sub_ids
+  int sub_id[SIZE]; // The sub ID for a given attribute value
+  int prev_sub_id[KMP_HW_LAST];
+  IndexFunc indexer;
+
+public:
+  kmp_sub_ids_t(int last_level) : last_level(last_level) {
+    KMP_ASSERT(last_level < KMP_HW_LAST);
+    for (size_t i = 0; i < SIZE; ++i)
+      sub_id[i] = -1;
+    for (size_t i = 0; i < KMP_HW_LAST; ++i)
+      prev_sub_id[i] = -1;
+  }
+  void update(const kmp_hw_thread_t &hw_thread) {
+    int idx = indexer(hw_thread);
+    KMP_ASSERT(idx < (int)SIZE);
+    for (int level = 0; level <= last_level; ++level) {
+      if (hw_thread.sub_ids[level] != prev_sub_id[level]) {
+        if (level < last_level)
+          sub_id[idx] = -1;
+        sub_id[idx]++;
+        break;
+      }
+    }
+    for (int level = 0; level <= last_level; ++level)
+      prev_sub_id[level] = hw_thread.sub_ids[level];
+  }
+  int get_sub_id(const kmp_hw_thread_t &hw_thread) const {
+    return sub_id[indexer(hw_thread)];
+  }
+};
+
+static kmp_str_buf_t *
+__kmp_hw_get_catalog_core_string(const kmp_hw_attr_t &attr, kmp_str_buf_t *buf,
+                                 bool plural) {
+  __kmp_str_buf_init(buf);
+  if (attr.is_core_type_valid())
+    __kmp_str_buf_print(buf, "%s %s",
+                        __kmp_hw_get_core_type_string(attr.get_core_type()),
+                        __kmp_hw_get_catalog_string(KMP_HW_CORE, plural));
+  else
+    __kmp_str_buf_print(buf, "%s eff=%d",
+                        __kmp_hw_get_catalog_string(KMP_HW_CORE, plural),
+                        attr.get_core_eff());
+  return buf;
+}
+
 // Apply the KMP_HW_SUBSET envirable to the topology
 // Returns true if KMP_HW_SUBSET filtered any processors
 // otherwise, returns false
@@ -828,17 +944,23 @@ bool kmp_topology_t::filter_hw_subset() {
   __kmp_hw_subset->sort();
 
   // Check to see if KMP_HW_SUBSET is a valid subset of the detected topology
+  bool using_core_types = false;
+  bool using_core_effs = false;
   int hw_subset_depth = __kmp_hw_subset->get_depth();
   kmp_hw_t specified[KMP_HW_LAST];
+  int topology_levels[hw_subset_depth];
   KMP_ASSERT(hw_subset_depth > 0);
   KMP_FOREACH_HW_TYPE(i) { specified[i] = KMP_HW_UNKNOWN; }
+  int core_level = get_level(KMP_HW_CORE);
   for (int i = 0; i < hw_subset_depth; ++i) {
     int max_count;
-    int num = __kmp_hw_subset->at(i).num;
-    int offset = __kmp_hw_subset->at(i).offset;
-    kmp_hw_t type = __kmp_hw_subset->at(i).type;
+    const kmp_hw_subset_t::item_t &item = __kmp_hw_subset->at(i);
+    int num = item.num[0];
+    int offset = item.offset[0];
+    kmp_hw_t type = item.type;
     kmp_hw_t equivalent_type = equivalent[type];
     int level = get_level(type);
+    topology_levels[i] = level;
 
     // Check to see if current layer is in detected machine topology
     if (equivalent_type != KMP_HW_UNKNOWN) {
@@ -849,8 +971,8 @@ bool kmp_topology_t::filter_hw_subset() {
       return false;
     }
 
-    // Check to see if current layer has already been specified
-    // either directly or through an equivalent type
+    // Check to see if current layer has already been
+    // specified either directly or through an equivalent type
     if (specified[equivalent_type] != KMP_HW_UNKNOWN) {
       KMP_WARNING(AffHWSubsetEqvLayers, __kmp_hw_get_catalog_string(type),
                   __kmp_hw_get_catalog_string(specified[equivalent_type]));
@@ -860,47 +982,245 @@ bool kmp_topology_t::filter_hw_subset() {
 
     // Check to see if each layer's num & offset parameters are valid
     max_count = get_ratio(level);
-    if (max_count < 0 || num + offset > max_count) {
+    if (max_count < 0 ||
+        (num != kmp_hw_subset_t::USE_ALL && num + offset > max_count)) {
       bool plural = (num > 1);
       KMP_WARNING(AffHWSubsetManyGeneric,
                   __kmp_hw_get_catalog_string(type, plural));
       return false;
     }
+
+    // Check to see if core attributes are consistent
+    if (core_level == level) {
+      // Determine which core attributes are specified
+      for (int j = 0; j < item.num_attrs; ++j) {
+        if (item.attr[j].is_core_type_valid())
+          using_core_types = true;
+        if (item.attr[j].is_core_eff_valid())
+          using_core_effs = true;
+      }
+
+      // Check if using a single core attribute on non-hybrid arch.
+      // Do not ignore all of KMP_HW_SUBSET, just ignore the attribute.
+      //
+      // Check if using multiple core attributes on non-hyrbid arch.
+      // Ignore all of KMP_HW_SUBSET if this is the case.
+      if ((using_core_effs || using_core_types) && !__kmp_is_hybrid_cpu()) {
+        if (item.num_attrs == 1) {
+          if (using_core_effs) {
+            KMP_WARNING(AffHWSubsetIgnoringAttr, "efficiency");
+          } else {
+            KMP_WARNING(AffHWSubsetIgnoringAttr, "core_type");
+          }
+          using_core_effs = false;
+          using_core_types = false;
+        } else {
+          KMP_WARNING(AffHWSubsetAttrsNonHybrid);
+          return false;
+        }
+      }
+
+      // Check if using both core types and core efficiencies together
+      if (using_core_types && using_core_effs) {
+        KMP_WARNING(AffHWSubsetIncompat, "core_type", "efficiency");
+        return false;
+      }
+
+      // Check that core efficiency values are valid
+      if (using_core_effs) {
+        for (int j = 0; j < item.num_attrs; ++j) {
+          if (item.attr[j].is_core_eff_valid()) {
+            int core_eff = item.attr[j].get_core_eff();
+            if (core_eff < 0 || core_eff >= num_core_efficiencies) {
+              kmp_str_buf_t buf;
+              __kmp_str_buf_init(&buf);
+              __kmp_str_buf_print(&buf, "%d", item.attr[j].get_core_eff());
+              __kmp_msg(kmp_ms_warning,
+                        KMP_MSG(AffHWSubsetAttrInvalid, "efficiency", buf.str),
+                        KMP_HNT(ValidValuesRange, 0, num_core_efficiencies - 1),
+                        __kmp_msg_null);
+              __kmp_str_buf_free(&buf);
+              return false;
+            }
+          }
+        }
+      }
+
+      // Check that the number of requested cores with attributes is valid
+      if (using_core_types || using_core_effs) {
+        for (int j = 0; j < item.num_attrs; ++j) {
+          int num = item.num[j];
+          int offset = item.offset[j];
+          int level_above = core_level - 1;
+          if (level_above >= 0) {
+            max_count = get_ncores_with_attr_per(item.attr[j], level_above);
+            if (max_count <= 0 ||
+                (num != kmp_hw_subset_t::USE_ALL && num + offset > max_count)) {
+              kmp_str_buf_t buf;
+              __kmp_hw_get_catalog_core_string(item.attr[j], &buf, num > 0);
+              KMP_WARNING(AffHWSubsetManyGeneric, buf.str);
+              __kmp_str_buf_free(&buf);
+              return false;
+            }
+          }
+        }
+      }
+
+      if ((using_core_types || using_core_effs) && item.num_attrs > 1) {
+        for (int j = 0; j < item.num_attrs; ++j) {
+          // Ambiguous use of specific core attribute + generic core
+          // e.g., 4c & 3c:intel_core or 4c & 3c:eff1
+          if (!item.attr[j]) {
+            kmp_hw_attr_t other_attr;
+            for (int k = 0; k < item.num_attrs; ++k) {
+              if (item.attr[k] != item.attr[j]) {
+                other_attr = item.attr[k];
+                break;
+              }
+            }
+            kmp_str_buf_t buf;
+            __kmp_hw_get_catalog_core_string(other_attr, &buf, item.num[j] > 0);
+            KMP_WARNING(AffHWSubsetIncompat,
+                        __kmp_hw_get_catalog_string(KMP_HW_CORE), buf.str);
+            __kmp_str_buf_free(&buf);
+            return false;
+          }
+          // Allow specifying a specific core type or core eff exactly once
+          for (int k = 0; k < j; ++k) {
+            if (!item.attr[j] || !item.attr[k])
+              continue;
+            if (item.attr[k] == item.attr[j]) {
+              kmp_str_buf_t buf;
+              __kmp_hw_get_catalog_core_string(item.attr[j], &buf,
+                                               item.num[j] > 0);
+              KMP_WARNING(AffHWSubsetAttrRepeat, buf.str);
+              __kmp_str_buf_free(&buf);
+              return false;
+            }
+          }
+        }
+      }
+    }
   }
 
-  // Apply the filtered hardware subset
-  int new_index = 0;
+  struct core_type_indexer {
+    int operator()(const kmp_hw_thread_t &t) const {
+      switch (t.attrs.get_core_type()) {
+#if KMP_ARCH_X86 || KMP_ARCH_X86_64
+      case KMP_HW_CORE_TYPE_ATOM:
+        return 1;
+      case KMP_HW_CORE_TYPE_CORE:
+        return 2;
+#endif
+      case KMP_HW_CORE_TYPE_UNKNOWN:
+        return 0;
+      }
+      KMP_ASSERT(0);
+      return 0;
+    }
+  };
+  struct core_eff_indexer {
+    int operator()(const kmp_hw_thread_t &t) const {
+      return t.attrs.get_core_eff();
+    }
+  };
+
+  kmp_sub_ids_t<KMP_HW_MAX_NUM_CORE_TYPES, core_type_indexer> core_type_sub_ids(
+      core_level);
+  kmp_sub_ids_t<KMP_HW_MAX_NUM_CORE_EFFS, core_eff_indexer> core_eff_sub_ids(
+      core_level);
+
+  // Determine which hardware threads should be filtered.
+  int num_filtered = 0;
+  bool *filtered = (bool *)__kmp_allocate(sizeof(bool) * num_hw_threads);
   for (int i = 0; i < num_hw_threads; ++i) {
     kmp_hw_thread_t &hw_thread = hw_threads[i];
+    // Update type_sub_id
+    if (using_core_types)
+      core_type_sub_ids.update(hw_thread);
+    if (using_core_effs)
+      core_eff_sub_ids.update(hw_thread);
+
     // Check to see if this hardware thread should be filtered
     bool should_be_filtered = false;
-    for (int level = 0, hw_subset_index = 0;
-         level < depth && hw_subset_index < hw_subset_depth; ++level) {
-      kmp_hw_t topology_type = types[level];
-      auto hw_subset_item = __kmp_hw_subset->at(hw_subset_index);
-      kmp_hw_t hw_subset_type = hw_subset_item.type;
-      if (topology_type != hw_subset_type)
+    for (int hw_subset_index = 0; hw_subset_index < hw_subset_depth;
+         ++hw_subset_index) {
+      const auto &hw_subset_item = __kmp_hw_subset->at(hw_subset_index);
+      int level = topology_levels[hw_subset_index];
+      if (level == -1)
         continue;
-      int num = hw_subset_item.num;
-      int offset = hw_subset_item.offset;
-      hw_subset_index++;
-      if (hw_thread.sub_ids[level] < offset ||
-          hw_thread.sub_ids[level] >= offset + num) {
-        should_be_filtered = true;
-        break;
+      if ((using_core_effs || using_core_types) && level == core_level) {
+        // Look for the core attribute in KMP_HW_SUBSET which corresponds
+        // to this hardware thread's core attribute. Use this num,offset plus
+        // the running sub_id for the particular core attribute of this hardware
+        // thread to determine if the hardware thread should be filtered or not.
+        int attr_idx;
+        kmp_hw_core_type_t core_type = hw_thread.attrs.get_core_type();
+        int core_eff = hw_thread.attrs.get_core_eff();
+        for (attr_idx = 0; attr_idx < hw_subset_item.num_attrs; ++attr_idx) {
+          if (using_core_types &&
+              hw_subset_item.attr[attr_idx].get_core_type() == core_type)
+            break;
+          if (using_core_effs &&
+              hw_subset_item.attr[attr_idx].get_core_eff() == core_eff)
+            break;
+        }
+        // This core attribute isn't in the KMP_HW_SUBSET so always filter it.
+        if (attr_idx == hw_subset_item.num_attrs) {
+          should_be_filtered = true;
+          break;
+        }
+        int sub_id;
+        int num = hw_subset_item.num[attr_idx];
+        int offset = hw_subset_item.offset[attr_idx];
+        if (using_core_types)
+          sub_id = core_type_sub_ids.get_sub_id(hw_thread);
+        else
+          sub_id = core_eff_sub_ids.get_sub_id(hw_thread);
+        if (sub_id < offset ||
+            (num != kmp_hw_subset_t::USE_ALL && sub_id >= offset + num)) {
+          should_be_filtered = true;
+          break;
+        }
+      } else {
+        int num = hw_subset_item.num[0];
+        int offset = hw_subset_item.offset[0];
+        if (hw_thread.sub_ids[level] < offset ||
+            (num != kmp_hw_subset_t::USE_ALL &&
+             hw_thread.sub_ids[level] >= offset + num)) {
+          should_be_filtered = true;
+          break;
+        }
       }
     }
-    if (!should_be_filtered) {
+    // Collect filtering information
+    filtered[i] = should_be_filtered;
+    if (should_be_filtered)
+      num_filtered++;
+  }
+
+  // One last check that we shouldn't allow filtering entire machine
+  if (num_filtered == num_hw_threads) {
+    KMP_WARNING(AffHWSubsetAllFiltered);
+    __kmp_free(filtered);
+    return false;
+  }
+
+  // Apply the filter
+  int new_index = 0;
+  for (int i = 0; i < num_hw_threads; ++i) {
+    if (!filtered[i]) {
       if (i != new_index)
-        hw_threads[new_index] = hw_thread;
+        hw_threads[new_index] = hw_threads[i];
       new_index++;
     } else {
 #if KMP_AFFINITY_SUPPORTED
-      KMP_CPU_CLR(hw_thread.os_id, __kmp_affin_fullMask);
+      KMP_CPU_CLR(hw_threads[i].os_id, __kmp_affin_fullMask);
 #endif
       __kmp_avail_proc--;
     }
   }
+
   KMP_DEBUG_ASSERT(new_index <= num_hw_threads);
   num_hw_threads = new_index;
 
@@ -909,6 +1229,7 @@ bool kmp_topology_t::filter_hw_subset() {
   _discover_uniformity();
   _set_globals();
   _set_last_level_cache();
+  __kmp_free(filtered);
   return true;
 }
 
@@ -1461,8 +1782,10 @@ static bool __kmp_affinity_create_hwloc_map(kmp_i18n_id_t *const msg_id) {
             break;
           }
         }
-        if (cpukind_index >= 0)
-          hw_thread.core_type = cpukinds[cpukind_index].core_type;
+        if (cpukind_index >= 0) {
+          hw_thread.attrs.set_core_type(cpukinds[cpukind_index].core_type);
+          hw_thread.attrs.set_core_eff(cpukinds[cpukind_index].efficiency);
+        }
       }
       index--;
     }
@@ -2040,11 +2363,21 @@ static bool __kmp_affinity_create_apicid_map(kmp_i18n_id_t *const msg_id) {
 
 // Hybrid cpu detection using CPUID.1A
 // Thread should be pinned to processor already
-static void __kmp_get_hybrid_info(kmp_hw_core_type_t *type,
+static void __kmp_get_hybrid_info(kmp_hw_core_type_t *type, int *efficiency,
                                   unsigned *native_model_id) {
   kmp_cpuid buf;
   __kmp_x86_cpuid(0x1a, 0, &buf);
   *type = (kmp_hw_core_type_t)__kmp_extract_bits<24, 31>(buf.eax);
+  switch (*type) {
+  case KMP_HW_CORE_TYPE_ATOM:
+    *efficiency = 0;
+    break;
+  case KMP_HW_CORE_TYPE_CORE:
+    *efficiency = 1;
+    break;
+  default:
+    *efficiency = 0;
+  }
   *native_model_id = __kmp_extract_bits<0, 23>(buf.eax);
 }
 
@@ -2321,8 +2654,10 @@ static bool __kmp_affinity_create_x2apicid_map(kmp_i18n_id_t *const msg_id) {
     if (__kmp_is_hybrid_cpu() && highest_leaf >= 0x1a) {
       kmp_hw_core_type_t type;
       unsigned native_model_id;
-      __kmp_get_hybrid_info(&type, &native_model_id);
-      hw_thread.core_type = type;
+      int efficiency;
+      __kmp_get_hybrid_info(&type, &efficiency, &native_model_id);
+      hw_thread.attrs.set_core_type(type);
+      hw_thread.attrs.set_core_eff(efficiency);
     }
     hw_thread_index++;
   }
diff --git a/openmp/runtime/src/kmp_affinity.h b/openmp/runtime/src/kmp_affinity.h
index 76ba38bc8fc2..ce00362f04ca 100644
--- a/openmp/runtime/src/kmp_affinity.h
+++ b/openmp/runtime/src/kmp_affinity.h
@@ -15,6 +15,7 @@
 
 #include "kmp.h"
 #include "kmp_os.h"
+#include <limits>
 
 #if KMP_AFFINITY_SUPPORTED
 #if KMP_USE_HWLOC
@@ -598,16 +599,62 @@ class KMPNativeAffinity : public KMPAffinity {
 #endif /* KMP_OS_WINDOWS */
 #endif /* KMP_AFFINITY_SUPPORTED */
 
-typedef enum kmp_hw_core_type_t {
-  KMP_HW_CORE_TYPE_UNKNOWN = 0x0,
-#if KMP_ARCH_X86 || KMP_ARCH_X86_64
-  KMP_HW_CORE_TYPE_ATOM = 0x20,
-  KMP_HW_CORE_TYPE_CORE = 0x40,
-  KMP_HW_MAX_NUM_CORE_TYPES = 3,
-#else
-  KMP_HW_MAX_NUM_CORE_TYPES = 1,
-#endif
-} kmp_hw_core_type_t;
+// Describe an attribute for a level in the machine topology
+struct kmp_hw_attr_t {
+  int core_type : 8;
+  int core_eff : 8;
+  unsigned valid : 1;
+  unsigned reserved : 15;
+
+  static const int UNKNOWN_CORE_EFF = -1;
+
+  kmp_hw_attr_t()
+      : core_type(KMP_HW_CORE_TYPE_UNKNOWN), core_eff(UNKNOWN_CORE_EFF),
+        valid(0), reserved(0) {}
+  void set_core_type(kmp_hw_core_type_t type) {
+    valid = 1;
+    core_type = type;
+  }
+  void set_core_eff(int eff) {
+    valid = 1;
+    core_eff = eff;
+  }
+  kmp_hw_core_type_t get_core_type() const {
+    return (kmp_hw_core_type_t)core_type;
+  }
+  int get_core_eff() const { return core_eff; }
+  bool is_core_type_valid() const {
+    return core_type != KMP_HW_CORE_TYPE_UNKNOWN;
+  }
+  bool is_core_eff_valid() const { return core_eff != UNKNOWN_CORE_EFF; }
+  operator bool() const { return valid; }
+  void clear() {
+    core_type = KMP_HW_CORE_TYPE_UNKNOWN;
+    core_eff = UNKNOWN_CORE_EFF;
+    valid = 0;
+  }
+  bool contains(const kmp_hw_attr_t &other) const {
+    if (!valid && !other.valid)
+      return true;
+    if (valid && other.valid) {
+      if (other.is_core_type_valid()) {
+        if (!is_core_type_valid() || (get_core_type() != other.get_core_type()))
+          return false;
+      }
+      if (other.is_core_eff_valid()) {
+        if (!is_core_eff_valid() || (get_core_eff() != other.get_core_eff()))
+          return false;
+      }
+      return true;
+    }
+    return false;
+  }
+  bool operator==(const kmp_hw_attr_t &rhs) const {
+    return (rhs.valid == valid && rhs.core_eff == core_eff &&
+            rhs.core_type == core_type);
+  }
+  bool operator!=(const kmp_hw_attr_t &rhs) const { return !operator==(rhs); }
+};
 
 class kmp_hw_thread_t {
 public:
@@ -618,14 +665,14 @@ public:
   int sub_ids[KMP_HW_LAST];
   bool leader;
   int os_id;
-  kmp_hw_core_type_t core_type;
+  kmp_hw_attr_t attrs;
 
   void print() const;
   void clear() {
     for (int i = 0; i < (int)KMP_HW_LAST; ++i)
       ids[i] = UNKNOWN_ID;
     leader = false;
-    core_type = KMP_HW_CORE_TYPE_UNKNOWN;
+    attrs.clear();
   }
 };
 
@@ -653,10 +700,11 @@ class kmp_topology_t {
   // Storage containing the absolute number of each topology layer
   int *count;
 
-  // Storage containing the core types and the number of
-  // each core type for hybrid processors
+  // The number of core efficiencies. This is only useful for hybrid
+  // topologies. Core efficiencies will range from 0 to num efficiencies - 1
+  int num_core_efficiencies;
+  int num_core_types;
   kmp_hw_core_type_t core_types[KMP_HW_MAX_NUM_CORE_TYPES];
-  int core_types_count[KMP_HW_MAX_NUM_CORE_TYPES];
 
   // The hardware threads array
   // hw_threads is num_hw_threads long
@@ -704,19 +752,11 @@ class kmp_topology_t {
   // Set the last level cache equivalent type
   void _set_last_level_cache();
 
-  // Increments the number of cores of type 'type'
-  void _increment_core_type(kmp_hw_core_type_t type) {
-    for (int i = 0; i < KMP_HW_MAX_NUM_CORE_TYPES; ++i) {
-      if (core_types[i] == KMP_HW_CORE_TYPE_UNKNOWN) {
-        core_types[i] = type;
-        core_types_count[i] = 1;
-        break;
-      } else if (core_types[i] == type) {
-        core_types_count[i]++;
-        break;
-      }
-    }
-  }
+  // Return the number of cores with a particular attribute, 'attr'.
+  // If 'find_all' is true, then find all cores on the machine, otherwise find
+  // all cores per the layer 'above'
+  int _get_ncores_with_attr(const kmp_hw_attr_t &attr, int above,
+                            bool find_all = false) const;
 
 public:
   // Force use of allocate()/deallocate()
@@ -807,6 +847,16 @@ public:
     KMP_DEBUG_ASSERT(level >= 0 && level < depth);
     return count[level];
   }
+  // Return the total number of cores with attribute 'attr'
+  int get_ncores_with_attr(const kmp_hw_attr_t &attr) const {
+    return _get_ncores_with_attr(attr, -1, true);
+  }
+  // Return the number of cores with attribute
+  // 'attr' per topology level 'above'
+  int get_ncores_with_attr_per(const kmp_hw_attr_t &attr, int above) const {
+    return _get_ncores_with_attr(attr, above, false);
+  }
+
 #if KMP_AFFINITY_SUPPORTED
   void sort_compact() {
     qsort(hw_threads, num_hw_threads, sizeof(kmp_hw_thread_t),
@@ -819,12 +869,19 @@ public:
 extern kmp_topology_t *__kmp_topology;
 
 class kmp_hw_subset_t {
+  const static size_t MAX_ATTRS = KMP_HW_MAX_NUM_CORE_EFFS;
+
 public:
+  // Describe a machine topology item in KMP_HW_SUBSET
   struct item_t {
-    int num;
     kmp_hw_t type;
-    int offset;
+    int num_attrs;
+    int num[MAX_ATTRS];
+    int offset[MAX_ATTRS];
+    kmp_hw_attr_t attr[MAX_ATTRS];
   };
+  // Put parenthesis around max to avoid accidental use of Windows max macro.
+  const static int USE_ALL = (std::numeric_limits<int>::max)();
 
 private:
   int depth;
@@ -869,7 +926,20 @@ public:
   }
   void set_absolute() { absolute = true; }
   bool is_absolute() const { return absolute; }
-  void push_back(int num, kmp_hw_t type, int offset) {
+  void push_back(int num, kmp_hw_t type, int offset, kmp_hw_attr_t attr) {
+    for (int i = 0; i < depth; ++i) {
+      // Found an existing item for this layer type
+      // Add the num, offset, and attr to this item
+      if (items[i].type == type) {
+        int idx = items[i].num_attrs++;
+        if ((size_t)idx >= MAX_ATTRS)
+          return;
+        items[i].num[idx] = num;
+        items[i].offset[idx] = offset;
+        items[i].attr[idx] = attr;
+        return;
+      }
+    }
     if (depth == capacity - 1) {
       capacity *= 2;
       item_t *new_items = (item_t *)__kmp_allocate(sizeof(item_t) * capacity);
@@ -878,9 +948,11 @@ public:
       __kmp_free(items);
       items = new_items;
     }
-    items[depth].num = num;
+    items[depth].num_attrs = 1;
     items[depth].type = type;
-    items[depth].offset = offset;
+    items[depth].num[0] = num;
+    items[depth].offset[0] = offset;
+    items[depth].attr[0] = attr;
     depth++;
     set |= (1ull << type);
   }
@@ -912,8 +984,19 @@ public:
     printf("* depth: %d\n", depth);
     printf("* items:\n");
     for (int i = 0; i < depth; ++i) {
-      printf("num: %d, type: %s, offset: %d\n", items[i].num,
-             __kmp_hw_get_keyword(items[i].type), items[i].offset);
+      printf(" type: %s\n", __kmp_hw_get_keyword(items[i].type));
+      for (int j = 0; j < items[i].num_attrs; ++j) {
+        printf("  num: %d, offset: %d, attr: ", items[i].num[j],
+               items[i].offset[j]);
+        if (!items[i].attr[j]) {
+          printf(" (none)\n");
+        } else {
+          printf(
+              " core_type = %s, core_eff = %d\n",
+              __kmp_hw_get_core_type_string(items[i].attr[j].get_core_type()),
+              items[i].attr[j].get_core_eff());
+        }
+      }
     }
     printf("* set: 0x%llx\n", set);
     printf("* absolute: %d\n", absolute);
diff --git a/openmp/runtime/src/kmp_runtime.cpp b/openmp/runtime/src/kmp_runtime.cpp
index 4505d269c2b6..6efc26df8de3 100644
--- a/openmp/runtime/src/kmp_runtime.cpp
+++ b/openmp/runtime/src/kmp_runtime.cpp
@@ -7710,6 +7710,11 @@ static void __kmp_push_thread_limit(kmp_info_t *thr, int num_teams,
       num_threads = 1;
     }
   } else {
+    if (num_threads < 0) {
+      __kmp_msg(kmp_ms_warning, KMP_MSG(CantFormThrTeam, num_threads, 1),
+                __kmp_msg_null);
+      num_threads = 1;
+    }
     // This thread will be the primary thread of the league primary threads
     // Store new thread limit; old limit is saved in th_cg_roots list
     thr->th.th_current_task->td_icvs.thread_limit = num_threads;
@@ -7741,9 +7746,13 @@ static void __kmp_push_thread_limit(kmp_info_t *thr, int num_teams,
 void __kmp_push_num_teams(ident_t *id, int gtid, int num_teams,
                           int num_threads) {
   kmp_info_t *thr = __kmp_threads[gtid];
-  KMP_DEBUG_ASSERT(num_teams >= 0);
-  KMP_DEBUG_ASSERT(num_threads >= 0);
-
+  if (num_teams < 0) {
+    // OpenMP specification requires requested values to be positive,
+    // but people can send us any value, so we'd better check
+    __kmp_msg(kmp_ms_warning, KMP_MSG(NumTeamsNotPositive, num_teams, 1),
+              __kmp_msg_null);
+    num_teams = 1;
+  }
   if (num_teams == 0) {
     if (__kmp_nteams > 0) {
       num_teams = __kmp_nteams;
@@ -7800,7 +7809,7 @@ void __kmp_push_num_teams_51(ident_t *id, int gtid, int num_teams_lb,
   } else if (num_teams_lb == num_teams_ub) { // requires exact number of teams
     num_teams = num_teams_ub;
   } else { // num_teams_lb <= num_teams <= num_teams_ub
-    if (num_threads == 0) {
+    if (num_threads <= 0) {
       if (num_teams_ub > __kmp_teams_max_nth) {
         num_teams = num_teams_lb;
       } else {
diff --git a/openmp/runtime/src/kmp_settings.cpp b/openmp/runtime/src/kmp_settings.cpp
index 8f7cee2382b4..302abb4042d8 100644
--- a/openmp/runtime/src/kmp_settings.cpp
+++ b/openmp/runtime/src/kmp_settings.cpp
@@ -4961,28 +4961,85 @@ static void __kmp_stg_parse_hw_subset(char const *name, char const *value,
 
   // Check each component
   for (int i = 0; i < level; ++i) {
-    int offset = 0;
-    int num = atoi(components[i]); // each component should start with a number
-    if (num <= 0) {
-      goto err; // only positive integers are valid for count
-    }
-    if ((pos = strchr(components[i], '@'))) {
-      offset = atoi(pos + 1); // save offset
-      *pos = '\0'; // cut the offset from the component
-    }
-    pos = components[i] + strspn(components[i], digits);
-    if (pos == components[i]) {
-      goto err;
-    }
-    // detect the component type
-    kmp_hw_t type = __kmp_stg_parse_hw_subset_name(pos);
-    if (type == KMP_HW_UNKNOWN) {
-      goto err;
-    }
-    if (__kmp_hw_subset->specified(type)) {
-      goto err;
+    int core_level = 0;
+    char *core_components[MAX_T_LEVEL];
+    // Split possible core components by '&' delimiter
+    pos = components[i];
+    core_components[core_level++] = pos;
+    while ((pos = strchr(pos, '&'))) {
+      if (core_level >= MAX_T_LEVEL)
+        goto err; // too many different core types
+      *pos = '\0'; // modify input and avoid more copying
+      core_components[core_level++] = ++pos; // expect something after '&'
+    }
+
+    for (int j = 0; j < core_level; ++j) {
+      char *offset_ptr;
+      char *attr_ptr;
+      int offset = 0;
+      kmp_hw_attr_t attr;
+      int num;
+      // components may begin with an optional count of the number of resources
+      if (isdigit(*core_components[j])) {
+        num = atoi(core_components[j]);
+        if (num <= 0) {
+          goto err; // only positive integers are valid for count
+        }
+        pos = core_components[j] + strspn(core_components[j], digits);
+      } else if (*core_components[j] == '*') {
+        num = kmp_hw_subset_t::USE_ALL;
+        pos = core_components[j] + 1;
+      } else {
+        num = kmp_hw_subset_t::USE_ALL;
+        pos = core_components[j];
+      }
+
+      offset_ptr = strchr(core_components[j], '@');
+      attr_ptr = strchr(core_components[j], ':');
+
+      if (offset_ptr) {
+        offset = atoi(offset_ptr + 1); // save offset
+        *offset_ptr = '\0'; // cut the offset from the component
+      }
+      if (attr_ptr) {
+        attr.clear();
+        // save the attribute
+#if KMP_ARCH_X86 || KMP_ARCH_X86_64
+        if (__kmp_str_match("intel_core", -1, attr_ptr + 1)) {
+          attr.set_core_type(KMP_HW_CORE_TYPE_CORE);
+        } else if (__kmp_str_match("intel_atom", -1, attr_ptr + 1)) {
+          attr.set_core_type(KMP_HW_CORE_TYPE_ATOM);
+        }
+#endif
+        if (__kmp_str_match("eff", 3, attr_ptr + 1)) {
+          const char *number = attr_ptr + 1;
+          // skip the eff[iciency] token
+          while (isalpha(*number))
+            number++;
+          if (!isdigit(*number)) {
+            goto err;
+          }
+          int efficiency = atoi(number);
+          attr.set_core_eff(efficiency);
+        } else {
+          goto err;
+        }
+        *attr_ptr = '\0'; // cut the attribute from the component
+      }
+      // detect the component type
+      kmp_hw_t type = __kmp_stg_parse_hw_subset_name(pos);
+      if (type == KMP_HW_UNKNOWN) {
+        goto err;
+      }
+      // Only the core type can have attributes
+      if (attr && type != KMP_HW_CORE)
+        goto err;
+      // Must allow core be specified more than once
+      if (type != KMP_HW_CORE && __kmp_hw_subset->specified(type)) {
+        goto err;
+      }
+      __kmp_hw_subset->push_back(num, type, offset, attr);
     }
-    __kmp_hw_subset->push_back(num, type, offset);
   }
   return;
 err:
@@ -4994,6 +5051,21 @@ err:
   return;
 }
 
+static inline const char *
+__kmp_hw_get_core_type_keyword(kmp_hw_core_type_t type) {
+  switch (type) {
+  case KMP_HW_CORE_TYPE_UNKNOWN:
+    return "unknown";
+#if KMP_ARCH_X86 || KMP_ARCH_X86_64
+  case KMP_HW_CORE_TYPE_ATOM:
+    return "intel_atom";
+  case KMP_HW_CORE_TYPE_CORE:
+    return "intel_core";
+#endif
+  }
+  return "unknown";
+}
+
 static void __kmp_stg_print_hw_subset(kmp_str_buf_t *buffer, char const *name,
                                       void *data) {
   kmp_str_buf_t buf;
@@ -5009,10 +5081,20 @@ static void __kmp_stg_print_hw_subset(kmp_str_buf_t *buffer, char const *name,
   depth = __kmp_hw_subset->get_depth();
   for (int i = 0; i < depth; ++i) {
     const auto &item = __kmp_hw_subset->at(i);
-    __kmp_str_buf_print(&buf, "%s%d%s", (i > 0 ? "," : ""), item.num,
-                        __kmp_hw_get_keyword(item.type));
-    if (item.offset)
-      __kmp_str_buf_print(&buf, "@%d", item.offset);
+    if (i > 0)
+      __kmp_str_buf_print(&buf, "%c", ',');
+    for (int j = 0; j < item.num_attrs; ++j) {
+      __kmp_str_buf_print(&buf, "%s%d%s", (j > 0 ? "&" : ""), item.num[j],
+                          __kmp_hw_get_keyword(item.type));
+      if (item.attr[j].is_core_type_valid())
+        __kmp_str_buf_print(
+            &buf, ":%s",
+            __kmp_hw_get_core_type_keyword(item.attr[j].get_core_type()));
+      if (item.attr[j].is_core_eff_valid())
+        __kmp_str_buf_print(&buf, ":eff%d", item.attr[j].get_core_eff());
+      if (item.offset[j])
+        __kmp_str_buf_print(&buf, "@%d", item.offset[j]);
+    }
   }
   __kmp_str_buf_print(buffer, "%s'\n", buf.str);
   __kmp_str_buf_free(&buf);